diff --git a/infra/docker/Dockerfile.foxhunt-runtime b/infra/docker/Dockerfile.foxhunt-runtime index 3a7d1f7fc..6e285181d 100644 --- a/infra/docker/Dockerfile.foxhunt-runtime +++ b/infra/docker/Dockerfile.foxhunt-runtime @@ -29,7 +29,7 @@ RUN curl -fsSL "https://dl.k8s.io/release/v1.31.4/bin/linux/amd64/kubectl" \ -o /usr/local/bin/kubectl \ && chmod +x /usr/local/bin/kubectl -RUN groupadd -g 1000 foxhunt \ - && useradd -u 1000 -g foxhunt -m -s /bin/false foxhunt +# Ubuntu 24.04 ships with ubuntu:ubuntu (UID/GID 1000) — rename instead of creating +RUN usermod -l foxhunt ubuntu && groupmod -n foxhunt ubuntu USER foxhunt diff --git a/infra/k8s/argo/compile-and-deploy-template.yaml b/infra/k8s/argo/compile-and-deploy-template.yaml index f2233f04a..957e64603 100644 --- a/infra/k8s/argo/compile-and-deploy-template.yaml +++ b/infra/k8s/argo/compile-and-deploy-template.yaml @@ -1,8 +1,10 @@ # Compile-and-Deploy: manual workflow for building and deploying service binaries. # # DAG: -# create-tag ──> compile-services ──> upload-release ──> deploy-services +# rebuild-ci-builder-cpu ──┐ +# rebuild-runtime ─────────┼──> create-tag ──> compile-services ──> upload-release ──> deploy-services # +# Image rebuilds use Kaniko layer cache — cached builds complete in ~15-30s. # Trigger manually when you want to build + deploy service binaries. # Training compilation lives in compile-and-train-template.yaml. --- @@ -47,10 +49,36 @@ spec: - name: pipeline dag: tasks: + # Image rebuilds run in parallel — Kaniko cache makes no-ops ~15-30s + - name: rebuild-ci-builder-cpu + templateRef: + name: build-ci-image + template: build + arguments: + parameters: + - name: commit-sha + value: "{{workflow.parameters.commit-sha}}" + - name: dockerfile + value: Dockerfile.ci-builder-cpu + - name: image-name + value: ci-builder-cpu + - name: rebuild-runtime + templateRef: + name: build-ci-image + template: build + arguments: + parameters: + - name: commit-sha + value: "{{workflow.parameters.commit-sha}}" + - name: dockerfile + value: Dockerfile.foxhunt-runtime + - name: image-name + value: foxhunt-runtime - name: create-tag + depends: "rebuild-runtime" template: create-tag - name: compile-services - depends: "create-tag" + depends: "create-tag && rebuild-ci-builder-cpu" template: compile-services arguments: parameters: diff --git a/infra/k8s/argo/compile-and-train-template.yaml b/infra/k8s/argo/compile-and-train-template.yaml index 5b998c77c..2cdd4cb2c 100644 --- a/infra/k8s/argo/compile-and-train-template.yaml +++ b/infra/k8s/argo/compile-and-train-template.yaml @@ -1,10 +1,10 @@ # Compile-and-Train: unified workflow for code change → training run. # # DAG: -# compile-training ──┐ -# ├──> fetch-binary ──> hyperopt ──> train-best ──> evaluate ──> upload-results -# gpu-warmup ────────┘ +# rebuild-ci-builder ────────> compile-training ──┐ +# rebuild-training-runtime ──> gpu-warmup ────────┼──> fetch-binary ──> hyperopt ──> train-best ──> evaluate ──> upload-results # +# Image rebuilds use Kaniko layer cache — cached builds complete in ~15-30s. # Compile and GPU warmup run in parallel (~5.5 min compile, ~3-5 min autoscale). # When both finish, fetch-binary downloads the freshly compiled binaries and # training starts immediately — no manual CI wait or package seeding. @@ -96,10 +96,37 @@ spec: - name: pipeline dag: tasks: + # Image rebuilds — Kaniko cache makes no-ops ~15-30s + - name: rebuild-ci-builder + templateRef: + name: build-ci-image + template: build + arguments: + parameters: + - name: commit-sha + value: "{{workflow.parameters.commit-sha}}" + - name: dockerfile + value: Dockerfile.ci-builder + - name: image-name + value: ci-builder + - name: rebuild-training-runtime + templateRef: + name: build-ci-image + template: build + arguments: + parameters: + - name: commit-sha + value: "{{workflow.parameters.commit-sha}}" + - name: dockerfile + value: Dockerfile.foxhunt-training-runtime + - name: image-name + value: foxhunt-training-runtime - name: compile-training template: compile-training + dependencies: [rebuild-ci-builder] - name: gpu-warmup template: gpu-warmup + dependencies: [rebuild-training-runtime] - name: fetch-binary template: fetch-binary dependencies: [compile-training] @@ -121,7 +148,11 @@ spec: dependencies: [evaluate] # ── compile-training: build from source on ci-compile-cpu ── + # Runs as root to match ci-builder image and PVC cargo-home ownership. - name: compile-training + securityContext: + runAsUser: 0 + runAsGroup: 0 outputs: parameters: - name: tag diff --git a/infra/k8s/argo/events/workflow-trigger-eventsource.yaml b/infra/k8s/argo/events/workflow-trigger-eventsource.yaml index 8b4bb9f78..4a6fecbff 100644 --- a/infra/k8s/argo/events/workflow-trigger-eventsource.yaml +++ b/infra/k8s/argo/events/workflow-trigger-eventsource.yaml @@ -1,6 +1,6 @@ # Manual workflow trigger EventSource. # -# Webhook endpoints for triggering compile/deploy and training workflows. +# Webhook endpoints for triggering compile/deploy, training, and image builds. # Each endpoint accepts a JSON POST body with workflow-specific parameters. # # Endpoints (all on port 12001): @@ -8,10 +8,12 @@ # POST /train/dqn — {"binary_tag": "latest"} # POST /train/ppo — {"binary_tag": "latest"} # POST /train/supervised — {"model": "tft", "binary_tag": "latest"} +# POST /build-image — {"dockerfile": "Dockerfile.ci-builder", "image_name": "ci-builder", "commit_sha": "HEAD"} # # Example: -# curl -X POST http://workflow-trigger-eventsource-svc.foxhunt:12001/train/dqn \ -# -H 'Content-Type: application/json' -d '{"binary_tag": "latest"}' +# curl -X POST http://workflow-trigger-eventsource-svc.foxhunt:12001/build-image \ +# -H 'Content-Type: application/json' \ +# -d '{"dockerfile": "Dockerfile.ci-builder", "image_name": "ci-builder", "commit_sha": "HEAD"}' --- apiVersion: argoproj.io/v1alpha1 kind: EventSource @@ -40,3 +42,7 @@ spec: port: "12001" endpoint: /train/supervised method: POST + build-image: + port: "12001" + endpoint: /build-image + method: POST diff --git a/infra/k8s/argo/events/workflow-trigger-sensor.yaml b/infra/k8s/argo/events/workflow-trigger-sensor.yaml index 794c3741e..7efac17d6 100644 --- a/infra/k8s/argo/events/workflow-trigger-sensor.yaml +++ b/infra/k8s/argo/events/workflow-trigger-sensor.yaml @@ -9,6 +9,7 @@ # train-dqn — POST /train/dqn {"binary_tag": "..."} # train-ppo — POST /train/ppo {"binary_tag": "..."} # train-supervised — POST /train/supervised {"model": "...", "binary_tag": "..."} +# build-image — POST /build-image {"dockerfile": "...", "image_name": "...", "commit_sha": "..."} --- apiVersion: argoproj.io/v1alpha1 kind: Sensor @@ -38,6 +39,9 @@ spec: - name: train-supervised-dep eventSourceName: workflow-trigger eventName: train-supervised + - name: build-image-dep + eventSourceName: workflow-trigger + eventName: build-image triggers: # ── Compile + Deploy ── @@ -170,3 +174,44 @@ spec: dependencyName: train-supervised-dep dataKey: body.binary_tag dest: spec.arguments.parameters.1.value + + # ── Build Image ── + # POST /build-image {"dockerfile": "Dockerfile.ci-builder", "image_name": "ci-builder", "commit_sha": "HEAD"} + - template: + name: build-image + conditions: build-image-dep + argoWorkflow: + operation: submit + source: + resource: + apiVersion: argoproj.io/v1alpha1 + kind: Workflow + metadata: + generateName: build-image- + namespace: foxhunt + spec: + serviceAccountName: argo-workflow + podGC: + strategy: OnPodCompletion + ttlStrategy: + secondsAfterCompletion: 3600 + workflowTemplateRef: + name: build-ci-image + arguments: + parameters: + - name: commit-sha + - name: dockerfile + - name: image-name + parameters: + - src: + dependencyName: build-image-dep + dataKey: body.commit_sha + dest: spec.arguments.parameters.0.value + - src: + dependencyName: build-image-dep + dataKey: body.dockerfile + dest: spec.arguments.parameters.1.value + - src: + dependencyName: build-image-dep + dataKey: body.image_name + dest: spec.arguments.parameters.2.value