feat(infra): auto-rebuild Docker images in CI pipelines, add build-image webhook

- Wire build-image endpoint into EventSource + Sensor for manual triggers - Add rebuild-* DAG tasks to compile-and-deploy and compile-and-train templates — Kaniko layer cache makes cached builds ~15-30s - Fix foxhunt-runtime Dockerfile: rename ubuntu user instead of groupadd (GID 1000 already exists in ubuntu:24.04) Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-12 08:35:55 +01:00
parent fbde4bb165
commit 45addfd35d
5 changed files with 120 additions and 10 deletions
--- a/infra/docker/Dockerfile.foxhunt-runtime
+++ b/infra/docker/Dockerfile.foxhunt-runtime
@@ -29,7 +29,7 @@ RUN curl -fsSL "https://dl.k8s.io/release/v1.31.4/bin/linux/amd64/kubectl" \
    -o /usr/local/bin/kubectl \
    && chmod +x /usr/local/bin/kubectl

-RUN groupadd -g 1000 foxhunt \
-    && useradd -u 1000 -g foxhunt -m -s /bin/false foxhunt
+# Ubuntu 24.04 ships with ubuntu:ubuntu (UID/GID 1000) — rename instead of creating
+RUN usermod -l foxhunt ubuntu && groupmod -n foxhunt ubuntu

 USER foxhunt
--- a/infra/k8s/argo/compile-and-deploy-template.yaml
+++ b/infra/k8s/argo/compile-and-deploy-template.yaml
@@ -1,8 +1,10 @@
 # Compile-and-Deploy: manual workflow for building and deploying service binaries.
 #
 # DAG:
-#   create-tag ──> compile-services ──> upload-release ──> deploy-services
+#   rebuild-ci-builder-cpu ──┐
+#   rebuild-runtime ─────────┼──> create-tag ──> compile-services ──> upload-release ──> deploy-services
 #
+# Image rebuilds use Kaniko layer cache — cached builds complete in ~15-30s.
 # Trigger manually when you want to build + deploy service binaries.
 # Training compilation lives in compile-and-train-template.yaml.
 ---
@@ -47,10 +49,36 @@ spec:
    - name: pipeline
      dag:
        tasks:
+          # Image rebuilds run in parallel — Kaniko cache makes no-ops ~15-30s
+          - name: rebuild-ci-builder-cpu
+            templateRef:
+              name: build-ci-image
+              template: build
+            arguments:
+              parameters:
+                - name: commit-sha
+                  value: "{{workflow.parameters.commit-sha}}"
+                - name: dockerfile
+                  value: Dockerfile.ci-builder-cpu
+                - name: image-name
+                  value: ci-builder-cpu
+          - name: rebuild-runtime
+            templateRef:
+              name: build-ci-image
+              template: build
+            arguments:
+              parameters:
+                - name: commit-sha
+                  value: "{{workflow.parameters.commit-sha}}"
+                - name: dockerfile
+                  value: Dockerfile.foxhunt-runtime
+                - name: image-name
+                  value: foxhunt-runtime
          - name: create-tag
+            depends: "rebuild-runtime"
            template: create-tag
          - name: compile-services
-            depends: "create-tag"
+            depends: "create-tag && rebuild-ci-builder-cpu"
            template: compile-services
            arguments:
              parameters:
--- a/infra/k8s/argo/compile-and-train-template.yaml
+++ b/infra/k8s/argo/compile-and-train-template.yaml
@@ -1,10 +1,10 @@
 # Compile-and-Train: unified workflow for code change → training run.
 #
 # DAG:
-#   compile-training ──┐
-#                      ├──> fetch-binary ──> hyperopt ──> train-best ──> evaluate ──> upload-results
-#   gpu-warmup ────────┘
+#   rebuild-ci-builder ────────> compile-training ──┐
+#   rebuild-training-runtime ──> gpu-warmup ────────┼──> fetch-binary ──> hyperopt ──> train-best ──> evaluate ──> upload-results
 #
+# Image rebuilds use Kaniko layer cache — cached builds complete in ~15-30s.
 # Compile and GPU warmup run in parallel (~5.5 min compile, ~3-5 min autoscale).
 # When both finish, fetch-binary downloads the freshly compiled binaries and
 # training starts immediately — no manual CI wait or package seeding.
@@ -96,10 +96,37 @@ spec:
    - name: pipeline
      dag:
        tasks:
+          # Image rebuilds — Kaniko cache makes no-ops ~15-30s
+          - name: rebuild-ci-builder
+            templateRef:
+              name: build-ci-image
+              template: build
+            arguments:
+              parameters:
+                - name: commit-sha
+                  value: "{{workflow.parameters.commit-sha}}"
+                - name: dockerfile
+                  value: Dockerfile.ci-builder
+                - name: image-name
+                  value: ci-builder
+          - name: rebuild-training-runtime
+            templateRef:
+              name: build-ci-image
+              template: build
+            arguments:
+              parameters:
+                - name: commit-sha
+                  value: "{{workflow.parameters.commit-sha}}"
+                - name: dockerfile
+                  value: Dockerfile.foxhunt-training-runtime
+                - name: image-name
+                  value: foxhunt-training-runtime
          - name: compile-training
            template: compile-training
+            dependencies: [rebuild-ci-builder]
          - name: gpu-warmup
            template: gpu-warmup
+            dependencies: [rebuild-training-runtime]
          - name: fetch-binary
            template: fetch-binary
            dependencies: [compile-training]
@@ -121,7 +148,11 @@ spec:
            dependencies: [evaluate]

    # ── compile-training: build from source on ci-compile-cpu ──
+    # Runs as root to match ci-builder image and PVC cargo-home ownership.
    - name: compile-training
+      securityContext:
+        runAsUser: 0
+        runAsGroup: 0
      outputs:
        parameters:
          - name: tag
--- a/infra/k8s/argo/events/workflow-trigger-eventsource.yaml
+++ b/infra/k8s/argo/events/workflow-trigger-eventsource.yaml
@@ -1,6 +1,6 @@
 # Manual workflow trigger EventSource.
 #
-# Webhook endpoints for triggering compile/deploy and training workflows.
+# Webhook endpoints for triggering compile/deploy, training, and image builds.
 # Each endpoint accepts a JSON POST body with workflow-specific parameters.
 #
 # Endpoints (all on port 12001):
@@ -8,10 +8,12 @@
 #   POST /train/dqn         — {"binary_tag": "latest"}
 #   POST /train/ppo         — {"binary_tag": "latest"}
 #   POST /train/supervised  — {"model": "tft", "binary_tag": "latest"}
+#   POST /build-image       — {"dockerfile": "Dockerfile.ci-builder", "image_name": "ci-builder", "commit_sha": "HEAD"}
 #
 # Example:
-#   curl -X POST http://workflow-trigger-eventsource-svc.foxhunt:12001/train/dqn \
-#     -H 'Content-Type: application/json' -d '{"binary_tag": "latest"}'
+#   curl -X POST http://workflow-trigger-eventsource-svc.foxhunt:12001/build-image \
+#     -H 'Content-Type: application/json' \
+#     -d '{"dockerfile": "Dockerfile.ci-builder", "image_name": "ci-builder", "commit_sha": "HEAD"}'
 ---
 apiVersion: argoproj.io/v1alpha1
 kind: EventSource
@@ -40,3 +42,7 @@ spec:
      port: "12001"
      endpoint: /train/supervised
      method: POST
+    build-image:
+      port: "12001"
+      endpoint: /build-image
+      method: POST
--- a/infra/k8s/argo/events/workflow-trigger-sensor.yaml
+++ b/infra/k8s/argo/events/workflow-trigger-sensor.yaml
@@ -9,6 +9,7 @@
 #   train-dqn         — POST /train/dqn         {"binary_tag": "..."}
 #   train-ppo         — POST /train/ppo         {"binary_tag": "..."}
 #   train-supervised  — POST /train/supervised  {"model": "...", "binary_tag": "..."}
+#   build-image       — POST /build-image       {"dockerfile": "...", "image_name": "...", "commit_sha": "..."}
 ---
 apiVersion: argoproj.io/v1alpha1
 kind: Sensor
@@ -38,6 +39,9 @@ spec:
    - name: train-supervised-dep
      eventSourceName: workflow-trigger
      eventName: train-supervised
+    - name: build-image-dep
+      eventSourceName: workflow-trigger
+      eventName: build-image

  triggers:
    # ── Compile + Deploy ──
@@ -170,3 +174,44 @@ spec:
                dependencyName: train-supervised-dep
                dataKey: body.binary_tag
              dest: spec.arguments.parameters.1.value
+
+    # ── Build Image ──
+    # POST /build-image {"dockerfile": "Dockerfile.ci-builder", "image_name": "ci-builder", "commit_sha": "HEAD"}
+    - template:
+        name: build-image
+        conditions: build-image-dep
+        argoWorkflow:
+          operation: submit
+          source:
+            resource:
+              apiVersion: argoproj.io/v1alpha1
+              kind: Workflow
+              metadata:
+                generateName: build-image-
+                namespace: foxhunt
+              spec:
+                serviceAccountName: argo-workflow
+                podGC:
+                  strategy: OnPodCompletion
+                ttlStrategy:
+                  secondsAfterCompletion: 3600
+                workflowTemplateRef:
+                  name: build-ci-image
+                arguments:
+                  parameters:
+                    - name: commit-sha
+                    - name: dockerfile
+                    - name: image-name
+          parameters:
+            - src:
+                dependencyName: build-image-dep
+                dataKey: body.commit_sha
+              dest: spec.arguments.parameters.0.value
+            - src:
+                dependencyName: build-image-dep
+                dataKey: body.dockerfile
+              dest: spec.arguments.parameters.1.value
+            - src:
+                dependencyName: build-image-dep
+                dataKey: body.image_name
+              dest: spec.arguments.parameters.2.value