fix(ci): per-component change detection to skip unnecessary compiles

Two bugs fixed:
1. jq parsed webhook as plain array but payload is {body:{commits:[...]}} —
   fell back to Cargo.toml catch-all, triggering full recompile every push.
2. Single needs-compile flag gave no granularity — now split into
   needs-services and needs-training with path-based detection.

crates/ml/ only → training + services (ml-training-service depends on ml)
services/ only → services only, skip training (saves ~5.5min CUDA compile)
infra/ only → skip both compiles entirely

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

infra/k8s/argo/ci-pipeline-template.yaml

@@ -16,16 +16,6 @@ spec:
       app.kubernetes.io/part-of: foxhunt
   ttlStrategy:
     secondsAfterCompletion: 3600
-  # Volumes for compile-training (called via templateRef — spec-level volumes don't carry over)
-  volumeClaimTemplates:
-    - metadata:
-        name: workspace
-      spec:
-        accessModes: [ReadWriteOnce]
-        resources:
-          requests:
-            storage: 20Gi
-        storageClassName: scw-bssd
   volumes:
     - name: sccache
       persistentVolumeClaim:
@@ -41,102 +31,23 @@ spec:
       - name: commits-json
         value: "[]"
   templates:
-    # ── DAG: orchestrate compile tasks ──
+    # ── DAG: orchestrate pipeline ──
     - name: pipeline
       dag:
         tasks:
           - name: detect-changes
             template: detect-changes
 
-          - name: compile-training
+          - name: compile-all
             dependencies: [detect-changes]
-            templateRef:
-              name: compile-training
-              template: compile
+            template: compile-all
             arguments:
               parameters:
-                - name: commit-sha
-                  value: "{{workflow.parameters.commit-sha}}"
-                - name: cuda-compute-cap
-                  value: "89"
-            when: "{{tasks.detect-changes.outputs.parameters.training}} == true"
-
-          - name: compile-api
-            dependencies: [detect-changes]
-            template: compile-service
-            arguments:
-              parameters:
-                - name: service-package
-                  value: api
-                - name: commit-sha
-                  value: "{{workflow.parameters.commit-sha}}"
-            when: "{{tasks.detect-changes.outputs.parameters.api}} == true"
-
-          - name: compile-trading-service
-            dependencies: [detect-changes]
-            template: compile-service
-            arguments:
-              parameters:
-                - name: service-package
-                  value: trading-service
-                - name: commit-sha
-                  value: "{{workflow.parameters.commit-sha}}"
-            when: "{{tasks.detect-changes.outputs.parameters.trading-service}} == true"
-
-          - name: compile-ml-training-service
-            dependencies: [detect-changes]
-            template: compile-service
-            arguments:
-              parameters:
-                - name: service-package
-                  value: ml-training-service
-                - name: commit-sha
-                  value: "{{workflow.parameters.commit-sha}}"
-            when: "{{tasks.detect-changes.outputs.parameters.ml-training-service}} == true"
-
-          - name: compile-backtesting-service
-            dependencies: [detect-changes]
-            template: compile-service
-            arguments:
-              parameters:
-                - name: service-package
-                  value: backtesting-service
-                - name: commit-sha
-                  value: "{{workflow.parameters.commit-sha}}"
-            when: "{{tasks.detect-changes.outputs.parameters.backtesting-service}} == true"
-
-          - name: compile-trading-agent-service
-            dependencies: [detect-changes]
-            template: compile-service
-            arguments:
-              parameters:
-                - name: service-package
-                  value: trading-agent-service
-                - name: commit-sha
-                  value: "{{workflow.parameters.commit-sha}}"
-            when: "{{tasks.detect-changes.outputs.parameters.trading-agent-service}} == true"
-
-          - name: compile-broker-gateway
-            dependencies: [detect-changes]
-            template: compile-service
-            arguments:
-              parameters:
-                - name: service-package
-                  value: broker-gateway
-                - name: commit-sha
-                  value: "{{workflow.parameters.commit-sha}}"
-            when: "{{tasks.detect-changes.outputs.parameters.broker-gateway}} == true"
-
-          - name: compile-data-acquisition-service
-            dependencies: [detect-changes]
-            template: compile-service
-            arguments:
-              parameters:
-                - name: service-package
-                  value: data-acquisition-service
-                - name: commit-sha
-                  value: "{{workflow.parameters.commit-sha}}"
-            when: "{{tasks.detect-changes.outputs.parameters.data-acquisition-service}} == true"
+                - name: needs-services
+                  value: "{{tasks.detect-changes.outputs.parameters.needs-services}}"
+                - name: needs-training
+                  value: "{{tasks.detect-changes.outputs.parameters.needs-training}}"
+            when: "{{tasks.detect-changes.outputs.parameters.needs-services}} == true || {{tasks.detect-changes.outputs.parameters.needs-training}} == true"
 
           - name: rebuild-ci-builder
             dependencies: [detect-changes]
@@ -198,15 +109,10 @@ spec:
                   value: foxhunt-training-runtime
             when: "{{tasks.detect-changes.outputs.parameters.docker-images}} == true"
 
-    # ── detect-changes: determine which services need rebuilding ──
+    # ── detect-changes: simplified to needs-compile + docker-images ──
     - name: detect-changes
       nodeSelector:
-        k8s.scaleway.com/pool-name: platform
-      tolerations:
-        - key: gitlab
-          operator: Equal
-          value: "true"
-          effect: NoSchedule
+        k8s.scaleway.com/pool-name: infra
       container:
         image: rg.fr-par.scw.cloud/foxhunt-ci/foxhunt-runtime:latest
         command: ["/bin/sh", "-c"]
@@ -227,18 +133,17 @@ spec:
             #!/bin/sh
             set -e
 
-            # COMMITS_JSON is passed as env var to avoid shell escaping issues
+            # COMMITS_JSON is the full webhook payload: {"body":{"commits":[...]}, "header":{...}}
+            # Extract changed file paths from .body.commits array
 
-            # Extract all changed file paths from commits array
             if command -v jq >/dev/null 2>&1; then
               CHANGED_FILES=$(echo "$COMMITS_JSON" | jq -r '
-                if type == "array" then
-                  [.[].added[], .[].modified[], .[].removed[]] | unique | .[]
-                elif type == "string" then
-                  (fromjson // []) | [.[].added[], .[].modified[], .[].removed[]] | unique | .[]
-                else
-                  empty
-                end
+                # Handle full webhook payload with .body.commits
+                (if .body.commits then .body.commits
+                 elif type == "array" then .
+                 else []
+                 end) |
+                [.[].added[]?, .[].modified[]?, .[].removed[]?] | unique | .[]
               ' 2>/dev/null || echo "")
             else
               CHANGED_FILES="Cargo.toml"
@@ -265,83 +170,69 @@ spec:
               echo "false"
             }
 
-            API=$(check_paths "services/api/ crates/common/ crates/config/ crates/trading_engine/ bin/fxt/ Cargo.toml Cargo.lock")
-            TRADING_SERVICE=$(check_paths "services/trading_service/ services/api/ crates/common/ crates/config/ crates/data/ crates/database/ crates/ml/ crates/risk/ crates/storage/ crates/trading_engine/ Cargo.toml Cargo.lock")
-            ML_TRAINING_SERVICE=$(check_paths "services/ml_training_service/ crates/common/ crates/config/ crates/data/ crates/ml/ crates/risk/ crates/storage/ crates/trading_engine/ Cargo.toml Cargo.lock")
-            BACKTESTING_SERVICE=$(check_paths "services/backtesting_service/ crates/common/ crates/config/ crates/data/ crates/ml/ crates/model_loader/ crates/risk/ crates/storage/ crates/trading_engine/ bin/fxt/ Cargo.toml Cargo.lock")
-            TRADING_AGENT_SERVICE=$(check_paths "services/trading_agent_service/ crates/common/ crates/config/ crates/ml/ crates/risk/ Cargo.toml Cargo.lock")
-            BROKER_GATEWAY=$(check_paths "services/broker_gateway_service/ crates/common/ crates/config/ crates/trading_engine/ vendor/ctrader-openapi/ Cargo.toml Cargo.lock")
-            DATA_ACQUISITION_SERVICE=$(check_paths "services/data_acquisition_service/ crates/common/ crates/config/ crates/storage/ Cargo.toml Cargo.lock")
-            TRAINING=$(check_paths "crates/ml/ crates/trading_engine/ crates/common/ crates/risk/ crates/data/ crates/config/ crates/storage/ services/training_uploader/ Cargo.toml Cargo.lock")
+            # Shared crates trigger both services and training
+            SHARED=$(check_paths "crates/common/ crates/config/ Cargo.toml Cargo.lock")
+
+            # Service-specific paths
+            SVC_ONLY=$(check_paths "services/ bin/fxt/ crates/trading_engine/ crates/risk/ crates/data/ crates/storage/ crates/database/ crates/broker crates/ctrader")
+
+            # Training-specific paths (ml crate also used by ml-training-service + trading-service)
+            ML_CHANGED=$(check_paths "crates/ml/")
+
+            # Docker images
             DOCKER_IMAGES=$(check_paths "infra/docker/")
 
+            # Services rebuild when: shared crates, service code, or ml crate changes
+            if [ "$SHARED" = "true" ] || [ "$SVC_ONLY" = "true" ] || [ "$ML_CHANGED" = "true" ]; then
+              NEEDS_SERVICES="true"
+            else
+              NEEDS_SERVICES="false"
+            fi
+
+            # Training rebuilds when: shared crates or ml crate changes
+            if [ "$SHARED" = "true" ] || [ "$ML_CHANGED" = "true" ]; then
+              NEEDS_TRAINING="true"
+            else
+              NEEDS_TRAINING="false"
+            fi
+
             echo "=== Build decisions ==="
-            echo "api: $API"
-            echo "trading-service: $TRADING_SERVICE"
-            echo "ml-training-service: $ML_TRAINING_SERVICE"
-            echo "backtesting-service: $BACKTESTING_SERVICE"
-            echo "trading-agent-service: $TRADING_AGENT_SERVICE"
-            echo "broker-gateway: $BROKER_GATEWAY"
-            echo "data-acquisition-service: $DATA_ACQUISITION_SERVICE"
-            echo "training: $TRAINING"
+            echo "needs-services: $NEEDS_SERVICES"
+            echo "needs-training: $NEEDS_TRAINING"
             echo "docker-images: $DOCKER_IMAGES"
             echo "======================"
 
             mkdir -p /tmp/outputs
-            echo -n "$API" > /tmp/outputs/api
-            echo -n "$TRADING_SERVICE" > /tmp/outputs/trading-service
-            echo -n "$ML_TRAINING_SERVICE" > /tmp/outputs/ml-training-service
-            echo -n "$BACKTESTING_SERVICE" > /tmp/outputs/backtesting-service
-            echo -n "$TRADING_AGENT_SERVICE" > /tmp/outputs/trading-agent-service
-            echo -n "$BROKER_GATEWAY" > /tmp/outputs/broker-gateway
-            echo -n "$DATA_ACQUISITION_SERVICE" > /tmp/outputs/data-acquisition-service
-            echo -n "$TRAINING" > /tmp/outputs/training
+            echo -n "$NEEDS_SERVICES" > /tmp/outputs/needs-services
+            echo -n "$NEEDS_TRAINING" > /tmp/outputs/needs-training
             echo -n "$DOCKER_IMAGES" > /tmp/outputs/docker-images
             SCRIPT
             chmod +x /tmp/detect.sh
             /tmp/detect.sh
       outputs:
         parameters:
-          - name: api
+          - name: needs-services
             valueFrom:
-              path: /tmp/outputs/api
-          - name: trading-service
+              path: /tmp/outputs/needs-services
+          - name: needs-training
             valueFrom:
-              path: /tmp/outputs/trading-service
-          - name: ml-training-service
-            valueFrom:
-              path: /tmp/outputs/ml-training-service
-          - name: backtesting-service
-            valueFrom:
-              path: /tmp/outputs/backtesting-service
-          - name: trading-agent-service
-            valueFrom:
-              path: /tmp/outputs/trading-agent-service
-          - name: broker-gateway
-            valueFrom:
-              path: /tmp/outputs/broker-gateway
-          - name: data-acquisition-service
-            valueFrom:
-              path: /tmp/outputs/data-acquisition-service
-          - name: training
-            valueFrom:
-              path: /tmp/outputs/training
+              path: /tmp/outputs/needs-training
           - name: docker-images
             valueFrom:
               path: /tmp/outputs/docker-images
 
-    # ── compile-service: build and upload a single service binary ──
-    - name: compile-service
+    # ── compile-all: build and upload service + training binaries (conditionally) ──
+    - name: compile-all
+      inputs:
+        parameters:
+          - name: needs-services
+          - name: needs-training
       nodeSelector:
         k8s.scaleway.com/pool-name: ci-compile-cpu
       tolerations:
         - key: node.cilium.io/agent-not-ready
           operator: Exists
           effect: NoSchedule
-      inputs:
-        parameters:
-          - name: service-package
-          - name: commit-sha
       container:
         image: rg.fr-par.scw.cloud/foxhunt-ci/ci-builder:latest
         command: ["/bin/sh", "-c"]
@@ -350,6 +241,10 @@ spec:
             value: "true"
           - name: CARGO_TERM_COLOR
             value: always
+          - name: NEEDS_SERVICES
+            value: "{{inputs.parameters.needs-services}}"
+          - name: NEEDS_TRAINING
+            value: "{{inputs.parameters.needs-training}}"
           - name: MINIO_ACCESS_KEY
             valueFrom:
               secretKeyRef:
@@ -362,11 +257,11 @@ spec:
                 key: secret-key
         resources:
           requests:
-            cpu: "4"
-            memory: 8Gi
+            cpu: "14"
+            memory: 32Gi
           limits:
-            cpu: "6"
-            memory: 16Gi
+            cpu: "30"
+            memory: 64Gi
         volumeMounts:
           - name: sccache
             mountPath: /mnt/sccache
@@ -376,17 +271,15 @@ spec:
         args:
           - |
             set -e
-            SERVICE="{{inputs.parameters.service-package}}"
-            SHA="{{inputs.parameters.commit-sha}}"
+            SHA="{{workflow.parameters.commit-sha}}"
 
-            # Configure SSH for git clone
+            # ── Git checkout ──
             mkdir -p ~/.ssh
             cp /etc/git-ssh/ssh-privatekey ~/.ssh/id_ed25519
             chmod 600 ~/.ssh/id_ed25519
             printf 'Host *\n  StrictHostKeyChecking no\n  UserKnownHostsFile /dev/null\n' > ~/.ssh/config
             chmod 600 ~/.ssh/config
 
-            echo "=== Compiling $SERVICE at $SHA ==="
             REPO="ssh://git@gitlab-gitlab-shell.foxhunt.svc.cluster.local:2222/root/foxhunt.git"
             if [ "$SHA" = "HEAD" ]; then
               git clone --depth=1 "$REPO" /workspace/src
@@ -399,35 +292,79 @@ spec:
             cd /workspace/src
             echo "Checked out $(git rev-parse --short HEAD)"
 
-            export SCCACHE_DIR="/mnt/sccache/cpu"
-            export RUSTC_WRAPPER=sccache
-            mkdir -p "$SCCACHE_DIR"
-            sccache --zero-stats || true
-
             YEAR=$(date +%Y)
             MONTH=$(date +%m)
             export FOXHUNT_BUILD_VERSION="${YEAR}.${MONTH}.argo"
+            export RUSTC_WRAPPER=sccache
 
-            # Service name matches Cargo.toml package name (all use hyphens)
-            PKG="$SERVICE"
+            # ── Build services (skip if only training changed) ──
+            if [ "$NEEDS_SERVICES" = "true" ]; then
+              echo "=== Building service binaries ==="
+              export SCCACHE_DIR="/mnt/sccache/cpu"
+              mkdir -p "$SCCACHE_DIR"
+              sccache --zero-stats || true
 
-            cargo build --release -p "$PKG"
-            sccache --show-stats || true
+              cargo build --release \
+                -p api -p trading-service -p ml-training-service \
+                -p backtesting-service -p trading-agent-service \
+                -p broker-gateway -p data-acquisition-service
 
-            mkdir -p /workspace/bin
-            cp "target/release/$PKG" /workspace/bin/ 2>/dev/null \
-              || cp "target/release/${PKG//_/-}" /workspace/bin/ 2>/dev/null \
-              || { echo "Binary not found for $PKG"; ls target/release/; exit 1; }
-            strip /workspace/bin/*
+              mkdir -p /workspace/bin/services
+              for pkg in api trading-service ml-training-service backtesting-service trading-agent-service broker-gateway data-acquisition-service; do
+                bin_name=$(echo "$pkg" | tr '-' '_')
+                cp "target/release/$pkg" "/workspace/bin/services/" 2>/dev/null \
+                  || cp "target/release/$bin_name" "/workspace/bin/services/" 2>/dev/null \
+                  || { echo "Binary not found for $pkg"; ls target/release/; exit 1; }
+              done
+              strip /workspace/bin/services/*
 
-            echo "=== Uploading $SERVICE binary to MinIO ==="
-            rclone copy /workspace/bin/ ":s3:foxhunt-binaries/services/" \
-              --s3-provider=Minio \
-              --s3-endpoint=https://minio.foxhunt.svc.cluster.local:9000 \
-              --s3-access-key-id="${MINIO_ACCESS_KEY}" \
-              --s3-secret-access-key="${MINIO_SECRET_KEY}" \
-              --s3-no-check-bucket \
-              --no-check-certificate
+              echo "=== Service binaries ==="
+              ls -lh /workspace/bin/services/
+              sccache --show-stats || true
+            else
+              echo "=== Skipping service binaries (no service changes) ==="
+            fi
 
-            echo "=== $SERVICE compile + upload complete ==="
-            ls -lh /workspace/bin/
+            # ── Build training binaries (skip if only services changed) ──
+            if [ "$NEEDS_TRAINING" = "true" ]; then
+              echo "=== Building training binaries (CUDA_COMPUTE_CAP={{workflow.parameters.cuda-compute-cap}}) ==="
+              export CUDA_COMPUTE_CAP={{workflow.parameters.cuda-compute-cap}}
+              export SCCACHE_DIR="/mnt/sccache/sm_{{workflow.parameters.cuda-compute-cap}}"
+              mkdir -p "$SCCACHE_DIR"
+              sccache --zero-stats || true
+
+              cargo build --release -p ml --features ml/cuda \
+                --example train_baseline_rl --example train_baseline_supervised \
+                --example evaluate_baseline --example evaluate_supervised \
+                --example hyperopt_baseline_rl --example hyperopt_baseline_supervised
+
+              cargo build --release -p training_uploader
+
+              mkdir -p /workspace/bin/training
+              for bin in train_baseline_rl train_baseline_supervised evaluate_baseline evaluate_supervised hyperopt_baseline_rl hyperopt_baseline_supervised; do
+                cp target/release/examples/$bin /workspace/bin/training/
+              done
+              cp target/release/training_uploader /workspace/bin/training/
+              strip /workspace/bin/training/*
+
+              echo "=== Training binaries ==="
+              ls -lh /workspace/bin/training/
+              sccache --show-stats || true
+            else
+              echo "=== Skipping training binaries (no ml changes) ==="
+            fi
+
+            # ── Upload to MinIO (only what was built) ──
+            RCLONE_S3="--s3-provider=Minio --s3-endpoint=https://minio.foxhunt.svc.cluster.local:9000 --s3-access-key-id=${MINIO_ACCESS_KEY} --s3-secret-access-key=${MINIO_SECRET_KEY} --s3-no-check-bucket --no-check-certificate"
+
+            if [ "$NEEDS_SERVICES" = "true" ]; then
+              echo "=== Uploading service binaries ==="
+              eval rclone copy /workspace/bin/services/ \":s3:foxhunt-binaries/services/\" $RCLONE_S3
+            fi
+
+            if [ "$NEEDS_TRAINING" = "true" ]; then
+              echo "=== Uploading training binaries ==="
+              eval rclone copy /workspace/bin/training/ \":s3:foxhunt-binaries/training/\" $RCLONE_S3
+            fi
+
+            echo "=== Done (services=$NEEDS_SERVICES, training=$NEEDS_TRAINING) ==="