foxhunt/infra/k8s/argo/ci-pipeline-template.yaml

apiVersion: argoproj.io/v1alpha1
kind: WorkflowTemplate
metadata:
  name: ci-pipeline
  namespace: foxhunt
  labels:
    app.kubernetes.io/name: ci-pipeline
    app.kubernetes.io/part-of: foxhunt
spec:
  activeDeadlineSeconds: 7200
  serviceAccountName: argo-workflow
  entrypoint: pipeline
  podMetadata:
    labels:
      app.kubernetes.io/component: ci-pipeline
      app.kubernetes.io/part-of: foxhunt
  podGC:
    strategy: OnPodCompletion
  ttlStrategy:
    secondsAfterCompletion: 3600
  volumes:
    - name: git-ssh-key
      secret:
        secretName: argo-git-ssh-key
        defaultMode: 256
    - name: registry-auth
      secret:
        secretName: gitlab-registry
        optional: true
        items:
          - key: .dockerconfigjson
            path: config.json
  arguments:
    parameters:
      - name: commit-sha
        value: HEAD
      - name: commits-json
        value: "[]"
      - name: cuda-compute-cap
        value: "90"
  templates:
    # ── DAG: orchestrate pipeline ──
    - name: pipeline
      dag:
        tasks:
          - name: detect-changes
            template: detect-changes

          # Parallel: each compile step is self-contained (own git clone)
          # sccache backed by MinIO — no PVC needed, shared across all nodes
          - name: compile-services
            dependencies: [detect-changes]
            template: compile-services
            when: "{{tasks.detect-changes.outputs.parameters.needs-services}} == true"

          - name: compile-training
            dependencies: [detect-changes]
            template: compile-training
            when: "{{tasks.detect-changes.outputs.parameters.needs-training}} == true"

          - name: build-web-dashboard
            dependencies: [detect-changes]
            template: build-web-dashboard
            when: "{{tasks.detect-changes.outputs.parameters.needs-dashboard}} == true"

          - name: deploy-services
            dependencies: [compile-services]
            template: deploy-services
            when: "{{tasks.detect-changes.outputs.parameters.needs-services}} == true"

          - name: rebuild-ci-builder
            dependencies: [detect-changes]
            templateRef:
              name: build-ci-image
              template: build
            arguments:
              parameters:
                - name: commit-sha
                  value: "{{workflow.parameters.commit-sha}}"
                - name: dockerfile
                  value: Dockerfile.ci-builder
                - name: image-name
                  value: ci-builder
            when: "{{tasks.detect-changes.outputs.parameters.docker-images}} == true"

          - name: rebuild-ci-builder-cpu
            dependencies: [detect-changes]
            templateRef:
              name: build-ci-image
              template: build
            arguments:
              parameters:
                - name: commit-sha
                  value: "{{workflow.parameters.commit-sha}}"
                - name: dockerfile
                  value: Dockerfile.ci-builder-cpu
                - name: image-name
                  value: ci-builder-cpu
            when: "{{tasks.detect-changes.outputs.parameters.docker-images}} == true"

          - name: rebuild-runtime
            dependencies: [detect-changes]
            templateRef:
              name: build-ci-image
              template: build
            arguments:
              parameters:
                - name: commit-sha
                  value: "{{workflow.parameters.commit-sha}}"
                - name: dockerfile
                  value: Dockerfile.foxhunt-runtime
                - name: image-name
                  value: foxhunt-runtime
            when: "{{tasks.detect-changes.outputs.parameters.docker-images}} == true"

          - name: rebuild-training-runtime
            dependencies: [detect-changes]
            templateRef:
              name: build-ci-image
              template: build
            arguments:
              parameters:
                - name: commit-sha
                  value: "{{workflow.parameters.commit-sha}}"
                - name: dockerfile
                  value: Dockerfile.foxhunt-training-runtime
                - name: image-name
                  value: foxhunt-training-runtime
            when: "{{tasks.detect-changes.outputs.parameters.docker-images}} == true"

    # ── detect-changes ──
    - name: detect-changes
      nodeSelector:
        k8s.scaleway.com/pool-name: platform
      container:
        image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest
        command: ["/bin/sh", "-c"]
        env:
          - name: COMMITS_JSON
            value: "{{workflow.parameters.commits-json}}"
        resources:
          requests:
            cpu: 100m
            memory: 128Mi
          limits:
            cpu: 500m
            memory: 256Mi
        args:
          - |
            set -e
            cat <<'SCRIPT' > /tmp/detect.sh
            #!/bin/sh
            set -e

            # Extract changed file paths from webhook JSON using grep+sed (no jq dependency).
            # Matches strings inside "added":[...], "modified":[...], "removed":[...] arrays.
            CHANGED_FILES=$(echo "$COMMITS_JSON" \
              | grep -oE '"(added|modified|removed)":\[[^]]*\]' \
              | sed 's/"added"://;s/"modified"://;s/"removed"://' \
              | tr ',' '\n' | tr -d '[]"' | sed '/^$/d' | sort -u \
              || echo "")

            if [ -z "$CHANGED_FILES" ]; then
              echo "No changed files detected — triggering all builds"
              CHANGED_FILES="Cargo.toml"
            fi

            echo "=== Changed files ==="
            echo "$CHANGED_FILES"
            echo "===================="

            check_paths() {
              local patterns="$1"
              for file in $CHANGED_FILES; do
                for pattern in $patterns; do
                  case "$file" in
                    ${pattern}*) echo "true"; return ;;
                  esac
                done
              done
              echo "false"
            }

            SHARED=$(check_paths "crates/common/ crates/config/ Cargo.toml Cargo.lock")
            SVC_ONLY=$(check_paths "services/ bin/fxt/ crates/trading_engine/ crates/risk/ crates/data/ crates/storage/ crates/database/ crates/broker crates/ctrader infra/k8s/argo/ci-pipeline infra/k8s/services/")
            ML_CHANGED=$(check_paths "crates/ml/")
            DOCKER_IMAGES=$(check_paths "infra/docker/")
            NEEDS_DASHBOARD=$(check_paths "web-dashboard/")

            if [ "$SHARED" = "true" ] || [ "$SVC_ONLY" = "true" ] || [ "$ML_CHANGED" = "true" ]; then
              NEEDS_SERVICES="true"
            else
              NEEDS_SERVICES="false"
            fi

            if [ "$SHARED" = "true" ] || [ "$ML_CHANGED" = "true" ]; then
              NEEDS_TRAINING="true"
            else
              NEEDS_TRAINING="false"
            fi

            echo "=== Build decisions ==="
            echo "needs-services: $NEEDS_SERVICES"
            echo "needs-training: $NEEDS_TRAINING"
            echo "needs-dashboard: $NEEDS_DASHBOARD"
            echo "docker-images: $DOCKER_IMAGES"
            echo "======================"

            mkdir -p /tmp/outputs
            echo -n "$NEEDS_SERVICES" > /tmp/outputs/needs-services
            echo -n "$NEEDS_TRAINING" > /tmp/outputs/needs-training
            echo -n "$NEEDS_DASHBOARD" > /tmp/outputs/needs-dashboard
            echo -n "$DOCKER_IMAGES" > /tmp/outputs/docker-images
            SCRIPT
            chmod +x /tmp/detect.sh
            /tmp/detect.sh
      outputs:
        parameters:
          - name: needs-services
            valueFrom:
              path: /tmp/outputs/needs-services
          - name: needs-training
            valueFrom:
              path: /tmp/outputs/needs-training
          - name: needs-dashboard
            valueFrom:
              path: /tmp/outputs/needs-dashboard
          - name: docker-images
            valueFrom:
              path: /tmp/outputs/docker-images

    # ── build-web-dashboard: npm build + upload to MinIO ──
    - name: build-web-dashboard
      nodeSelector:
        k8s.scaleway.com/pool-name: platform
      container:
        image: node:22-alpine
        command: ["/bin/sh", "-c"]
        env:
          - name: MINIO_ACCESS_KEY
            valueFrom:
              secretKeyRef:
                name: minio-credentials
                key: access-key
          - name: MINIO_SECRET_KEY
            valueFrom:
              secretKeyRef:
                name: minio-credentials
                key: secret-key
        resources:
          requests:
            cpu: "1"
            memory: 1Gi
          limits:
            cpu: "2"
            memory: 2Gi
        volumeMounts:
          - name: git-ssh-key
            mountPath: /etc/git-ssh
            readOnly: true
        args:
          - |
            set -e
            SHA="{{workflow.parameters.commit-sha}}"

            # Git clone
            apk add --no-cache git openssh rclone
            mkdir -p ~/.ssh
            cp /etc/git-ssh/ssh-privatekey ~/.ssh/id_ed25519
            chmod 600 ~/.ssh/id_ed25519
            printf 'Host *\n  StrictHostKeyChecking no\n  UserKnownHostsFile /dev/null\n' > ~/.ssh/config
            chmod 600 ~/.ssh/config

            REPO="ssh://git@gitlab-gitlab-shell.foxhunt.svc.cluster.local:2222/root/foxhunt.git"
            WORKSPACE="/tmp/workspace"
            if [ "$SHA" = "HEAD" ]; then
              git clone --depth=1 "$REPO" "$WORKSPACE"
            else
              git clone --depth=1 "$REPO" "$WORKSPACE"
              cd "$WORKSPACE"
              git fetch --depth=1 origin "$SHA"
              git checkout "$SHA"
            fi

            cd "$WORKSPACE/web-dashboard"
            echo "=== Building web dashboard ==="
            npm ci
            npm run build

            echo "=== Uploading to MinIO ==="
            rclone copy dist/ :s3:foxhunt-binaries/web-dashboard/ \
              --s3-provider=Minio \
              --s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 \
              --s3-access-key-id="${MINIO_ACCESS_KEY}" \
              --s3-secret-access-key="${MINIO_SECRET_KEY}" \
              --s3-no-check-bucket

            echo "=== Web dashboard build + upload done ==="

    # ── compile-services: CPU-only, runs on ci-compile-cpu alongside compile-training ──
    # sccache backed by MinIO (s3://foxhunt-sccache/cpu/) — shared across nodes
    - name: compile-services
      nodeSelector:
        k8s.scaleway.com/pool-name: ci-compile-cpu
      tolerations:
        - key: node.cilium.io/agent-not-ready
          operator: Exists
          effect: NoSchedule
      container:
        image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/ci-builder-cpu:latest
        imagePullPolicy: Always
        command: ["/bin/sh", "-c"]
        env:
          - name: SQLX_OFFLINE
            value: "true"
          - name: CARGO_TERM_COLOR
            value: always
          - name: SCCACHE_BUCKET
            value: foxhunt-sccache
          - name: SCCACHE_ENDPOINT
            value: http://minio.foxhunt.svc.cluster.local:9000
          - name: SCCACHE_REGION
            value: us-east-1
          - name: SCCACHE_S3_KEY_PREFIX
            value: cpu
          - name: SCCACHE_S3_USE_SSL
            value: "false"
          - name: SCCACHE_S3_NO_CREDENTIALS
            value: "false"
          - name: AWS_ACCESS_KEY_ID
            valueFrom:
              secretKeyRef:
                name: minio-credentials
                key: access-key
          - name: AWS_SECRET_ACCESS_KEY
            valueFrom:
              secretKeyRef:
                name: minio-credentials
                key: secret-key
          - name: MINIO_ACCESS_KEY
            valueFrom:
              secretKeyRef:
                name: minio-credentials
                key: access-key
          - name: MINIO_SECRET_KEY
            valueFrom:
              secretKeyRef:
                name: minio-credentials
                key: secret-key
        resources:
          requests:
            cpu: "14"
            memory: 16Gi
          limits:
            cpu: "30"
            memory: 32Gi
        volumeMounts:
          - name: git-ssh-key
            mountPath: /etc/git-ssh
            readOnly: true
        args:
          - |
            set -e
            SHA="{{workflow.parameters.commit-sha}}"

            # Git clone (self-contained)
            mkdir -p ~/.ssh
            cp /etc/git-ssh/ssh-privatekey ~/.ssh/id_ed25519
            chmod 600 ~/.ssh/id_ed25519
            printf 'Host *\n  StrictHostKeyChecking no\n  UserKnownHostsFile /dev/null\n' > ~/.ssh/config
            chmod 600 ~/.ssh/config

            REPO="ssh://git@gitlab-gitlab-shell.foxhunt.svc.cluster.local:2222/root/foxhunt.git"
            WORKSPACE="/tmp/workspace"
            if [ "$SHA" = "HEAD" ]; then
              git clone --depth=1 "$REPO" "$WORKSPACE"
            else
              git clone --depth=1 "$REPO" "$WORKSPACE"
              cd "$WORKSPACE"
              git fetch --depth=1 origin "$SHA"
              git checkout "$SHA"
            fi
            cd "$WORKSPACE"
            echo "Checked out $(git rev-parse --short HEAD)"

            YEAR=$(date +%Y)
            MONTH=$(date +%m)
            export FOXHUNT_BUILD_VERSION="${YEAR}.${MONTH}.argo"
            export RUSTC_WRAPPER=sccache
            sccache --zero-stats || true

            echo "=== Building service binaries (sccache → MinIO s3://foxhunt-sccache/cpu/) ==="
            cargo build --release \
              -p api -p trading-service -p ml-training-service \
              -p backtesting-service -p trading-agent-service \
              -p broker-gateway -p data-acquisition-service

            mkdir -p "$WORKSPACE/bin/services"
            for pkg in api trading-service ml-training-service backtesting-service trading-agent-service broker-gateway data-acquisition-service; do
              bin_name=$(echo "$pkg" | tr '-' '_')
              cp "target/release/$pkg" "$WORKSPACE/bin/services/" 2>/dev/null \
                || cp "target/release/$bin_name" "$WORKSPACE/bin/services/" 2>/dev/null \
                || { echo "Binary not found for $pkg"; ls target/release/; exit 1; }
            done
            strip "$WORKSPACE/bin/services/"*

            echo "=== Service binaries ==="
            ls -lh "$WORKSPACE/bin/services/"
            sccache --show-stats || true

            echo "=== Uploading service binaries ==="
            rclone copy "$WORKSPACE/bin/services/" :s3:foxhunt-binaries/services/ \
              --s3-provider=Minio \
              --s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 \
              --s3-access-key-id="${MINIO_ACCESS_KEY}" \
              --s3-secret-access-key="${MINIO_SECRET_KEY}" \
              --s3-no-check-bucket

            echo "=== Service compile + upload done ==="

    # ── compile-training: CUDA build, needs POP2 for speed ──
    # sccache backed by MinIO (s3://foxhunt-sccache/cuda/sm_XX/) — persists across ephemeral nodes
    - name: compile-training
      nodeSelector:
        k8s.scaleway.com/pool-name: ci-compile-cpu
      tolerations:
        - key: node.cilium.io/agent-not-ready
          operator: Exists
          effect: NoSchedule
      container:
        image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/ci-builder:latest
        imagePullPolicy: Always
        command: ["/bin/sh", "-c"]
        env:
          - name: SQLX_OFFLINE
            value: "true"
          - name: CARGO_TERM_COLOR
            value: always
          - name: SCCACHE_BUCKET
            value: foxhunt-sccache
          - name: SCCACHE_ENDPOINT
            value: http://minio.foxhunt.svc.cluster.local:9000
          - name: SCCACHE_REGION
            value: us-east-1
          - name: SCCACHE_S3_USE_SSL
            value: "false"
          - name: SCCACHE_S3_NO_CREDENTIALS
            value: "false"
          - name: AWS_ACCESS_KEY_ID
            valueFrom:
              secretKeyRef:
                name: minio-credentials
                key: access-key
          - name: AWS_SECRET_ACCESS_KEY
            valueFrom:
              secretKeyRef:
                name: minio-credentials
                key: secret-key
          - name: MINIO_ACCESS_KEY
            valueFrom:
              secretKeyRef:
                name: minio-credentials
                key: access-key
          - name: MINIO_SECRET_KEY
            valueFrom:
              secretKeyRef:
                name: minio-credentials
                key: secret-key
        resources:
          requests:
            cpu: "14"
            memory: 32Gi
          limits:
            cpu: "30"
            memory: 64Gi
        volumeMounts:
          - name: git-ssh-key
            mountPath: /etc/git-ssh
            readOnly: true
        args:
          - |
            set -e
            SHA="{{workflow.parameters.commit-sha}}"

            # Git clone (self-contained — ephemeral node)
            mkdir -p ~/.ssh
            cp /etc/git-ssh/ssh-privatekey ~/.ssh/id_ed25519
            chmod 600 ~/.ssh/id_ed25519
            printf 'Host *\n  StrictHostKeyChecking no\n  UserKnownHostsFile /dev/null\n' > ~/.ssh/config
            chmod 600 ~/.ssh/config

            REPO="ssh://git@gitlab-gitlab-shell.foxhunt.svc.cluster.local:2222/root/foxhunt.git"
            WORKSPACE="/tmp/workspace"
            if [ "$SHA" = "HEAD" ]; then
              git clone --depth=1 "$REPO" "$WORKSPACE"
            else
              git clone --depth=1 "$REPO" "$WORKSPACE"
              cd "$WORKSPACE"
              git fetch --depth=1 origin "$SHA"
              git checkout "$SHA"
            fi
            cd "$WORKSPACE"
            echo "Checked out $(git rev-parse --short HEAD)"

            YEAR=$(date +%Y)
            MONTH=$(date +%m)
            export FOXHUNT_BUILD_VERSION="${YEAR}.${MONTH}.argo"
            export RUSTC_WRAPPER=sccache
            export CUDA_COMPUTE_CAP={{workflow.parameters.cuda-compute-cap}}
            export SCCACHE_S3_KEY_PREFIX="cuda/sm_${CUDA_COMPUTE_CAP}"
            sccache --zero-stats || true

            echo "=== Building training binaries (CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP, sccache → MinIO) ==="
            cargo build --release -p ml --features ml/cuda \
              --example train_baseline_rl --example train_baseline_supervised \
              --example evaluate_baseline --example evaluate_supervised \
              --example hyperopt_baseline_rl --example hyperopt_baseline_supervised

            cargo build --release -p training_uploader

            mkdir -p "$WORKSPACE/bin/training"
            for bin in train_baseline_rl train_baseline_supervised evaluate_baseline evaluate_supervised hyperopt_baseline_rl hyperopt_baseline_supervised; do
              cp target/release/examples/$bin "$WORKSPACE/bin/training/"
            done
            cp target/release/training_uploader "$WORKSPACE/bin/training/"
            strip "$WORKSPACE/bin/training/"*

            echo "=== Training binaries ==="
            ls -lh "$WORKSPACE/bin/training/"
            sccache --show-stats || true

            echo "=== Uploading training binaries ==="
            rclone copy "$WORKSPACE/bin/training/" :s3:foxhunt-binaries/training/ \
              --s3-provider=Minio \
              --s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 \
              --s3-access-key-id="${MINIO_ACCESS_KEY}" \
              --s3-secret-access-key="${MINIO_SECRET_KEY}" \
              --s3-no-check-bucket

            echo "=== Training compile + upload done ==="

    # ── deploy-services: rolling restart after new binaries are uploaded ──
    - name: deploy-services
      serviceAccountName: argo-workflow
      nodeSelector:
        k8s.scaleway.com/pool-name: platform
      container:
        image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest
        command: ["/bin/sh", "-c"]
        resources:
          requests:
            cpu: 100m
            memory: 64Mi
          limits:
            cpu: 200m
            memory: 128Mi
        args:
          - |
            set -e
            # kubectl baked into foxhunt-runtime image; fallback to /tmp for non-root
            if ! command -v kubectl >/dev/null 2>&1; then
              echo "=== Installing kubectl ==="
              curl -sLo /tmp/kubectl "https://dl.k8s.io/release/v1.31.4/bin/linux/amd64/kubectl"
              chmod +x /tmp/kubectl
              export PATH="/tmp:$PATH"
            fi

            echo "=== Rolling restart of service deployments ==="
            SERVICES="api trading-service ml-training-service backtesting-service trading-agent-service broker-gateway data-acquisition-service"
            for svc in $SERVICES; do
              echo "Restarting $svc..."
              kubectl -n foxhunt rollout restart deployment "$svc" || echo "WARN: $svc restart failed (may not exist)"
            done

            echo "=== Waiting for rollouts ==="
            for svc in $SERVICES; do
              kubectl -n foxhunt rollout status deployment "$svc" --timeout=120s || echo "WARN: $svc rollout timeout"
            done

            echo "=== Deploy complete ==="