foxhunt/infra/k8s/argo/training-workflow-template.yaml

apiVersion: argoproj.io/v1alpha1
kind: WorkflowTemplate
metadata:
  name: training-pipeline
  namespace: foxhunt
  labels:
    app.kubernetes.io/name: training-pipeline
    app.kubernetes.io/part-of: foxhunt
spec:
  entrypoint: train-model
  serviceAccountName: argo-workflow
  # Label all workflow pods so network policies (MinIO, DNS) allow traffic
  podMetadata:
    labels:
      app.kubernetes.io/part-of: foxhunt
      app.kubernetes.io/component: training-workflow
  securityContext:
    runAsUser: 1000
    runAsGroup: 1000
    fsGroup: 1000
  ttlStrategy:
    secondsAfterCompletion: 3600
  activeDeadlineSeconds: 21600

  arguments:
    parameters:
      - name: model
      - name: gpu-pool
        value: ci-training-h100  # Pools: ci-training-h100, ci-training-l40s, ci-training-h100x2
      - name: hyperopt-trials
        value: "20"
      - name: hyperopt-epochs
        value: "8"
      - name: train-epochs
        value: "50"
      - name: symbol
        value: ES.FUT
      - name: data-dir
        value: /tmp/futures-baseline
      - name: tx-cost-bps
        value: "0.1"
      - name: tick-size
        value: "0.25"
      - name: spread-ticks
        value: "1.0"
      - name: initial-capital
        value: "35000"
      - name: ensemble-top-k
        value: "5"
      - name: cuda-compute-cap
        value: "90"

  # Ephemeral shared storage for passing data between DAG steps.
  # Auto-deleted when workflow completes (all artifacts uploaded to MinIO).
  volumeClaimTemplates:
    - metadata:
        name: workspace
      spec:
        accessModes: ["ReadWriteOnce"]
        storageClassName: scw-bssd
        resources:
          requests:
            storage: 5Gi

  templates:
    - name: train-model
      dag:
        tasks:
          - name: fetch-binary
            template: fetch-binary
          - name: gpu-warmup
            template: gpu-warmup
          - name: hyperopt
            template: hyperopt
            dependencies: [fetch-binary, gpu-warmup]
          - name: train-best
            template: train-best
            dependencies: [hyperopt]
          - name: evaluate
            template: evaluate
            dependencies: [train-best]
          - name: upload-results
            template: upload-results
            dependencies: [evaluate]

    # GPU warmup: triggers H100 autoscale during compilation so the node
    # is ready when hyperopt starts. Exits immediately after nvidia-smi.
    - name: gpu-warmup
      nodeSelector:
        k8s.scaleway.com/pool-name: "{{workflow.parameters.gpu-pool}}"
      tolerations:
        - key: nvidia.com/gpu
          operator: Exists
          effect: NoSchedule
        - key: node.cilium.io/agent-not-ready
          operator: Exists
          effect: NoSchedule
      container:
        image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-training-runtime:latest
        imagePullPolicy: IfNotPresent
        command: ["/bin/sh", "-c"]
        args:
          - |
            echo "GPU warmup: triggering node autoscale..."
            nvidia-smi --query-gpu=name,memory.total --format=csv,noheader
            echo "GPU node ready, exiting to free resources"
      resources:
        requests:
          nvidia.com/gpu: "1"
          cpu: 100m
          memory: 128Mi
        limits:
          nvidia.com/gpu: "1"
          cpu: 200m
          memory: 256Mi

    - name: fetch-binary
      nodeSelector:
        k8s.scaleway.com/pool-name: platform
      container:
        image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest
        imagePullPolicy: IfNotPresent
        command: ["/bin/sh", "-c"]
        args:
          - |
            set -e
            mkdir -p /workspace/bin
            echo "Fetching training binaries from MinIO..."
            rclone copy :s3:foxhunt-binaries/training/ /workspace/bin/ \
              --s3-provider=Minio \
              --s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 \
              --s3-access-key-id="${MINIO_ACCESS_KEY}" \
              --s3-secret-access-key="${MINIO_SECRET_KEY}" \
              --s3-no-check-bucket \
            chmod +x /workspace/bin/*
            echo "Fetched binaries:"
            ls -lh /workspace/bin/
        env:
          - name: MINIO_ACCESS_KEY
            valueFrom:
              secretKeyRef:
                name: minio-credentials
                key: access-key
          - name: MINIO_SECRET_KEY
            valueFrom:
              secretKeyRef:
                name: minio-credentials
                key: secret-key
        volumeMounts:
          - name: workspace
            mountPath: /workspace
        resources:
          requests:
            cpu: 100m
            memory: 128Mi
          limits:
            cpu: 500m
            memory: 256Mi

    - name: hyperopt
      nodeSelector:
        k8s.scaleway.com/pool-name: "{{workflow.parameters.gpu-pool}}"
      tolerations:
        - key: nvidia.com/gpu
          operator: Exists
          effect: NoSchedule
        - key: node.cilium.io/agent-not-ready
          operator: Exists
          effect: NoSchedule
      container:
        image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-training-runtime:latest
        imagePullPolicy: IfNotPresent
        command: ["/bin/sh", "-c"]
        args:
          - |
            set -e
            export PATH="/workspace/bin:$PATH"
            export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//')
            nvidia-smi

            # Download training data from MinIO → /tmp (no PVC needed)
            echo "=== Downloading training data from MinIO ==="
            rclone copy :s3:foxhunt-training-data/futures-baseline/ /tmp/futures-baseline/ \
              --s3-provider=Minio \
              --s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 \
              --s3-access-key-id="${MINIO_ACCESS_KEY}" \
              --s3-secret-access-key="${MINIO_SECRET_KEY}" \
              --s3-no-check-bucket \
              --transfers=8
            echo "Training data: $(find /tmp/futures-baseline -name '*.dbn.zst' | wc -l) files"

            mkdir -p /workspace/output/hyperopt

            MODEL="{{workflow.parameters.model}}"
            case "$MODEL" in
              dqn|ppo) BINARY=hyperopt_baseline_rl ;;
              *)       BINARY=hyperopt_baseline_supervised ;;
            esac

            TRIALS={{workflow.parameters.hyperopt-trials}}
            N_INITIAL=5
            if [ "$TRIALS" -le "$N_INITIAL" ]; then
              N_INITIAL=$((TRIALS > 1 ? TRIALS - 1 : 1))
            fi

            echo "Running $BINARY --model $MODEL ($TRIALS trials, $N_INITIAL initial x {{workflow.parameters.hyperopt-epochs}} epochs)"
            $BINARY \
              --model "$MODEL" \
              --trials "$TRIALS" \
              --n-initial "$N_INITIAL" \
              --epochs {{workflow.parameters.hyperopt-epochs}} \
              --parallel 0 \
              --symbol {{workflow.parameters.symbol}} \
              --tx-cost-bps {{workflow.parameters.tx-cost-bps}} \
              --tick-size {{workflow.parameters.tick-size}} \
              --spread-ticks {{workflow.parameters.spread-ticks}} \
              --initial-capital {{workflow.parameters.initial-capital}} \
              --data-dir {{workflow.parameters.data-dir}} \
              --base-dir /workspace/output/hyperopt \
              --output /workspace/output/${MODEL}_hyperopt_results.json

            echo "=== Hyperopt results ==="
            cat /workspace/output/${MODEL}_hyperopt_results.json 2>/dev/null || echo "No results file"
        env:
          - name: RUST_LOG
            value: info
          - name: SQLX_OFFLINE
            value: "true"
          - name: OTEL_EXPORTER_OTLP_ENDPOINT
            value: "http://tempo.foxhunt.svc.cluster.local:4317"
          - name: CUBLAS_WORKSPACE_CONFIG
            value: ":4096:8"
          - name: MINIO_ACCESS_KEY
            valueFrom:
              secretKeyRef:
                name: minio-credentials
                key: access-key
          - name: MINIO_SECRET_KEY
            valueFrom:
              secretKeyRef:
                name: minio-credentials
                key: secret-key
        volumeMounts:
          - name: workspace
            mountPath: /workspace
        resources:
          requests:
            nvidia.com/gpu: "1"
            cpu: "7"
            memory: 16Gi
          limits:
            nvidia.com/gpu: "1"
            cpu: "8"
            memory: 48Gi

    - name: train-best
      nodeSelector:
        k8s.scaleway.com/pool-name: "{{workflow.parameters.gpu-pool}}"
      tolerations:
        - key: nvidia.com/gpu
          operator: Exists
          effect: NoSchedule
        - key: node.cilium.io/agent-not-ready
          operator: Exists
          effect: NoSchedule
      container:
        image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-training-runtime:latest
        imagePullPolicy: IfNotPresent
        command: ["/bin/sh", "-c"]
        args:
          - |
            set -e
            export PATH="/workspace/bin:$PATH"
            export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//')
            nvidia-smi

            # Download training data from MinIO → /tmp
            echo "=== Downloading training data from MinIO ==="
            rclone copy :s3:foxhunt-training-data/futures-baseline/ /tmp/futures-baseline/ \
              --s3-provider=Minio \
              --s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 \
              --s3-access-key-id="${MINIO_ACCESS_KEY}" \
              --s3-secret-access-key="${MINIO_SECRET_KEY}" \
              --s3-no-check-bucket \
              --transfers=8
            echo "Training data: $(find /tmp/futures-baseline -name '*.dbn.zst' | wc -l) files"

            MODEL="{{workflow.parameters.model}}"
            case "$MODEL" in
              dqn|ppo) BINARY=train_baseline_rl ;;
              *)       BINARY=train_baseline_supervised ;;
            esac

            HYPEROPT_FILE="/workspace/output/${MODEL}_hyperopt_results.json"
            HYPEROPT_FLAG=""
            if [ -f "$HYPEROPT_FILE" ]; then
              HYPEROPT_FLAG="--hyperopt-params $HYPEROPT_FILE"
            fi

            echo "Training $MODEL with best params ({{workflow.parameters.train-epochs}} epochs)"
            $BINARY \
              --model "$MODEL" \
              --symbol {{workflow.parameters.symbol}} \
              --tx-cost-bps {{workflow.parameters.tx-cost-bps}} \
              --tick-size {{workflow.parameters.tick-size}} \
              --spread-ticks {{workflow.parameters.spread-ticks}} \
              --data-dir {{workflow.parameters.data-dir}} \
              --output-dir /workspace/output \
              --max-steps-per-epoch {{workflow.parameters.train-epochs}} \
              $HYPEROPT_FLAG \
              --ensemble-top-k {{workflow.parameters.ensemble-top-k}}

            echo "=== Training complete ==="
            ls -lh /workspace/output/
        env:
          - name: RUST_LOG
            value: info
          - name: SQLX_OFFLINE
            value: "true"
          - name: OTEL_EXPORTER_OTLP_ENDPOINT
            value: "http://tempo.foxhunt.svc.cluster.local:4317"
          - name: CUBLAS_WORKSPACE_CONFIG
            value: ":4096:8"
          - name: MINIO_ACCESS_KEY
            valueFrom:
              secretKeyRef:
                name: minio-credentials
                key: access-key
          - name: MINIO_SECRET_KEY
            valueFrom:
              secretKeyRef:
                name: minio-credentials
                key: secret-key
        volumeMounts:
          - name: workspace
            mountPath: /workspace
        resources:
          requests:
            nvidia.com/gpu: "1"
            cpu: "7"
            memory: 16Gi
          limits:
            nvidia.com/gpu: "1"
            cpu: "8"
            memory: 48Gi

    - name: evaluate
      nodeSelector:
        k8s.scaleway.com/pool-name: "{{workflow.parameters.gpu-pool}}"
      tolerations:
        - key: nvidia.com/gpu
          operator: Exists
          effect: NoSchedule
        - key: node.cilium.io/agent-not-ready
          operator: Exists
          effect: NoSchedule
      container:
        image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-training-runtime:latest
        imagePullPolicy: IfNotPresent
        command: ["/bin/sh", "-c"]
        args:
          - |
            set -e
            export PATH="/workspace/bin:$PATH"
            export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//')

            # Download training data from MinIO → /tmp
            echo "=== Downloading training data from MinIO ==="
            rclone copy :s3:foxhunt-training-data/futures-baseline/ /tmp/futures-baseline/ \
              --s3-provider=Minio \
              --s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 \
              --s3-access-key-id="${MINIO_ACCESS_KEY}" \
              --s3-secret-access-key="${MINIO_SECRET_KEY}" \
              --s3-no-check-bucket \
              --transfers=8

            MODEL="{{workflow.parameters.model}}"
            HYPEROPT_FILE="/workspace/output/${MODEL}_hyperopt_results.json"
            HYPEROPT_FLAG=""
            if [ -f "$HYPEROPT_FILE" ]; then
              HYPEROPT_FLAG="--hyperopt-params $HYPEROPT_FILE"
            fi

            # Evaluate all ensemble members (evaluator auto-discovers *_ensemble_*.safetensors)
            echo "Evaluating $MODEL"
            evaluate_baseline \
              --model "$MODEL" \
              --models-dir /workspace/output \
              --data-dir {{workflow.parameters.data-dir}} \
              --output /workspace/output/${MODEL}_eval_report.json \
              --symbol {{workflow.parameters.symbol}} \
              --tx-cost-bps {{workflow.parameters.tx-cost-bps}} \
              --tick-size {{workflow.parameters.tick-size}} \
              --spread-ticks {{workflow.parameters.spread-ticks}} \
              $HYPEROPT_FLAG \
            || true

            echo "=== Eval report ==="
            cat /workspace/output/${MODEL}_eval_report.json 2>/dev/null || echo "No eval report"
        env:
          - name: RUST_LOG
            value: info
          - name: SQLX_OFFLINE
            value: "true"
          - name: MINIO_ACCESS_KEY
            valueFrom:
              secretKeyRef:
                name: minio-credentials
                key: access-key
          - name: MINIO_SECRET_KEY
            valueFrom:
              secretKeyRef:
                name: minio-credentials
                key: secret-key
        volumeMounts:
          - name: workspace
            mountPath: /workspace
        resources:
          requests:
            nvidia.com/gpu: "1"
            cpu: "4"
            memory: 16Gi
          limits:
            nvidia.com/gpu: "1"
            cpu: "8"
            memory: 32Gi

    - name: upload-results
      nodeSelector:
        k8s.scaleway.com/pool-name: platform
      container:
        image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest
        imagePullPolicy: IfNotPresent
        command: ["/bin/sh", "-c"]
        args:
          - |
            set -e
            MODEL="{{workflow.parameters.model}}"
            SYMBOL="{{workflow.parameters.symbol}}"
            TIMESTAMP=$(date +%Y%m%d-%H%M%S)
            DEST="s3:foxhunt-training-results/${MODEL}/${SYMBOL}/${TIMESTAMP}/"

            echo "Uploading all training artifacts to MinIO: $DEST"
            echo "  checkpoints (*.safetensors), norm stats, hyperopt results, eval reports"
            rclone copy /workspace/output/ ":${DEST}" \
              --s3-provider=Minio \
              --s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 \
              --s3-access-key-id="${MINIO_ACCESS_KEY}" \
              --s3-secret-access-key="${MINIO_SECRET_KEY}" \
              --s3-no-check-bucket \

            echo "=== Upload complete ==="
            rclone ls ":${DEST}" \
              --s3-provider=Minio \
              --s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 \
              --s3-access-key-id="${MINIO_ACCESS_KEY}" \
              --s3-secret-access-key="${MINIO_SECRET_KEY}" \
              --s3-no-check-bucket \
        env:
          - name: MINIO_ACCESS_KEY
            valueFrom:
              secretKeyRef:
                name: minio-credentials
                key: access-key
          - name: MINIO_SECRET_KEY
            valueFrom:
              secretKeyRef:
                name: minio-credentials
                key: secret-key
        volumeMounts:
          - name: workspace
            mountPath: /workspace
        resources:
          requests:
            cpu: 100m
            memory: 128Mi
          limits:
            cpu: 500m
            memory: 256Mi