From f8e0f459fcd70277deff205cf26a1d0a9cf67cde Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Fri, 10 Apr 2026 20:34:08 +0200 Subject: [PATCH] =?UTF-8?q?feat:=20unified=20train=20workflow=20template?= =?UTF-8?q?=20=E2=80=94=20cache-or-compile=20+=20cache-or-precompute?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.6 (1M context) --- infra/k8s/argo/train-template.yaml | 679 +++++++++++++++++++++++++++++ 1 file changed, 679 insertions(+) create mode 100644 infra/k8s/argo/train-template.yaml diff --git a/infra/k8s/argo/train-template.yaml b/infra/k8s/argo/train-template.yaml new file mode 100644 index 000000000..5628d28df --- /dev/null +++ b/infra/k8s/argo/train-template.yaml @@ -0,0 +1,679 @@ +# Unified training workflow — replaces all model-specific training templates. +# +# Key innovation: binary caching by commit SHA on PVC. +# Check /data/bin/$SHA first; compile only on cache miss. +# +# DAG: +# ensure-binary ──┐ +# gpu-warmup ─────┼──> ensure-fxcache ──> hyperopt ──> train-best ──> evaluate ──> upload-results +# +# Usage: +# argo submit -n foxhunt --from=wftmpl/train +# argo submit -n foxhunt --from=wftmpl/train -p model=ppo -p train-epochs=100 +# argo submit -n foxhunt --from=wftmpl/train -p hyperopt-trials=0 # skip hyperopt +--- +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: train + namespace: foxhunt + labels: + app.kubernetes.io/name: train + app.kubernetes.io/part-of: foxhunt + app.kubernetes.io/component: training +spec: + entrypoint: pipeline + onExit: notify-result + serviceAccountName: argo-workflow + podMetadata: + labels: + app.kubernetes.io/part-of: foxhunt + app.kubernetes.io/component: training + securityContext: + fsGroup: 0 + ttlStrategy: + secondsAfterCompletion: 3600 + activeDeadlineSeconds: 28800 # 8 hours + + arguments: + parameters: + - name: commit-sha + value: HEAD + - name: git-branch + value: main + - name: cuda-compute-cap + value: "90" + - name: model + value: dqn + - name: gpu-pool + value: ci-training-h100 + - name: hyperopt-trials + value: "20" + - name: hyperopt-epochs + value: "8" + - name: train-epochs + value: "50" + - name: symbol + value: ES.FUT + - name: initial-capital + value: "35000" + - name: tx-cost-bps + value: "0.1" + - name: tick-size + value: "0.25" + - name: spread-ticks + value: "1.0" + - name: min-hold-bars + value: "5" + + volumes: + - name: git-ssh-key + secret: + secretName: argo-git-ssh-key + defaultMode: 256 + - name: training-data + persistentVolumeClaim: + claimName: training-data-pvc + - name: cargo-target-cuda + persistentVolumeClaim: + claimName: cargo-target-cuda + - name: feature-cache + persistentVolumeClaim: + claimName: feature-cache-pvc + + volumeClaimTemplates: + - metadata: + name: workspace + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: scw-bssd + resources: + requests: + storage: 5Gi + + templates: + # ── DAG: orchestrate all steps ── + - name: pipeline + dag: + tasks: + - name: ensure-binary + template: ensure-binary + - name: gpu-warmup + template: gpu-warmup + - name: ensure-fxcache + template: ensure-fxcache + dependencies: [ensure-binary] + - name: hyperopt + template: hyperopt + dependencies: [ensure-binary, gpu-warmup, ensure-fxcache] + when: "{{workflow.parameters.hyperopt-trials}} != 0" + - name: train-best + template: train-best + dependencies: [ensure-binary, gpu-warmup, ensure-fxcache, hyperopt] + - name: evaluate + template: evaluate + dependencies: [train-best] + - name: upload-results + template: upload-results + dependencies: [evaluate] + + # ── ensure-binary: cache-or-compile training binaries by commit SHA ── + - name: ensure-binary + outputs: + parameters: + - name: sha + valueFrom: + path: /tmp/sha + nodeSelector: + k8s.scaleway.com/pool-name: ci-compile-cpu + tolerations: + - key: node.cilium.io/agent-not-ready + operator: Exists + effect: NoSchedule + container: + image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/ci-builder:latest + imagePullPolicy: Always + command: ["/bin/bash", "-c"] + env: + - name: SQLX_OFFLINE + value: "true" + - name: CARGO_TERM_COLOR + value: always + - name: CARGO_TARGET_DIR + value: /cargo-target + - name: CARGO_HOME + value: /cargo-target/cargo-home + - name: GITLAB_PAT + valueFrom: + secretKeyRef: + name: gitlab-pat + key: token + resources: + requests: + cpu: "14" + memory: 32Gi + limits: + cpu: "30" + memory: 64Gi + volumeMounts: + - name: git-ssh-key + mountPath: /etc/git-ssh + readOnly: true + - name: cargo-target-cuda + mountPath: /cargo-target + - name: training-data + mountPath: /data + args: + - | + set -e + SHA="{{workflow.parameters.commit-sha}}" + BRANCH="{{workflow.parameters.git-branch}}" + MODEL="{{workflow.parameters.model}}" + + # ── SSH setup ── + mkdir -p ~/.ssh + cp /etc/git-ssh/ssh-privatekey ~/.ssh/id_ed25519 + chmod 600 ~/.ssh/id_ed25519 + printf 'Host *\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n' > ~/.ssh/config + chmod 600 ~/.ssh/config + + REPO="ssh://git@gitlab-gitlab-shell.foxhunt.svc.cluster.local:2222/root/foxhunt.git" + BUILD="/cargo-target/src" + + git config --global --add safe.directory "$BUILD" + + # ── Resolve SHA ── + if [ "$SHA" = "HEAD" ]; then + if [ -d "$BUILD/.git" ]; then + cd "$BUILD" + git fetch origin + SHA=$(git rev-parse "origin/$BRANCH") + cd / + else + git clone --filter=blob:none "$REPO" "$BUILD" + cd "$BUILD" + git checkout "origin/$BRANCH" + SHA=$(git rev-parse HEAD) + cd / + fi + fi + SHORT_SHA=$(echo "$SHA" | cut -c1-9) + echo "Resolved SHA: $SHA (short: $SHORT_SHA)" + + # ── Derive needed binaries from model ── + case "$MODEL" in + dqn|ppo) + BINARIES="hyperopt_baseline_rl train_baseline_rl evaluate_baseline precompute_features" + ;; + *) + BINARIES="hyperopt_baseline_supervised train_baseline_supervised evaluate_supervised precompute_features" + ;; + esac + + # ── Cache check ── + BIN_DIR="/data/bin/$SHORT_SHA" + ALL_CACHED=true + for bin in $BINARIES; do + if [ ! -x "$BIN_DIR/$bin" ]; then + ALL_CACHED=false + break + fi + done + + if [ "$ALL_CACHED" = "true" ]; then + echo "=== Cache HIT: all binaries present in $BIN_DIR ===" + ls -lh "$BIN_DIR/" + echo "$SHORT_SHA" > /tmp/sha + exit 0 + fi + + echo "=== Cache MISS: compiling binaries for $SHORT_SHA ===" + + # ── Clone / checkout ── + if [ -d "$BUILD/.git" ]; then + cd "$BUILD" + git fetch origin + CURRENT=$(git rev-parse HEAD 2>/dev/null || echo "none") + if [ "$CURRENT" != "$SHA" ]; then + echo "Updating checkout: $(echo $CURRENT | cut -c1-8) -> $SHORT_SHA" + git checkout --force "$SHA" + git clean -fd + fi + else + git clone --filter=blob:none "$REPO" "$BUILD" + cd "$BUILD" + git checkout "$SHA" + fi + + # ── Build ── + export PATH="${CARGO_HOME}/bin:${PATH}" + export CUDA_COMPUTE_CAP={{workflow.parameters.cuda-compute-cap}} + + ML_EXAMPLE_ARGS="" + for ex in $BINARIES; do + ML_EXAMPLE_ARGS="$ML_EXAMPLE_ARGS --example $ex" + done + + echo "Building: $BINARIES" + echo " CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP" + cargo build --release -p ml --features ml/cuda $ML_EXAMPLE_ARGS + + # ── Install to cache dir ── + mkdir -p "$BIN_DIR" + for bin in $BINARIES; do + cp "$CARGO_TARGET_DIR/release/examples/$bin" "$BIN_DIR/" + done + strip "$BIN_DIR/"* + + echo "=== Cached binaries ===" + ls -lh "$BIN_DIR/" + + # ── Prune old SHAs: keep last 5 ── + cd /data/bin + ls -1t | tail -n +6 | while read -r old; do + echo "Pruning old cache: $old" + rm -rf "$old" + done + + echo "$SHORT_SHA" > /tmp/sha + + # ── ensure-fxcache: precompute feature cache if needed ── + - name: ensure-fxcache + nodeSelector: + k8s.scaleway.com/pool-name: ci-compile-cpu + tolerations: + - key: node.cilium.io/agent-not-ready + operator: Exists + effect: NoSchedule + container: + image: ubuntu:24.04 + command: ["/bin/bash", "-c"] + env: + - name: RUST_LOG + value: info + resources: + requests: + cpu: "4" + memory: 16Gi + limits: + cpu: "28" + memory: 56Gi + volumeMounts: + - name: training-data + mountPath: /data + readOnly: true + - name: feature-cache + mountPath: /feature-cache + args: + - | + set -e + SHA="{{tasks.ensure-binary.outputs.parameters.sha}}" + BINARY="/data/bin/$SHA/precompute_features" + + if [ ! -x "$BINARY" ]; then + echo "ERROR: precompute_features not found at $BINARY" + exit 1 + fi + + echo "=== Running precompute_features (SHA: $SHA) ===" + $BINARY \ + --data-dir /data/futures-baseline \ + --mbp10-data-dir /data/futures-baseline-mbp10 \ + --trades-data-dir /data/futures-baseline-trades \ + --output-dir /feature-cache \ + --symbol {{workflow.parameters.symbol}} \ + --yes + + echo "=== Feature cache ready ===" + + # ── gpu-warmup: trigger GPU node autoscale during compile ── + - name: gpu-warmup + nodeSelector: + k8s.scaleway.com/pool-name: "{{workflow.parameters.gpu-pool}}" + tolerations: + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + - key: node.cilium.io/agent-not-ready + operator: Exists + effect: NoSchedule + container: + image: busybox:1.37 + command: ["/bin/sh", "-c"] + args: + - | + echo "GPU warmup: triggering node autoscale..." + echo "GPU node scheduled, exiting to free resources" + resources: + requests: + nvidia.com/gpu: "1" + cpu: 100m + memory: 64Mi + limits: + nvidia.com/gpu: "1" + cpu: 200m + memory: 128Mi + + # ── hyperopt: PSO/TPE hyperparameter optimization on GPU ── + - name: hyperopt + nodeSelector: + k8s.scaleway.com/pool-name: "{{workflow.parameters.gpu-pool}}" + tolerations: + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + - key: node.cilium.io/agent-not-ready + operator: Exists + effect: NoSchedule + container: + image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/ci-builder:latest + imagePullPolicy: IfNotPresent + command: ["/bin/sh", "-c"] + env: + - name: RUST_LOG + value: info + - name: SQLX_OFFLINE + value: "true" + - name: CUBLAS_WORKSPACE_CONFIG + value: ":4096:8" + - name: FOXHUNT_FEATURE_CACHE_DIR + value: /feature-cache + resources: + requests: + nvidia.com/gpu: "1" + cpu: "4" + memory: 32Gi + limits: + nvidia.com/gpu: "1" + cpu: "8" + memory: 64Gi + volumeMounts: + - name: workspace + mountPath: /workspace + - name: training-data + mountPath: /data + readOnly: true + - name: feature-cache + mountPath: /feature-cache + readOnly: true + args: + - | + set -e + SHA="{{tasks.ensure-binary.outputs.parameters.sha}}" + export PATH="/data/bin/$SHA:$PATH" + export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//') + nvidia-smi + + mkdir -p /workspace/output/hyperopt + + MODEL="{{workflow.parameters.model}}" + case "$MODEL" in + dqn|ppo) BINARY=hyperopt_baseline_rl ;; + *) BINARY=hyperopt_baseline_supervised ;; + esac + + echo "=== Running hyperopt: $MODEL ($BINARY) ===" + echo " Trials: {{workflow.parameters.hyperopt-trials}}, Epochs: {{workflow.parameters.hyperopt-epochs}}" + + ${BINARY} \ + --model "$MODEL" \ + --phase fast \ + --trials {{workflow.parameters.hyperopt-trials}} \ + --epochs {{workflow.parameters.hyperopt-epochs}} \ + --symbol {{workflow.parameters.symbol}} \ + --tx-cost-bps {{workflow.parameters.tx-cost-bps}} \ + --tick-size {{workflow.parameters.tick-size}} \ + --spread-ticks {{workflow.parameters.spread-ticks}} \ + --initial-capital {{workflow.parameters.initial-capital}} \ + --min-hold-bars {{workflow.parameters.min-hold-bars}} \ + --data-dir /data/futures-baseline \ + --mbp10-data-dir /data/futures-baseline-mbp10 \ + --trades-data-dir /data/futures-baseline-trades \ + --base-dir /workspace/output/hyperopt \ + --output /workspace/output/${MODEL}_hyperopt_results.json + + echo "=== Hyperopt complete ===" + cat /workspace/output/${MODEL}_hyperopt_results.json 2>/dev/null || echo "No results file" + + # ── train-best: full training with best hyperparams ── + - name: train-best + nodeSelector: + k8s.scaleway.com/pool-name: "{{workflow.parameters.gpu-pool}}" + tolerations: + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + - key: node.cilium.io/agent-not-ready + operator: Exists + effect: NoSchedule + container: + image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/ci-builder:latest + imagePullPolicy: IfNotPresent + command: ["/bin/sh", "-c"] + env: + - name: RUST_LOG + value: info + - name: SQLX_OFFLINE + value: "true" + - name: CUBLAS_WORKSPACE_CONFIG + value: ":4096:8" + - name: FOXHUNT_FEATURE_CACHE_DIR + value: /feature-cache + resources: + requests: + nvidia.com/gpu: "1" + cpu: "4" + memory: 32Gi + limits: + nvidia.com/gpu: "1" + cpu: "8" + memory: 64Gi + volumeMounts: + - name: workspace + mountPath: /workspace + - name: training-data + mountPath: /data + readOnly: true + - name: feature-cache + mountPath: /feature-cache + readOnly: true + args: + - | + set -e + SHA="{{tasks.ensure-binary.outputs.parameters.sha}}" + export PATH="/data/bin/$SHA:$PATH" + export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//') + nvidia-smi + + MODEL="{{workflow.parameters.model}}" + case "$MODEL" in + dqn|ppo) BINARY=train_baseline_rl ;; + *) BINARY=train_baseline_supervised ;; + esac + + HYPEROPT_FLAG="" + if [ -f "/workspace/output/${MODEL}_hyperopt_results.json" ]; then + HYPEROPT_FLAG="--hyperopt-params /workspace/output/${MODEL}_hyperopt_results.json" + echo " Using hyperopt results: ${MODEL}_hyperopt_results.json" + else + echo " No hyperopt results — training with default hyperparams" + fi + + echo "=== Training: $MODEL ({{workflow.parameters.train-epochs}} epochs) ===" + + stdbuf -oL ${BINARY} \ + --model "$MODEL" \ + --symbol {{workflow.parameters.symbol}} \ + --tx-cost-bps {{workflow.parameters.tx-cost-bps}} \ + --tick-size {{workflow.parameters.tick-size}} \ + --spread-ticks {{workflow.parameters.spread-ticks}} \ + --initial-capital {{workflow.parameters.initial-capital}} \ + --min-hold-bars {{workflow.parameters.min-hold-bars}} \ + --data-dir /data/futures-baseline \ + --mbp10-data-dir /data/futures-baseline-mbp10 \ + --trades-data-dir /data/futures-baseline-trades \ + --output-dir /workspace/output \ + --epochs {{workflow.parameters.train-epochs}} \ + $HYPEROPT_FLAG + + echo "=== Training complete ===" + + # ── evaluate: run evaluation on trained model ── + - name: evaluate + nodeSelector: + k8s.scaleway.com/pool-name: "{{workflow.parameters.gpu-pool}}" + tolerations: + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + - key: node.cilium.io/agent-not-ready + operator: Exists + effect: NoSchedule + container: + image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/ci-builder:latest + imagePullPolicy: IfNotPresent + command: ["/bin/sh", "-c"] + env: + - name: RUST_LOG + value: info + - name: SQLX_OFFLINE + value: "true" + resources: + requests: + nvidia.com/gpu: "1" + cpu: "2" + memory: 16Gi + limits: + nvidia.com/gpu: "1" + cpu: "4" + memory: 32Gi + volumeMounts: + - name: workspace + mountPath: /workspace + - name: training-data + mountPath: /data + readOnly: true + - name: feature-cache + mountPath: /feature-cache + readOnly: true + args: + - | + set -e + SHA="{{tasks.ensure-binary.outputs.parameters.sha}}" + export PATH="/data/bin/$SHA:$PATH" + export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//') + + MODEL="{{workflow.parameters.model}}" + case "$MODEL" in + dqn|ppo) BINARY=evaluate_baseline ;; + *) BINARY=evaluate_supervised ;; + esac + + echo "=== Evaluating: $MODEL ===" + ${BINARY} \ + --model "$MODEL" \ + --symbol {{workflow.parameters.symbol}} \ + --data-dir /data/futures-baseline \ + --checkpoint-dir /workspace/output \ + --output-dir /workspace/output/eval \ + --initial-capital {{workflow.parameters.initial-capital}} \ + --min-hold-bars {{workflow.parameters.min-hold-bars}} \ + --tx-cost-bps {{workflow.parameters.tx-cost-bps}} \ + --tick-size {{workflow.parameters.tick-size}} \ + --spread-ticks {{workflow.parameters.spread-ticks}} || { + echo "WARN: Evaluation failed, continuing" + } + + echo "=== Evaluation complete ===" + + # ── upload-results: push artifacts to GitLab packages ── + - name: upload-results + nodeSelector: + k8s.scaleway.com/pool-name: platform + container: + image: curlimages/curl:8.12.1 + command: ["/bin/sh", "-c"] + env: + - name: GITLAB_PAT + valueFrom: + secretKeyRef: + name: gitlab-pat + key: token + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 256Mi + volumeMounts: + - name: workspace + mountPath: /workspace + args: + - | + set -e + MODEL="{{workflow.parameters.model}}" + SYMBOL="{{workflow.parameters.symbol}}" + TIMESTAMP=$(date +%Y%m%d-%H%M%S) + GITLAB_API="http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181/api/v4" + PKG_NAME="foxhunt-training-results" + PKG_VERSION="${MODEL}-${SYMBOL}-${TIMESTAMP}" + + echo "Uploading training artifacts: ${PKG_NAME}/${PKG_VERSION}" + + UPLOADED=0 + find /workspace/output -type f | while read -r file; do + REL_PATH="${file#/workspace/output/}" + SAFE_NAME=$(echo "$REL_PATH" | tr '/' '--') + echo " Uploading ${REL_PATH} as ${SAFE_NAME}..." + curl -f --upload-file "$file" \ + -H "PRIVATE-TOKEN: ${GITLAB_PAT}" \ + "${GITLAB_API}/projects/1/packages/generic/${PKG_NAME}/${PKG_VERSION}/${SAFE_NAME}" && \ + UPLOADED=$((UPLOADED + 1)) || \ + echo " WARN: Failed to upload ${REL_PATH}" + done + + echo "=== Upload complete (${UPLOADED} files) ===" + echo "Package: ${PKG_NAME}/${PKG_VERSION}" + + # ── notify-result: post workflow outcome to Mattermost (onExit) ── + - name: notify-result + nodeSelector: + k8s.scaleway.com/pool-name: platform + container: + image: curlimages/curl:8.12.1 + command: ["/bin/sh", "-c"] + env: + - name: WEBHOOK_URL + valueFrom: + secretKeyRef: + name: notification-webhook + key: webhook-url + optional: true + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + args: + - | + STATUS="{{workflow.status}}" + NAME="{{workflow.name}}" + + if [ -z "$WEBHOOK_URL" ] || echo "$WEBHOOK_URL" | grep -q "PLACEHOLDER"; then + echo "No webhook configured, skipping notification" + exit 0 + fi + + if [ "$STATUS" = "Succeeded" ]; then + EMOJI=":white_check_mark:" + else + EMOJI=":x:" + fi + + PAYLOAD="{\"username\":\"Argo CI\",\"text\":\"${EMOJI} **${NAME}** — ${STATUS} ({{workflow.duration}}s)\"}" + + curl -sf -X POST -H 'Content-Type: application/json' \ + -d "$PAYLOAD" "$WEBHOOK_URL" || echo "WARN: webhook post failed"