diff --git a/docs/superpowers/plans/2026-04-10-argo-pipeline-redesign.md b/docs/superpowers/plans/2026-04-10-argo-pipeline-redesign.md new file mode 100644 index 000000000..57e398804 --- /dev/null +++ b/docs/superpowers/plans/2026-04-10-argo-pipeline-redesign.md @@ -0,0 +1,647 @@ +# Argo Training Pipeline Redesign Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Replace 7 overlapping Argo workflow templates with a single unified `train` template that caches binaries by commit SHA on PVC and skips fxcache regeneration when valid. + +**Architecture:** One workflow template (`train-template.yaml`) with a DAG: `ensure-binary` (cache-or-compile) and `gpu-warmup` run in parallel, then `ensure-fxcache` (cache-or-precompute), then `hyperopt` → `train-best` → `evaluate` → `upload-results`. One script (`argo-train.sh`) submits it. Five dead templates deleted. + +**Tech Stack:** Argo Workflows, Kubernetes, bash, PVC storage + +--- + +## File Structure + +| Action | File | Responsibility | +|--------|------|----------------| +| Create | `infra/k8s/argo/train-template.yaml` | Unified training workflow template | +| Rewrite | `scripts/argo-train.sh` | Simplified CLI that submits the unified template | +| Delete | `infra/k8s/argo/compile-and-train-template.yaml` | Replaced by train-template | +| Delete | `infra/k8s/argo/train-dqn-template.yaml` | Unused duplicate | +| Delete | `infra/k8s/argo/train-baseline-rl-template.yaml` | Unused duplicate | +| Delete | `infra/k8s/argo/train-supervised-template.yaml` | Unused duplicate | +| Delete | `infra/k8s/argo/training-workflow-template.yaml` | Unused duplicate | +| Delete | `infra/k8s/argo/precompute-features-template.yaml` | Absorbed into ensure-fxcache step | +| Delete | `scripts/argo-precompute.sh` | Absorbed into ensure-fxcache step | +| Modify | `infra/k8s/argo/kustomization.yaml` | Remove deleted templates, add new one | + +--- + +### Task 1: Create the unified `train-template.yaml` + +**Files:** +- Create: `infra/k8s/argo/train-template.yaml` + +- [ ] **Step 1: Write the template header, parameters, volumes, and DAG** + +```yaml +# Unified training pipeline: ensure-binary → ensure-fxcache → train +# +# Smart caching: +# - Binaries cached on PVC at /data/bin/{sha}/ — skips compile if present +# - fxcache cached on feature-cache PVC — skips precompute if present +# - GPU warmup runs in parallel with ensure-binary +# +# Usage: +# ./scripts/argo-train.sh dqn +# ./scripts/argo-train.sh dqn --sha abc1234 --epochs 100 +--- +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: train + namespace: foxhunt + labels: + app.kubernetes.io/name: train + app.kubernetes.io/part-of: foxhunt +spec: + entrypoint: pipeline + onExit: notify-result + serviceAccountName: argo-workflow + podMetadata: + labels: + app.kubernetes.io/part-of: foxhunt + app.kubernetes.io/component: train + securityContext: + fsGroup: 0 + ttlStrategy: + secondsAfterCompletion: 3600 + activeDeadlineSeconds: 28800 + + arguments: + parameters: + - name: commit-sha + value: HEAD + - name: git-branch + value: main + - name: cuda-compute-cap + value: "90" + - name: model + value: dqn + - name: gpu-pool + value: ci-training-h100 + - name: hyperopt-trials + value: "20" + - name: hyperopt-epochs + value: "8" + - name: train-epochs + value: "50" + - name: symbol + value: ES.FUT + - name: initial-capital + value: "35000" + - name: tx-cost-bps + value: "0.1" + - name: tick-size + value: "0.25" + - name: spread-ticks + value: "1.0" + - name: min-hold-bars + value: "5" + + volumes: + - name: git-ssh-key + secret: + secretName: argo-git-ssh-key + defaultMode: 256 + - name: training-data + persistentVolumeClaim: + claimName: training-data-pvc + - name: cargo-target-cuda + persistentVolumeClaim: + claimName: cargo-target-cuda + - name: feature-cache + persistentVolumeClaim: + claimName: feature-cache-pvc + + volumeClaimTemplates: + - metadata: + name: workspace + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: scw-bssd + resources: + requests: + storage: 5Gi + + templates: + - name: pipeline + dag: + tasks: + - name: ensure-binary + template: ensure-binary + - name: gpu-warmup + template: gpu-warmup + - name: ensure-fxcache + template: ensure-fxcache + dependencies: [ensure-binary] + - name: hyperopt + template: hyperopt + dependencies: [ensure-binary, ensure-fxcache, gpu-warmup] + when: "{{workflow.parameters.hyperopt-trials}} != 0" + - name: train-best + template: train-best + dependencies: [ensure-binary, ensure-fxcache, gpu-warmup, hyperopt] + - name: evaluate + template: evaluate + dependencies: [train-best] + - name: upload-results + template: upload-results + dependencies: [evaluate] +``` + +- [ ] **Step 2: Write the `ensure-binary` template** + +This is the key innovation — checks PVC cache first, compiles only on miss. Append to the same file: + +```yaml + # ── ensure-binary: cache-or-compile training binaries ── + # Checks /data/bin/{sha}/ on PVC. If binaries exist, exits immediately. + # If not, clones repo, builds, copies to /data/bin/{sha}/, cleans old SHAs. + - name: ensure-binary + nodeSelector: + k8s.scaleway.com/pool-name: ci-compile-cpu + tolerations: + - key: node.cilium.io/agent-not-ready + operator: Exists + effect: NoSchedule + container: + image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/ci-builder:latest + imagePullPolicy: Always + command: ["/bin/bash", "-c"] + env: + - name: SQLX_OFFLINE + value: "true" + - name: CARGO_TERM_COLOR + value: always + - name: CARGO_TARGET_DIR + value: /cargo-target + - name: CARGO_HOME + value: /cargo-target/cargo-home + - name: GITLAB_PAT + valueFrom: + secretKeyRef: + name: gitlab-pat + key: token + resources: + requests: + cpu: "14" + memory: 32Gi + limits: + cpu: "30" + memory: 64Gi + volumeMounts: + - name: git-ssh-key + mountPath: /etc/git-ssh + readOnly: true + - name: cargo-target-cuda + mountPath: /cargo-target + - name: training-data + mountPath: /data + args: + - | + set -e + BRANCH="{{workflow.parameters.git-branch}}" + REPO="ssh://git@gitlab-gitlab-shell.foxhunt.svc.cluster.local:2222/root/foxhunt.git" + BUILD="/cargo-target/src" + + # Resolve commit SHA + mkdir -p ~/.ssh + cp /etc/git-ssh/ssh-privatekey ~/.ssh/id_ed25519 + chmod 600 ~/.ssh/id_ed25519 + printf 'Host *\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n' > ~/.ssh/config + chmod 600 ~/.ssh/config + + git config --global --add safe.directory "$BUILD" + + SHA="{{workflow.parameters.commit-sha}}" + if [ "$SHA" = "HEAD" ]; then + # Need to resolve HEAD to actual SHA + if [ -d "$BUILD/.git" ]; then + cd "$BUILD" && git fetch origin + SHA=$(git rev-parse "origin/$BRANCH") + cd / + else + git clone --filter=blob:none --depth=1 "$REPO" /tmp/sha-resolve + SHA=$(git -C /tmp/sha-resolve rev-parse HEAD) + rm -rf /tmp/sha-resolve + fi + fi + SHORT_SHA=$(echo "$SHA" | cut -c1-9) + echo "Resolved SHA: $SHORT_SHA" + + # ── Cache check ── + BIN_DIR="/data/bin/$SHORT_SHA" + MODEL="{{workflow.parameters.model}}" + case "$MODEL" in + dqn|ppo) BINARIES="hyperopt_baseline_rl train_baseline_rl evaluate_baseline precompute_features" ;; + *) BINARIES="hyperopt_baseline_supervised train_baseline_supervised evaluate_supervised precompute_features" ;; + esac + + ALL_CACHED=true + for bin in $BINARIES; do + if [ ! -x "$BIN_DIR/$bin" ]; then + ALL_CACHED=false + break + fi + done + + if $ALL_CACHED; then + echo "=== Binary cache HIT for $SHORT_SHA ===" + ls -lh "$BIN_DIR/" + echo "$SHORT_SHA" > /tmp/sha + exit 0 + fi + + echo "=== Binary cache MISS for $SHORT_SHA — compiling ===" + + # Checkout + if [ -d "$BUILD/.git" ]; then + cd "$BUILD" + git fetch origin + CURRENT=$(git rev-parse HEAD 2>/dev/null || echo "none") + if [ "$CURRENT" != "$SHA" ]; then + git checkout --force "$SHA" + git clean -fd + fi + else + git clone --filter=blob:none "$REPO" "$BUILD" + cd "$BUILD" + git checkout "$SHA" + fi + + export PATH="${CARGO_HOME}/bin:${PATH}" + export CUDA_COMPUTE_CAP={{workflow.parameters.cuda-compute-cap}} + + # Prune build artifacts if PVC exceeds 25GB + TARGET_SIZE_MB=$(du -sm "$CARGO_TARGET_DIR" 2>/dev/null | cut -f1 || echo 0) + if [ "$TARGET_SIZE_MB" -gt 25000 ]; then + echo "PVC exceeds 25GB, pruning..." + rm -rf "$CARGO_TARGET_DIR/release" "$CARGO_TARGET_DIR/debug" + fi + + ML_EXAMPLE_ARGS="" + for ex in $BINARIES; do + ML_EXAMPLE_ARGS="$ML_EXAMPLE_ARGS --example $ex" + done + + echo "=== Building: $BINARIES ===" + cargo build --release -p ml --features ml/cuda $ML_EXAMPLE_ARGS + + mkdir -p "$BIN_DIR" + for bin in $BINARIES; do + cp "$CARGO_TARGET_DIR/release/examples/$bin" "$BIN_DIR/" + done + strip "$BIN_DIR/"* + + echo "=== Binaries cached at $BIN_DIR ===" + ls -lh "$BIN_DIR/" + + # Cleanup: keep only last 5 SHA dirs + ls -dt /data/bin/*/ 2>/dev/null | tail -n +6 | xargs rm -rf 2>/dev/null || true + + echo "$SHORT_SHA" > /tmp/sha + outputs: + parameters: + - name: sha + valueFrom: + path: /tmp/sha +``` + +- [ ] **Step 3: Write the `ensure-fxcache` template** + +```yaml + # ── ensure-fxcache: run precompute if cache miss ── + # The precompute binary computes its own cache key and skips if output exists. + - name: ensure-fxcache + nodeSelector: + k8s.scaleway.com/pool-name: ci-compile-cpu + tolerations: + - key: node.cilium.io/agent-not-ready + operator: Exists + effect: NoSchedule + container: + image: ubuntu:24.04 + command: ["/bin/bash", "-c"] + env: + - name: RUST_LOG + value: info + resources: + requests: + cpu: "4" + memory: 16Gi + limits: + cpu: "28" + memory: 56Gi + volumeMounts: + - name: training-data + mountPath: /data + readOnly: true + - name: feature-cache + mountPath: /feature-cache + args: + - | + set -e + SHA="{{tasks.ensure-binary.outputs.parameters.sha}}" + BINARY="/data/bin/$SHA/precompute_features" + + if [ ! -x "$BINARY" ]; then + echo "ERROR: $BINARY not found" + exit 1 + fi + + echo "=== Feature precompute (skip-if-exists) ===" + $BINARY \ + --data-dir /data/futures-baseline \ + --mbp10-data-dir /data/futures-baseline-mbp10 \ + --trades-data-dir /data/futures-baseline-trades \ + --output-dir /feature-cache \ + --symbol {{workflow.parameters.symbol}} \ + --yes +``` + +- [ ] **Step 4: Write the `gpu-warmup` template** + +```yaml + # ── gpu-warmup: trigger GPU node autoscale in parallel ── + - name: gpu-warmup + nodeSelector: + k8s.scaleway.com/pool-name: "{{workflow.parameters.gpu-pool}}" + tolerations: + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + - key: node.cilium.io/agent-not-ready + operator: Exists + effect: NoSchedule + container: + image: busybox:1.37 + command: ["/bin/sh", "-c"] + args: + - | + echo "GPU warmup: node scheduled, autoscale triggered" + resources: + requests: + nvidia.com/gpu: "1" + cpu: 100m + memory: 64Mi + limits: + nvidia.com/gpu: "1" + cpu: 200m + memory: 128Mi +``` + +- [ ] **Step 5: Write `hyperopt`, `train-best`, `evaluate`, `upload-results`, `notify-result` templates** + +These are copied from `compile-and-train-template.yaml` with one change: binaries are read from `/data/bin/{sha}/` instead of `/workspace/bin/`. Replace all `export PATH="/workspace/bin:$PATH"` with: + +```bash +SHA="{{tasks.ensure-binary.outputs.parameters.sha}}" +export PATH="/data/bin/$SHA:$PATH" +``` + +And replace `feature-cache-dir` parameter references with the hardcoded `/feature-cache` path. + +The `hyperopt` template (lines 411-503 of old file), `train-best` (505-587), `evaluate` (589-654), `upload-results` (656-704), and `notify-result` (706-747) are otherwise identical — copy them with the PATH fix above. + +- [ ] **Step 6: Verify the template is valid YAML** + +Run: `python3 -c "import yaml; yaml.safe_load(open('infra/k8s/argo/train-template.yaml'))" && echo "Valid YAML"` +Expected: `Valid YAML` + +- [ ] **Step 7: Commit** + +```bash +git add infra/k8s/argo/train-template.yaml +git commit -m "feat: unified train workflow template — cache-or-compile + cache-or-precompute" +``` + +--- + +### Task 2: Rewrite `scripts/argo-train.sh` + +**Files:** +- Modify: `scripts/argo-train.sh` + +- [ ] **Step 1: Replace the entire script** + +```bash +#!/usr/bin/env bash +# Train a model via Argo Workflows. +# +# Usage: +# ./scripts/argo-train.sh dqn # defaults: HEAD, H100, 50 epochs +# ./scripts/argo-train.sh dqn --sha abc1234 # specific commit +# ./scripts/argo-train.sh dqn --epochs 100 --trials 40 # override training params +# ./scripts/argo-train.sh ppo --gpu-pool ci-training # L40S instead of H100 +# ./scripts/argo-train.sh dqn --baseline # skip hyperopt +# ./scripts/argo-train.sh dqn --watch # follow logs +# +# Supported models: +# RL: dqn, ppo +# Supervised: tft, mamba2, tggn, tlob, liquid, kan, xlstm, diffusion +# +# Requires: argo CLI +set -euo pipefail + +SHA="HEAD" +BRANCH="main" +TRIALS="" +EPOCHS="" +GPU_POOL="" +SYMBOL="" +WATCH=false +BASELINE=false +CAPITAL="" + +usage() { + cat < [OPTIONS] + +Models: + dqn, ppo (RL) + tft, mamba2, tggn, tlob, liquid, kan, xlstm (supervised) + +Options: + --sha Git commit SHA (default: HEAD) + --branch Git branch (default: main) + --trials Hyperopt trials (default: 20) + --epochs Training epochs (default: 50) + --gpu-pool GPU node pool (default: ci-training-h100) + --symbol Trading symbol (default: ES.FUT) + --capital Initial capital (default: 35000) + --baseline Skip hyperopt (trials=0) + --watch Follow workflow logs + -h, --help Show this help +EOF + exit 0 +} + +[[ $# -eq 0 ]] && { echo "Error: model argument required"; usage; } + +MODEL="$1"; shift + +case "$MODEL" in + dqn|ppo|tft|mamba2|tggn|tlob|liquid|kan|xlstm|diffusion) ;; + *) echo "Error: unknown model '$MODEL'"; usage ;; +esac + +while [[ $# -gt 0 ]]; do + case $1 in + --sha) SHA="$2"; shift 2 ;; + --branch) BRANCH="$2"; shift 2 ;; + --trials) TRIALS="$2"; shift 2 ;; + --epochs) EPOCHS="$2"; shift 2 ;; + --gpu-pool) GPU_POOL="$2"; shift 2 ;; + --symbol) SYMBOL="$2"; shift 2 ;; + --capital) CAPITAL="$2"; shift 2 ;; + --baseline) BASELINE=true; shift ;; + --watch) WATCH=true; shift ;; + -h|--help) usage ;; + *) echo "Unknown option: $1"; usage ;; + esac +done + +CMD="argo submit -n foxhunt --from=wftmpl/train" +CMD="$CMD -p commit-sha=$SHA" +CMD="$CMD -p git-branch=$BRANCH" +CMD="$CMD -p model=$MODEL" + +[[ -n "$TRIALS" ]] && CMD="$CMD -p hyperopt-trials=$TRIALS" +[[ -n "$EPOCHS" ]] && CMD="$CMD -p train-epochs=$EPOCHS" +[[ -n "$GPU_POOL" ]] && CMD="$CMD -p gpu-pool=$GPU_POOL" +[[ -n "$SYMBOL" ]] && CMD="$CMD -p symbol=$SYMBOL" +[[ -n "$CAPITAL" ]] && CMD="$CMD -p initial-capital=$CAPITAL" + +$BASELINE && CMD="$CMD -p hyperopt-trials=0" +$WATCH && CMD="$CMD --watch" + +echo "Submitting $MODEL training workflow..." +echo " sha: $SHA" +echo " branch: $BRANCH" +echo " model: $MODEL" +$BASELINE && echo " mode: baseline (no hyperopt)" +[[ -n "$TRIALS" ]] && echo " trials: $TRIALS" +[[ -n "$EPOCHS" ]] && echo " epochs: $EPOCHS" +[[ -n "$GPU_POOL" ]] && echo " gpu: $GPU_POOL" +echo "" +eval "$CMD" +``` + +- [ ] **Step 2: Verify script is executable** + +Run: `bash -n scripts/argo-train.sh && echo "Syntax OK"` +Expected: `Syntax OK` + +- [ ] **Step 3: Commit** + +```bash +git add scripts/argo-train.sh +git commit -m "feat: rewrite argo-train.sh — single template, --sha for commit pinning" +``` + +--- + +### Task 3: Delete dead templates and scripts + +**Files:** +- Delete: `infra/k8s/argo/compile-and-train-template.yaml` +- Delete: `infra/k8s/argo/train-dqn-template.yaml` +- Delete: `infra/k8s/argo/train-baseline-rl-template.yaml` +- Delete: `infra/k8s/argo/train-supervised-template.yaml` +- Delete: `infra/k8s/argo/training-workflow-template.yaml` +- Delete: `infra/k8s/argo/precompute-features-template.yaml` +- Delete: `infra/k8s/argo/train-ppo-template.yaml` +- Delete: `scripts/argo-precompute.sh` +- Modify: `infra/k8s/argo/kustomization.yaml` + +- [ ] **Step 1: Delete the template files** + +```bash +rm infra/k8s/argo/compile-and-train-template.yaml +rm infra/k8s/argo/train-dqn-template.yaml +rm infra/k8s/argo/train-baseline-rl-template.yaml +rm infra/k8s/argo/train-supervised-template.yaml +rm infra/k8s/argo/training-workflow-template.yaml +rm infra/k8s/argo/precompute-features-template.yaml +rm infra/k8s/argo/train-ppo-template.yaml +rm scripts/argo-precompute.sh +``` + +- [ ] **Step 2: Update kustomization.yaml** + +Read `infra/k8s/argo/kustomization.yaml`, remove references to deleted templates, add `train-template.yaml`. + +- [ ] **Step 3: Commit** + +```bash +git add -A infra/k8s/argo/ scripts/ +git commit -m "cleanup: delete 7 dead training templates + precompute script, update kustomization" +``` + +--- + +### Task 4: Deploy to cluster and delete old templates + +**Files:** +- No file changes — cluster operations only + +- [ ] **Step 1: Apply the new template to the cluster** + +```bash +kubectl apply -f infra/k8s/argo/train-template.yaml +``` + +Expected: `workflowtemplate.argoproj.io/train created` + +- [ ] **Step 2: Delete old templates from the cluster** + +```bash +kubectl delete wftmpl -n foxhunt compile-and-train train-dqn train-baseline-rl train-supervised training-pipeline precompute-features train-ppo 2>/dev/null || true +``` + +- [ ] **Step 3: Verify only the new template exists (plus the non-training ones)** + +```bash +kubectl get wftmpl -n foxhunt +``` + +Expected: `train` plus `compile-and-deploy`, `databento-download`, `gpu-test-pipeline`, `build-ci-image`, `ci-pipeline` + +- [ ] **Step 4: Commit (no file changes — just verify)** + +No commit needed. Cluster state is updated. + +--- + +### Task 5: Test the new workflow — baseline DQN on H100 + +**Files:** +- No file changes — integration test + +- [ ] **Step 1: Run a baseline DQN training (no hyperopt, 5 epochs)** + +```bash +./scripts/argo-train.sh dqn --baseline --epochs 5 --watch +``` + +Expected: +- `ensure-binary`: cache HIT (binary was compiled by the terminated workflow earlier) OR cache MISS → compile ~10 min +- `ensure-fxcache`: cache HIT (fxcache was generated earlier) OR run precompute ~4 min +- `gpu-warmup`: completes in ~3 min (node autoscale) +- `train-best`: runs 5 epochs on H100 with TF32 tensor cores +- Watch for epoch timing — should be ~10s/epoch on H100 (was 77s with old CUDA core path) + +- [ ] **Step 2: Verify epoch timing in logs** + +```bash +argo logs -n foxhunt | grep 'duration_secs\|duration=' +``` + +Expected: epoch times significantly faster than 77s (the old bf16/CUDA core baseline). + +- [ ] **Step 3: Push all changes** + +```bash +git push origin main +``` diff --git a/infra/k8s/argo/compile-and-train-template.yaml b/infra/k8s/argo/compile-and-train-template.yaml deleted file mode 100644 index ef4f17923..000000000 --- a/infra/k8s/argo/compile-and-train-template.yaml +++ /dev/null @@ -1,746 +0,0 @@ -# Compile-and-Train: unified workflow for code change → training run. -# -# DAG: -# compile-training ──┐ -# gpu-warmup ────────┼──> fetch-binary ──> hyperopt ──> train-best ──> evaluate ──> upload-results -# -# Image builds handled by ci-pipeline, not here. -# Compile and GPU warmup run in parallel (~5.5 min compile, ~3-5 min autoscale). -# When both finish, fetch-binary downloads the freshly compiled binaries and -# training starts immediately — no manual CI wait or package seeding. ---- -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate -metadata: - name: compile-and-train - namespace: foxhunt - labels: - app.kubernetes.io/name: compile-and-train - app.kubernetes.io/part-of: foxhunt - app.kubernetes.io/component: gpu-test -spec: - entrypoint: pipeline - onExit: notify-result - serviceAccountName: argo-workflow - podMetadata: - labels: - app.kubernetes.io/part-of: foxhunt - app.kubernetes.io/component: compile-and-train - # No runAsUser — compile (ci-builder) needs root, training runtime defaults - # to its image user. fsGroup ensures workspace PVC is writable by all pods. - securityContext: - fsGroup: 0 - ttlStrategy: - secondsAfterCompletion: 3600 - activeDeadlineSeconds: 28800 # 8 hours (compile + full training) - - arguments: - parameters: - # ── Compile ── - - name: commit-sha - value: HEAD - - name: git-branch - value: main - - name: cuda-compute-cap - value: "90" - # ── Training ── - - name: model - value: dqn - - name: gpu-pool - value: ci-training-h100 - - name: hyperopt-trials - value: "20" - - name: hyperopt-epochs - value: "8" - - name: train-epochs - value: "50" - - name: symbol - value: ES.FUT - - name: data-dir - value: /data/futures-baseline - - name: mbp10-data-dir - value: /data/futures-baseline-mbp10 - - name: trades-data-dir - value: /data/futures-baseline-trades - - name: feature-cache-dir - value: /feature-cache - - name: tx-cost-bps - value: "0.1" - - name: tick-size - value: "0.25" - - name: spread-ticks - value: "1.0" - - name: min-hold-bars - value: "5" - - name: initial-capital - value: "35000" - - name: ensemble-top-k - value: "5" - - volumes: - - name: git-ssh-key - secret: - secretName: argo-git-ssh-key - defaultMode: 256 - - name: training-data - persistentVolumeClaim: - claimName: training-data-pvc - - name: cargo-target-cuda - persistentVolumeClaim: - claimName: cargo-target-cuda - - name: feature-cache - persistentVolumeClaim: - claimName: feature-cache-pvc - - volumeClaimTemplates: - - metadata: - name: workspace - spec: - accessModes: ["ReadWriteOnce"] - storageClassName: scw-bssd - resources: - requests: - storage: 5Gi - - templates: - - name: pipeline - dag: - tasks: - - name: compile-training - template: compile-training - - name: gpu-warmup - template: gpu-warmup - - name: fetch-binary - template: fetch-binary - dependencies: [compile-training] - arguments: - parameters: - - name: tag - value: "{{tasks.compile-training.outputs.parameters.tag}}" - - name: precompute - template: precompute - dependencies: [fetch-binary] - - name: hyperopt - template: hyperopt - dependencies: [fetch-binary, gpu-warmup, precompute] - when: "{{workflow.parameters.hyperopt-trials}} != 0" - - name: train-best - template: train-best - dependencies: [fetch-binary, gpu-warmup, hyperopt] - - name: evaluate - template: evaluate - dependencies: [train-best] - - name: upload-results - template: upload-results - dependencies: [evaluate] - - # ── compile-training: build from source on ci-compile-cpu ── - - name: compile-training - outputs: - parameters: - - name: tag - valueFrom: - path: /tmp/tag - nodeSelector: - k8s.scaleway.com/pool-name: ci-compile-cpu - tolerations: - - key: node.cilium.io/agent-not-ready - operator: Exists - effect: NoSchedule - container: - image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/ci-builder:latest - imagePullPolicy: Always - command: ["/bin/sh", "-c"] - env: - - name: SQLX_OFFLINE - value: "true" - - name: CARGO_TERM_COLOR - value: always - - name: CARGO_TARGET_DIR - value: /cargo-target - - name: CARGO_HOME - value: /cargo-target/cargo-home - - name: GITLAB_PAT - valueFrom: - secretKeyRef: - name: gitlab-pat - key: token - resources: - requests: - cpu: "14" - memory: 32Gi - limits: - cpu: "30" - memory: 64Gi - volumeMounts: - - name: git-ssh-key - mountPath: /etc/git-ssh - readOnly: true - - name: cargo-target-cuda - mountPath: /cargo-target - args: - - | - set -e - SHA="{{workflow.parameters.commit-sha}}" - BRANCH="{{workflow.parameters.git-branch}}" - - # SSH setup - mkdir -p ~/.ssh - cp /etc/git-ssh/ssh-privatekey ~/.ssh/id_ed25519 - chmod 600 ~/.ssh/id_ed25519 - printf 'Host *\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n' > ~/.ssh/config - chmod 600 ~/.ssh/config - - REPO="ssh://git@gitlab-gitlab-shell.foxhunt.svc.cluster.local:2222/root/foxhunt.git" - BUILD="/cargo-target/src" - - # PVC may be owned by a different UID from a previous run - git config --global --add safe.directory "$BUILD" - - # Persistent checkout on PVC — only changed files get new mtimes, - # so cargo skips recompiling unchanged workspace crates. - if [ -d "$BUILD/.git" ]; then - cd "$BUILD" - git fetch origin - # Resolve SHA: "HEAD" means remote branch tip, not local HEAD - if [ "$SHA" = "HEAD" ]; then - SHA=$(git rev-parse "origin/$BRANCH") - fi - CURRENT=$(git rev-parse HEAD 2>/dev/null || echo "none") - if [ "$CURRENT" = "$SHA" ]; then - echo "=== Already at $SHA, skipping checkout ===" - else - echo "=== Updating checkout: $(echo $CURRENT | cut -c1-8) -> $(echo $SHA | cut -c1-8) ===" - git checkout --force "$SHA" - git clean -fd - fi - else - echo "=== Initial clone (first run) ===" - git clone --filter=blob:none "$REPO" "$BUILD" - cd "$BUILD" - if [ "$SHA" = "HEAD" ]; then - git checkout "origin/$BRANCH" - else - git checkout "$SHA" - fi - fi - SHORT_SHA=$(git rev-parse --short HEAD) - echo "Checked out ${SHORT_SHA}" - - # Ensure cargo home registry is on PVC - export PATH="${CARGO_HOME}/bin:${PATH}" - - export CUDA_COMPUTE_CAP={{workflow.parameters.cuda-compute-cap}} - - # Prune build artifacts if PVC exceeds 25GB (prevents unbounded growth) - TARGET_SIZE_MB=$(du -sm "$CARGO_TARGET_DIR" 2>/dev/null | cut -f1 || echo 0) - echo "PVC usage: ${TARGET_SIZE_MB}MB" - if [ "$TARGET_SIZE_MB" -gt 25000 ]; then - echo "PVC exceeds 25GB limit, pruning build artifacts..." - rm -rf "$CARGO_TARGET_DIR/release" "$CARGO_TARGET_DIR/debug" - fi - - # Derive needed binaries from model parameter (only build what's used) - MODEL="{{workflow.parameters.model}}" - case "$MODEL" in - dqn|ppo) - EXAMPLES="hyperopt_baseline_rl train_baseline_rl evaluate_baseline precompute_features" - ;; - *) - EXAMPLES="hyperopt_baseline_supervised train_baseline_supervised evaluate_supervised" - ;; - esac - - ML_EXAMPLE_ARGS="" - for ex in $EXAMPLES; do - ML_EXAMPLE_ARGS="$ML_EXAMPLE_ARGS --example $ex" - done - - echo "=== Building training binaries for $MODEL: $EXAMPLES (incremental) ===" - echo " CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP, target dir on PVC /cargo-target" - cargo build --release -p ml --features ml/cuda $ML_EXAMPLE_ARGS - - mkdir -p "$BUILD/bin/training" - for bin in $EXAMPLES; do - cp "$CARGO_TARGET_DIR/release/examples/$bin" "$BUILD/bin/training/" - done - strip "$BUILD/bin/training/"* - - echo "=== Training binaries ===" - ls -lh "$BUILD/bin/training/" - - # Upload to GitLab packages under commit SHA - GITLAB="http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181" - TAG="dev-${SHORT_SHA}" - echo "=== Uploading as ${TAG} ===" - for bin in "$BUILD/bin/training/"*; do - BIN_NAME=$(basename "$bin") - curl -f --upload-file "$bin" \ - -H "PRIVATE-TOKEN: ${GITLAB_PAT}" \ - "${GITLAB}/api/v4/projects/1/packages/generic/foxhunt-training/${TAG}/${BIN_NAME}" - done - - echo "=== Compile + upload done (${TAG}) ===" - echo "$TAG" > /tmp/tag - - # ── precompute: generate fxcache if data changed (skips if cache exists) ── - # Pure CPU work — no GPU needed. Runs on compile node (same as compile-training) - # to reuse workspace PVC attachment. - - name: precompute - nodeSelector: - k8s.scaleway.com/pool-name: ci-compile-cpu - tolerations: - - key: node.cilium.io/agent-not-ready - operator: Exists - effect: NoSchedule - container: - image: ubuntu:24.04 - command: ["/bin/bash", "-c"] - env: - - name: RUST_LOG - value: info - args: - - | - set -e - BINARY=/workspace/bin/precompute_features - if [ ! -f "$BINARY" ]; then - echo "ERROR: precompute_features not found in /workspace/bin/" - exit 1 - fi - $BINARY \ - --data-dir /data/futures-baseline \ - --mbp10-data-dir /data/futures-baseline-mbp10 \ - --trades-data-dir /data/futures-baseline-trades \ - --output-dir /feature-cache \ - --symbol ES.FUT \ - --yes - volumeMounts: - - name: training-data - mountPath: /data - readOnly: true - - name: feature-cache - mountPath: /feature-cache - - name: workspace - mountPath: /workspace - resources: - requests: - cpu: "4" - memory: 16Gi - limits: - cpu: "28" - memory: 56Gi - - # ── gpu-warmup: trigger GPU autoscale during compile ── - - name: gpu-warmup - nodeSelector: - k8s.scaleway.com/pool-name: "{{workflow.parameters.gpu-pool}}" - tolerations: - - key: nvidia.com/gpu - operator: Exists - effect: NoSchedule - - key: node.cilium.io/agent-not-ready - operator: Exists - effect: NoSchedule - container: - image: busybox:1.37 - command: ["/bin/sh", "-c"] - args: - - | - echo "GPU warmup: triggering node autoscale..." - echo "GPU node scheduled, exiting to free resources" - resources: - requests: - nvidia.com/gpu: "1" - cpu: 100m - memory: 64Mi - limits: - nvidia.com/gpu: "1" - cpu: 200m - memory: 128Mi - - # ── fetch-binary: download freshly compiled binaries to workspace ── - - name: fetch-binary - inputs: - parameters: - - name: tag - nodeSelector: - k8s.scaleway.com/pool-name: platform - container: - image: curlimages/curl:8.12.1 - command: ["/bin/sh", "-c"] - args: - - | - set -e - mkdir -p /workspace/bin - GITLAB_API="http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181/api/v4" - - TAG="{{inputs.parameters.tag}}" - echo "Fetching training binaries from GitLab package: foxhunt-training/${TAG}" - - BINARIES="hyperopt_baseline_rl hyperopt_baseline_supervised train_baseline_rl train_baseline_supervised evaluate_baseline evaluate_supervised precompute_features" - for bin in $BINARIES; do - echo " Downloading ${bin}..." - curl -fSL -o "/workspace/bin/${bin}" \ - -H "PRIVATE-TOKEN: ${GITLAB_PAT}" \ - "${GITLAB_API}/projects/1/packages/generic/foxhunt-training/${TAG}/${bin}" || { - echo "WARN: ${bin} not found, skipping" - continue - } - done - - chmod +x /workspace/bin/* - echo "Fetched binaries:" - ls -lh /workspace/bin/ - env: - - name: GITLAB_PAT - valueFrom: - secretKeyRef: - name: gitlab-pat - key: token - volumeMounts: - - name: workspace - mountPath: /workspace - resources: - requests: - cpu: 100m - memory: 128Mi - limits: - cpu: 500m - memory: 256Mi - - # ── hyperopt: PSO/TPE optimization on GPU ── - - name: hyperopt - nodeSelector: - k8s.scaleway.com/pool-name: "{{workflow.parameters.gpu-pool}}" - tolerations: - - key: nvidia.com/gpu - operator: Exists - effect: NoSchedule - - key: node.cilium.io/agent-not-ready - operator: Exists - effect: NoSchedule - container: - image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/ci-builder:latest - imagePullPolicy: IfNotPresent - command: ["/bin/sh", "-c"] - env: - - name: RUST_LOG - value: info - - name: SQLX_OFFLINE - value: "true" - - name: CUBLAS_WORKSPACE_CONFIG - value: ":4096:8" - - name: CARGO_TARGET_DIR - value: /workspace - - name: FOXHUNT_FEATURE_CACHE_DIR - value: "{{workflow.parameters.feature-cache-dir}}" - args: - - | - set -e - export PATH="/workspace/bin:$PATH" - # Remove CUDA stubs from LD_LIBRARY_PATH — stubs shadow real NVIDIA drivers - export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//') - nvidia-smi - - # Training data available via training-data-pvc mounted at /data - echo "=== Training data (PVC) ===" - echo "OHLCV: $(ls {{workflow.parameters.data-dir}}/ES.FUT/*.dbn.zst 2>/dev/null | wc -l) files" - echo "MBP-10: $(ls {{workflow.parameters.mbp10-data-dir}}/ES.FUT/*.dbn.zst 2>/dev/null | wc -l) files" - echo "Trades: $(ls {{workflow.parameters.trades-data-dir}}/ES.FUT/*.dbn.zst 2>/dev/null | wc -l) files" - - mkdir -p /workspace/output/hyperopt - - echo "=== Feature cache: $(ls ${FOXHUNT_FEATURE_CACHE_DIR}/*.fxcache 2>/dev/null | wc -l) fxcache files ===" - - MODEL="{{workflow.parameters.model}}" - case "$MODEL" in - dqn|ppo) BINARY=hyperopt_baseline_rl ;; - *) BINARY=hyperopt_baseline_supervised ;; - esac - - echo "=== Running hyperopt: $MODEL (${BINARY}) ===" - echo " Trials: {{workflow.parameters.hyperopt-trials}}, Epochs: {{workflow.parameters.hyperopt-epochs}}" - - # Single-pass 14D hyperopt: family-based search covers all dims at once. - # Fast phase searches all 14 dimensions (3 breakouts + 11 family intensities). - # Phased search was needed for the old 30D space; 14D is within PSO's efficient range. - ${BINARY} \ - --model "$MODEL" \ - --phase fast \ - --trials {{workflow.parameters.hyperopt-trials}} \ - --epochs {{workflow.parameters.hyperopt-epochs}} \ - --symbol {{workflow.parameters.symbol}} \ - --tx-cost-bps {{workflow.parameters.tx-cost-bps}} \ - --tick-size {{workflow.parameters.tick-size}} \ - --spread-ticks {{workflow.parameters.spread-ticks}} \ - --initial-capital {{workflow.parameters.initial-capital}} \ - --min-hold-bars {{workflow.parameters.min-hold-bars}} \ - --data-dir {{workflow.parameters.data-dir}} \ - --mbp10-data-dir {{workflow.parameters.mbp10-data-dir}} \ - --trades-data-dir {{workflow.parameters.trades-data-dir}} \ - --base-dir /workspace/output/hyperopt \ - --output /workspace/output/${MODEL}_hyperopt_results.json - - echo "=== Hyperopt complete (14D single-pass) ===" - cat /workspace/output/${MODEL}_hyperopt_results.json 2>/dev/null || echo "No results file" - resources: - requests: - nvidia.com/gpu: "1" - cpu: "4" - memory: 32Gi - limits: - nvidia.com/gpu: "1" - cpu: "8" - memory: 64Gi - volumeMounts: - - name: workspace - mountPath: /workspace - - name: training-data - mountPath: /data - readOnly: true - - name: feature-cache - mountPath: /feature-cache - readOnly: true - - # ── train-best: full training with best hyperparams ── - - name: train-best - nodeSelector: - k8s.scaleway.com/pool-name: "{{workflow.parameters.gpu-pool}}" - tolerations: - - key: nvidia.com/gpu - operator: Exists - effect: NoSchedule - - key: node.cilium.io/agent-not-ready - operator: Exists - effect: NoSchedule - container: - image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/ci-builder:latest - imagePullPolicy: IfNotPresent - command: ["/bin/sh", "-c"] - env: - - name: RUST_LOG - value: info - - name: SQLX_OFFLINE - value: "true" - - name: CUBLAS_WORKSPACE_CONFIG - value: ":4096:8" - - name: FOXHUNT_FEATURE_CACHE_DIR - value: "{{workflow.parameters.feature-cache-dir}}" - args: - - | - set -e - export PATH="/workspace/bin:$PATH" - export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//') - nvidia-smi - - MODEL="{{workflow.parameters.model}}" - - # Determine binary and hyperopt flag - case "$MODEL" in - dqn|ppo) BINARY=train_baseline_rl ;; - *) BINARY=train_baseline_supervised ;; - esac - - HYPEROPT_FLAG="" - if [ -f "/workspace/output/${MODEL}_hyperopt_results.json" ]; then - HYPEROPT_FLAG="--hyperopt-params /workspace/output/${MODEL}_hyperopt_results.json" - echo " Using hyperopt results: ${MODEL}_hyperopt_results.json" - else - echo " No hyperopt results — training with default hyperparams" - fi - - echo "=== Training best: $MODEL ({{workflow.parameters.train-epochs}} epochs) ===" - - stdbuf -oL ${BINARY} \ - --model "$MODEL" \ - --symbol {{workflow.parameters.symbol}} \ - --tx-cost-bps {{workflow.parameters.tx-cost-bps}} \ - --tick-size {{workflow.parameters.tick-size}} \ - --spread-ticks {{workflow.parameters.spread-ticks}} \ - --initial-capital {{workflow.parameters.initial-capital}} \ - --min-hold-bars {{workflow.parameters.min-hold-bars}} \ - --data-dir {{workflow.parameters.data-dir}} \ - --mbp10-data-dir {{workflow.parameters.mbp10-data-dir}} \ - --trades-data-dir {{workflow.parameters.trades-data-dir}} \ - --output-dir /workspace/output \ - --epochs {{workflow.parameters.train-epochs}} \ - $HYPEROPT_FLAG - - echo "=== Training complete ===" - resources: - requests: - nvidia.com/gpu: "1" - cpu: "4" - memory: 32Gi - limits: - nvidia.com/gpu: "1" - cpu: "8" - memory: 64Gi - volumeMounts: - - name: workspace - mountPath: /workspace - - name: training-data - mountPath: /data - readOnly: true - - name: feature-cache - mountPath: /feature-cache - readOnly: true - - # ── evaluate: run evaluation on trained model ── - - name: evaluate - nodeSelector: - k8s.scaleway.com/pool-name: "{{workflow.parameters.gpu-pool}}" - tolerations: - - key: nvidia.com/gpu - operator: Exists - effect: NoSchedule - - key: node.cilium.io/agent-not-ready - operator: Exists - effect: NoSchedule - container: - image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/ci-builder:latest - imagePullPolicy: IfNotPresent - command: ["/bin/sh", "-c"] - env: - - name: RUST_LOG - value: info - - name: SQLX_OFFLINE - value: "true" - args: - - | - set -e - export PATH="/workspace/bin:$PATH" - export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//') - - MODEL="{{workflow.parameters.model}}" - case "$MODEL" in - dqn|ppo) BINARY=evaluate_baseline ;; - *) BINARY=evaluate_supervised ;; - esac - - echo "=== Evaluating: $MODEL ===" - ${BINARY} \ - --model "$MODEL" \ - --symbol {{workflow.parameters.symbol}} \ - --data-dir {{workflow.parameters.data-dir}} \ - --checkpoint-dir /workspace/output \ - --output-dir /workspace/output/eval \ - --initial-capital {{workflow.parameters.initial-capital}} \ - --min-hold-bars {{workflow.parameters.min-hold-bars}} \ - --tx-cost-bps {{workflow.parameters.tx-cost-bps}} \ - --tick-size {{workflow.parameters.tick-size}} \ - --spread-ticks {{workflow.parameters.spread-ticks}} || { - echo "WARN: Evaluation failed, continuing to upload available results" - } - - echo "=== Evaluation complete ===" - resources: - requests: - nvidia.com/gpu: "1" - cpu: "2" - memory: 16Gi - limits: - nvidia.com/gpu: "1" - cpu: "4" - memory: 32Gi - volumeMounts: - - name: workspace - mountPath: /workspace - - name: training-data - mountPath: /data - readOnly: true - - name: feature-cache - mountPath: /feature-cache - readOnly: true - - # ── upload-results: push artifacts to GitLab packages ── - - name: upload-results - nodeSelector: - k8s.scaleway.com/pool-name: platform - container: - image: curlimages/curl:8.12.1 - command: ["/bin/sh", "-c"] - args: - - | - set -e - MODEL="{{workflow.parameters.model}}" - SYMBOL="{{workflow.parameters.symbol}}" - TIMESTAMP=$(date +%Y%m%d-%H%M%S) - GITLAB_API="http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181/api/v4" - PKG_NAME="foxhunt-training-results" - PKG_VERSION="${MODEL}-${SYMBOL}-${TIMESTAMP}" - - echo "Uploading training artifacts to GitLab packages: ${PKG_NAME}/${PKG_VERSION}" - - UPLOADED=0 - find /workspace/output -type f | while read -r file; do - REL_PATH="${file#/workspace/output/}" - SAFE_NAME=$(echo "$REL_PATH" | tr '/' '--') - echo " Uploading ${REL_PATH} as ${SAFE_NAME}..." - curl -f --upload-file "$file" \ - -H "PRIVATE-TOKEN: ${GITLAB_PAT}" \ - "${GITLAB_API}/projects/1/packages/generic/${PKG_NAME}/${PKG_VERSION}/${SAFE_NAME}" && \ - UPLOADED=$((UPLOADED + 1)) || \ - echo " WARN: Failed to upload ${REL_PATH}" - done - - echo "=== Upload complete (${UPLOADED} files) ===" - echo "Package: ${PKG_NAME}/${PKG_VERSION}" - env: - - name: GITLAB_PAT - valueFrom: - secretKeyRef: - name: gitlab-pat - key: token - volumeMounts: - - name: workspace - mountPath: /workspace - resources: - requests: - cpu: 100m - memory: 128Mi - limits: - cpu: 500m - memory: 256Mi - - # ── notify-result: post workflow outcome to Mattermost ── - - name: notify-result - nodeSelector: - k8s.scaleway.com/pool-name: platform - container: - image: curlimages/curl:8.12.1 - command: ["/bin/sh", "-c"] - env: - - name: WEBHOOK_URL - valueFrom: - secretKeyRef: - name: notification-webhook - key: webhook-url - optional: true - resources: - requests: - cpu: 50m - memory: 32Mi - limits: - cpu: 100m - memory: 64Mi - args: - - | - STATUS="{{workflow.status}}" - NAME="{{workflow.name}}" - - if [ -z "$WEBHOOK_URL" ] || echo "$WEBHOOK_URL" | grep -q "PLACEHOLDER"; then - echo "No webhook configured, skipping notification" - exit 0 - fi - - if [ "$STATUS" = "Succeeded" ]; then - EMOJI=":white_check_mark:" - else - EMOJI=":x:" - fi - - PAYLOAD="{\"username\":\"Argo CI\",\"text\":\"${EMOJI} **${NAME}** — ${STATUS} ({{workflow.duration}}s)\"}" - - curl -sf -X POST -H 'Content-Type: application/json' \ - -d "$PAYLOAD" "$WEBHOOK_URL" || echo "WARN: webhook post failed" diff --git a/infra/k8s/argo/kustomization.yaml b/infra/k8s/argo/kustomization.yaml index 49ecfbf34..57eb8269c 100644 --- a/infra/k8s/argo/kustomization.yaml +++ b/infra/k8s/argo/kustomization.yaml @@ -4,12 +4,8 @@ resources: - sccache-pvcs.yaml - ci-pipeline-template.yaml - compile-and-deploy-template.yaml - - compile-and-train-template.yaml + - train-template.yaml - build-ci-image-template.yaml - - training-workflow-template.yaml - - train-dqn-template.yaml - - train-ppo-template.yaml - - train-supervised-template.yaml - argo-workflow-netpol.yaml - ci-deploy-rbac.yaml - archive-rbac.yaml diff --git a/infra/k8s/argo/precompute-features-template.yaml b/infra/k8s/argo/precompute-features-template.yaml deleted file mode 100644 index b225d32bb..000000000 --- a/infra/k8s/argo/precompute-features-template.yaml +++ /dev/null @@ -1,122 +0,0 @@ -# Pre-compute DQN training features to .fxcache format (f32). -# -# Runs after data downloads complete. Reads DBN files from training-data PVC, -# extracts all features (OHLCV 42-dim + targets 4-dim + OFI 8-dim), writes -# flat binary cache for zero-overhead GPU loading. -# -# Usage: -# argo submit -n foxhunt --from=wftmpl/precompute-features -# ./scripts/argo-precompute.sh ES.FUT ---- -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate -metadata: - name: precompute-features - namespace: foxhunt - labels: - app.kubernetes.io/name: precompute-features - app.kubernetes.io/part-of: foxhunt -spec: - entrypoint: precompute - serviceAccountName: argo-workflow - podMetadata: - labels: - app.kubernetes.io/part-of: foxhunt - app.kubernetes.io/component: precompute-features - securityContext: - fsGroup: 1000 - ttlStrategy: - secondsAfterCompletion: 3600 - activeDeadlineSeconds: 7200 - - arguments: - parameters: - - name: symbol - value: ES.FUT - - name: data-dir - value: /data/futures-baseline - - name: mbp10-dir - value: /data/futures-baseline-mbp10 - - name: trades-dir - value: /data/futures-baseline-trades - - name: output-dir - value: /feature-cache - - name: node-pool - value: ci-compile-cpu - - volumes: - - name: training-data - persistentVolumeClaim: - claimName: training-data-pvc - - name: feature-cache - persistentVolumeClaim: - claimName: feature-cache-pvc - - templates: - - name: precompute - inputs: - parameters: - - name: symbol - - name: data-dir - - name: mbp10-dir - - name: trades-dir - - name: output-dir - - name: node-pool - nodeSelector: - k8s.scaleway.com/pool-name: "{{inputs.parameters.node-pool}}" - container: - image: ubuntu:24.04 - command: ["/bin/bash", "-c"] - env: - - name: RUST_LOG - value: info - resources: - requests: - cpu: "4" - memory: 16Gi - limits: - cpu: "28" - memory: 56Gi - volumeMounts: - - name: training-data - mountPath: /data - readOnly: true - - name: feature-cache - mountPath: /feature-cache - args: - - | - set -e - # Binary deployed to PVC by compile-and-train workflow - BINARY=/data/bin/precompute_features - if [ ! -x "$BINARY" ]; then - echo "ERROR: $BINARY not found or not executable. Run compile-and-train first." - exit 1 - fi - - SYMBOL="{{inputs.parameters.symbol}}" - DATA_DIR="{{inputs.parameters.data-dir}}" - MBP10_DIR="{{inputs.parameters.mbp10-dir}}" - TRADES_DIR="{{inputs.parameters.trades-dir}}" - OUTPUT_DIR="{{inputs.parameters.output-dir}}" - - echo "=== Feature Pre-Compute (f32) ===" - echo "Symbol: $SYMBOL" - echo "OHLCV: $DATA_DIR" - echo "MBP-10: $MBP10_DIR" - echo "Trades: $TRADES_DIR" - echo "Output: $OUTPUT_DIR" - echo "Node: $(hostname), CPUs: $(nproc), RAM: $(free -g | awk '/Mem:/{print $2}')G" - echo "" - - $BINARY \ - --data-dir "$DATA_DIR" \ - --mbp10-data-dir "$MBP10_DIR" \ - --trades-data-dir "$TRADES_DIR" \ - --output-dir "$OUTPUT_DIR" \ - --symbol "$SYMBOL" \ - --yes - - echo "" - echo "=== Output ===" - find "$OUTPUT_DIR" -name '*.fxcache' -exec ls -lh {} \; - du -sh "$OUTPUT_DIR" diff --git a/infra/k8s/argo/train-baseline-rl-template.yaml b/infra/k8s/argo/train-baseline-rl-template.yaml deleted file mode 100644 index 62f39eafc..000000000 --- a/infra/k8s/argo/train-baseline-rl-template.yaml +++ /dev/null @@ -1,102 +0,0 @@ -# Baseline RL training — compile-and-train with hyperopt disabled. -# -# Identical to train-dqn but hardcodes hyperopt-trials=0. -# Compiles from source, skips hyperopt, trains with default params. -# -# Usage: -# argo submit -n foxhunt --from=wftmpl/train-baseline-rl -# argo submit -n foxhunt --from=wftmpl/train-baseline-rl -p train-epochs=100 ---- -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate -metadata: - name: train-baseline-rl - namespace: foxhunt - labels: - app.kubernetes.io/name: train-baseline-rl - app.kubernetes.io/part-of: foxhunt -spec: - entrypoint: pipeline - serviceAccountName: argo-workflow - podMetadata: - labels: - app.kubernetes.io/part-of: foxhunt - app.kubernetes.io/component: compile-and-train - securityContext: - fsGroup: 0 - ttlStrategy: - secondsAfterCompletion: 3600 - activeDeadlineSeconds: 14400 - - # Mirror ALL compile-and-train parameters — Argo resolves - # {{workflow.parameters.*}} in the caller's scope. - arguments: - parameters: - - name: commit-sha - value: HEAD - - name: git-branch - value: main - - name: cuda-compute-cap - value: "90" - - name: model - value: dqn - - name: gpu-pool - value: ci-training-h100 - - name: hyperopt-trials - value: "0" - - name: hyperopt-epochs - value: "0" - - name: train-epochs - value: "50" - - name: symbol - value: ES.FUT - - name: data-dir - value: /data/futures-baseline - - name: mbp10-data-dir - value: /data/futures-baseline-mbp10 - - name: trades-data-dir - value: /data/futures-baseline-trades - - name: feature-cache-dir - value: /data/feature-cache - - name: tx-cost-bps - value: "0.1" - - name: tick-size - value: "0.25" - - name: spread-ticks - value: "1.0" - - name: min-hold-bars - value: "5" - - name: initial-capital - value: "35000" - - name: ensemble-top-k - value: "5" - - volumes: - - name: git-ssh-key - secret: - secretName: argo-git-ssh-key - defaultMode: 256 - - name: training-data - persistentVolumeClaim: - claimName: training-data-pvc - - name: cargo-target-cuda - persistentVolumeClaim: - claimName: cargo-target-cuda - - volumeClaimTemplates: - - metadata: - name: workspace - spec: - accessModes: ["ReadWriteOnce"] - storageClassName: scw-bssd - resources: - requests: - storage: 5Gi - - templates: - - name: pipeline - steps: - - - name: run - templateRef: - name: compile-and-train - template: pipeline diff --git a/infra/k8s/argo/train-dqn-template.yaml b/infra/k8s/argo/train-dqn-template.yaml deleted file mode 100644 index 297c2947f..000000000 --- a/infra/k8s/argo/train-dqn-template.yaml +++ /dev/null @@ -1,105 +0,0 @@ -# Train DQN: compile → hyperopt → train → evaluate → upload. -# -# Usage: -# argo submit --from=wftmpl/train-dqn -# argo submit --from=wftmpl/train-dqn -p commit-ref=my-branch -# -# Thin wrapper around compile-and-train with DQN-tuned defaults. -# Compiles from source (no pre-built binary dependency). ---- -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate -metadata: - name: train-dqn - namespace: foxhunt - labels: - app.kubernetes.io/name: train-dqn - app.kubernetes.io/part-of: foxhunt -spec: - entrypoint: pipeline - serviceAccountName: argo-workflow - podMetadata: - labels: - app.kubernetes.io/part-of: foxhunt - app.kubernetes.io/component: compile-and-train - securityContext: - fsGroup: 1000 - ttlStrategy: - secondsAfterCompletion: 3600 - activeDeadlineSeconds: 28800 # 8 hours - - arguments: - parameters: - - name: model - value: dqn - - name: commit-sha - value: HEAD - - name: git-branch - value: main - - name: gpu-pool - value: ci-training-h100 - - name: hyperopt-trials - value: "50" - - name: hyperopt-epochs - value: "20" - - name: train-epochs - value: "200" - - name: symbol - value: ES.FUT - - name: data-dir - value: /data/futures-baseline - - name: mbp10-data-dir - value: /data/futures-baseline-mbp10 - - name: trades-data-dir - value: /data/futures-baseline-trades - - name: feature-cache-dir - value: /feature-cache - - name: tx-cost-bps - value: "0.1" - - name: tick-size - value: "0.25" - - name: spread-ticks - value: "1.0" - - name: min-hold-bars - value: "5" - - name: initial-capital - value: "35000" - - name: ensemble-top-k - value: "5" - - name: cuda-compute-cap - value: "90" - - # Volumes required by compile-and-train steps - volumes: - - name: git-ssh-key - secret: - secretName: argo-git-ssh-key - defaultMode: 256 - - name: training-data - persistentVolumeClaim: - claimName: training-data-pvc - readOnly: true - - name: cargo-target-cuda - persistentVolumeClaim: - claimName: cargo-target-cuda - - name: feature-cache - persistentVolumeClaim: - claimName: feature-cache-pvc - - volumeClaimTemplates: - - metadata: - name: workspace - spec: - accessModes: ["ReadWriteOnce"] - storageClassName: scw-bssd - resources: - requests: - storage: 5Gi - - templates: - - name: pipeline - steps: - - - name: compile-and-train - templateRef: - name: compile-and-train - template: pipeline diff --git a/infra/k8s/argo/train-ppo-template.yaml b/infra/k8s/argo/train-ppo-template.yaml deleted file mode 100644 index 0b2a5c9b9..000000000 --- a/infra/k8s/argo/train-ppo-template.yaml +++ /dev/null @@ -1,107 +0,0 @@ -# Train PPO: fetch binaries → hyperopt → train → evaluate → upload. -# -# Usage: -# argo submit --from=wftmpl/train-ppo -p binary-tag=dev-abc1234 -# argo submit --from=wftmpl/train-ppo -p binary-tag=latest -p train-epochs=100 -# -# Thin wrapper around training-pipeline (templateRef) with PPO-tuned defaults. -# PPO uses 45 factored actions (5 exposure × 3 order × 3 urgency). ---- -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate -metadata: - name: train-ppo - namespace: foxhunt - labels: - app.kubernetes.io/name: train-ppo - app.kubernetes.io/part-of: foxhunt - app.kubernetes.io/component: gpu-test -spec: - entrypoint: pipeline - serviceAccountName: argo-workflow - podMetadata: - labels: - app.kubernetes.io/part-of: foxhunt - app.kubernetes.io/component: gpu-test - app.kubernetes.io/component: training - securityContext: - runAsUser: 1000 - runAsGroup: 1000 - fsGroup: 1000 - ttlStrategy: - secondsAfterCompletion: 3600 - activeDeadlineSeconds: 28800 # 8 hours - - arguments: - parameters: - - name: model - value: ppo - - name: commit-sha - value: HEAD - - name: git-branch - value: main - - name: gpu-pool - value: ci-training-h100 - - name: hyperopt-trials - value: "20" - - name: hyperopt-epochs - value: "8" - - name: train-epochs - value: "50" - - name: symbol - value: ES.FUT - - name: data-dir - value: /data/futures-baseline - - name: mbp10-data-dir - value: /data/futures-baseline-mbp10 - - name: trades-data-dir - value: /data/futures-baseline-trades - - name: tx-cost-bps - value: "0.1" - - name: tick-size - value: "0.25" - - name: spread-ticks - value: "1.0" - - name: initial-capital - value: "35000" - - name: feature-cache-dir - value: /feature-cache - - name: ensemble-top-k - value: "5" - - name: cuda-compute-cap - value: "90" - - # Volumes required by compile-and-train steps - volumes: - - name: git-ssh-key - secret: - secretName: argo-git-ssh-key - defaultMode: 256 - - name: training-data - persistentVolumeClaim: - claimName: training-data-pvc - readOnly: true - - name: cargo-target-cuda - persistentVolumeClaim: - claimName: cargo-target-cuda - - name: feature-cache - persistentVolumeClaim: - claimName: feature-cache-pvc - - volumeClaimTemplates: - - metadata: - name: workspace - spec: - accessModes: ["ReadWriteOnce"] - storageClassName: scw-bssd - resources: - requests: - storage: 5Gi - - templates: - - name: pipeline - steps: - - - name: compile-and-train - templateRef: - name: compile-and-train - template: pipeline diff --git a/infra/k8s/argo/train-supervised-template.yaml b/infra/k8s/argo/train-supervised-template.yaml deleted file mode 100644 index 87ca1d557..000000000 --- a/infra/k8s/argo/train-supervised-template.yaml +++ /dev/null @@ -1,111 +0,0 @@ -# Train supervised model: fetch binaries → hyperopt → train → evaluate → upload. -# -# Usage (any of the 8 supervised models): -# argo submit --from=wftmpl/train-supervised -p model=tft -p binary-tag=latest -# argo submit --from=wftmpl/train-supervised -p model=mamba2 -p binary-tag=dev-abc1234 -# argo submit --from=wftmpl/train-supervised -p model=kan -p binary-tag=latest -p train-epochs=100 -# -# Supported models: tft, mamba2, tggn, tlob, liquid, kan, xlstm, diffusion -# -# Thin wrapper around training-pipeline (templateRef) with supervised defaults. -# Uses hyperopt_baseline_supervised / train_baseline_supervised / evaluate_supervised. ---- -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate -metadata: - name: train-supervised - namespace: foxhunt - labels: - app.kubernetes.io/name: train-supervised - app.kubernetes.io/part-of: foxhunt - app.kubernetes.io/component: gpu-test -spec: - entrypoint: pipeline - serviceAccountName: argo-workflow - podMetadata: - labels: - app.kubernetes.io/part-of: foxhunt - app.kubernetes.io/component: gpu-test - app.kubernetes.io/component: training - securityContext: - runAsUser: 1000 - runAsGroup: 1000 - fsGroup: 1000 - ttlStrategy: - secondsAfterCompletion: 3600 - activeDeadlineSeconds: 28800 # 8 hours - - arguments: - parameters: - - name: model - # One of: tft, mamba2, tggn, tlob, liquid, kan, xlstm, diffusion - value: tft - - name: commit-sha - value: HEAD - - name: git-branch - value: main - - name: gpu-pool - value: ci-training-h100 - - name: hyperopt-trials - value: "20" - - name: hyperopt-epochs - value: "8" - - name: train-epochs - value: "50" - - name: symbol - value: ES.FUT - - name: data-dir - value: /data/futures-baseline - - name: mbp10-data-dir - value: /data/futures-baseline-mbp10 - - name: trades-data-dir - value: /data/futures-baseline-trades - - name: tx-cost-bps - value: "0.1" - - name: tick-size - value: "0.25" - - name: spread-ticks - value: "1.0" - - name: initial-capital - value: "35000" - - name: feature-cache-dir - value: /feature-cache - - name: ensemble-top-k - value: "5" - - name: cuda-compute-cap - value: "90" - - # Volumes required by compile-and-train steps - volumes: - - name: git-ssh-key - secret: - secretName: argo-git-ssh-key - defaultMode: 256 - - name: training-data - persistentVolumeClaim: - claimName: training-data-pvc - readOnly: true - - name: cargo-target-cuda - persistentVolumeClaim: - claimName: cargo-target-cuda - - name: feature-cache - persistentVolumeClaim: - claimName: feature-cache-pvc - - volumeClaimTemplates: - - metadata: - name: workspace - spec: - accessModes: ["ReadWriteOnce"] - storageClassName: scw-bssd - resources: - requests: - storage: 5Gi - - templates: - - name: pipeline - steps: - - - name: compile-and-train - templateRef: - name: compile-and-train - template: pipeline diff --git a/infra/k8s/argo/training-workflow-template.yaml b/infra/k8s/argo/training-workflow-template.yaml deleted file mode 100644 index 680b9aaee..000000000 --- a/infra/k8s/argo/training-workflow-template.yaml +++ /dev/null @@ -1,489 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate -metadata: - name: training-pipeline - namespace: foxhunt - labels: - app.kubernetes.io/name: training-pipeline - app.kubernetes.io/part-of: foxhunt - app.kubernetes.io/component: gpu-test -spec: - entrypoint: train-model - serviceAccountName: argo-workflow - # Label all workflow pods so network policies (MinIO, DNS) allow traffic - podMetadata: - labels: - app.kubernetes.io/part-of: foxhunt - app.kubernetes.io/component: training-workflow - securityContext: - runAsUser: 1000 - runAsGroup: 1000 - fsGroup: 1000 - ttlStrategy: - secondsAfterCompletion: 3600 - activeDeadlineSeconds: 21600 - - arguments: - parameters: - - name: model - - name: binary-tag - # Required: full commit SHA, e.g. "dev-6ab9e184abcd1234..." - # No default — caller must specify which compiled binaries to use. - - name: gpu-pool - value: ci-training-h100 # Pools: ci-training-h100, ci-training-l40s, ci-training-h100x2 - - name: hyperopt-trials - value: "20" - - name: hyperopt-epochs - value: "8" - - name: train-epochs - value: "50" - - name: symbol - value: ES.FUT - - name: data-dir - value: /data/futures-baseline - - name: mbp10-data-dir - value: /data/futures-baseline-mbp10 - - name: trades-data-dir - value: /data/futures-baseline-trades - - name: tx-cost-bps - value: "0.1" - - name: tick-size - value: "0.25" - - name: spread-ticks - value: "1.0" - - name: initial-capital - value: "35000" - - name: ensemble-top-k - value: "5" - - name: cuda-compute-cap - value: "90" - - # Training data PVC — already populated by download jobs (OHLCV, MBP-10, trades). - # RWO: only one pod at a time, but steps run sequentially so no conflict. - volumes: - - name: training-data - persistentVolumeClaim: - claimName: training-data-pvc - readOnly: true - - # Ephemeral shared storage for passing artifacts between DAG steps. - # Auto-deleted when workflow completes (all artifacts uploaded to MinIO). - volumeClaimTemplates: - - metadata: - name: workspace - spec: - accessModes: ["ReadWriteOnce"] - storageClassName: scw-bssd - resources: - requests: - storage: 5Gi - - templates: - - name: train-model - dag: - tasks: - - name: fetch-binary - template: fetch-binary - # gpu-warmup disabled — re-enable when GPU nodes stay warm - # - name: gpu-warmup - # template: gpu-warmup - - name: hyperopt - template: hyperopt - dependencies: [fetch-binary] - - name: train-best - template: train-best - dependencies: [hyperopt] - - name: evaluate - template: evaluate - dependencies: [train-best] - - name: upload-results - template: upload-results - dependencies: [evaluate] - - # ── gpu-warmup: disabled — re-enable when GPU nodes stay warm ── - # - name: gpu-warmup - # nodeSelector: - # k8s.scaleway.com/pool-name: "{{workflow.parameters.gpu-pool}}" - # tolerations: - # - key: nvidia.com/gpu - # operator: Exists - # effect: NoSchedule - # - key: node.cilium.io/agent-not-ready - # operator: Exists - # effect: NoSchedule - # container: - # image: busybox:1.37 - # command: ["/bin/sh", "-c"] - # args: - # - | - # echo "GPU warmup: triggering node autoscale..." - # echo "GPU node scheduled, exiting to free resources" - # resources: - # requests: - # nvidia.com/gpu: "1" - # cpu: 100m - # memory: 64Mi - # limits: - # nvidia.com/gpu: "1" - # cpu: 200m - # memory: 128Mi - - - name: fetch-binary - nodeSelector: - k8s.scaleway.com/pool-name: platform - container: - image: curlimages/curl:8.12.1 - command: ["/bin/sh", "-c"] - args: - - | - set -e - mkdir -p /workspace/bin - GITLAB_API="http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181/api/v4" - - TAG="{{workflow.parameters.binary-tag}}" - if [ -z "$TAG" ]; then - echo "ERROR: binary-tag parameter is required (full commit SHA, e.g. dev-6ab9e184...)" - exit 1 - fi - echo "Fetching training binaries from GitLab package: foxhunt-training/${TAG}" - - BINARIES="hyperopt_baseline_rl hyperopt_baseline_supervised train_baseline_rl train_baseline_supervised evaluate_baseline evaluate_supervised training_uploader" - for bin in $BINARIES; do - echo " Downloading ${bin}..." - curl -fSL -o "/workspace/bin/${bin}" \ - -H "PRIVATE-TOKEN: ${GITLAB_PAT}" \ - "${GITLAB_API}/projects/1/packages/generic/foxhunt-training/${TAG}/${bin}" || { - echo "WARN: ${bin} not found in package foxhunt-training/${TAG}, skipping" - continue - } - done - - chmod +x /workspace/bin/* - echo "Fetched binaries:" - ls -lh /workspace/bin/ - env: - - name: GITLAB_PAT - valueFrom: - secretKeyRef: - name: gitlab-pat - key: token - volumeMounts: - - name: workspace - mountPath: /workspace - resources: - requests: - cpu: 100m - memory: 128Mi - limits: - cpu: 500m - memory: 256Mi - - - name: hyperopt - nodeSelector: - k8s.scaleway.com/pool-name: "{{workflow.parameters.gpu-pool}}" - tolerations: - - key: nvidia.com/gpu - operator: Exists - effect: NoSchedule - - key: node.cilium.io/agent-not-ready - operator: Exists - effect: NoSchedule - container: - image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/ci-builder:latest - imagePullPolicy: IfNotPresent - command: ["/bin/sh", "-c"] - args: - - | - set -e - export PATH="/workspace/bin:$PATH" - export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//') - nvidia-smi - - # Training data available via training-data-pvc mounted at /data - echo "=== Training data (PVC) ===" - echo "OHLCV: $(ls {{workflow.parameters.data-dir}}/ES.FUT/*.dbn.zst 2>/dev/null | wc -l) files" - echo "MBP-10: $(ls {{workflow.parameters.mbp10-data-dir}}/ES.FUT/*.dbn.zst 2>/dev/null | wc -l) files" - echo "Trades: $(ls {{workflow.parameters.trades-data-dir}}/ES.FUT/*.dbn.zst 2>/dev/null | wc -l) files" - - mkdir -p /workspace/output/hyperopt - - MODEL="{{workflow.parameters.model}}" - case "$MODEL" in - dqn|ppo) BINARY=hyperopt_baseline_rl ;; - *) BINARY=hyperopt_baseline_supervised ;; - esac - - TRIALS={{workflow.parameters.hyperopt-trials}} - N_INITIAL=5 - if [ "$TRIALS" -le "$N_INITIAL" ]; then - N_INITIAL=$((TRIALS > 1 ? TRIALS - 1 : 1)) - fi - - echo "Running $BINARY --model $MODEL ($TRIALS trials, $N_INITIAL initial x {{workflow.parameters.hyperopt-epochs}} epochs)" - # Phase 1 (fast): fix architecture, search learning dynamics - $BINARY \ - --model "$MODEL" \ - --phase fast \ - --trials "$TRIALS" \ - --n-initial "$N_INITIAL" \ - --epochs {{workflow.parameters.hyperopt-epochs}} \ - --parallel 0 \ - --symbol {{workflow.parameters.symbol}} \ - --tx-cost-bps {{workflow.parameters.tx-cost-bps}} \ - --tick-size {{workflow.parameters.tick-size}} \ - --spread-ticks {{workflow.parameters.spread-ticks}} \ - --initial-capital {{workflow.parameters.initial-capital}} \ - --data-dir {{workflow.parameters.data-dir}} \ - --mbp10-data-dir {{workflow.parameters.mbp10-data-dir}} \ - --trades-data-dir {{workflow.parameters.trades-data-dir}} \ - --base-dir /workspace/output/hyperopt \ - --output /workspace/output/${MODEL}_phase1_results.json - - echo "=== Phase 1 complete ===" - cat /workspace/output/${MODEL}_phase1_results.json 2>/dev/null || echo "No Phase 1 results" - - # Phase 2 (full): fix dynamics from Phase 1, search architecture - PHASE2_TRIALS=$((TRIALS / 2)) - [ "$PHASE2_TRIALS" -lt 5 ] && PHASE2_TRIALS=5 - PHASE2_EPOCHS=$(({{workflow.parameters.hyperopt-epochs}} * 2)) - - $BINARY \ - --model "$MODEL" \ - --phase full \ - --hyperopt-params /workspace/output/${MODEL}_phase1_results.json \ - --trials $PHASE2_TRIALS \ - --epochs $PHASE2_EPOCHS \ - --parallel 0 \ - --symbol {{workflow.parameters.symbol}} \ - --tx-cost-bps {{workflow.parameters.tx-cost-bps}} \ - --tick-size {{workflow.parameters.tick-size}} \ - --spread-ticks {{workflow.parameters.spread-ticks}} \ - --initial-capital {{workflow.parameters.initial-capital}} \ - --data-dir {{workflow.parameters.data-dir}} \ - --mbp10-data-dir {{workflow.parameters.mbp10-data-dir}} \ - --trades-data-dir {{workflow.parameters.trades-data-dir}} \ - --base-dir /workspace/output/hyperopt \ - --output /workspace/output/${MODEL}_hyperopt_results.json - - echo "=== Two-phase hyperopt complete ===" - cat /workspace/output/${MODEL}_hyperopt_results.json 2>/dev/null || echo "No results file" - env: - - name: RUST_LOG - value: info - - name: SQLX_OFFLINE - value: "true" - - name: OTEL_EXPORTER_OTLP_ENDPOINT - value: "http://tempo.foxhunt.svc.cluster.local:4317" - - name: CUBLAS_WORKSPACE_CONFIG - value: ":4096:8" - volumeMounts: - - name: workspace - mountPath: /workspace - - name: training-data - mountPath: /data - readOnly: true - resources: - requests: - nvidia.com/gpu: "1" - cpu: "7" - memory: 16Gi - limits: - nvidia.com/gpu: "1" - cpu: "8" - memory: 48Gi - - - name: train-best - nodeSelector: - k8s.scaleway.com/pool-name: "{{workflow.parameters.gpu-pool}}" - tolerations: - - key: nvidia.com/gpu - operator: Exists - effect: NoSchedule - - key: node.cilium.io/agent-not-ready - operator: Exists - effect: NoSchedule - container: - image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/ci-builder:latest - imagePullPolicy: IfNotPresent - command: ["/bin/sh", "-c"] - args: - - | - set -e - export PATH="/workspace/bin:$PATH" - export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//') - nvidia-smi - - # Training data available via training-data-pvc mounted at /data - - MODEL="{{workflow.parameters.model}}" - case "$MODEL" in - dqn|ppo) BINARY=train_baseline_rl ;; - *) BINARY=train_baseline_supervised ;; - esac - - HYPEROPT_FILE="/workspace/output/${MODEL}_hyperopt_results.json" - HYPEROPT_FLAG="" - if [ -f "$HYPEROPT_FILE" ]; then - HYPEROPT_FLAG="--hyperopt-params $HYPEROPT_FILE" - fi - - echo "Training $MODEL with best params ({{workflow.parameters.train-epochs}} epochs)" - $BINARY \ - --model "$MODEL" \ - --symbol {{workflow.parameters.symbol}} \ - --tx-cost-bps {{workflow.parameters.tx-cost-bps}} \ - --tick-size {{workflow.parameters.tick-size}} \ - --spread-ticks {{workflow.parameters.spread-ticks}} \ - --data-dir {{workflow.parameters.data-dir}} \ - --mbp10-data-dir {{workflow.parameters.mbp10-data-dir}} \ - --trades-data-dir {{workflow.parameters.trades-data-dir}} \ - --output-dir /workspace/output \ - --epochs {{workflow.parameters.train-epochs}} \ - $HYPEROPT_FLAG \ - --ensemble-top-k {{workflow.parameters.ensemble-top-k}} - - echo "=== Training complete ===" - ls -lh /workspace/output/ - env: - - name: RUST_LOG - value: info - - name: SQLX_OFFLINE - value: "true" - - name: FOXHUNT_TRAINING_PROFILE - value: /workspace/config/training/dqn-production.toml - - name: OTEL_EXPORTER_OTLP_ENDPOINT - value: "http://tempo.foxhunt.svc.cluster.local:4317" - - name: CUBLAS_WORKSPACE_CONFIG - value: ":4096:8" - volumeMounts: - - name: workspace - mountPath: /workspace - - name: training-data - mountPath: /data - readOnly: true - resources: - requests: - nvidia.com/gpu: "1" - cpu: "7" - memory: 16Gi - limits: - nvidia.com/gpu: "1" - cpu: "8" - memory: 48Gi - - - name: evaluate - nodeSelector: - k8s.scaleway.com/pool-name: "{{workflow.parameters.gpu-pool}}" - tolerations: - - key: nvidia.com/gpu - operator: Exists - effect: NoSchedule - - key: node.cilium.io/agent-not-ready - operator: Exists - effect: NoSchedule - container: - image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/ci-builder:latest - imagePullPolicy: IfNotPresent - command: ["/bin/sh", "-c"] - args: - - | - set -e - export PATH="/workspace/bin:$PATH" - export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//') - - # Training data available via training-data-pvc mounted at /data - - MODEL="{{workflow.parameters.model}}" - HYPEROPT_FILE="/workspace/output/${MODEL}_hyperopt_results.json" - HYPEROPT_FLAG="" - if [ -f "$HYPEROPT_FILE" ]; then - HYPEROPT_FLAG="--hyperopt-params $HYPEROPT_FILE" - fi - - # Evaluate all ensemble members (evaluator auto-discovers *_ensemble_*.safetensors) - echo "Evaluating $MODEL" - evaluate_baseline \ - --model "$MODEL" \ - --models-dir /workspace/output \ - --data-dir {{workflow.parameters.data-dir}} \ - --output /workspace/output/${MODEL}_eval_report.json \ - --symbol {{workflow.parameters.symbol}} \ - --tx-cost-bps {{workflow.parameters.tx-cost-bps}} \ - --tick-size {{workflow.parameters.tick-size}} \ - --spread-ticks {{workflow.parameters.spread-ticks}} \ - $HYPEROPT_FLAG \ - || true - - echo "=== Eval report ===" - cat /workspace/output/${MODEL}_eval_report.json 2>/dev/null || echo "No eval report" - env: - - name: RUST_LOG - value: info - - name: SQLX_OFFLINE - value: "true" - volumeMounts: - - name: workspace - mountPath: /workspace - - name: training-data - mountPath: /data - readOnly: true - resources: - requests: - nvidia.com/gpu: "1" - cpu: "4" - memory: 16Gi - limits: - nvidia.com/gpu: "1" - cpu: "8" - memory: 32Gi - - - name: upload-results - nodeSelector: - k8s.scaleway.com/pool-name: platform - container: - image: curlimages/curl:8.12.1 - command: ["/bin/sh", "-c"] - args: - - | - set -e - MODEL="{{workflow.parameters.model}}" - SYMBOL="{{workflow.parameters.symbol}}" - TIMESTAMP=$(date +%Y%m%d-%H%M%S) - GITLAB_API="http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181/api/v4" - PKG_NAME="foxhunt-training-results" - PKG_VERSION="${MODEL}-${SYMBOL}-${TIMESTAMP}" - - echo "Uploading training artifacts to GitLab packages: ${PKG_NAME}/${PKG_VERSION}" - echo " checkpoints (*.safetensors), norm stats, hyperopt results, eval reports" - - UPLOADED=0 - find /workspace/output -type f | while read -r file; do - REL_PATH="${file#/workspace/output/}" - # Replace / with -- for flat package file naming - SAFE_NAME=$(echo "$REL_PATH" | tr '/' '--') - echo " Uploading ${REL_PATH} as ${SAFE_NAME}..." - curl -f --upload-file "$file" \ - -H "PRIVATE-TOKEN: ${GITLAB_PAT}" \ - "${GITLAB_API}/projects/1/packages/generic/${PKG_NAME}/${PKG_VERSION}/${SAFE_NAME}" && \ - UPLOADED=$((UPLOADED + 1)) || \ - echo " WARN: Failed to upload ${REL_PATH}" - done - - echo "=== Upload complete (${UPLOADED} files) ===" - echo "Package: ${PKG_NAME}/${PKG_VERSION}" - env: - - name: GITLAB_PAT - valueFrom: - secretKeyRef: - name: gitlab-pat - key: token - volumeMounts: - - name: workspace - mountPath: /workspace - resources: - requests: - cpu: 100m - memory: 128Mi - limits: - cpu: 500m - memory: 256Mi diff --git a/scripts/argo-precompute.sh b/scripts/argo-precompute.sh deleted file mode 100755 index 852a1276a..000000000 --- a/scripts/argo-precompute.sh +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env bash -# Pre-compute DQN features via Argo Workflows. -# -# Usage: -# ./scripts/argo-precompute.sh ES.FUT -# ./scripts/argo-precompute.sh ES.FUT --pool platform --watch -# -# Requires: argo CLI -set -euo pipefail - -TEMPLATE="precompute-features" -POOL="" -DATA_DIR="" -MBP10_DIR="" -TRADES_DIR="" -OUTPUT_DIR="" -WATCH=false - -usage() { - cat < [OPTIONS] - -Options: - --pool Node pool (default: ci-compile-cpu) - --data-dir OHLCV data dir (default: /data/futures-baseline) - --mbp10-dir MBP-10 data dir (default: /data/futures-baseline-mbp10) - --trades-dir Trades data dir (default: /data/futures-baseline-trades) - --output-dir Output dir (default: /data/feature-cache) - --watch Follow workflow logs after submission - -h, --help Show this help -EOF - exit 0 -} - -[[ $# -eq 0 ]] && { echo "Error: symbol argument required"; usage; } -[[ "$1" == "-h" || "$1" == "--help" ]] && usage - -SYMBOL="$1"; shift - -while [[ $# -gt 0 ]]; do - case $1 in - --pool) POOL="$2"; shift 2 ;; - --data-dir) DATA_DIR="$2"; shift 2 ;; - --mbp10-dir) MBP10_DIR="$2"; shift 2 ;; - --trades-dir) TRADES_DIR="$2"; shift 2 ;; - --output-dir) OUTPUT_DIR="$2"; shift 2 ;; - --watch) WATCH=true; shift ;; - -h|--help) usage ;; - *) echo "Unknown option: $1"; usage ;; - esac -done - -CMD="argo submit -n foxhunt --from=wftmpl/$TEMPLATE" -CMD="$CMD -p symbol=$SYMBOL" - -[[ -n "$POOL" ]] && CMD="$CMD -p node-pool=$POOL" -[[ -n "$DATA_DIR" ]] && CMD="$CMD -p data-dir=$DATA_DIR" -[[ -n "$MBP10_DIR" ]] && CMD="$CMD -p mbp10-dir=$MBP10_DIR" -[[ -n "$TRADES_DIR" ]] && CMD="$CMD -p trades-dir=$TRADES_DIR" -[[ -n "$OUTPUT_DIR" ]] && CMD="$CMD -p output-dir=$OUTPUT_DIR" - -if $WATCH; then - CMD="$CMD --watch" -fi - -echo "Submitting feature pre-compute workflow..." -echo " symbol: $SYMBOL" -[[ -n "$POOL" ]] && echo " pool: $POOL" -echo "" -eval "$CMD" diff --git a/scripts/argo-train.sh b/scripts/argo-train.sh index ba6aae075..22b5668dc 100755 --- a/scripts/argo-train.sh +++ b/scripts/argo-train.sh @@ -1,29 +1,30 @@ #!/usr/bin/env bash -# Train a model via Argo Workflows (DQN, PPO, or any supervised model). +# Train a model via Argo Workflows. # # Usage: -# ./scripts/argo-train.sh dqn # DQN with defaults -# ./scripts/argo-train.sh ppo --tag dev-abc1234 # PPO with specific binary -# ./scripts/argo-train.sh tft --trials 40 --epochs 100 # supervised TFT -# ./scripts/argo-train.sh mamba2 --webhook # trigger via webhook +# ./scripts/argo-train.sh dqn # defaults: HEAD, H100, 50 epochs +# ./scripts/argo-train.sh dqn --sha abc1234 # specific commit +# ./scripts/argo-train.sh dqn --epochs 100 --trials 40 # override training params +# ./scripts/argo-train.sh ppo --gpu-pool ci-training # L40S instead of H100 +# ./scripts/argo-train.sh dqn --baseline # skip hyperopt +# ./scripts/argo-train.sh dqn --watch # follow logs # # Supported models: # RL: dqn, ppo # Supervised: tft, mamba2, tggn, tlob, liquid, kan, xlstm, diffusion # -# Requires: argo CLI (default) or curl (webhook mode) +# Requires: argo CLI set -euo pipefail -TAG="latest" +SHA="HEAD" +BRANCH="main" TRIALS="" EPOCHS="" GPU_POOL="" SYMBOL="" -WEBHOOK=false -WEBHOOK_BASE="http://workflow-trigger-eventsource-svc.foxhunt:12001" WATCH=false BASELINE=false -CACHE_DIR="" +CAPITAL="" usage() { cat < [OPTIONS] Models: dqn, ppo (RL) - tft, mamba2, tggn, tlob, liquid, kan, xlstm, diffusion (supervised) + tft, mamba2, tggn, tlob, liquid, kan, xlstm (supervised) Options: - --tag Binary tag to use (default: latest) - --trials Hyperopt trials (default: model-specific) - --epochs Training epochs (default: model-specific) - --gpu-pool GPU node pool (default: ci-training-h100) - --symbol Trading symbol (default: ES.FUT) - --baseline Skip hyperopt (set trials=0), train with defaults - --cache-dir Override feature cache directory - --webhook Trigger via webhook instead of argo CLI - --watch Follow workflow logs after submission - -h, --help Show this help + --sha Git commit SHA (default: HEAD) + --branch Git branch (default: main) + --trials Hyperopt trials (default: 20) + --epochs Training epochs (default: 50) + --gpu-pool GPU node pool (default: ci-training-h100) + --symbol Trading symbol (default: ES.FUT) + --capital Initial capital (default: 35000) + --baseline Skip hyperopt (trials=0) + --watch Follow workflow logs + -h, --help Show this help EOF exit 0 } @@ -52,7 +53,6 @@ EOF MODEL="$1"; shift -# Validate model name case "$MODEL" in dqn|ppo|tft|mamba2|tggn|tlob|liquid|kan|xlstm|diffusion) ;; *) echo "Error: unknown model '$MODEL'"; usage ;; @@ -60,72 +60,41 @@ esac while [[ $# -gt 0 ]]; do case $1 in - --tag) TAG="$2"; shift 2 ;; + --sha) SHA="$2"; shift 2 ;; + --branch) BRANCH="$2"; shift 2 ;; --trials) TRIALS="$2"; shift 2 ;; --epochs) EPOCHS="$2"; shift 2 ;; --gpu-pool) GPU_POOL="$2"; shift 2 ;; --symbol) SYMBOL="$2"; shift 2 ;; + --capital) CAPITAL="$2"; shift 2 ;; --baseline) BASELINE=true; shift ;; - --cache-dir) CACHE_DIR="$2"; shift 2 ;; - --webhook) WEBHOOK=true; shift ;; --watch) WATCH=true; shift ;; -h|--help) usage ;; *) echo "Unknown option: $1"; usage ;; esac done -# Determine workflow template name -case "$MODEL" in - dqn) TEMPLATE="train-dqn" ;; - ppo) TEMPLATE="train-ppo" ;; - *) TEMPLATE="train-supervised" ;; -esac - -if $WEBHOOK; then - case "$MODEL" in - dqn) ENDPOINT="/train/dqn"; PAYLOAD="{\"binary_tag\": \"${TAG}\"}" ;; - ppo) ENDPOINT="/train/ppo"; PAYLOAD="{\"binary_tag\": \"${TAG}\"}" ;; - *) ENDPOINT="/train/supervised"; PAYLOAD="{\"model\": \"${MODEL}\", \"binary_tag\": \"${TAG}\"}" ;; - esac - - echo "Triggering $MODEL training via webhook..." - curl -sf -X POST "${WEBHOOK_BASE}${ENDPOINT}" \ - -H 'Content-Type: application/json' \ - -d "$PAYLOAD" - echo "Webhook sent: $PAYLOAD" - exit 0 -fi - -# Build argo submit command -CMD="argo submit -n foxhunt --from=wftmpl/$TEMPLATE" -CMD="$CMD -p binary-tag=$TAG" - -# Supervised models need --model param -if [[ "$MODEL" != "dqn" && "$MODEL" != "ppo" ]]; then - CMD="$CMD -p model=$MODEL" -fi +CMD="argo submit -n foxhunt --from=wftmpl/train" +CMD="$CMD -p commit-sha=$SHA" +CMD="$CMD -p git-branch=$BRANCH" +CMD="$CMD -p model=$MODEL" [[ -n "$TRIALS" ]] && CMD="$CMD -p hyperopt-trials=$TRIALS" [[ -n "$EPOCHS" ]] && CMD="$CMD -p train-epochs=$EPOCHS" [[ -n "$GPU_POOL" ]] && CMD="$CMD -p gpu-pool=$GPU_POOL" [[ -n "$SYMBOL" ]] && CMD="$CMD -p symbol=$SYMBOL" -[[ -n "$CACHE_DIR" ]] && CMD="$CMD -p feature-cache-dir=$CACHE_DIR" +[[ -n "$CAPITAL" ]] && CMD="$CMD -p initial-capital=$CAPITAL" -if $BASELINE; then - CMD="$CMD -p hyperopt-trials=0" -fi +$BASELINE && CMD="$CMD -p hyperopt-trials=0" +$WATCH && CMD="$CMD --watch" -if $WATCH; then - CMD="$CMD --watch" -fi - -echo "Submitting $MODEL training workflow ($TEMPLATE)..." -echo " binary-tag: $TAG" -$BASELINE && echo " mode: baseline (hyperopt skipped)" -[[ -n "$TRIALS" ]] && echo " trials: $TRIALS" -[[ -n "$EPOCHS" ]] && echo " epochs: $EPOCHS" -[[ -n "$GPU_POOL" ]] && echo " gpu-pool: $GPU_POOL" -[[ -n "$SYMBOL" ]] && echo " symbol: $SYMBOL" -[[ -n "$CACHE_DIR" ]] && echo " cache-dir: $CACHE_DIR" +echo "Submitting $MODEL training workflow..." +echo " sha: $SHA" +echo " branch: $BRANCH" +echo " model: $MODEL" +$BASELINE && echo " mode: baseline (no hyperopt)" +[[ -n "$TRIALS" ]] && echo " trials: $TRIALS" +[[ -n "$EPOCHS" ]] && echo " epochs: $EPOCHS" +[[ -n "$GPU_POOL" ]] && echo " gpu: $GPU_POOL" echo "" eval "$CMD"