diff --git a/infra/k8s/argo/alpha-perception-template.yaml b/infra/k8s/argo/alpha-perception-template.yaml new file mode 100644 index 000000000..3d438ed15 --- /dev/null +++ b/infra/k8s/argo/alpha-perception-template.yaml @@ -0,0 +1,290 @@ +# Alpha perception trainer workflow. +# +# Stacked Mamba2 -> CfC -> heads supervised pretrain on MBP-10 from the +# training-data PVC. Emits `alpha_train_summary.json` to MinIO-backed +# feature-cache PVC with per-horizon val AUC for downstream gate +# consumption. Runs on a single L40S; ~30-90 min wall depending on +# n-train-seqs. +# +# DAG: +# ensure-binary ──> train +# +# Usage: +# argo submit -n foxhunt --from=wftmpl/alpha-perception \ +# -p commit-sha=HEAD -p git-branch=ml-alpha-phase-a +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: alpha-perception + namespace: foxhunt + labels: + app.kubernetes.io/name: alpha-perception + app.kubernetes.io/part-of: foxhunt + app.kubernetes.io/component: train +spec: + entrypoint: pipeline + serviceAccountName: argo-workflow + archiveLogs: true + podMetadata: + labels: + app.kubernetes.io/part-of: foxhunt + app.kubernetes.io/component: train + securityContext: + fsGroup: 0 + ttlStrategy: + secondsAfterCompletion: 3600 + activeDeadlineSeconds: 14400 + + arguments: + parameters: + - name: commit-sha + value: HEAD + - name: git-branch + value: ml-alpha-phase-a + - name: cuda-compute-cap + value: "89" + - name: gpu-pool + value: ci-training-l40s + # Trainer hyperparameters (mirror the validated smoke config). + - name: epochs + value: "5" + - name: seq-len + value: "32" + - name: mamba2-state-dim + value: "16" + - name: lr-cfc + value: "3e-3" + - name: lr-mamba2 + value: "1e-3" + - name: n-train-seqs + value: "8000" + - name: n-val-seqs + value: "1000" + - name: seed + value: "0x4242" + + volumes: + - name: git-ssh-key + secret: + secretName: argo-git-ssh-key + defaultMode: 256 + - name: training-data + persistentVolumeClaim: + claimName: training-data-pvc + - name: cargo-target-cuda + persistentVolumeClaim: + claimName: cargo-target-cuda + - name: feature-cache + persistentVolumeClaim: + claimName: feature-cache-pvc + + templates: + # ── DAG ── + - name: pipeline + dag: + tasks: + - name: ensure-binary + template: ensure-binary + - name: train + template: train + dependencies: [ensure-binary] + arguments: + parameters: + - name: sha + value: "{{tasks.ensure-binary.outputs.parameters.sha}}" + + # ── ensure-binary: compile `alpha_train` example via sccache ── + - name: ensure-binary + outputs: + parameters: + - name: sha + valueFrom: + path: /tmp/sha + nodeSelector: + k8s.scaleway.com/pool-name: ci-compile-cpu + topology.kubernetes.io/zone: fr-par-2 + tolerations: + - key: node.cilium.io/agent-not-ready + operator: Exists + effect: NoSchedule + container: + image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/ci-builder:latest + imagePullPolicy: Always + command: ["/bin/bash", "-c"] + env: + - name: SQLX_OFFLINE + value: "true" + - name: CARGO_TERM_COLOR + value: always + - name: CARGO_TARGET_DIR + value: /cargo-target + - name: CARGO_HOME + value: /cargo-target/cargo-home + - name: RUSTC_WRAPPER + value: sccache + - name: SCCACHE_DIR + value: /cargo-target/sccache + - name: SCCACHE_CACHE_SIZE + value: "40G" + - name: CARGO_INCREMENTAL + value: "0" + - name: CUDA_COMPUTE_CAP + value: "{{workflow.parameters.cuda-compute-cap}}" + resources: + requests: + cpu: "14" + memory: 32Gi + limits: + cpu: "30" + memory: 64Gi + volumeMounts: + - name: git-ssh-key + mountPath: /etc/git-ssh + readOnly: true + - name: cargo-target-cuda + mountPath: /cargo-target + - name: training-data + mountPath: /data + args: + - | + set -e + SHA="{{workflow.parameters.commit-sha}}" + BRANCH="{{workflow.parameters.git-branch}}" + mkdir -p ~/.ssh + cp /etc/git-ssh/ssh-privatekey ~/.ssh/id_ed25519 + chmod 600 ~/.ssh/id_ed25519 + printf 'Host *\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n' > ~/.ssh/config + chmod 600 ~/.ssh/config + REPO="ssh://git@gitlab-gitlab-shell.foxhunt.svc.cluster.local:2222/root/foxhunt.git" + BUILD="/cargo-target/src" + git config --global --add safe.directory "$BUILD" + + if [ "$SHA" = "HEAD" ]; then + if [ -d "$BUILD/.git" ]; then + cd "$BUILD"; git fetch origin; SHA=$(git rev-parse "origin/$BRANCH"); cd / + else + git clone --filter=blob:none "$REPO" "$BUILD"; cd "$BUILD" + git checkout "origin/$BRANCH"; SHA=$(git rev-parse HEAD); cd / + fi + fi + SHORT_SHA=$(echo "$SHA" | cut -c1-9) + echo "Resolved SHA: $SHA (short: $SHORT_SHA)" + + BIN_DIR="/data/bin/$SHORT_SHA" + if [ -x "$BIN_DIR/alpha_train" ]; then + echo "=== Cache HIT: $BIN_DIR/alpha_train ===" + ls -lh "$BIN_DIR/" + echo "$SHORT_SHA" > /tmp/sha + exit 0 + fi + echo "=== Cache MISS: compiling alpha_train for $SHORT_SHA ===" + + if [ -d "$BUILD/.git" ]; then + cd "$BUILD"; git fetch origin + CURRENT=$(git rev-parse HEAD 2>/dev/null || echo "none") + if [ "$CURRENT" != "$SHA" ]; then + git checkout --force "$SHA"; git clean -fd + fi + else + git clone --filter=blob:none "$REPO" "$BUILD"; cd "$BUILD"; git checkout "$SHA" + fi + + export PATH="${CARGO_HOME}/bin:${PATH}" + echo "Building alpha_train (ml-alpha)..." + cargo build --release -p ml-alpha --example alpha_train + + mkdir -p "$BIN_DIR" + cp "$CARGO_TARGET_DIR/release/examples/alpha_train" "$BIN_DIR/" + strip "$BIN_DIR/alpha_train" + ls -lh "$BIN_DIR/" + + # Prune old cache (keep last 5) + cd /data/bin + ls -1t | tail -n +6 | while read -r old; do + echo "Pruning old cache: $old"; rm -rf "$old" + done + echo "$SHORT_SHA" > /tmp/sha + + # ── train: run alpha_train on L40S ── + - name: train + inputs: + parameters: + - name: sha + nodeSelector: + k8s.scaleway.com/pool-name: "{{workflow.parameters.gpu-pool}}" + topology.kubernetes.io/zone: fr-par-2 + tolerations: + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + - key: node.cilium.io/agent-not-ready + operator: Exists + effect: NoSchedule + container: + image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/training-runtime:latest + imagePullPolicy: Always + command: ["/bin/bash", "-c"] + env: + - name: RUST_LOG + value: info + - name: SQLX_OFFLINE + value: "true" + - name: CUDA_COMPUTE_CAP + value: "{{workflow.parameters.cuda-compute-cap}}" + resources: + requests: + cpu: "8" + memory: 32Gi + nvidia.com/gpu: "1" + limits: + cpu: "16" + memory: 80Gi + nvidia.com/gpu: "1" + volumeMounts: + - name: training-data + mountPath: /data + - name: feature-cache + mountPath: /feature-cache + args: + - | + set -e + SHA="{{inputs.parameters.sha}}" + BIN="/data/bin/$SHA/alpha_train" + MBP10_DIR="/data/futures-baseline/mbp10" + PREDECODED_DIR="/feature-cache/predecoded" + OUT_DIR="/feature-cache/alpha-perception-runs/$SHA" + mkdir -p "$OUT_DIR" "$PREDECODED_DIR" + + if [ ! -x "$BIN" ]; then + echo "ERROR: binary not found at $BIN" + ls -lh /data/bin/ || true + exit 1 + fi + if [ ! -d "$MBP10_DIR" ]; then + echo "ERROR: MBP-10 data directory not found at $MBP10_DIR" + ls -lh /data/ || true + exit 1 + fi + echo "Running alpha_train (stacked Mamba2 -> CfC -> heads)" + echo " binary: $BIN" + echo " mbp10: $MBP10_DIR ($(find "$MBP10_DIR" -name '*.dbn.zst' | wc -l) files)" + echo " predecoded: $PREDECODED_DIR" + echo " out: $OUT_DIR" + + "$BIN" \ + --mbp10-data-dir "$MBP10_DIR" \ + --predecoded-dir "$PREDECODED_DIR" \ + --out "$OUT_DIR" \ + --epochs {{workflow.parameters.epochs}} \ + --seq-len {{workflow.parameters.seq-len}} \ + --mamba2-state-dim {{workflow.parameters.mamba2-state-dim}} \ + --lr-cfc {{workflow.parameters.lr-cfc}} \ + --lr-mamba2 {{workflow.parameters.lr-mamba2}} \ + --n-train-seqs {{workflow.parameters.n-train-seqs}} \ + --n-val-seqs {{workflow.parameters.n-val-seqs}} \ + --seed {{workflow.parameters.seed}} + + echo "=== Training complete ===" + ls -lh "$OUT_DIR/" + echo "--- alpha_train_summary.json ---" + cat "$OUT_DIR/alpha_train_summary.json" diff --git a/scripts/argo-alpha-perception.sh b/scripts/argo-alpha-perception.sh new file mode 100755 index 000000000..cd1205083 --- /dev/null +++ b/scripts/argo-alpha-perception.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +# Submit the alpha-perception workflow. +# +# Trains the stacked Mamba2 -> CfC -> heads perception model on MBP-10 +# from the training-data PVC. Emits alpha_train_summary.json with +# per-horizon validation AUC to the feature-cache PVC. +# +# Defaults match the validated synthetic-overfit smoke config; cluster +# runs can override for sweep work. +# +# Usage: +# ./scripts/argo-alpha-perception.sh # current HEAD on ml-alpha-phase-a +# ./scripts/argo-alpha-perception.sh --branch main +# ./scripts/argo-alpha-perception.sh --epochs 1 --n-train-seqs 1000 # quick smoke +# ./scripts/argo-alpha-perception.sh --watch + +set -euo pipefail + +SHA=HEAD +BRANCH=$(git symbolic-ref --short HEAD 2>/dev/null || echo "ml-alpha-phase-a") +GPU_POOL=ci-training-l40s +EPOCHS=5 +SEQ_LEN=32 +MAMBA2_STATE_DIM=16 +LR_CFC=3e-3 +LR_MAMBA2=1e-3 +N_TRAIN_SEQS=8000 +N_VAL_SEQS=1000 +SEED=0x4242 +WATCH=false + +usage() { + cat < Git SHA (default: HEAD on the current branch) + --branch Git branch (default: $BRANCH) + --gpu-pool GPU pool (default: $GPU_POOL) + --epochs Training epochs (default: $EPOCHS) + --seq-len Snapshots per sequence window (default: $SEQ_LEN) + --mamba2-state-dim Mamba2 SSM state dim (default: $MAMBA2_STATE_DIM) + --lr-cfc CfC learning rate (default: $LR_CFC) + --lr-mamba2 Mamba2 learning rate (default: $LR_MAMBA2) + --n-train-seqs Train sequences per epoch (default: $N_TRAIN_SEQS) + --n-val-seqs Val sequences per epoch (default: $N_VAL_SEQS) + --seed Random seed (default: $SEED) + --watch Follow logs via argo watch +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --sha) SHA="$2"; shift 2 ;; + --branch) BRANCH="$2"; shift 2 ;; + --gpu-pool) GPU_POOL="$2"; shift 2 ;; + --epochs) EPOCHS="$2"; shift 2 ;; + --seq-len) SEQ_LEN="$2"; shift 2 ;; + --mamba2-state-dim) MAMBA2_STATE_DIM="$2"; shift 2 ;; + --lr-cfc) LR_CFC="$2"; shift 2 ;; + --lr-mamba2) LR_MAMBA2="$2"; shift 2 ;; + --n-train-seqs) N_TRAIN_SEQS="$2"; shift 2 ;; + --n-val-seqs) N_VAL_SEQS="$2"; shift 2 ;; + --seed) SEED="$2"; shift 2 ;; + --watch) WATCH=true; shift ;; + -h|--help) usage; exit 0 ;; + *) echo "Unknown option: $1"; usage; exit 1 ;; + esac +done + +case "$GPU_POOL" in + ci-training-l40s) + SM=89 ;; + ci-training-h100*|ci-training-h100x2|ci-training-h100-sxm) + SM=90 ;; + *) + echo "Unknown gpu-pool: $GPU_POOL" + exit 1 ;; +esac + +echo "Submitting alpha-perception workflow..." +echo " branch: $BRANCH" +echo " sha: $SHA" +echo " gpu-pool: $GPU_POOL (sm_$SM)" +echo " epochs: $EPOCHS" +echo " seq-len: $SEQ_LEN" +echo " mamba2-state-dim: $MAMBA2_STATE_DIM" +echo " lr-cfc: $LR_CFC" +echo " lr-mamba2: $LR_MAMBA2" +echo " n-train-seqs: $N_TRAIN_SEQS" +echo " n-val-seqs: $N_VAL_SEQS" +echo " seed: $SEED" + +WATCH_FLAG="" +if [[ "$WATCH" == "true" ]]; then + WATCH_FLAG="--watch" +fi + +argo submit -n foxhunt --from=wftmpl/alpha-perception \ + -p commit-sha="$SHA" \ + -p git-branch="$BRANCH" \ + -p cuda-compute-cap="$SM" \ + -p gpu-pool="$GPU_POOL" \ + -p epochs="$EPOCHS" \ + -p seq-len="$SEQ_LEN" \ + -p mamba2-state-dim="$MAMBA2_STATE_DIM" \ + -p lr-cfc="$LR_CFC" \ + -p lr-mamba2="$LR_MAMBA2" \ + -p n-train-seqs="$N_TRAIN_SEQS" \ + -p n-val-seqs="$N_VAL_SEQS" \ + -p seed="$SEED" \ + $WATCH_FLAG