infra(argo): alpha-perception workflow + submission script
Argo WorkflowTemplate at infra/k8s/argo/alpha-perception-template.yaml
runs the stacked Mamba2 -> CfC -> heads PerceptionTrainer on a single
L40S in fr-par-2. Two-stage DAG:
ensure-binary (ci-compile-cpu pool, sccache-backed cargo build of
alpha_train example, SHA-keyed binary cache under
/data/bin/$SHORT_SHA/)
train (ci-training-l40s pool, runs the cached binary against
/data/futures-baseline/mbp10 with predecoded sidecar
cache at /feature-cache/predecoded, writes
alpha_train_summary.json to
/feature-cache/alpha-perception-runs/$SHA/)
Defaults mirror the validated synthetic-overfit smoke config:
epochs=5, seq_len=32, mamba2_state_dim=16, lr_cfc=3e-3,
lr_mamba2=1e-3, n_train_seqs=8000, n_val_seqs=1000, seed=0x4242
Submission script scripts/argo-alpha-perception.sh wraps argo submit
with the standard L40S/H100 cuda-compute-cap mapping. --watch
follows logs.
Workflow nodeSelector pinned to fr-par-2 (consistent with the cluster
topology constraint). ttlStrategy 1h after completion;
activeDeadlineSeconds 4h cap (well above expected ~30-90 min wall).
This is the cluster entrypoint for the stacked perception design.
Once it lands a summary on MinIO, the gate runner (alpha_gate, Task
17) can consume it.
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
290
infra/k8s/argo/alpha-perception-template.yaml
Normal file
290
infra/k8s/argo/alpha-perception-template.yaml
Normal file
@@ -0,0 +1,290 @@
|
||||
# Alpha perception trainer workflow.
|
||||
#
|
||||
# Stacked Mamba2 -> CfC -> heads supervised pretrain on MBP-10 from the
|
||||
# training-data PVC. Emits `alpha_train_summary.json` to MinIO-backed
|
||||
# feature-cache PVC with per-horizon val AUC for downstream gate
|
||||
# consumption. Runs on a single L40S; ~30-90 min wall depending on
|
||||
# n-train-seqs.
|
||||
#
|
||||
# DAG:
|
||||
# ensure-binary ──> train
|
||||
#
|
||||
# Usage:
|
||||
# argo submit -n foxhunt --from=wftmpl/alpha-perception \
|
||||
# -p commit-sha=HEAD -p git-branch=ml-alpha-phase-a
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: WorkflowTemplate
|
||||
metadata:
|
||||
name: alpha-perception
|
||||
namespace: foxhunt
|
||||
labels:
|
||||
app.kubernetes.io/name: alpha-perception
|
||||
app.kubernetes.io/part-of: foxhunt
|
||||
app.kubernetes.io/component: train
|
||||
spec:
|
||||
entrypoint: pipeline
|
||||
serviceAccountName: argo-workflow
|
||||
archiveLogs: true
|
||||
podMetadata:
|
||||
labels:
|
||||
app.kubernetes.io/part-of: foxhunt
|
||||
app.kubernetes.io/component: train
|
||||
securityContext:
|
||||
fsGroup: 0
|
||||
ttlStrategy:
|
||||
secondsAfterCompletion: 3600
|
||||
activeDeadlineSeconds: 14400
|
||||
|
||||
arguments:
|
||||
parameters:
|
||||
- name: commit-sha
|
||||
value: HEAD
|
||||
- name: git-branch
|
||||
value: ml-alpha-phase-a
|
||||
- name: cuda-compute-cap
|
||||
value: "89"
|
||||
- name: gpu-pool
|
||||
value: ci-training-l40s
|
||||
# Trainer hyperparameters (mirror the validated smoke config).
|
||||
- name: epochs
|
||||
value: "5"
|
||||
- name: seq-len
|
||||
value: "32"
|
||||
- name: mamba2-state-dim
|
||||
value: "16"
|
||||
- name: lr-cfc
|
||||
value: "3e-3"
|
||||
- name: lr-mamba2
|
||||
value: "1e-3"
|
||||
- name: n-train-seqs
|
||||
value: "8000"
|
||||
- name: n-val-seqs
|
||||
value: "1000"
|
||||
- name: seed
|
||||
value: "0x4242"
|
||||
|
||||
volumes:
|
||||
- name: git-ssh-key
|
||||
secret:
|
||||
secretName: argo-git-ssh-key
|
||||
defaultMode: 256
|
||||
- name: training-data
|
||||
persistentVolumeClaim:
|
||||
claimName: training-data-pvc
|
||||
- name: cargo-target-cuda
|
||||
persistentVolumeClaim:
|
||||
claimName: cargo-target-cuda
|
||||
- name: feature-cache
|
||||
persistentVolumeClaim:
|
||||
claimName: feature-cache-pvc
|
||||
|
||||
templates:
|
||||
# ── DAG ──
|
||||
- name: pipeline
|
||||
dag:
|
||||
tasks:
|
||||
- name: ensure-binary
|
||||
template: ensure-binary
|
||||
- name: train
|
||||
template: train
|
||||
dependencies: [ensure-binary]
|
||||
arguments:
|
||||
parameters:
|
||||
- name: sha
|
||||
value: "{{tasks.ensure-binary.outputs.parameters.sha}}"
|
||||
|
||||
# ── ensure-binary: compile `alpha_train` example via sccache ──
|
||||
- name: ensure-binary
|
||||
outputs:
|
||||
parameters:
|
||||
- name: sha
|
||||
valueFrom:
|
||||
path: /tmp/sha
|
||||
nodeSelector:
|
||||
k8s.scaleway.com/pool-name: ci-compile-cpu
|
||||
topology.kubernetes.io/zone: fr-par-2
|
||||
tolerations:
|
||||
- key: node.cilium.io/agent-not-ready
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
container:
|
||||
image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/ci-builder:latest
|
||||
imagePullPolicy: Always
|
||||
command: ["/bin/bash", "-c"]
|
||||
env:
|
||||
- name: SQLX_OFFLINE
|
||||
value: "true"
|
||||
- name: CARGO_TERM_COLOR
|
||||
value: always
|
||||
- name: CARGO_TARGET_DIR
|
||||
value: /cargo-target
|
||||
- name: CARGO_HOME
|
||||
value: /cargo-target/cargo-home
|
||||
- name: RUSTC_WRAPPER
|
||||
value: sccache
|
||||
- name: SCCACHE_DIR
|
||||
value: /cargo-target/sccache
|
||||
- name: SCCACHE_CACHE_SIZE
|
||||
value: "40G"
|
||||
- name: CARGO_INCREMENTAL
|
||||
value: "0"
|
||||
- name: CUDA_COMPUTE_CAP
|
||||
value: "{{workflow.parameters.cuda-compute-cap}}"
|
||||
resources:
|
||||
requests:
|
||||
cpu: "14"
|
||||
memory: 32Gi
|
||||
limits:
|
||||
cpu: "30"
|
||||
memory: 64Gi
|
||||
volumeMounts:
|
||||
- name: git-ssh-key
|
||||
mountPath: /etc/git-ssh
|
||||
readOnly: true
|
||||
- name: cargo-target-cuda
|
||||
mountPath: /cargo-target
|
||||
- name: training-data
|
||||
mountPath: /data
|
||||
args:
|
||||
- |
|
||||
set -e
|
||||
SHA="{{workflow.parameters.commit-sha}}"
|
||||
BRANCH="{{workflow.parameters.git-branch}}"
|
||||
mkdir -p ~/.ssh
|
||||
cp /etc/git-ssh/ssh-privatekey ~/.ssh/id_ed25519
|
||||
chmod 600 ~/.ssh/id_ed25519
|
||||
printf 'Host *\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n' > ~/.ssh/config
|
||||
chmod 600 ~/.ssh/config
|
||||
REPO="ssh://git@gitlab-gitlab-shell.foxhunt.svc.cluster.local:2222/root/foxhunt.git"
|
||||
BUILD="/cargo-target/src"
|
||||
git config --global --add safe.directory "$BUILD"
|
||||
|
||||
if [ "$SHA" = "HEAD" ]; then
|
||||
if [ -d "$BUILD/.git" ]; then
|
||||
cd "$BUILD"; git fetch origin; SHA=$(git rev-parse "origin/$BRANCH"); cd /
|
||||
else
|
||||
git clone --filter=blob:none "$REPO" "$BUILD"; cd "$BUILD"
|
||||
git checkout "origin/$BRANCH"; SHA=$(git rev-parse HEAD); cd /
|
||||
fi
|
||||
fi
|
||||
SHORT_SHA=$(echo "$SHA" | cut -c1-9)
|
||||
echo "Resolved SHA: $SHA (short: $SHORT_SHA)"
|
||||
|
||||
BIN_DIR="/data/bin/$SHORT_SHA"
|
||||
if [ -x "$BIN_DIR/alpha_train" ]; then
|
||||
echo "=== Cache HIT: $BIN_DIR/alpha_train ==="
|
||||
ls -lh "$BIN_DIR/"
|
||||
echo "$SHORT_SHA" > /tmp/sha
|
||||
exit 0
|
||||
fi
|
||||
echo "=== Cache MISS: compiling alpha_train for $SHORT_SHA ==="
|
||||
|
||||
if [ -d "$BUILD/.git" ]; then
|
||||
cd "$BUILD"; git fetch origin
|
||||
CURRENT=$(git rev-parse HEAD 2>/dev/null || echo "none")
|
||||
if [ "$CURRENT" != "$SHA" ]; then
|
||||
git checkout --force "$SHA"; git clean -fd
|
||||
fi
|
||||
else
|
||||
git clone --filter=blob:none "$REPO" "$BUILD"; cd "$BUILD"; git checkout "$SHA"
|
||||
fi
|
||||
|
||||
export PATH="${CARGO_HOME}/bin:${PATH}"
|
||||
echo "Building alpha_train (ml-alpha)..."
|
||||
cargo build --release -p ml-alpha --example alpha_train
|
||||
|
||||
mkdir -p "$BIN_DIR"
|
||||
cp "$CARGO_TARGET_DIR/release/examples/alpha_train" "$BIN_DIR/"
|
||||
strip "$BIN_DIR/alpha_train"
|
||||
ls -lh "$BIN_DIR/"
|
||||
|
||||
# Prune old cache (keep last 5)
|
||||
cd /data/bin
|
||||
ls -1t | tail -n +6 | while read -r old; do
|
||||
echo "Pruning old cache: $old"; rm -rf "$old"
|
||||
done
|
||||
echo "$SHORT_SHA" > /tmp/sha
|
||||
|
||||
# ── train: run alpha_train on L40S ──
|
||||
- name: train
|
||||
inputs:
|
||||
parameters:
|
||||
- name: sha
|
||||
nodeSelector:
|
||||
k8s.scaleway.com/pool-name: "{{workflow.parameters.gpu-pool}}"
|
||||
topology.kubernetes.io/zone: fr-par-2
|
||||
tolerations:
|
||||
- key: nvidia.com/gpu
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
- key: node.cilium.io/agent-not-ready
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
container:
|
||||
image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/training-runtime:latest
|
||||
imagePullPolicy: Always
|
||||
command: ["/bin/bash", "-c"]
|
||||
env:
|
||||
- name: RUST_LOG
|
||||
value: info
|
||||
- name: SQLX_OFFLINE
|
||||
value: "true"
|
||||
- name: CUDA_COMPUTE_CAP
|
||||
value: "{{workflow.parameters.cuda-compute-cap}}"
|
||||
resources:
|
||||
requests:
|
||||
cpu: "8"
|
||||
memory: 32Gi
|
||||
nvidia.com/gpu: "1"
|
||||
limits:
|
||||
cpu: "16"
|
||||
memory: 80Gi
|
||||
nvidia.com/gpu: "1"
|
||||
volumeMounts:
|
||||
- name: training-data
|
||||
mountPath: /data
|
||||
- name: feature-cache
|
||||
mountPath: /feature-cache
|
||||
args:
|
||||
- |
|
||||
set -e
|
||||
SHA="{{inputs.parameters.sha}}"
|
||||
BIN="/data/bin/$SHA/alpha_train"
|
||||
MBP10_DIR="/data/futures-baseline/mbp10"
|
||||
PREDECODED_DIR="/feature-cache/predecoded"
|
||||
OUT_DIR="/feature-cache/alpha-perception-runs/$SHA"
|
||||
mkdir -p "$OUT_DIR" "$PREDECODED_DIR"
|
||||
|
||||
if [ ! -x "$BIN" ]; then
|
||||
echo "ERROR: binary not found at $BIN"
|
||||
ls -lh /data/bin/ || true
|
||||
exit 1
|
||||
fi
|
||||
if [ ! -d "$MBP10_DIR" ]; then
|
||||
echo "ERROR: MBP-10 data directory not found at $MBP10_DIR"
|
||||
ls -lh /data/ || true
|
||||
exit 1
|
||||
fi
|
||||
echo "Running alpha_train (stacked Mamba2 -> CfC -> heads)"
|
||||
echo " binary: $BIN"
|
||||
echo " mbp10: $MBP10_DIR ($(find "$MBP10_DIR" -name '*.dbn.zst' | wc -l) files)"
|
||||
echo " predecoded: $PREDECODED_DIR"
|
||||
echo " out: $OUT_DIR"
|
||||
|
||||
"$BIN" \
|
||||
--mbp10-data-dir "$MBP10_DIR" \
|
||||
--predecoded-dir "$PREDECODED_DIR" \
|
||||
--out "$OUT_DIR" \
|
||||
--epochs {{workflow.parameters.epochs}} \
|
||||
--seq-len {{workflow.parameters.seq-len}} \
|
||||
--mamba2-state-dim {{workflow.parameters.mamba2-state-dim}} \
|
||||
--lr-cfc {{workflow.parameters.lr-cfc}} \
|
||||
--lr-mamba2 {{workflow.parameters.lr-mamba2}} \
|
||||
--n-train-seqs {{workflow.parameters.n-train-seqs}} \
|
||||
--n-val-seqs {{workflow.parameters.n-val-seqs}} \
|
||||
--seed {{workflow.parameters.seed}}
|
||||
|
||||
echo "=== Training complete ==="
|
||||
ls -lh "$OUT_DIR/"
|
||||
echo "--- alpha_train_summary.json ---"
|
||||
cat "$OUT_DIR/alpha_train_summary.json"
|
||||
110
scripts/argo-alpha-perception.sh
Executable file
110
scripts/argo-alpha-perception.sh
Executable file
@@ -0,0 +1,110 @@
|
||||
#!/usr/bin/env bash
|
||||
# Submit the alpha-perception workflow.
|
||||
#
|
||||
# Trains the stacked Mamba2 -> CfC -> heads perception model on MBP-10
|
||||
# from the training-data PVC. Emits alpha_train_summary.json with
|
||||
# per-horizon validation AUC to the feature-cache PVC.
|
||||
#
|
||||
# Defaults match the validated synthetic-overfit smoke config; cluster
|
||||
# runs can override for sweep work.
|
||||
#
|
||||
# Usage:
|
||||
# ./scripts/argo-alpha-perception.sh # current HEAD on ml-alpha-phase-a
|
||||
# ./scripts/argo-alpha-perception.sh --branch main
|
||||
# ./scripts/argo-alpha-perception.sh --epochs 1 --n-train-seqs 1000 # quick smoke
|
||||
# ./scripts/argo-alpha-perception.sh --watch
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SHA=HEAD
|
||||
BRANCH=$(git symbolic-ref --short HEAD 2>/dev/null || echo "ml-alpha-phase-a")
|
||||
GPU_POOL=ci-training-l40s
|
||||
EPOCHS=5
|
||||
SEQ_LEN=32
|
||||
MAMBA2_STATE_DIM=16
|
||||
LR_CFC=3e-3
|
||||
LR_MAMBA2=1e-3
|
||||
N_TRAIN_SEQS=8000
|
||||
N_VAL_SEQS=1000
|
||||
SEED=0x4242
|
||||
WATCH=false
|
||||
|
||||
usage() {
|
||||
cat <<EOF
|
||||
Usage: $0 [OPTIONS]
|
||||
--sha <commit> Git SHA (default: HEAD on the current branch)
|
||||
--branch <branch> Git branch (default: $BRANCH)
|
||||
--gpu-pool <pool> GPU pool (default: $GPU_POOL)
|
||||
--epochs <n> Training epochs (default: $EPOCHS)
|
||||
--seq-len <n> Snapshots per sequence window (default: $SEQ_LEN)
|
||||
--mamba2-state-dim <n> Mamba2 SSM state dim (default: $MAMBA2_STATE_DIM)
|
||||
--lr-cfc <f> CfC learning rate (default: $LR_CFC)
|
||||
--lr-mamba2 <f> Mamba2 learning rate (default: $LR_MAMBA2)
|
||||
--n-train-seqs <n> Train sequences per epoch (default: $N_TRAIN_SEQS)
|
||||
--n-val-seqs <n> Val sequences per epoch (default: $N_VAL_SEQS)
|
||||
--seed <n> Random seed (default: $SEED)
|
||||
--watch Follow logs via argo watch
|
||||
EOF
|
||||
}
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--sha) SHA="$2"; shift 2 ;;
|
||||
--branch) BRANCH="$2"; shift 2 ;;
|
||||
--gpu-pool) GPU_POOL="$2"; shift 2 ;;
|
||||
--epochs) EPOCHS="$2"; shift 2 ;;
|
||||
--seq-len) SEQ_LEN="$2"; shift 2 ;;
|
||||
--mamba2-state-dim) MAMBA2_STATE_DIM="$2"; shift 2 ;;
|
||||
--lr-cfc) LR_CFC="$2"; shift 2 ;;
|
||||
--lr-mamba2) LR_MAMBA2="$2"; shift 2 ;;
|
||||
--n-train-seqs) N_TRAIN_SEQS="$2"; shift 2 ;;
|
||||
--n-val-seqs) N_VAL_SEQS="$2"; shift 2 ;;
|
||||
--seed) SEED="$2"; shift 2 ;;
|
||||
--watch) WATCH=true; shift ;;
|
||||
-h|--help) usage; exit 0 ;;
|
||||
*) echo "Unknown option: $1"; usage; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
case "$GPU_POOL" in
|
||||
ci-training-l40s)
|
||||
SM=89 ;;
|
||||
ci-training-h100*|ci-training-h100x2|ci-training-h100-sxm)
|
||||
SM=90 ;;
|
||||
*)
|
||||
echo "Unknown gpu-pool: $GPU_POOL"
|
||||
exit 1 ;;
|
||||
esac
|
||||
|
||||
echo "Submitting alpha-perception workflow..."
|
||||
echo " branch: $BRANCH"
|
||||
echo " sha: $SHA"
|
||||
echo " gpu-pool: $GPU_POOL (sm_$SM)"
|
||||
echo " epochs: $EPOCHS"
|
||||
echo " seq-len: $SEQ_LEN"
|
||||
echo " mamba2-state-dim: $MAMBA2_STATE_DIM"
|
||||
echo " lr-cfc: $LR_CFC"
|
||||
echo " lr-mamba2: $LR_MAMBA2"
|
||||
echo " n-train-seqs: $N_TRAIN_SEQS"
|
||||
echo " n-val-seqs: $N_VAL_SEQS"
|
||||
echo " seed: $SEED"
|
||||
|
||||
WATCH_FLAG=""
|
||||
if [[ "$WATCH" == "true" ]]; then
|
||||
WATCH_FLAG="--watch"
|
||||
fi
|
||||
|
||||
argo submit -n foxhunt --from=wftmpl/alpha-perception \
|
||||
-p commit-sha="$SHA" \
|
||||
-p git-branch="$BRANCH" \
|
||||
-p cuda-compute-cap="$SM" \
|
||||
-p gpu-pool="$GPU_POOL" \
|
||||
-p epochs="$EPOCHS" \
|
||||
-p seq-len="$SEQ_LEN" \
|
||||
-p mamba2-state-dim="$MAMBA2_STATE_DIM" \
|
||||
-p lr-cfc="$LR_CFC" \
|
||||
-p lr-mamba2="$LR_MAMBA2" \
|
||||
-p n-train-seqs="$N_TRAIN_SEQS" \
|
||||
-p n-val-seqs="$N_VAL_SEQS" \
|
||||
-p seed="$SEED" \
|
||||
$WATCH_FLAG
|
||||
Reference in New Issue
Block a user