25c78c5913
Matches the alpha-perception-template (which is the known-good template in this namespace). Without the fix, ensure-binary + run-cell + aggregate pods all stayed Init for ~20m with 'MountVolume.SetUp failed for volume "git-ssh-key" : secret "git-ssh-key" not found' before the workflow timed out. The actual secret in the foxhunt namespace is named 'argo-git-ssh-key' (see kubectl get secrets -n foxhunt). Same template misconfig as the training-data PVC name.
349 lines
12 KiB
YAML
349 lines
12 KiB
YAML
# Real-LOB backtest sweep workflow.
|
|
#
|
|
# Fans out a parameter grid across N parallel GPU pods, each running
|
|
# `fxt-backtest run` against the same MBP-10 data + checkpoint with one
|
|
# cell's worth of overrides. A final CPU pod runs `fxt-backtest aggregate`
|
|
# to produce aggregate.parquet + pareto_frontier.json at the sweep root.
|
|
#
|
|
# The `# __SWEEP_CELLS__` marker on the dag.tasks line is replaced by
|
|
# scripts/argo-lob-sweep.sh with N generated WorkflowTask stanzas before
|
|
# submission. Same convention as train-multi-seed-template.yaml's
|
|
# # __MATRIX_TASKS__ marker.
|
|
#
|
|
# DAG:
|
|
# ensure-binary ──> [N parallel run-cell-<i> tasks on ci-training-l40s,
|
|
# each with its own outputDir on the shared PVC]
|
|
# │
|
|
# └──> aggregate (CPU node, runs fxt-backtest aggregate
|
|
# against the sweep root)
|
|
#
|
|
# Per `feedback_default_to_l40s_pool.md` (2026-05-09): defaults to
|
|
# ci-training-l40s + sm_89. Override via --gpu-pool ci-training-h100
|
|
# for sm_90 / 80 GB.
|
|
#
|
|
# Usage:
|
|
# ./scripts/argo-lob-sweep.sh --grid config/ml/sweep_decision_stride_example.yaml
|
|
# ./scripts/argo-lob-sweep.sh --grid <path> --sha abc1234 --gpu-pool ci-training-h100
|
|
# ./scripts/argo-lob-sweep.sh --grid <path> --watch
|
|
# ./scripts/argo-lob-sweep.sh --grid <path> --dry-run > /tmp/wf.yaml
|
|
|
|
apiVersion: argoproj.io/v1alpha1
|
|
kind: WorkflowTemplate
|
|
metadata:
|
|
name: lob-backtest-sweep
|
|
namespace: foxhunt
|
|
labels:
|
|
app.kubernetes.io/name: lob-backtest-sweep
|
|
app.kubernetes.io/part-of: foxhunt
|
|
app.kubernetes.io/component: backtest
|
|
spec:
|
|
entrypoint: sweep-matrix
|
|
onExit: notify-result
|
|
serviceAccountName: argo-workflow
|
|
archiveLogs: true
|
|
podMetadata:
|
|
labels:
|
|
app.kubernetes.io/part-of: foxhunt
|
|
app.kubernetes.io/component: backtest
|
|
securityContext:
|
|
fsGroup: 0
|
|
ttlStrategy:
|
|
secondsAfterCompletion: 3600
|
|
# LOB sweep cells are much faster than training (~minutes vs hours);
|
|
# 2h walltime is generous even for N=128 cells.
|
|
activeDeadlineSeconds: 7200
|
|
|
|
arguments:
|
|
parameters:
|
|
- name: commit-sha
|
|
value: HEAD
|
|
- name: git-branch
|
|
value: main
|
|
- name: cuda-compute-cap
|
|
value: "89"
|
|
- name: gpu-pool
|
|
value: ci-training-l40s
|
|
- name: data-root
|
|
value: /mnt/training-data/futures-baseline/ES.FUT
|
|
- name: predecoded-dir
|
|
value: /mnt/training-data/futures-baseline/ES.FUT
|
|
- name: checkpoint
|
|
value: "" # empty = trunk runs with --seed (random init);
|
|
# set to a path on the data PVC for trained weights
|
|
- name: sweep-root
|
|
value: /mnt/training-data/sweeps/lob-backtest
|
|
- name: sweep-tag
|
|
value: default # subdirectory: <sweep-root>/<sweep-tag>/
|
|
# Operational knobs. argo-lob-sweep.sh sets these per-grid.
|
|
- name: n-parallel
|
|
value: "1"
|
|
- name: latency-ns
|
|
value: "100000000"
|
|
- name: target-annual-vol-units
|
|
value: "50.0"
|
|
- name: annualisation-factor
|
|
value: "825.0"
|
|
- name: max-lots
|
|
value: "5"
|
|
- name: max-events
|
|
value: "0"
|
|
|
|
volumes:
|
|
- name: training-data
|
|
persistentVolumeClaim:
|
|
claimName: training-data-pvc
|
|
- name: git-ssh-key
|
|
secret:
|
|
secretName: argo-git-ssh-key
|
|
defaultMode: 0400
|
|
- name: cargo-target-cuda
|
|
persistentVolumeClaim:
|
|
claimName: cargo-target-cuda
|
|
|
|
templates:
|
|
# ── DAG entrypoint ──────────────────────────────────────────────
|
|
- name: sweep-matrix
|
|
dag:
|
|
tasks:
|
|
- name: ensure-binary
|
|
template: ensure-binary
|
|
# __SWEEP_CELLS__ — replaced by argo-lob-sweep.sh
|
|
- name: aggregate
|
|
template: aggregate
|
|
dependencies: [ensure-binary] # plus all run-cell-* via shell-injected list
|
|
arguments:
|
|
parameters:
|
|
- name: sha
|
|
value: "{{tasks.ensure-binary.outputs.parameters.sha}}"
|
|
|
|
# ── ensure-binary: cache-or-compile fxt-backtest by commit SHA ─
|
|
# Same shape as train-multi-seed-template.yaml's ensure-binary but
|
|
# for the fxt-backtest binary only.
|
|
- name: ensure-binary
|
|
outputs:
|
|
parameters:
|
|
- name: sha
|
|
valueFrom:
|
|
path: /tmp/sha
|
|
nodeSelector:
|
|
k8s.scaleway.com/pool-name: ci-compile-cpu
|
|
topology.kubernetes.io/zone: fr-par-2
|
|
tolerations:
|
|
- key: node.cilium.io/agent-not-ready
|
|
operator: Exists
|
|
effect: NoSchedule
|
|
container:
|
|
image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/ci-builder:latest
|
|
imagePullPolicy: Always
|
|
command: ["/bin/bash", "-c"]
|
|
env:
|
|
- name: SQLX_OFFLINE
|
|
value: "true"
|
|
- name: CARGO_TARGET_DIR
|
|
value: /cargo-target
|
|
- name: CARGO_HOME
|
|
value: /cargo-target/cargo-home
|
|
- name: RUSTC_WRAPPER
|
|
value: sccache
|
|
- name: SCCACHE_DIR
|
|
value: /cargo-target/sccache
|
|
- name: SCCACHE_CACHE_SIZE
|
|
value: "40G"
|
|
- name: CARGO_INCREMENTAL
|
|
value: "0"
|
|
- name: FOXHUNT_CUDA_ARCH
|
|
value: "sm_{{workflow.parameters.cuda-compute-cap}}"
|
|
- name: GITLAB_PAT
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: gitlab-pat
|
|
key: token
|
|
resources:
|
|
requests:
|
|
cpu: "14"
|
|
memory: 32Gi
|
|
limits:
|
|
cpu: "30"
|
|
memory: 64Gi
|
|
volumeMounts:
|
|
- name: git-ssh-key
|
|
mountPath: /etc/git-ssh
|
|
readOnly: true
|
|
- name: cargo-target-cuda
|
|
mountPath: /cargo-target
|
|
- name: training-data
|
|
mountPath: /mnt/training-data
|
|
args:
|
|
- |
|
|
set -e
|
|
SHA="{{workflow.parameters.commit-sha}}"
|
|
BRANCH="{{workflow.parameters.git-branch}}"
|
|
|
|
mkdir -p ~/.ssh
|
|
cp /etc/git-ssh/ssh-privatekey ~/.ssh/id_ed25519
|
|
chmod 600 ~/.ssh/id_ed25519
|
|
printf 'Host *\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n' > ~/.ssh/config
|
|
|
|
REPO="ssh://git@gitlab-gitlab-shell.foxhunt.svc.cluster.local:2222/root/foxhunt.git"
|
|
BUILD="/cargo-target/src"
|
|
git config --global --add safe.directory "$BUILD"
|
|
|
|
if [ "$SHA" = "HEAD" ]; then
|
|
if [ -d "$BUILD/.git" ]; then
|
|
cd "$BUILD"; git fetch origin
|
|
SHA=$(git rev-parse "origin/$BRANCH"); cd /
|
|
else
|
|
git clone --filter=blob:none "$REPO" "$BUILD"
|
|
cd "$BUILD"; git checkout "origin/$BRANCH"
|
|
SHA=$(git rev-parse HEAD); cd /
|
|
fi
|
|
fi
|
|
SHORT_SHA=$(echo "$SHA" | cut -c1-9)
|
|
echo "Resolved SHA: $SHA (short: $SHORT_SHA)"
|
|
|
|
BIN_DIR="/mnt/training-data/bin/$SHORT_SHA"
|
|
mkdir -p "$BIN_DIR"
|
|
if [ -x "$BIN_DIR/fxt-backtest" ]; then
|
|
echo "=== Cache HIT: fxt-backtest present in $BIN_DIR ==="
|
|
ls -lh "$BIN_DIR/fxt-backtest"
|
|
echo "$SHORT_SHA" > /tmp/sha
|
|
exit 0
|
|
fi
|
|
|
|
echo "=== Cache MISS: compiling fxt-backtest for $SHORT_SHA ==="
|
|
if [ -d "$BUILD/.git" ]; then
|
|
cd "$BUILD"; git fetch origin; git checkout "$SHA"
|
|
else
|
|
git clone --filter=blob:none "$REPO" "$BUILD"
|
|
cd "$BUILD"; git checkout "$SHA"
|
|
fi
|
|
|
|
cargo build -p fxt-backtest --release
|
|
cp "$BUILD/target/release/fxt-backtest" "$BIN_DIR/fxt-backtest"
|
|
echo "$SHORT_SHA" > /tmp/sha
|
|
ls -lh "$BIN_DIR/fxt-backtest"
|
|
|
|
# ── run-cell: one sweep cell against a GPU pod ─────────────────
|
|
# Inputs: cell name + every Run arg that varies across the grid.
|
|
# The shell-rendered DAG tasks supply these per-cell.
|
|
- name: run-cell
|
|
inputs:
|
|
parameters:
|
|
- name: sha
|
|
- name: cell-name
|
|
- name: decision-stride
|
|
value: "4"
|
|
- name: latency-ns
|
|
value: "{{workflow.parameters.latency-ns}}"
|
|
- name: target-annual-vol-units
|
|
value: "{{workflow.parameters.target-annual-vol-units}}"
|
|
- name: annualisation-factor
|
|
value: "{{workflow.parameters.annualisation-factor}}"
|
|
- name: max-lots
|
|
value: "{{workflow.parameters.max-lots}}"
|
|
- name: max-events
|
|
value: "{{workflow.parameters.max-events}}"
|
|
- name: seed
|
|
value: "0xC0FFEE"
|
|
nodeSelector:
|
|
k8s.scaleway.com/pool-name: "{{workflow.parameters.gpu-pool}}"
|
|
topology.kubernetes.io/zone: fr-par-2
|
|
tolerations:
|
|
- key: nvidia.com/gpu
|
|
operator: Exists
|
|
effect: NoSchedule
|
|
container:
|
|
image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/training-runtime:latest
|
|
imagePullPolicy: Always
|
|
command: ["/bin/bash", "-c"]
|
|
resources:
|
|
requests:
|
|
cpu: "4"
|
|
memory: 16Gi
|
|
nvidia.com/gpu: "1"
|
|
limits:
|
|
cpu: "8"
|
|
memory: 32Gi
|
|
nvidia.com/gpu: "1"
|
|
volumeMounts:
|
|
- name: training-data
|
|
mountPath: /mnt/training-data
|
|
args:
|
|
- |
|
|
set -e
|
|
SHA="{{inputs.parameters.sha}}"
|
|
CELL="{{inputs.parameters.cell-name}}"
|
|
BIN="/mnt/training-data/bin/$SHA/fxt-backtest"
|
|
CKPT="{{workflow.parameters.checkpoint}}"
|
|
OUT="{{workflow.parameters.sweep-root}}/{{workflow.parameters.sweep-tag}}/$CELL"
|
|
mkdir -p "$OUT"
|
|
echo "=== sweep cell $CELL on $(hostname) → $OUT ==="
|
|
|
|
CKPT_FLAG=""
|
|
if [ -n "$CKPT" ]; then
|
|
CKPT_FLAG="--checkpoint $CKPT"
|
|
fi
|
|
|
|
"$BIN" run \
|
|
--data "{{workflow.parameters.data-root}}" \
|
|
--predecoded-dir "{{workflow.parameters.predecoded-dir}}" \
|
|
--n-parallel "{{workflow.parameters.n-parallel}}" \
|
|
--decision-stride "{{inputs.parameters.decision-stride}}" \
|
|
--latency-ns "{{inputs.parameters.latency-ns}}" \
|
|
--target-annual-vol-units "{{inputs.parameters.target-annual-vol-units}}" \
|
|
--annualisation-factor "{{inputs.parameters.annualisation-factor}}" \
|
|
--max-lots "{{inputs.parameters.max-lots}}" \
|
|
--max-events "{{inputs.parameters.max-events}}" \
|
|
--seed "{{inputs.parameters.seed}}" \
|
|
$CKPT_FLAG \
|
|
--out "$OUT"
|
|
|
|
echo "=== cell $CELL done ==="
|
|
ls -lh "$OUT"
|
|
|
|
# ── aggregate: final CPU pod runs fxt-backtest aggregate ───────
|
|
- name: aggregate
|
|
inputs:
|
|
parameters:
|
|
- name: sha
|
|
nodeSelector:
|
|
k8s.scaleway.com/pool-name: ci-compile-cpu
|
|
topology.kubernetes.io/zone: fr-par-2
|
|
tolerations:
|
|
- key: node.cilium.io/agent-not-ready
|
|
operator: Exists
|
|
effect: NoSchedule
|
|
container:
|
|
image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/training-runtime:latest
|
|
imagePullPolicy: Always
|
|
command: ["/bin/bash", "-c"]
|
|
resources:
|
|
requests:
|
|
cpu: "2"
|
|
memory: 4Gi
|
|
limits:
|
|
cpu: "4"
|
|
memory: 8Gi
|
|
volumeMounts:
|
|
- name: training-data
|
|
mountPath: /mnt/training-data
|
|
args:
|
|
- |
|
|
set -e
|
|
SHA="{{inputs.parameters.sha}}"
|
|
BIN="/mnt/training-data/bin/$SHA/fxt-backtest"
|
|
SWEEP_DIR="{{workflow.parameters.sweep-root}}/{{workflow.parameters.sweep-tag}}"
|
|
echo "=== aggregate sweep at $SWEEP_DIR ==="
|
|
"$BIN" aggregate "$SWEEP_DIR"
|
|
ls -lh "$SWEEP_DIR"
|
|
|
|
# ── notify-result: exit hook (placeholder; real impl emits to Slack/MinIO) ─
|
|
- name: notify-result
|
|
container:
|
|
image: alpine:3.20
|
|
command: ["sh", "-c"]
|
|
args:
|
|
- |
|
|
echo "lob-backtest-sweep workflow {{workflow.name}} finished with status {{workflow.status}}"
|
|
echo "results under {{workflow.parameters.sweep-root}}/{{workflow.parameters.sweep-tag}}"
|