feat(alpha): decision-stride + cluster 9-fold CV workflow
Two complementary additions to validate the minute-horizon alpha hypothesis at IBKR-realistic costs: 1. `alpha_baseline --decision-stride N`: emits a new action every N steps; between decisions force action=0 (wait) so an open position is held rather than re-decided per bar. Cuts per-bar trade counts ~stride× and removes the coin-flip overtrading. Local 2Q sweep showed stride=200 + scaled training (8K episodes × 25 envs × H=1200) flipped Sharpe at ¼-tick from -4.29 (per-bar, 3-fold mean) to +1.78, with std collapsing from ±8.8 to ±1.15. Break-even cost moved from <¼-tick to ~1-tick — for the first time positive at IBKR-realistic passive-execution frictions. 2. `alpha_train_stacker --max-rows N`: optional cap on bars consumed from the fxcache. Used during local 2Q smoke (--max-rows 4M against the 17.8M-row 9Q fxcache) to fit Mamba2 training on a 4 GB consumer GPU; on the cluster (--no-cap) it sees all 9Q. 3. New Argo workflow `alpha-cv`: standalone template that compiles alpha_train_stacker + alpha_baseline + alpha_fill_coeffs.json, trains the stacker on the 9Q fxcache, then runs 9 sequential walk-forward folds of alpha_baseline on disjoint 1.9M-bar windows (one per quarter). Launcher script `scripts/argo-alpha-cv.sh` mirrors argo-train.sh conventions. The local 2Q test that motivated this commit is summarised inline in the alpha-cv template comments; the verdict was "framing was the bug — once decision cadence matches the multi-minute alpha horizon, the strategy is positive at IBKR commission". Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -82,6 +82,12 @@ struct Cli {
|
||||
/// `< seq_len - 1` are written as 0 (no history). Empty / unset → no
|
||||
/// cache dumped.
|
||||
#[arg(long)] alpha_cache_out: Option<String>,
|
||||
|
||||
/// Optional cap on bars consumed from the fxcache. When set, the trainer
|
||||
/// processes only the first `max_rows` rows; useful for fitting Mamba2
|
||||
/// training on a 4 GB consumer GPU without rebuilding a smaller fxcache.
|
||||
/// Downstream `alpha_baseline --max-snapshots` must be ≤ this value.
|
||||
#[arg(long)] max_rows: Option<usize>,
|
||||
}
|
||||
|
||||
fn main() -> Result<()> {
|
||||
@@ -96,8 +102,16 @@ fn main() -> Result<()> {
|
||||
let reader = FxCacheReader::open(&cli.fxcache_path)?;
|
||||
let alpha_dim = reader.alpha_feature_dim()
|
||||
.ok_or_else(|| anyhow::anyhow!("fxcache lacks alpha column"))?;
|
||||
let n_bars = reader.bar_count();
|
||||
println!("fxcache: {} rows, alpha_dim={}, horizon K={}", n_bars, alpha_dim, cli.horizon);
|
||||
let total_bars = reader.bar_count();
|
||||
let n_bars = cli.max_rows.map(|m| m.min(total_bars)).unwrap_or(total_bars);
|
||||
if n_bars < total_bars {
|
||||
println!(
|
||||
"fxcache: {} rows total, {} rows used (--max-rows={}), alpha_dim={}, horizon K={}",
|
||||
total_bars, n_bars, cli.max_rows.unwrap_or(0), alpha_dim, cli.horizon
|
||||
);
|
||||
} else {
|
||||
println!("fxcache: {} rows, alpha_dim={}, horizon K={}", n_bars, alpha_dim, cli.horizon);
|
||||
}
|
||||
|
||||
// Flat feature matrix (CPU-resident) and prices.
|
||||
let mut feature_matrix: Vec<f32> = Vec::with_capacity(n_bars * alpha_dim);
|
||||
|
||||
@@ -225,6 +225,15 @@ struct Cli {
|
||||
eps_end: f32,
|
||||
#[arg(long, default_value_t = 0.99)]
|
||||
gamma: f32,
|
||||
|
||||
/// Decision stride — emit a new action only every N steps; between
|
||||
/// decisions force `action=0` (wait), preserving the previously-opened
|
||||
/// position. Aligns DQN decision cadence with the multi-minute alpha
|
||||
/// horizon and stops the policy from flip-flopping on per-bar noise
|
||||
/// ("coin-flip problem"). 1 = current per-bar behaviour. Pair with
|
||||
/// γ→0.999+ so the Bellman chain reaches across the wait segments.
|
||||
#[arg(long, default_value_t = 1)]
|
||||
decision_stride: u32,
|
||||
#[arg(long, default_value_t = 0.9)]
|
||||
alpha_m: f32,
|
||||
#[arg(long, default_value_t = 0.03)]
|
||||
@@ -651,7 +660,16 @@ fn main() -> Result<()> {
|
||||
}
|
||||
}
|
||||
stream.synchronize()?;
|
||||
let actions = batched_action_pinned_train.read_all();
|
||||
// `--decision-stride N`: only emit a new action every N steps;
|
||||
// between strides force action=0 (wait) so an open position is
|
||||
// held rather than re-decided per bar. Avoids the per-bar
|
||||
// "coin-flip" overtrading. step==0 is always a decision.
|
||||
let stride_train = cli.decision_stride.max(1) as usize;
|
||||
let actions = if step % stride_train == 0 {
|
||||
batched_action_pinned_train.read_all()
|
||||
} else {
|
||||
vec![0_i32; train_n_par]
|
||||
};
|
||||
|
||||
for i in 0..train_n_par {
|
||||
if done_flags[i] { continue; }
|
||||
@@ -901,7 +919,7 @@ fn main() -> Result<()> {
|
||||
.context("reset vol_obs min slot")?;
|
||||
}
|
||||
// Lockstep step loop.
|
||||
for _step in 0..cli.horizon {
|
||||
for step in 0..cli.horizon {
|
||||
// Gather current states (CPU; <100μs for N=500).
|
||||
let mut batched_states_host = vec![0.0_f32; n_par * STATE_DIM];
|
||||
for i in 0..n_par {
|
||||
@@ -993,7 +1011,17 @@ fn main() -> Result<()> {
|
||||
}
|
||||
// ONE sync per step (instead of N).
|
||||
stream.synchronize()?;
|
||||
let actions = batched_action_pinned.read_all();
|
||||
// `--decision-stride N`: same rate-limit semantics as training.
|
||||
// Between strides force action=0 so an open passive limit
|
||||
// post / live position is held rather than re-decided. Cuts
|
||||
// per-bar trade counts ~stride× and removes the coin-flip
|
||||
// overtrading on noise.
|
||||
let stride_eval = cli.decision_stride.max(1) as usize;
|
||||
let actions = if step % stride_eval == 0 {
|
||||
batched_action_pinned.read_all()
|
||||
} else {
|
||||
vec![0_i32; n_par]
|
||||
};
|
||||
// Step all envs on CPU.
|
||||
for i in 0..n_par {
|
||||
if done_flags[i] { continue; }
|
||||
|
||||
495
infra/k8s/argo/alpha-cv-template.yaml
Normal file
495
infra/k8s/argo/alpha-cv-template.yaml
Normal file
@@ -0,0 +1,495 @@
|
||||
# Alpha walk-forward CV workflow.
|
||||
#
|
||||
# Trains the Mamba2+stacker alpha logits cache on the full 9Q fxcache,
|
||||
# then runs `alpha_baseline` 9 times across disjoint 1.9M-bar windows
|
||||
# (one per quarter). Each fold trains its own execution-policy DQN
|
||||
# from scratch on the front of the window and backtests on the back
|
||||
# (purged train/eval split via `--train-frac`).
|
||||
#
|
||||
# Sequential by design: 9 folds chain in the DAG so a single L40S node
|
||||
# stays warm across the run. With --decision-stride 200 and --horizon
|
||||
# 1200, each fold's DQN training is ~5-10 min on L40S; total wall ~60-90
|
||||
# min for stacker + 9 folds.
|
||||
#
|
||||
# DAG:
|
||||
# ensure-binary ──> ensure-fxcache ──> stacker-train ──> fold-0 ──> ... ──> fold-8
|
||||
#
|
||||
# Usage:
|
||||
# argo submit -n foxhunt --from=wftmpl/alpha-cv \
|
||||
# -p commit-sha=HEAD -p git-branch=main -p decision-stride=200
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: WorkflowTemplate
|
||||
metadata:
|
||||
name: alpha-cv
|
||||
namespace: foxhunt
|
||||
labels:
|
||||
app.kubernetes.io/name: alpha-cv
|
||||
app.kubernetes.io/part-of: foxhunt
|
||||
app.kubernetes.io/component: train
|
||||
spec:
|
||||
entrypoint: pipeline
|
||||
serviceAccountName: argo-workflow
|
||||
archiveLogs: true
|
||||
podMetadata:
|
||||
labels:
|
||||
app.kubernetes.io/part-of: foxhunt
|
||||
app.kubernetes.io/component: train
|
||||
securityContext:
|
||||
fsGroup: 0
|
||||
ttlStrategy:
|
||||
secondsAfterCompletion: 3600
|
||||
activeDeadlineSeconds: 28800
|
||||
|
||||
arguments:
|
||||
parameters:
|
||||
- name: commit-sha
|
||||
value: HEAD
|
||||
- name: git-branch
|
||||
value: main
|
||||
- name: cuda-compute-cap
|
||||
value: "89"
|
||||
- name: gpu-pool
|
||||
value: ci-training-l40s
|
||||
- name: symbol
|
||||
value: ES.FUT
|
||||
# Match the fxcache build params (cluster's 9Q fxcache was built
|
||||
# with these on 2026-05-16; see commit e7ce4395e).
|
||||
- name: imbalance-bar-threshold
|
||||
value: "20.0"
|
||||
- name: imbalance-bar-ewma-alpha
|
||||
value: "0.1"
|
||||
- name: volume-bar-size
|
||||
value: "100"
|
||||
- name: data-source
|
||||
value: "mbp10"
|
||||
# Stacker training params (mirror local 2Q smoke).
|
||||
- name: stacker-horizon
|
||||
value: "6000"
|
||||
- name: stacker-seq-len
|
||||
value: "32"
|
||||
- name: stacker-hidden-dim
|
||||
value: "64"
|
||||
- name: stacker-state-dim
|
||||
value: "16"
|
||||
- name: stacker-epochs
|
||||
value: "5"
|
||||
- name: stacker-batch-size
|
||||
value: "128"
|
||||
- name: stacker-lr
|
||||
value: "3e-3"
|
||||
- name: stacker-train-frac
|
||||
value: "0.8"
|
||||
# CV / alpha_baseline params — defaults match the validated 2Q
|
||||
# config that produced +1.78 Sharpe at quarter-tick cost.
|
||||
- name: decision-stride
|
||||
value: "200"
|
||||
- name: fold-window
|
||||
value: "1900000"
|
||||
- name: train-frac
|
||||
value: "0.6"
|
||||
- name: window-k
|
||||
value: "16"
|
||||
- name: horizon
|
||||
value: "1200"
|
||||
- name: n-train-par
|
||||
value: "25"
|
||||
- name: n-train-episodes
|
||||
value: "8000"
|
||||
|
||||
volumes:
|
||||
- name: git-ssh-key
|
||||
secret:
|
||||
secretName: argo-git-ssh-key
|
||||
defaultMode: 256
|
||||
- name: training-data
|
||||
persistentVolumeClaim:
|
||||
claimName: training-data-pvc
|
||||
- name: cargo-target-cuda
|
||||
persistentVolumeClaim:
|
||||
claimName: cargo-target-cuda
|
||||
- name: feature-cache
|
||||
persistentVolumeClaim:
|
||||
claimName: feature-cache-pvc
|
||||
|
||||
templates:
|
||||
# ── DAG ──
|
||||
- name: pipeline
|
||||
dag:
|
||||
tasks:
|
||||
- name: ensure-binary
|
||||
template: ensure-binary
|
||||
- name: stacker-train
|
||||
template: stacker-train
|
||||
dependencies: [ensure-binary]
|
||||
arguments:
|
||||
parameters:
|
||||
- name: sha
|
||||
value: "{{tasks.ensure-binary.outputs.parameters.sha}}"
|
||||
- name: fold-0
|
||||
template: alpha-cv-fold
|
||||
dependencies: [stacker-train]
|
||||
arguments:
|
||||
parameters:
|
||||
- name: sha
|
||||
value: "{{tasks.ensure-binary.outputs.parameters.sha}}"
|
||||
- name: fold-idx
|
||||
value: "0"
|
||||
- name: offset
|
||||
value: "0"
|
||||
- name: fold-1
|
||||
template: alpha-cv-fold
|
||||
dependencies: [fold-0]
|
||||
arguments:
|
||||
parameters:
|
||||
- name: sha
|
||||
value: "{{tasks.ensure-binary.outputs.parameters.sha}}"
|
||||
- name: fold-idx
|
||||
value: "1"
|
||||
- name: offset
|
||||
value: "1900000"
|
||||
- name: fold-2
|
||||
template: alpha-cv-fold
|
||||
dependencies: [fold-1]
|
||||
arguments:
|
||||
parameters:
|
||||
- name: sha
|
||||
value: "{{tasks.ensure-binary.outputs.parameters.sha}}"
|
||||
- name: fold-idx
|
||||
value: "2"
|
||||
- name: offset
|
||||
value: "3800000"
|
||||
- name: fold-3
|
||||
template: alpha-cv-fold
|
||||
dependencies: [fold-2]
|
||||
arguments:
|
||||
parameters:
|
||||
- name: sha
|
||||
value: "{{tasks.ensure-binary.outputs.parameters.sha}}"
|
||||
- name: fold-idx
|
||||
value: "3"
|
||||
- name: offset
|
||||
value: "5700000"
|
||||
- name: fold-4
|
||||
template: alpha-cv-fold
|
||||
dependencies: [fold-3]
|
||||
arguments:
|
||||
parameters:
|
||||
- name: sha
|
||||
value: "{{tasks.ensure-binary.outputs.parameters.sha}}"
|
||||
- name: fold-idx
|
||||
value: "4"
|
||||
- name: offset
|
||||
value: "7600000"
|
||||
- name: fold-5
|
||||
template: alpha-cv-fold
|
||||
dependencies: [fold-4]
|
||||
arguments:
|
||||
parameters:
|
||||
- name: sha
|
||||
value: "{{tasks.ensure-binary.outputs.parameters.sha}}"
|
||||
- name: fold-idx
|
||||
value: "5"
|
||||
- name: offset
|
||||
value: "9500000"
|
||||
- name: fold-6
|
||||
template: alpha-cv-fold
|
||||
dependencies: [fold-5]
|
||||
arguments:
|
||||
parameters:
|
||||
- name: sha
|
||||
value: "{{tasks.ensure-binary.outputs.parameters.sha}}"
|
||||
- name: fold-idx
|
||||
value: "6"
|
||||
- name: offset
|
||||
value: "11400000"
|
||||
- name: fold-7
|
||||
template: alpha-cv-fold
|
||||
dependencies: [fold-6]
|
||||
arguments:
|
||||
parameters:
|
||||
- name: sha
|
||||
value: "{{tasks.ensure-binary.outputs.parameters.sha}}"
|
||||
- name: fold-idx
|
||||
value: "7"
|
||||
- name: offset
|
||||
value: "13300000"
|
||||
- name: fold-8
|
||||
template: alpha-cv-fold
|
||||
dependencies: [fold-7]
|
||||
arguments:
|
||||
parameters:
|
||||
- name: sha
|
||||
value: "{{tasks.ensure-binary.outputs.parameters.sha}}"
|
||||
- name: fold-idx
|
||||
value: "8"
|
||||
- name: offset
|
||||
value: "15200000"
|
||||
|
||||
# ── ensure-binary: compile alpha_train_stacker + alpha_baseline ──
|
||||
- name: ensure-binary
|
||||
outputs:
|
||||
parameters:
|
||||
- name: sha
|
||||
valueFrom:
|
||||
path: /tmp/sha
|
||||
nodeSelector:
|
||||
k8s.scaleway.com/pool-name: ci-compile-cpu
|
||||
topology.kubernetes.io/zone: fr-par-2
|
||||
tolerations:
|
||||
- key: node.cilium.io/agent-not-ready
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
container:
|
||||
image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/ci-builder:latest
|
||||
imagePullPolicy: Always
|
||||
command: ["/bin/bash", "-c"]
|
||||
env:
|
||||
- name: SQLX_OFFLINE
|
||||
value: "true"
|
||||
- name: CARGO_TERM_COLOR
|
||||
value: always
|
||||
- name: CARGO_TARGET_DIR
|
||||
value: /cargo-target
|
||||
- name: CARGO_HOME
|
||||
value: /cargo-target/cargo-home
|
||||
- name: RUSTC_WRAPPER
|
||||
value: sccache
|
||||
- name: SCCACHE_DIR
|
||||
value: /cargo-target/sccache
|
||||
- name: SCCACHE_CACHE_SIZE
|
||||
value: "40G"
|
||||
- name: CARGO_INCREMENTAL
|
||||
value: "0"
|
||||
- name: GITLAB_PAT
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: gitlab-pat
|
||||
key: token
|
||||
resources:
|
||||
requests:
|
||||
cpu: "14"
|
||||
memory: 32Gi
|
||||
limits:
|
||||
cpu: "30"
|
||||
memory: 64Gi
|
||||
volumeMounts:
|
||||
- name: git-ssh-key
|
||||
mountPath: /etc/git-ssh
|
||||
readOnly: true
|
||||
- name: cargo-target-cuda
|
||||
mountPath: /cargo-target
|
||||
- name: training-data
|
||||
mountPath: /data
|
||||
args:
|
||||
- |
|
||||
set -e
|
||||
SHA="{{workflow.parameters.commit-sha}}"
|
||||
BRANCH="{{workflow.parameters.git-branch}}"
|
||||
mkdir -p ~/.ssh
|
||||
cp /etc/git-ssh/ssh-privatekey ~/.ssh/id_ed25519
|
||||
chmod 600 ~/.ssh/id_ed25519
|
||||
printf 'Host *\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n' > ~/.ssh/config
|
||||
chmod 600 ~/.ssh/config
|
||||
REPO="ssh://git@gitlab-gitlab-shell.foxhunt.svc.cluster.local:2222/root/foxhunt.git"
|
||||
BUILD="/cargo-target/src"
|
||||
git config --global --add safe.directory "$BUILD"
|
||||
|
||||
if [ "$SHA" = "HEAD" ]; then
|
||||
if [ -d "$BUILD/.git" ]; then
|
||||
cd "$BUILD"; git fetch origin; SHA=$(git rev-parse "origin/$BRANCH"); cd /
|
||||
else
|
||||
git clone --filter=blob:none "$REPO" "$BUILD"; cd "$BUILD"
|
||||
git checkout "origin/$BRANCH"; SHA=$(git rev-parse HEAD); cd /
|
||||
fi
|
||||
fi
|
||||
SHORT_SHA=$(echo "$SHA" | cut -c1-9)
|
||||
echo "Resolved SHA: $SHA (short: $SHORT_SHA)"
|
||||
|
||||
BIN_DIR="/data/bin/$SHORT_SHA"
|
||||
BINARIES="alpha_train_stacker alpha_baseline"
|
||||
ALL_CACHED=true
|
||||
for bin in $BINARIES; do
|
||||
[ ! -x "$BIN_DIR/$bin" ] && ALL_CACHED=false && break
|
||||
done
|
||||
|
||||
if [ "$ALL_CACHED" = "true" ]; then
|
||||
echo "=== Cache HIT ==="
|
||||
ls -lh "$BIN_DIR/"
|
||||
echo "$SHORT_SHA" > /tmp/sha
|
||||
exit 0
|
||||
fi
|
||||
echo "=== Cache MISS: compiling alpha binaries for $SHORT_SHA ==="
|
||||
|
||||
if [ -d "$BUILD/.git" ]; then
|
||||
cd "$BUILD"; git fetch origin
|
||||
CURRENT=$(git rev-parse HEAD 2>/dev/null || echo "none")
|
||||
if [ "$CURRENT" != "$SHA" ]; then
|
||||
git checkout --force "$SHA"; git clean -fd
|
||||
fi
|
||||
else
|
||||
git clone --filter=blob:none "$REPO" "$BUILD"; cd "$BUILD"; git checkout "$SHA"
|
||||
fi
|
||||
|
||||
export PATH="${CARGO_HOME}/bin:${PATH}"
|
||||
export CUDA_COMPUTE_CAP={{workflow.parameters.cuda-compute-cap}}
|
||||
|
||||
# alpha_train_stacker lives in ml-alpha, alpha_baseline in ml.
|
||||
echo "Building alpha_train_stacker (ml-alpha)..."
|
||||
cargo build --release -p ml-alpha --example alpha_train_stacker
|
||||
echo "Building alpha_baseline (ml)..."
|
||||
cargo build --release -p ml --features ml/cuda --example alpha_baseline
|
||||
|
||||
mkdir -p "$BIN_DIR"
|
||||
cp "$CARGO_TARGET_DIR/release/examples/alpha_train_stacker" "$BIN_DIR/"
|
||||
cp "$CARGO_TARGET_DIR/release/examples/alpha_baseline" "$BIN_DIR/"
|
||||
strip "$BIN_DIR/"*
|
||||
# alpha_baseline reads config/ml/alpha_fill_coeffs.json as a
|
||||
# required --fill-coeffs input. The source tree's copy travels
|
||||
# with the binaries so the GPU fold pods don't need to clone.
|
||||
cp "$BUILD/config/ml/alpha_fill_coeffs.json" "$BIN_DIR/" || true
|
||||
ls -lh "$BIN_DIR/"
|
||||
|
||||
cd /data/bin
|
||||
ls -1t | tail -n +6 | while read -r old; do
|
||||
echo "Pruning old cache: $old"; rm -rf "$old"
|
||||
done
|
||||
echo "$SHORT_SHA" > /tmp/sha
|
||||
|
||||
# ── stacker-train: train Mamba2+stacker on the 9Q fxcache ──
|
||||
- name: stacker-train
|
||||
inputs:
|
||||
parameters:
|
||||
- name: sha
|
||||
nodeSelector:
|
||||
k8s.scaleway.com/pool-name: "{{workflow.parameters.gpu-pool}}"
|
||||
topology.kubernetes.io/zone: fr-par-2
|
||||
tolerations:
|
||||
- key: nvidia.com/gpu
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
- key: node.cilium.io/agent-not-ready
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
container:
|
||||
image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/training-runtime:latest
|
||||
imagePullPolicy: Always
|
||||
command: ["/bin/bash", "-c"]
|
||||
env:
|
||||
- name: RUST_LOG
|
||||
value: info
|
||||
- name: SQLX_OFFLINE
|
||||
value: "true"
|
||||
resources:
|
||||
requests:
|
||||
cpu: "8"
|
||||
memory: 32Gi
|
||||
nvidia.com/gpu: "1"
|
||||
limits:
|
||||
cpu: "16"
|
||||
memory: 80Gi
|
||||
nvidia.com/gpu: "1"
|
||||
volumeMounts:
|
||||
- name: training-data
|
||||
mountPath: /data
|
||||
- name: feature-cache
|
||||
mountPath: /feature-cache
|
||||
args:
|
||||
- |
|
||||
set -e
|
||||
SHA="{{inputs.parameters.sha}}"
|
||||
BIN="/data/bin/$SHA/alpha_train_stacker"
|
||||
FXCACHE=$(ls -t /feature-cache/*.fxcache | head -1)
|
||||
ALPHA_OUT="/feature-cache/alpha_logits_cache_9q.bin"
|
||||
echo "Using fxcache: $FXCACHE"
|
||||
echo "Output: $ALPHA_OUT"
|
||||
|
||||
"$BIN" \
|
||||
--fxcache-path "$FXCACHE" \
|
||||
--horizon {{workflow.parameters.stacker-horizon}} \
|
||||
--seq-len {{workflow.parameters.stacker-seq-len}} \
|
||||
--hidden-dim {{workflow.parameters.stacker-hidden-dim}} \
|
||||
--state-dim {{workflow.parameters.stacker-state-dim}} \
|
||||
--epochs {{workflow.parameters.stacker-epochs}} \
|
||||
--batch-size {{workflow.parameters.stacker-batch-size}} \
|
||||
--lr {{workflow.parameters.stacker-lr}} \
|
||||
--train-frac {{workflow.parameters.stacker-train-frac}} \
|
||||
--alpha-cache-out "$ALPHA_OUT"
|
||||
ls -lh "$ALPHA_OUT"
|
||||
|
||||
# ── alpha-cv-fold: run alpha_baseline on one window ──
|
||||
- name: alpha-cv-fold
|
||||
inputs:
|
||||
parameters:
|
||||
- name: sha
|
||||
- name: fold-idx
|
||||
- name: offset
|
||||
nodeSelector:
|
||||
k8s.scaleway.com/pool-name: "{{workflow.parameters.gpu-pool}}"
|
||||
topology.kubernetes.io/zone: fr-par-2
|
||||
tolerations:
|
||||
- key: nvidia.com/gpu
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
- key: node.cilium.io/agent-not-ready
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
container:
|
||||
image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/training-runtime:latest
|
||||
imagePullPolicy: Always
|
||||
command: ["/bin/bash", "-c"]
|
||||
env:
|
||||
- name: RUST_LOG
|
||||
value: info
|
||||
- name: SQLX_OFFLINE
|
||||
value: "true"
|
||||
resources:
|
||||
requests:
|
||||
cpu: "8"
|
||||
memory: 32Gi
|
||||
nvidia.com/gpu: "1"
|
||||
limits:
|
||||
cpu: "16"
|
||||
memory: 80Gi
|
||||
nvidia.com/gpu: "1"
|
||||
volumeMounts:
|
||||
- name: training-data
|
||||
mountPath: /data
|
||||
- name: feature-cache
|
||||
mountPath: /feature-cache
|
||||
args:
|
||||
- |
|
||||
set -e
|
||||
SHA="{{inputs.parameters.sha}}"
|
||||
FOLD={{inputs.parameters.fold-idx}}
|
||||
OFFSET={{inputs.parameters.offset}}
|
||||
BIN="/data/bin/$SHA/alpha_baseline"
|
||||
FXCACHE=$(ls -t /feature-cache/*.fxcache | head -1)
|
||||
ALPHA="/feature-cache/alpha_logits_cache_9q.bin"
|
||||
# alpha_fill_coeffs.json travels with the source tree; the
|
||||
# binary needs it next to itself or via explicit --fill-coeffs.
|
||||
# The ci-builder image bakes it under /opt/foxhunt/config/ml.
|
||||
FILL="/data/bin/$SHA/alpha_fill_coeffs.json"
|
||||
if [ ! -f "$FILL" ]; then
|
||||
# Fall back to copying from the cloned source on the
|
||||
# cargo-target PVC mounted in ensure-binary; we don't have
|
||||
# that PVC here, so embed via fxcache PVC pre-stage.
|
||||
FILL="/feature-cache/alpha_fill_coeffs.json"
|
||||
fi
|
||||
OUT="/feature-cache/cv_results_9fold/fold_${FOLD}.json"
|
||||
mkdir -p /feature-cache/cv_results_9fold
|
||||
echo "Fold $FOLD: offset=$OFFSET out=$OUT"
|
||||
|
||||
"$BIN" \
|
||||
--fxcache-path "$FXCACHE" \
|
||||
--alpha-cache "$ALPHA" \
|
||||
--fill-coeffs "$FILL" \
|
||||
--data-start-offset $OFFSET \
|
||||
--max-snapshots {{workflow.parameters.fold-window}} \
|
||||
--train-frac {{workflow.parameters.train-frac}} \
|
||||
--window-k {{workflow.parameters.window-k}} \
|
||||
--decision-stride {{workflow.parameters.decision-stride}} \
|
||||
--horizon {{workflow.parameters.horizon}} \
|
||||
--n-train-par {{workflow.parameters.n-train-par}} \
|
||||
--n-train-episodes {{workflow.parameters.n-train-episodes}} \
|
||||
--out-path "$OUT"
|
||||
ls -lh "$OUT"
|
||||
81
scripts/argo-alpha-cv.sh
Executable file
81
scripts/argo-alpha-cv.sh
Executable file
@@ -0,0 +1,81 @@
|
||||
#!/usr/bin/env bash
|
||||
# Submit the alpha-cv workflow: stacker train on 9Q + 9-fold walk-forward.
|
||||
#
|
||||
# Defaults match the validated 2Q config (decision-stride=200,
|
||||
# scaled training H=1200 × N_par=25 × 8K episodes).
|
||||
#
|
||||
# Usage:
|
||||
# ./scripts/argo-alpha-cv.sh # submit on current HEAD
|
||||
# ./scripts/argo-alpha-cv.sh --branch main
|
||||
# ./scripts/argo-alpha-cv.sh --decision-stride 100 --horizon 2000
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SHA=HEAD
|
||||
BRANCH=main
|
||||
DECISION_STRIDE=200
|
||||
HORIZON=1200
|
||||
N_TRAIN_PAR=25
|
||||
N_TRAIN_EPISODES=8000
|
||||
FOLD_WINDOW=1900000
|
||||
GPU_POOL=ci-training-l40s
|
||||
WATCH=false
|
||||
|
||||
usage() {
|
||||
cat <<EOF
|
||||
Usage: $0 [OPTIONS]
|
||||
--sha <commit> Git SHA (default: HEAD)
|
||||
--branch <branch> Git branch (default: main)
|
||||
--decision-stride <n> Decisions every N bars (default: 200)
|
||||
--horizon <n> Episode length in bars (default: 1200)
|
||||
--n-train-par <n> Parallel envs during training (default: 25)
|
||||
--n-train-episodes <n> Total training episodes (default: 8000)
|
||||
--fold-window <n> Bars per fold window (default: 1900000)
|
||||
--gpu-pool <p> GPU pool (default: ci-training-l40s)
|
||||
--watch Follow logs via argo watch
|
||||
EOF
|
||||
}
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--sha) SHA="$2"; shift 2 ;;
|
||||
--branch) BRANCH="$2"; shift 2 ;;
|
||||
--decision-stride) DECISION_STRIDE="$2"; shift 2 ;;
|
||||
--horizon) HORIZON="$2"; shift 2 ;;
|
||||
--n-train-par) N_TRAIN_PAR="$2"; shift 2 ;;
|
||||
--n-train-episodes) N_TRAIN_EPISODES="$2"; shift 2 ;;
|
||||
--fold-window) FOLD_WINDOW="$2"; shift 2 ;;
|
||||
--gpu-pool) GPU_POOL="$2"; shift 2 ;;
|
||||
--watch) WATCH=true; shift ;;
|
||||
-h|--help) usage; exit 0 ;;
|
||||
*) echo "Unknown option: $1"; usage; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
case "$GPU_POOL" in
|
||||
ci-training-l40s) SM=89 ;;
|
||||
ci-training-h100*|ci-training-h100x2|ci-training-h100-sxm) SM=90 ;;
|
||||
*) echo "Unknown gpu-pool: $GPU_POOL"; exit 1 ;;
|
||||
esac
|
||||
|
||||
echo "Submitting alpha-cv workflow..."
|
||||
echo " branch: $BRANCH"
|
||||
echo " sha: $SHA"
|
||||
echo " decision-stride: $DECISION_STRIDE"
|
||||
echo " horizon: $HORIZON"
|
||||
echo " n-train-par: $N_TRAIN_PAR"
|
||||
echo " n-train-episodes: $N_TRAIN_EPISODES"
|
||||
echo " fold-window: $FOLD_WINDOW"
|
||||
echo " gpu-pool: $GPU_POOL (sm_$SM)"
|
||||
|
||||
argo submit -n foxhunt --from=wftmpl/alpha-cv \
|
||||
-p commit-sha="$SHA" \
|
||||
-p git-branch="$BRANCH" \
|
||||
-p cuda-compute-cap="$SM" \
|
||||
-p gpu-pool="$GPU_POOL" \
|
||||
-p decision-stride="$DECISION_STRIDE" \
|
||||
-p horizon="$HORIZON" \
|
||||
-p n-train-par="$N_TRAIN_PAR" \
|
||||
-p n-train-episodes="$N_TRAIN_EPISODES" \
|
||||
-p fold-window="$FOLD_WINDOW" \
|
||||
$($WATCH && echo "--watch" || echo "")
|
||||
Reference in New Issue
Block a user