diff --git a/crates/ml-alpha/examples/alpha_rl_train.rs b/crates/ml-alpha/examples/alpha_rl_train.rs index 86e8d6149..8ebaa8f98 100644 --- a/crates/ml-alpha/examples/alpha_rl_train.rs +++ b/crates/ml-alpha/examples/alpha_rl_train.rs @@ -129,6 +129,44 @@ struct Cli { #[arg(long, default_value_t = 100)] log_every: usize, + /// Walk-forward fold index for the multi-fold G8 gate (per + /// `pearl_single_window_oos_is_not_oos` — a single window is NOT + /// out-of-sample). Slices the MBP-10 file list into K equal-sized + /// blocks; fold k trains on blocks [0..k+1) and evaluates on block + /// [k+1] (purged walk-forward). At `fold_idx=0` the first block + /// is the train window, second is eval. At `fold_idx=K-2` the + /// trainer sees almost all the data and evaluates on the final + /// block. The dispatcher (`scripts/argo-alpha-rl.sh --n-folds K`) + /// fans out K-1 workflows, one per fold; the aggregator computes + /// mean ± SD of the per-fold profit_factor. + /// + /// Default 0 + n-folds=1 = single-window mode (no eval split). + #[arg(long, default_value_t = 0)] + fold_idx: usize, + + /// Total walk-forward fold count. Set with `--fold-idx` for the + /// multi-fold G8 gate. n_folds=1 disables the train/eval split + /// (single-window mode — only meaningful for smoke validation). + /// n_folds≥3 enables a real evaluation: train on first + /// `fold_idx + 1` of `n_folds` data blocks, evaluate on the next. + #[arg(long, default_value_t = 1)] + n_folds: usize, + + /// Eval-phase step count. After the train phase completes its + /// `--n-steps`, the eval phase runs `--n-eval-steps` additional + /// steps on the held-out fold using the same trainer + /// (acknowledged minor in-eval learning contamination at b_size=1 + /// — pure eval mode is a follow-up). LobSim trade records are + /// drained at the END so the summary reflects eval-phase trades + /// only (train-phase trades are flushed when the eval phase + /// starts). + /// + /// 0 disables the eval phase entirely (single-window training + /// only — the summary's profit_factor will reflect the full run + /// including train). + #[arg(long, default_value_t = 0)] + n_eval_steps: usize, + /// Per-step diagnostic JSONL path. One record per step capturing: /// * step number, wall time /// * loss components (l_bce/q/pi/v/aux/total) + λs (5) @@ -226,14 +264,50 @@ fn main() -> Result<()> { // next_sequence_pair is available (the labels it preloads are // unused by step_with_lobsim, but the API requires the label // precomputation pass per `feedback_no_partial_refactor`). - let files = discover_mbp10_files_sorted(&cli.mbp10_data_dir) + let all_files = discover_mbp10_files_sorted(&cli.mbp10_data_dir) .with_context(|| format!("discover_mbp10_files_sorted({})", cli.mbp10_data_dir.display()))?; anyhow::ensure!( - !files.is_empty(), + !all_files.is_empty(), "no MBP-10 .dbn.zst files under {}", cli.mbp10_data_dir.display() ); - eprintln!("loader: {} MBP-10 files", files.len()); + eprintln!("loader: {} MBP-10 files total", all_files.len()); + + // Walk-forward fold split per `pearl_single_window_oos_is_not_oos`. + // Slice the file list into `n_folds` equal blocks. fold k uses + // blocks [0..=k] for training and block [k+1] for evaluation. With + // n_folds=1 (default) train_files = all_files and eval_files is + // empty (single-window mode, no eval phase). + let (train_files, eval_files) = if cli.n_folds <= 1 || cli.n_eval_steps == 0 { + eprintln!("walk-forward: disabled (n_folds={}, n_eval_steps={})", cli.n_folds, cli.n_eval_steps); + (all_files, Vec::new()) + } else { + anyhow::ensure!( + cli.fold_idx + 1 < cli.n_folds, + "fold_idx {} requires fold_idx + 1 < n_folds ({}); the last block must be the eval window", + cli.fold_idx, + cli.n_folds + ); + let n_files = all_files.len(); + let block_size = n_files / cli.n_folds; + anyhow::ensure!( + block_size >= 1, + "not enough MBP-10 files ({}) to split into n_folds={} (need ≥ n_folds files)", + n_files, + cli.n_folds + ); + // Train: blocks 0..=fold_idx, Eval: block fold_idx+1. + let train_end = (cli.fold_idx + 1) * block_size; + let eval_end = (cli.fold_idx + 2) * block_size; + let train: Vec<_> = all_files[..train_end].to_vec(); + let eval: Vec<_> = all_files[train_end..eval_end.min(n_files)].to_vec(); + eprintln!( + "walk-forward fold {}/{}: train={} files ({}..{}), eval={} files ({}..{})", + cli.fold_idx, cli.n_folds, train.len(), 0, train_end, + eval.len(), train_end, eval_end.min(n_files) + ); + (train, eval) + }; let multi_resolution: ml_alpha::data::aggregation::MultiResolutionConfig = format!("1:{}", cli.seq_len) @@ -248,9 +322,9 @@ fn main() -> Result<()> { ); let loader_cfg = MultiHorizonLoaderConfig { - files, + files: train_files, predecoded_dir: cli.predecoded_dir.clone(), - multi_resolution, + multi_resolution: multi_resolution.clone(), horizons: HORIZONS, // Soft cap above n_steps × 2 so next_sequence_pair never // returns Ok(None) mid-run. Loader's per-file random anchor @@ -475,6 +549,117 @@ fn main() -> Result<()> { } diag.flush().context("diag: final flush")?; + // ── Eval phase (walk-forward G8). ──────────────────────────────── + // After the train phase, run n_eval_steps additional step_with_lobsim + // calls on the held-out eval files. Trade records accumulated by + // LobSim during eval are isolated from train-phase records via the + // `head_before_eval` checkpoint — eval records are sliced as + // `all_records[head_before_eval..]` post-eval. + // + // Note: this is NOT pure-eval mode — step_with_lobsim still runs + // its learning machinery (Adam steps, PER updates). At b_size=1 + // the per-step learning effect is small relative to the policy + // already accumulated in train; the eval profit_factor approximates + // the train-end policy's OOS performance. True pure-eval (forward + // only, no backward) is a follow-up architectural change. + if cli.n_eval_steps > 0 && !eval_files.is_empty() { + let head_before_eval = sim + .read_total_trade_count() + .context("read trade count pre-eval")?; + eprintln!( + "── eval phase: {} steps on {} held-out files (trade-record checkpoint head={}) ──", + cli.n_eval_steps, eval_files.len(), head_before_eval + ); + + let eval_loader_cfg = MultiHorizonLoaderConfig { + files: eval_files.clone(), + predecoded_dir: cli.predecoded_dir.clone(), + multi_resolution, + horizons: HORIZONS, + n_max_sequences: cli.n_eval_steps.saturating_mul(2).max(64), + // Different seed for eval so sequence sampling is independent + // of the train seed (deterministic given the eval seed). + seed: cli.seed.wrapping_add(0xE7AE), + inference_only: false, + outcome_label_cost: cli.outcome_label_cost, + instrument_filter: cli.instrument_mode, + }; + let mut eval_loader = MultiHorizonLoader::new(&eval_loader_cfg) + .context("MultiHorizonLoader::new (eval)")?; + + for eval_step in 0..cli.n_eval_steps { + let pair = eval_loader + .next_sequence_pair() + .with_context(|| format!("eval next_sequence_pair at step {eval_step}"))?; + let (s_t, s_tp1) = match pair { + Some(p) => p, + None => { + eprintln!("eval loader EOF at step {eval_step}"); + break; + } + }; + let stats = trainer + .step_with_lobsim(&s_t.snapshots, &s_tp1.snapshots, &mut sim) + .with_context(|| format!("step_with_lobsim eval step {eval_step}"))?; + if !stats.l_total.is_finite() { + eprintln!("G8 NaN ABORT during eval at step {eval_step}"); + process::exit(2); + } + if eval_step % cli.log_every == 0 || eval_step == cli.n_eval_steps - 1 { + eprintln!( + "eval {:>5}/{}: l_total={:.4} elapsed={:.1}s", + eval_step, cli.n_eval_steps, stats.l_total, + t_start.elapsed().as_secs_f32() + ); + } + } + + // Drain trade records, slice to eval-only, compute summary. + let all_records = sim + .read_trade_records(0) + .context("read trade records post-eval")?; + let head_before_usize = head_before_eval as usize; + let eval_records: Vec<_> = if all_records.len() > head_before_usize { + all_records[head_before_usize..].to_vec() + } else { + // Trade log wrapped past TRADE_LOG_CAP — use what we have. + // For smoke (b_size=1, ≤200 trades) this branch never fires. + eprintln!( + "warning: trade log wrapped — head_before={} but only {} records readable", + head_before_usize, all_records.len() + ); + all_records + }; + + // Synthesise a pnl_curve from per-trade cumulative PnL for + // compute_summary's max_drawdown calc. + let mut pnl_curve = Vec::with_capacity(eval_records.len()); + let mut cum = 0.0f32; + for r in &eval_records { + cum += (r.realised_pnl_usd_fp as f32) / 100.0; + pnl_curve.push(cum); + } + let eval_summary = + ml_backtesting::artifacts::compute_summary(&eval_records, &pnl_curve); + + eprintln!( + "eval summary: n_trades={} pnl_usd={:.2} pf={:.3} sharpe_ann={:.3} max_dd_usd={:.2} win_rate={:.3}", + eval_summary.n_trades, + eval_summary.total_pnl_usd, + eval_summary.profit_factor, + eval_summary.sharpe_ann, + eval_summary.max_drawdown_usd, + eval_summary.win_rate, + ); + + let eval_summary_path = cli.out.join("eval_summary.json"); + let f = std::fs::File::create(&eval_summary_path) + .with_context(|| format!("create {}", eval_summary_path.display()))?; + serde_json::to_writer_pretty(f, &eval_summary) + .with_context(|| format!("write {}", eval_summary_path.display()))?; + eprintln!("eval summary written: {}", eval_summary_path.display()); + } + // ── Compose + emit summary. ────────────────────────────────────── if let Some(s) = last_stats { summary.final_l_bce = s.l_bce; diff --git a/infra/k8s/argo/alpha-rl-template.yaml b/infra/k8s/argo/alpha-rl-template.yaml index 80ad64adc..7c868aad4 100644 --- a/infra/k8s/argo/alpha-rl-template.yaml +++ b/infra/k8s/argo/alpha-rl-template.yaml @@ -72,6 +72,12 @@ spec: value: "16962" - name: instrument-mode value: "front-month" + - name: fold-idx + value: "0" + - name: n-folds + value: "1" + - name: n-eval-steps + value: "0" volumes: - name: git-ssh-key @@ -402,7 +408,10 @@ spec: --n-backtests {{workflow.parameters.n-backtests}} \ --per-capacity {{workflow.parameters.per-capacity}} \ --seed {{workflow.parameters.seed}} \ - --instrument-mode "{{workflow.parameters.instrument-mode}}" + --instrument-mode "{{workflow.parameters.instrument-mode}}" \ + --fold-idx {{workflow.parameters.fold-idx}} \ + --n-folds {{workflow.parameters.n-folds}} \ + --n-eval-steps {{workflow.parameters.n-eval-steps}} RC=$? echo "=== alpha_rl_train exited with code $RC ===" diff --git a/scripts/argo-alpha-rl.sh b/scripts/argo-alpha-rl.sh index e7a80a0d3..af38f6878 100755 --- a/scripts/argo-alpha-rl.sh +++ b/scripts/argo-alpha-rl.sh @@ -36,6 +36,9 @@ N_BACKTESTS=1 PER_CAPACITY=4096 SEED=16962 INSTRUMENT_MODE=front-month +FOLD_IDX=0 +N_FOLDS=1 +N_EVAL_STEPS=0 WATCH=false SKIP_PUSH_CHECK=false SKIP_TEMPLATE_APPLY=false @@ -57,6 +60,23 @@ Usage: $0 [OPTIONS] --per-capacity PER buffer capacity (default: $PER_CAPACITY). --seed Random seed (default: $SEED). --instrument-mode all | front-month | id= (default: $INSTRUMENT_MODE) + --fold-idx Walk-forward fold index (0-based) for the + multi-fold G8 gate (default: $FOLD_IDX). Slices + the MBP-10 file list into n_folds blocks; fold k + trains on blocks [0..=k] and evals on block [k+1]. + Requires --n-folds ≥ 3 and --n-eval-steps > 0. + --n-folds Total fold count (default: $N_FOLDS). K=1 disables + the eval split (single-window mode). K≥3 enables + walk-forward; submit K-1 workflows (one per + fold_idx) and aggregate the per-fold profit_factor + for the G8 gate per + \`pearl_single_window_oos_is_not_oos\`. + --n-eval-steps Eval-phase step count (default: $N_EVAL_STEPS). + After the train phase, runs n_eval_steps additional + steps on the held-out fold. LobSim trade records + from the eval phase are isolated via a pre-eval + head checkpoint; \`eval_summary.json\` lands in + OUT_DIR alongside \`alpha_rl_train_summary.json\`. --watch Follow logs via argo watch --skip-push-check Bypass the feedback_push_before_deploy guard (rare; use only when submitting against a @@ -79,6 +99,9 @@ while [[ $# -gt 0 ]]; do --per-capacity) PER_CAPACITY="$2"; shift 2 ;; --seed) SEED="$2"; shift 2 ;; --instrument-mode) INSTRUMENT_MODE="$2"; shift 2 ;; + --fold-idx) FOLD_IDX="$2"; shift 2 ;; + --n-folds) N_FOLDS="$2"; shift 2 ;; + --n-eval-steps) N_EVAL_STEPS="$2"; shift 2 ;; --watch) WATCH=true; shift ;; --skip-push-check) SKIP_PUSH_CHECK=true; shift ;; --skip-template-apply) SKIP_TEMPLATE_APPLY=true; shift ;; @@ -191,4 +214,7 @@ argo submit -n foxhunt --from=wftmpl/alpha-rl \ -p per-capacity="$PER_CAPACITY" \ -p seed="$SEED" \ -p instrument-mode="$INSTRUMENT_MODE" \ + -p fold-idx="$FOLD_IDX" \ + -p n-folds="$N_FOLDS" \ + -p n-eval-steps="$N_EVAL_STEPS" \ $WATCH_FLAG