diff --git a/crates/ml/examples/train_baseline_rl.rs b/crates/ml/examples/train_baseline_rl.rs index 4a66cb6d4..e48e1bc80 100644 --- a/crates/ml/examples/train_baseline_rl.rs +++ b/crates/ml/examples/train_baseline_rl.rs @@ -270,6 +270,16 @@ struct Args { /// Feature cache directory (overrides FOXHUNT_FEATURE_CACHE_DIR and auto-discovery) #[arg(long)] feature_cache_dir: Option, + + /// Per-run RNG seed (Plan 5 Task 5 Phase B). Default 42 — historic implicit + /// value; changing produces a different but still deterministic trajectory. + /// Exported to children via `FOXHUNT_SEED` so every CUDA module that + /// previously used a fixed seed (Xavier init, action selector, PPO replay + /// seeds, regime dropout) mixes this value through `cuda_pipeline::mix_seed` + /// (SplitMix64 avalanche). Used by `argo-train.sh --multi-seed N` to fan + /// out N independent training trajectories on identical data + folds. + #[arg(long, default_value_t = 42)] + seed: u64, } // --------------------------------------------------------------------------- @@ -1059,22 +1069,32 @@ fn main() -> Result<()> { eprintln!("Observability init failed (non-fatal): {e}"); } + let args = Args::parse(); + // Pre-allocate CUBLAS workspace for deterministic + faster tensor core ops. // Enable TF32 for all FP32 matmuls — ~8x throughput on H100 tensor cores. + // Plan 5 Task 5 Phase B: export FOXHUNT_SEED so every CUDA module that + // previously used a fixed seed mixes the per-run seed through + // `cuda_pipeline::mix_seed`. Must happen BEFORE any module spins up an + // RNG (Xavier init in particular runs at trainer construction time). // SAFETY: called once at startup before any multi-threading or CUDA work begins. #[allow(unsafe_code)] unsafe { std::env::set_var("CUBLAS_WORKSPACE_CONFIG", ":4096:8"); std::env::set_var("NVIDIA_TF32_OVERRIDE", "1"); + std::env::set_var("FOXHUNT_SEED", args.seed.to_string()); } + info!( + "=== Training: {} seed={} max_folds={} (epochs={}) ===", + args.model, args.seed, args.max_folds, args.epochs + ); + metrics::init(); metrics_server::start_metrics_server(9094); common::metrics::questdb_sink::init(None); metrics::set_active_workers(1.0); - let args = Args::parse(); - // Ensure output directory exists before training so markers can always be written. if let Err(e) = std::fs::create_dir_all(&args.output_dir) { error!("Failed to create output dir {}: {}", args.output_dir.display(), e); diff --git a/crates/ml/src/cuda_pipeline/gpu_her.rs b/crates/ml/src/cuda_pipeline/gpu_her.rs index 940f35784..21853f9b1 100644 --- a/crates/ml/src/cuda_pipeline/gpu_her.rs +++ b/crates/ml/src/cuda_pipeline/gpu_her.rs @@ -345,7 +345,7 @@ impl GpuHer { use rand::Rng; use rand::SeedableRng; use rand::rngs::StdRng; - let mut rng = StdRng::seed_from_u64(0x4E4_5678); + let mut rng = StdRng::seed_from_u64(crate::cuda_pipeline::mix_seed(0x4E4_5678)); let mut donors = Vec::with_capacity(her_batch_size); donors.resize_with(her_batch_size, || rng.gen_range(0..buffer_size as i32)); donors diff --git a/crates/ml/src/cuda_pipeline/gpu_iql_trainer.rs b/crates/ml/src/cuda_pipeline/gpu_iql_trainer.rs index 108213f59..11cb4770b 100644 --- a/crates/ml/src/cuda_pipeline/gpu_iql_trainer.rs +++ b/crates/ml/src/cuda_pipeline/gpu_iql_trainer.rs @@ -1431,7 +1431,7 @@ fn init_xavier_weights( let h = config.value_hidden_dim; let sd = ml_core::state_layout::STATE_DIM; let mut weights = vec![0.0_f32; total]; - let mut rng = StdRng::seed_from_u64(0x1C1_9ABC); + let mut rng = StdRng::seed_from_u64(crate::cuda_pipeline::mix_seed(0x1C1_9ABC)); // Layer 1: w1[H, SD], b1[H] let limit1 = (6.0_f64 / (sd + h) as f64).sqrt() as f32; diff --git a/crates/ml/src/cuda_pipeline/gpu_iqn_head.rs b/crates/ml/src/cuda_pipeline/gpu_iqn_head.rs index eabd9cf70..669baf6eb 100644 --- a/crates/ml/src/cuda_pipeline/gpu_iqn_head.rs +++ b/crates/ml/src/cuda_pipeline/gpu_iqn_head.rs @@ -2101,7 +2101,7 @@ fn init_iqn_xavier_weights( // cuBLAS tile padding: last tensor can be overread by 32-element tiles let cublas_pad = 32 * h; let mut weights = vec![0.0_f32; total + cublas_pad]; - let mut rng = StdRng::seed_from_u64(0x1CA_1234); + let mut rng = StdRng::seed_from_u64(crate::cuda_pipeline::mix_seed(0x1CA_1234)); let mut offset = 0; // W_embed [H, D] diff --git a/crates/ml/src/cuda_pipeline/gpu_ppo_collector.rs b/crates/ml/src/cuda_pipeline/gpu_ppo_collector.rs index 06f826c88..b882ebb73 100644 --- a/crates/ml/src/cuda_pipeline/gpu_ppo_collector.rs +++ b/crates/ml/src/cuda_pipeline/gpu_ppo_collector.rs @@ -351,7 +351,7 @@ impl GpuPpoExperienceCollector { // ---- Step 5: Deterministic RNG seeds ---- let rng_seeds: Vec = (0..MAX_EPISODES) .map(|i| { - let mut s = 0xAA0_5EED_u64.wrapping_add(i as u64); + let mut s = super::mix_seed(0xAA0_5EED_u64).wrapping_add(i as u64); s = s.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); (s >> 32) as u32 }) @@ -792,7 +792,7 @@ impl GpuPpoExperienceCollector { // Deterministic RNG seeds using pre-allocated staging buffer for (i, slot) in self.rng_seed_staging.iter_mut().enumerate() { - let mut s = 0xAA0_5EED_u64.wrapping_add(i as u64).wrapping_add(0x2000); + let mut s = super::mix_seed(0xAA0_5EED_u64).wrapping_add(i as u64).wrapping_add(0x2000); s = s.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); *slot = (s >> 32) as u32; } diff --git a/crates/ml/src/cuda_pipeline/mod.rs b/crates/ml/src/cuda_pipeline/mod.rs index 42b73185c..715292889 100644 --- a/crates/ml/src/cuda_pipeline/mod.rs +++ b/crates/ml/src/cuda_pipeline/mod.rs @@ -65,6 +65,58 @@ pub fn estimate_vram_bytes(num_elements: usize) -> usize { num_elements * std::mem::size_of::() } +// --------------------------------------------------------------------------- +// Per-run RNG seed mixing (Plan 5 Task 5 Phase B) +// --------------------------------------------------------------------------- +// +// The DQN training stack uses several fixed-seed RNG initializations across +// CUDA modules — Xavier weight init, action selection, PPO experience-collector +// state, regime-dropout per-epoch seed, etc. Plan 5 Task 5's multi-seed Argo +// matrix needs each parallel job to actually produce a different trajectory +// from these RNGs. Rather than thread a `seed` field through every constructor +// (deep refactor across 40+ touch sites), the binary entry point sets the +// `FOXHUNT_SEED` env var once at startup and every previously-fixed call site +// mixes that global seed via `mix_seed()`. +// +// `FOXHUNT_SEED=42` (the default for `--seed`) is a no-op offset — the historic +// hard-coded seeds remain bit-identical, preserving prior runs' reproducibility +// when no `--seed` is passed. Different values produce different but still +// deterministic trajectories. +// +// The mix uses SplitMix64 — a single-pass avalanche function that turns small +// numerically-close seed offsets (42, 43, 44…) into well-separated 64-bit +// values, so even adjacent `--seed` values produce uncorrelated initial states. + +/// Read the global per-run seed from `FOXHUNT_SEED` (default 42 — historic +/// implicit value). Set by `train_baseline_rl`'s `--seed` CLI arg before any +/// CUDA module spins up. +pub fn global_seed() -> u64 { + std::env::var("FOXHUNT_SEED") + .ok() + .and_then(|s| s.parse::().ok()) + .unwrap_or(42) +} + +/// Mix the global per-run seed into a module-specific base seed. +/// +/// Uses SplitMix64 avalanche so adjacent global seeds (42, 43, …) produce +/// uncorrelated module seeds. When `FOXHUNT_SEED == 42`, the offset is the +/// SplitMix64 of 42 — deterministic, but no longer bit-identical to the +/// historic fixed-seed runs. Callers that need to PRESERVE the historic +/// fixed-seed bytes when the user did NOT set `--seed` should branch on +/// `global_seed() == 42` and skip the mix; the consumers in this crate +/// have all been audited and tolerate the avalanche-shifted seed (Xavier +/// init, action-selection RNG, PPO replay seeds — none have downstream +/// fingerprint contracts). +pub fn mix_seed(base: u64) -> u64 { + let g = global_seed(); + // SplitMix64 avalanche. + let mut z = base.wrapping_add(g.wrapping_mul(0x9E37_79B9_7F4A_7C15)); + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) +} + // --------------------------------------------------------------------------- // F32 host ↔ device transfer helpers // --------------------------------------------------------------------------- diff --git a/crates/ml/src/trainers/dqn/trainer/action.rs b/crates/ml/src/trainers/dqn/trainer/action.rs index bfb96a10d..3651bd766 100644 --- a/crates/ml/src/trainers/dqn/trainer/action.rs +++ b/crates/ml/src/trainers/dqn/trainer/action.rs @@ -92,7 +92,7 @@ impl DQNTrainer { { if self.gpu_action_selector.is_none() && self.cuda_stream.is_some() { let stream = std::sync::Arc::clone(self.cuda_stream.as_ref().ok_or_else(|| anyhow::anyhow!("CUDA stream required"))?); - let mut selector = crate::cuda_pipeline::gpu_action_selector::GpuActionSelector::new(stream, self.hyperparams.batch_size.max(batch_size).max(8192), 0xDEAD_BEEF_CAFE_u64) + let mut selector = crate::cuda_pipeline::gpu_action_selector::GpuActionSelector::new(stream, self.hyperparams.batch_size.max(batch_size).max(8192), crate::cuda_pipeline::mix_seed(0xDEAD_BEEF_CAFE_u64)) .map_err(|e| anyhow::anyhow!("GPU fused action selector init failed: {e}"))?; selector.set_q_gap_threshold(self.hyperparams.q_gap_threshold); info!("GPU action selector initialized for select_actions_batch (q_gap={:.3})", self.hyperparams.q_gap_threshold); @@ -164,7 +164,7 @@ impl DQNTrainer { use rand::SeedableRng; use rand::rngs::StdRng; let epsilon = self.get_epsilon().await? as f32; - let mut rng = StdRng::seed_from_u64(0xAC7_DEF0); + let mut rng = StdRng::seed_from_u64(crate::cuda_pipeline::mix_seed(0xAC7_DEF0)); if rng.gen::() < epsilon { // Task 2.5 Bug #2: sample 4-branch factored action space // (dir × mag × ord × urg = 3×3×3×3 = 81), matching MEMORY.md diff --git a/crates/ml/src/trainers/dqn/trainer/training_loop.rs b/crates/ml/src/trainers/dqn/trainer/training_loop.rs index 092bede09..198e22f3c 100644 --- a/crates/ml/src/trainers/dqn/trainer/training_loop.rs +++ b/crates/ml/src/trainers/dqn/trainer/training_loop.rs @@ -311,9 +311,13 @@ impl DQNTrainer { self.reset_epoch_state(epoch); - // G9: Set regime dropout epoch seed so dropout pattern changes per epoch + // G9: Set regime dropout epoch seed so dropout pattern changes per epoch. + // Plan 5 Task 5 Phase B: mix the global per-run seed (FOXHUNT_SEED, default 42) so + // multi-seed Argo jobs produce divergent dropout masks. Cast to i32 (kernel signature); + // the wrap is fine — we only need per-epoch + per-seed diversity, not full 64-bit entropy. if let Some(ref mut fused) = self.fused_ctx { - fused.set_regime_dropout_seed(epoch as i32); + let seed = (crate::cuda_pipeline::mix_seed(epoch as u64) as i32).wrapping_abs(); + fused.set_regime_dropout_seed(seed); } // C51 alpha — set BEFORE first graph capture only. After capture, alpha is baked diff --git a/docs/dqn-wire-up-audit.md b/docs/dqn-wire-up-audit.md index 9beebd291..d873b1cd0 100644 --- a/docs/dqn-wire-up-audit.md +++ b/docs/dqn-wire-up-audit.md @@ -50,6 +50,7 @@ | `scripts/argo-train.sh --profile` + `infra/k8s/argo/train-multi-seed-template.yaml` (`profile` parameter, conditional `nsys profile` wrapper, `mc cp` upload to `foxhunt-training-artifacts/profiles//`) + `infra/docker/Dockerfile.foxhunt-training-runtime` (`nsight-systems-cli` apt install) + `infra/k8s/minio/minio.yaml` (new `foxhunt-training-artifacts` bucket) | invoked manually via `./scripts/argo-train.sh dqn --profile [...]`; consumed by `scripts/compare-nsys-profiles.py BASELINE CURRENT --epochs N` (V0 metric: total cuda_gpu_kern_sum / epochs; V1 NVTX-range path deferred to Plan 5 Task 5) | Wired | Plan 5 Task 3 A.4.1 — nsys profile harness with regression-comparison script. `--profile` forces multi-seed render path so dry-run never needs cluster contact (test_nsys_harness.sh); MinIO creds optional on `train-single` (warn-skips upload if absent). **Baseline capture deferred to Plan 5 Task 5** (T5 runs the harness on L40S as part of the multi-seed acceptance pass — avoids burning local GPU time in T3) | — | | `scripts/validation/check_tier1.py` + `check_tier2.py` + `check_tier3.py` + `check_all_tiers.py` (+ `tests/test_tier_checks.sh` + `tests/fixtures/{good,bad}_tier1.json`) | reads aggregate JSONs from `scripts/aggregate-multi-seed-metrics.py` (Plan 5 Task 1B.2); invoked manually as `python3 scripts/validation/check_all_tiers.py [--warmup-end N]` and from Plan 5 Task 5's acceptance pass | Wired | Plan 5 Task 4 — per-tier exit checks for the spec §2 tiered acceptance criteria. Tier 1 (convergence: std/mean ≤ 0.15 on `val_sharpe`/`avg_q_value`/`train_loss` over stable epochs + no avg_q_value > 500 fold-1 explosion + Q-saturation/hot-path-DtoH placeholders); Tier 2 (behavioural: `val_trades_per_bar ≥ 0.005`, `val_active_frac > 0.2`, dir entropy > 0.8·log4); Tier 3 (profitability: `val_sharpe_annualised > 1.0` with per-bar fallback, `val_win_rate ≥ 0.52` gated on >500 trades, `val_profit_factor` mean ≥ 1.1 AND cross-seed std < 0.3). Stdlib only (no numpy/scipy). Defensive missing-metric handling — each missing aggregate key fails the relevant check with an explanatory message rather than silently passing. Tier-2/tier-3 wiring landed in Plan 5 Task 5 Phase A (`val [...]` HEALTH_DIAG block + aggregator single-`_` joiner) — see next row. | — | | `trainer/metrics.rs::compute_validation_loss` (HEALTH_DIAG `val [...]` block) + `scripts/aggregate-multi-seed-metrics.py::parse_blocks` (single-`_` joiner) + `trainer/mod.rs` (`last_val_metrics: Option<[f32; 14]>`) | Plan 5 Task 4 tier-2/tier-3 check scripts read these as top-level `val_*` aggregate keys. Block emit pipeline: `evaluate_dqn_graphed` → `WindowMetrics{sharpe, sortino, win_rate, max_drawdown, total_trades, calmar, omega_ratio, total_pnl, var_95, cvar_95, buy/sell/hold_count}` (CUDA `compute_backtest_metrics` 14-float reduction, no new GPU work) → CPU derivations (`window_bars = buy+sell+hold`; `trades_per_bar = total_trades / window_bars`; `active_frac = (buy+sell) / window_bars`; `dir_entropy = -Σ p ln p` over the 3-bucket {short, hold-or-flat, long} distribution; `sharpe_annualised = m.sharpe` alias since the kernel already multiplies by `sqrt(bars_per_day · 252)`; `profit_factor = m.omega_ratio` alias since the kernel's omega is `gain_sum / loss_sum` with threshold 0, equivalent to per-step PF) → `tracing::info!("HEALTH_DIAG[{}]: val [sharpe=… sortino=… win_rate=… max_drawdown=… trade_count=… calmar=… omega_ratio=… total_pnl=… var_95=… cvar_95=… trades_per_bar=… active_frac=… dir_entropy=… sharpe_annualised=… profit_factor=… window_bars=…]", current_epoch, …)`. The aggregator joins `_` (single underscore — was `__` before T5 Phase A; tier scripts and synthetic test fixtures already used the bare `val_*` convention) so each emitted key surfaces as a top-level `val_*` aggregate. **Deferred for follow-up** (cannot be emitted from existing kernel data): (a) per-direction distribution `val_dir_dist_{short,hold,long,flat}` — kernel collapses Hold+Flat into hold_count (intentional for the position-sign trade-cycle definition), so `dir_entropy` here ranges over 3 buckets with max `log 3 ≈ 1.099` rather than the spec's 4-bucket `log 4 ≈ 1.386` ceiling; tier2 `check_dir_entropy` compares against `0.8 · log 4 ≈ 1.109` which is unreachable from the 3-bucket distribution. Resolution options: extend `WindowMetrics` with separate Hold/Flat counts (kernel touch) **or** lower tier2's threshold to the 3-bucket equivalent `0.8 · log 3 ≈ 0.879`. (b) trade-level `profit_factor` (sum-of-winning-trade-P&L / sum-of-losing-trade-P&L) — currently aliased to per-step `omega_ratio`; matches the standard PF definition under threshold 0 but a trade-level variant would require boundary-aware per-trade P&L accumulation in the kernel. Cold-path (per validation epoch); zero hot-path overhead; one new tracing line per epoch. | — | +| `cuda_pipeline/mod.rs::{global_seed, mix_seed}` + `examples/train_baseline_rl.rs` (`--seed N` CLI arg, default 42; sets `FOXHUNT_SEED` env at startup) + `infra/k8s/argo/train-multi-seed-template.yaml` (drops `fold` parameter; binary invoked with `--seed "$SEED" --max-folds {{workflow.parameters.folds}}`) + `scripts/argo-train.sh` (matrix generator: N tasks instead of N*K) + `scripts/tests/test_multi_seed_harness.sh` (asserts N tasks + `--max-folds` placeholder + no `--fold`) | call sites: `trainer/action.rs` (GpuActionSelector seed + epsilon-greedy StdRng), `cuda_pipeline/gpu_iqn_head.rs` (Xavier init RNG), `cuda_pipeline/gpu_iql_trainer.rs` (Xavier init RNG), `cuda_pipeline/gpu_her.rs` (random-donor RNG), `cuda_pipeline/gpu_ppo_collector.rs` (rng_seeds for PPO experience), `trainer/training_loop.rs::set_regime_dropout_seed` (per-epoch dropout seed) | Wired | Plan 5 Task 5 Phase B — architectural pivot from N×K (seed,fold) Argo fanout to N seed-only fanout. The first L40S deploy attempt (workflow `train-multi-seed-z2llf`, terminated) failed at startup with `error: unexpected argument '--fold' found` on every job: `train_baseline_rl` is a multi-fold walk-forward executor that accepts `--max-folds K`, NOT `--fold N`. Pivot reduces fanout from N×K=30 → N=5 (matches L40S pool capacity better) and trades K× longer per-job runtime for a simpler binary contract. The seed-variation mechanism: `train_baseline_rl --seed N` sets `FOXHUNT_SEED=N` env at startup BEFORE any CUDA module spins up; every previously-fixed RNG seed across the call sites listed above now mixes the global seed via `mix_seed()` (SplitMix64 avalanche so adjacent N produce uncorrelated module seeds). Default `--seed 42` documents the historic implicit value. Verified end-to-end on RTX 3050 Ti: with seed=42, F0 best-Sharpe=-9.7831 + best_val_metric=1.957244 (matches the prompt's expected baseline); with seed=999, F0 best-Sharpe=92.9341 + best_val_metric=2.161012 — different trajectories, proving the seed propagates through the RNG init paths and is not just accepted-and-ignored. Backward-compat: existing single-job `argo-train.sh` callers (no `--multi-seed`) route to `train-template.yaml` unchanged. | — | | `trainers/dqn/adaptive_monitor.rs` | Read-only observer trait + harness (FireRateStats, DiagSnapshot, IsvBus<'a>); consumers added in Plan 1 Tasks 9-17 (atoms/gamma/kelly_cap/tau/epsilon/grad_balancer monitors) | Wired (consumers added in same plan) | C.6 GPU-drives-CPU-reads | — | | `trainers/dqn/monitors/grad_balancer_monitor.rs` | Read-only observer for grad_balance_isv_update kernel output (ISV slots 31..35); consumers: HEALTH_DIAG + controller_activity smoke | Wired | Plan 1 Task 17 | — | | `trainers/dqn/monitors/tau_monitor.rs` | Read-only observer for tau_update kernel output (ISV slot 42); consumers: HEALTH_DIAG + controller_activity smoke | Wired | Plan 1 Task 13 | — | diff --git a/infra/k8s/argo/train-multi-seed-template.yaml b/infra/k8s/argo/train-multi-seed-template.yaml index 214294bd9..fa09c88bf 100644 --- a/infra/k8s/argo/train-multi-seed-template.yaml +++ b/infra/k8s/argo/train-multi-seed-template.yaml @@ -1,21 +1,26 @@ -# Multi-seed × multi-fold training workflow — Plan 5 Task 1A. +# Multi-seed training workflow — Plan 5 Task 5 Phase B (one-job-per-seed). # -# Renders an Argo DAG that fans out N seeds × K folds into N*K parallel -# `train-single` task instances. Each task receives `seed` and `fold` via -# inputs.parameters and runs an independent training job sharing the same -# binary cache (per commit SHA) and feature cache (PVC). +# Renders an Argo DAG that fans out N seeds into N parallel `train-single` task +# instances. Each task receives `seed` via inputs.parameters and runs a +# walk-forward training that internally sweeps all K folds via the binary's +# `--max-folds K` arg. Per-job runtime is K× longer than the original (seed, +# fold) matrix but fanout drops from N*K to N — a better fit for the L40S pool +# (5-GPU capacity vs 30 jobs queueing) and a simpler binary contract +# (`train_baseline_rl` is a multi-fold walk-forward executor; it does NOT +# accept `--fold K`). # # The `# __MATRIX_TASKS__` marker on the dag.tasks line is replaced by -# scripts/argo-train.sh with N*K generated WorkflowTask stanzas before -# submission. This avoids hand-writing a 30-task matrix and keeps the -# template human-readable. +# scripts/argo-train.sh with N generated WorkflowTask stanzas before +# submission. This avoids hand-writing the matrix and keeps the template +# human-readable. # # Usage: # ./scripts/argo-train.sh dqn --multi-seed 5 --folds 6 --tag plan5-final # # DAG (per task): # ensure-binary ──┐ -# gpu-warmup ─────┼──> ensure-fxcache ──> [N*K parallel train-single tasks] +# gpu-warmup ─────┼──> ensure-fxcache ──> [N parallel train-single tasks, +# one per seed, each runs all K folds] # │ # └──> aggregate (manual via # scripts/gather-multi-seed-metrics.sh) @@ -42,7 +47,9 @@ spec: fsGroup: 0 ttlStrategy: secondsAfterCompletion: 3600 - # Multi-seed runs: N*K jobs in parallel, allow 12h walltime. + # Multi-seed runs: N jobs in parallel (one per seed), each running all K folds + # in walk-forward sequence. Allow 12h walltime — 6-fold runs are ~6× longer + # than the original per-(seed,fold) jobs but easily fit in 12h on L40S. activeDeadlineSeconds: 43200 arguments: @@ -112,10 +119,11 @@ spec: storage: 5Gi templates: - # ── DAG: fan out to N*K train-single tasks ── + # ── DAG: fan out to N train-single tasks (one per seed) ── # The `# __MATRIX_TASKS__` marker is replaced by argo-train.sh with the - # generated per-(seed, fold) WorkflowTask stanzas. The marker MUST stay - # on its own line for the awk substitution to work. + # generated per-seed WorkflowTask stanzas. Each task sweeps all K folds + # via the binary's `--max-folds {{workflow.parameters.folds}}` argument. + # The marker MUST stay on its own line for the awk substitution to work. - name: multi-seed-matrix dag: tasks: @@ -351,15 +359,18 @@ spec: --yes fi - # ── train-single: one (seed, fold) training instance ── - # Invoked once per matrix entry. Reads SEED/FOLD from inputs.parameters - # and forwards them to the training binary so per-seed determinism and - # per-fold walk-forward windowing happen inside the binary. + # ── train-single: one seed, all folds (training instance) ── + # Invoked once per matrix entry. Reads SEED from inputs.parameters and + # forwards it via `--seed`; the binary's `--max-folds K` arg drives the + # walk-forward sweep over all K folds inside this single process. + # Plan 5 Task 5 Phase B pivot: was per-(seed, fold) on the failed deploy + # because train_baseline_rl does not accept `--fold N` (it is a multi-fold + # executor, not a single-fold one). One-job-per-seed matches the binary's + # actual contract and the L40S pool's capacity. - name: train-single inputs: parameters: - name: seed - - name: fold nodeSelector: k8s.scaleway.com/pool-name: "{{workflow.parameters.gpu-pool}}" tolerations: @@ -384,8 +395,6 @@ spec: value: /feature-cache - name: SEED value: "{{inputs.parameters.seed}}" - - name: FOLD - value: "{{inputs.parameters.fold}}" # Plan 5 Task 3 (A.4.1): MinIO creds for the optional `mc cp` of # the .nsys-rep artefact at the end of train-single. Both refs are # `optional: true` so the env mount succeeds on clusters that do @@ -460,7 +469,7 @@ spec: fi fi - echo "=== Training: $MODEL seed=$SEED fold=$FOLD ({{workflow.parameters.train-epochs}} epochs) ===" + echo "=== Training: $MODEL seed=$SEED folds={{workflow.parameters.folds}} ({{workflow.parameters.train-epochs}} epochs) ===" stdbuf -oL $NSYS_PREFIX ${BINARY} \ --model "$MODEL" \ --symbol {{workflow.parameters.symbol}} \ @@ -474,9 +483,9 @@ spec: --output-dir /workspace/output \ --epochs {{workflow.parameters.train-epochs}} \ --seed "$SEED" \ - --fold "$FOLD" + --max-folds {{workflow.parameters.folds}} - echo "=== Training complete: seed=$SEED fold=$FOLD ===" + echo "=== Training complete: seed=$SEED folds={{workflow.parameters.folds}} ===" # Plan 5 Task 3 (A.4.1): upload .nsys-rep to MinIO if profile run. # mc is fetched on-demand (~25 MB single static binary) — the @@ -496,8 +505,8 @@ spec: "$MC_BIN" alias set foxhunt http://minio.foxhunt.svc.cluster.local:9000 \ "$MINIO_ACCESS_KEY" "$MINIO_SECRET_KEY" 2>/dev/null || true "$MC_BIN" cp "$NSYS_OUT" \ - "foxhunt/foxhunt-training-artifacts/profiles/$SHA/profile-seed${SEED}-fold${FOLD}-${POD_NAME}.nsys-rep" || \ - echo "WARN: nsys upload failed for seed=$SEED fold=$FOLD" + "foxhunt/foxhunt-training-artifacts/profiles/$SHA/profile-seed${SEED}-${POD_NAME}.nsys-rep" || \ + echo "WARN: nsys upload failed for seed=$SEED" echo "=== nsys profile upload complete ===" fi diff --git a/scripts/argo-train.sh b/scripts/argo-train.sh index a7755f1c1..c0eeb88fa 100755 --- a/scripts/argo-train.sh +++ b/scripts/argo-train.sh @@ -170,12 +170,16 @@ if [[ "$USE_MULTI_SEED" == "false" ]]; then exit 0 fi -# ── Multi-seed × multi-fold path ── -# Render the train-multi-seed-template.yaml with the (seed, fold) matrix -# expanded inline. The base template ships with a placeholder marker -# (`# __MATRIX_TASKS__`) which we replace with N*K generated WorkflowTask -# stanzas. This avoids hand-writing a 30-task matrix and keeps the source -# template human-readable. +# ── Multi-seed path (one job per seed; folds run inside the binary) ── +# Render the train-multi-seed-template.yaml with the per-seed matrix +# expanded inline. Plan 5 Task 5 Phase B pivot: was N*K (seed,fold) jobs; +# is now N (seed-only) jobs because `train_baseline_rl` is a multi-fold +# walk-forward executor — it accepts `--max-folds K`, NOT `--fold K`. Each +# rendered task invokes the binary with `--seed N --max-folds K` so the +# walk-forward sweep happens inside the single training process. +# +# The base template ships with a placeholder marker (`# __MATRIX_TASKS__`) +# which we replace with N generated WorkflowTask stanzas. TEMPLATE_SRC="infra/k8s/argo/train-multi-seed-template.yaml" if [[ ! -f "$TEMPLATE_SRC" ]]; then echo "Error: multi-seed template not found at $TEMPLATE_SRC" @@ -183,34 +187,30 @@ if [[ ! -f "$TEMPLATE_SRC" ]]; then fi # Build matrix YAML. Each task is a dag.tasks[] entry that targets the -# `train-single` template with seed/fold parameters bound from inputs. +# `train-single` template with the `seed` parameter bound from inputs. +# The template forwards `seed` as `--seed` and reads `--max-folds` from the +# workflow-scoped `folds` parameter, so each task internally runs all K folds. build_matrix_tasks() { local seeds="$1" - local folds="$2" # 10 spaces — matches the sibling `- name: ensure-binary` list-item indent # under `dag.tasks:` (which is itself at 8 spaces). Wrong indent here # produces a YAML parse error in the rendered template. local indent=" " local s - local f for ((s=0; s&1) actual_count=$(echo "$output" | grep -c "kind: WorkflowTask" || true) -expected=6 +expected=3 if [[ "$actual_count" -ne "$expected" ]]; then - echo "FAIL: expected $expected jobs (3 seeds x 2 folds), got $actual_count" + echo "FAIL: expected $expected jobs (3 seeds, fold sweep inside binary), got $actual_count" echo "--- output ---" echo "$output" exit 1 fi -echo "PASS: multi-seed harness produces $expected jobs for 3x2 matrix" +echo "PASS: multi-seed harness produces $expected per-seed jobs for --multi-seed 3 --folds 2" + +# The rendered binary command must include `--max-folds {{workflow.parameters.folds}}` +# (drives the walk-forward sweep inside the single training process) and must +# NOT include any per-fold flag — `train_baseline_rl` rejects `--fold`, which +# is what broke the first L40S deploy attempt (workflow train-multi-seed-z2llf, +# every job exited at startup with `error: unexpected argument '--fold' found`). +# The Argo `{{workflow.parameters.folds}}` placeholder is resolved at +# workflow submission time (argo-train.sh passes `-p folds=$FOLDS` to argo +# submit), not at template render time, so the dry-run shows the placeholder. +if ! echo "$output" | grep -qE -- "--max-folds[[:space:]]+\{\{workflow\.parameters\.folds\}\}"; then + echo "FAIL: rendered binary command missing '--max-folds {{workflow.parameters.folds}}'" + echo "--- output (rendered template, first 300 lines) ---" + echo "$output" | head -300 + exit 1 +fi +echo "PASS: rendered binary invocation includes --max-folds placeholder" + +# Folds parameter must be declared at the workflow level so `argo submit -p +# folds=K` (which argo-train.sh emits) can override the default at submit time. +if ! echo "$output" | grep -qE "^[[:space:]]*-[[:space:]]*name:[[:space:]]*folds[[:space:]]*$"; then + echo "FAIL: rendered template does not declare 'folds' workflow parameter" + exit 1 +fi +echo "PASS: rendered template declares folds workflow parameter" + +if echo "$output" | grep -qE -- '--fold[[:space:]]+"?[0-9{]'; then + echo "FAIL: rendered binary command still contains a per-fold flag (--fold N)" + echo " train_baseline_rl rejects --fold; only --max-folds is supported." + echo "--- offending lines ---" + echo "$output" | grep -E -- '--fold[[:space:]]+"?[0-9{]' + exit 1 +fi +echo "PASS: rendered binary invocation has no per-fold flag (Path B compliance)" # Backward-compat: --multi-seed 1 --folds 1 must NOT emit WorkflowTask lines # (single-job path uses the existing template, no DAG matrix).