diff --git a/bin/fxt-backtest/src/main.rs b/bin/fxt-backtest/src/main.rs index 21f9ec346..5499c6ae7 100644 --- a/bin/fxt-backtest/src/main.rs +++ b/bin/fxt-backtest/src/main.rs @@ -174,7 +174,17 @@ struct SweepGrid { #[derive(Debug, serde::Deserialize, Clone)] struct SweepBase { - data: PathBuf, + /// Single data file path. Used by the legacy fan-out flow (one Argo + /// task per cell). For the P6 batched flow that varies window per + /// cell, prefer `data_template` with `{window}` interpolation. + #[serde(default)] + data: Option, + /// P6: data path template with `{window}` placeholder. When set, + /// each cell's `window` field interpolates into this template to + /// produce the per-cell data path. + /// Example: /mnt/training-data/futures-baseline-mbp10/ES.FUT/ES.FUT_{window}.dbn.zst + #[serde(default)] + data_template: Option, #[serde(default)] predecoded_dir: Option, #[serde(default = "default_n_parallel")] n_parallel: usize, @@ -188,6 +198,28 @@ struct SweepBase { #[serde(default)] checkpoint: Option, #[serde(default = "default_kelly_frac_floor")] kelly_frac_floor: f32, #[serde(default = "default_sharpe_weight_floor")] sharpe_weight_floor: f32, + /// P6: list of sim variants. When non-empty, each cell runs ONE + /// harness at n_parallel=variants.len() instead of fanning out one + /// harness per cell. Cells become a window axis; variants become the + /// (cost, latency, threshold) axis. + #[serde(default)] sim_variants: Vec, +} + +/// P6: per-sim-variant overrides for the batched-cell sweep flow. +/// `threshold` and `cost_per_lot_per_side` are required (no defaults — +/// these are the spec's primary sweep axes); other fields are optional +/// overrides on top of the SweepBase scalars. +#[derive(Debug, serde::Deserialize, Clone)] +struct SimVariant { + name: String, + threshold: f32, + cost_per_lot_per_side: f32, + #[serde(default)] latency_ns: Option, + #[serde(default)] target_annual_vol_units: Option, + #[serde(default)] annualisation_factor: Option, + #[serde(default)] max_lots: Option, + #[serde(default)] kelly_frac_floor: Option, + #[serde(default)] sharpe_weight_floor: Option, } fn default_n_parallel() -> usize { 1 } @@ -210,6 +242,10 @@ fn default_sharpe_weight_floor() -> f32 { 0.10 } #[derive(Debug, serde::Deserialize, Clone)] struct SweepCell { name: String, + /// P6: window identifier (e.g., "2025-Q2") substituted into + /// SweepBase.data_template. When set, the cell's data path is + /// data_template.replace("{window}", window). + #[serde(default)] window: Option, #[serde(default)] decision_stride: Option, #[serde(default)] latency_ns: Option, #[serde(default)] target_annual_vol_units: Option, @@ -280,6 +316,15 @@ fn sweep(args: SweepArgs) -> Result<()> { std::fs::create_dir_all(&args.out) .with_context(|| format!("create sweep out {}", args.out.display()))?; + // P6: resolve sim variants once. When non-empty, each cell runs ONE + // harness at n_parallel=variants.len() with BatchedSimConfig::from_grid. + let resolved_variants = if grid.base.sim_variants.is_empty() { + Vec::new() + } else { + resolve_sim_variants(&grid.base) + }; + let batched_mode = !resolved_variants.is_empty(); + for (i, cell) in grid.cells.iter().take(n_cells).enumerate() { let cell_out = args.out.join(&cell.name); tracing::info!( @@ -287,8 +332,31 @@ fn sweep(args: SweepArgs) -> Result<()> { progress = format!("{}/{}", i + 1, n_cells), "running sweep cell" ); + // Resolve per-cell data path. data_template + cell.window interpolation + // takes precedence over scalar `data` when both are present. + let cell_data = match (&grid.base.data_template, &cell.window) { + (Some(tpl), Some(window)) => PathBuf::from(tpl.replace("{window}", window)), + _ => grid.base.data.clone() + .ok_or_else(|| anyhow::anyhow!( + "sweep base must set either `data` (legacy) or `data_template + cell.window`" + ))?, + }; + if batched_mode { + // P6 batched flow: build BatchedSimConfig from grid, run ONE + // harness at n_parallel=variants.len() with variant_names plumbed + // through for sim_/ output dirs. + run_batched_cell( + &resolved_variants, + cell, + &grid.base, + &cell_data, + &cell_out, + )?; + tracing::info!(cell = cell.name.as_str(), "batched cell complete"); + continue; + } let run_args = RunArgs { - data: grid.base.data.clone(), + data: cell_data, predecoded_dir: grid.base.predecoded_dir.clone(), n_parallel: grid.base.n_parallel, policy_grid: None, // sweep cells use the default policy; nested @@ -325,6 +393,84 @@ fn sweep(args: SweepArgs) -> Result<()> { Ok(()) } +/// P6: resolve sim_variants from SweepBase, layering per-variant overrides +/// over base scalars. Produces a flat Vec suitable for +/// BatchedSimConfig::from_grid. +fn resolve_sim_variants(base: &SweepBase) -> Vec { + base.sim_variants.iter().map(|v| ml_backtesting::sim::ResolvedSimVariant { + name: v.name.clone(), + target_annual_vol_units: v.target_annual_vol_units.unwrap_or(base.target_annual_vol_units), + annualisation_factor: v.annualisation_factor.unwrap_or(base.annualisation_factor), + max_lots: v.max_lots.unwrap_or(base.max_lots), + latency_ns: v.latency_ns.unwrap_or(base.latency_ns), + kelly_frac_floor: v.kelly_frac_floor.unwrap_or(base.kelly_frac_floor), + sharpe_weight_floor: v.sharpe_weight_floor.unwrap_or(base.sharpe_weight_floor), + threshold: v.threshold, + cost_per_lot_per_side: v.cost_per_lot_per_side, + }).collect() +} + +/// P6: run ONE cell with n_parallel=variants.len() backtests sharing the +/// forward pass. Each backtest gets distinct (cost, latency, threshold, +/// ...) from its ResolvedSimVariant. Output dirs are named `sim_` +/// per spec §3.3. +fn run_batched_cell( + variants: &[ml_backtesting::sim::ResolvedSimVariant], + cell: &SweepCell, + base: &SweepBase, + cell_data: &std::path::Path, + cell_out: &std::path::Path, +) -> Result<()> { + use ml_alpha::trainer::perception::{PerceptionTrainer, PerceptionTrainerConfig}; + use ml_core::device::MlDevice; + let n = variants.len(); + let names: Vec = variants.iter().map(|v| v.name.clone()).collect(); + let batched = ml_backtesting::sim::BatchedSimConfig::from_grid(variants); + + let dev = MlDevice::cuda(0).context("CUDA device unavailable")?; + let seed = cell.seed.unwrap_or(base.seed); + let trainer_cfg = PerceptionTrainerConfig { seed, n_batch: 1, ..Default::default() }; + let checkpoint = cell.checkpoint.clone().or_else(|| base.checkpoint.clone()); + let trainer = if let Some(ckpt_path) = &checkpoint { + PerceptionTrainer::from_checkpoint(&dev, &trainer_cfg, ckpt_path) + .with_context(|| format!("load checkpoint {}", ckpt_path.display()))? + } else { + let mut cfg2 = trainer_cfg.clone(); + cfg2.seed = seed; + PerceptionTrainer::new(&dev, &cfg2).context("PerceptionTrainer::new (random init)")? + }; + + let cfg = BacktestHarnessConfig { + data_root: cell_data.to_path_buf(), + predecoded_dir: base.predecoded_dir.clone().unwrap_or_else(|| cell_data.to_path_buf()), + n_parallel: n, + decision_stride: cell.decision_stride.unwrap_or(base.decision_stride), + target_annual_vol_units: base.target_annual_vol_units, // overridden by sim_config + annualisation_factor: base.annualisation_factor, + max_lots: base.max_lots, + max_events: cell.max_events.unwrap_or(base.max_events), + latency_ns: base.latency_ns, + kelly_frac_floor: base.kelly_frac_floor, + sharpe_weight_floor: base.sharpe_weight_floor, + threshold: 0.0, // overridden by sim_config + cost_per_lot_per_side: 0.0, // overridden by sim_config + variant_names: Some(names), + sim_config_override: Some(batched), + strategies: Vec::new(), // batched flow uses default policy per backtest + }; + + std::fs::create_dir_all(cell_out) + .with_context(|| format!("create cell out {}", cell_out.display()))?; + let mut harness = BacktestHarness::new(cfg, &dev, trainer).context("BacktestHarness::new (batched)")?; + let stats = harness.run().context("BacktestHarness::run (batched)")?; + harness.write_artifacts(cell_out).context("write_artifacts (batched)")?; + println!( + "batched cell {} done: {} events, {} decisions, {} variants → {}", + cell.name, stats.events_processed, stats.decisions_taken, n, cell_out.display(), + ); + Ok(()) +} + fn run(args: RunArgs) -> Result<()> { // Resolve strategies — either from YAML or default_for. let strategies: Vec = if let Some(grid_path) = &args.policy_grid { @@ -395,6 +541,8 @@ fn run(args: RunArgs) -> Result<()> { sharpe_weight_floor: args.sharpe_weight_floor, threshold: 0.0, // P4: default = gate disabled cost_per_lot_per_side: 0.0, // P4: default = frictionless + variant_names: None, // P6: legacy single-cell flow uses cell_NNNN naming + sim_config_override: None, // P6: legacy single-cell flow uses from_uniform strategies: strategy_grid, }; diff --git a/config/ml/sweep_deployability.yaml b/config/ml/sweep_deployability.yaml index 513d43991..12330f11d 100644 --- a/config/ml/sweep_deployability.yaml +++ b/config/ml/sweep_deployability.yaml @@ -1,22 +1,164 @@ -# Spec §3.6 + §3.1 — 4 walk-forward held-out windows × full grid. -# 7 × 4 × 5 × 4 = 560 cells. Fan out via scripts/argo-lob-sweep.sh. -# Verdict reads the cells at the realistic (1.0 tick, 200 ms) and stress -# (1.5 tick, 400 ms) anchors against the pre-registered threshold from -# config/ml/v2_prod_thresholds.json. -checkpoint: /feature-cache/alpha-perception-runs/__SHA__/trunk_best_h6000.bin -windows: - - id: W1 - dbn_file: /data/futures-baseline-mbp10/ES.FUT/ES.FUT_2025-Q2.dbn.zst - - id: W2 - dbn_file: /data/futures-baseline-mbp10/ES.FUT/ES.FUT_2025-Q3.dbn.zst - - id: W3 - dbn_file: /data/futures-baseline-mbp10/ES.FUT/ES.FUT_2025-Q4.dbn.zst - - id: W4 - dbn_file: /data/futures-baseline-mbp10/ES.FUT/ES.FUT_2026-Q1.dbn.zst -axes: - cost_tick: [0.0, 0.25, 0.5, 0.75, 1.0, 1.25, 1.5] - latency_ms: [50, 100, 200, 400] - threshold: [0.70, 0.75, 0.80, 0.85, 0.90] -decision_stride: 1 -horizons_used: [h6000] -gpu_pool: ci-training-l40s +# P6 deployability sweep — 4 windows × 140 sim_variants (7 cost × 4 latency × 5 threshold). +# Generated by scripts/generate_sweep_variants.py — do NOT hand-edit. +# Re-run after threshold-tuning to refresh threshold absolute values. + +base: + data_template: /mnt/training-data/futures-baseline-mbp10/ES.FUT/ES.FUT_{window}.dbn.zst + predecoded_dir: /feature-cache/predecoded + n_parallel: 1 # batched flow overrides with variants.len()=140 + decision_stride: 4 + latency_ns: 200000000 # default (overridden per variant) + target_annual_vol_units: 50.0 + annualisation_factor: 825.0 + max_lots: 5 + max_events: 0 + seed: 12648430 + # __SHA__ replaced by submitter with post-trunk-grows training short-SHA. + checkpoint: /feature-cache/alpha-perception-runs/__SHA__/trunk_best_h6000.bin + sim_variants: + - { name: c0_l0_p60, threshold: 0.4, cost_per_lot_per_side: 0.0625, latency_ns: 100000000 } + - { name: c0_l0_p70, threshold: 0.55, cost_per_lot_per_side: 0.0625, latency_ns: 100000000 } + - { name: c0_l0_p80, threshold: 0.7, cost_per_lot_per_side: 0.0625, latency_ns: 100000000 } + - { name: c0_l0_p90, threshold: 0.85, cost_per_lot_per_side: 0.0625, latency_ns: 100000000 } + - { name: c0_l0_p95, threshold: 0.95, cost_per_lot_per_side: 0.0625, latency_ns: 100000000 } + - { name: c0_l1_p60, threshold: 0.4, cost_per_lot_per_side: 0.0625, latency_ns: 200000000 } + - { name: c0_l1_p70, threshold: 0.55, cost_per_lot_per_side: 0.0625, latency_ns: 200000000 } + - { name: c0_l1_p80, threshold: 0.7, cost_per_lot_per_side: 0.0625, latency_ns: 200000000 } + - { name: c0_l1_p90, threshold: 0.85, cost_per_lot_per_side: 0.0625, latency_ns: 200000000 } + - { name: c0_l1_p95, threshold: 0.95, cost_per_lot_per_side: 0.0625, latency_ns: 200000000 } + - { name: c0_l2_p60, threshold: 0.4, cost_per_lot_per_side: 0.0625, latency_ns: 300000000 } + - { name: c0_l2_p70, threshold: 0.55, cost_per_lot_per_side: 0.0625, latency_ns: 300000000 } + - { name: c0_l2_p80, threshold: 0.7, cost_per_lot_per_side: 0.0625, latency_ns: 300000000 } + - { name: c0_l2_p90, threshold: 0.85, cost_per_lot_per_side: 0.0625, latency_ns: 300000000 } + - { name: c0_l2_p95, threshold: 0.95, cost_per_lot_per_side: 0.0625, latency_ns: 300000000 } + - { name: c0_l3_p60, threshold: 0.4, cost_per_lot_per_side: 0.0625, latency_ns: 400000000 } + - { name: c0_l3_p70, threshold: 0.55, cost_per_lot_per_side: 0.0625, latency_ns: 400000000 } + - { name: c0_l3_p80, threshold: 0.7, cost_per_lot_per_side: 0.0625, latency_ns: 400000000 } + - { name: c0_l3_p90, threshold: 0.85, cost_per_lot_per_side: 0.0625, latency_ns: 400000000 } + - { name: c0_l3_p95, threshold: 0.95, cost_per_lot_per_side: 0.0625, latency_ns: 400000000 } + - { name: c1_l0_p60, threshold: 0.4, cost_per_lot_per_side: 0.125, latency_ns: 100000000 } + - { name: c1_l0_p70, threshold: 0.55, cost_per_lot_per_side: 0.125, latency_ns: 100000000 } + - { name: c1_l0_p80, threshold: 0.7, cost_per_lot_per_side: 0.125, latency_ns: 100000000 } + - { name: c1_l0_p90, threshold: 0.85, cost_per_lot_per_side: 0.125, latency_ns: 100000000 } + - { name: c1_l0_p95, threshold: 0.95, cost_per_lot_per_side: 0.125, latency_ns: 100000000 } + - { name: c1_l1_p60, threshold: 0.4, cost_per_lot_per_side: 0.125, latency_ns: 200000000 } + - { name: c1_l1_p70, threshold: 0.55, cost_per_lot_per_side: 0.125, latency_ns: 200000000 } + - { name: c1_l1_p80, threshold: 0.7, cost_per_lot_per_side: 0.125, latency_ns: 200000000 } + - { name: c1_l1_p90, threshold: 0.85, cost_per_lot_per_side: 0.125, latency_ns: 200000000 } + - { name: c1_l1_p95, threshold: 0.95, cost_per_lot_per_side: 0.125, latency_ns: 200000000 } + - { name: c1_l2_p60, threshold: 0.4, cost_per_lot_per_side: 0.125, latency_ns: 300000000 } + - { name: c1_l2_p70, threshold: 0.55, cost_per_lot_per_side: 0.125, latency_ns: 300000000 } + - { name: c1_l2_p80, threshold: 0.7, cost_per_lot_per_side: 0.125, latency_ns: 300000000 } + - { name: c1_l2_p90, threshold: 0.85, cost_per_lot_per_side: 0.125, latency_ns: 300000000 } + - { name: c1_l2_p95, threshold: 0.95, cost_per_lot_per_side: 0.125, latency_ns: 300000000 } + - { name: c1_l3_p60, threshold: 0.4, cost_per_lot_per_side: 0.125, latency_ns: 400000000 } + - { name: c1_l3_p70, threshold: 0.55, cost_per_lot_per_side: 0.125, latency_ns: 400000000 } + - { name: c1_l3_p80, threshold: 0.7, cost_per_lot_per_side: 0.125, latency_ns: 400000000 } + - { name: c1_l3_p90, threshold: 0.85, cost_per_lot_per_side: 0.125, latency_ns: 400000000 } + - { name: c1_l3_p95, threshold: 0.95, cost_per_lot_per_side: 0.125, latency_ns: 400000000 } + - { name: c2_l0_p60, threshold: 0.4, cost_per_lot_per_side: 0.1875, latency_ns: 100000000 } + - { name: c2_l0_p70, threshold: 0.55, cost_per_lot_per_side: 0.1875, latency_ns: 100000000 } + - { name: c2_l0_p80, threshold: 0.7, cost_per_lot_per_side: 0.1875, latency_ns: 100000000 } + - { name: c2_l0_p90, threshold: 0.85, cost_per_lot_per_side: 0.1875, latency_ns: 100000000 } + - { name: c2_l0_p95, threshold: 0.95, cost_per_lot_per_side: 0.1875, latency_ns: 100000000 } + - { name: c2_l1_p60, threshold: 0.4, cost_per_lot_per_side: 0.1875, latency_ns: 200000000 } + - { name: c2_l1_p70, threshold: 0.55, cost_per_lot_per_side: 0.1875, latency_ns: 200000000 } + - { name: c2_l1_p80, threshold: 0.7, cost_per_lot_per_side: 0.1875, latency_ns: 200000000 } + - { name: c2_l1_p90, threshold: 0.85, cost_per_lot_per_side: 0.1875, latency_ns: 200000000 } + - { name: c2_l1_p95, threshold: 0.95, cost_per_lot_per_side: 0.1875, latency_ns: 200000000 } + - { name: c2_l2_p60, threshold: 0.4, cost_per_lot_per_side: 0.1875, latency_ns: 300000000 } + - { name: c2_l2_p70, threshold: 0.55, cost_per_lot_per_side: 0.1875, latency_ns: 300000000 } + - { name: c2_l2_p80, threshold: 0.7, cost_per_lot_per_side: 0.1875, latency_ns: 300000000 } + - { name: c2_l2_p90, threshold: 0.85, cost_per_lot_per_side: 0.1875, latency_ns: 300000000 } + - { name: c2_l2_p95, threshold: 0.95, cost_per_lot_per_side: 0.1875, latency_ns: 300000000 } + - { name: c2_l3_p60, threshold: 0.4, cost_per_lot_per_side: 0.1875, latency_ns: 400000000 } + - { name: c2_l3_p70, threshold: 0.55, cost_per_lot_per_side: 0.1875, latency_ns: 400000000 } + - { name: c2_l3_p80, threshold: 0.7, cost_per_lot_per_side: 0.1875, latency_ns: 400000000 } + - { name: c2_l3_p90, threshold: 0.85, cost_per_lot_per_side: 0.1875, latency_ns: 400000000 } + - { name: c2_l3_p95, threshold: 0.95, cost_per_lot_per_side: 0.1875, latency_ns: 400000000 } + - { name: c3_l0_p60, threshold: 0.4, cost_per_lot_per_side: 0.25, latency_ns: 100000000 } + - { name: c3_l0_p70, threshold: 0.55, cost_per_lot_per_side: 0.25, latency_ns: 100000000 } + - { name: c3_l0_p80, threshold: 0.7, cost_per_lot_per_side: 0.25, latency_ns: 100000000 } + - { name: c3_l0_p90, threshold: 0.85, cost_per_lot_per_side: 0.25, latency_ns: 100000000 } + - { name: c3_l0_p95, threshold: 0.95, cost_per_lot_per_side: 0.25, latency_ns: 100000000 } + - { name: c3_l1_p60, threshold: 0.4, cost_per_lot_per_side: 0.25, latency_ns: 200000000 } + - { name: c3_l1_p70, threshold: 0.55, cost_per_lot_per_side: 0.25, latency_ns: 200000000 } + - { name: c3_l1_p80, threshold: 0.7, cost_per_lot_per_side: 0.25, latency_ns: 200000000 } + - { name: c3_l1_p90, threshold: 0.85, cost_per_lot_per_side: 0.25, latency_ns: 200000000 } + - { name: c3_l1_p95, threshold: 0.95, cost_per_lot_per_side: 0.25, latency_ns: 200000000 } + - { name: c3_l2_p60, threshold: 0.4, cost_per_lot_per_side: 0.25, latency_ns: 300000000 } + - { name: c3_l2_p70, threshold: 0.55, cost_per_lot_per_side: 0.25, latency_ns: 300000000 } + - { name: c3_l2_p80, threshold: 0.7, cost_per_lot_per_side: 0.25, latency_ns: 300000000 } + - { name: c3_l2_p90, threshold: 0.85, cost_per_lot_per_side: 0.25, latency_ns: 300000000 } + - { name: c3_l2_p95, threshold: 0.95, cost_per_lot_per_side: 0.25, latency_ns: 300000000 } + - { name: c3_l3_p60, threshold: 0.4, cost_per_lot_per_side: 0.25, latency_ns: 400000000 } + - { name: c3_l3_p70, threshold: 0.55, cost_per_lot_per_side: 0.25, latency_ns: 400000000 } + - { name: c3_l3_p80, threshold: 0.7, cost_per_lot_per_side: 0.25, latency_ns: 400000000 } + - { name: c3_l3_p90, threshold: 0.85, cost_per_lot_per_side: 0.25, latency_ns: 400000000 } + - { name: c3_l3_p95, threshold: 0.95, cost_per_lot_per_side: 0.25, latency_ns: 400000000 } + - { name: c4_l0_p60, threshold: 0.4, cost_per_lot_per_side: 0.375, latency_ns: 100000000 } + - { name: c4_l0_p70, threshold: 0.55, cost_per_lot_per_side: 0.375, latency_ns: 100000000 } + - { name: c4_l0_p80, threshold: 0.7, cost_per_lot_per_side: 0.375, latency_ns: 100000000 } + - { name: c4_l0_p90, threshold: 0.85, cost_per_lot_per_side: 0.375, latency_ns: 100000000 } + - { name: c4_l0_p95, threshold: 0.95, cost_per_lot_per_side: 0.375, latency_ns: 100000000 } + - { name: c4_l1_p60, threshold: 0.4, cost_per_lot_per_side: 0.375, latency_ns: 200000000 } + - { name: c4_l1_p70, threshold: 0.55, cost_per_lot_per_side: 0.375, latency_ns: 200000000 } + - { name: c4_l1_p80, threshold: 0.7, cost_per_lot_per_side: 0.375, latency_ns: 200000000 } + - { name: c4_l1_p90, threshold: 0.85, cost_per_lot_per_side: 0.375, latency_ns: 200000000 } + - { name: c4_l1_p95, threshold: 0.95, cost_per_lot_per_side: 0.375, latency_ns: 200000000 } + - { name: c4_l2_p60, threshold: 0.4, cost_per_lot_per_side: 0.375, latency_ns: 300000000 } + - { name: c4_l2_p70, threshold: 0.55, cost_per_lot_per_side: 0.375, latency_ns: 300000000 } + - { name: c4_l2_p80, threshold: 0.7, cost_per_lot_per_side: 0.375, latency_ns: 300000000 } + - { name: c4_l2_p90, threshold: 0.85, cost_per_lot_per_side: 0.375, latency_ns: 300000000 } + - { name: c4_l2_p95, threshold: 0.95, cost_per_lot_per_side: 0.375, latency_ns: 300000000 } + - { name: c4_l3_p60, threshold: 0.4, cost_per_lot_per_side: 0.375, latency_ns: 400000000 } + - { name: c4_l3_p70, threshold: 0.55, cost_per_lot_per_side: 0.375, latency_ns: 400000000 } + - { name: c4_l3_p80, threshold: 0.7, cost_per_lot_per_side: 0.375, latency_ns: 400000000 } + - { name: c4_l3_p90, threshold: 0.85, cost_per_lot_per_side: 0.375, latency_ns: 400000000 } + - { name: c4_l3_p95, threshold: 0.95, cost_per_lot_per_side: 0.375, latency_ns: 400000000 } + - { name: c5_l0_p60, threshold: 0.4, cost_per_lot_per_side: 0.5, latency_ns: 100000000 } + - { name: c5_l0_p70, threshold: 0.55, cost_per_lot_per_side: 0.5, latency_ns: 100000000 } + - { name: c5_l0_p80, threshold: 0.7, cost_per_lot_per_side: 0.5, latency_ns: 100000000 } + - { name: c5_l0_p90, threshold: 0.85, cost_per_lot_per_side: 0.5, latency_ns: 100000000 } + - { name: c5_l0_p95, threshold: 0.95, cost_per_lot_per_side: 0.5, latency_ns: 100000000 } + - { name: c5_l1_p60, threshold: 0.4, cost_per_lot_per_side: 0.5, latency_ns: 200000000 } + - { name: c5_l1_p70, threshold: 0.55, cost_per_lot_per_side: 0.5, latency_ns: 200000000 } + - { name: c5_l1_p80, threshold: 0.7, cost_per_lot_per_side: 0.5, latency_ns: 200000000 } + - { name: c5_l1_p90, threshold: 0.85, cost_per_lot_per_side: 0.5, latency_ns: 200000000 } + - { name: c5_l1_p95, threshold: 0.95, cost_per_lot_per_side: 0.5, latency_ns: 200000000 } + - { name: c5_l2_p60, threshold: 0.4, cost_per_lot_per_side: 0.5, latency_ns: 300000000 } + - { name: c5_l2_p70, threshold: 0.55, cost_per_lot_per_side: 0.5, latency_ns: 300000000 } + - { name: c5_l2_p80, threshold: 0.7, cost_per_lot_per_side: 0.5, latency_ns: 300000000 } + - { name: c5_l2_p90, threshold: 0.85, cost_per_lot_per_side: 0.5, latency_ns: 300000000 } + - { name: c5_l2_p95, threshold: 0.95, cost_per_lot_per_side: 0.5, latency_ns: 300000000 } + - { name: c5_l3_p60, threshold: 0.4, cost_per_lot_per_side: 0.5, latency_ns: 400000000 } + - { name: c5_l3_p70, threshold: 0.55, cost_per_lot_per_side: 0.5, latency_ns: 400000000 } + - { name: c5_l3_p80, threshold: 0.7, cost_per_lot_per_side: 0.5, latency_ns: 400000000 } + - { name: c5_l3_p90, threshold: 0.85, cost_per_lot_per_side: 0.5, latency_ns: 400000000 } + - { name: c5_l3_p95, threshold: 0.95, cost_per_lot_per_side: 0.5, latency_ns: 400000000 } + - { name: c6_l0_p60, threshold: 0.4, cost_per_lot_per_side: 1.0, latency_ns: 100000000 } + - { name: c6_l0_p70, threshold: 0.55, cost_per_lot_per_side: 1.0, latency_ns: 100000000 } + - { name: c6_l0_p80, threshold: 0.7, cost_per_lot_per_side: 1.0, latency_ns: 100000000 } + - { name: c6_l0_p90, threshold: 0.85, cost_per_lot_per_side: 1.0, latency_ns: 100000000 } + - { name: c6_l0_p95, threshold: 0.95, cost_per_lot_per_side: 1.0, latency_ns: 100000000 } + - { name: c6_l1_p60, threshold: 0.4, cost_per_lot_per_side: 1.0, latency_ns: 200000000 } + - { name: c6_l1_p70, threshold: 0.55, cost_per_lot_per_side: 1.0, latency_ns: 200000000 } + - { name: c6_l1_p80, threshold: 0.7, cost_per_lot_per_side: 1.0, latency_ns: 200000000 } + - { name: c6_l1_p90, threshold: 0.85, cost_per_lot_per_side: 1.0, latency_ns: 200000000 } + - { name: c6_l1_p95, threshold: 0.95, cost_per_lot_per_side: 1.0, latency_ns: 200000000 } + - { name: c6_l2_p60, threshold: 0.4, cost_per_lot_per_side: 1.0, latency_ns: 300000000 } + - { name: c6_l2_p70, threshold: 0.55, cost_per_lot_per_side: 1.0, latency_ns: 300000000 } + - { name: c6_l2_p80, threshold: 0.7, cost_per_lot_per_side: 1.0, latency_ns: 300000000 } + - { name: c6_l2_p90, threshold: 0.85, cost_per_lot_per_side: 1.0, latency_ns: 300000000 } + - { name: c6_l2_p95, threshold: 0.95, cost_per_lot_per_side: 1.0, latency_ns: 300000000 } + - { name: c6_l3_p60, threshold: 0.4, cost_per_lot_per_side: 1.0, latency_ns: 400000000 } + - { name: c6_l3_p70, threshold: 0.55, cost_per_lot_per_side: 1.0, latency_ns: 400000000 } + - { name: c6_l3_p80, threshold: 0.7, cost_per_lot_per_side: 1.0, latency_ns: 400000000 } + - { name: c6_l3_p90, threshold: 0.85, cost_per_lot_per_side: 1.0, latency_ns: 400000000 } + - { name: c6_l3_p95, threshold: 0.95, cost_per_lot_per_side: 1.0, latency_ns: 400000000 } + +cells: + - { name: W1, window: 2025-Q2 } + - { name: W2, window: 2025-Q3 } + - { name: W3, window: 2025-Q4 } + - { name: W4, window: 2026-Q1 } diff --git a/config/ml/sweep_threshold_tuning.yaml b/config/ml/sweep_threshold_tuning.yaml index d2590ad95..3abdebf3e 100644 --- a/config/ml/sweep_threshold_tuning.yaml +++ b/config/ml/sweep_threshold_tuning.yaml @@ -1,15 +1,37 @@ -# Spec §3.3 — threshold pre-registration on W0 only. -# Single-process fxt-backtest sweep (not Argo fan-out). 8 cells. -# Selects the production threshold by maximising in-sample Sharpe. -checkpoint: /feature-cache/alpha-perception-runs/__SHA__/trunk_best_h6000.bin -windows: - - id: W0 - dbn_file: /data/futures-baseline-mbp10/ES.FUT/ES.FUT_2025-Q1.dbn.zst -axes: - cost_tick: [1.0] # anchor cost (1 tick all-in) - latency_ms: [200] # anchor latency (Scaleway PAR → IBKR realistic RTT) - threshold: [0.60, 0.65, 0.70, 0.75, 0.80, 0.85, 0.90, 0.95] -decision_stride: 1 -horizons_used: [h6000] -# Selected threshold is persisted to config/ml/v2_prod_thresholds.json -# and committed before the deployability sweep runs. +# P6 batched flow — threshold pre-registration on W0 only. +# Single-cell, 8 sim variants (one per threshold) sharing one forward pass +# on the validation window. Selects the production threshold value by +# maximising in-sample Sharpe; selected value is persisted to +# config/ml/v2_prod_thresholds.json and committed before the deployability +# sweep runs. + +base: + # P6 batched flow: data_template + cell.window interpolation produces the per-cell data path. + data_template: /mnt/training-data/futures-baseline-mbp10/ES.FUT/ES.FUT_{window}.dbn.zst + predecoded_dir: /feature-cache/predecoded + n_parallel: 1 # legacy default; overridden by variants.len() per cell + decision_stride: 4 + latency_ns: 200000000 # anchor (Scaleway PAR → IBKR realistic RTT) + target_annual_vol_units: 50.0 + annualisation_factor: 825.0 + max_lots: 5 + max_events: 0 # exhaust loader (full quarter) + seed: 12648430 + # __SHA__ is replaced by argo-lob-sweep.sh / the operator at submission time + # with the post-trunk-grows training short-SHA (9 chars). + checkpoint: /feature-cache/alpha-perception-runs/__SHA__/trunk_best_h6000.bin + # 8 thresholds spanning p60-p95 (5pt steps). Cost held at the realistic + # anchor (1 tick = 0.125 price units) so threshold-tuning Sharpe reflects + # what the deployability sweep will see at that cost. + sim_variants: + - { name: t60, threshold: 0.60, cost_per_lot_per_side: 0.125 } + - { name: t65, threshold: 0.65, cost_per_lot_per_side: 0.125 } + - { name: t70, threshold: 0.70, cost_per_lot_per_side: 0.125 } + - { name: t75, threshold: 0.75, cost_per_lot_per_side: 0.125 } + - { name: t80, threshold: 0.80, cost_per_lot_per_side: 0.125 } + - { name: t85, threshold: 0.85, cost_per_lot_per_side: 0.125 } + - { name: t90, threshold: 0.90, cost_per_lot_per_side: 0.125 } + - { name: t95, threshold: 0.95, cost_per_lot_per_side: 0.125 } + +cells: + - { name: W0, window: 2025-Q1 } diff --git a/crates/ml-backtesting/src/harness.rs b/crates/ml-backtesting/src/harness.rs index e2ef0e7fc..5405127ea 100644 --- a/crates/ml-backtesting/src/harness.rs +++ b/crates/ml-backtesting/src/harness.rs @@ -64,6 +64,16 @@ pub struct BacktestHarnessConfig { /// pos.realized_pnl and accumulated into total_fees_per_b_d at /// every fill. Default 0.0 = no cost (frictionless). pub cost_per_lot_per_side: f32, + /// P6: optional per-backtest variant names. When Some, write_artifacts + /// writes subdirs as `sim_` instead of `cell_NNNN`. Length MUST + /// equal n_parallel when set. None preserves the legacy cell_NNNN + /// naming for non-grid-pack callers (smoke, fixtures). + pub variant_names: Option>, + /// P6: optional pre-built BatchedSimConfig (one entry per backtest = + /// per variant). When Some, the harness uses this directly instead + /// of building from_uniform off the scalar fields above. Length MUST + /// equal n_parallel when set. + pub sim_config_override: Option, } pub struct BacktestHarness { @@ -140,22 +150,32 @@ impl BacktestHarness { sim.upload_program(b, &prog)?; } - // P1: BatchedSimConfig built from uniform broadcast of the harness cfg's - // scalar fields. In P6 the sweep runner constructs a per-variant - // BatchedSimConfig via BatchedSimConfig::from_grid instead. - let sim_config = crate::sim::BatchedSimConfig::from_uniform( - cfg.n_parallel, - &crate::sim::UniformSimParams { - target_annual_vol_units: cfg.target_annual_vol_units, - annualisation_factor: cfg.annualisation_factor, - max_lots: cfg.max_lots, - latency_ns: cfg.latency_ns, - kelly_frac_floor: cfg.kelly_frac_floor, - sharpe_weight_floor: cfg.sharpe_weight_floor, - threshold: cfg.threshold, - cost_per_lot_per_side: cfg.cost_per_lot_per_side, - }, - ); + // P6: if sim_config_override is provided (sweep runner grid-pack + // flow), use it directly; otherwise build a uniform broadcast from + // scalar cfg fields (smoke + fixture flows). + let sim_config = match cfg.sim_config_override.clone() { + Some(bc) => { + anyhow::ensure!( + bc.n_backtests() == cfg.n_parallel, + "sim_config_override has {} variants but n_parallel = {}", + bc.n_backtests(), cfg.n_parallel, + ); + bc + } + None => crate::sim::BatchedSimConfig::from_uniform( + cfg.n_parallel, + &crate::sim::UniformSimParams { + target_annual_vol_units: cfg.target_annual_vol_units, + annualisation_factor: cfg.annualisation_factor, + max_lots: cfg.max_lots, + latency_ns: cfg.latency_ns, + kelly_frac_floor: cfg.kelly_frac_floor, + sharpe_weight_floor: cfg.sharpe_weight_floor, + threshold: cfg.threshold, + cost_per_lot_per_side: cfg.cost_per_lot_per_side, + }, + ), + }; let pnl_curves = (0..cfg.n_parallel).map(|_| Vec::with_capacity(1024)).collect(); Ok(Self { @@ -253,8 +273,21 @@ impl BacktestHarness { pub fn write_artifacts(&self, out_dir: &std::path::Path) -> Result<()> { std::fs::create_dir_all(out_dir) .with_context(|| format!("create out dir {}", out_dir.display()))?; + if let Some(names) = self.cfg.variant_names.as_ref() { + anyhow::ensure!( + names.len() == self.cfg.n_parallel, + "variant_names len {} != n_parallel {}", + names.len(), self.cfg.n_parallel, + ); + } for b in 0..self.cfg.n_parallel { - let cell_dir = out_dir.join(format!("cell_{b:04}")); + // P6: when sweep runner provides variant names, dir = sim_ + // per spec §3.3. Otherwise fall back to legacy cell_NNNN naming. + let subdir_name = match self.cfg.variant_names.as_ref() { + Some(names) => format!("sim_{}", names[b]), + None => format!("cell_{b:04}"), + }; + let cell_dir = out_dir.join(subdir_name); std::fs::create_dir_all(&cell_dir) .with_context(|| format!("create cell dir {}", cell_dir.display()))?; let records = self.sim.read_trade_records(b)?; diff --git a/crates/ml-backtesting/src/sim/batched_config.rs b/crates/ml-backtesting/src/sim/batched_config.rs index 4c6f43407..fced0f219 100644 --- a/crates/ml-backtesting/src/sim/batched_config.rs +++ b/crates/ml-backtesting/src/sim/batched_config.rs @@ -24,6 +24,22 @@ pub struct BatchedSimConfig { pub cost_per_lot_per_side: Vec, } +/// P6: one fully-resolved sim variant for the sweep runner's grid-pack +/// flow. Built from sweep-YAML `SimVariant` entries (per-cell threshold + +/// cost + per-variant overrides) layered over the SweepBase scalars. +#[derive(Clone, Debug)] +pub struct ResolvedSimVariant { + pub name: String, + pub target_annual_vol_units: f32, + pub annualisation_factor: f32, + pub max_lots: u16, + pub latency_ns: u32, + pub kelly_frac_floor: f32, + pub sharpe_weight_floor: f32, + pub threshold: f32, + pub cost_per_lot_per_side: f32, +} + /// Convenience scalar input for `from_uniform`. Mirror of the historical /// BacktestHarnessConfig sim fields. #[derive(Clone, Debug)] @@ -56,6 +72,22 @@ impl BatchedSimConfig { } } + /// P6: pack a list of resolved sim variants into one BatchedSimConfig. + /// Used by the sweep runner when a cell carries N variants (140 typical) + /// to run n_parallel=N backtests in one pod with shared forward. + pub fn from_grid(variants: &[ResolvedSimVariant]) -> Self { + Self { + target_annual_vol_units: variants.iter().map(|v| v.target_annual_vol_units).collect(), + annualisation_factor: variants.iter().map(|v| v.annualisation_factor).collect(), + max_lots: variants.iter().map(|v| v.max_lots).collect(), + latency_ns: variants.iter().map(|v| v.latency_ns).collect(), + kelly_frac_floor: variants.iter().map(|v| v.kelly_frac_floor).collect(), + sharpe_weight_floor: variants.iter().map(|v| v.sharpe_weight_floor).collect(), + threshold: variants.iter().map(|v| v.threshold).collect(), + cost_per_lot_per_side: variants.iter().map(|v| v.cost_per_lot_per_side).collect(), + } + } + pub fn validate(&self) -> Result<()> { let n = self.target_annual_vol_units.len(); let lens = [ diff --git a/crates/ml-backtesting/src/sim/mod.rs b/crates/ml-backtesting/src/sim/mod.rs index 281bb229c..f66d34b5a 100644 --- a/crates/ml-backtesting/src/sim/mod.rs +++ b/crates/ml-backtesting/src/sim/mod.rs @@ -7,7 +7,7 @@ //! See docs/superpowers/specs/2026-05-18-real-lob-integration-design.md §2 + §5b. mod batched_config; -pub use batched_config::{BatchedSimConfig, UniformSimParams}; +pub use batched_config::{BatchedSimConfig, ResolvedSimVariant, UniformSimParams}; use anyhow::{Context, Result}; use cudarc::driver::{CudaSlice, CudaStream, LaunchConfig, PushKernelArg}; diff --git a/scripts/generate_sweep_variants.py b/scripts/generate_sweep_variants.py new file mode 100755 index 000000000..a73edac20 --- /dev/null +++ b/scripts/generate_sweep_variants.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +"""P6 helper: emit the 140 sim_variants for the deployability sweep. + +Cross-product: 7 costs × 4 latencies × 5 thresholds = 140 variants. + +Threshold values are placeholders — the actual values come from +config/ml/v2_prod_thresholds.json after the threshold-tuning step. +Re-run this script post-tuning to refresh sweep_deployability.yaml with +calibrated thresholds. + +Usage: + scripts/generate_sweep_variants.py > config/ml/sweep_deployability.yaml +""" +from __future__ import annotations +import json, sys +from pathlib import Path + +# 7 costs (price-units / lot / side). 0.125 = 1 tick (ES), 0.0625 = 0.5 tick (best-case), +# stepping through realistic retail (0.25, 0.375 = 2-3 ticks), and worst-case (1.0 = 8 ticks). +COSTS = [0.0625, 0.125, 0.1875, 0.25, 0.375, 0.5, 1.0] + +# 4 latencies (ns). 100M = 100ms (best), 200M = realistic anchor, 300M = degraded, +# 400M = stress anchor. +LATENCIES_NS = [100_000_000, 200_000_000, 300_000_000, 400_000_000] + +# 5 thresholds — REPLACE THESE WITH THE OUTPUT OF THE THRESHOLD-TUNING PASS. +# Read from config/ml/v2_prod_thresholds.json if it exists; otherwise use +# the canonical p60-p95 placeholder (5 steps spanning the range). +THRESH_JSON = Path(__file__).resolve().parent.parent / "config" / "ml" / "v2_prod_thresholds.json" +if THRESH_JSON.exists(): + blob = json.loads(THRESH_JSON.read_text()) + THRESHOLDS = blob["thresholds_absolute"] # e.g. {"p60": 0.42, "p70": 0.55, ...} + THRESHOLD_NAMES = list(THRESHOLDS.keys()) + THRESHOLD_VALUES = list(THRESHOLDS.values()) + assert len(THRESHOLD_VALUES) == 5, f"expected 5 thresholds; got {len(THRESHOLD_VALUES)}" +else: + print("# WARNING: v2_prod_thresholds.json missing — using p60-p95 placeholder thresholds", + file=sys.stderr) + THRESHOLD_NAMES = ["p60", "p70", "p80", "p90", "p95"] + THRESHOLD_VALUES = [0.40, 0.55, 0.70, 0.85, 0.95] + +assert len(COSTS) == 7 +assert len(LATENCIES_NS) == 4 + +print("# P6 deployability sweep — 4 windows × 140 sim_variants (7 cost × 4 latency × 5 threshold).") +print("# Generated by scripts/generate_sweep_variants.py — do NOT hand-edit.") +print("# Re-run after threshold-tuning to refresh threshold absolute values.") +print() +print("base:") +print(" data_template: /mnt/training-data/futures-baseline-mbp10/ES.FUT/ES.FUT_{window}.dbn.zst") +print(" predecoded_dir: /feature-cache/predecoded") +print(" n_parallel: 1 # batched flow overrides with variants.len()=140") +print(" decision_stride: 4") +print(" latency_ns: 200000000 # default (overridden per variant)") +print(" target_annual_vol_units: 50.0") +print(" annualisation_factor: 825.0") +print(" max_lots: 5") +print(" max_events: 0") +print(" seed: 12648430") +print(" # __SHA__ replaced by submitter with post-trunk-grows training short-SHA.") +print(" checkpoint: /feature-cache/alpha-perception-runs/__SHA__/trunk_best_h6000.bin") +print(" sim_variants:") + +for ci, cost in enumerate(COSTS): + for li, latency in enumerate(LATENCIES_NS): + for ti, (tname, tval) in enumerate(zip(THRESHOLD_NAMES, THRESHOLD_VALUES)): + name = f"c{ci}_l{li}_{tname}" + print(f" - {{ name: {name}, threshold: {tval}, cost_per_lot_per_side: {cost}, latency_ns: {latency} }}") + +print() +print("cells:") +print(" - { name: W1, window: 2025-Q2 }") +print(" - { name: W2, window: 2025-Q3 }") +print(" - { name: W3, window: 2025-Q4 }") +print(" - { name: W4, window: 2026-Q1 }")