diff --git a/bin/fxt-data-audit/src/main.rs b/bin/fxt-data-audit/src/main.rs index 891a4a23f..bef70961f 100644 --- a/bin/fxt-data-audit/src/main.rs +++ b/bin/fxt-data-audit/src/main.rs @@ -37,7 +37,9 @@ fn main() -> Result<()> { let predecoded_dir = PathBuf::from(&args[2]); eprintln!("loading: {} (predecoded_dir={})", source.display(), predecoded_dir.display()); - let snapshots = load_or_predecode_mbp10(&source, &predecoded_dir) + // The audit tool inspects raw multi-instrument sidecars — we want to see + // every instrument in the file, so the filter is `None` by design. + let snapshots = load_or_predecode_mbp10(&source, &predecoded_dir, None) .map_err(|e| anyhow::anyhow!("load_or_predecode_mbp10: {e}"))?; eprintln!("loaded: {} snapshots", snapshots.len()); diff --git a/crates/data/src/providers/databento/dbn_parser.rs b/crates/data/src/providers/databento/dbn_parser.rs index 24d35db20..828eb7b72 100644 --- a/crates/data/src/providers/databento/dbn_parser.rs +++ b/crates/data/src/providers/databento/dbn_parser.rs @@ -780,6 +780,12 @@ impl DbnParser { /// /// * `path` - Path to the .dbn or .dbn.zst file /// * `snapshot_interval` - Create a snapshot every N raw updates (e.g. 100) + /// * `instrument_id_filter` - When `Some(id)`, only records where + /// `record.hd.instrument_id == id` are processed; all others are + /// skipped before any snapshot mutation. `None` preserves the legacy + /// all-instruments behavior. Multi-instrument DBN files (ES front + + /// back month + calendar spreads in the same archive) require a filter + /// to avoid cross-instrument ΔP contamination in downstream features. /// * `callback` - Called with each aggregated snapshot (by reference, not cloned) /// /// # Returns @@ -789,6 +795,7 @@ impl DbnParser { &self, path: P, snapshot_interval: usize, + instrument_id_filter: Option, mut callback: F, ) -> Result where @@ -823,6 +830,11 @@ impl DbnParser { let mut current_snapshot = Mbp10Snapshot::empty(symbol); let mut update_count: usize = 0; let mut snapshot_count: usize = 0; + // Diagnostic counters: how many records matched vs. were skipped by + // the instrument_id filter. Logged at end-of-parse so multi-instrument + // DBN files are visible in training-job stdout. + let mut kept: u64 = 0; + let mut skipped: u64 = 0; loop { match decoder.decode_record_ref() { @@ -832,6 +844,13 @@ impl DbnParser { })?; if let RecordRefEnum::Mbp10(mbp10) = record_enum { + if let Some(filter) = instrument_id_filter { + if mbp10.hd.instrument_id != filter { + skipped += 1; + continue; + } + } + kept += 1; update_count += 1; let is_bid = mbp10.side == b'B' as i8; @@ -898,6 +917,16 @@ impl DbnParser { snapshot_count += 1; } + if let Some(filter) = instrument_id_filter { + tracing::info!( + kept, + skipped, + filter, + path = %path.display(), + "instrument filter applied" + ); + } + Ok(snapshot_count) } } diff --git a/crates/ml-alpha/examples/alpha_train.rs b/crates/ml-alpha/examples/alpha_train.rs index a03e05154..cecfadc85 100644 --- a/crates/ml-alpha/examples/alpha_train.rs +++ b/crates/ml-alpha/examples/alpha_train.rs @@ -191,6 +191,17 @@ struct Cli { /// useful — defeats the cost-aware objective of the aux supervision. #[arg(long, default_value_t = DEFAULT_OUTCOME_LABEL_COST_ES)] outcome_label_cost: f32, + + /// Filter MBP-10 records to a single `instrument_id`. For ES.FUT + /// use 17077 (front-month ESH4 for Q1 2024). Without this flag, + /// ALL instruments in the file are processed sequentially — + /// multi-instrument ΔP contamination silently inflates K-step + /// labels (front-month ↔ back-month price gaps treated as moves + /// of the same series). The predecoder caches filtered output in + /// a separate sidecar (`*.mbp10_instr.predecoded.bin`) so + /// unfiltered runs keep their existing cache. + #[arg(long)] + instrument_id: Option, } #[derive(Serialize, serde::Deserialize, Default)] @@ -461,6 +472,7 @@ fn main() -> Result<()> { seed: cli.seed, inference_only: false, outcome_label_cost: cli.outcome_label_cost, + instrument_id_filter: cli.instrument_id, }) .context("train loader")?; let mut val_loader = MultiHorizonLoader::new(&MultiHorizonLoaderConfig { @@ -472,6 +484,7 @@ fn main() -> Result<()> { seed: cli.seed.wrapping_add(0xC0FFEE), inference_only: false, outcome_label_cost: cli.outcome_label_cost, + instrument_id_filter: cli.instrument_id, }) .context("val loader")?; tracing::info!( diff --git a/crates/ml-alpha/src/data/loader.rs b/crates/ml-alpha/src/data/loader.rs index 3a3daac54..2c144af73 100644 --- a/crates/ml-alpha/src/data/loader.rs +++ b/crates/ml-alpha/src/data/loader.rs @@ -139,6 +139,18 @@ pub struct MultiHorizonLoaderConfig { /// the aux head learn a cost-free outcome and rarely matches the /// inference-time trading objective. pub outcome_label_cost: f32, + /// When `Some(id)`, the underlying MBP-10 predecoder keeps only the + /// records whose `record.hd.instrument_id == id`. `None` preserves + /// the legacy all-records behavior. + /// + /// ES.FUT DBN archives are multi-instrument: the same file carries + /// the front-month contract (e.g. `ESH4` for Q1 2024, instrument_id + /// 17077), the back month (`ESM4`, 5602), plus calendar spreads / + /// related futures. Without a filter the loader interleaves all of + /// them and computes ΔP across instruments — which is meaningless + /// and silently inflates K-step labels. For ES set this to the + /// front-month id; for single-instrument datasets leave it `None`. + pub instrument_id_filter: Option, } /// Default round-trip cost (price units) baked into D-style outcome labels @@ -275,8 +287,10 @@ impl MultiHorizonLoader { }; let mut files_loaded: Vec = Vec::with_capacity(files.len()); for path in &files { - let snapshots = load_or_predecode_mbp10(path, &cfg.predecoded_dir) - .with_context(|| format!("load mbp10 {}", path.display()))?; + let snapshots = load_or_predecode_mbp10(path, &cfg.predecoded_dir, cfg.instrument_id_filter) + .with_context(|| format!( + "load mbp10 {} (filter={:?})", path.display(), cfg.instrument_id_filter + ))?; if snapshots.len() < min_size { tracing::warn!( path = %path.display(), @@ -683,6 +697,7 @@ mod inference_mode_tests { seed: 0xCAFEF00D, inference_only, outcome_label_cost: DEFAULT_OUTCOME_LABEL_COST_ES, + instrument_id_filter: None, }) } @@ -764,6 +779,7 @@ mod inference_mode_tests { seed: 0, inference_only: false, outcome_label_cost: DEFAULT_OUTCOME_LABEL_COST_ES, + instrument_id_filter: None, }; if let Ok(mut loader) = MultiHorizonLoader::new(&cfg) { let err = loader.next_inference_input(); diff --git a/crates/ml-alpha/tests/multi_horizon_loader.rs b/crates/ml-alpha/tests/multi_horizon_loader.rs index 3b1750763..675f038fb 100644 --- a/crates/ml-alpha/tests/multi_horizon_loader.rs +++ b/crates/ml-alpha/tests/multi_horizon_loader.rs @@ -26,6 +26,7 @@ fn cfg_from_env() -> Option { seed: 0xA1A2_A3A4, inference_only: false, outcome_label_cost: DEFAULT_OUTCOME_LABEL_COST_ES, + instrument_id_filter: None, }) } @@ -69,6 +70,7 @@ fn loader_errors_on_empty_files() { seed: 0, inference_only: false, outcome_label_cost: DEFAULT_OUTCOME_LABEL_COST_ES, + instrument_id_filter: None, }; let res = MultiHorizonLoader::new(&cfg); assert!(res.is_err(), "expected error for empty file list"); diff --git a/crates/ml-backtesting/src/harness.rs b/crates/ml-backtesting/src/harness.rs index ab0f8959d..856339a89 100644 --- a/crates/ml-backtesting/src/harness.rs +++ b/crates/ml-backtesting/src/harness.rs @@ -257,6 +257,10 @@ impl BacktestHarness { // field is required; carry the ES default so a future training-mode // switch in the harness doesn't silently drop the cost. outcome_label_cost: ml_alpha::data::loader::DEFAULT_OUTCOME_LABEL_COST_ES, + // No instrument filter at the harness level — backtest sidecars + // are still the legacy all-records files. A follow-up that plumbs + // a CLI flag through `BacktestHarnessConfig` can lift this. + instrument_id_filter: None, }; let loader = MultiHorizonLoader::new(&loader_cfg)?; diff --git a/crates/ml-backtesting/tests/ring3_replay.rs b/crates/ml-backtesting/tests/ring3_replay.rs index c9cd6db34..3a5530531 100644 --- a/crates/ml-backtesting/tests/ring3_replay.rs +++ b/crates/ml-backtesting/tests/ring3_replay.rs @@ -49,6 +49,7 @@ fn try_loader() -> Option { seed: 0xCAFE_F00D, inference_only: true, outcome_label_cost: ml_alpha::data::loader::DEFAULT_OUTCOME_LABEL_COST_ES, + instrument_id_filter: None, }; match MultiHorizonLoader::new(&cfg) { Ok(l) => Some(l), diff --git a/crates/ml-backtesting/tests/trainer_parity.rs b/crates/ml-backtesting/tests/trainer_parity.rs index f6ddcc4a4..1355cfc56 100644 --- a/crates/ml-backtesting/tests/trainer_parity.rs +++ b/crates/ml-backtesting/tests/trainer_parity.rs @@ -36,6 +36,7 @@ fn try_loader(inference_only: bool) -> Option { seed: 0xCAFEF00D, inference_only, outcome_label_cost: ml_alpha::data::loader::DEFAULT_OUTCOME_LABEL_COST_ES, + instrument_id_filter: None, }; match MultiHorizonLoader::new(&cfg) { Ok(l) => Some(l), diff --git a/crates/ml-features/src/mbp10_loader.rs b/crates/ml-features/src/mbp10_loader.rs index 5f5f07b10..bd93cf707 100644 --- a/crates/ml-features/src/mbp10_loader.rs +++ b/crates/ml-features/src/mbp10_loader.rs @@ -98,7 +98,7 @@ pub fn compute_ofi_from_file(file_path: &Path) -> Result, MLError> let mut features = Vec::new(); let snapshot_count = parser - .parse_mbp10_streaming(file_path, 100, |snapshot| { + .parse_mbp10_streaming(file_path, 100, None, |snapshot| { if let Ok(f) = calculator.calculate(snapshot) { features.push(f.to_array()); } // Skip failed calculations (e.g. first snapshot with no prev) @@ -159,7 +159,7 @@ pub fn compute_ofi_with_trades( let mut trade_cursor: usize = 0; let snapshot_count = parser - .parse_mbp10_streaming(mbp10_file, 100, |snapshot| { + .parse_mbp10_streaming(mbp10_file, 100, None, |snapshot| { // Feed all trades with timestamp <= this snapshot's timestamp. // This ensures VPIN/Kyle's Lambda/trade_imbalance are populated // before the OFI calculator processes the snapshot. diff --git a/crates/ml-features/src/predecoded.rs b/crates/ml-features/src/predecoded.rs index eff1b6eba..4ea3aa54a 100644 --- a/crates/ml-features/src/predecoded.rs +++ b/crates/ml-features/src/predecoded.rs @@ -161,11 +161,29 @@ pub fn load_or_predecode_trades( /// On miss: streams the file via `DbnParser::parse_mbp10_streaming`, accumulates /// `Vec`, persists a sidecar, returns the snapshots. On hit: /// deserializes the sidecar (skips zstd). +/// +/// # `instrument_id_filter` +/// +/// When `Some(id)`, only DBN records whose `record.hd.instrument_id == id` +/// are kept; all others are dropped at decode time. This is required for +/// multi-instrument DBN files (e.g. ES.FUT archives that contain front +/// month + back month + calendar spreads in one stream) — without it, +/// downstream features (ΔP, Mid, K-step labels) span across instruments +/// and produce garbage. +/// +/// The sidecar filename embeds the filter id (`.mbp10_instr.predecoded.bin`) +/// so filtered and unfiltered runs use disjoint cache files and never +/// invalidate each other. `None` keeps the legacy `.mbp10.predecoded.bin` +/// name so existing sidecars on disk remain valid for unfiltered callers. pub fn load_or_predecode_mbp10( source: &Path, predecoded_dir: &Path, + instrument_id_filter: Option, ) -> Result, MLError> { - let sidecar = sidecar_path(source, predecoded_dir, "mbp10"); + let sidecar = match instrument_id_filter { + Some(filter) => sidecar_path(source, predecoded_dir, &format!("mbp10_instr{}", filter)), + None => sidecar_path(source, predecoded_dir, "mbp10"), + }; match try_read_sidecar::>(source, &sidecar) { Ok(Some(snapshots)) => { tracing::info!( @@ -188,7 +206,7 @@ pub fn load_or_predecode_mbp10( .map_err(|e| MLError::InsufficientData(format!("DbnParser init failed: {e}")))?; let mut snapshots = Vec::new(); parser - .parse_mbp10_streaming(source, 100, |snap| { + .parse_mbp10_streaming(source, 100, instrument_id_filter, |snap| { snapshots.push(snap.clone()); }) .map_err(|e| MLError::InsufficientData(format!("parse_mbp10_streaming failed: {e}")))?; diff --git a/crates/ml/examples/alpha_dqn_h600_smoke.rs b/crates/ml/examples/alpha_dqn_h600_smoke.rs index c0a2f45c4..352b7641d 100644 --- a/crates/ml/examples/alpha_dqn_h600_smoke.rs +++ b/crates/ml/examples/alpha_dqn_h600_smoke.rs @@ -391,7 +391,7 @@ fn load_snapshots( let mut rows: Vec = Vec::with_capacity(max_snapshots); 'files: for file in &files { let mut hit_limit = false; - let result = parser.parse_mbp10_streaming(file, snapshot_interval, |snap: &Mbp10Snapshot| { + let result = parser.parse_mbp10_streaming(file, snapshot_interval, None, |snap: &Mbp10Snapshot| { if rows.len() >= max_snapshots { hit_limit = true; return; diff --git a/crates/ml/examples/alpha_fit_fill_model.rs b/crates/ml/examples/alpha_fit_fill_model.rs index 1fff6c051..d80904bf5 100644 --- a/crates/ml/examples/alpha_fit_fill_model.rs +++ b/crates/ml/examples/alpha_fit_fill_model.rs @@ -163,6 +163,7 @@ fn main() -> Result<()> { let result = parser.parse_mbp10_streaming( file, cli.snapshot_interval, + None, |snap: &Mbp10Snapshot| { if n_processed >= limit { hit_limit = true; diff --git a/crates/ml/examples/alpha_random_baseline.rs b/crates/ml/examples/alpha_random_baseline.rs index 6c33e4f15..930174f7a 100644 --- a/crates/ml/examples/alpha_random_baseline.rs +++ b/crates/ml/examples/alpha_random_baseline.rs @@ -168,7 +168,7 @@ fn main() -> Result<()> { 'files: for file in &files { let mut hit_limit = false; let result = - parser.parse_mbp10_streaming(file, cli.snapshot_interval, |snap: &Mbp10Snapshot| { + parser.parse_mbp10_streaming(file, cli.snapshot_interval, None, |snap: &Mbp10Snapshot| { if rows.len() >= cli.max_snapshots { hit_limit = true; return; diff --git a/crates/ml/examples/precompute_features.rs b/crates/ml/examples/precompute_features.rs index a80e18d23..ff5bc9ea1 100644 --- a/crates/ml/examples/precompute_features.rs +++ b/crates/ml/examples/precompute_features.rs @@ -577,7 +577,10 @@ async fn main() -> Result<()> { let per_file: Vec> = { use rayon::prelude::*; mbp10_files.par_iter().filter_map(|file| { - match ml::features::predecoded::load_or_predecode_mbp10(file, &predecoded_dir) { + // precompute_features is the multi-symbol feature-cache + // builder; it intentionally consumes every instrument in + // each file. No filter. + match ml::features::predecoded::load_or_predecode_mbp10(file, &predecoded_dir, None) { Ok(snapshots) => { info!( " {} -> {} snapshots", diff --git a/crates/ml/src/trainers/dqn/data_loading.rs b/crates/ml/src/trainers/dqn/data_loading.rs index 10a230c4f..2594c234b 100644 --- a/crates/ml/src/trainers/dqn/data_loading.rs +++ b/crates/ml/src/trainers/dqn/data_loading.rs @@ -336,7 +336,10 @@ impl DQNTrainer { Err(e) => { warn!(" DBN parser init failed: {e}"); return None; } }; let mut snapshots = Vec::new(); - match parser.parse_mbp10_streaming(file, 100, |snap| { + // DQN data loading predates the instrument-filter + // refactor; keep legacy all-instruments behavior + // until the DQN trainer needs filtering. + match parser.parse_mbp10_streaming(file, 100, None, |snap| { snapshots.push(snap.clone()); }) { Ok(_) => { diff --git a/crates/ml/tests/gpu_per_integration_test.rs b/crates/ml/tests/gpu_per_integration_test.rs index 9529e54cd..fc10ae58a 100644 --- a/crates/ml/tests/gpu_per_integration_test.rs +++ b/crates/ml/tests/gpu_per_integration_test.rs @@ -124,7 +124,8 @@ fn fill_buffer(buf: &mut GpuReplayBuffer, n: usize, state_dim: usize) { let aux_sign: CudaSlice = stream.alloc_zeros::(n).unwrap(); let aux_conf: CudaSlice = stream.alloc_zeros::(n).unwrap(); - buf.insert_batch(&sf, &nsf, &af, &rf, &df, &aux_sign, &aux_conf, n).unwrap(); + let aux_outcome: CudaSlice = stream.alloc_zeros::(n).unwrap(); + buf.insert_batch(&sf, &nsf, &af, &rf, &df, &aux_sign, &aux_conf, &aux_outcome, n).unwrap(); } #[test] diff --git a/crates/ml/tests/sp15_phase1_oracle_tests.rs b/crates/ml/tests/sp15_phase1_oracle_tests.rs index fa4eeda1f..42f20306a 100644 --- a/crates/ml/tests/sp15_phase1_oracle_tests.rs +++ b/crates/ml/tests/sp15_phase1_oracle_tests.rs @@ -2167,7 +2167,8 @@ mod gpu { let ax: CudaSlice = stream.clone_htod(&aux_sign_host).expect("htod aux_sign"); let ac: CudaSlice = stream.alloc_zeros::(n1).expect("aux_conf alloc"); - buf.insert_batch(&sf, &nsf, &af, &rf, &df, &ax, &ac, n1) + let ao_zero: CudaSlice = stream.alloc_zeros::(n1).expect("aux_outcome alloc"); + buf.insert_batch(&sf, &nsf, &af, &rf, &df, &ax, &ac, &ao_zero, n1) .expect("insert_batch (recovery=1)"); stream.synchronize().expect("sync after insert_batch (recovery=1)"); @@ -2186,7 +2187,8 @@ mod gpu { let ax2: CudaSlice = stream.clone_htod(&vec![0_i32; n2]).expect("htod ax2"); let ac2: CudaSlice = stream.alloc_zeros::(n2).expect("aux_conf alloc 2"); - buf.insert_batch(&sf2, &nsf2, &af2, &rf2, &df2, &ax2, &ac2, n2) + let ao2_zero: CudaSlice = stream.alloc_zeros::(n2).expect("aux_outcome alloc"); + buf.insert_batch(&sf2, &nsf2, &af2, &rf2, &df2, &ax2, &ac2, &ao2_zero, n2) .expect("insert_batch (recovery=0)"); stream.synchronize().expect("sync after insert_batch (recovery=0)"); @@ -2353,7 +2355,8 @@ mod gpu { let ax: CudaSlice = stream.clone_htod(&aux_sign_host).expect("htod aux_sign"); let ac: CudaSlice = stream.alloc_zeros::(batch_size).expect("aux_conf alloc"); - buf.insert_batch(&sf, &nsf, &af, &rf, &df, &ax, &ac, batch_size) + let ao_zero: CudaSlice = stream.alloc_zeros::(batch_size).expect("aux_outcome alloc"); + buf.insert_batch(&sf, &nsf, &af, &rf, &df, &ax, &ac, &ao_zero, batch_size) .expect("insert_batch (mixed-env)"); stream.synchronize().expect("sync after insert_batch"); diff --git a/docs/dqn-wire-up-audit.md b/docs/dqn-wire-up-audit.md index 60b156af6..49bf4ad1d 100644 --- a/docs/dqn-wire-up-audit.md +++ b/docs/dqn-wire-up-audit.md @@ -18722,3 +18722,16 @@ Lib + examples now have zero removable warnings. The remaining `unsafe_block` lints (each cudarc kernel launch needs `unsafe`) are structural and not actionable under the project's `-W unsafe-code` policy. + +## 2026-05-22 — chore(dqn): forward None to parse_mbp10_streaming after API expansion + +**Branch:** `ml-alpha-phase-a`. **Trigger:** `parse_mbp10_streaming` signature gained +`instrument_id_filter: Option` to enable multi-instrument filtering for ml-alpha's +MBP-10 loader. DQN data loading path predates the refactor. + +**Change:** `crates/ml/src/trainers/dqn/data_loading.rs:339` passes `None` to preserve +legacy all-instruments behavior. DQN trainer is not currently exercised against +multi-instrument data; if/when ES.FUT calendar spreads become a training signal for +DQN, the call site should be updated to forward an instrument_id from config. + +**Wire-up impact:** None (semantic preservation only). diff --git a/docs/sp18-wireup-audit.md b/docs/sp18-wireup-audit.md index 0a32250de..c847818c7 100644 --- a/docs/sp18-wireup-audit.md +++ b/docs/sp18-wireup-audit.md @@ -337,89 +337,112 @@ After Phase 1's atomic deletion lands, this fingerprint will shrink dramatically re-capture is committed as part of Task 1.6 (close-out). -``` === D-leg: Slot 380 (HOLD_COST_INDEX) consumers === total_hits=39 crates/ml/build.rs 1 - crates/ml/src/cuda_pipeline/state_layout.cuh 1 - crates/ml/src/cuda_pipeline/hold_rate_observer_kernel.cu 1 crates/ml/src/cuda_pipeline/sp5_isv_slots.rs 1 - crates/ml/src/cuda_pipeline/sp13_isv_slots.rs 1 + crates/ml/src/cuda_pipeline/hold_rate_observer_kernel.cu 1 crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs 2 + crates/ml/src/cuda_pipeline/sp13_isv_slots.rs 1 + crates/ml/src/cuda_pipeline/state_layout.cuh 1 crates/ml/src/trainers/dqn/trainer/training_loop.rs 3 crates/ml/src/trainers/dqn/state_reset_registry.rs 1 - docs/sp18-wireup-audit.md 5 docs/dqn-wire-up-audit.md 10 docs/superpowers/specs/2026-05-04-sp13-redefine-success-for-predictive-skill.md 4 - docs/superpowers/plans/2026-05-04-sp13-redefine-success-for-predictive-skill.md 5 docs/superpowers/plans/2026-05-08-sp18-reward-shape-hold-attractor.md 4 + docs/superpowers/plans/2026-05-04-sp13-redefine-success-for-predictive-skill.md 5 + docs/sp18-wireup-audit.md 5 === D-leg: Slot 461 (HOLD_COST_SCALE_INDEX) consumers === - total_hits=28 - docs/sp18-wireup-audit.md 6 - docs/dqn-wire-up-audit.md 9 - docs/superpowers/plans/2026-05-08-sp16-hold-collapse-structural-fix.md 10 + total_hits=79 + crates/ml/src/cuda_pipeline/sp20_controllers_compute.rs 5 + crates/ml/src/cuda_pipeline/sp20_hold_baseline.cuh 2 + crates/ml/src/cuda_pipeline/gpu_experience_collector.rs 2 + crates/ml/src/cuda_pipeline/sp20_hold_baseline_test_kernel.cu 1 + crates/ml/src/cuda_pipeline/sp14_isv_slots.rs 2 + crates/ml/src/cuda_pipeline/sp20_controllers_compute_kernel.cu 2 + crates/ml/src/trainers/dqn/trainer/training_loop.rs 4 + crates/ml/src/trainers/dqn/state_reset_registry.rs 2 + crates/ml/tests/sp20_controllers_compute_test.rs 17 + crates/ml/tests/sp20_phase1_4_wireup_test.rs 2 + docs/dqn-wire-up-audit.md 11 + docs/superpowers/specs/2026-05-09-sp19-20-wr-first-design.md 1 + docs/superpowers/plans/2026-05-09-sp19-20-wr-first.md 8 docs/superpowers/plans/2026-05-08-sp18-reward-shape-hold-attractor.md 3 + docs/superpowers/plans/2026-05-08-sp16-hold-collapse-structural-fix.md 10 + docs/sp18-wireup-audit.md 6 + docs/isv-slots.md 1 === D-leg: Slots [462..468) (HCS_* Welford) consumers === total_hits=9 docs/dqn-wire-up-audit.md 8 docs/superpowers/plans/2026-05-08-sp18-reward-shape-hold-attractor.md 1 === D-leg: hold_cost_scale_update_kernel references === - total_hits=58 - docs/sp18-wireup-audit.md 11 - docs/dqn-wire-up-audit.md 22 + total_hits=78 + crates/ml/src/cuda_pipeline/sp20_controllers_compute.rs 2 + crates/ml/src/cuda_pipeline/sp20_controllers_compute_kernel.cu 3 + crates/ml/src/trainers/dqn/trainer/training_loop.rs 5 + crates/ml/src/trainers/dqn/state_reset_registry.rs 2 + crates/ml/tests/sp20_controllers_compute_test.rs 3 + docs/plans/2026-05-10-sp21-train-eval-coherence-isv-defrost.md 2 + docs/dqn-wire-up-audit.md 23 docs/superpowers/specs/2026-05-08-sp18-reward-shape-hold-attractor-design.md 4 - docs/superpowers/plans/2026-05-08-sp16-hold-collapse-structural-fix.md 9 + docs/superpowers/plans/2026-05-09-sp19-20-wr-first.md 2 docs/superpowers/plans/2026-05-08-sp18-reward-shape-hold-attractor.md 12 + docs/superpowers/plans/2026-05-08-sp16-hold-collapse-structural-fix.md 9 + docs/sp18-wireup-audit.md 11 === D-leg: hold_rate_observer_kernel (RETAINED — diag chain stays per DD7=c) === - total_hits=160 - crates/ml/tests/sp13_phase0_oracle_tests.rs 19 - crates/ml/tests/sp14_oracle_tests.rs 20 + total_hits=164 crates/ml/build.rs 5 - crates/ml/src/cuda_pipeline/min_hold_temperature_update_kernel.cu 6 - crates/ml/src/cuda_pipeline/state_layout.cuh 1 - crates/ml/src/cuda_pipeline/gpu_experience_collector.rs 12 - crates/ml/src/cuda_pipeline/hold_rate_observer_kernel.cu 4 crates/ml/src/cuda_pipeline/sp5_isv_slots.rs 3 - crates/ml/src/cuda_pipeline/sp13_isv_slots.rs 2 - crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs 8 crates/ml/src/cuda_pipeline/gpu_aux_trunk.rs 5 + crates/ml/src/cuda_pipeline/hold_rate_observer_kernel.cu 4 + crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs 8 + crates/ml/src/cuda_pipeline/gpu_experience_collector.rs 13 crates/ml/src/cuda_pipeline/sp14_isv_slots.rs 5 + crates/ml/src/cuda_pipeline/sp20_aggregate_inputs.rs 1 + crates/ml/src/cuda_pipeline/sp13_isv_slots.rs 2 + crates/ml/src/cuda_pipeline/min_hold_temperature_update_kernel.cu 6 + crates/ml/src/cuda_pipeline/sp20_aggregate_inputs_kernel.cu 1 + crates/ml/src/cuda_pipeline/state_layout.cuh 1 crates/ml/src/trainers/dqn/trainer/training_loop.rs 13 crates/ml/src/trainers/dqn/state_reset_registry.rs 3 + crates/ml/tests/sp13_phase0_oracle_tests.rs 19 + crates/ml/tests/sp14_oracle_tests.rs 20 + docs/dqn-wire-up-audit.md 12 + docs/superpowers/specs/2026-05-08-sp18-reward-shape-hold-attractor-design.md 1 + docs/superpowers/specs/2026-05-04-sp13-redefine-success-for-predictive-skill.md 6 + docs/superpowers/plans/2026-05-05-sp14-aux-q-wire-earned-gradient-flow.md 5 + docs/superpowers/plans/2026-05-08-sp18-reward-shape-hold-attractor.md 3 + docs/superpowers/plans/2026-05-04-sp13-redefine-success-for-predictive-skill.md 20 docs/sp18-wireup-audit.md 7 docs/isv-slots.md 1 - docs/dqn-wire-up-audit.md 11 - docs/superpowers/specs/2026-05-04-sp13-redefine-success-for-predictive-skill.md 6 - docs/superpowers/specs/2026-05-08-sp18-reward-shape-hold-attractor-design.md 1 - docs/superpowers/plans/2026-05-05-sp14-aux-q-wire-earned-gradient-flow.md 5 - docs/superpowers/plans/2026-05-04-sp13-redefine-success-for-predictive-skill.md 20 - docs/superpowers/plans/2026-05-08-sp18-reward-shape-hold-attractor.md 3 === D-leg: Per-bar Hold-cost subtraction sites (3 lines we change in Phase 1.5) === - total_hits= + total_hits=5 + crates/ml/src/cuda_pipeline/experience_kernels.cu 5 === D-leg: state_layout.cuh mirror constants (HOLD_COST + HCS) === total_hits=1 crates/ml/src/cuda_pipeline/state_layout.cuh 1 === D-leg: state_reset_registry slot 461 entries === - total_hits= + total_hits=2 + crates/ml/src/trainers/dqn/state_reset_registry.rs 2 === D-leg: Cubin manifest references in build.rs === total_hits=2 crates/ml/build.rs 2 === B-leg: td_lambda_kernel launch sites === total_hits=20 - crates/ml/tests/sp18_td_lambda_q_next_oracle_tests.rs 8 - crates/ml/src/cuda_pipeline/gpu_experience_collector.rs 11 crates/ml/src/cuda_pipeline/nstep_kernel.cu 1 + crates/ml/src/cuda_pipeline/gpu_experience_collector.rs 11 + crates/ml/tests/sp18_td_lambda_q_next_oracle_tests.rs 8 === B-leg: q_next argument origin (the line ~4143 site) === total_hits= === B-leg: rewards_out consumers (downstream of TD(λ) output) === total_hits= === B-leg: PER priority computation sites (verify B-DD4: NO migration needed) === total_hits=18 - crates/ml-dqn/src/gpu_replay_buffer.rs 7 - crates/ml-dqn/src/replay_buffer_type.rs 3 - crates/ml/tests/gpu_per_integration_test.rs 3 - crates/ml/src/trainers/dqn/config.rs 4 crates/ml/src/trainers/dqn/fused_training.rs 1 + crates/ml/src/trainers/dqn/config.rs 4 + crates/ml/tests/gpu_per_integration_test.rs 3 + crates/ml-dqn/src/replay_buffer_type.rs 3 + crates/ml-dqn/src/gpu_replay_buffer.rs 7 === B-leg: Replay buffer schema — q_next storage check (verify NO migration) === total_hits= === B-leg: c51_loss_kernel target-Q origin (verify already correct at loss time) === @@ -428,11 +451,12 @@ re-capture is committed as part of Task 1.6 (close-out). total_hits= === B-leg: PopArt slot 63 references (verify B-DD11 reset is the only PopArt change) === total_hits=20 - crates/ml/tests/sp18_hold_reward_oracle_tests.rs 17 - crates/ml/tests/sp11_producer_unit_tests.rs 3 + crates/ml/build.rs 8 + crates/ml/src/training_profile.rs 8 + crates/ml/src/cuda_pipeline/trade_physics.cuh 1 + crates/ml/src/cuda_pipeline/dqn_utility_kernels.cu 3 === END === total_hits= -``` ## Phase 4 close-out — B-leg target-net forward additive infrastructure (2026-05-09) diff --git a/docs/superpowers/plans/2026-05-22-aux-candidate-b-pivot.md b/docs/superpowers/plans/2026-05-22-aux-candidate-b-pivot.md new file mode 100644 index 000000000..bca3ea4a8 --- /dev/null +++ b/docs/superpowers/plans/2026-05-22-aux-candidate-b-pivot.md @@ -0,0 +1,68 @@ +# Aux Supervision Candidate B Pivot — A+B Paired (Profitable Binary + Sized Regression) + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development to implement this plan task-by-task. + +**Goal:** Replace the D-style asymmetric labels (Layer B of prior plan) with Candidate B's A+B paired design from E3's original recommendation: binary "profitable-after-cost" + σ-normalized magnitude regression, per (long, short) direction per horizon. Driven by Smoke-1-v2 empirical finding: D labels are 99.99% negative at K=100 and 94-96% negative at K=1000, making them un-learnable. A+B's class-weighted BCE on a balanced (per-horizon-prior-corrected) binary plus a conditional-Huber on magnitude is empirically learnable. + +**Architecture:** Keep aux trunk infrastructure (B3 — AUX_HIDDEN=64 CfC). Replace D label generator (B1), replace aux loss kernel (B4), add second head to aux_heads.cu (B4), adjust trainer wiring (B5) to feed both losses through separate Adam moments. + +## Pre-conditions + +Branch: `ml-alpha-phase-a` HEAD `2e0629767`. Working tree clean. Smoke-1-v2 (`alpha-perception-dz6r4`) validated: +- BCE training non-regressed (bit-identical to baseline) +- Aux supervision FIRES on real data (Huber drops 2029→1268) +- Stop-grad never lifts (threshold never reached due to D-label scale mismatch) + +## Sub-decisions (resolve before implementation) + +| # | Decision | Default | Rationale | +|---|---|---|---| +| 1 | y_prof label definition | `y_prof[t,k,d] = 1 if signed_pnl_d > 2×cost else 0` | E3 default. The `2×cost` threshold (vs just `>0`) lifts the bar above breakeven noise. | +| 2 | y_size label definition | `y_size[t,k,d] = signed_pnl_d / σ_K[t]` | σ_K[t] = rolling 1000-bar std of K-step log-returns (Welford EMA). Per-horizon σ. | +| 3 | Aux head output structure | 4 outputs per (direction): `[prof_long, size_long, prof_short, size_short] × N_HORIZONS` = 12 total | Single fused head for cache locality; loss kernel splits at output. Alternative: 4 separate heads (rejected — more launches, no perf gain). | +| 4 | Loss balancing | `total_aux = α_prof × BCE(prof) + α_size × Huber(size if prof else 0)` per direction; per-Adam-group LR scaling | E3 + `pearl_adam_normalizes_loss_weights`: don't sum-weight, use separate optimizer groups OR per-loss-function gradient scaling. Simplest first cut: single aux Adam group, but BCE and Huber inherently produce different gradient scales — no need for explicit α. | +| 5 | Class-weight for BCE | `pos_weight = n_neg / n_pos` per horizon, recomputed each epoch from EMA-tracked counts | Per E3. Without this, the 99.99% negative class drives all-negative predictions (same trap D fell into). | +| 6 | Conditional-Huber gating | Mask y_size loss if `y_prof_true == 0` (don't penalize size estimates for unprofitable trades) | Per E3. Keeps the size head focused on the actually-profitable subset. Alternative: train size on ALL samples (rejected — would force size head to predict large negative magnitudes for unprofitable cases, defeating the purpose). | +| 7 | σ_K window | 1000-snapshot Welford EMA | Matches K=1000 horizon naturally. Smaller windows would be noisier at K=10. | +| 8 | Aux head initialization | Smaller Xavier scale (0.1× instead of standard) to keep initial outputs close to 0 | Empirical fix from D-label run: standard Xavier gave outputs ~5× label magnitude. Smaller init = faster convergence. | +| 9 | Aux gate in policy (B7) | Gate trade entry on `prof_long_prob > 0.55 AND size_long_pred > 0.3 × σ_K` (or the short equivalent) | Two-threshold gate per direction: profitability confidence + minimum expected size. | + +## File scope + +### Modified files (most reused from B1-B5) +- `crates/ml-alpha/src/multi_horizon_labels.rs` — REPLACE `generate_outcome_labels_d` → `generate_outcome_labels_ab` (add σ_K rolling computation) +- `crates/ml-alpha/src/data/loader.rs` — `OutcomeLabelsD` → `OutcomeLabelsAB` field rename + rolling-σ_K wiring +- `crates/ml-alpha/cuda/aux_heads.cu` — change output structure (4 values per direction × N_HORIZONS = 12 outputs from 64-dim aux trunk hidden) +- `crates/ml-alpha/cuda/aux_loss.cu` — REPLACE Huber with class-weighted BCE for prof + conditional-Huber for size +- `crates/ml-alpha/src/aux_heads.rs` — adjust weight structs, fwd/bwd wrappers +- `crates/ml-alpha/src/trainer/perception.rs` — wire 2 losses + ISV signals (`aux_prof_bce_ema`, `aux_size_huber_ema`, `aux_prof_dir_acc_ema` per horizon) +- `crates/ml-alpha/examples/alpha_train.rs` — tracing log lines for new aux metrics +- `crates/ml-backtesting/cuda/decision_policy.cu` — TBD in B7 (after this lands) + +### Reused as-is +- `crates/ml-alpha/src/cfc/aux_trunk.rs` + `aux_trunk.cu` (no change — same 64-hidden CfC) +- `crates/ml-alpha/cuda/aux_vec_add.cu` (no change — still needed for stop-grad lift) +- Tests in `aux_heads.rs`, `aux_trunk.rs` — need updates for new output structure + +## Estimated cost + +| Task | Hours | +|---|---| +| Label generator: D → A+B with σ_K | 3 | +| Loader integration changes | 1.5 | +| aux_loss.cu rewrite (BCE + conditional Huber + class weights) | 3 | +| aux_heads.cu output structure change | 1.5 | +| Trainer wiring + ISV signals | 2 | +| Local smoke validation | 0.5 | +| Argo Smoke 1 | 0.5 (wallclock wait) | +| **Total** | **~12 hours** | + +## Risks + +| Risk | Likelihood | Mitigation | +|---|---|---| +| Class-weighted BCE causes pos-pred explosion | Medium | Cap `pos_weight ≤ 50` per horizon | +| σ_K is too noisy at K=10 | Medium | Floor σ_K at cost-tick scale per `pearl_trade_level_vol_for_stop_distance` | +| Conditional-Huber masks too much data, size head doesn't train | Low | Verify positive-class fraction is reasonable (≥5%) before masking | +| Loss-scale mismatch between BCE and Huber overweights one | Low-Medium | Track per-loss EMAs separately. If divergence, add per-loss LR per `pearl_adam_normalizes_loss_weights` | +| Aux head init still too large after 0.1× Xavier | Medium | Bias init = 0 + small Xavier should put outputs near 0; verify on local smoke | diff --git a/docs/superpowers/plans/2026-05-22-aux-diagnosis-deeper.md b/docs/superpowers/plans/2026-05-22-aux-diagnosis-deeper.md new file mode 100644 index 000000000..06228ec23 --- /dev/null +++ b/docs/superpowers/plans/2026-05-22-aux-diagnosis-deeper.md @@ -0,0 +1,247 @@ +# Deeper Diagnosis: Aux Supervision Mysteries Before Next Pivot + +> **For agentic workers:** Diagnostic plan, not implementation. Read each step's pass/fail criteria; branch decisions are explicit. + +**Goal:** Resolve four unexplained mysteries before committing to another aux-supervision iteration OR pivoting to Layer C (regime gate). Avoid the "fix-and-pray" loop that's already burned 5 Argo smokes without convergence. + +**Architecture of the diagnosis:** Cheap falsifiers first. Each step narrows the hypothesis space; later steps run only if earlier ones don't yield a verdict. + +## The four mysteries + +| # | Mystery | What we know | What we don't | +|---|---|---|---| +| M1 | `pos_fraction` bit-identical pre/post F2 fix (0.351→0.351) | Different seeds, different code, same value to 6 d.p. | Whether F2 actually fired at runtime | +| M2 | Why is `pos_fraction ≈ 40%` at K=10 (~700µs)? | At ES tick=0.25, "ΔP > 4 ticks in 700µs" should be rare | Whether one-sided books / data quality cause spurious positives, OR if the data really has this property | +| M3 | Epoch-4 NaN explosion (BCE 0.84→18.74, size→NaN) | First 4 epochs bit-identical to all prior runs; epoch 4 diverges only after F2 fix | Where the NaN first appears + which buffer chain propagates it | +| M4 | What's the structural ceiling? | Synthetic test passes; real-data fails consistently; ES heavily arbitraged | Whether ANY aux signal exists at K∈[10,100,1000] worth pursuing | + +## Pre-conditions + +- Branch: `ml-alpha-phase-a` HEAD `85d3ca5c7` (F1+F2 applied; epoch-4 NaN observed in `alpha-perception-7shgw`) +- Working tree clean +- Local validation: 43 lib tests + 15 multi_horizon_labels pass +- 6 prior aux-supervision smokes available for cross-comparison + +--- + +## Step 1 — Resolve M1 (did F2 actually fire?) [10 min, no compute] + +**Question:** Is `mid_price_f32 → NaN` actually being executed on real data, or is the code path bypassed? + +### 1.1 Check binary actually rebuilt + +```bash +git log -1 --format=%H crates/ml-alpha/src/data/loader.rs +# Expected: 85d3ca5c7 (the F2 commit) + +cd /tmp && mc ls fh/argo-logs/alpha-perception-7shgw/ 2>&1 | head -3 +# Confirm the workflow at commit 85d3ca5c7 actually ran (build timestamp matches) +``` + +### 1.2 Inspect the actual computed code path + +Read `crates/ml-alpha/src/data/loader.rs::mid_price_f32` — confirm the NaN branch landed in the current commit. Read `Mbp10Snapshot.levels[0]` source — verify it's the right level. Read where `mid_price_f32` is called — verify the result feeds `generate_outcome_labels_ab`. + +**PASS:** Code path verified, F2 fix is in the binary. +**FAIL:** F2 fix isn't in the binary (revert / re-push and skip to Step 4). + +### 1.3 The pos_fraction bit-identity test + +Run a Rust unit test that constructs a 1000-sample fixture with N% one-sided books and computes pos_fraction with both old (blind avg) and new (NaN-guarded) mid_price_f32. If the resulting pos_fractions are within 0.01 of each other, **one-sided books are NOT the contamination source** — the loader's input data has near-zero one-sided rate, and F2's hypothesis was wrong. + +```rust +#[test] +fn f2_pos_fraction_sensitivity_to_one_sided_books() { + // Construct synthetic Mbp10Snapshot stream with X% one-sided books. + // Compute pos_fraction with old vs new mid_price_f32. + // Assert: |new - old| < 0.01 when X < 0.5%, gap visible when X > 5%. +} +``` + +**Decision branch:** +- If gap visible at X=5% but not at X=0.5%: F2 fix WORKS, real data must have <0.5% one-sided. Skip to Step 2. +- If no gap at X=10%: F2 fix doesn't work as designed. Inspect why (cascade issue). + +--- + +## Step 2 — Resolve M2 (is 40% pos_fraction real?) [30 min, NumPy on local data] + +**Question:** Does ES real data actually have ~40% of K=10 windows with ΔP > 4 ticks, or is this an artifact of decoder/contamination? + +### 2.1 Compute pos_fraction LOCALLY using the Rust loader logic + +Write a self-contained Python script reusing the existing decoded local fixture (`test_data/futures-baseline-mbp10/ES.FUT/ES.FUT_2024-Q1.dbn.zst` via databento): + +```python +# Decode K snapshots from the local Q1 file +# Compute mid = (bid_px[0] + ask_px[0]) / 2 (raw, no guards) +# For each K in [10, 100, 1000]: +# Compute delta_K = mid[t+K] - mid[t] +# y_prof_long = (delta_K > 1.0).mean() # 2 ticks cost = 1.0 price unit +# y_prof_short = (delta_K < -1.0).mean() +# Print pos_fraction per (direction, K) +``` + +This mirrors what the loader does. Expected: if matches cluster's 0.351/0.389/0.456 → data really has this property. If different → cluster data path differs from local. + +### 2.2 Histogram of |ΔP| at K=10 on local data + +Plot or print quantile statistics of `|mid[t+10] - mid[t]|`. If p50 > 1.0, most K=10 windows naturally exceed the cost threshold — the 40% pos_fraction is correct. If p50 << 1.0, something's wrong. + +**Decision branch:** +- If matches cluster + p50 > 1.0: 40% pos_fraction is REAL ES microstructure. F2's contamination hypothesis was wrong. Don't fix what isn't broken; focus on F1. +- If matches cluster + p50 << 1.0: there's a SHARED bug in mid-price computation that's NOT one-sided books. Need to find it. +- If doesn't match cluster: cluster data is contaminated differently (decoder issue, normalized prices, etc.) — investigate cluster decode path. + +--- + +## Step 3 — Resolve M3 (epoch-4 NaN explosion) [45 min, log analysis + code trace] + +**Question:** What buffer first showed NaN at epoch 4? + +### 3.1 Cross-compare epoch-by-epoch metrics across runs + +| Run | HEAD | Epoch 0 BCE | Epoch 4 BCE | Notes | +|---|---|---|---|---| +| dz6r4 | 0f5d5c7b4 (F1+F2 pre) | 0.845 | 0.806 | Pre F1+F2 | +| 779fp | 24289f40a (pw=10) | 0.845 | 0.806 | Same as above | +| 7shgw | 85d3ca5c7 (F1+F2 fix) | 0.845 | **18.74** | NaN at epoch 4 | + +**Observation:** Epochs 0-3 BCE values ARE bit-identical across all three runs. The NaN appears ONLY at epoch 4 of the F1+F2 run. + +This is highly suspicious: identical labels and gradients for 3 epochs, then suddenly diverge. Possible causes: +- **Hidden statefulness**: σ_K Welford EMA accumulates NaN over epochs without resetting between runs +- **Random seed exhaustion**: validation data shifts to a new region at epoch 4 with more zero-priced books +- **Numerical accumulation**: head weights drift to a regime where one bad batch tips them to NaN + +### 3.2 Add NaN-detection tracing (write code, but only diagnostic, not in critical path) + +```rust +// At end of each train_step in step_batched, after BCE/Huber loss readback: +if !mean_prof_bce.is_finite() || !mean_size_huber.is_finite() { + tracing::warn!( + step = self.train_step_count, + epoch_step = epoch_step, + prof_long_loss = total_loss_pl, + prof_short_loss = total_loss_ps, + size_long_loss = total_loss_sl, + size_short_loss = total_loss_ss, + n_pl = n_pl, n_ps = n_ps, n_sl = n_sl, n_ss = n_ss, + "AUX NAN DETECTED — first occurrence" + ); +} +``` + +Re-run the smoke. The first NaN warning will tell us EXACTLY at which step + which loss buffer it appeared. Then we trace BACKWARD: what input made that buffer NaN? + +### 3.3 If NaN appears in size_huber first (likely): trace σ_K cascade + +The σ_K computation in `generate_outcome_labels_ab` uses a rolling Welford. If a NaN enters the window, it propagates. The current code: + +```rust +// Look at the log_ret computation: log(p[t+K] / p[t]) +// If p[t] == 0 OR p[t+K] == 0 → log_ret = -inf or NaN +// Welford running sum gets NaN → all subsequent σ_K[h][t'] = NaN +``` + +Check `crates/ml-alpha/src/multi_horizon_labels.rs::generate_outcome_labels_ab` Pass 1 (σ_K computation). Find where log_ret is computed and the Welford update. Confirm there's no NaN-skip. + +**Decision branch:** +- If σ_K Welford propagates NaN: fix σ_K to NaN-skip (the proper F2 strengthening). +- If σ_K is fine but size head still NaNs: investigate `aux_huber_masked` kernel for divide-by-zero or overflow. + +--- + +## Step 4 — Resolve M4 (structural ceiling test) [conditional on Steps 1-3] + +**Run only if Steps 1-3 conclude "F1+F2 should work properly" but we want to bound the upside.** + +### 4.1 Cheapest structural ceiling test: label-swap experiment + +Per F4's recommendation: swap `y_prof_long ↔ y_prof_short` at the loader emission and run a 3-epoch smoke. If dir_acc inverts to >0.5, the model has learnable signal. If still at chance, the aux signal at this trunk size is structurally absent. + +```rust +// One-line gated swap in alpha_train.rs +if std::env::var("AUX_LABEL_SWAP_DIAGNOSTIC").is_ok() { + std::mem::swap(&mut prof_long_row, &mut prof_short_row); + std::mem::swap(&mut size_long_row, &mut size_short_row); +} +``` + +3-epoch Argo smoke takes ~7 minutes. The verdict is decisive: +- dir_acc > 0.55 → real signal exists, find the right metric / loss +- dir_acc near 0.5 → no signal at this trunk size + +### 4.2 Pivot decision + +Based on M1-M4 verdicts: + +| M1 | M2 | M3 | M4 | Next | +|---|---|---|---|---| +| Pass | 40% real | σ_K NaN | dir_acc > 0.55 | Fix σ_K, keep aux supervision | +| Pass | 40% real | σ_K NaN | dir_acc ~0.5 | Aux has no signal at this size; pivot to Layer C | +| Pass | 40% NOT real | * | * | Cluster data bug; fix decoder first | +| Fail | * | * | * | F2 fix wasn't applied; either re-apply or revert | + +--- + +## Tasks (executable order) + +- [ ] **Task 1** — Step 1.1 + 1.2: verify F2 fix actually landed in binary. Read code. (5 min) +- [ ] **Task 2** — Step 1.3: write Rust unit test for pos_fraction sensitivity to one-sided book rate. (15 min) +- [ ] **Task 3** — Step 2.1: write Python script to recompute pos_fraction on LOCAL test_data Q1 file, compare to cluster's bit-identical value. (20 min) +- [ ] **Task 4** — Step 2.2: histogram of |ΔP| at K=10 on local data. (10 min) +- [ ] **Task 5** — Step 3.2: add NaN-detection tracing to step_batched, re-dispatch smoke (only if Tasks 1-4 don't already explain everything). (20 min code + 12 min Argo) +- [ ] **Task 6** — Step 3.3: inspect σ_K Welford for NaN-propagation IF Task 5 confirms σ_K is the cascade point. (15 min) +- [ ] **Task 7** — Decision: based on Tasks 1-6 outcomes, choose: + - (a) Strengthen F2 with σ_K NaN-skip → re-run smoke + - (b) Revert F2, keep F1 only → re-run smoke + - (c) Run label-swap diagnostic (Step 4.1) → branch on dir_acc + - (d) Pivot to Layer C +- [ ] **Task 8** — Pearls: capture findings regardless of outcome (don't lose the diagnostic work). + +## Estimated cost + +- Steps 1-2: 30 min (no compute beyond local Python/cargo) +- Step 3 (NaN tracing): 30-45 min if needed, 0 if M1/M2 closes the question +- Step 4 (label-swap): 45 min if needed + +**Total floor: 30 min if Steps 1-2 give us a clear answer.** +**Total ceiling: 2 hours if we go through all steps.** + +## Memory rules applicable + +- `feedback_no_quickfixes` — diagnose root cause before fixing +- `pearl_no_deferrals_for_complementary_fixes` — combine related fixes when scope overlaps +- `pearl_canary_input_freshness_launch_order` — staging vs kernel ordering bugs +- `pearl_trade_level_vol_for_stop_distance` — σ_K should be floored at cost-tick scale (we did this; verify it's not being bypassed) +- `pearl_first_observation_bootstrap` — sentinel handling for rolling stats + +## Risks + +| Risk | Mitigation | +|---|---| +| Step 1 reveals F2 fix didn't apply at runtime — wasted earlier compute | Cheap to detect (Task 1), and the learning is real | +| Step 2 shows local data ≠ cluster data — broader decoder investigation needed | Document the divergence; cluster-side investigation gets its own plan | +| Step 5 NaN tracing modifies hot path | Tracing is conditional on `!is_finite()` — negligible perf cost | +| Label-swap diagnostic doesn't decisively answer (dir_acc oscillates near 0.5) | Run for 5 epochs instead of 3; if still ambiguous, the answer IS "ambiguous" — pivot to Layer C anyway | + +## Falsification gates (when to STOP) + +- After Step 2: if 40% pos_fraction confirmed REAL on local data, stop digging F2 — it was a wrong hypothesis. Focus on F1 + σ_K cascade. +- After Step 3: if NaN cascade source identified, the fix is direct. +- After Step 4: if label-swap gives dir_acc still ~0.5, aux signal at this trunk size is bounded by architectural ceiling. Pivot to Layer C. + +## Out-of-scope (DO NOT do here) + +- Implement CB6 (per-horizon Huber kernel split) — defer until aux supervision is validated as worth pursuing +- Build Layer C regime gate — that's the post-diagnosis pivot, not the diagnosis itself +- Modify aux trunk dimensions — that's an upsize architectural change, not a diagnosis step + +## After this plan + +Either: +- **Aux supervision validated**: continue to B7 (decision policy aux gate) with confidence +- **Aux supervision falsified**: pivot to Layer C (regime gate) per the original anti-calibration plan; aux infrastructure stays in code as future-work scaffolding + +Plan saved at `docs/superpowers/plans/2026-05-22-aux-diagnosis-deeper.md`. diff --git a/docs/superpowers/plans/2026-05-22-horizon-rebase-n3-100-300-1000-and-aux-d-labels.md b/docs/superpowers/plans/2026-05-22-horizon-rebase-n3-100-300-1000-and-aux-d-labels.md new file mode 100644 index 000000000..30e093892 --- /dev/null +++ b/docs/superpowers/plans/2026-05-22-horizon-rebase-n3-100-300-1000-and-aux-d-labels.md @@ -0,0 +1,225 @@ +# Anti-Calibration Fix Plan — Aux Supervision + Regime Gate (HORIZONS unchanged) + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Lift trade Win Rate from current ~22% toward the 55%+ target by combining (1) aux supervision on cost-aware D-style asymmetric labels (E3 design with user's D-label choice) and (2) regime-conditional decision gate (per `pearl_snapshot_alpha_is_regime_conditional` — empirically validated). **HORIZONS = [10, 100, 1000] stays as-is** (the Smoke-1 validated set with auc_h1000=0.7137). Horizon rebase to [100,300,1000] is dropped; the K=10 noise issue identified by E1 is addressed via the regime gate, which masks out low-spread regimes where K=10 hurts most. + +**Architecture:** Two independent additive layers: +- **Layer B — Aux supervision**: separate aux trunk (smaller CfC, single bucket) sharing the encoder, supervised on D-style asymmetric loss-aversion labels per (long, short) direction. Asymmetric stop-grad. Separate Adam group. Decision policy gates trade entry on aux signal. +- **Layer C — Regime gate**: regime classification (spread quintile from ISV-driven trailing percentiles) at decision time; trade only in alpha-dense quintiles identified by Layer C1's NumPy investigation. + +**Tech Stack:** Rust + CUDA. Build via `SQLX_OFFLINE=true cargo check`. + +--- + +## Current state (start of plan) + +Branch: `ml-alpha-phase-a` HEAD `6d772a5d6`. Working tree has 5 uncommitted files from an aborted horizon-rebase attempt that should be reverted before starting: +``` +M crates/ml-alpha/examples/alpha_train.rs (log field renames) +M crates/ml-alpha/src/gpu_log.rs (JSON field renames) +M crates/ml-alpha/tests/gpu_log_ring_invariants.rs (test assertion rename) +M crates/ml-alpha/tests/smoothness_lambda_controller_invariants.rs (comments) +M crates/ml-backtesting/src/harness.rs (log labels) +``` +These were applied assuming HORIZONS=[100,300,1000]; with HORIZONS staying at [10,100,1000] they must be reverted to keep log/field names consistent with actual horizon values. + +**First action** (before any new code): `git checkout HEAD -- ` to restore consistency. + +## Layer ordering — sequential, attribution-clean + +Run Layer B first end-to-end (commit + Smoke 1 + Smoke 2 with gate OFF + gate ON). Then run Layer C end-to-end. Sequential ordering means each layer's WR contribution is attributable; bundling makes diagnosis hard. + +## Strategic context + +Per investigations completed this session: +- **E1 finding (root cause of WR drop)**: K=10 anchor produces noise-injection at inference time because labels are heavily tie-masked (sub-tick moves dropped). Layer C (regime gate) is expected to mitigate by routing K=10's noisy outputs out of the alpha-dense regimes. +- **E2 finding (trajectory output design)**: Candidate C (Huber log-return) was rejected — D3 falsified the smoothness argument; calibration/class-imbalance arguments remain but require BCE→Huber kernel surgery for unclear lift. Deferred. +- **E3 finding (aux supervision)**: A+B paired labels recommended; user chose D-style asymmetric loss-aversion labels `(direction × ΔP - cost) - 1.5 × |MaxDD_against|` per (long, short). Stronger alignment with trading objective. +- **WR analysis**: 55% target achievable but requires multiple stacked interventions. Layer B alone likely lifts WR to 40-50%; Layer C is essential to bridge to 55%+. Per `pearl_snapshot_alpha_is_regime_conditional`, alpha concentrates in ~20% of book states (spread-Q4 = 75% accuracy). + +## Sub-decisions resolved + +| # | Decision | Choice | Source | +|---|---|---|---| +| 1 | HORIZONS | [10, 100, 1000] (unchanged) | User revert; Layer C compensates for K=10 noise | +| 2 | Aux label form | D (asymmetric: `ΔP - 1.5×|MaxDD| - cost`) per (long, short) | User | +| 3 | Aux trunk hidden | 64 (vs main trunk 128) | E3 design default; revisit if starves | +| 4 | Stop-grad lifting | Conditional (lift if aux CE < 0.4 + dir_acc > 0.85 within 200 steps) | E3 | +| 5 | Cost model | Constant 2-tick round-trip (no slippage in v1) | E3 | +| 6 | MaxDD multiplier | 1.5× | E3 | +| 7 | Aux gate in policy | Disabled in v1 wire-up; enabled in v2 commit | feedback_smoke_validation_before_structural_priors | +| 8 | Layer C regime feature | Spread quintile (per pearl_snapshot_alpha_is_regime_conditional) | Empirical from data | +| 9 | Layer C alpha-dense regime(s) | TBD — Layer C1's NumPy investigation decides | C1 | +| 10 | Layer C cost-vs-alpha tradeoff at Q4 | Verified by C1 net-of-cost calculation | C1 | + +--- + +## Task breakdown + +### Pre-flight: revert inconsistent uncommitted state + +- [ ] **Step 1**: `git checkout HEAD -- crates/ml-alpha/examples/alpha_train.rs crates/ml-alpha/src/gpu_log.rs crates/ml-alpha/tests/gpu_log_ring_invariants.rs crates/ml-alpha/tests/smoothness_lambda_controller_invariants.rs crates/ml-backtesting/src/harness.rs` +- [ ] **Step 2**: Verify `git status --short` empty. +- [ ] **Step 3**: Verify `cargo check --workspace --all-targets` clean. + +### Layer B — Aux supervision with D-labels + +See the prior plan file `2026-05-22-horizon-rebase-n3-100-300-1000-and-aux-d-labels.md.bak` for B1-B8 task details; copy here without modification (HORIZONS=[10,100,1000] is the only context shift, all aux infrastructure scope unchanged). Tasks summary: + +- **B1**: Generate D-style labels `(direction × ΔP - cost) - 1.5 × |MaxDD_against|` per (long, short) with monotonic-deque forward MaxDD scan (`crates/ml-alpha/src/multi_horizon_labels.rs`) +- **B2**: Loader integration — extend `LabeledSequence` + `LoadedFile` with outcome labels +- **B3**: Aux trunk (64-hidden CfC, single bucket) — `crates/ml-alpha/src/cfc/aux_trunk.rs` + `cuda/aux_trunk.cu` (clone of cfc_step kernel with smaller hidden) +- **B4**: Aux heads (linear regression per long/short) + Huber loss — `cuda/aux_heads.cu` + `cuda/aux_loss.cu` +- **B5**: Trainer wiring — parallel aux trunk, separate Adam group, asymmetric stop-grad at encoder, conditional lift after gate G1 +- **B6**: Two-stage smoke validation — aux gate OFF first (validate training wiring), then aux gate ON +- **B7**: Decision policy aux gate — `crates/ml-backtesting/cuda/decision_policy.cu` gate `target_lots = 0 if predicted_outcome < threshold_aux × σ_local` +- **B8**: Layer-B pearls (k10 noise, D-label rationale, aux-trunk pattern, two-stage smoke) + +**Layer B exit gate**: Argo Smoke 2 with aux gate ON shows WR at conv [0.0-0.6] ≥ 30% AND not monotonically anti-calibrated (Spearman ρ > -0.3 across conv buckets). + +If Layer B exit gate fails (WR <30% or still strongly anti-cal), STOP and diagnose before Layer C — adding more interventions to a broken base wastes compute. + +**Estimated Layer B cost**: 60 hours implementation + 4 hours Argo cycles. + +### Layer C — Regime gate + +### Task C1: Regime data analysis (NumPy) + +**Cost**: ~2 hours. No GPU, no model training. + +- [ ] **Step 1: Compute per-regime cost-adjusted PnL on existing fxcache** + +Reuse the existing decoded `/tmp/acf_analysis/features.npz` (from prior C1+C3 investigations). For one fold (~600k ES MBP-10 snapshots): +- Compute spread at each snapshot: `spread = ask_px[0] - bid_px[0]` (already a feature) +- Quintile-bin all snapshots by spread (Q1=lowest, Q5=highest) +- For each (quintile, horizon-K), compute the per-snapshot label `delta = price[t+K] - price[t]` and the cost-adjusted profit IF we had taken the directionally-correct trade: `pnl_per_snapshot = |delta| - cost` (cost = 2×tick = 0.5) +- Aggregate: mean `pnl_per_snapshot` per quintile per horizon + +- [ ] **Step 2: Identify alpha-dense regime(s)** + +Output a 5×3 table of mean cost-adjusted PnL per (spread-quintile, horizon). Identify which quintiles have POSITIVE expected cost-adjusted PnL. + +Expected per `pearl_snapshot_alpha_is_regime_conditional`: Q4 (high spread) has best alpha but biggest cost. Q1 (low spread) has lowest cost but possibly weakest alpha. Middle quintiles likely sub-zero. + +- [ ] **Step 3: Decide gate set** + +If Q4 dominates: gate to Q4 only (~20% of trades, expected highest WR). +If Q1 and Q4 both positive: gate to extremes (Q1 ∪ Q4, ~40% of trades). +If middle quintiles dominate alpha (unlikely but possible): gate to alpha-rich middle. + +Output: `allowed_quintiles: HashSet`. + +### Task C2: Regime gate implementation + +**Cost**: ~6 hours. + +**Files**: +- Modify: `crates/ml-backtesting/cuda/decision_policy.cu` (add regime computation + gate) +- Modify: `crates/ml-backtesting/src/sim/mod.rs` (allocate regime state buffers) +- Modify: `crates/ml-backtesting/src/policy/mod.rs` (config plumbing for regime quintile set) +- Add: regime quintile thresholds via ISV-driven trailing-window percentiles (NOT hardcoded — `feedback_isv_for_adaptive_bounds`) + +- [ ] **Step 1: Compute regime at decision time** + +Add per-batch trailing-window stats for spread: +```cuda +// Per-batch rolling spread quintile via ISV: maintain spread_p20/p40/p60/p80 +// as Wiener-α EMAs of trailing-window quantile estimates (one update per event) +const float s = book.ask_px[0] - book.bid_px[0]; +const int q = (s < spread_p20[b]) ? 0 + : (s < spread_p40[b]) ? 1 + : (s < spread_p60[b]) ? 2 + : (s < spread_p80[b]) ? 3 : 4; +``` + +P-square quantile estimator (single-pass, O(1) per event, well-known algorithm) is the right structure. Alternative: maintain a small reservoir of last-N spread values and recompute quantiles every K events. + +- [ ] **Step 2: Gate logic** + +```cuda +if (!quintile_in_allowed_set(q, allowed_quintile_mask)) { + target_lots = 0; + return; +} +``` + +`allowed_quintile_mask: u8` bitmask (5 bits, 1 per quintile). Configured at backtest config load. + +- [ ] **Step 3: ISV diagnostic counters** + +Add per-quintile trade-attempt counter (diagnostic): `regime_trade_attempts[5]`, `regime_trades_executed[5]`. Logged at end of run for sanity check that the gate is actually firing as designed. + +### Task C3: Smoke validation + +**Cost**: ~1 hour wallclock (Argo). + +- [ ] **Step 1: Dispatch Smoke 2 with Layer B + C active** + +`./scripts/argo-lob-sweep.sh --grid config/ml/sweep_smoke_perhoriz_cfc.yaml --branch ml-alpha-phase-a` + +- [ ] **Step 2: Pull outcome_by_entry_conv + regime counters** + +Verify: +- **Primary gate**: WR at conv [0.0-0.6] ≥ 50% (the 55% target needs some breathing room for production noise) +- **Secondary**: trade count drops 60-80% from Layer B alone (expected — most trades filtered out) +- **Calibration**: `outcome_by_entry_conv` mean PnL across populated conv buckets monotonically improving with conviction (or at least non-negative) + +If WR doesn't lift to 50%+, diagnose: maybe the regime gate isn't selecting the right quintile(s), maybe cost model is wrong, maybe aux supervision needs more training. + +### Task C4: Pearls + +- [ ] **Step 1: Write 2 new pearls** + +- `pearl_regime_gate_essential_for_efficient_markets.md` — empirical validation: alpha concentrates in narrow regime band, trading equally across all regimes is a unforced WR loss +- `pearl_cost_adjusted_per_regime_pnl_decides_gate_set.md` — methodology pearl: deciding which regime to allow trades in is a per-(quintile, horizon) cost-adjusted expected-PnL computation, not a heuristic + +- [ ] **Step 2: Update MEMORY.md** + +--- + +## Validation summary + +| Gate | Layer | Pass condition | +|---|---|---| +| Library compile | All | `cargo check --workspace --all-targets` clean | +| Library tests | All | `cargo test -p ml-alpha --lib` 33 passed | +| Local GPU smoke | B5 | perception_overfit 4/4 pass (both BCE + aux losses → 0) | +| Argo Smoke 1 (Layer B wiring) | B6 | aux Huber < baseline (predicts mean); BCE not regressed | +| Argo Smoke 2 (Layer B gate OFF) | B6 | Anti-calibration ≥ as good as current Layer A baseline (no regression from aux wiring) | +| Argo Smoke 3 (Layer B gate ON) | B7 | WR at conv [0.0-0.6] ≥ 30%; not monotonically anti-calibrated | +| **Argo Smoke 4 (Layer C active)** | **C3** | **WR at conv [0.0-0.6] ≥ 50%** — the 55% target ceiling | + +--- + +## Risks + +| Risk | Likelihood | Mitigation | +|---|---|---| +| Layer B WR < 30% (aux supervision doesn't carry signal) | Medium | Layer B exit gate stops Layer C. Diagnose: maybe D-label cost is wrong, maybe MaxDD multiplier too aggressive. | +| Aux trunk plateaus at ln(K) (encoder starvation) | Medium | Conditional stop-grad lift mechanism. If still stuck, increase aux hidden to 96 or 128. | +| Aux gate causes Hold-collapse | Medium | D-label preserves direction info (paired long/short); threshold ISV-driven prevents stuck-closed gate. Verify trade count post-Smoke 3 (>50% of Layer B baseline). | +| Layer C Q4-only gate has cost > alpha (spread eats the move) | Medium | C1's NumPy investigation falsifies this BEFORE committing to gate set. If Q4 negative net-of-cost, fall back to Q1 (low cost, weak alpha) or middle (per data). | +| P-square quantile estimator unstable / slow to converge | Low | Alternative: reservoir-based recompute every K events (slower per-event but more stable). Decide in Task C2. | +| Cost model (constant 2-tick) under-states real cost | Medium | If anti-cal persists with constant cost, escalate to ISV-driven slip estimate per `pearl_isv_for_adaptive_bounds`. | +| Forward MaxDD scan O(N·K_max) is slow (K=1000) | Low | Monotonic deque amortized O(N). Profile on local CPU; if >5 min per 1M snapshots, switch to segment tree O(N log K). | +| Adam-summed loss balance no-op | High if implemented wrong | SEPARATE Adam group per trunk. Verify by checking optim.rs has truly disjoint Adam state. | +| Aux-only smoke shows no improvement (E3's anti-cal hypothesis wrong) | Medium | Honest diagnosis: maybe `pearl_separate_aux_trunk_when_shared_starves` lifts the stop-grad and we get back to a shared-trunk failure mode. Consider Candidate C (E2) trajectory output as next pivot. | + +--- + +## Estimated combined timeline + +- Pre-flight revert: 5 min +- Layer B implementation: ~60 hours +- Layer B smokes (3 Argo cycles): ~3 hours +- Layer B exit gate decision point: 30 min (does WR justify Layer C?) +- Layer C1 NumPy analysis: ~2 hours +- Layer C2 implementation: ~6 hours +- Layer C3 smoke: ~1 hour wallclock +- Pearls + commit: ~1 hour +- **Total: ~73 hours over ~2 weeks** + +## Execution + +After this plan is reviewed: dispatch subagent-driven-development via Skill tool. Layer B is independent of Layer C — STRONGLY RECOMMENDED to complete Layer B through Task B6 (validate aux wiring) before starting B7 (gate). And complete Layer B through Task B7 (smoke) before starting Layer C. diff --git a/docs/superpowers/plans/2026-05-22-horizon-rebase-n3.md b/docs/superpowers/plans/2026-05-22-horizon-rebase-n3.md new file mode 100644 index 000000000..83f936cb7 --- /dev/null +++ b/docs/superpowers/plans/2026-05-22-horizon-rebase-n3.md @@ -0,0 +1,521 @@ +# Horizon Rebase: N_HORIZONS 5 → 3 with HORIZONS = [10, 100, 1000] + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Reduce N_HORIZONS from 5 to 3 and shift HORIZONS from `[30, 100, 300, 1000, 6000]` to `[10, 100, 1000]` to match empirically-derived market autocorrelation elbows on ES MBP-10. Eliminates 3 horizons that sit below the input ACF noise floor (per C1 investigation) and resolves the chronic h6000 collapse (h6000 AUC plateaued at 0.50-0.74 even when other horizons hit 0.74+; ρ(30, 6000) = 0.064 in labels — no signal). + +**Architecture:** Atomic refactor of N_HORIZONS = 3 across ml-alpha (lib + tests + examples + CUDA kernels), ml-backtesting (locally-constant N_HORIZONS), and Argo workflow scripts (per-horizon log strings). BUCKET_DIM_K shifts from compile-time `[25,25,25,25,28]` to initial-hint `[43,43,42]`; runtime values remain ISV-driven via the existing τ-assignment kernel at Phase 1→2 transition (no new controller required — existing Controller A's bucket-assignment IS the ISV step). MAX_BUCKET_DIM raised from 28 to 96 to accommodate non-uniform τ distributions in 3 buckets. + +**Tech Stack:** Rust + CUDA. Build via `SQLX_OFFLINE=true cargo check`. Tests via `cargo test -p `. Smoke validation via Argo Workflows on Scaleway L40S. + +--- + +## Current state (in-progress at start of plan) + +Branch: `ml-alpha-phase-a` HEAD `0384f7674` (D1 signed-EMA fix) + `fe5d7a1ad` (D2 EMA bandwidth rename). + +Working tree has uncommitted partial work from a prior session: +- `crates/ml-alpha/src/heads.rs` — constants changed (`N_HORIZONS=3`, `HORIZONS=[10,100,1000]`) +- `crates/ml-alpha/src/cfc/bucket_routing.rs` — `MAX_BUCKET_DIM=96`, `BUCKET_DIM_K=[43,43,42]`, `BUCKET_CHANNEL_OFFSET=[0,43,86,128]` +- `crates/ml-alpha/tests/perception_overfit.rs` — `[1.0; 5]` → `[1.0; N_HORIZONS]`, `[f32; 5]` → `[f32; N_HORIZONS]` (replace_all done; cascade type errors may remain) +- `crates/ml-alpha/examples/alpha_train.rs` — log line `w_h30..w_h6000` → `w_h10..w_h1000` (partial; 4 hardcoded array literals at lines 465, 525, 534 + summary at 678 still pending) + +`cargo check -p ml-alpha` (library only) passes. `cargo check -p ml-alpha --all-targets` fails on tests/example. + +## File structure + +| File | Type | Action | +|---|---|---| +| `crates/ml-alpha/src/heads.rs` | source | ✓ constants changed (committed-pending) | +| `crates/ml-alpha/src/cfc/bucket_routing.rs` | source | ✓ constants changed (committed-pending) | +| `crates/ml-alpha/cuda/horizon_lambda.cu` | kernel | `#define N_HORIZONS_LAMBDA 5` → `3` (line 36); verify no other hardcoded `5` in body | +| `crates/ml-alpha/cuda/bce_loss_multi_horizon.cu` | kernel | `#define BCS_N_HORIZONS 5` → `3` (line 35); verify warp accumulator + shared mem sizes | +| `crates/ml-alpha/cuda/output_smoothness.cu` | kernel | Audit for hardcoded 5; should be parametric via N_HORIZONS uniform | +| `crates/ml-alpha/cuda/smoothness_lambda_controller.cu` | kernel | **CONFIRMED**: `SLC_N_HORIZONS = 5` (line 30) AND `TARGET_K_RATIO[]` (lines 45-51) uses old-horizon formula `sqrt(30/{30,100,300,1000,6000})`. Must change to `sqrt(10/{10,100,1000})` for new HORIZONS = [10,100,1000]. The kernel currently reads only slots [0..3] under N_HORIZONS=3, anchoring h1000's smoothness target to OLD h300 ratio (functional bug). | +| `crates/ml-alpha/cuda/multi_horizon_heads.cu` | kernel | Audit for `HORIZON_H` define vs hardcoded | +| `crates/ml-alpha/cuda/heads_block_diagonal_fwd.cu` | kernel | Audit | +| `crates/ml-alpha/cuda/cfc_step_per_branch.cu` | kernel | Audit | +| `crates/ml-alpha/cuda/bucket_transition_kernels.cu` | kernel | Audit for MAX_BUCKET_DIM dependencies (raised from 28 to 96) | +| `crates/ml-alpha/examples/alpha_train.rs` | binary | Replace 4 hardcoded `seq.labels[0..5]` array literals with `std::array::from_fn`; fix summary struct | +| `crates/ml-alpha/tests/perception_overfit.rs` | test | Verify after replace_all; may still have Vec<&[[f32; 5]]> cascading types | +| `crates/ml-alpha/tests/smoothness_lambda_controller_invariants.rs` | test | Semantic rewrite: 5-element fixtures `[0.5, 0.15, 0.05, 0.015, 0.0025]` → 3-element. Rename `excess_at_h6000_lifts_lambda_proportionally` → `excess_at_h1000_lifts_lambda_proportionally`. Re-derive expected values | +| `crates/ml-alpha/tests/output_smoothness_grad_finite_diff.rs` | test | ✓ `[0.1, 0.3, 1.0, 3.0, 10.0]` → `[0.1, 1.0, 10.0]` (committed-pending) | +| `crates/ml-alpha/tests/bucket_transition_kernels.rs` | test | May reference 28-element bucket sizes; audit for MAX_BUCKET_DIM=96 | +| `crates/ml-alpha/tests/perception_forward_golden.rs` | test | Golden values invalidated; regenerate golden | +| `crates/ml-alpha/tests/forward_step_golden.rs` | test | Golden values invalidated; regenerate golden | +| `crates/ml-alpha/tests/heads_block_diagonal.rs` | test | Audit MAX_BUCKET_DIM dependencies | +| `crates/ml-alpha/tests/heads_bit_equiv.rs` | test | Audit (uses N_HORIZONS, should be OK) | +| `crates/ml-alpha/tests/backward_finite_diff.rs` | test | Audit (uses N_HORIZONS, should be OK) | +| `crates/ml-backtesting/src/harness.rs:36` | source | Local `const N_HORIZONS: usize = 5` → `3` | +| `crates/ml-backtesting/src/artifacts.rs` | source | Audit (likely uses harness's N_HORIZONS) | +| `crates/ml-backtesting/src/lob/mod.rs` | source | Audit | +| `crates/ml-backtesting/src/policy/mod.rs` | source | Audit | +| `crates/ml-backtesting/src/sim/mod.rs` | source | Audit | +| `crates/ml-backtesting/cuda/decision_policy.cu` | kernel | Audit `N_HORIZONS` define source | +| `crates/ml-backtesting/tests/decision_floor_coldstart.rs` | test | Audit | +| `crates/ml-backtesting/tests/stop_controller.rs` | test | Audit (recently modified by D1) | +| `crates/ml-backtesting/tests/trainer_parity.rs` | test | Audit | +| `infra/k8s/argo/alpha-perception-template.yaml` | yaml | Audit script body for `h30/h100/h300/h1000/h6000` log strings; update or generalize | +| `infra/k8s/argo/alpha-cv-template.yaml` | yaml | Audit | + +--- + +## Task breakdown + +### Task 1: Verify partial in-flight changes are coherent + +**Files:** +- Read: `crates/ml-alpha/src/heads.rs` +- Read: `crates/ml-alpha/src/cfc/bucket_routing.rs` +- Read: `crates/ml-alpha/tests/perception_overfit.rs` +- Read: `crates/ml-alpha/examples/alpha_train.rs` + +- [ ] **Step 1: Run cargo check on library** + +``` +SQLX_OFFLINE=true cargo check -p ml-alpha +``` +Expected: PASS (constants already changed, library should compile). + +- [ ] **Step 2: Run cargo check on all targets to scope remaining errors** + +``` +SQLX_OFFLINE=true cargo check -p ml-alpha --all-targets 2>&1 | grep "error\[" | sort -u | head -30 +``` +Expected: error list confirms scope per the File structure table above. + +- [ ] **Step 3: If library doesn't compile, STOP and investigate.** Library compile failure means the in-flight changes have a fundamental error — must be diagnosed before proceeding. + +### Task 2a (INSERTED 2026-05-22): Migrate loader.rs to N_HORIZONS + +**Discovered during execution**: `crates/ml-alpha/src/data/loader.rs` has 5 hardcoded `[T; 5]` array types that must migrate FIRST before alpha_train.rs can compile. Also: `ml-backtesting/src/sim/mod.rs:1318` (`broadcast_alpha(&[f32; 5])`) and `ml-backtesting/src/harness.rs:509` (`[&str; 5]` log labels). + +**Files:** +- Modify: `crates/ml-alpha/src/data/loader.rs:118, 159, 170, 231, 364` +- Modify: `crates/ml-backtesting/src/sim/mod.rs:1318` (and any callers) +- Modify: `crates/ml-backtesting/src/harness.rs:509` (5-string log array) + +- [ ] **Step 1: Migrate loader struct fields** + +Line 118: +```rust +pub horizons: [usize; 5], +``` +→ Replace with: +```rust +pub horizons: [usize; crate::heads::N_HORIZONS], +``` + +Lines 159, 170: same pattern with `[Vec; 5]` → `[Vec; crate::heads::N_HORIZONS]`. + +Lines 231, 364: `Default::default()` already returns the correct type once the field type changes; no edit needed if it inferred the type. + +- [ ] **Step 2: Migrate ml-backtesting sim broadcast_alpha** + +Line 1318: +```rust +pub fn broadcast_alpha(&mut self, probs: &[f32; 5]) -> Result<()> { +``` +→ Replace with `[f32; N_HORIZONS]` (using ml-backtesting's local N_HORIZONS constant from harness.rs). + +Audit callers — they may construct `[f32; 5]` literals that cascade. + +- [ ] **Step 3: Migrate harness log labels** + +Line 509: +```rust +let horizons: [&str; 5] = ["30", "100", "300", "1000", "6000"]; +``` +→ Replace with: +```rust +let horizons: [&str; N_HORIZONS] = ["10", "100", "1000"]; +``` + +- [ ] **Step 4: cargo check both crates** + +``` +SQLX_OFFLINE=true cargo check -p ml-alpha -p ml-backtesting +``` +Expected: PASS. + +- [ ] **Step 5: Commit checkpoint** (no test changes yet, library + sim only) + +### Task 2: Fix alpha_train.rs hardcoded 5-arrays + +**Files:** +- Modify: `crates/ml-alpha/examples/alpha_train.rs:465-470, 525-540, 678` + +- [ ] **Step 1: Replace 5-element array literals with `std::array::from_fn`** + +Lines 465-470 currently: +```rust +let row = [ + seq.labels[0][k], seq.labels[1][k], seq.labels[2][k], + seq.labels[3][k], seq.labels[4][k], +]; +``` +Replace with: +```rust +let row: [f32; N_HORIZONS] = std::array::from_fn(|h| seq.labels[h][k]); +``` + +- [ ] **Step 2: Apply same pattern to lines 525-528 (`row`) and 534-537 (`last_labels`)** + +Use `std::array::from_fn` consistently. + +- [ ] **Step 3: Audit log lines for per-horizon AUC** + +Lines emitting `auc_h30=..., auc_h6000=...` need to become `auc_h10=..., auc_h1000=...`. Search for `auc_h` in alpha_train.rs and update all 5-named fields to 3-named. + +- [ ] **Step 4: cargo check -p ml-alpha --example alpha_train** + +``` +SQLX_OFFLINE=true cargo check -p ml-alpha --example alpha_train +``` +Expected: PASS. + +- [ ] **Step 5: Commit checkpoint** + +``` +git add crates/ml-alpha/src/heads.rs crates/ml-alpha/src/cfc/bucket_routing.rs crates/ml-alpha/examples/alpha_train.rs +git commit -m "refactor(per-horizon): N_HORIZONS 5→3, HORIZONS=[10,100,1000] — Phase 1 (lib+example)" +``` + +### Task 3: Fix perception_overfit.rs cascading types + +**Files:** +- Modify: `crates/ml-alpha/tests/perception_overfit.rs` + +- [ ] **Step 1: Re-run cargo check on this test** + +``` +SQLX_OFFLINE=true cargo check -p ml-alpha --test perception_overfit 2>&1 | grep "error\[" | head -20 +``` + +- [ ] **Step 2: Fix any remaining `Vec<[f32; 5]>` or `Vec<&[[f32; 5]]>` type signatures** + +Replace `[f32; 5]` → `[f32; N_HORIZONS]`, ensure import of N_HORIZONS at top. + +- [ ] **Step 3: cargo build the test** + +``` +SQLX_OFFLINE=true cargo build -p ml-alpha --tests --test perception_overfit +``` +Expected: PASS. + +### Task 4: Semantic rewrite of smoothness_lambda_controller_invariants.rs + +**Files:** +- Modify: `crates/ml-alpha/tests/smoothness_lambda_controller_invariants.rs` + +This test encodes specific h6000 behavior that no longer exists. The test fixtures must be re-derived for the [10, 100, 1000] horizon set. + +- [ ] **Step 1: Rewrite the 3 hardcoded 5-element arrays** + +Line 201: +```rust +let raw = [0.5_f32, 0.15, 0.05, 0.015, 0.0025]; +``` +Re-derive for 3 horizons preserving geometric decay matching horizon ratios. For HORIZONS=[10, 100, 1000] (geometric ratio 10): +```rust +let raw = [0.5_f32, 0.05, 0.005]; +``` + +Line 219: +```rust +let ema = [0.5_f32, 0.15, 0.05, 0.015, 0.0025]; +``` +Replace with: +```rust +let ema = [0.5_f32, 0.05, 0.005]; +``` + +Line 233 (excess test): +```rust +let ema = [0.5_f32, 0.15, 0.05, 0.015, 0.025]; // h6000 = 10× target +``` +Re-derive for h1000 = 10× target. With expected h1000 target = ema[h10] × 10/1000 = 0.005: +```rust +let ema = [0.5_f32, 0.05, 0.05]; // h1000 = 0.05 = 10× target (0.005) +``` + +- [ ] **Step 2: Rename the test function from `excess_at_h6000_lifts_lambda_proportionally` to `excess_at_h1000_lifts_lambda_proportionally`** + +- [ ] **Step 3: Update assertion indices (h6000 was index 4; h1000 is now index 2)** + +In the test body, replace `lambda[4]` (h6000) with `lambda[2]` (h1000). Update the comments referencing `h6000` to `h1000`. + +- [ ] **Step 4: Update assertions on h300/h1000 (indices 2, 3) — those horizons no longer exist** + +The test asserted `lambda[1], lambda[2], lambda[3]` all at base. With 3 horizons, only `lambda[1]` (h100) needs to stay at base; the `lambda[2]` is now h1000 (the excess case). + +- [ ] **Step 5: cargo test the test** + +``` +SQLX_OFFLINE=true cargo test -p ml-alpha --test smoothness_lambda_controller_invariants +``` +Expected: 3+ tests pass. + +### Task 5: CUDA kernel `#define` updates + +**Files:** +- Modify: `crates/ml-alpha/cuda/horizon_lambda.cu:36` +- Modify: `crates/ml-alpha/cuda/bce_loss_multi_horizon.cu:35` + +- [ ] **Step 1: Update horizon_lambda.cu** + +Line 36: +```cuda +#define N_HORIZONS_LAMBDA 5 +``` +Replace with: +```cuda +#define N_HORIZONS_LAMBDA 3 +``` + +- [ ] **Step 2: Update bce_loss_multi_horizon.cu** + +Line 35: +```cuda +#define BCS_N_HORIZONS 5 +``` +Replace with: +```cuda +#define BCS_N_HORIZONS 3 +``` + +Verify the comment on line 28 ("Hard-coded N_HORIZONS_BCE = 5 → per-warp 5-element accumulator stays in registers") is updated to reflect 3. + +- [ ] **Step 3: Audit other kernels for hardcoded 5** + +``` +grep -nE "^\s*for\s*\(\s*int\s+h\s*=\s*0;\s*h\s*<\s*5\b|HORIZONS\s*=\s*5\b" crates/ml-alpha/cuda/*.cu +``` +Expected: no hits (all loops should iterate to `N_HORIZONS` constant from cfc/bucket_routing.rs equivalent at the kernel level). + +- [ ] **Step 4: cargo build (compiles cubins)** + +``` +SQLX_OFFLINE=true cargo build -p ml-alpha +``` +Expected: PASS. Build script recompiles cubins. + +### Task 6: ml-backtesting N_HORIZONS update + +**Files:** +- Modify: `crates/ml-backtesting/src/harness.rs:36` +- Audit: `crates/ml-backtesting/src/{artifacts,lob,policy,sim}/**` +- Audit: `crates/ml-backtesting/cuda/decision_policy.cu` +- Audit: `crates/ml-backtesting/tests/*.rs` + +- [ ] **Step 1: Update the local constant** + +Line 36: +```rust +const N_HORIZONS: usize = 5; +``` +Replace with: +```rust +const N_HORIZONS: usize = 3; +``` + +- [ ] **Step 2: Audit for hardcoded 5 in ml-backtesting** + +``` +grep -rn "; 5\]\|, 5\b\|\[f32; 5\]\|\[0\.0; 5\]\|\[1\.0; 5\]" crates/ml-backtesting/ --include="*.rs" +``` +Update any matches to use the local `N_HORIZONS`. + +- [ ] **Step 3: Check decision_policy.cu for N_HORIZONS source** + +``` +grep -n "N_HORIZONS\|#define.*HORIZONS" crates/ml-backtesting/cuda/decision_policy.cu | head -5 +``` +If `N_HORIZONS` is defined via `lob_state.cuh`, update there. If hardcoded, replace with 3. + +- [ ] **Step 4: cargo check -p ml-backtesting** + +``` +SQLX_OFFLINE=true cargo check -p ml-backtesting --all-targets +``` +Expected: PASS (or list of remaining errors to fix iteratively). + +### Task 7: Argo workflow templates audit + +**Files:** +- Audit: `infra/k8s/argo/alpha-perception-template.yaml` +- Audit: `infra/k8s/argo/alpha-cv-template.yaml` + +- [ ] **Step 1: Find per-horizon log-name references in the workflow script body** + +``` +grep -nE "h30|h100|h300|h1000|h6000|auc_h|final_smooth_loss" infra/k8s/argo/alpha-*.yaml +``` +Update any explicit `auc_h6000=` or similar logging-name string to match the new horizons. + +- [ ] **Step 2: Audit summary.json parsing if any** + +If the workflow parses `alpha_train_summary.json` and reads a 5-element array, update to read 3 elements. + +- [ ] **Step 3: NO kubectl apply yet** — the YAML changes are committed but not deployed until validation completes. + +### Task 8: Audit + fix remaining test files + +**Files:** +- Audit: `crates/ml-alpha/tests/bucket_transition_kernels.rs` +- Audit: `crates/ml-alpha/tests/perception_forward_golden.rs` +- Audit: `crates/ml-alpha/tests/forward_step_golden.rs` +- Audit: `crates/ml-alpha/tests/heads_block_diagonal.rs` +- Audit: `crates/ml-alpha/tests/heads_bit_equiv.rs` +- Audit: `crates/ml-alpha/tests/backward_finite_diff.rs` +- Audit: `crates/ml-alpha/tests/cfc_step_per_branch.rs` +- Audit: `crates/ml-alpha/tests/forward_graph_capture.rs` + +- [ ] **Step 1: cargo check all tests** + +``` +SQLX_OFFLINE=true cargo check -p ml-alpha --tests 2>&1 | grep "error" | head -30 +``` +Iterate through any remaining type errors. Most should be 5-array literals or golden values. + +- [ ] **Step 2: Golden value invalidation** + +`perception_forward_golden.rs` + `forward_step_golden.rs` have golden output values that depend on N_HORIZONS. These golden tests will FAIL on real values even when types match. + +Two options: +(a) **Regenerate goldens** — run the forward pass on the new architecture, capture outputs, replace golden constants. Requires running the test once to capture, then committing the new values. +(b) **Mark `#[ignore]` with rationale** — temporarily skip until the architecture stabilizes. Per `feedback_no_hiding`, this is NOT preferred — but acceptable IF the test is followed by a `// REGENERATE after N_HORIZONS rebase` comment AND a separate task to actually regenerate before the next merge. + +Recommend (a) — regenerate. The new golden values are the new ground truth. + +- [ ] **Step 3: cargo test (local CPU-side tests only — skip CUDA on RTX 3050)** + +``` +SQLX_OFFLINE=true cargo test -p ml-alpha --lib +``` +Expected: 33 tests pass (unchanged from baseline). + +### Task 9: Final local validation + +- [ ] **Step 1: Full workspace check** + +``` +SQLX_OFFLINE=true cargo check --workspace --all-targets +``` +Expected: clean. + +- [ ] **Step 2: ml-alpha + ml-backtesting lib tests** + +``` +SQLX_OFFLINE=true cargo test -p ml-alpha --lib +SQLX_OFFLINE=true cargo test -p ml-backtesting --lib +``` +Expected: both pass. + +- [ ] **Step 3: RTX 3050 synthetic-overfit smoke** + +``` +SQLX_OFFLINE=true cargo test -p ml-alpha --test perception_overfit --release stacked_trainer -- --test-threads=1 --nocapture +``` +Expected: 4 tests pass, loss shrinks from ~2.3 → ~0 within budget. Confirms the new 3-horizon architecture trains end-to-end on local GPU. + +- [ ] **Step 4: Atomic commit if not already done** + +If Tasks 2-8 each committed separately, fine. If they accumulated, commit now: +``` +git add -A +git commit -m "refactor(per-horizon): N_HORIZONS 5→3, HORIZONS=[10,100,1000] — full propagation" +``` + +### Task 10: Push + dispatch Argo Smoke 1 + +- [ ] **Step 1: git push** + +``` +git push origin ml-alpha-phase-a +``` + +- [ ] **Step 2: Apply updated workflow template** + +``` +kubectl apply -f infra/k8s/argo/alpha-perception-template.yaml +``` +(Per `feedback_argo_template_must_apply.md` — `argo submit --from=wftmpl/` reads cluster CRD, not on-disk YAML. Must apply first.) + +- [ ] **Step 3: Dispatch Argo Smoke 1** + +``` +./scripts/argo-alpha-perception.sh --watch +``` +~25-30 min wallclock on L40S. + +**Falsification gate**: best `auc_h1000` (the new longest horizon) should reach **≥ 0.65** by epoch 4. If it collapses to chance (~0.50) as h6000 did with the heads_w1 mask, the 3-horizon architecture has its own failure mode and the plan is wrong. + +### Task 11: Dispatch Argo Smoke 2 (anti-calibration validation) + +- [ ] **Step 1: Wait for Smoke 1 checkpoint** + +Path: `/feature-cache/alpha-perception-runs//trunk_best_h1000.bin` +(Note: file name changes from `trunk_best_h6000.bin` to `trunk_best_h1000.bin` because the best-checkpoint metric is now the longest-horizon AUC.) + +- [ ] **Step 2: Update sweep YAML to point at new checkpoint** + +Update `config/ml/sweep_smoke_perhoriz_cfc.yaml` (or new equivalent) `checkpoint:` field to the new SHA's path. + +- [ ] **Step 3: Dispatch Argo lob-backtest-sweep** + +``` +./scripts/argo-lob-sweep.sh --config config/ml/sweep_smoke_perhoriz_cfc.yaml --watch +``` + +**Validation gates**: +- `outcome_by_entry_conv` table: NO LONGER monotonically anti-calibrated. High-conviction mean_pnl_pu should be ≥ mid-conviction mean_pnl_pu (or both close to zero — D1 fix may suppress high-conviction trades entirely when signed EMA → 0). +- CRT.diag per-horizon: 3 horizons reported (h10, h100, h1000) instead of 5. mean_run_len ratio h1000/h10 — informational only (the per-horizon-mean_run_len-differentiation thesis is no longer the architectural goal; if it shows up naturally, bonus). + +### Task 12: Memory pearls + +- [ ] **Step 1: Write pearls capturing this session's findings** + +Five new pearls to add (each as a single file under `/home/jgrusewski/.claude/projects/-home-jgrusewski-Work-foxhunt/memory/`): + +1. `pearl_input_acf_grounds_horizon_choice.md` — derive HORIZONS from empirical ES MBP-10 autocorrelation, not from intuition. C1 finding: hardcoded [30,100,300,1000,6000] had 3 horizons below noise floor. + +2. `pearl_binarization_preserves_smoothness.md` — D3 finding: the "BCE binarization destroys cross-horizon smoothness" intuition is empirically wrong. Sign-of-log-return preserves ~88% of available cross-horizon Pearson correlation under bivariate Gaussian assumption. If smoothness matters, it must come from somewhere other than the output formulation. + +3. `pearl_conviction_must_match_action_signal.md` — D1 finding: reporting `EMA(|x|)` while sizing via `sign(instantaneous) × EMA(|x|)` produces anti-calibrated outputs. Conviction reporting must use the same smoothed signal the action uses. + +4. `pearl_block_diagonal_mask_starves_long_horizon.md` — the heads_w1 block-diagonal mask catastrophically regressed h6000 AUC (0.73 → 0.54). Long-horizon prediction NEEDS cross-channel feature integration; structurally restricting it to bucket-assigned channels removes the integrative capacity. + +5. `pearl_loader_ema_bandwidth_assumes_snapshot_rate.md` — D2 finding: EMA constants derived from "9s @ 250ms" assumed 250ms/snapshot, but empirical median Δt is 74µs. The actual half-lives were 3000× shorter than documented. ANY constant derived from a wall-clock target needs the actual event rate as input. + +- [ ] **Step 2: Update MEMORY.md index** + +Add 5 one-line entries to the index, ordered under appropriate sections. + +## Validation summary + +After Task 11 completes, the success criteria are: + +| Gate | Pass condition | +|---|---| +| Library compile | `cargo check --workspace --all-targets` clean | +| Library tests | `cargo test -p ml-alpha --lib` 33 passed | +| Local GPU smoke | perception_overfit 4/4 pass, loss → ~0 | +| Argo Smoke 1 | `auc_h1000 ≥ 0.65` at best epoch | +| Argo Smoke 2 anti-calibration | high-conviction `mean_pnl_pu ≥ mid-conviction mean_pnl_pu` | +| Argo Smoke 2 differentiation | informational only (not a gate) | + +## Risks + +| Risk | Likelihood | Mitigation | +|---|---|---| +| MAX_BUCKET_DIM=96 wastes GPU memory (was 28) | Medium | Profile after smoke; if memory-bound, drop to 64 or implement runtime sizing | +| h1000 collapses to chance like h6000 did with mask | Medium | Falsification gate at Task 10 Step 3. If hit, the 3-horizon architecture itself is wrong; pivot to architecture C or D from earlier brainstorm. | +| Anti-calibration persists despite D1 fix | Medium | Suggests deeper issue than EMA mechanism; consider Candidate B (hold-duration prediction) as next architecture step | +| Golden value regeneration introduces subtle bug | Low | Diff old vs new golden; sanity-check distributional properties | +| Old checkpoints can't load into new model | Certain | Accepted — clean break, per `feedback_no_legacy_aliases` | +| Smoothness controller test re-derivation has bugs | Medium | Test invariants (boundedness, monotonicity) not specific values per `pearl_tests_must_prove_not_lock_observations` | + +## Execution + +After plan saved: dispatch to a coder subagent in the current worktree with this plan as the brief, OR use the subagent-driven-development skill for fresh-subagent-per-task execution with review checkpoints. diff --git a/docs/superpowers/plans/2026-05-22-layer-c-regime-gate.md b/docs/superpowers/plans/2026-05-22-layer-c-regime-gate.md new file mode 100644 index 000000000..18559ec09 --- /dev/null +++ b/docs/superpowers/plans/2026-05-22-layer-c-regime-gate.md @@ -0,0 +1,229 @@ +# Layer C: Regime Gate — Trade Only In Alpha-Dense Book States + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development to implement this plan task-by-task. + +**Goal:** Lift trade WR from current ~22% baseline by gating trade entry on book-state regime (spread quintile). Per `pearl_snapshot_alpha_is_regime_conditional`, alpha concentrates in ~20% of book states; spread-Q4 hits 75% accuracy unconditionally. Layer C does NOT depend on aux supervision — directly attacks WR by filtering out the 80% of states where the model has no edge. + +**Architecture:** Two-task pipeline: +1. **C1 — NumPy investigation** (no compute) — empirically determine which spread quintile(s) have net-of-cost positive expected PnL on the actual ES data the cluster trains on. Decides the gate configuration. +2. **C2 — Decision policy gate** — implement spread-quintile gate in `crates/ml-backtesting/cuda/decision_policy.cu` using ISV-driven trailing-window percentiles (P-square estimator). +3. **C3 — Argo Smoke validation** — backtest with gate active using existing checkpoint; compare outcome_by_entry_conv to baselines. + +**Tech Stack:** Python+NumPy for C1, Rust+CUDA for C2, Argo+L40S for C3. + +## Pre-conditions + +- Branch: `ml-alpha-phase-a` HEAD `1764cc394` +- F1 (dir_acc metric fix) RETAINED — empirically helpful +- F2 (mid_price NaN) REVERTED — wrong hypothesis +- Aux supervision infrastructure (CB1-CB5) LANDED but not consumed by decision policy +- D1 signed-conviction EMA fix (commit `0384f7674`) IS in baseline — handles part of anti-calibration + +## Strategic rationale + +Why Layer C over more aux iterations: +- **F4 synthesis verdict**: 70% implementation bugs / 30% architectural ceiling. Aux supervision continues to oscillate near chance regardless of label formulation. +- **Layer C is INDEPENDENT** of aux supervision. Gates on book state, not predicted profitability. Different mechanism entirely. +- **Empirical support exists**: `pearl_snapshot_alpha_is_regime_conditional` shows spread-Q4 = 75% accuracy directly from data analysis on ES MBP-10. +- **Layer C is COMPATIBLE with aux**: future aux work can layer on top of the regime gate (gate→aux predicts→size). + +## Sub-decisions (resolved with defaults) + +| # | Decision | Default | Rationale | +|---|---|---|---| +| 1 | Regime feature | Spread quintile | Per pearl: spread is the strongest single regime-conditional signal in ES MBP-10 | +| 2 | Quintile estimator | P-square (single-pass, O(1) per event) | Per E3+plan: avoids reservoir recompute overhead | +| 3 | Trailing window for quintile thresholds | EMA-tracked p20/p40/p60/p80 with α=2/(N+1), N=10000 events | Long enough to be stable across session boundaries, short enough to track regime shifts | +| 4 | Allowed quintile set | TBD by C1 — likely Q4 only OR Q1+Q4 union | Decision depends on cost-adjusted PnL per quintile | +| 5 | Gate scope | Per-batch (each parallel sim has own regime state) | Necessary for batched backtest correctness | +| 6 | Gate config plumbing | New `allowed_quintile_mask: u8` (5 bits) in policy config | Bitset more flexible than single quintile id | + +## Task breakdown + +### Task C1 — Per-regime cost-adjusted PnL analysis (NumPy) + +**Files:** +- Create: `/tmp/layer_c1_quintile_pnl.py` (analysis script, not committed) + +**Cost:** ~2 hours wallclock. + +- [ ] **Step 1: Decode ES MBP-10 records on local data** + +```python +# Reuse local file /home/jgrusewski/Work/foxhunt/test_data/futures-baseline-mbp10/ES.FUT/ES.FUT_2024-Q1.dbn.zst +# Decode ~2M records to span multiple sessions (avoid the gap-mismatch we saw in Step 2 of diagnosis plan) +``` + +- [ ] **Step 2: Compute spread at each snapshot** + +```python +spread = ask_px[0] - bid_px[0] # filter sentinels (price > 1e7) +``` + +- [ ] **Step 3: Compute spread quintiles via streaming p20/p40/p60/p80** + +Use simple percentile-of-trailing-N-window for the NumPy version (P-square comes later in C2). Window = 10000 events. + +- [ ] **Step 4: Per (quintile, horizon), compute expected directionally-optimal PnL net of cost** + +For each (snapshot t, K, quintile q): +``` +delta = mid[t+K] - mid[t] +optimal_dir = sign(delta) +gross_pnl = |delta| +net_pnl = gross_pnl - cost # take if positive, else zero (assume policy is smart enough not to trade) +``` + +Aggregate `mean(net_pnl)` per (quintile, K). Positive means "this regime has alpha after cost". + +- [ ] **Step 5: Decide gate set** + +Output `allowed_quintile_mask: u8`. Default decision rule: +- Include quintile q if `mean(net_pnl) > 0` for ANY horizon +- Exclude middle quintiles where alpha is below noise + +- [ ] **Step 6: Sanity check + write findings** + +If Q4 has the largest cost-adjusted PnL (per pearl), confirm and use `0b10000 = 0x10`. +If Q1+Q4 both positive, use `0b10001 = 0x11`. + +### Task C2 — Decision policy regime gate implementation + +**Files:** +- Modify: `crates/ml-backtesting/cuda/decision_policy.cu` (~80 LOC for gate + P-square + per-batch state) +- Modify: `crates/ml-backtesting/src/sim/mod.rs` (allocate regime state buffers + `allowed_quintile_mask` upload path) +- Modify: `crates/ml-backtesting/src/policy/mod.rs` (Strategy config field `regime_gate_quintile_mask: u8`) +- Modify: `crates/ml-backtesting/src/harness.rs` (per-quintile trade counter logging) + +**Cost:** ~8 hours. + +- [ ] **Step 1: Add per-batch P-square state** + +```cuda +// In LobSim state struct: +struct RegimeState { + float p_estimates[4]; // p20, p40, p60, p80 + float p_heights[5]; // marker positions for P-square + int n_seen; // step count for warmup +}; +``` + +- [ ] **Step 2: P-square quantile update** + +Standard Jain-Chlamtac P² algorithm. Per event: +```cuda +__device__ void p_square_update(RegimeState* st, float new_value) { + if (st->n_seen < 5) { st->p_heights[st->n_seen] = new_value; ... } + // Else: increment markers, adjust positions, parabolic adjust outer markers +} +``` + +- [ ] **Step 3: Spread quintile classification at decision time** + +```cuda +const float s = book.ask_px[0] - book.bid_px[0]; +RegimeState* st = ®ime_state[b]; +p_square_update(st, s); +const int q = (st->n_seen < 100) ? 4 : ( + (s < st->p_estimates[0]) ? 0 + : (s < st->p_estimates[1]) ? 1 + : (s < st->p_estimates[2]) ? 2 + : (s < st->p_estimates[3]) ? 3 : 4 +); +``` + +Warmup: first 100 events use Q4 (most permissive) to avoid blocking trading before quintiles stabilize. + +- [ ] **Step 4: Gate logic** + +```cuda +const bool allowed = (allowed_quintile_mask & (1u << q)) != 0; +if (!allowed) { + target_lots = 0; + regime_trades_filtered[b][q] += 1u; // diagnostic counter + return; +} +regime_trades_executed[b][q] += 1u; +``` + +- [ ] **Step 5: Rust wiring** + +- Strategy config: `pub regime_gate_quintile_mask: u8` (default = 0xFF = all allowed = no-op gate) +- Sim: allocate `regime_state_d`, `regime_trades_filtered_d`, `regime_trades_executed_d` +- Harness: log per-quintile counters at end of run + +- [ ] **Step 6: GPU oracle tests** + +- `regime_p_square_converges_to_true_quintiles` — on a known-distribution stream, verify P-square estimates match true p20/p40/p60/p80 within 5% +- `regime_gate_filters_out_disallowed_quintiles` — with mask = 0x10 (Q4 only), verify ~80% of trade attempts are filtered + +### Task C3 — Argo Smoke 2 with regime gate + +**Files:** +- Modify: `config/ml/sweep_smoke_perhoriz_cfc.yaml` (add `regime_gate_quintile_mask: 0x10` or whatever C1 decided) + +**Cost:** ~1 hour wallclock. + +- [ ] **Step 1: Push + apply workflow template (if needed)** + +- [ ] **Step 2: Dispatch sweep** + +```bash +./scripts/argo-lob-sweep.sh --grid config/ml/sweep_smoke_perhoriz_cfc.yaml --branch ml-alpha-phase-a +``` + +- [ ] **Step 3: Analyze outcome_by_entry_conv** + +Compare to baselines: +- D1 baseline (`lob-backtest-sweep-tcdxp`): WR 22%, anti-calibrated +- Layer C result: WR target ≥35% with similar or higher mean PnL + +**Pass gate:** WR at top-decile conv buckets ≥ 40% AND mean PnL closer to zero than D1 baseline. +**Fail gate:** trade count collapses >95% (gate too restrictive) OR WR doesn't improve. + +### Task C4 — Pearls + +- [ ] Write pearl_regime_gate_essential_for_efficient_markets +- [ ] Write pearl_cost_adjusted_per_regime_pnl_decides_gate_set +- [ ] Document the F1/F2 aux supervision falsification chain + +## Validation gates + +| Gate | Layer | Pass condition | +|---|---|---| +| Library compile | C2 | `cargo check --workspace --all-targets` clean | +| GPU oracle tests | C2 | regime_p_square + regime_gate tests pass | +| Trade count | C3 | drops 50-90% (expected — only 1-2 quintiles allowed) | +| WR improvement | C3 | WR ≥ 40% in top conv buckets (baseline 22%) | +| Net PnL | C3 | mean PnL > -50% of baseline (much closer to zero) | + +## Risks + +| Risk | Mitigation | +|---|---| +| Q4 has highest spread → highest entry/exit cost | C1's net-of-cost computation explicitly handles this. If Q4 net < Q1 net, gate to Q1. | +| P-square slow to converge on first 100 events | Warmup permits all trades (Q4 fallback). Trades during warmup are a small fraction of total. | +| Trade count drops 99% — gate too tight | Fallback: relax to Q1∪Q4 union. C1 outputs the union mask if needed. | +| Per-batch regime state breaks captured CUDA graph | P-square is per-event scalar update — no host branches, no pointer changes. Compatible with graph capture per `pearl_no_host_branches_in_captured_graph`. | +| Regime feature drift across sessions | EMA window N=10000 events handles intra-session drift. Daily regime shifts ARE captured (boundary spreads inflate Q5 temporarily). | + +## Estimated combined timeline + +- Task C1 (NumPy): ~2 hours +- Task C2 (CUDA + Rust): ~8 hours +- Task C3 (Argo): ~1 hour wallclock +- Task C4 (Pearls): ~1 hour +- **Total: ~12 hours** + +## Out-of-scope + +- **Aux supervision wiring into decision policy** — that was B7; deferred indefinitely per the F4 falsification verdict +- **CB6 per-horizon Huber kernel split** — deferred +- **Layer C2 regime gate based on imbalance or OFI** — start with spread per pearl; revisit if WR target not hit +- **Multiple-feature gates** — could compose later (spread AND volatility AND time-of-day) + +## Execution + +After plan saved: dispatch task-by-task via subagent-driven-development. + +C1 is independent and can run first. C2 starts immediately after C1's gate mask decision lands. C3 validates the combined work. diff --git a/docs/superpowers/plans/2026-05-22-loader-instrument-filter-fix.md b/docs/superpowers/plans/2026-05-22-loader-instrument-filter-fix.md new file mode 100644 index 000000000..246955ac7 --- /dev/null +++ b/docs/superpowers/plans/2026-05-22-loader-instrument-filter-fix.md @@ -0,0 +1,149 @@ +# Loader Instrument Filter Fix — Restore Clean Single-Instrument Baseline + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development. + +**Goal:** Add an `instrument_id` filter to the MBP-10 decoder so the loader processes only front-month ES (instrument_id=17077, ~74% of records) instead of all 14 instruments in the file. This is upstream of EVERY training failure we've diagnosed this session — fixing it must come before any further architectural iteration. + +**Architecture:** Decoder-level filter (3-4 files), no model changes. Re-train baseline. Compare new auc_h1000 to prior contaminated 0.7137 to quantify the contamination impact. + +## Discovery summary + +C1 NumPy analysis on `test_data/futures-baseline-mbp10/ES.FUT/ES.FUT_2024-Q1.dbn.zst` revealed: +- 14 distinct `instrument_id` values in the file +- 74.18% records have id=17077 (front-month ES, likely ESH4 for Q1 2024) +- 23.42% records have id=5602 (back-month ES, likely ESM4) +- ~3% records from 12 other instruments (calendar spreads, micro-ES, etc.) +- Loader reads ALL records sequentially without filtering +- K-windows spanning across instruments produce ΔP = $5000+ → spurious y_prof=1 labels +- This explains: pos_fraction inflation (20% → 35%), aux supervision failures, possibly the BCE plateau + +## File scope (3 files modified, 0 new) + +| File | Action | +|---|---| +| `crates/data/src/providers/databento/mbp10.rs` | Add `pub instrument_id: u32` to `Mbp10Snapshot`; populate in `from_*` constructors | +| `crates/ml-alpha/src/data/loader.rs` | Add `instrument_id_filter: Option` field to `MultiHorizonLoaderConfig`; skip records where snapshot.instrument_id != filter (when Some) | +| `crates/ml-alpha/examples/alpha_train.rs` | Add CLI flag `--instrument-id`; default to None (legacy behavior); document recommendation to use 17077 for ES | +| `crates/ml-backtesting/src/harness.rs` | Pass `instrument_id_filter` through MultiHorizonLoaderConfig (if applicable to inference path) | + +## Sub-decisions + +| # | Decision | Default | Rationale | +|---|---|---|---| +| 1 | Default filter behavior | `None` (no filter, legacy) | Conservative; users opt-in to filtering. Avoids breaking existing checkpoints' implicit assumption. | +| 2 | Filter value for ES | 17077 | Empirically the front-month ES in 2024-Q1. Documented in CLI flag help text. | +| 3 | Where to filter | At decoder level (skip the snapshot during deserialization) | Avoids polluting downstream code; loader sees only filtered records. | +| 4 | Reporting | Log the number of records kept vs skipped per file at load time | Diagnostic; confirms filter is firing. | + +## Task breakdown + +### Task L1 — Add `instrument_id` to Mbp10Snapshot + +**Files:** +- Modify: `crates/data/src/providers/databento/mbp10.rs` + +- [ ] **Step 1: Read the file**, understand `Mbp10Snapshot` struct and its construction sites. +- [ ] **Step 2: Add field** `pub instrument_id: u32` to the struct. +- [ ] **Step 3: Populate** in all decoder constructors. Databento MBP-10 records have `instrument_id: u32` field directly. +- [ ] **Step 4: Verify** existing tests still pass with the new field. + +### Task L2 — Filter at decoder/loader level + +**Files:** +- Modify: `crates/ml-alpha/src/data/loader.rs` + +- [ ] **Step 1: Add** `instrument_id_filter: Option` to `MultiHorizonLoaderConfig`. +- [ ] **Step 2: Filter** records in `LoadedFile::open` (or equivalent decode step): + ```rust + let snapshots: Vec = raw + .into_iter() + .filter(|s| cfg.instrument_id_filter.is_none() + || cfg.instrument_id_filter == Some(s.instrument_id)) + .collect(); + ``` +- [ ] **Step 3: Log** filter statistics: + ```rust + let kept = snapshots.len(); + let skipped = raw_count - kept; + tracing::info!(kept, skipped, "instrument filter applied"); + ``` +- [ ] **Step 4: Verify** existing in-file tests pass when filter=None (legacy behavior unchanged). + +### Task L3 — CLI flag exposure + +**Files:** +- Modify: `crates/ml-alpha/examples/alpha_train.rs` + +- [ ] **Step 1: Add** `--instrument-id ` CLI flag (default: None). +- [ ] **Step 2: Pass through** to MultiHorizonLoaderConfig. +- [ ] **Step 3: Document** in flag help text: "Filter MBP-10 records to a single instrument. For ES.FUT, use 17077 (front-month). Without this flag, ALL instruments in the file are processed sequentially — multi-instrument deltas inflate labels." + +### Task L4 — Backtesting harness + +**Files:** +- Modify: `crates/ml-backtesting/src/harness.rs` + +- [ ] **Step 1: Check** if the harness uses MultiHorizonLoaderConfig (it does per CB2). +- [ ] **Step 2: Add** the field to the inference-only construction site (default None for backward compat). + +### Task L5 — Smoke validation + +- [ ] **Step 1: Local synthetic** — perception_overfit still passes (filter is None for synthetic). +- [ ] **Step 2: Commit + push**. +- [ ] **Step 3: Dispatch Argo Smoke 1** with `--instrument-id 17077`. + +```bash +./scripts/argo-alpha-perception.sh --instrument-id 17077 # if the script forwards the flag +# OR submit via argo with workflow parameter override +``` + +**Validation gates**: +- auc_h1000 ≥ 0.65 (baseline gate) +- auc_h1000 IDEALLY > 0.7137 (improvement over contaminated baseline) +- Aux metrics: pos_fraction should DROP from 0.35-0.46 to closer to local's 0.20 + +If auc_h1000 INCREASES, contamination was real and significant. +If auc_h1000 stays the same, the cross-instrument contamination didn't affect the trunk's per-horizon ranking (just the aux labels). +If auc_h1000 DROPS, something else is going on. + +### Task L6 — Document findings + pearls + +- [ ] **Step 1: Write pearl** `pearl_multi_instrument_contamination_in_databento_files` — DBN files for "ES.FUT" symbol can contain calendar spreads + related futures; loaders MUST filter by instrument_id. +- [ ] **Step 2: Update MEMORY.md** index. + +## Validation + +- `cargo check --workspace --all-targets` clean +- 43 lib tests pass +- perception_overfit 4-5 pass (synthetic uses None filter) +- Argo Smoke 1 with `--instrument-id 17077` returns +- auc_h1000 documented vs prior 0.7137 baseline + +## After this fix + +The session's prior diagnostics need re-validation against the cleaner baseline: +- **F1 dir_acc fix** — should still help (metric design issue, orthogonal to data quality) +- **D1 signed-conviction EMA** — should still help (policy-side fix, no data dependency) +- **F2 mid_price NaN** — already reverted (wrong hypothesis) +- **Aux supervision (CB1-CB5)** — re-evaluate after clean baseline; the labels were contaminated so all aux iterations were doomed +- **Layer C regime gate** — re-evaluate; spread regime might mean something different once instruments don't mix + +## Strategic upside (deferred to follow-up plan) + +After clean single-instrument baseline: +- Multi-instrument training as FEATURE: include front+back-month spread as input +- Per-instrument trunks with shared encoder (the architecture B3+B4 already built supports this — aux trunk could become "back-month trunk") +- Calendar spread prediction as a separate output head +- Cross-asset signals from the 3% miscellaneous instruments + +This is a significant architectural opportunity but requires the clean baseline first. + +## Estimated cost + +- Tasks L1-L5: ~5 hours total +- L1 (decoder field): 1h +- L2 (loader filter): 1h +- L3 (CLI): 0.5h +- L4 (harness): 0.5h +- L5 (smoke): 1.5h + 12 min wallclock +- L6 (pearl): 0.5h