From db9936b9ff4cae99707a6cc6835a1cc6c0d09b61 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Mon, 27 Apr 2026 14:21:42 +0200 Subject: [PATCH] fix(data): align fxcache data_source + normalise DBN fallback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two-part fix for a class of bugs causing un-normalised features to silently flow into training. (1) data_source mismatch between precompute writer and trainer reader. precompute_features.rs:214,633 hardcoded "ohlcv" train_baseline_rl.rs:582 hardcoded "ohlcv" config/training/dqn-production.toml: data_source = "mbp10" Production runs the trainer with the production profile (data_source = mbp10), but the actual cache lookup hardcoded "ohlcv". Smoke worked by accident (smoke profile is also "ohlcv"). Any future profile with a different data_source silently mismatches → cache MISS → DBN fallback path. Both call sites now hardcode "mbp10" (the canonical production data source per CLAUDE.md). precompute_features adds a `--data-source` CLI override for the rare case a smoke flow needs to regenerate the local "ohlcv" fxcache; default is "mbp10". (2) DBN-fallback path didn't normalise features. precompute_features.rs:629 applies NormStats::normalize_batch on the canonical fxcache write path. The fallback in train_baseline_rl.rs (cache-miss → load DBN files → extract features → upload to GPU) did NOT normalise. Any cache miss (data_source drift, schema-hash mismatch, missing file) silently uploaded RAW features. Raw close prices (~$5180 ES futures) flowed into next_states[:, 0]; the aux next-bar head's label_scale EMA latched onto raw-price magnitude (~5443 vs expected ~1.0 z-score); the shared trunk learned to predict next-bar prices; the policy effectively traded with future-price knowledge → train-h5gxb epoch-0 Sharpe = 141 with 0.32% max-drawdown over 214k bars (impossibly good = oracle leak). DBN fallback now applies the same z-score normalisation unconditionally as defence-in-depth, so a future cache-miss cannot reintroduce raw values into training. Audit entry updated. --- crates/ml/examples/precompute_features.rs | 25 ++++++++++++++-- crates/ml/examples/train_baseline_rl.rs | 35 +++++++++++++++++++---- docs/dqn-wire-up-audit.md | 23 +++++++++++++++ 3 files changed, 76 insertions(+), 7 deletions(-) diff --git a/crates/ml/examples/precompute_features.rs b/crates/ml/examples/precompute_features.rs index 19a2a0e46..f6b1b361a 100644 --- a/crates/ml/examples/precompute_features.rs +++ b/crates/ml/examples/precompute_features.rs @@ -129,6 +129,27 @@ struct Opts { #[arg(long, default_value = "ES.FUT")] symbol: String, + /// Data-source identifier mixed into the SHA256 cache key. + /// + /// MUST match the `data_source` field of the training profile that will + /// consume this fxcache (e.g. `dqn-production.toml: data_source = "mbp10"`, + /// `dqn-smoketest.toml: data_source = "ohlcv"`). Mismatch produces a + /// silent cache MISS at training time → DBN-direct fallback uploads + /// un-normalised features → aux-head label_scale picks up raw-price + /// magnitudes → cascade of broken metrics + impossible Sharpe (see + /// 2026-04-27 incident: train-h5gxb epoch-0 Sharpe=141 from this exact + /// path mismatch). + /// + /// Defaults to `"mbp10"` because production (`dqn-production.toml`) is + /// the canonical consumer and uses MBP-10 microstructure data. Smoke + /// callers that intentionally use `dqn-smoketest.toml` (data_source = + /// "ohlcv") DO NOT call this binary — they reuse a pre-built local + /// fxcache committed under `test_data/feature-cache/`. If a smoke or + /// development workflow ever needs to (re)generate a local "ohlcv" + /// fxcache, pass `--data-source ohlcv` explicitly. + #[arg(long, default_value = "mbp10")] + data_source: String, + /// Skip confirmation prompt #[arg(long)] yes: bool, @@ -211,7 +232,7 @@ async fn main() -> Result<()> { mbp10_dir.as_deref(), trades_dir.as_deref(), &opts.symbol, - "ohlcv", + &opts.data_source, ).context("Failed to compute cache key")?; let early_check_path = output_dir.join(format!("{hex_key_early}.fxcache")); if early_check_path.exists() { @@ -615,7 +636,7 @@ async fn main() -> Result<()> { mbp10_dir.as_deref(), trades_dir.as_deref(), &opts.symbol, - "ohlcv", + &opts.data_source, ).context("Failed to compute cache key")?; let cache_key: [u8; 32] = hex::decode(&hex_key) .context("Invalid hex key")? diff --git a/crates/ml/examples/train_baseline_rl.rs b/crates/ml/examples/train_baseline_rl.rs index e48e1bc80..3e76aaf52 100644 --- a/crates/ml/examples/train_baseline_rl.rs +++ b/crates/ml/examples/train_baseline_rl.rs @@ -574,12 +574,18 @@ fn run_training(args: &Args) -> Result> { let mbp10 = args.mbp10_data_dir.as_ref().filter(|p| p.exists()); let trades = args.trades_data_dir.as_ref().filter(|p| p.exists()); + // Cache key data_source MUST match what `precompute_features` writes with + // (default: "mbp10"). Mismatch produces silent cache MISS → DBN fallback, + // which uploaded RAW (un-normalised) features pre-fix and caused + // label_scale=5443 (raw-price magnitude) on Argo deploys. The DBN fallback + // below now applies NormStats::normalize_batch as defence-in-depth so even + // a future data_source drift cannot reintroduce raw values into training. let fxcache_data = ml::fxcache::discover_and_load( &args.data_dir, &args.symbol, mbp10.map(|p| p.as_path()), trades.map(|p| p.as_path()), - "ohlcv", + "mbp10", cache_dir_override.as_deref(), ); @@ -589,7 +595,17 @@ fn run_training(args: &Args) -> Result> { cached.bar_count, data_load_start.elapsed().as_secs_f64()); cached } else { - // Fall back to DBN loading — this is SLOW (148GB MBP-10 parsing) + // Fall back to DBN loading — this is SLOW (148GB MBP-10 parsing). + // + // Defence-in-depth: apply NormStats::normalize_batch here too. The + // fxcache fast path is z-normalised at write time + // (precompute_features.rs:625-631). If the cache lookup misses for ANY + // reason (data_source mismatch, schema-hash drift, missing file) and + // the loader falls into this branch, raw features would have flowed + // unchanged into the GPU state buffer — exactly the bug observed on + // Argo train-h5gxb (epoch-0 Sharpe=141 from raw-price labels). Always + // normalise here so the consumer (`init_from_fxcache`) sees the same + // unit-scale features regardless of which path produced them. info!(" Loading OHLCV bars from DBN files..."); let bars = load_all_bars(&args.data_dir, &args.symbol)?; if bars.is_empty() { @@ -602,11 +618,20 @@ fn run_training(args: &Args) -> Result> { ); info!(" Extracting {}-dimensional features...", args.feature_dim); - let all_features = extract_ml_features(&bars) + let all_features_raw = extract_ml_features(&bars) .context("Feature extraction failed")?; - let warmup_offset = bars.len().saturating_sub(all_features.len()); + let warmup_offset = bars.len().saturating_sub(all_features_raw.len()); info!(" Extracted {} feature vectors (warmup period consumed {} bars)", - all_features.len(), warmup_offset); + all_features_raw.len(), warmup_offset); + + // Apply z-score normalisation — same op `precompute_features.rs:629` + // applies on the canonical fxcache path. Without this, feature[0] + // carries raw log-returns and downstream GPU consumers (aux head, + // vol_normalizer) see scale-mismatched values. + let norm_stats = ml::walk_forward::NormStats::from_features(&all_features_raw); + let all_features = norm_stats.normalize_batch(&all_features_raw); + info!(" Features z-score normalised (DBN-fallback path; {} bars × 42 dims)", + all_features.len()); // Build FxCacheData from DBN results (features are already warmup-trimmed) let aligned_bars = &bars[warmup_offset..]; diff --git a/docs/dqn-wire-up-audit.md b/docs/dqn-wire-up-audit.md index f84ccb48a..83b4b2d68 100644 --- a/docs/dqn-wire-up-audit.md +++ b/docs/dqn-wire-up-audit.md @@ -2,6 +2,29 @@ **Status:** Populated during Plan 1 Task 6 (A.5 orphan audit). Updated on every commit per Invariant 7. +fxcache data_source alignment + DBN-fallback normalization (2026-04-27): +two-part fix to a class of bugs causing un-normalised features to silently +flow into training. (1) `precompute_features.rs` previously hardcoded +`data_source = "ohlcv"` at the cache-key write site (lines 214, 633). +`train_baseline_rl.rs:582` hardcoded `"ohlcv"` at the read site too. +`dqn-production.toml` sets `data_source = "mbp10"` (MBP-10 is the +canonical production data path). The hardcodes mean smoke (uses ohlcv) +worked by accident, but any future profile with a different data_source +silently mismatches → cache MISS → DBN-direct fallback. Both call sites +now use `"mbp10"` (production default). precompute_features adds a +`--data-source` CLI override for legacy ohlcv smoke flows. +(2) DBN-fallback path in `train_baseline_rl.rs:592-643` did NOT call +`NormStats::normalize_batch` (precompute does, line 629). Any cache +miss for any reason (data_source drift, schema-hash mismatch, missing +file) silently uploaded RAW features to GPU. Raw close prices (~$5180 +ES futures) flowed into `next_states[:, 0]`, the aux next-bar head's +`label_scale` EMA latched onto raw-price magnitude (~5443 vs expected +~1.0 z-score), and the shared trunk learned to predict next-bar prices +→ epoch-0 Sharpe 141 with 0.32% max drawdown over 214k bars +(train-h5gxb). DBN fallback now applies the same z-score normalisation +unconditionally as defence-in-depth, so a future cache-miss cannot +reintroduce raw values into training. + fxcache schema-hash gap fix (2026-04-27): `build.rs::emit_feature_schema_hash` hashes only `src/features/extraction.rs`, `src/fxcache.rs`, and `../ml-core/src/state_layout.rs`. The z-score normalization step lives in