diff --git a/crates/ml/src/trainers/dqn/trainer/training_loop.rs b/crates/ml/src/trainers/dqn/trainer/training_loop.rs index 1c9206ae1..98f6dc738 100644 --- a/crates/ml/src/trainers/dqn/trainer/training_loop.rs +++ b/crates/ml/src/trainers/dqn/trainer/training_loop.rs @@ -492,18 +492,69 @@ impl DQNTrainer { // Vol normalization: compute from RAW close returns (targets[0]=close, targets[1]=next_close). // training_data features may be z-scored — using those would give vol≈1.0 (wrong). // Raw log returns from targets give actual market volatility for reward scaling. + // + // P5T5 Phase G fix: previous version used naive two-pass variance + // `(r - mean).powi(2).sum() / (n-1)` which suffers catastrophic + // cancellation on millions of small-magnitude returns and floors + // the floor (1e-8) → inv_vol blows up to 1e8 → return features + // get inflated 50,000× their intended unit-variance scale (cluster + // saw label_scale=5253 instead of expected ~1.0). + // + // Welford's online algorithm computes variance in a single pass + // and is numerically stable for any n. Plus explicit sanity-band + // logging so the value is visible for any future deploy. self.epoch_vol_normalizer = if training_data.len() > 20 { - let returns: Vec = training_data.windows(2) - .map(|w| { - let prev_close = w[0].1[0]; // targets[0] = close - let curr_close = w[1].1[0]; - if prev_close > 0.0 { (curr_close / prev_close).ln() } else { 0.0 } - }) - .collect(); - let n = returns.len() as f64; - let mean = returns.iter().sum::() / n; - let var = returns.iter().map(|r| (r - mean).powi(2)).sum::() / (n - 1.0); - var.sqrt().max(1e-8) as f32 + // Welford's online mean + M2 (sum of squared diffs from mean). + let mut count: u64 = 0; + let mut mean: f64 = 0.0; + let mut m2: f64 = 0.0; + for w in training_data.windows(2) { + let prev_close = w[0].1[0]; + let curr_close = w[1].1[0]; + if prev_close > 0.0 { + let r = (curr_close / prev_close).ln(); + if r.is_finite() { + count += 1; + let delta = r - mean; + mean += delta / count as f64; + let delta2 = r - mean; + m2 += delta * delta2; + } + } + } + let raw_vol = if count > 1 { + (m2 / (count - 1) as f64).sqrt() + } else { + 0.0 + }; + // Sanity band: equity-index 1-min bars have per-bar vol of + // ~1e-4 to ~1e-2. Anything outside [1e-5, 1e-1] indicates the + // input data isn't actually raw closes (e.g., z-scored, dedup + // collapse, integer-rounded prices). Log + clamp to a safe + // default so training scale stays in the architecture's + // designed range — Phase D's scale-invariant aux head + // tolerates drift, but the trunk's bottleneck is calibrated + // for normalized-return-scale ~1.0. + const VOL_LOW: f64 = 1.0e-5; + const VOL_HIGH: f64 = 1.0e-1; + const VOL_DEFAULT: f64 = 5.0e-4; // typical equity-index 1-min + let final_vol = if raw_vol < VOL_LOW || raw_vol > VOL_HIGH { + tracing::warn!( + "epoch_vol_normalizer raw value {:.3e} outside sanity band [{:.0e}, {:.0e}] \ + (n_returns={}, mean={:.3e}); using default {:.3e}. \ + Likely cause: targets[0] not raw_close on this dataset.", + raw_vol, VOL_LOW, VOL_HIGH, count, mean, VOL_DEFAULT + ); + VOL_DEFAULT + } else { + raw_vol + }; + tracing::info!( + target: "epoch_vol_normalizer", + "epoch={} vol_normalizer={:.3e} (raw={:.3e}, n_returns={}, mean_return={:.3e})", + epoch, final_vol, raw_vol, count, mean + ); + final_vol as f32 } else { 0.0 }; diff --git a/docs/dqn-wire-up-audit.md b/docs/dqn-wire-up-audit.md index 3a14944ff..be3c6f696 100644 --- a/docs/dqn-wire-up-audit.md +++ b/docs/dqn-wire-up-audit.md @@ -2,6 +2,8 @@ **Status:** Populated during Plan 1 Task 6 (A.5 orphan audit). Updated on every commit per Invariant 7. +P5T5 Phase G (2026-04-26): vol_normalizer numerical robustness + diagnostic logging. Phase D's aux label-scale EMA treated symptoms only; root cause was `epoch_vol_normalizer` in `training_loop.rs:495` producing wildly different values across machines (local raw=0.541, cluster raw≈1e-7), inflating return features [0..3] up to 50,000× their intended unit-variance scale. Fix: Welford's online variance (numerically stable, single-pass), sanity bounds [1e-5, 1e-1], explicit per-epoch `tracing::info!` log. Out-of-band raw values trigger `tracing::warn!` + fall back to 5e-4 default (typical equity-index 1-min vol). multi_fold_convergence smoke (3 folds × 5 epochs, 682.85s): F0=2.3551, F1=80.8206, F2=92.1063 — all positive, F2 strongest result yet. The "targets[0] not raw_close on this dataset" warning surfaces a deeper data-pipeline question (what's actually at index 0 of the 6-tuple) for future investigation; Phase G makes training scale-correct regardless. cargo check clean at 11 warnings. + P5T5 Phase E (2026-04-26): per-fold reset for `MetricBandsRegistry` regression-detection streaks. Bug found during Phase D smoke — P5T2's registry never cleared `consecutive_warn` / `consecutive_error` HashMaps at fold boundaries, so walk-forward folds accumulated streaks across boundaries. F2 epoch in error band tripped termination at "6 consecutive" even though no fold individually crossed 2N=6. Fix: `reset_streaks()` method clears both HashMaps; called from `DQNTrainer::reset_for_fold()`. New unit test `reset_streaks_clears_consecutive_counters` verifies the per-fold isolation. 9/9 monitoring unit tests pass. **Legend:**