diff --git a/crates/ml/src/trainers/dqn/trainer/mod.rs b/crates/ml/src/trainers/dqn/trainer/mod.rs index 90ca3dfe6..e0fc8eef3 100644 --- a/crates/ml/src/trainers/dqn/trainer/mod.rs +++ b/crates/ml/src/trainers/dqn/trainer/mod.rs @@ -1538,6 +1538,15 @@ impl DQNTrainer { .map_err(|e| anyhow::anyhow!("guard reset: {e}"))?; } + // P5T2 fix: clear regression-detection consecutive-warn/error streaks + // at fold boundary. Walk-forward folds are independent training runs; + // without this reset a fold-2 epoch in the error band can trip + // termination because earlier folds each contributed accumulated + // streak counts. Found during P5T5 Phase D smoke (avg_grad_norm 3e7 + // tripped after 6 cumulative cross-fold epochs even though no single + // fold individually crossed the 2N threshold). + self.metric_bands.reset_streaks(); + // Task 2.5 Bug #5 — reset controller fire counters + total-epochs + // prev-controller snapshot at fold boundary. Without this, Track 3 // C2/C5 fold-boundary artefacts (cosine-annealed tau jumps when diff --git a/crates/ml/src/trainers/dqn/trainer/monitoring.rs b/crates/ml/src/trainers/dqn/trainer/monitoring.rs index b604c9d1d..19a985ccc 100644 --- a/crates/ml/src/trainers/dqn/trainer/monitoring.rs +++ b/crates/ml/src/trainers/dqn/trainer/monitoring.rs @@ -152,6 +152,20 @@ impl MetricBandsRegistry { Self::new(HashMap::new(), BandSettings::default()) } + /// Clear consecutive-warn and consecutive-error streak counters for every + /// known metric. Bands and settings are preserved. + /// + /// Called from `DQNTrainer::reset_for_fold` so different walk-forward + /// folds — which are independent training runs over disjoint date ranges + /// — never share streaks. Without this, a fold-2 epoch in the error band + /// could trigger termination because fold-0 and fold-1 each contributed + /// some out-of-band epochs that accumulated into the count, even though + /// neither fold individually crossed the 2N threshold. + pub(crate) fn reset_streaks(&mut self) { + self.consecutive_warn.clear(); + self.consecutive_error.clear(); + } + /// Load bands + settings from a toml file. The `[bands]` section may be /// empty (returns an empty registry); a missing file is a hard error. pub(crate) fn load_from_toml(path: &Path) -> Result { @@ -311,6 +325,38 @@ mod tests { } } + #[test] + fn reset_streaks_clears_consecutive_counters() { + // P5T5 Phase E: walk-forward folds are independent training runs; + // reset_streaks must clear cross-fold accumulation so a fold-N + // out-of-band epoch can't trip termination off counts inherited + // from folds 0..N-1. + let mut reg = MetricBandsRegistry::new( + bands_one("m", band(-1.0, 1.0, -100.0, 100.0)), + BandSettings { consecutive_epochs_for_warn: 3, consecutive_epochs_for_error: 6 }, + ); + // Accumulate 5 out-of-error-band epochs (1 short of termination) + for _ in 0..5 { + assert!(reg.update_and_check("m", 1e9).is_none()); + } + // Fold boundary + reg.reset_streaks(); + // Streak must now be zero — 5 more out-of-band epochs still no + // termination (would have terminated at the 1st without the reset) + for i in 0..5 { + let result = reg.update_and_check("m", 1e9); + assert!( + result.is_none(), + "post-reset epoch {i}: expected None (streak < 6), got {result:?}", + ); + } + // 6th post-reset epoch terminates + let term = reg + .update_and_check("m", 1e9) + .expect("expected RegressionError on 6th post-reset out-of-band epoch"); + assert!(matches!(term, TerminationReason::RegressionError { .. })); + } + #[test] fn six_consecutive_error_epochs_terminate() { let mut reg = MetricBandsRegistry::new( diff --git a/docs/dqn-wire-up-audit.md b/docs/dqn-wire-up-audit.md index 5317deec1..bb8ab53ca 100644 --- a/docs/dqn-wire-up-audit.md +++ b/docs/dqn-wire-up-audit.md @@ -2,6 +2,8 @@ **Status:** Populated during Plan 1 Task 6 (A.5 orphan audit). Updated on every commit per Invariant 7. +P5T5 Phase E (2026-04-26): per-fold reset for `MetricBandsRegistry` regression-detection streaks. Bug found during Phase D smoke — P5T2's registry never cleared `consecutive_warn` / `consecutive_error` HashMaps at fold boundaries, so walk-forward folds accumulated streaks across boundaries. F2 epoch in error band tripped termination at "6 consecutive" even though no fold individually crossed 2N=6. Fix: `reset_streaks()` method clears both HashMaps; called from `DQNTrainer::reset_for_fold()`. New unit test `reset_streaks_clears_consecutive_counters` verifies the per-fold isolation. 9/9 monitoring unit tests pass. + **Legend:** - `Wired` — consumed by production training + val path. - `Partial` — consumed on one side only (training OR val, or forward OR backward).