diff --git a/crates/ml/src/trainers/dqn/smoke_tests/exploration_coverage.rs b/crates/ml/src/trainers/dqn/smoke_tests/exploration_coverage.rs index f8b44166f..bf8b1a464 100644 --- a/crates/ml/src/trainers/dqn/smoke_tests/exploration_coverage.rs +++ b/crates/ml/src/trainers/dqn/smoke_tests/exploration_coverage.rs @@ -42,17 +42,52 @@ fn test_exploration_covers_magnitude_space() -> Result<()> { let history = trainer.exploration_entropy_history(); println!("[EXPLORE] magnitude entropy history: {:?}", history); + // ── History must actually be populated ────────────────────────────── + // Prior weakness: `.unwrap_or(0.0)` on a `find` for ep5/ep20 would + // silently return 0.0 on an empty history, then FAIL the ≥ 0.5 check + // — which looked like "exploration collapsed" but actually meant "no + // entropy was ever recorded". Distinguish the two explicitly. + assert!( + history.len() >= 20, + "exploration_entropy_history has only {} entries after 20 epochs \ + — diagnostic did not emit per-epoch entropy. Full history: {:?}", + history.len(), history + ); + for &(e, v) in history { + assert!( + v.is_finite(), + "entropy history has non-finite value at epoch {}: {} — \ + normalized entropy should be in [0, 1]. Full: {:?}", + e, v, history + ); + assert!( + (0.0..=1.0).contains(&v), + "entropy history value at epoch {} = {} out of normalized \ + range [0, 1]. Full: {:?}", + e, v, history + ); + } + // Epoch numbering in history is 0-indexed per the training loop. + // If the epoch isn't present in the populated history, that's a + // different bug than low entropy — bail with a clear message rather + // than substituting 0.0 and failing the threshold check. let ep5 = history .iter() .find(|&&(e, _)| e == 4 /* 0-indexed epoch 5 */) .map(|&(_, v)| v) - .unwrap_or(0.0); + .unwrap_or_else(|| panic!( + "epoch 5 (0-indexed 4) missing from exploration history — \ + entropy emission regressed. Full history: {:?}", history + )); let ep20 = history .iter() .find(|&&(e, _)| e == 19 /* 0-indexed epoch 20 */) .map(|&(_, v)| v) - .unwrap_or(0.0); + .unwrap_or_else(|| panic!( + "epoch 20 (0-indexed 19) missing from exploration history — \ + entropy emission regressed. Full history: {:?}", history + )); println!("[EXPLORE] entropy @ep5 = {:.3}, @ep20 = {:.3}", ep5, ep20); @@ -68,5 +103,24 @@ fn test_exploration_covers_magnitude_space() -> Result<()> { Full history: {:?}", ep20, history ); + + // ── History must not be DEGENERATE ────────────────────────────────── + // A trainer that returns a constant value (say 1.0) across all epochs + // would pass the per-epoch thresholds trivially. Require at least some + // variance across the trajectory. Typical entropy decays during training + // as the policy becomes more confident, so the early-vs-late difference + // (or the min-max spread) should be > 1e-6. A perfectly flat history + // is a sign the emitter is stuck. + let values: Vec = history.iter().map(|&(_, v)| v).collect(); + let min_v = values.iter().copied().fold(f32::INFINITY, f32::min); + let max_v = values.iter().copied().fold(f32::NEG_INFINITY, f32::max); + let spread = max_v - min_v; + assert!( + spread > 1e-6, + "exploration entropy history is completely flat (min={:.6} max={:.6} \ + spread={:.6}) across 20 epochs — emitter may be stuck on a constant. \ + Full: {:?}", + min_v, max_v, spread, history + ); Ok(()) } diff --git a/crates/ml/src/trainers/dqn/smoke_tests/reward_component_audit.rs b/crates/ml/src/trainers/dqn/smoke_tests/reward_component_audit.rs index c6b9863e4..4b96baf6d 100644 --- a/crates/ml/src/trainers/dqn/smoke_tests/reward_component_audit.rs +++ b/crates/ml/src/trainers/dqn/smoke_tests/reward_component_audit.rs @@ -49,15 +49,85 @@ fn test_reward_components_contribute() -> Result<()> { // The smoke's job is to ensure the reporting scaffolding never regresses // with NaN/Inf values, which would break log parsing downstream. let summary = trainer.reward_component_audit_summary(); + let [popart, cf_flip, trail_r, micro, loss_aversion] = summary; println!( "[REWARD_AUDIT] popart={:.4} cf_flip={:.4} trail={:.4} micro={:.4} \ loss_aversion={:.4}", - summary[0], summary[1], summary[2], summary[3], summary[4] + popart, cf_flip, trail_r, micro, loss_aversion ); + + // ── Finite + non-negative gate (structural, unchanged intent) ── + // Magnitudes should be unsigned by construction (abs-sums, a/b with b≥0). + let names = ["popart", "cf_flip", "trail", "micro", "loss_aversion"]; for (i, &v) in summary.iter().enumerate() { - let names = ["popart", "cf_flip", "trail", "micro", "loss_aversion"]; - assert!(v.is_finite(), "reward_contrib[{}] (={}) is non-finite — diagnostic infra broken", names[i], v); - assert!(v >= 0.0, "reward_contrib[{}] (={}) is negative — magnitudes should be unsigned", names[i], v); + assert!( + v.is_finite(), + "reward_contrib[{}] (={}) is non-finite — diagnostic infra broken", + names[i], v + ); + assert!( + v >= 0.0, + "reward_contrib[{}] (={}) is negative — magnitudes should be unsigned", + names[i], v + ); } + + // ── Meaningful-signal gate ────────────────────────────────────────── + // Passes-on-zeros was the old weakness: a broken wiring that silently + // returned all-0 would have passed. Assert that the slots known to be + // wired in smoke config are *actually populated*. + // + // cf_flip: with `cf_ratio` default 0.5 in HEALTH_DIAG, ~half of the + // counterfactual slots should flip per epoch. The reducer normalizes + // against `n_touched` (reached slots), not N*L — so the rate is + // stable around 0.5 regardless of early termination. 20 epochs at + // batch_size from the smoke TOML gives >>1k samples, well past any + // small-N sampling noise. Assert ≥ 0.1 as a generous floor (10× below + // the ~0.5 target); anything lower means CF-flip is decoupled from + // the per-sample counter. If this fails, investigate the + // `cf_flip_per_sample` kernel write + `slot_live_per_sample` counter. + assert!( + cf_flip > 0.1, + "reward_contrib cf_flip = {:.4} ≤ 0.1 — CF-flip diagnostic is \ + effectively zero across 20 epochs. Either (a) CF path disabled, \ + (b) `cf_flip_per_sample` kernel write broke, or (c) \ + `slot_live_per_sample` counter is wrong. summary={:?}", + cf_flip, summary + ); + + // trail_r: the magnitude_distribution smoke shows `trail [fire_q=0.5-0.7]` + // on default smoke config, so some fraction of trades will hit the + // trailing-stop over 20 epochs. The reducer is fire_count / traded_count, + // gated by `traded > 0`. Assert ≥ 0.01 as a floor — anything below that + // means trail-stop never fires on trades, which would be a regression of + // the Task 0.5 wiring (`trail_triggered_per_sample` / + // `traded_per_sample` buffers). + assert!( + trail_r > 0.01, + "reward_contrib trail = {:.4} ≤ 0.01 — trailing-stop diagnostic is \ + effectively zero across 20 epochs. Per magnitude_distribution, \ + fire_q should be 0.5-0.7. Investigate `trail_triggered_per_sample` \ + or `traded_per_sample` wiring in experience_env_step. summary={:?}", + trail_r, summary + ); + + // popart and micro are known-zero in smoke config: + // - popart: requires multi-epoch variance drift; first few epochs see + // `prev_var == 0` so slot[0] is honestly 0.0. + // - micro: smoke TOML sets `micro_reward_scale = 0` in td_propagation, + // though default smoke may have a small value. Still dominated by + // main reward, so the ratio is small. + // Assert they are finite + non-negative only (already covered above). + // No additional floor: these slots legitimately stay near zero in smoke. + let _ = (popart, micro); + + // loss_aversion fires whenever a trade has a negative return that gets + // clamped by the asymmetric soft-clamp. With 20 epochs and default + // trading, at least some loss clamps should occur. loss_aversion can + // legitimately be small (the bulk of rewards are non-negative PnL), so + // assert ≥ 0 only (covered above). No floor — a run with zero losing + // trades is statistically unlikely but not indicative of a wiring bug. + let _ = loss_aversion; + Ok(()) } diff --git a/crates/ml/src/trainers/dqn/smoke_tests/training_stability.rs b/crates/ml/src/trainers/dqn/smoke_tests/training_stability.rs index b88d12391..c220bfb4f 100644 --- a/crates/ml/src/trainers/dqn/smoke_tests/training_stability.rs +++ b/crates/ml/src/trainers/dqn/smoke_tests/training_stability.rs @@ -251,15 +251,22 @@ fn test_trading_model_behavior() -> anyhow::Result<()> { ); // Action entropy: model explores multiple actions, doesn't collapse - // H(uniform over 9) = ln(9) ≈ 2.2. H > 0.5 means at least 2-3 actions used regularly + // H(uniform over 9) = ln(9) ≈ 2.2. H > 0.5 means at least 2-3 actions used regularly. + // + // Prior weakness: guarded the assertion on `is_finite()` — a NaN entropy + // (which is a MORE severe failure than low entropy) would silently pass. + // Surface NaN as its own distinct failure mode. let action_entropy = metrics.additional_metrics.get("action_entropy").copied().unwrap_or(f64::NAN); - if action_entropy.is_finite() { - assert!( - action_entropy > 0.3, - "BEHAVIORAL: Action entropy {action_entropy:.3} too low — model collapsed to \ - single action (lottery trading). Expected > 0.3" - ); - } + assert!( + action_entropy.is_finite(), + "BEHAVIORAL: action_entropy missing or NaN — diagnostic regressed. Got: {:?}", + metrics.additional_metrics.get("action_entropy") + ); + assert!( + action_entropy > 0.3, + "BEHAVIORAL: Action entropy {action_entropy:.3} too low — model collapsed to \ + single action (lottery trading). Expected > 0.3" + ); // GPU trade stats: model must actually complete trades (not stuck on Flat) let total_trades = metrics.additional_metrics.get("total_trades").copied().unwrap_or(0.0) as usize; @@ -424,14 +431,21 @@ fn test_50_epoch_convergence() -> anyhow::Result<()> { // ═══════════════════════════════════════════════════════════ // Anomaly 5: Action entropy must not collapse // ═══════════════════════════════════════════════════════════ + // Prior weakness: guarded the assertion on `entropy.is_finite()` so + // NaN entropy — which is a WORSE failure than low entropy — + // silently skipped the gate. Fail on missing/NaN first. let entropy = metrics.additional_metrics.get("action_entropy").copied().unwrap_or(f64::NAN); - if entropy.is_finite() { - assert!( - entropy > 0.15, - "ANOMALY 5: Action entropy {:.3} → model collapsed to single action.", - entropy - ); - } + assert!( + entropy.is_finite(), + "ANOMALY 5: action_entropy missing/NaN after 50 epochs — diagnostic \ + regressed or entropy computation exploded. Got: {:?}", + metrics.additional_metrics.get("action_entropy") + ); + assert!( + entropy > 0.15, + "ANOMALY 5: Action entropy {:.3} → model collapsed to single action.", + entropy + ); drop(trainer); drop(rt); @@ -455,6 +469,25 @@ fn test_gpu_collector_auto_initializes() -> anyhow::Result<()> { Ok(String::new()) })); assert!(result.is_ok(), "Training must succeed with auto-initialized GPU collector: {:?}", result.err()); + + // Strengthened: don't just check "Ok" — the auto-initialized collector + // must have actually collected experiences (reward-contrib reducer reads + // its per-sample buffers). A collector that silently produced zeros + // would still return Ok. + let metrics = result.unwrap(); + assert!( + metrics.epochs_trained >= 1, + "auto-init collector ran but produced 0 epochs: {metrics:?}" + ); + assert_finite(metrics.loss, "auto-init collector loss"); + let grad_norm = metrics.additional_metrics.get("avg_gradient_norm") + .copied().unwrap_or(f64::NAN); + assert!( + grad_norm.is_finite() && grad_norm > 0.0, + "auto-init collector produced grad_norm={grad_norm} — collector \ + buffers likely empty, gradients did not flow" + ); + drop(trainer); drop(rt); Ok(()) diff --git a/crates/ml/src/trainers/dqn/smoke_tests/walk_forward.rs b/crates/ml/src/trainers/dqn/smoke_tests/walk_forward.rs index b9782f668..7d24b0185 100644 --- a/crates/ml/src/trainers/dqn/smoke_tests/walk_forward.rs +++ b/crates/ml/src/trainers/dqn/smoke_tests/walk_forward.rs @@ -170,15 +170,23 @@ fn test_walk_forward_no_overfitting_50_epochs() -> anyhow::Result<()> { // Walk-forward fold resets make first_epoch_loss unreliable for // comparison (pre-trained weights produce low initial loss), so we // check that training didn't DIVERGE rather than that it converged. + // + // Prior assertion was a no-op tautology (`x < x + 1`) commented + // "finite check" — actually verify finiteness, not a self-truth. let avg_loss = (first_epoch_loss + last_epoch_loss) / 2.0; let loss_ratio = last_epoch_loss / avg_loss; - assert!(loss_ratio < 3.0 * loss_ratio.signum().max(0.0) + 3.0, // finite check - "loss_ratio not finite: last={last_epoch_loss:.4}, avg={avg_loss:.4}"); - // Neither extreme should be more than 10× the other + assert!(loss_ratio.is_finite(), + "loss_ratio not finite: last={last_epoch_loss:.4}, avg={avg_loss:.4}, ratio={loss_ratio}"); + // Neither extreme should be more than 10× the other (real stability gate, + // not the tautological `x < x + 1` that was here before). let max_loss = first_epoch_loss.max(last_epoch_loss); let min_loss = first_epoch_loss.min(last_epoch_loss).max(1e-10); - assert!(max_loss / min_loss < max_loss / min_loss + 1.0, // finite check - "Loss divergence ratio not finite"); + let div_ratio = max_loss / min_loss; + assert!(div_ratio.is_finite(), + "Loss divergence ratio not finite: max={max_loss:.4} min={min_loss:.4} ratio={div_ratio}"); + assert!(div_ratio < 10.0, + "Loss divergence >10× across 50 epochs: first={first_epoch_loss:.4} \ + last={last_epoch_loss:.4} ratio={div_ratio:.2}× — training has diverged"); // 4. Gradient health: norm must be positive (not collapsed to zero) assert!(grad_norm > 0.0,