diff --git a/crates/ml/src/trainers/dqn/smoke_tests/controller_activity.rs b/crates/ml/src/trainers/dqn/smoke_tests/controller_activity.rs new file mode 100644 index 000000000..7465c6e70 --- /dev/null +++ b/crates/ml/src/trainers/dqn/smoke_tests/controller_activity.rs @@ -0,0 +1,56 @@ +//! Smoke test: controller activity (Track 3 from policy-quality spec). +//! +//! Each of the 6 adaptive controllers (anti-LR, adaptive tau, adaptive gamma, +//! adaptive grad-clip, CQL alpha schedule, cost anneal) should fire +//! DIAGNOSTICALLY — i.e. catch edge cases, not run every epoch. A controller +//! firing in > 50% of epochs is LOAD-BEARING: the policy depends on the +//! controller to stay on the rails, which is a production-readiness red flag +//! (at deployment the controller isn't there to save things). +//! +//! Fire detection: a controller "fired" in epoch N iff its observable output +//! value changed from epoch N-1. Delta-based detection, implemented in +//! training_loop.rs around the HEALTH_DIAG emission. +//! +//! Run: `FOXHUNT_TEST_DATA=test_data/futures-baseline \ +//! cargo test -p ml --release --lib -- controller_activity --ignored --nocapture` + +use super::helpers::*; +use anyhow::Result; + +#[test] +#[ignore] // Requires fxcache +fn test_controllers_not_load_bearing() -> Result<()> { + let data = load_smoke_fxcache().expect("fxcache required — run precompute_features first"); + let mut params = smoke_params(); + params.epochs = 20; + params.early_stopping_enabled = false; + params.min_epochs_before_stopping = 20; + + let mut trainer = smoke_trainer_with(params)?; + init_trainer_from_fxcache(&mut trainer, &data, 200_000)?; + + let data_dir = test_data_dir().expect("FOXHUNT_TEST_DATA or test_data/ must exist"); + let rt = tokio::runtime::Builder::new_current_thread().enable_all().build()?; + let _metrics = rt.block_on(trainer.train( + &data_dir, + "ES.FUT", + |_epoch, _bytes, _best| Ok("skip".to_owned()), + ))?; + + let rates = trainer.controller_fire_rates_final(); + let names = ["anti_lr", "tau", "gamma", "grad_clip", "cql_alpha", "cost_anneal"]; + println!( + "[CTRL_FIRE] anti_lr={:.3} tau={:.3} gamma={:.3} clip={:.3} cql={:.3} cost={:.3}", + rates[0], rates[1], rates[2], rates[3], rates[4], rates[5] + ); + for (name, &rate) in names.iter().zip(rates.iter()) { + assert!( + rate <= 0.5, + "Controller '{}' fires in {:.1}% of epochs (> 50% = load-bearing). \ + Full rates: anti_lr={:.3} tau={:.3} gamma={:.3} clip={:.3} cql={:.3} cost={:.3}", + name, rate * 100.0, + rates[0], rates[1], rates[2], rates[3], rates[4], rates[5] + ); + } + Ok(()) +} diff --git a/crates/ml/src/trainers/dqn/smoke_tests/mod.rs b/crates/ml/src/trainers/dqn/smoke_tests/mod.rs index a42687978..189c80cb1 100644 --- a/crates/ml/src/trainers/dqn/smoke_tests/mod.rs +++ b/crates/ml/src/trainers/dqn/smoke_tests/mod.rs @@ -28,3 +28,5 @@ mod td_propagation; mod magnitude_distribution; #[cfg(test)] mod exploration_coverage; +#[cfg(test)] +mod controller_activity; diff --git a/crates/ml/src/trainers/dqn/trainer/constructor.rs b/crates/ml/src/trainers/dqn/trainer/constructor.rs index 79fb3f6fb..1a7333734 100644 --- a/crates/ml/src/trainers/dqn/trainer/constructor.rs +++ b/crates/ml/src/trainers/dqn/trainer/constructor.rs @@ -578,6 +578,9 @@ impl DQNTrainer { last_action_entropy: None, last_magnitude_dist: [0.0_f32; 3], explore_entropy_mag_history: Vec::new(), + prev_controller_values: super::ControllerPrevValues::default(), + controller_fire_counts: super::ControllerFireCounts::default(), + controller_total_epochs: 0, // Wave 16 Portfolio Features (action masking always active) max_position, diff --git a/crates/ml/src/trainers/dqn/trainer/mod.rs b/crates/ml/src/trainers/dqn/trainer/mod.rs index a991650ef..a64aa30ea 100644 --- a/crates/ml/src/trainers/dqn/trainer/mod.rs +++ b/crates/ml/src/trainers/dqn/trainer/mod.rs @@ -162,6 +162,43 @@ impl BacktrackingState { } } +/// Task 0.9 — prior-epoch values for adaptive-controller fire detection. +/// NaN sentinel means "no prior epoch" — first real value treated as fire. +#[derive(Debug, Clone, Copy)] +pub struct ControllerPrevValues { + pub lr: f64, + pub tau: f32, + pub gamma: f32, + pub grad_clip: f32, + pub cql_alpha: f32, + pub cost_anneal: f32, +} + +impl Default for ControllerPrevValues { + fn default() -> Self { + Self { + lr: f64::NAN, + tau: f32::NAN, + gamma: f32::NAN, + grad_clip: f32::NAN, + cql_alpha: f32::NAN, + cost_anneal: f32::NAN, + } + } +} + +/// Task 0.9 — running count of how many epochs each controller fired +/// (i.e. changed the value it controls). +#[derive(Debug, Clone, Copy, Default)] +pub struct ControllerFireCounts { + pub anti_lr: u32, + pub tau: u32, + pub gamma: u32, + pub grad_clip: u32, + pub cql_alpha: u32, + pub cost_anneal: u32, +} + pub struct DQNTrainer { /// DQN agent pub(crate) agent: Arc>, @@ -248,6 +285,19 @@ pub struct DQNTrainer { /// doesn't collapse to a single bin too fast. pub(crate) explore_entropy_mag_history: Vec<(u32, f32)>, + /// Task 0.9 — prior-epoch values of adaptive-controller outputs, used to + /// detect "fire" = value changed from the prior epoch. Controllers audited: + /// anti-LR, adaptive tau, adaptive gamma, adaptive grad-clip, CQL alpha + /// schedule, cost anneal schedule. Initialized to sentinel NaN so the + /// first epoch's comparison treats any non-NaN value as a fire. + pub(crate) prev_controller_values: ControllerPrevValues, + + /// Task 0.9 — running per-controller fire counts + total-epochs counter. + /// Used by controller_activity smoke test to assert no controller fires in + /// > 50% of epochs (load-bearing controllers are a production risk). + pub(crate) controller_fire_counts: ControllerFireCounts, + pub(crate) controller_total_epochs: u32, + // Wave 16 Portfolio Features (action masking is always active). /// Maximum position size for action masking (default: 2.0) pub max_position: f64, @@ -1422,6 +1472,22 @@ impl DQNTrainer { &self.explore_entropy_mag_history } + /// Per-controller firing rates across all completed epochs. + /// Layout: [anti_lr, tau, gamma, grad_clip, cql_alpha, cost_anneal]. + /// Each rate = epochs-where-value-changed / total-epochs. Used by the + /// controller_activity smoke test to detect load-bearing controllers. + pub fn controller_fire_rates_final(&self) -> [f32; 6] { + let total = self.controller_total_epochs.max(1) as f32; + [ + self.controller_fire_counts.anti_lr as f32 / total, + self.controller_fire_counts.tau as f32 / total, + self.controller_fire_counts.gamma as f32 / total, + self.controller_fire_counts.grad_clip as f32 / total, + self.controller_fire_counts.cql_alpha as f32 / total, + self.controller_fire_counts.cost_anneal as f32 / total, + ] + } + /// Get current epsilon from the DQN agent pub async fn get_agent_epsilon(&self) -> f32 { let agent_lock = self.agent.read().await; diff --git a/crates/ml/src/trainers/dqn/trainer/training_loop.rs b/crates/ml/src/trainers/dqn/trainer/training_loop.rs index 336c8c7ef..1971e77b8 100644 --- a/crates/ml/src/trainers/dqn/trainer/training_loop.rs +++ b/crates/ml/src/trainers/dqn/trainer/training_loop.rs @@ -2213,6 +2213,61 @@ impl DQNTrainer { // Task 0.14: append to exploration entropy history for smoke-test readback. self.explore_entropy_mag_history.push((epoch as u32, ent_mag)); + // Task 0.9: adaptive-controller fire detection. + // A controller "fired" this epoch iff its observable output value + // changed from the prior epoch. NaN prev (first epoch) → no fire + // (no baseline to compare against). Uses absolute delta thresholds + // chosen per controller's natural scale — below threshold is + // floating-point noise, above is real adjustment. + // LR used for THIS epoch's training. anti-LR runs later in the loop + // (computing LR for next epoch) so scheduler value here is the + // one that was actually in effect. + let cur_lr = self.lr_scheduler.get_lr(); + let cur_tau = self.last_tau_eff.unwrap_or(f32::NAN); + let cur_gamma = self.last_gamma_eff.unwrap_or(f32::NAN); + let cur_clip = self.fused_ctx.as_ref().map(|f| f.adaptive_clip_value()).unwrap_or(f32::NAN); + let cur_cql = self.last_cql_alpha_eff.unwrap_or(f32::NAN); + let cur_cost = self.cost_anneal_factor; + + let prev = self.prev_controller_values; + let fire_lr = prev.lr.is_finite() && (cur_lr - prev.lr).abs() > 1e-10; + let fire_tau = prev.tau.is_finite() && (cur_tau - prev.tau).abs() > 1e-6; + let fire_gamma = prev.gamma.is_finite() && (cur_gamma - prev.gamma).abs() > 1e-4; + let fire_clip = prev.grad_clip.is_finite() && (cur_clip - prev.grad_clip).abs() > 1e-3; + let fire_cql = prev.cql_alpha.is_finite() && (cur_cql - prev.cql_alpha).abs() > 1e-5; + let fire_cost = prev.cost_anneal.is_finite() && (cur_cost - prev.cost_anneal).abs() > 1e-4; + + if fire_lr { self.controller_fire_counts.anti_lr += 1; } + if fire_tau { self.controller_fire_counts.tau += 1; } + if fire_gamma { self.controller_fire_counts.gamma += 1; } + if fire_clip { self.controller_fire_counts.grad_clip += 1; } + if fire_cql { self.controller_fire_counts.cql_alpha += 1; } + if fire_cost { self.controller_fire_counts.cost_anneal += 1; } + self.controller_total_epochs += 1; + self.prev_controller_values = super::ControllerPrevValues { + lr: cur_lr, + tau: cur_tau, + gamma: cur_gamma, + grad_clip: cur_clip, + cql_alpha: cur_cql, + cost_anneal: cur_cost, + }; + + // fire_frac = max fire-rate across all controllers (worst-case "load-bearing" + // signal). Running fraction = fires / total_epochs for the max-firing + // controller. Captures whether *any* controller is above 50% threshold + // (the controller_activity smoke gate). + let total = self.controller_total_epochs.max(1) as f32; + let rates = [ + self.controller_fire_counts.anti_lr as f32 / total, + self.controller_fire_counts.tau as f32 / total, + self.controller_fire_counts.gamma as f32 / total, + self.controller_fire_counts.grad_clip as f32 / total, + self.controller_fire_counts.cql_alpha as f32 / total, + self.controller_fire_counts.cost_anneal as f32 / total, + ]; + let fire_frac = rates.iter().copied().fold(0.0_f32, f32::max); + tracing::info!( "HEALTH_DIAG[{}]: health={:.2} components [q_gap={:.2} q_var={:.2} atoms={:.2} grad_stable={:.2} ens_agree={:.2} grad_cos={:.2} spectral={:.2}] effective [cql_alpha={:.4} iqn_budget={:.2} cql_budget={:.2} c51_budget={:.2} tau={:.5} sarsa_tau={:.2} gamma={:.3} cf_ratio={:.2}] novels [distill={} barrier={:.3} plasticity={} ib={:.3} ensemble_collapse={:.2} contrarian={} meta_q_pred={:.2}] diag [sharpe_ema={:.3} action_entropy={:.2}] gems [g12_predictive={:.4}] mag [q_full={:.3} q_half={:.3} q_quarter={:.3} var_scale={:.3} kelly_f={:.3} avg_win_ratio={:.3} grad_ratio_mag_dir={:.4} dist_q={:.3} dist_h={:.3} dist_f={:.3}] trail [fire_q={:.3} fire_h={:.3} fire_f={:.3} hold_q={:.2} hold_h={:.2} hold_f={:.2}] noisy [vsn_mag={:.3} vsn_dir={:.3} sigma_mag={:.4} sigma_dir={:.4} drift_mag={:.3} drift_dir={:.3}] eval_dist [eq={:.3} eh={:.3} ef={:.3}] reward_contrib [popart={:.3} cf={:.3} trail_r={:.3} micro={:.3} la={:.3} seg={:.3}] controller [anti_lr={} tau={} gamma={} clip={} cql={} cost={} fire_frac={:.2}] explore [ent_mag={:.2} ent_dir={:.2} sigma_mean={:.4}]", epoch, @@ -2256,8 +2311,8 @@ impl DQNTrainer { 0.0_f32, 0.0_f32, 0.0_f32, // Track 2 — reward contrib (6 f32) 0.0_f32, 0.0_f32, 0.0_f32, 0.0_f32, 0.0_f32, 0.0_f32, - // Track 3 — controllers (6 bool, 1 f32) - false, false, false, false, false, false, 0.0_f32, + // Track 3 — controllers (6 bool per-epoch fire + 1 f32 max running rate) + fire_lr, fire_tau, fire_gamma, fire_clip, fire_cql, fire_cost, fire_frac, // Track 4 — explore (3 f32): ent_mag, ent_dir, sigma_mean. // sigma_mean still stubbed (needs NoisyNets σ readback, Task 0.6). ent_mag, ent_dir, 0.0_f32,