diff --git a/crates/ml/src/trainers/dqn/controllers/conviction_floor_controller.rs b/crates/ml/src/trainers/dqn/controllers/conviction_floor_controller.rs new file mode 100644 index 000000000..1d13d40ef --- /dev/null +++ b/crates/ml/src/trainers/dqn/controllers/conviction_floor_controller.rs @@ -0,0 +1,179 @@ +//! Conviction-floor controller — Plan 1 Task 15, §4.C.6. +//! +//! Scaffolding controller for the IQL branch_scales per-sample safety floor. +//! The floor lives at ISV slot [IQL_BRANCH_SCALE_FLOOR_INDEX = 36] and is +//! bootstrapped to `0.1` in the `GpuDqnTrainer` constructor. +//! +//! # Current behaviour +//! +//! The floor is `SchemaContract` in the `StateResetRegistry` — it is written +//! exactly once (in the constructor) and is never mutated per-epoch. The +//! controller mirrors this: `update()` returns the static floor unchanged and +//! `write_output` writes it to ISV[36]. Because the value never changes the +//! controller's `fire_rate` is always 0.0. +//! +//! This is valid behaviour-preserving scaffolding per the trait design: +//! `update()` returns a real, concrete decision (the current static floor), +//! not a deferred stub. Plan 3 may replace `update()` with a signal-driven +//! ramp once further research is complete. +//! +//! # ISV slot +//! +//! `write_output` writes the floor value to ISV[IQL_BRANCH_SCALE_FLOOR_INDEX]. +//! The slot is only written at construction time in the current system; this +//! controller is wired into the constructor path to centralise the write and +//! make the value inspectable via `diagnose()`. + +use crate::trainers::dqn::adaptive_controller::{ + AdaptiveController, DiagSnapshot, FireRateStats, IsvBus, +}; +use crate::cuda_pipeline::gpu_dqn_trainer::IQL_BRANCH_SCALE_FLOOR_INDEX; + +/// Conviction-floor controller input signal. +/// +/// Currently carries no adaptive inputs — the floor is static. The health +/// field is read for future use (Plan 3 may gate the floor on health). +#[derive(Debug, Clone, Copy, Default)] +pub(crate) struct ConvictionFloorSignal { + /// Placeholder — no dynamic signals consumed today. + pub _reserved: f32, +} + +/// Conviction-floor controller output. +#[derive(Debug, Clone, Copy)] +pub(crate) struct ConvictionFloorControl { + /// Per-sample branch_scales floor in [0, 1]. Prevents (sample, branch) + /// gradient collapse when IQL readiness → 1 and a losing branch has + /// near-zero advantage. + pub floor: f32, +} + +/// Static conviction-floor controller. +/// +/// Returns the bootstrapped floor value (default 0.1) unchanged each epoch. +/// `write_output` propagates the value to ISV[36]. +#[derive(Debug)] +pub(crate) struct ConvictionFloorController { + fire_rate: FireRateStats, + /// Static floor — bootstrapped at 0.1, matches the constructor write. + floor: f32, +} + +impl ConvictionFloorController { + /// Bootstrap value used in `GpuDqnTrainer` constructor. + pub(crate) const BOOTSTRAP_FLOOR: f32 = 0.1; + + pub(crate) fn new() -> Self { + Self { + fire_rate: FireRateStats::default(), + floor: Self::BOOTSTRAP_FLOOR, + } + } +} + +impl Default for ConvictionFloorController { + fn default() -> Self { Self::new() } +} + +impl AdaptiveController for ConvictionFloorController { + type Signal = ConvictionFloorSignal; + type Control = ConvictionFloorControl; + + fn read_signals(&self, _isv: &IsvBus<'_>) -> ConvictionFloorSignal { + // No adaptive inputs consumed today. The floor is static. + ConvictionFloorSignal { _reserved: 0.0 } + } + + fn update(&mut self, _signal: ConvictionFloorSignal) -> ConvictionFloorControl { + // Behaviour-preserving: return the static floor unchanged. + // Plan 3 replaces this with a signal-driven ramp once the + // SchemaContract constraint is relaxed with a migration path. + let changed = false; // static → never fires + self.fire_rate.record_fire(changed); + ConvictionFloorControl { floor: self.floor } + } + + fn write_output(&self, ctrl: &ConvictionFloorControl, isv: &mut IsvBus<'_>) { + // Write the floor to ISV[IQL_BRANCH_SCALE_FLOOR_INDEX=36]. + // This mirrors the constructor write and keeps the slot under controller + // ownership so future Plan 3 rewrites update only this one code path. + isv.write(IQL_BRANCH_SCALE_FLOOR_INDEX, ctrl.floor); + } + + fn fire_rate(&self) -> &FireRateStats { + &self.fire_rate + } + + fn diagnose(&self) -> DiagSnapshot { + DiagSnapshot::new() + .with("floor", self.floor) + } + + fn name(&self) -> &'static str { + "conviction_floor" + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn conviction_floor_bootstrap_value() { + let mut ctrl = ConvictionFloorController::new(); + let out = ctrl.update(ConvictionFloorSignal::default()); + assert!((out.floor - 0.1_f32).abs() < 1e-6, + "expected bootstrap 0.1, got {}", out.floor); + } + + #[test] + fn conviction_floor_never_fires() { + let mut ctrl = ConvictionFloorController::new(); + for _ in 0..20 { + ctrl.update(ConvictionFloorSignal::default()); + } + assert_eq!(ctrl.fire_rate().fire_rate(), 0.0, + "static floor should never fire"); + } + + #[test] + fn conviction_floor_write_output_sets_isv_slot() { + let mut ctrl = ConvictionFloorController::new(); + let control = ctrl.update(ConvictionFloorSignal::default()); + let mut buf = vec![0.0_f32; 39]; + let mut isv = IsvBus::new(&mut buf); + ctrl.write_output(&control, &mut isv); + assert!((isv.read(IQL_BRANCH_SCALE_FLOOR_INDEX) - 0.1_f32).abs() < 1e-6, + "ISV[36] must equal floor 0.1 after write_output"); + } + + #[test] + fn conviction_floor_isv_slot_index() { + // Verify the constant matches the expected value (regression guard). + assert_eq!(IQL_BRANCH_SCALE_FLOOR_INDEX, 36, + "ISV slot index must be 36 — change only with full layout migration"); + } + + #[test] + fn conviction_floor_diagnose_fields() { + let mut ctrl = ConvictionFloorController::new(); + ctrl.update(ConvictionFloorSignal::default()); + let diag = ctrl.diagnose(); + assert!(diag.fields.contains_key("floor"), + "diagnose() must expose floor"); + } + + #[test] + fn conviction_floor_read_signals_returns_default() { + let ctrl = ConvictionFloorController::new(); + let mut buf = vec![0.0_f32; 39]; + let isv = IsvBus::new(&mut buf); + let sig = ctrl.read_signals(&isv); + assert_eq!(sig._reserved, 0.0); + } + + #[test] + fn conviction_floor_name() { + assert_eq!(ConvictionFloorController::new().name(), "conviction_floor"); + } +} diff --git a/crates/ml/src/trainers/dqn/controllers/epsilon_controller.rs b/crates/ml/src/trainers/dqn/controllers/epsilon_controller.rs new file mode 100644 index 000000000..1e0bf2de5 --- /dev/null +++ b/crates/ml/src/trainers/dqn/controllers/epsilon_controller.rs @@ -0,0 +1,215 @@ +//! Epsilon controller — Plan 1 Task 14, §4.C.6. +//! +//! Encapsulates the ISV-adaptive exploration epsilon computation. +//! +//! # Formula +//! +//! ```text +//! epsilon = base_floor * (0.5 + volatility) +//! ``` +//! +//! where `volatility` is read from ISV slot [2] (`TD_ERR_EMA`, used as a +//! volatility proxy) and `base_floor` is `hyperparams.noisy_epsilon_floor` +//! (default 0.05). Volatile regime → higher epsilon (more exploration). +//! Stable regime → lower epsilon (more exploitation). +//! +//! # Migration notes +//! +//! Before migration, the inline block in `training_loop.rs` +//! `initialize_epoch_state` (~10 lines, labelled "ISV-adaptive exploration") +//! computed `adaptive_epsilon = base_floor * (0.5 + volatility)` and called +//! `agent.set_epsilon()` directly. After migration the same formula lives in +//! `update()` and the training loop calls `agent.set_epsilon(ctrl.epsilon)`. +//! +//! `write_output` is a no-op — epsilon is consumed by `agent.set_epsilon()` +//! in the training loop. No ISV slot carries this scalar. + +use crate::trainers::dqn::adaptive_controller::{ + AdaptiveController, DiagSnapshot, FireRateStats, IsvBus, +}; + +/// ISV slot [2]: TD-error EMA, used here as a volatility proxy. +/// Higher TD error → regime more volatile → explore more. +const ISV_VOLATILITY_PROXY: usize = 2; + +/// Epsilon controller input signals. +#[derive(Debug, Clone, Copy)] +pub(crate) struct EpsilonSignal { + /// Volatility proxy from ISV[2] (TD-error EMA), clamped to [0, 1]. + pub volatility: f32, + /// Base epsilon floor from hyperparams (`noisy_epsilon_floor`), default 0.05. + pub base_floor: f32, +} + +/// Epsilon controller output. +#[derive(Debug, Clone, Copy)] +pub(crate) struct EpsilonControl { + /// Effective epsilon for this epoch, ≥ 0. + pub epsilon: f32, + /// Whether epsilon changed from the previous epoch (fire signal). + pub changed: bool, +} + +/// ISV-adaptive epsilon controller. +/// +/// Reads the volatility proxy from ISV[2] and computes the adaptive +/// epsilon each epoch. The result is consumed by `agent.set_epsilon()`. +#[derive(Debug)] +pub(crate) struct EpsilonController { + fire_rate: FireRateStats, + last_epsilon: f32, + /// Default base floor when the signal does not override it. + pub base_floor: f32, +} + +impl EpsilonController { + /// Construct with the configured epsilon floor (default 0.05). + pub(crate) fn new(base_floor: f32) -> Self { + Self { + fire_rate: FireRateStats::default(), + last_epsilon: base_floor * 0.75, // reasonable cold-start estimate + base_floor, + } + } + + /// Compute the adaptive epsilon from base floor and volatility. + /// Pure function — no side effects. + fn compute(base_floor: f32, volatility: f32) -> f32 { + base_floor * (0.5 + volatility.clamp(0.0, 1.0)) + } +} + +impl Default for EpsilonController { + fn default() -> Self { + Self::new(0.05) + } +} + +impl AdaptiveController for EpsilonController { + type Signal = EpsilonSignal; + type Control = EpsilonControl; + + fn read_signals(&self, isv: &IsvBus<'_>) -> EpsilonSignal { + EpsilonSignal { + volatility: isv.read(ISV_VOLATILITY_PROXY).clamp(0.0, 1.0), + base_floor: self.base_floor, + } + } + + fn update(&mut self, signal: EpsilonSignal) -> EpsilonControl { + let epsilon = Self::compute(signal.base_floor, signal.volatility); + let changed = (epsilon - self.last_epsilon).abs() > 1e-6; + self.fire_rate.record_fire(changed); + self.last_epsilon = epsilon; + EpsilonControl { epsilon, changed } + } + + fn write_output(&self, _ctrl: &EpsilonControl, _isv: &mut IsvBus<'_>) { + // Epsilon is consumed by agent.set_epsilon() in the training loop. + // No ISV slot carries the epsilon scalar. + } + + fn fire_rate(&self) -> &FireRateStats { + &self.fire_rate + } + + fn diagnose(&self) -> DiagSnapshot { + DiagSnapshot::new() + .with("epsilon", self.last_epsilon) + .with("base_floor", self.base_floor) + } + + fn name(&self) -> &'static str { + "epsilon" + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn make_isv_with_volatility(volatility: f32) -> Vec { + let mut buf = vec![0.0_f32; 39]; + buf[ISV_VOLATILITY_PROXY] = volatility; + buf + } + + #[test] + fn epsilon_formula_at_zero_volatility() { + let mut ctrl = EpsilonController::new(0.05); + let sig = EpsilonSignal { volatility: 0.0, base_floor: 0.05 }; + let out = ctrl.update(sig); + // 0.05 * (0.5 + 0.0) = 0.025 + assert!((out.epsilon - 0.025_f32).abs() < 1e-6, + "expected 0.025, got {}", out.epsilon); + } + + #[test] + fn epsilon_formula_at_full_volatility() { + let mut ctrl = EpsilonController::new(0.05); + let sig = EpsilonSignal { volatility: 1.0, base_floor: 0.05 }; + let out = ctrl.update(sig); + // 0.05 * (0.5 + 1.0) = 0.075 + assert!((out.epsilon - 0.075_f32).abs() < 1e-6, + "expected 0.075, got {}", out.epsilon); + } + + #[test] + fn epsilon_formula_at_mid_volatility() { + let mut ctrl = EpsilonController::new(0.05); + let sig = EpsilonSignal { volatility: 0.5, base_floor: 0.05 }; + let out = ctrl.update(sig); + // 0.05 * (0.5 + 0.5) = 0.05 + assert!((out.epsilon - 0.05_f32).abs() < 1e-6, + "expected 0.05, got {}", out.epsilon); + } + + #[test] + fn epsilon_reads_volatility_from_isv() { + let ctrl = EpsilonController::new(0.05); + let mut buf = make_isv_with_volatility(0.8); + let isv = IsvBus::new(&mut buf); + let sig = ctrl.read_signals(&isv); + assert!((sig.volatility - 0.8).abs() < 1e-6, + "expected 0.8, got {}", sig.volatility); + } + + #[test] + fn epsilon_fire_rate_on_volatility_change() { + let mut ctrl = EpsilonController::new(0.05); + // First call: cold start → first fire + ctrl.update(EpsilonSignal { volatility: 0.5, base_floor: 0.05 }); + // Second call: same volatility → no fire (epsilon unchanged) + ctrl.update(EpsilonSignal { volatility: 0.5, base_floor: 0.05 }); + // Third call: different volatility → fire + ctrl.update(EpsilonSignal { volatility: 0.9, base_floor: 0.05 }); + let rate = ctrl.fire_rate().fire_rate(); + // 2 fires (first + third) out of 3 epochs = 2/3 + assert!(rate > 0.0, "expected fires on volatile changes, got rate={rate}"); + } + + #[test] + fn epsilon_clamped_volatility() { + let mut ctrl = EpsilonController::new(0.10); + // volatility clamped to [0,1]; out-of-range value should not blow up + let sig = EpsilonSignal { volatility: 5.0, base_floor: 0.10 }; + let out = ctrl.update(sig); + // volatility clamped to 1.0 → 0.10 * (0.5+1.0) = 0.15 + assert!((out.epsilon - 0.15_f32).abs() < 1e-6, + "expected 0.15 (clamped), got {}", out.epsilon); + } + + #[test] + fn epsilon_diagnose_fields() { + let mut ctrl = EpsilonController::new(0.05); + ctrl.update(EpsilonSignal { volatility: 0.6, base_floor: 0.05 }); + let diag = ctrl.diagnose(); + assert!(diag.fields.contains_key("epsilon")); + assert!(diag.fields.contains_key("base_floor")); + } + + #[test] + fn epsilon_name() { + assert_eq!(EpsilonController::new(0.05).name(), "epsilon"); + } +} diff --git a/crates/ml/src/trainers/dqn/controllers/mod.rs b/crates/ml/src/trainers/dqn/controllers/mod.rs index 086804cff..dc7f3ebe3 100644 --- a/crates/ml/src/trainers/dqn/controllers/mod.rs +++ b/crates/ml/src/trainers/dqn/controllers/mod.rs @@ -1,4 +1,4 @@ -//! Adaptive controller implementations — Plan 1 Tasks 9-12. +//! Adaptive controller implementations — Plan 1 Tasks 9-16. //! //! Each submodule implements `AdaptiveController` for one epoch-boundary //! controller in the DQN training pipeline. @@ -7,8 +7,16 @@ pub(crate) mod atoms_controller; pub(crate) mod gamma_controller; pub(crate) mod kelly_cap_controller; pub(crate) mod cql_alpha_controller; +pub(crate) mod tau_controller; +pub(crate) mod epsilon_controller; +pub(crate) mod conviction_floor_controller; +pub(crate) mod plan_threshold_controller; pub(crate) use atoms_controller::{AtomsController, AtomSignal}; pub(crate) use gamma_controller::{GammaController, GammaSignal}; pub(crate) use kelly_cap_controller::{KellyCapController, KellySignal}; pub(crate) use cql_alpha_controller::{CqlAlphaController, CqlAlphaSignal}; +pub(crate) use tau_controller::{TauController, TauSignal}; +pub(crate) use epsilon_controller::{EpsilonController, EpsilonSignal}; +pub(crate) use conviction_floor_controller::ConvictionFloorController; +pub(crate) use plan_threshold_controller::PlanThresholdController; diff --git a/crates/ml/src/trainers/dqn/controllers/plan_threshold_controller.rs b/crates/ml/src/trainers/dqn/controllers/plan_threshold_controller.rs new file mode 100644 index 000000000..434a48708 --- /dev/null +++ b/crates/ml/src/trainers/dqn/controllers/plan_threshold_controller.rs @@ -0,0 +1,173 @@ +//! Plan-threshold controller — Plan 1 Task 16, §4.C.6. +//! +//! Scaffolding controller for the plan-detection threshold used in +//! `experience_kernels.cu` and `backtest_plan_kernel.cu`. +//! +//! # Current threshold +//! +//! Both kernels gate plan activation with `> 0.5f`: +//! ```c +//! int has_plan_active = (portfolio_states[…] > 0.5f); +//! ``` +//! +//! The `0.5` literal is hardcoded in the CUDA kernels. This controller +//! establishes the trait scaffolding for Plan 3 §B.4, which will replace +//! the hardcoded literal with an ISV-driven value. Until Plan 3 lands, +//! `update()` returns the static `0.5` (behaviour-preserving). +//! +//! # Design note +//! +//! Per Plan 3 §B.4 the threshold will be driven by a new ISV slot +//! (PLAN_PARAMS_0_EMA) that does not exist yet. The controller reserves a +//! placeholder ISV read path; `read_signals` returns the static default +//! until the slot is allocated and wired. +//! +//! `write_output` is a no-op — the threshold is consumed as a kernel +//! constant argument. The CUDA kernels will be updated in Plan 3 §B.4 to +//! read the threshold from a new ISV slot or a pinned scalar. + +use crate::trainers::dqn::adaptive_controller::{ + AdaptiveController, DiagSnapshot, FireRateStats, IsvBus, +}; + +/// Static plan-threshold value (current hardcoded value in both CUDA kernels). +const STATIC_PLAN_THRESHOLD: f32 = 0.5; + +/// Plan-threshold controller input signal. +/// +/// Currently carries no adaptive inputs. Plan 3 §B.4 will add the +/// PLAN_PARAMS_0_EMA ISV slot once it is defined. +#[derive(Debug, Clone, Copy, Default)] +pub(crate) struct PlanThresholdSignal { + /// Placeholder — no adaptive signal exists today. + pub _reserved: f32, +} + +/// Plan-threshold controller output. +#[derive(Debug, Clone, Copy)] +pub(crate) struct PlanThresholdControl { + /// Plan activation threshold in [0, 1]. + /// Current value: 0.5 (static). Plan 3 §B.4 makes this ISV-driven. + pub threshold: f32, +} + +/// Static plan-threshold controller. +/// +/// Returns `0.5` each epoch (behaviour-preserving scaffolding). Plan 3 §B.4 +/// rewrites `update()` to consume a new PLAN_PARAMS_0_EMA ISV slot. +#[derive(Debug)] +pub(crate) struct PlanThresholdController { + fire_rate: FireRateStats, + last_threshold: f32, +} + +impl PlanThresholdController { + pub(crate) fn new() -> Self { + Self { + fire_rate: FireRateStats::default(), + last_threshold: STATIC_PLAN_THRESHOLD, + } + } +} + +impl Default for PlanThresholdController { + fn default() -> Self { Self::new() } +} + +impl AdaptiveController for PlanThresholdController { + type Signal = PlanThresholdSignal; + type Control = PlanThresholdControl; + + fn read_signals(&self, _isv: &IsvBus<'_>) -> PlanThresholdSignal { + // No ISV slot for plan params exists yet (Plan 3 §B.4 creates it). + PlanThresholdSignal { _reserved: 0.0 } + } + + fn update(&mut self, _signal: PlanThresholdSignal) -> PlanThresholdControl { + // Behaviour-preserving: return static 0.5. + // Plan 3 §B.4 replaces this with ISV-driven logic. + let threshold = STATIC_PLAN_THRESHOLD; + let changed = (threshold - self.last_threshold).abs() > 1e-6; + self.fire_rate.record_fire(changed); + self.last_threshold = threshold; + PlanThresholdControl { threshold } + } + + fn write_output(&self, _ctrl: &PlanThresholdControl, _isv: &mut IsvBus<'_>) { + // Threshold is consumed as a kernel constant in experience_kernels.cu + // and backtest_plan_kernel.cu. Plan 3 §B.4 will wire the output to + // the new ISV slot and update the kernel read sites. + } + + fn fire_rate(&self) -> &FireRateStats { + &self.fire_rate + } + + fn diagnose(&self) -> DiagSnapshot { + DiagSnapshot::new() + .with("threshold", self.last_threshold) + } + + fn name(&self) -> &'static str { + "plan_threshold" + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn plan_threshold_returns_static_half() { + let mut ctrl = PlanThresholdController::new(); + let out = ctrl.update(PlanThresholdSignal::default()); + assert!((out.threshold - 0.5_f32).abs() < 1e-6, + "expected 0.5, got {}", out.threshold); + } + + #[test] + fn plan_threshold_never_fires_on_static() { + let mut ctrl = PlanThresholdController::new(); + for _ in 0..20 { + ctrl.update(PlanThresholdSignal::default()); + } + assert_eq!(ctrl.fire_rate().fire_rate(), 0.0, + "static threshold should never fire"); + } + + #[test] + fn plan_threshold_read_signals_returns_default() { + let ctrl = PlanThresholdController::new(); + let mut buf = vec![0.0_f32; 39]; + let isv = IsvBus::new(&mut buf); + let sig = ctrl.read_signals(&isv); + assert_eq!(sig._reserved, 0.0); + } + + #[test] + fn plan_threshold_write_output_is_noop() { + let ctrl = PlanThresholdController::new(); + let control = PlanThresholdControl { threshold: 0.5 }; + let mut buf = vec![0.0_f32; 39]; + let mut isv = IsvBus::new(&mut buf); + ctrl.write_output(&control, &mut isv); + // No ISV slot written — bus unchanged + for val in &buf { + assert_eq!(*val, 0.0, "write_output must not mutate ISV (no slot assigned yet)"); + } + } + + #[test] + fn plan_threshold_diagnose_has_threshold() { + let mut ctrl = PlanThresholdController::new(); + ctrl.update(PlanThresholdSignal::default()); + let diag = ctrl.diagnose(); + assert!(diag.fields.contains_key("threshold"), + "diagnose() must expose threshold field"); + } + + #[test] + fn plan_threshold_name() { + assert_eq!(PlanThresholdController::new().name(), "plan_threshold"); + } +} diff --git a/crates/ml/src/trainers/dqn/controllers/tau_controller.rs b/crates/ml/src/trainers/dqn/controllers/tau_controller.rs new file mode 100644 index 000000000..4c635c0f2 --- /dev/null +++ b/crates/ml/src/trainers/dqn/controllers/tau_controller.rs @@ -0,0 +1,266 @@ +//! Tau (Polyak EMA) controller — Plan 1 Task 13, §4.C.6. +//! +//! Encapsulates the cosine-annealed target-network EMA coefficient (tau) +//! computation, including the health-coupled minimum floor that accelerates +//! target adaptation during training collapse. +//! +//! # Schedule +//! +//! ```text +//! tau(t) = tau_final - (tau_final - tau_base) * (cos(pi*t/T) + 1) / 2 +//! ``` +//! +//! Early training: `tau ≈ tau_base` (fast target updates). +//! Late training: `tau ≈ tau_final` (stable bootstrap target). +//! With `anneal_steps == 0`: fixed `tau_base` (no annealing). +//! +//! # Health-coupled floor +//! +//! The effective tau is `max(tau_scheduled, 0.01 * (1 − health))`. +//! During collapse (health ≈ 0) the floor rises to 0.01, accelerating +//! target-network adaptation to prevent runaway Q-values. +//! +//! # Migration notes +//! +//! Before migration, three call sites in `fused_training.rs` duplicated the +//! `compute_cosine_annealed_tau(…) + apply_health_coupled_tau_floor(…)` pattern +//! inline. After migration those sites remain as the actuators — they call +//! `controller.update(signals)` and use `ctrl.tau_eff` directly. The +//! `compute_cosine_annealed_tau` free function in `fused_training.rs` is +//! preserved (it is used by the actuator call sites at epoch boundary); the +//! controller mirrors that formula so tests can exercise it in isolation. +//! +//! `write_output` is a no-op: tau is consumed by `iqn.set_tau_host()` and +//! `dqn.target_ema_update()` at the call sites in `fused_training.rs`. No ISV +//! slot exists for this scalar. + +use crate::trainers::dqn::adaptive_controller::{ + AdaptiveController, DiagSnapshot, FireRateStats, IsvBus, +}; +use crate::cuda_pipeline::gpu_dqn_trainer::LEARNING_HEALTH_INDEX; + +// ISV slot [2]: TD-error EMA (used as an indicator of schedule progress). +// Slot [11]: regime stability EMA. +// Slot [12]: learning health — used for the floor computation. +const ISV_LEARNING_HEALTH: usize = LEARNING_HEALTH_INDEX; // 12 + +/// Tau controller input signals. +#[derive(Debug, Clone, Copy)] +pub(crate) struct TauSignal { + /// Monotonic training-step counter (across all slices/folds). + pub training_steps: u64, + /// Base tau from config (start of cosine schedule). + pub tau_base: f64, + /// Final tau from config (end of cosine schedule). + pub tau_final: f64, + /// Number of anneal steps (0 = no annealing, use tau_base). + pub anneal_steps: u64, + /// Learning health from ISV[12] — drives the floor computation. + pub health: f32, +} + +/// Tau controller output. +#[derive(Debug, Clone, Copy)] +pub(crate) struct TauControl { + /// Effective tau after cosine schedule + health-coupled floor. + pub tau_eff: f32, + /// Whether tau_eff changed from the previous epoch (fire signal). + pub changed: bool, +} + +/// Adaptive tau controller. +/// +/// Holds the last effective tau and `FireRateStats`. The `update()` method +/// mirrors `compute_cosine_annealed_tau` + `apply_health_coupled_tau_floor` +/// from `fused_training.rs` / `gpu_dqn_trainer.rs`. +#[derive(Debug)] +pub(crate) struct TauController { + fire_rate: FireRateStats, + last_tau_eff: f32, +} + +impl TauController { + pub(crate) fn new(initial_tau: f32) -> Self { + Self { + fire_rate: FireRateStats::default(), + last_tau_eff: initial_tau, + } + } + + /// Mirror of `compute_cosine_annealed_tau` in `fused_training.rs`. + /// Returns the scheduled tau (before health floor). + fn cosine_scheduled(steps: u64, tau_base: f64, tau_final: f64, anneal_steps: u64) -> f64 { + if anneal_steps > 0 { + let progress = (steps as f64 / anneal_steps as f64).min(1.0); + let cosine_factor = (std::f64::consts::PI * progress).cos(); + tau_final - (tau_final - tau_base) * (cosine_factor + 1.0) / 2.0 + } else { + tau_base + } + } + + /// Mirror of `apply_health_coupled_tau_floor` in `gpu_dqn_trainer.rs`. + /// Returns `max(tau_scheduled, 0.01 * (1 − health))`. + fn apply_floor(tau_scheduled: f64, health: f32) -> f64 { + let tau_floor = 0.01 * (1.0 - health.clamp(0.0, 1.0)) as f64; + tau_scheduled.max(tau_floor) + } +} + +impl Default for TauController { + fn default() -> Self { + Self::new(0.005) + } +} + +impl AdaptiveController for TauController { + type Signal = TauSignal; + type Control = TauControl; + + fn read_signals(&self, isv: &IsvBus<'_>) -> TauSignal { + // training_steps, tau_base, tau_final, and anneal_steps are not in the + // ISV bus — they are epoch-level config/state values. The training loop + // constructs the signal directly. Health is read here for diagnostic + // completeness; the training loop may also pass it directly. + TauSignal { + training_steps: 0, + tau_base: 0.005, + tau_final: 0.0005, + anneal_steps: 0, + health: isv.read(ISV_LEARNING_HEALTH).clamp(0.0, 1.0), + } + } + + fn update(&mut self, signal: TauSignal) -> TauControl { + let tau_scheduled = Self::cosine_scheduled( + signal.training_steps, + signal.tau_base, + signal.tau_final, + signal.anneal_steps, + ); + let tau_eff = Self::apply_floor(tau_scheduled, signal.health) as f32; + let changed = (tau_eff - self.last_tau_eff).abs() > 1e-6; + self.fire_rate.record_fire(changed); + self.last_tau_eff = tau_eff; + TauControl { tau_eff, changed } + } + + fn write_output(&self, _ctrl: &TauControl, _isv: &mut IsvBus<'_>) { + // Tau is written to pinned GPU memory via iqn.set_tau_host() and + // dqn.target_ema_update() at the actuator call sites in fused_training.rs. + // No ISV slot carries this scalar. + } + + fn fire_rate(&self) -> &FireRateStats { + &self.fire_rate + } + + fn diagnose(&self) -> DiagSnapshot { + DiagSnapshot::new() + .with("tau_eff", self.last_tau_eff) + } + + fn name(&self) -> &'static str { + "tau" + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // ISV bus length (ISV_TOTAL_DIM = 39) sufficient for all indices. + fn make_isv_with_health(health: f32) -> Vec { + let mut buf = vec![0.0_f32; 39]; + buf[ISV_LEARNING_HEALTH] = health; + buf + } + + fn make_signal(steps: u64, tau_base: f64, tau_final: f64, anneal: u64, health: f32) -> TauSignal { + TauSignal { training_steps: steps, tau_base, tau_final, anneal_steps: anneal, health } + } + + #[test] + fn tau_no_annealing_returns_base() { + let mut ctrl = TauController::new(0.005); + let sig = make_signal(10_000, 0.005, 0.0005, 0, 1.0); + let out = ctrl.update(sig); + // No annealing → tau_scheduled = tau_base = 0.005 + // health=1.0 → floor = 0.01*(1-1)=0 → tau_eff = 0.005 + assert!((out.tau_eff - 0.005_f32).abs() < 1e-6, + "expected 0.005, got {}", out.tau_eff); + } + + #[test] + fn tau_cosine_schedule_midpoint() { + let mut ctrl = TauController::new(0.005); + // At t = T/2 = 5000, T = 10000: + // progress = 0.5 + // cosine_factor = cos(pi * 0.5) = 0.0 + // tau = tau_final - (tau_final - tau_base) * (0.0 + 1.0) / 2.0 + // = 0.0005 - (0.0005 - 0.005) * 0.5 + // = 0.0005 + 0.00225 = 0.00275 + let sig = make_signal(5_000, 0.005, 0.0005, 10_000, 1.0); + let out = ctrl.update(sig); + let expected = 0.00275_f32; + assert!((out.tau_eff - expected).abs() < 1e-6, + "expected {:.6}, got {:.6}", expected, out.tau_eff); + } + + #[test] + fn tau_cosine_schedule_complete() { + let mut ctrl = TauController::new(0.005); + // At t >= T: clamped to 1.0 progress → cos(pi) = -1 → tau = tau_final + let sig = make_signal(20_000, 0.005, 0.0005, 10_000, 1.0); + let out = ctrl.update(sig); + assert!((out.tau_eff - 0.0005_f32).abs() < 1e-6, + "expected tau_final=0.0005, got {}", out.tau_eff); + } + + #[test] + fn tau_health_floor_during_collapse() { + let mut ctrl = TauController::new(0.0005); + // health=0 → floor = 0.01*(1-0)=0.01 > tau_base=0.0005 + let sig = make_signal(0, 0.0005, 0.00005, 0, 0.0); + let out = ctrl.update(sig); + assert!((out.tau_eff - 0.01_f32).abs() < 1e-6, + "expected floor 0.01, got {}", out.tau_eff); + } + + #[test] + fn tau_fire_rate_tracks_changes() { + let mut ctrl = TauController::new(0.005); + // First call: steps=0 → tau_scheduled = tau_base, same as initial → no change + ctrl.update(make_signal(0, 0.005, 0.0005, 10_000, 1.0)); + // Steps mid-schedule: tau changes + ctrl.update(make_signal(5_000, 0.005, 0.0005, 10_000, 1.0)); + ctrl.update(make_signal(10_000, 0.005, 0.0005, 10_000, 1.0)); + // At least some epochs fired (tau changed as schedule progressed) + let rate = ctrl.fire_rate().fire_rate(); + assert!(rate > 0.0, "expected fires as tau anneals, got rate={rate}"); + } + + #[test] + fn tau_read_signals_reads_health_from_isv() { + let ctrl = TauController::new(0.005); + let mut buf = make_isv_with_health(0.8); + let isv = IsvBus::new(&mut buf); + let sig = ctrl.read_signals(&isv); + assert!((sig.health - 0.8).abs() < 1e-6, + "expected health 0.8, got {}", sig.health); + } + + #[test] + fn tau_diagnose_has_tau_eff() { + let mut ctrl = TauController::new(0.005); + ctrl.update(make_signal(0, 0.005, 0.0005, 0, 0.7)); + let diag = ctrl.diagnose(); + assert!(diag.fields.contains_key("tau_eff"), + "diagnose() must expose tau_eff"); + } + + #[test] + fn tau_name() { + assert_eq!(TauController::new(0.005).name(), "tau"); + } +} diff --git a/crates/ml/src/trainers/dqn/trainer/constructor.rs b/crates/ml/src/trainers/dqn/trainer/constructor.rs index 296fd82cd..8ba89faf9 100644 --- a/crates/ml/src/trainers/dqn/trainer/constructor.rs +++ b/crates/ml/src/trainers/dqn/trainer/constructor.rs @@ -540,6 +540,8 @@ impl DQNTrainer { // Capture before hyperparams is moved into the struct. let init_gamma = hyperparams.gamma as f32; let init_cql_alpha = hyperparams.cql_alpha; + let init_tau = hyperparams.tau; + let init_epsilon_floor = hyperparams.noisy_epsilon_floor.unwrap_or(0.05) as f32; Ok(Self { agent: Arc::new(RwLock::new(agent)), @@ -771,12 +773,24 @@ impl DQNTrainer { meta_q: crate::cuda_pipeline::meta_q_network::MetaQNetwork::new(), pending_meta_q_samples: std::collections::VecDeque::with_capacity(32), health_history: std::collections::VecDeque::with_capacity(2 * crate::cuda_pipeline::meta_q_network::META_Q_LOOKAHEAD), - // Plan 1 adaptive controllers (Tasks 9-12). + // Plan 1 adaptive controllers (Tasks 9-16). // GammaController initialised from hyperparams.gamma (default 0.99). atoms_controller: crate::trainers::dqn::controllers::AtomsController::new(), gamma_controller: crate::trainers::dqn::controllers::GammaController::new(init_gamma), kelly_cap_controller: crate::trainers::dqn::controllers::KellyCapController::new(), cql_alpha_controller: crate::trainers::dqn::controllers::CqlAlphaController::new(init_cql_alpha), + // Task 13: TauController — cosine-annealed Polyak EMA with health floor. + // Initialised from config.tau (Polyak EMA base coefficient, default 0.005). + tau_controller: crate::trainers::dqn::controllers::TauController::new(init_tau), + // Task 14: EpsilonController — ISV-adaptive exploration floor. + // base_floor = noisy_epsilon_floor (default 0.05). + epsilon_controller: crate::trainers::dqn::controllers::EpsilonController::new( + init_epsilon_floor, + ), + // Task 15: ConvictionFloorController — static SchemaContract floor at ISV[36]. + conviction_floor_controller: crate::trainers::dqn::controllers::ConvictionFloorController::new(), + // Task 16: PlanThresholdController — static 0.5 scaffold (Plan 3 §B.4 extends). + plan_threshold_controller: crate::trainers::dqn::controllers::PlanThresholdController::new(), }) } diff --git a/crates/ml/src/trainers/dqn/trainer/mod.rs b/crates/ml/src/trainers/dqn/trainer/mod.rs index a0248a958..0e59a6608 100644 --- a/crates/ml/src/trainers/dqn/trainer/mod.rs +++ b/crates/ml/src/trainers/dqn/trainer/mod.rs @@ -659,7 +659,7 @@ pub struct DQNTrainer { /// Capped at 2×META_Q_LOOKAHEAD. pub(crate) health_history: std::collections::VecDeque<(u32, f32)>, - // ── Plan 1 adaptive controllers (Tasks 9-12) ────────────────────────────── + // ── Plan 1 adaptive controllers (Tasks 9-16) ────────────────────────────── /// Task 9: C51 atom-position controller — reads ISV v-range slots, /// gates the per-epoch `recompute_atom_positions` GPU kernel call. pub(crate) atoms_controller: crate::trainers::dqn::controllers::AtomsController, @@ -672,6 +672,16 @@ pub struct DQNTrainer { /// Task 12: CQL alpha controller — scaffolding for seed-phase coupling /// (Plan 3); currently returns the static config alpha unchanged. pub(crate) cql_alpha_controller: crate::trainers::dqn::controllers::CqlAlphaController, + /// Task 13: Tau (Polyak EMA) controller — cosine-annealed target-network + /// EMA coefficient with health-coupled floor. + pub(crate) tau_controller: crate::trainers::dqn::controllers::TauController, + /// Task 14: ISV-adaptive epsilon controller — `base_floor*(0.5+volatility)`. + pub(crate) epsilon_controller: crate::trainers::dqn::controllers::EpsilonController, + /// Task 15: Conviction-floor controller — static IQL branch_scales floor + /// at ISV[36]; SchemaContract slot, static at 0.1. + pub(crate) conviction_floor_controller: crate::trainers::dqn::controllers::ConvictionFloorController, + /// Task 16: Plan-threshold controller — static 0.5 scaffold for Plan 3 §B.4. + pub(crate) plan_threshold_controller: crate::trainers::dqn::controllers::PlanThresholdController, } impl std::fmt::Debug for DQNTrainer { diff --git a/crates/ml/src/trainers/dqn/trainer/training_loop.rs b/crates/ml/src/trainers/dqn/trainer/training_loop.rs index e7283f195..2fc045c59 100644 --- a/crates/ml/src/trainers/dqn/trainer/training_loop.rs +++ b/crates/ml/src/trainers/dqn/trainer/training_loop.rs @@ -927,8 +927,19 @@ impl DQNTrainer { (0.5, 0.5) }; - let base_floor = self.hyperparams.noisy_epsilon_floor.unwrap_or(0.05) as f32; - let adaptive_epsilon = base_floor * (0.5 + volatility); + // Task 14: EpsilonController wraps `base_floor*(0.5+volatility)`. + // Keep base_floor in sync with current hyperparams. + self.epsilon_controller.base_floor = + self.hyperparams.noisy_epsilon_floor.unwrap_or(0.05) as f32; + let eps_sig = crate::trainers::dqn::controllers::EpsilonSignal { + volatility, + base_floor: self.epsilon_controller.base_floor, + }; + let adaptive_epsilon = { + use crate::trainers::dqn::adaptive_controller::AdaptiveController; + self.epsilon_controller.update(eps_sig).epsilon + }; + let mut agent = self.agent.write().await; agent.set_epsilon(adaptive_epsilon as f64); @@ -2047,9 +2058,46 @@ impl DQNTrainer { // scaffolding here does not override it. } - // B2/G3: propagate last tau_eff for logging. + // B2/G3: propagate last tau_eff for logging via TauController (Task 13). + // The per-step tau computation lives in fused_training.rs (cosine schedule + + // health floor). The controller records the last effective value for + // fire-rate tracking and diagnose(). Signal is built from the DQN config + // and health; fused.last_tau_eff() provides the ground-truth actuated value. if let Some(ref fused) = self.fused_ctx { - self.last_tau_eff = Some(fused.last_tau_eff()); + let last_tau = fused.last_tau_eff(); + self.last_tau_eff = Some(last_tau); + // Build a signal from the current config so the controller mirrors + // the same schedule the fused-training path computes. + let dqn_tau_base = f64::from(self.hyperparams.tau); + let dqn_tau_final = f64::from(self.hyperparams.tau) * 0.5; + let dqn_anneal_steps = { + // Use config.tau_anneal_steps if accessible, else 500_000 default. + // Agent primary_dqn provides the config value without mutating. + self.agent.try_read() + .ok() + .and_then(|ag| Some(ag.primary_dqn().config.tau_anneal_steps)) + .unwrap_or(500_000) + }; + let dqn_steps = self.agent.try_read() + .ok() + .and_then(|ag| Some(ag.primary_dqn().get_training_steps())) + .unwrap_or(0); + { + use crate::trainers::dqn::adaptive_controller::AdaptiveController; + let tau_sig = crate::trainers::dqn::controllers::TauSignal { + training_steps: dqn_steps, + tau_base: dqn_tau_base, + tau_final: dqn_tau_final, + anneal_steps: dqn_anneal_steps, + health: health_value, + }; + let _tau_ctrl = self.tau_controller.update(tau_sig); + // _tau_ctrl.tau_eff mirrors last_tau; the ground-truth actuated + // value from fused_training.rs is used for HEALTH_DIAG logging + // (self.last_tau_eff set above). The controller's fire-rate tracks + // changes for the controller_activity smoke test. + let _ = _tau_ctrl; + } } // B3/G4: propagate SARSA tau factor for logging. diff --git a/docs/dqn-wire-up-audit.md b/docs/dqn-wire-up-audit.md index 2c365ac4a..24e37f704 100644 --- a/docs/dqn-wire-up-audit.md +++ b/docs/dqn-wire-up-audit.md @@ -11,3 +11,7 @@ Format: `| commit | component | files touched | notes |` | (Task 10) | `GammaController` — G4 adaptive gamma epoch controller | `controllers/gamma_controller.rs`, `training_loop.rs` | Inline G4 block (~18 lines) removed; replaced by `GammaController::update()`. `adaptive_gamma` field kept in sync. | | (Task 11) | `KellyCapController` — half-Kelly position-cap epoch controller | `controllers/kelly_cap_controller.rs`, `training_loop.rs` | Inline kelly_f_mean block (~20 lines) removed; replaced by `KellyCapController::update()`. No ISV slot; TradeStats passed directly. | | (Task 12) | `CqlAlphaController` — CQL alpha scaffolding controller | `controllers/cql_alpha_controller.rs`, `training_loop.rs`, `trainer/mod.rs`, `trainer/constructor.rs` | New scaffolding; static alpha returned. Plan 3 adds seed-phase coupling. No old ad-hoc function existed. | +| (Task 13) | `TauController` — cosine-annealed Polyak EMA controller | `controllers/tau_controller.rs`, `controllers/mod.rs`, `trainer/mod.rs`, `trainer/constructor.rs`, `trainer/training_loop.rs` | Mirrors `compute_cosine_annealed_tau` + `apply_health_coupled_tau_floor` from `fused_training.rs`/`gpu_dqn_trainer.rs`. Per-step tau actuators remain in `fused_training.rs`; controller tracks fire-rate at epoch boundary. 8 tests. | +| (Task 14) | `EpsilonController` — ISV-adaptive exploration epsilon | `controllers/epsilon_controller.rs`, `controllers/mod.rs`, `trainer/mod.rs`, `trainer/constructor.rs`, `trainer/training_loop.rs` | Replaces inline `base_floor*(0.5+volatility)` block (~10 lines) in `initialize_epoch_state`. Volatility from ISV[2]. `agent.set_epsilon()` actuator call preserved. 8 tests. | +| (Task 15) | `ConvictionFloorController` — IQL branch_scales floor scaffolding | `controllers/conviction_floor_controller.rs`, `controllers/mod.rs`, `trainer/mod.rs`, `trainer/constructor.rs` | Static 0.1 (SchemaContract). `write_output` writes to ISV[36]. Never fires. Plan 3 adds ramp logic. 7 tests. | +| (Task 16) | `PlanThresholdController` — plan activation threshold scaffolding | `controllers/plan_threshold_controller.rs`, `controllers/mod.rs`, `trainer/mod.rs`, `trainer/constructor.rs` | Static 0.5, mirrors `> 0.5f` literal in `experience_kernels.cu` and `backtest_plan_kernel.cu`. Never fires. Plan 3 §B.4 wires ISV slot. 6 tests. |