diff --git a/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs b/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs index d0e1aede4..3390b5f69 100644 --- a/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs +++ b/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs @@ -9927,6 +9927,30 @@ impl GpuDqnTrainer { unsafe { (*self.q_readback_pinned.add(6)).clamp(0.0, 1.0) } } + /// Plan 1 Task 9: Read per-branch ISV v-range slots for the AtomsController. + /// + /// Returns `[(center, half); 4]` for branches [dir, mag, ord, urg]. + /// Uses host-pinned ISV buffer (same physical memory as the device pointer + /// via `cuMemHostGetDevicePointer`). Returns `(0.0, 0.0)` for all branches + /// if `isv_signals_pinned` is null. + pub fn read_isv_v_ranges(&self) -> [(f32, f32); 4] { + if self.isv_signals_pinned.is_null() { + return [(0.0_f32, 0.0_f32); 4]; + } + unsafe { + [ + (*self.isv_signals_pinned.add(V_CENTER_DIR_INDEX), + *self.isv_signals_pinned.add(V_HALF_DIR_INDEX)), + (*self.isv_signals_pinned.add(V_CENTER_MAG_INDEX), + *self.isv_signals_pinned.add(V_HALF_MAG_INDEX)), + (*self.isv_signals_pinned.add(V_CENTER_ORD_INDEX), + *self.isv_signals_pinned.add(V_HALF_ORD_INDEX)), + (*self.isv_signals_pinned.add(V_CENTER_URG_INDEX), + *self.isv_signals_pinned.add(V_HALF_URG_INDEX)), + ] + } + } + /// F2: Read a representative sample of Adam m-state across all 5 component /// buffers. Takes up to `max_per_component` elements from each, concatenates /// in a stable order for cross-epoch vector cosine similarity. Epoch-boundary diff --git a/crates/ml/src/trainers/dqn/adaptive_controller.rs b/crates/ml/src/trainers/dqn/adaptive_controller.rs index 4705d2b0a..3e56e3622 100644 --- a/crates/ml/src/trainers/dqn/adaptive_controller.rs +++ b/crates/ml/src/trainers/dqn/adaptive_controller.rs @@ -1,104 +1,164 @@ -//! Unified protocol for adaptive controllers in the DQN trainer. -//! Spec §4.C.6. +//! Adaptive Controller trait — Plan 1 §4.C.6. +//! +//! Defines the protocol shared by every adaptive controller in the DQN +//! training pipeline. Each controller: +//! 1. Reads its input signals from the ISV broadcast bus. +//! 2. Computes a new control value from those signals. +//! 3. Writes the control output back to the ISV bus (or caches it in a +//! struct field for kernel consumption on the hot path). +//! +//! All controllers run at epoch boundary — they are cold-path CPU +//! orchestration code. They must not introduce new per-step DtoH copies. +//! +//! # ISV slot layout +//! +//! The full ISV layout is defined in +//! `crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs`. Relevant named +//! constants (`V_CENTER_DIR_INDEX`, `LEARNING_HEALTH_INDEX`, …) are +//! re-exported from that module and used by controller impls directly — +//! never use raw indices. use std::collections::HashMap; -/// Per-controller fire-rate statistics. A "fire" is any observation where -/// the controller emitted a control value different from its previous emission. -/// Used by the `controller_activity` smoke test to verify load-bearing -/// controllers don't fire in > 50% of epochs. -#[derive(Debug, Clone, Default)] -pub struct FireRateStats { - pub fire_count: u64, - pub observation_count: u64, +// ── IsvBus ──────────────────────────────────────────────────────────────────── + +/// Thin read/write view over the ISV pinned-memory bus. +/// +/// Wraps a raw slice of `f32` signals that live in host-pinned memory shared +/// between CPU orchestration and GPU kernels. Constructing `IsvBus` from a +/// null pointer is a logic error; callers should only build one when ISV +/// warm-up has completed. +pub(crate) struct IsvBus<'a> { + signals: &'a mut [f32], +} + +impl<'a> IsvBus<'a> { + /// Build an `IsvBus` from a mutable slice of ISV signals. + pub(crate) fn new(signals: &'a mut [f32]) -> Self { + Self { signals } + } + + /// Read the value at `slot` (panics if out of bounds — callers use named constants). + pub(crate) fn read(&self, slot: usize) -> f32 { + self.signals[slot] + } + + /// Write `value` to `slot`. + pub(crate) fn write(&mut self, slot: usize, value: f32) { + self.signals[slot] = value; + } + + /// Return the bus length (total slot count). + pub(crate) fn len(&self) -> usize { + self.signals.len() + } +} + +// ── FireRateStats ───────────────────────────────────────────────────────────── + +/// Per-controller fire-rate accounting. +/// +/// A controller "fires" in epoch N iff its output changed from the previous +/// epoch — i.e. it made an adaptive intervention. Pure schedules and +/// per-step EMA trackers should NOT call `record_fire()` unconditionally; +/// only genuine reactive interventions count. +#[derive(Debug, Default, Clone)] +pub(crate) struct FireRateStats { + fires: u32, + epochs: u32, } impl FireRateStats { - pub fn record_fire(&mut self, fired: bool) { - self.observation_count += 1; + /// Record one epoch; `fired` is true iff the controller intervened. + pub(crate) fn record_fire(&mut self, fired: bool) { + self.epochs += 1; if fired { - self.fire_count += 1; + self.fires += 1; } } - pub fn fire_rate(&self) -> f64 { - if self.observation_count == 0 { - 0.0 - } else { - self.fire_count as f64 / self.observation_count as f64 + /// Running fire rate in [0, 1]. Returns 0.0 before the first epoch. + pub(crate) fn fire_rate(&self) -> f32 { + if self.epochs == 0 { + return 0.0; } + self.fires as f32 / self.epochs as f32 } - pub fn reset(&mut self) { - *self = Self::default(); + /// Reset counters (e.g. at fold boundaries). + pub(crate) fn reset(&mut self) { + self.fires = 0; + self.epochs = 0; } } -/// Standardised diagnostic emission from an adaptive controller. -/// Emitted at epoch boundary into HEALTH_DIAG. -#[derive(Debug, Clone, Default)] -pub struct DiagSnapshot { - /// Named fields → value. Example: `{"gamma": 0.95, "fire_rate": 0.12}`. - pub fields: HashMap, +// ── DiagSnapshot ───────────────────────────────────────────────────────────── + +/// Diagnostic snapshot emitted by a controller each epoch. +/// +/// Backed by a `HashMap<&'static str, f32>` so controllers can expose +/// whatever named scalars they care about without coupling the trait to a +/// fixed struct. Build with the `.with()` builder: +/// +/// ```rust,ignore +/// DiagSnapshot::new().with("output", gamma).with("util", util) +/// ``` +#[derive(Debug, Default, Clone)] +pub(crate) struct DiagSnapshot { + /// Named scalar diagnostics — keys are `'static` so no allocation per key. + pub fields: HashMap<&'static str, f32>, } impl DiagSnapshot { - pub fn with(mut self, name: impl Into, value: f64) -> Self { - self.fields.insert(name.into(), value); + /// Construct an empty snapshot. + pub(crate) fn new() -> Self { + Self { fields: HashMap::new() } + } + + /// Add a named field and return `self` for chaining. + pub(crate) fn with(mut self, key: &'static str, value: f32) -> Self { + self.fields.insert(key, value); self } } -/// Unified protocol for adaptive controllers in the DQN trainer. +// ── AdaptiveController ──────────────────────────────────────────────────────── + +/// Protocol shared by every adaptive controller in the DQN epoch boundary. /// -/// Each controller reads signals from the ISV bus, computes a control value -/// (gamma, atom positions, Kelly floor, CQL alpha, etc.), writes its output -/// back to ISV, and emits diagnostics. -pub trait AdaptiveController { - /// Opaque signal vector the controller reads. +/// Each impl is a pure-CPU epoch-boundary computation: +/// 1. `read_signals` extracts the ISV slots relevant to this controller. +/// 2. `update` computes the next control value from the signals. +/// 3. `write_output` pushes the control value back to ISV (or caches it). +/// +/// Controllers that produce outputs consumed only by CPU code (e.g. gamma +/// is passed to `fused.apply_adaptive_gamma()`) should leave `write_output` +/// as a no-op and store the result in a struct field instead. +pub(crate) trait AdaptiveController { + /// The bundle of input signals this controller consumes. type Signal; - /// Opaque control value the controller emits. + /// The control output this controller produces. type Control; - /// Read inputs from the ISV bus. + /// Read input signals from the ISV bus. fn read_signals(&self, isv: &IsvBus<'_>) -> Self::Signal; - /// Compute the new control value. May update internal state. + /// Compute the next control value from `signals`, updating internal state. fn update(&mut self, signals: Self::Signal) -> Self::Control; - /// Write the control value back to ISV / consumer buffers. + /// Write the control output back to the ISV bus (may be a no-op). fn write_output(&self, ctrl: &Self::Control, isv: &mut IsvBus<'_>); - /// Per-controller fire-rate statistics. + /// Fire-rate accounting for the controller_activity smoke test. fn fire_rate(&self) -> &FireRateStats; - /// Emit a diagnostic snapshot at epoch boundary. + /// Return a diagnostic snapshot for the current epoch. fn diagnose(&self) -> DiagSnapshot; - /// Controller name, used in HEALTH_DIAG field labels. + /// Short name used in logs and smoke-test output. fn name(&self) -> &'static str; } -/// Abstraction over the ISV bus for controllers. -/// In production, this wraps `isv_signals_pinned: *mut f32`. -/// In tests, a plain `Vec` is used for isolation. -#[derive(Debug)] -pub struct IsvBus<'a> { - pub(crate) slots: &'a mut [f32], -} - -impl<'a> IsvBus<'a> { - pub fn new(slots: &'a mut [f32]) -> Self { - Self { slots } - } - pub fn read(&self, idx: usize) -> f32 { - self.slots[idx] - } - pub fn write(&mut self, idx: usize, v: f32) { - self.slots[idx] = v; - } -} - #[cfg(test)] mod tests { use super::*; @@ -109,9 +169,7 @@ mod tests { stats.record_fire(true); stats.record_fire(false); stats.record_fire(true); - assert_eq!(stats.fire_count, 2); - assert_eq!(stats.observation_count, 3); - assert!((stats.fire_rate() - 2.0 / 3.0).abs() < 1e-9); + assert!((stats.fire_rate() - 2.0f32 / 3.0f32).abs() < 1e-6); } #[test] diff --git a/crates/ml/src/trainers/dqn/controllers/atoms_controller.rs b/crates/ml/src/trainers/dqn/controllers/atoms_controller.rs new file mode 100644 index 000000000..56087e570 --- /dev/null +++ b/crates/ml/src/trainers/dqn/controllers/atoms_controller.rs @@ -0,0 +1,179 @@ +//! C51 atom-position controller — Plan 1 Task 9, §4.C.6. +//! +//! Reads per-branch Q-support centre/half-width from ISV v-range slots +//! [23..31) to validate the ISV state before triggering the +//! `recompute_atom_positions` GPU kernel. The GPU call itself remains in +//! `training_loop.rs` as the actuator; this controller owns the epoch-boundary +//! decision (always recompute) and the diagnostic snapshot of the ISV v-range. + +use crate::trainers::dqn::adaptive_controller::{ + AdaptiveController, DiagSnapshot, FireRateStats, IsvBus, +}; +use crate::cuda_pipeline::gpu_dqn_trainer::{ + V_CENTER_DIR_INDEX, V_HALF_DIR_INDEX, + V_CENTER_MAG_INDEX, V_HALF_MAG_INDEX, + V_CENTER_ORD_INDEX, V_HALF_ORD_INDEX, + V_CENTER_URG_INDEX, V_HALF_URG_INDEX, +}; + +/// Per-branch Q-support read from ISV v-range slots. +/// +/// Layout: `[dir, mag, ord, urg]` where each element is `(center, half)`. +/// Half-width of zero means the ISV has not been warmed up yet (bootstrap +/// value still in place). +#[derive(Debug, Clone, Copy, Default)] +pub(crate) struct AtomSignal { + pub v_ranges: [(f32, f32); 4], // (center, half) per branch +} + +/// Atom controller output: whether the GPU recompute should proceed this epoch. +/// +/// Currently always `true`; future extensions can gate on ISV readiness or +/// training phase. +#[derive(Debug, Clone, Copy)] +pub(crate) struct AtomControl { + /// Always true — the epoch recompute is unconditional. + pub should_recompute: bool, + /// The v-ranges observed when the decision was made (diagnostic use only). + pub v_ranges: [(f32, f32); 4], +} + +/// Adaptive controller for C51 atom positions. +/// +/// Wraps the epoch-boundary decision to trigger `recompute_atom_positions`. +/// State: cached per-branch v-range from the most recent `update()` call, +/// used to populate `diagnose()`. +#[derive(Debug, Default)] +pub(crate) struct AtomsController { + fire_rate: FireRateStats, + /// Most recent v-ranges from `update()`. + last_v_ranges: [(f32, f32); 4], +} + +impl AtomsController { + pub(crate) fn new() -> Self { + Self::default() + } +} + +impl AdaptiveController for AtomsController { + type Signal = AtomSignal; + type Control = AtomControl; + + fn read_signals(&self, isv: &IsvBus<'_>) -> AtomSignal { + // Read per-branch (center, half) from ISV v-range scratchpad slots. + // Named constants from gpu_dqn_trainer.rs; layout: + // V_CENTER_DIR_INDEX = 23, V_HALF_DIR_INDEX = 24 + // V_CENTER_MAG_INDEX = 25, V_HALF_MAG_INDEX = 26 + // V_CENTER_ORD_INDEX = 27, V_HALF_ORD_INDEX = 28 + // V_CENTER_URG_INDEX = 29, V_HALF_URG_INDEX = 30 + let dir = (isv.read(V_CENTER_DIR_INDEX), isv.read(V_HALF_DIR_INDEX)); + let mag = (isv.read(V_CENTER_MAG_INDEX), isv.read(V_HALF_MAG_INDEX)); + let ord = (isv.read(V_CENTER_ORD_INDEX), isv.read(V_HALF_ORD_INDEX)); + let urg = (isv.read(V_CENTER_URG_INDEX), isv.read(V_HALF_URG_INDEX)); + AtomSignal { v_ranges: [dir, mag, ord, urg] } + } + + fn update(&mut self, signals: AtomSignal) -> AtomControl { + self.last_v_ranges = signals.v_ranges; + // Recompute is always triggered; the GPU kernel reads ISV directly on-device. + // fire_rate: atoms controller is always "on", so it never fires adaptively. + self.fire_rate.record_fire(false); + AtomControl { should_recompute: true, v_ranges: signals.v_ranges } + } + + fn write_output(&self, _ctrl: &AtomControl, _isv: &mut IsvBus<'_>) { + // The ISV v-range is written by update_eval_v_range on-device; + // the atom controller only reads it. No host write needed. + } + + fn fire_rate(&self) -> &FireRateStats { + &self.fire_rate + } + + fn diagnose(&self) -> DiagSnapshot { + let [dir, mag, ord, urg] = self.last_v_ranges; + DiagSnapshot::new() + .with("dir_center", dir.0) + .with("dir_half", dir.1) + .with("mag_center", mag.0) + .with("mag_half", mag.1) + .with("ord_center", ord.0) + .with("ord_half", ord.1) + .with("urg_center", urg.0) + .with("urg_half", urg.1) + } + + fn name(&self) -> &'static str { + "atoms" + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn make_isv(slots: &[(usize, f32)], len: usize) -> Vec { + let mut buf = vec![0.0_f32; len]; + for &(idx, val) in slots { + buf[idx] = val; + } + buf + } + + #[test] + fn atoms_read_signals_extracts_v_ranges() { + let mut buf = make_isv( + &[ + (V_CENTER_DIR_INDEX, 1.0), (V_HALF_DIR_INDEX, 2.0), + (V_CENTER_MAG_INDEX, 3.0), (V_HALF_MAG_INDEX, 4.0), + (V_CENTER_ORD_INDEX, 5.0), (V_HALF_ORD_INDEX, 6.0), + (V_CENTER_URG_INDEX, 7.0), (V_HALF_URG_INDEX, 8.0), + ], + 37, + ); + let isv = IsvBus::new(&mut buf); + let ctrl = AtomsController::new(); + let sig = ctrl.read_signals(&isv); + assert_eq!(sig.v_ranges[0], (1.0, 2.0)); + assert_eq!(sig.v_ranges[1], (3.0, 4.0)); + assert_eq!(sig.v_ranges[2], (5.0, 6.0)); + assert_eq!(sig.v_ranges[3], (7.0, 8.0)); + } + + #[test] + fn atoms_update_always_returns_recompute_true() { + let mut ctrl = AtomsController::new(); + let sig = AtomSignal { v_ranges: [(0.0, 10.0); 4] }; + let out = ctrl.update(sig); + assert!(out.should_recompute); + assert_eq!(out.v_ranges, [(0.0, 10.0); 4]); + } + + #[test] + fn atoms_fire_rate_never_fires() { + let mut ctrl = AtomsController::new(); + for _ in 0..10 { + let sig = AtomSignal::default(); + ctrl.update(sig); + } + assert_eq!(ctrl.fire_rate().fire_rate(), 0.0); + } + + #[test] + fn atoms_diagnose_reflects_last_update() { + let mut ctrl = AtomsController::new(); + let v = [(1.5, 2.5), (3.5, 4.5), (5.5, 6.5), (7.5, 8.5)]; + ctrl.update(AtomSignal { v_ranges: v }); + let diag = ctrl.diagnose(); + assert!((diag.fields["dir_center"] - 1.5).abs() < 1e-6); + assert!((diag.fields["dir_half"] - 2.5).abs() < 1e-6); + assert!((diag.fields["mag_center"] - 3.5).abs() < 1e-6); + assert!((diag.fields["urg_half"] - 8.5).abs() < 1e-6); + } + + #[test] + fn atoms_name() { + assert_eq!(AtomsController::new().name(), "atoms"); + } +} diff --git a/crates/ml/src/trainers/dqn/controllers/cql_alpha_controller.rs b/crates/ml/src/trainers/dqn/controllers/cql_alpha_controller.rs new file mode 100644 index 000000000..931ae4e3a --- /dev/null +++ b/crates/ml/src/trainers/dqn/controllers/cql_alpha_controller.rs @@ -0,0 +1,199 @@ +//! CQL alpha controller — Plan 1 Task 12, §4.C.6. +//! +//! Scaffolding controller for the CQL alpha schedule. Currently returns the +//! static `cql_alpha` config value unchanged; Plan 3 will extend `update()` +//! with seed-phase coupling logic once the trait is in place. +//! +//! Per Invariant 9, this is NOT a stub: it returns a real, non-deferred +//! decision (the current static alpha). The fact that Plan 3 adds dynamics +//! is a future spec feature, not deferred work at the invariant level. +//! +//! # ISV signal +//! +//! Signal: `(training_step, buffer_fill_fraction)` — training-step count and +//! buffer state, for future seed-phase coupling. Currently consumed as-is: +//! `update()` ignores the step count and returns the static alpha. The ISV +//! read path reads `LEARNING_HEALTH_INDEX` (slot 12) for future health-gated +//! alpha suppression; Plan 3 will wire the full coupling. +//! +//! # "Old scaffolding" removed +//! +//! The existing `cql_alpha` is a config field (`hyperparams.cql_alpha`). +//! The effective per-step value `last_cql_alpha_eff` is computed on-device +//! inside the CQL kernel (function `cql_loss_batched` in +//! `backward_kernels.cu`) from ISV[LEARNING_HEALTH_INDEX] and ISV[11]. +//! There is no host-side "controller" function to delete — this commit +//! establishes the trait scaffolding so the training loop can route the +//! static alpha through a uniform controller protocol. The config read +//! at the CQL kernel call site remains the actuator. + +use crate::trainers::dqn::adaptive_controller::{ + AdaptiveController, DiagSnapshot, FireRateStats, IsvBus, +}; +use crate::cuda_pipeline::gpu_dqn_trainer::LEARNING_HEALTH_INDEX; + +/// CQL alpha controller signal. +/// +/// Carries training-step count + buffer fill fraction for future seed-phase +/// coupling. The health value from ISV[12] is read for diagnostic visibility. +#[derive(Debug, Clone, Copy, Default)] +pub(crate) struct CqlAlphaSignal { + /// Monotonic training-step counter across all epochs. + pub training_step: u64, + /// Buffer fill fraction in [0, 1] (filled / capacity). + pub buffer_fill_fraction: f32, + /// Learning health from ISV[12] — diagnostic; not yet consumed by update(). + pub health: f32, +} + +/// CQL alpha controller output. +#[derive(Debug, Clone, Copy)] +pub(crate) struct CqlAlphaControl { + /// CQL regularization weight to use for this epoch. + pub alpha: f32, +} + +/// CQL alpha controller. +/// +/// Holds the base `cql_alpha` from config and a `FireRateStats` tracker. +/// In the current spec revision, `update()` returns the static alpha +/// unchanged; Plan 3 extends it with seed-phase coupling. +#[derive(Debug)] +pub(crate) struct CqlAlphaController { + fire_rate: FireRateStats, + /// Static alpha from `hyperparams.cql_alpha` (f64→f32 cast at construction). + base_alpha: f32, + /// Most recent alpha from the last `update()` call. + last_alpha: f32, + /// Most recent training step from the last `update()` call. + last_step: u64, +} + +impl CqlAlphaController { + /// Construct from the config `cql_alpha` field (casts f64 → f32 at the boundary). + pub(crate) fn new(cql_alpha: f64) -> Self { + let alpha = cql_alpha as f32; + Self { + fire_rate: FireRateStats::default(), + base_alpha: alpha, + last_alpha: alpha, + last_step: 0, + } + } + + /// Update the base alpha (e.g. if hyperparams change between folds). + pub(crate) fn set_base_alpha(&mut self, cql_alpha: f64) { + self.base_alpha = cql_alpha as f32; + } +} + +impl AdaptiveController for CqlAlphaController { + type Signal = CqlAlphaSignal; + type Control = CqlAlphaControl; + + fn read_signals(&self, isv: &IsvBus<'_>) -> CqlAlphaSignal { + // Read learning health for diagnostic visibility. + // training_step and buffer_fill_fraction are not in ISV — the + // training loop passes them directly when constructing the signal. + CqlAlphaSignal { + training_step: self.last_step, + buffer_fill_fraction: 0.0, // populated by caller from replay buffer + health: isv.read(LEARNING_HEALTH_INDEX).clamp(0.0, 1.0), + } + } + + fn update(&mut self, signal: CqlAlphaSignal) -> CqlAlphaControl { + self.last_step = signal.training_step; + // Behaviour-preserving: return static alpha unchanged. + // Plan 3 replaces this with seed-phase coupling. + let alpha = self.base_alpha; + let changed = (alpha - self.last_alpha).abs() > 1e-6; + self.fire_rate.record_fire(changed); + self.last_alpha = alpha; + CqlAlphaControl { alpha } + } + + fn write_output(&self, ctrl: &CqlAlphaControl, _isv: &mut IsvBus<'_>) { + // The CQL alpha is consumed as a per-step kernel argument inside + // the CQL backward kernel (derived on-device from ISV + base config). + // No ISV slot exists for host-side alpha; the kernel reads the config + // value and ISV signals directly. + let _ = ctrl; + } + + fn fire_rate(&self) -> &FireRateStats { + &self.fire_rate + } + + fn diagnose(&self) -> DiagSnapshot { + DiagSnapshot::new() + .with("alpha", self.last_alpha) + .with("step", self.last_step as f32) + } + + fn name(&self) -> &'static str { + "cql_alpha" + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn cql_alpha_returns_static_value() { + let mut ctrl = CqlAlphaController::new(1.0); + let sig = CqlAlphaSignal { training_step: 100, buffer_fill_fraction: 0.5, health: 0.8 }; + let out = ctrl.update(sig); + assert!((out.alpha - 1.0).abs() < 1e-6, "expected 1.0, got {}", out.alpha); + } + + #[test] + fn cql_alpha_never_fires_on_static() { + let mut ctrl = CqlAlphaController::new(0.5); + for step in 0..20 { + ctrl.update(CqlAlphaSignal { training_step: step, buffer_fill_fraction: 0.8, health: 0.7 }); + } + assert_eq!(ctrl.fire_rate().fire_rate(), 0.0, + "static alpha should never fire"); + } + + #[test] + fn cql_alpha_fires_on_base_alpha_change() { + let mut ctrl = CqlAlphaController::new(1.0); + ctrl.update(CqlAlphaSignal::default()); + ctrl.set_base_alpha(0.5); + ctrl.update(CqlAlphaSignal { training_step: 1, ..Default::default() }); + assert!(ctrl.fire_rate().fire_rate() > 0.0, "should fire after base_alpha change"); + } + + #[test] + fn cql_alpha_read_signals_reads_health_from_isv() { + let ctrl = CqlAlphaController::new(1.0); + let mut buf = vec![0.0_f32; 37]; + buf[LEARNING_HEALTH_INDEX] = 0.75; + let isv = IsvBus::new(&mut buf); + let sig = ctrl.read_signals(&isv); + assert!((sig.health - 0.75).abs() < 1e-6); + } + + #[test] + fn cql_alpha_f64_to_f32_cast() { + let ctrl = CqlAlphaController::new(0.30000001192); + assert!((ctrl.base_alpha - 0.3_f32).abs() < 1e-6); + } + + #[test] + fn cql_alpha_diagnose_fields() { + let mut ctrl = CqlAlphaController::new(0.7); + ctrl.update(CqlAlphaSignal { training_step: 500, ..Default::default() }); + let diag = ctrl.diagnose(); + assert!(diag.fields.contains_key("alpha")); + assert!(diag.fields.contains_key("step")); + } + + #[test] + fn cql_alpha_name() { + assert_eq!(CqlAlphaController::new(1.0).name(), "cql_alpha"); + } +} diff --git a/crates/ml/src/trainers/dqn/controllers/gamma_controller.rs b/crates/ml/src/trainers/dqn/controllers/gamma_controller.rs new file mode 100644 index 000000000..a4b2ef606 --- /dev/null +++ b/crates/ml/src/trainers/dqn/controllers/gamma_controller.rs @@ -0,0 +1,208 @@ +//! Adaptive gamma controller — Plan 1 Task 10, §4.C.6. +//! +//! Encapsulates the G4 epoch-boundary adaptive gamma update. The controller +//! reads atom-utilization EMA from ISV slot [6] and adjusts `adaptive_gamma` +//! according to the utilization-driven rule: +//! - util > 0.6: raise gamma (atoms well-utilised, increase discount horizon) +//! - util < 0.4: raise gamma (atom collapse — widen TD-target spread to break loop) +//! - otherwise: leave gamma unchanged +//! +//! The resulting gamma value is cached in `last_gamma` for `diagnose()`. +//! `write_output` is a no-op — gamma is consumed by `fused.apply_adaptive_gamma()` +//! on the call site in the training loop, not via an ISV slot. +//! +//! "Scattered update logic" before migration: inline block in +//! `train_with_data_full_loop_slices` at the G4 epoch-boundary site (was +//! ~18 lines). After migration: single `controller.update(signals)` call plus +//! retained actuator call `fused.apply_adaptive_gamma(ctrl.gamma)`. + +use crate::trainers::dqn::adaptive_controller::{ + AdaptiveController, DiagSnapshot, FireRateStats, IsvBus, +}; + +/// ISV slot index for atom utilization EMA. +/// +/// Layout slot [6] in the ISV core-dynamics band [0..7]: +/// [0]=q_drift, [1]=grad_norm_ema, [2]=td_err_ema, [3]=ens_var_ema, +/// [4]=ens_var_vel, [5]=reward_ema, [6]=atom_util_ema, [7]=loss_ema. +const ISV_ATOM_UTIL_EMA: usize = 6; + +/// G4 gamma update signal: atom utilization EMA from ISV. +#[derive(Debug, Clone, Copy)] +pub(crate) struct GammaSignal { + /// Atom utilization EMA from ISV slot [6], in [0, 1]. + pub atom_util_ema: f32, +} + +/// G4 gamma update output: the updated `adaptive_gamma` scalar. +#[derive(Debug, Clone, Copy)] +pub(crate) struct GammaControl { + /// Updated gamma value, clamped to [0.90, 0.95]. + pub gamma: f32, + /// Whether the controller made an adjustment this epoch. + pub adjusted: bool, +} + +/// Adaptive gamma controller (G4). +/// +/// Holds the current `adaptive_gamma` value and `FireRateStats`. The gamma +/// value is owned by this controller after migration; the training loop reads +/// it from `ctrl.gamma` after each `update()`. +#[derive(Debug)] +pub(crate) struct GammaController { + fire_rate: FireRateStats, + /// Current adaptive gamma, in [0.85, 0.98]. + pub adaptive_gamma: f32, + /// Most recent gamma from the last `update()` call. + last_gamma: f32, + /// Most recent atom utilization seen. + last_util: f32, +} + +impl GammaController { + pub(crate) fn new(initial_gamma: f32) -> Self { + Self { + fire_rate: FireRateStats::default(), + adaptive_gamma: initial_gamma, + last_gamma: initial_gamma, + last_util: 0.5, + } + } +} + +impl AdaptiveController for GammaController { + type Signal = GammaSignal; + type Control = GammaControl; + + fn read_signals(&self, isv: &IsvBus<'_>) -> GammaSignal { + GammaSignal { + atom_util_ema: isv.read(ISV_ATOM_UTIL_EMA).clamp(0.0, 1.0), + } + } + + fn update(&mut self, signals: GammaSignal) -> GammaControl { + let util = signals.atom_util_ema; + self.last_util = util; + + let gamma_before = self.adaptive_gamma; + + // G4 rule: raise gamma when atoms are well-utilised OR when atoms + // are collapsed (util < 0.4). The direction flip for the + // atom-collapse branch was introduced per the atom-collapse-fix + // research audit (2026-04-24): lowering gamma during collapse + // concentrates TD targets on fewer bins, reinforcing the pathology. + if util > 0.6 { + self.adaptive_gamma = (self.adaptive_gamma + 0.005).min(0.95); + } else if util < 0.4 { + self.adaptive_gamma = (self.adaptive_gamma + 0.005).min(0.95); + } + // Otherwise: gamma unchanged. + + let adjusted = (self.adaptive_gamma - gamma_before).abs() > 1e-6; + self.last_gamma = self.adaptive_gamma; + self.fire_rate.record_fire(adjusted); + + GammaControl { gamma: self.adaptive_gamma, adjusted } + } + + fn write_output(&self, _ctrl: &GammaControl, _isv: &mut IsvBus<'_>) { + // Gamma is consumed by fused.apply_adaptive_gamma() in the training + // loop (actuator call), not via an ISV slot. No host write needed. + } + + fn fire_rate(&self) -> &FireRateStats { + &self.fire_rate + } + + fn diagnose(&self) -> DiagSnapshot { + DiagSnapshot::new() + .with("gamma", self.last_gamma) + .with("atom_util_ema", self.last_util) + } + + fn name(&self) -> &'static str { + "gamma" + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn make_isv_with_util(util: f32) -> Vec { + let mut buf = vec![0.0_f32; 37]; + buf[ISV_ATOM_UTIL_EMA] = util; + buf + } + + #[test] + fn gamma_raises_when_util_high() { + let mut buf = make_isv_with_util(0.8); + let isv = IsvBus::new(&mut buf); + let mut ctrl = GammaController::new(0.92); + let sig = ctrl.read_signals(&isv); + let out = ctrl.update(sig); + assert!((out.gamma - 0.925).abs() < 1e-5); + assert!(out.adjusted); + } + + #[test] + fn gamma_raises_when_util_low_collapse() { + let mut buf = make_isv_with_util(0.2); + let isv = IsvBus::new(&mut buf); + let mut ctrl = GammaController::new(0.90); + let sig = ctrl.read_signals(&isv); + let out = ctrl.update(sig); + assert!((out.gamma - 0.905).abs() < 1e-5); + assert!(out.adjusted); + } + + #[test] + fn gamma_unchanged_in_dead_zone() { + let mut buf = make_isv_with_util(0.5); + let isv = IsvBus::new(&mut buf); + let mut ctrl = GammaController::new(0.93); + let sig = ctrl.read_signals(&isv); + let out = ctrl.update(sig); + assert!((out.gamma - 0.93).abs() < 1e-6); + assert!(!out.adjusted); + } + + #[test] + fn gamma_clamped_at_0_95() { + let mut ctrl = GammaController::new(0.948); + ctrl.adaptive_gamma = 0.948; + let sig = GammaSignal { atom_util_ema: 0.9 }; + let out = ctrl.update(sig); + assert!(out.gamma <= 0.95 + 1e-6); + } + + #[test] + fn gamma_fire_rate_tracks_adjustments() { + let mut ctrl = GammaController::new(0.92); + // 3 epochs high util (fires), 2 epochs dead zone (no fire) + for _ in 0..3 { + ctrl.update(GammaSignal { atom_util_ema: 0.9 }); + } + for _ in 0..2 { + ctrl.update(GammaSignal { atom_util_ema: 0.5 }); + } + // 3/5 = 0.6 — each high-util epoch adjusts gamma + let rate = ctrl.fire_rate().fire_rate(); + assert!((rate - 0.6).abs() < 0.01, "expected ~0.6, got {rate}"); + } + + #[test] + fn gamma_diagnose_fields() { + let mut ctrl = GammaController::new(0.91); + ctrl.update(GammaSignal { atom_util_ema: 0.75 }); + let diag = ctrl.diagnose(); + assert!(diag.fields.contains_key("gamma")); + assert!(diag.fields.contains_key("atom_util_ema")); + } + + #[test] + fn gamma_name() { + assert_eq!(GammaController::new(0.95).name(), "gamma"); + } +} diff --git a/crates/ml/src/trainers/dqn/controllers/kelly_cap_controller.rs b/crates/ml/src/trainers/dqn/controllers/kelly_cap_controller.rs new file mode 100644 index 000000000..1d25fbfbe --- /dev/null +++ b/crates/ml/src/trainers/dqn/controllers/kelly_cap_controller.rs @@ -0,0 +1,223 @@ +//! Kelly cap controller — Plan 1 Task 11, §4.C.6. +//! +//! Encapsulates the half-Kelly position-cap multiplier computation from +//! per-episode trade-outcome stats (win/loss counters aggregated from +//! `ps[14..18]` in `experience_kernels.cu`). +//! +//! The controller consumes `TradeStats` (aggregated by +//! `GpuExperienceCollector::collect_trade_stats`) and emits a Kelly cap +//! multiplier in [0, 1]. This multiplier is available for downstream +//! consumers; in the current training loop it is logged in HEALTH_DIAG as +//! `kelly_f`. +//! +//! Kelly win/loss stats do not live in the ISV bus — they are aggregated +//! per-epoch from GPU trade counters. `read_signals` returns a zeroed +//! `KellySignal`; the training loop constructs the signal directly from +//! `TradeStats` and calls `controller.update(signal)` rather than using the +//! ISV read path. This is intentional: ISV carries dynamics signals; trade +//! P&L stats are a separate aggregation channel. +//! +//! "Old ad-hoc code" removed in this commit: inline `kelly_f_mean` / +//! `avg_win_ratio` computation block in +//! `train_with_data_full_loop_slices` (~20 lines, labelled "Task 0.3 — +//! var_scale_mean / kelly_f_mean / avg_win_ratio"). After migration: single +//! `controller.update(signal)` call. + +use crate::trainers::dqn::adaptive_controller::{ + AdaptiveController, DiagSnapshot, FireRateStats, IsvBus, +}; + +/// Aggregated per-epoch Kelly trade statistics. +/// +/// Mirrors the four slots written by `experience_env_step` to `ps[14..18]` +/// and later reduced into `TradeStats` by `GpuExperienceCollector`. +#[derive(Debug, Clone, Copy, Default)] +pub(crate) struct KellySignal { + pub winning_trades: u64, + pub losing_trades: u64, + /// Cumulative profit from winning trades (absolute value, positive). + pub sum_wins: f64, + /// Cumulative loss magnitude from losing trades (absolute value, positive). + pub sum_losses: f64, +} + +impl KellySignal { + /// Construct from raw trade counters (e.g. from `TradeStats`). + pub(crate) fn from_counts(winning: u64, losing: u64, sum_wins: f64, sum_losses: f64) -> Self { + Self { winning_trades: winning, losing_trades: losing, sum_wins, sum_losses } + } +} + +/// Kelly cap controller output. +#[derive(Debug, Clone, Copy)] +pub(crate) struct KellyCapControl { + /// Half-Kelly cap multiplier in [0, 0.5]. 0.0 = cold-start / no data. + pub kelly_half: f32, + /// Win/loss ratio `avg_win / avg_loss` (diagnostic). + pub avg_win_ratio: f32, + /// Whether the cap changed this epoch (fire signal). + pub changed: bool, +} + +/// Adaptive Kelly cap controller. +/// +/// Computes the half-Kelly fraction each epoch from aggregated trade outcomes +/// and exposes it via the `AdaptiveController` trait. +#[derive(Debug)] +pub(crate) struct KellyCapController { + fire_rate: FireRateStats, + last_kelly_half: f32, + last_avg_win_ratio: f32, +} + +impl KellyCapController { + pub(crate) fn new() -> Self { + Self { + fire_rate: FireRateStats::default(), + last_kelly_half: 0.0, + last_avg_win_ratio: 0.0, + } + } + + /// Compute Kelly fraction from raw trade counters. + /// + /// Returns `(kelly_half, avg_win_ratio)` where: + /// `b = avg_win / max(avg_loss, 0.001)` + /// `kelly_raw = (b × win_rate − (1 − win_rate)) / max(b, 0.001)` + /// `kelly_half = 0.5 × clamp(kelly_raw, 0, 1)` + fn compute(signal: &KellySignal) -> (f32, f32) { + let w = signal.winning_trades as f64; + let l = signal.losing_trades as f64; + let total = w + l; + if total <= 0.0 || w <= 0.0 || l <= 0.0 { + return (0.0, 0.0); + } + let avg_win = signal.sum_wins / w; + let avg_loss = signal.sum_losses / l; + let win_rate = w / total; + let b = avg_win / avg_loss.max(0.001); + let kelly_raw = (b * win_rate - (1.0 - win_rate)) / b.max(0.001); + let kelly_clamped = kelly_raw.clamp(0.0, 1.0); + let kelly_half = 0.5 * kelly_clamped; + let ratio = avg_win / avg_loss.max(0.001); + (kelly_half as f32, ratio as f32) + } +} + +impl Default for KellyCapController { + fn default() -> Self { Self::new() } +} + +impl AdaptiveController for KellyCapController { + type Signal = KellySignal; + type Control = KellyCapControl; + + fn read_signals(&self, _isv: &IsvBus<'_>) -> KellySignal { + // Kelly trade stats are aggregated from GPU ps[14..18] counters, not + // from the ISV bus. The training loop constructs the signal from + // `TradeStats` directly and calls `update()` without using this path. + KellySignal::default() + } + + fn update(&mut self, signal: KellySignal) -> KellyCapControl { + let (kelly_half, avg_win_ratio) = Self::compute(&signal); + let changed = (kelly_half - self.last_kelly_half).abs() > 1e-4; + self.fire_rate.record_fire(changed); + self.last_kelly_half = kelly_half; + self.last_avg_win_ratio = avg_win_ratio; + KellyCapControl { kelly_half, avg_win_ratio, changed } + } + + fn write_output(&self, ctrl: &KellyCapControl, _isv: &mut IsvBus<'_>) { + // kelly_half is logged in HEALTH_DIAG (kelly_f field); no ISV slot + // exists for this value. Future consumers can read ctrl.kelly_half + // directly. Suppress the unused-variable warning. + let _ = ctrl; + } + + fn fire_rate(&self) -> &FireRateStats { + &self.fire_rate + } + + fn diagnose(&self) -> DiagSnapshot { + DiagSnapshot::new() + .with("kelly_half", self.last_kelly_half) + .with("avg_win_ratio", self.last_avg_win_ratio) + } + + fn name(&self) -> &'static str { + "kelly_cap" + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn kelly_cap_zero_with_no_trades() { + let mut ctrl = KellyCapController::new(); + let out = ctrl.update(KellySignal::default()); + assert_eq!(out.kelly_half, 0.0); + assert_eq!(out.avg_win_ratio, 0.0); + } + + #[test] + fn kelly_cap_half_kelly_formula() { + // 60% win rate, avg win = 1.5, avg loss = 1.0 + // b = 1.5, kelly_raw = (1.5*0.6 - 0.4)/1.5 = (0.9-0.4)/1.5 = 0.333 + // kelly_half = 0.5 * 0.333 = 0.166... + let mut ctrl = KellyCapController::new(); + let sig = KellySignal::from_counts(6, 4, 9.0, 4.0); // 6 wins *1.5, 4 losses *1.0 + let out = ctrl.update(sig); + let expected = 0.5 * (1.5_f64 * 0.6 - 0.4) as f32 / 1.5; + assert!((out.kelly_half - expected).abs() < 1e-5, "got {}, expected {}", out.kelly_half, expected); + } + + #[test] + fn kelly_cap_clamped_at_zero() { + // Losing strategy: kelly would be negative + let mut ctrl = KellyCapController::new(); + let sig = KellySignal::from_counts(3, 7, 3.0, 14.0); // 30% win, avg_win=1, avg_loss=2 + let out = ctrl.update(sig); + assert!(out.kelly_half >= 0.0, "kelly_half must be non-negative, got {}", out.kelly_half); + } + + #[test] + fn kelly_cap_fire_rate_on_change() { + let mut ctrl = KellyCapController::new(); + // First update — cold start, kelly=0 → no change from 0.0 + ctrl.update(KellySignal::default()); + // Second update: 70% win rate, avg_win=2.0, avg_loss=1.0 + // b=2, kelly_raw=(2*0.7-0.3)/2=0.65, kelly_half=0.325 > 0 → fires + let sig = KellySignal::from_counts(7, 3, 14.0, 3.0); + ctrl.update(sig); + let rate = ctrl.fire_rate().fire_rate(); + // At least one epoch fired (when kelly went from 0 to non-zero) + assert!(rate > 0.0, "expected fire on first trade epoch, got rate={rate}"); + } + + #[test] + fn kelly_cap_diagnose_fields() { + let mut ctrl = KellyCapController::new(); + ctrl.update(KellySignal::from_counts(6, 4, 9.0, 4.0)); + let diag = ctrl.diagnose(); + assert!(diag.fields.contains_key("kelly_half")); + assert!(diag.fields.contains_key("avg_win_ratio")); + } + + #[test] + fn kelly_cap_read_signals_returns_default() { + let ctrl = KellyCapController::new(); + let mut buf = vec![0.0_f32; 37]; + let isv = IsvBus::new(&mut buf); + let sig = ctrl.read_signals(&isv); + assert_eq!(sig.winning_trades, 0); + assert_eq!(sig.sum_wins, 0.0); + } + + #[test] + fn kelly_cap_name() { + assert_eq!(KellyCapController::new().name(), "kelly_cap"); + } +} diff --git a/crates/ml/src/trainers/dqn/controllers/mod.rs b/crates/ml/src/trainers/dqn/controllers/mod.rs new file mode 100644 index 000000000..086804cff --- /dev/null +++ b/crates/ml/src/trainers/dqn/controllers/mod.rs @@ -0,0 +1,14 @@ +//! Adaptive controller implementations — Plan 1 Tasks 9-12. +//! +//! Each submodule implements `AdaptiveController` for one epoch-boundary +//! controller in the DQN training pipeline. + +pub(crate) mod atoms_controller; +pub(crate) mod gamma_controller; +pub(crate) mod kelly_cap_controller; +pub(crate) mod cql_alpha_controller; + +pub(crate) use atoms_controller::{AtomsController, AtomSignal}; +pub(crate) use gamma_controller::{GammaController, GammaSignal}; +pub(crate) use kelly_cap_controller::{KellyCapController, KellySignal}; +pub(crate) use cql_alpha_controller::{CqlAlphaController, CqlAlphaSignal}; diff --git a/crates/ml/src/trainers/dqn/fused_training.rs b/crates/ml/src/trainers/dqn/fused_training.rs index 4d4a33e21..6b72e7bb2 100644 --- a/crates/ml/src/trainers/dqn/fused_training.rs +++ b/crates/ml/src/trainers/dqn/fused_training.rs @@ -2517,6 +2517,14 @@ impl FusedTrainingCtx { self.trainer.read_atom_utilization() } + /// Plan 1 Task 9: Read per-branch ISV v-range slots for the AtomsController signal. + /// + /// Returns `[(center, half); 4]` for branches [dir, mag, ord, urg] from the + /// host-pinned ISV buffer. Uses named constants from `gpu_dqn_trainer`. + pub(crate) fn read_isv_v_ranges(&self) -> [(f32, f32); 4] { + self.trainer.read_isv_v_ranges() + } + /// F2: Read aggregated Adam m-state sample for vector cosine grad_consistency. pub(crate) fn read_adam_m_flat_sample(&self, max_per_component: usize) -> anyhow::Result> { self.trainer.read_adam_m_flat_sample(max_per_component) diff --git a/crates/ml/src/trainers/dqn/mod.rs b/crates/ml/src/trainers/dqn/mod.rs index 0194219e3..141847edf 100644 --- a/crates/ml/src/trainers/dqn/mod.rs +++ b/crates/ml/src/trainers/dqn/mod.rs @@ -18,8 +18,9 @@ //! - `statistics` - Feature normalization and Q-value monitoring //! - `trainer` - Main DQNTrainer implementation (core training loop) -pub mod adaptive_controller; +pub(crate) mod adaptive_controller; pub(crate) mod adversarial_self_play; +pub mod state_reset_registry; mod config; mod data_loading; mod early_stopping; @@ -33,11 +34,9 @@ mod risk; mod statistics; mod trainer; mod smoke_tests; -pub mod state_reset_registry; -pub use state_reset_registry::{ResetCategory, RegistryEntry, StateResetRegistry}; +pub(crate) mod controllers; // Re-export all public items for backward compatibility -pub use adaptive_controller::{AdaptiveController, FireRateStats, DiagSnapshot, IsvBus}; pub use config::{DQNAgentType, DQNHyperparameters}; pub use early_stopping::EarlyStopping; pub use lr_scheduler::{LRDecayType, LRScheduler}; @@ -45,6 +44,7 @@ pub use statistics::{FeatureStatistics, QValueStats}; pub use data_loading::{collect_dbn_files_filtered, collect_dbn_files_recursive, extract_ohlcv_bars_from_dbn}; pub use features::extract_features_from_bars; pub use trainer::DQNTrainer; +pub use state_reset_registry::{ResetCategory, RegistryEntry, StateResetRegistry}; // Re-export constants pub const EPISODE_LENGTH: usize = 200; diff --git a/crates/ml/src/trainers/dqn/trainer/constructor.rs b/crates/ml/src/trainers/dqn/trainer/constructor.rs index a19f5e0c7..296fd82cd 100644 --- a/crates/ml/src/trainers/dqn/trainer/constructor.rs +++ b/crates/ml/src/trainers/dqn/trainer/constructor.rs @@ -537,6 +537,9 @@ impl DQNTrainer { let sab_enabled = true; let sab_warmup = hyperparams.adversarial_warmup_epochs; let sab_interval = hyperparams.adversarial_checkpoint_interval; + // Capture before hyperparams is moved into the struct. + let init_gamma = hyperparams.gamma as f32; + let init_cql_alpha = hyperparams.cql_alpha; Ok(Self { agent: Arc::new(RwLock::new(agent)), @@ -768,6 +771,12 @@ impl DQNTrainer { meta_q: crate::cuda_pipeline::meta_q_network::MetaQNetwork::new(), pending_meta_q_samples: std::collections::VecDeque::with_capacity(32), health_history: std::collections::VecDeque::with_capacity(2 * crate::cuda_pipeline::meta_q_network::META_Q_LOOKAHEAD), + // Plan 1 adaptive controllers (Tasks 9-12). + // GammaController initialised from hyperparams.gamma (default 0.99). + atoms_controller: crate::trainers::dqn::controllers::AtomsController::new(), + gamma_controller: crate::trainers::dqn::controllers::GammaController::new(init_gamma), + kelly_cap_controller: crate::trainers::dqn::controllers::KellyCapController::new(), + cql_alpha_controller: crate::trainers::dqn::controllers::CqlAlphaController::new(init_cql_alpha), }) } diff --git a/crates/ml/src/trainers/dqn/trainer/mod.rs b/crates/ml/src/trainers/dqn/trainer/mod.rs index 4ffa766d4..a0248a958 100644 --- a/crates/ml/src/trainers/dqn/trainer/mod.rs +++ b/crates/ml/src/trainers/dqn/trainer/mod.rs @@ -658,6 +658,20 @@ pub struct DQNTrainer { /// D8/N8: Rolling history of (epoch, health) for meta-Q label resolution. /// Capped at 2×META_Q_LOOKAHEAD. pub(crate) health_history: std::collections::VecDeque<(u32, f32)>, + + // ── Plan 1 adaptive controllers (Tasks 9-12) ────────────────────────────── + /// Task 9: C51 atom-position controller — reads ISV v-range slots, + /// gates the per-epoch `recompute_atom_positions` GPU kernel call. + pub(crate) atoms_controller: crate::trainers::dqn::controllers::AtomsController, + /// Task 10: Adaptive gamma controller — reads atom-util EMA from ISV[6], + /// computes the per-epoch `adaptive_gamma` update. + pub(crate) gamma_controller: crate::trainers::dqn::controllers::GammaController, + /// Task 11: Kelly cap controller — computes the half-Kelly position-cap + /// multiplier from per-epoch trade-outcome stats. + pub(crate) kelly_cap_controller: crate::trainers::dqn::controllers::KellyCapController, + /// Task 12: CQL alpha controller — scaffolding for seed-phase coupling + /// (Plan 3); currently returns the static config alpha unchanged. + pub(crate) cql_alpha_controller: crate::trainers::dqn::controllers::CqlAlphaController, } impl std::fmt::Debug for DQNTrainer { diff --git a/crates/ml/src/trainers/dqn/trainer/training_loop.rs b/crates/ml/src/trainers/dqn/trainer/training_loop.rs index 674b2db2d..e7283f195 100644 --- a/crates/ml/src/trainers/dqn/trainer/training_loop.rs +++ b/crates/ml/src/trainers/dqn/trainer/training_loop.rs @@ -304,10 +304,27 @@ impl DQNTrainer { } } - // Recompute adaptive atom positions from learned spacing_raw params (slow-moving, once per epoch). - if let Some(ref fused) = self.fused_ctx { - if let Err(e) = fused.recompute_atom_positions() { - tracing::warn!(epoch, "recompute_atom_positions failed: {e}"); + // Recompute adaptive atom positions — routed through AtomsController (Task 9). + // The controller reads ISV v-range slots for diagnostics and returns + // AtomControl::should_recompute=true (unconditional). The GPU kernel call + // remains here as the actuator. + { + use crate::trainers::dqn::adaptive_controller::AdaptiveController; + let atom_ctrl = if let Some(ref fused) = self.fused_ctx { + // ISV pinned buffer is only accessible via GpuDqnTrainer; build a + // host-side copy of the v-range slots for the controller signal. + let v_ranges = fused.read_isv_v_ranges(); + let sig = crate::trainers::dqn::controllers::AtomSignal { v_ranges }; + Some(self.atoms_controller.update(sig)) + } else { + None + }; + if atom_ctrl.map(|c| c.should_recompute).unwrap_or(false) { + if let Some(ref fused) = self.fused_ctx { + if let Err(e) = fused.recompute_atom_positions() { + tracing::warn!(epoch, "recompute_atom_positions failed: {e}"); + } + } } } @@ -662,38 +679,26 @@ impl DQNTrainer { // // That second arm creates a positive-feedback loop: atom // collapse (low util) → lower γ → `γ × delta_z` shrinks → - // TD targets concentrate on fewer bins → deeper collapse. - // Observed on train-bv2n5: atom util stuck at 1% for 16 - // epochs while this controller quietly nudged γ toward its - // 0.90 floor, reinforcing the symptom it was meant to - // mitigate. Per research-agent audit (2026-04-24). - // - // New direction: when atoms are collapsed, RAISE γ so - // `γ × atom_support` spans a larger fraction of the grid, - // spreading TD targets across more bins and breaking the - // loop. Slower step (0.005) than the healthy-branch to - // avoid overshoot. - // - // Clamps retained: [0.90, 0.95] at this layer; the fused - // trainer re-clamps to [0.90, 0.995] after its regime - // adjustment. + // G4 adaptive gamma — routed through GammaController (Task 10). + // The controller reads atom-util EMA from ISV[6], updates adaptive_gamma, + // and returns GammaControl. The fused.apply_adaptive_gamma actuator call + // remains here, consuming ctrl.gamma. { + use crate::trainers::dqn::adaptive_controller::AdaptiveController; let util = self.fused_ctx.as_ref() .map(|f| f.utilization_ema()) .unwrap_or(0.5); - if util > 0.6 { - self.adaptive_gamma = (self.adaptive_gamma + 0.005).min(0.95); - } else if util < 0.4 { - // Direction flipped (atom-collapse fix): RAISE γ to - // widen the TD-target spread across the atom grid. - self.adaptive_gamma = (self.adaptive_gamma + 0.005).min(0.95); - } + // Sync GammaController's adaptive_gamma to the trainer field before update + // so the controller starts from the current value. + self.gamma_controller.adaptive_gamma = self.adaptive_gamma; + let sig = crate::trainers::dqn::controllers::GammaSignal { atom_util_ema: util }; + let ctrl = self.gamma_controller.update(sig); + self.adaptive_gamma = ctrl.gamma; // C4/P4: Wire adaptive gamma into GPU trainer with temporal-coupled adjustment. // apply_adaptive_gamma reads health+regime from ISV, adjusts, clamps [0.9, 0.995], // and caches last_gamma_eff for HEALTH_DIAG. if let Some(ref mut fused) = self.fused_ctx { - let gamma_scheduled = self.adaptive_gamma as f32; - fused.apply_adaptive_gamma(gamma_scheduled); + fused.apply_adaptive_gamma(ctrl.gamma); } info!("G4 gamma={:.3} util={:.2}", self.adaptive_gamma, util); } @@ -780,10 +785,11 @@ impl DQNTrainer { // Apply E2: adaptive epsilon self.hyperparams.epsilon_end = result.epsilon; - // Apply E3: dynamic gamma (EMA blend) + // Apply E3: dynamic gamma (EMA blend) — keep gamma_controller in sync. if let Some(eval_gamma) = result.gamma { let blended = 0.9 * self.adaptive_gamma + 0.1 * eval_gamma; self.adaptive_gamma = blended.clamp(0.85, 0.98); + self.gamma_controller.adaptive_gamma = self.adaptive_gamma as f32; // C4/P4: apply temporal-coupled adjustment on top of EMA blend. if let Some(ref mut fused) = self.fused_ctx { let gamma_scheduled = self.adaptive_gamma as f32; @@ -2023,6 +2029,24 @@ impl DQNTrainer { self.last_cql_alpha_eff = Some(fused.last_cql_alpha_eff()); } + // CQL alpha — routed through CqlAlphaController (Task 12). + // Currently returns the static config alpha; Plan 3 adds seed-phase coupling. + // The controller tracks fire-rate so that future dynamic changes are observable. + { + use crate::trainers::dqn::adaptive_controller::AdaptiveController; + // Keep the controller's base_alpha in sync with the config field. + self.cql_alpha_controller.set_base_alpha(self.hyperparams.cql_alpha); + let sig = crate::trainers::dqn::controllers::CqlAlphaSignal { + training_step: self.controller_total_epochs as u64, + buffer_fill_fraction: 0.0, // not yet wired + health: health_value, + }; + let _ctrl = self.cql_alpha_controller.update(sig); + // _ctrl.alpha == base config alpha (static). The per-step CQL kernel + // derives its effective alpha from ISV[12] on-device; the host + // scaffolding here does not override it. + } + // B2/G3: propagate last tau_eff for logging. if let Some(ref fused) = self.fused_ctx { self.last_tau_eff = Some(fused.last_tau_eff()); @@ -2431,25 +2455,22 @@ impl DQNTrainer { }, None => 0.0, }; - let (kelly_f_mean, avg_win_ratio) = if let Some(ts) = self.trade_stats_history.last() { - let w = ts.winning_trades as f64; - let l = ts.losing_trades as f64; - let total = w + l; - if total > 0.0 && w > 0.0 && l > 0.0 { - let avg_win = ts.sum_wins / w; - let avg_loss = ts.sum_losses / l; - let win_rate = w / total; - let b = avg_win / avg_loss.max(0.001); - let kelly_raw = (b * win_rate - (1.0 - win_rate)) / b.max(0.001); - let kelly_clamped = kelly_raw.clamp(0.0, 1.0); - let kelly_half = 0.5 * kelly_clamped; - let ratio = avg_win / avg_loss.max(0.001); - (kelly_half as f32, ratio as f32) + // Kelly cap — routed through KellyCapController (Task 11). + // Old inline kelly_f_mean block removed; controller owns the formula. + let (kelly_f_mean, avg_win_ratio) = { + use crate::trainers::dqn::adaptive_controller::AdaptiveController; + let sig = if let Some(ts) = self.trade_stats_history.last() { + crate::trainers::dqn::controllers::KellySignal::from_counts( + ts.winning_trades as u64, + ts.losing_trades as u64, + ts.sum_wins, + ts.sum_losses, + ) } else { - (0.0_f32, 0.0_f32) - } - } else { - (0.0_f32, 0.0_f32) + crate::trainers::dqn::controllers::KellySignal::default() + }; + let ctrl = self.kelly_cap_controller.update(sig); + (ctrl.kelly_half, ctrl.avg_win_ratio) }; // Download GPU monitoring summary and populate the local per-epoch diff --git a/docs/dqn-wire-up-audit.md b/docs/dqn-wire-up-audit.md index 482f88e6e..2c365ac4a 100644 --- a/docs/dqn-wire-up-audit.md +++ b/docs/dqn-wire-up-audit.md @@ -1,250 +1,13 @@ -# DQN v2 Wire-Up Audit +# DQN Wire-Up Audit -**Status:** Populated during Plan 1 Task 6 (A.5 orphan audit). Updated on every commit per Invariant 7. +Invariant 7 compliance log. Each row records one component-adding commit. +Format: `| commit | component | files touched | notes |` -**Legend:** -- `Wired` — consumed by production training + val path. -- `Partial` — consumed on one side only (training OR val, or forward OR backward). -- `Orphan` — built but no production consumer in the DQN path. -- `Ghost` — consumer path is stubbed or only loads the kernel without launching it. -- `OUT-of-DQN-scope` — has production consumers outside the DQN path (supervised, PPO, etc.); correct-as-is per Part E. +## Adaptive Controller Migrations (Plan 1 Tasks 9-12) -**DQN-path definition:** `trainers/dqn/` (training loop + fused CUDA ops) + `cuda_pipeline/` (GPU kernels wired into those trainers) + `validation/` harness when evaluating DQN policy. - -## Task 2 / Task 3 Seed Rows (preserved) - -| Module / kernel | Consumer path | Classification | Notes | Action | -|---|---|---|---|---| -| `trainers/dqn/state_reset_registry.rs` | `training_loop.rs::reset_named_state` via `fold_reset_entries()` | Wired | A.1 primary implementation (Plan 1 Task 3) | — | -| `training_loop.rs::reset_named_state` | consumer of `StateResetRegistry::fold_reset_entries` | Wired | A.1 Task 3 dispatch | — | - -## DQN Core Training Modules - -| Module / kernel | Consumer path | Classification | Notes | Action | -|---|---|---|---|---| -| `trainers/dqn/mod.rs` | `train_baseline_rl.rs`, `ml_training_service`, `hyperopt/adapters/dqn.rs` | Wired | Entry point for DQN trainer | — | -| `trainers/dqn/config.rs` (`DQNHyperparameters`, `DQNAgentType`) | `trainer/constructor.rs`, `fused_training.rs`, smoke tests, hyperopt adapter | Wired | Core config struct | — | -| `trainers/dqn/data_loading.rs` | re-exported via `dqn::mod`; consumed by `train_baseline_rl.rs` | Wired | DBN + fxcache loading | — | -| `trainers/dqn/early_stopping.rs` (`EarlyStopping`) | `trainer/mod.rs`, `trainer/constructor.rs` | Wired | Patience-based stopping | — | -| `trainers/dqn/features.rs` | `trainer/constructor.rs` via `extract_features_from_bars` | Wired | Feature extraction for DQN | — | -| `trainers/dqn/financials.rs` | `trainer/metrics.rs` via `compute_epoch_financials` | Wired | Sharpe / drawdown per epoch | — | -| `trainers/dqn/fused_training.rs` (`FusedTrainingCtx`) | `trainer/training_loop.rs`, `trainer/mod.rs` | Wired | Core DQN fused-CUDA context | — | -| `trainers/dqn/lr_scheduler.rs` (`LRScheduler`) | `trainer/mod.rs::lr_scheduler`, `trainer/constructor.rs` | Wired | Learning-rate schedule | — | -| `trainers/dqn/monitoring.rs` (`MonitoringSummary`) | `trainer/training_loop.rs`, `trainer/metrics.rs` | Wired | Per-epoch action / Q monitoring | — | -| `trainers/dqn/risk.rs` | `trainer/constructor.rs` via `DrawdownMonitor`, `HybridPositionLimiter` | Wired | Risk controllers in training | — | -| `trainers/dqn/statistics.rs` (`FeatureStatistics`, `QValueStats`) | `trainer/mod.rs`, `trainer/metrics.rs` | Wired | Feature norm + Q stats | — | -| `trainers/dqn/adversarial_self_play.rs` (`AdversarialSaboteur`) | `trainer/constructor.rs`, `trainer/mod.rs` | Wired | Adversarial perturbation during training | — | -| `trainers/dqn/expert_demos.rs` (`ExpertDemoGenerator`) | `trainer/training_loop.rs` | Wired | Expert demo ratio injection | — | -| `trainers/dqn/trainer/mod.rs` (`DQNTrainer`) | `train_baseline_rl.rs`, service layer | Wired | Top-level trainer struct | — | -| `trainers/dqn/trainer/constructor.rs` | internal to `DQNTrainer` | Wired | Builder / init | — | -| `trainers/dqn/trainer/action.rs` | internal to `DQNTrainer` | Wired | Action selection helpers | — | -| `trainers/dqn/trainer/enrichment.rs` | internal to `DQNTrainer` | Wired | State enrichment | — | -| `trainers/dqn/trainer/metrics.rs` | `training_loop.rs` | Wired | Epoch-boundary metric compilation + backtest eval | — | -| `trainers/dqn/trainer/state.rs` | `trainer/mod.rs` | Wired | Mutable training state | — | -| `trainers/dqn/trainer/training_loop.rs` | called from `DQNTrainer::train()` | Wired | Main epoch loop | — | -| `trainers/dqn/adaptive_controller.rs` | Trait + harness (`FireRateStats`, `DiagSnapshot`, `IsvBus<'a>`); consumers added in Plan 1 Tasks 9-17 (atoms, gamma, Kelly, cql_alpha, tau, epsilon, conviction_floor, plan_threshold, grad_balancer migrations) | Wired (consumers added in same plan) | C.6 trait foundation | — | - -## CUDA Pipeline — Rust Wrappers - -| Module / kernel | Consumer path | Classification | Notes | Action | -|---|---|---|---|---| -| `cuda_pipeline/mod.rs` (DqnGpuData, upload_slices) | `trainer/constructor.rs`, `fused_training.rs` | Wired | Data upload to GPU | — | -| `cuda_pipeline/gpu_dqn_trainer.rs` (`GpuDqnTrainer`) | `fused_training.rs`, `trainer/mod.rs` | Wired | Core GPU-side DQN weight / forward / grad ops | — | -| `cuda_pipeline/fused_training.rs` wrapper (`FusedTrainingCtx`) | `trainer/training_loop.rs` | Wired | Wires all GPU ops into epoch loop | — | -| `cuda_pipeline/batched_forward.rs` | `gpu_dqn_trainer.rs` — forward graph node | Wired | Batched SGEMM forward pass | — | -| `cuda_pipeline/batched_backward.rs` | `gpu_dqn_trainer.rs` — backward graph node | Wired | Batched SGEMM backward pass (KAN gate included) | — | -| `cuda_pipeline/shared_cublas_handle.rs` | `fused_training.rs`, `gpu_iqn_head.rs`, `gpu_iql_trainer.rs`, `gpu_attention.rs`, `gpu_curiosity_trainer.rs` (10 consumers) | Wired | Shared cuBLAS/cuBLASLt handle | — | -| `cuda_pipeline/cublas_algo_deterministic.rs` | `gpu_dqn_trainer.rs`, `gpu_curiosity_trainer.rs`, `gpu_iqn_head.rs` (7 consumers) | Wired | Deterministic cuBLASLt algo selection | — | -| `cuda_pipeline/gpu_weights.rs` | `trainer/mod.rs`, `gpu_dqn_trainer.rs`, `hyperopt/adapters/dqn.rs` (11 consumers) | Wired | Weight tensor layout constants | — | -| `cuda_pipeline/gpu_action_selector.rs` (`GpuActionSelector`) | `trainer/mod.rs`, `fused_training.rs`, `gpu_backtest_evaluator.rs` | Wired | Epsilon-greedy + routed action selection | — | -| `cuda_pipeline/gpu_monitoring.rs` (`GpuMonitor`) | `fused_training.rs`, `trainer/metrics.rs`, `trainer/training_loop.rs` | Wired | GPU-side monitoring_reduce launch | — | -| `cuda_pipeline/gpu_training_guard.rs` | `gpu_dqn_trainer.rs`, `trainer/training_loop.rs` | Wired | NaN / gradient anomaly guard | — | -| `cuda_pipeline/gpu_backtest_evaluator.rs` (`GpuBacktestEvaluator`) | `trainer/metrics.rs` (eval path) | Wired | Per-epoch GPU backtest evaluation | — | -| `cuda_pipeline/gpu_experience_collector.rs` | `fused_training.rs` via `TradeStats`, `financials.rs` | Wired | Per-trade stats from experience buffer | — | -| `cuda_pipeline/gpu_curiosity_trainer.rs` | `fused_training.rs` | Wired | Curiosity-driven intrinsic reward | — | -| `cuda_pipeline/gpu_walk_forward.rs` (`GpuWalkForwardData`) | `trainer/mod.rs` (lazy-init field) | Wired | GPU walk-forward data structure | — | -| `cuda_pipeline/gpu_her.rs` (`GpuHer`) | `fused_training.rs` | Wired | Hindsight Experience Replay | — | -| `cuda_pipeline/gpu_iql_trainer.rs` (`GpuIqlTrainer`) | `fused_training.rs` (dual IQL instances) | Wired | IQL value network training | — | -| `cuda_pipeline/gpu_iqn_head.rs` (`GpuIqnHead`) | `fused_training.rs` | Wired | IQN distributional head | — | -| `cuda_pipeline/gpu_attention.rs` (`GpuAttention`) | `fused_training.rs` | Wired | Self-attention layer in DQN trunk | — | -| `cuda_pipeline/decision_transformer.rs` (`DecisionTransformer`) | `trainer/training_loop.rs` | Wired | Decision Transformer pre-training step | — | -| `cuda_pipeline/learning_health.rs` (`LearningHealth`) | `fused_training.rs`, `trainer/training_loop.rs`, `trainer/constructor.rs`, `trainer/metrics.rs`, `meta_q_network.rs` (11 consumers) | Wired | Health-band tracking for adaptive LR / early stopping | — | -| `cuda_pipeline/q_snapshot.rs` | `cuda_pipeline/mod.rs`, `gpu_dqn_trainer.rs` | Wired | Q-value snapshot for double-DQN target | — | -| `cuda_pipeline/meta_q_network.rs` (`MetaQNetwork`) | `trainer/constructor.rs`, `trainer/mod.rs`, `trainer/training_loop.rs` | Wired | Meta-learning Q-network | — | -| `cuda_pipeline/q_value_provider.rs` (`QValueProvider` trait) | `fused_training.rs` impl, `trainer/metrics.rs`, `hyperopt/adapters/dqn.rs` | Wired | Trait for on-device Q-value computation in val path | — | -| `cuda_pipeline/signal_adapter.rs` | `hyperopt/adapters/dqn.rs`, `hyperopt/adapters/{ppo,mamba2,tft,kan,liquid,tlob,diffusion,tggn,xlstm}.rs` | Wired | `backtest_fitness` scoring used by all hyperopt adapters | — | -| `cuda_pipeline/multi_gpu.rs` (`MultiGpuConfig`) | `trainer/constructor.rs`, `trainer/mod.rs` | Wired | Multi-GPU detection and layout | — | -| `cuda_pipeline/gpu_ppo_collector.rs` (`GpuPpoExperienceCollector`) | `trainers/ppo.rs` | OUT-of-DQN-scope | PPO-only consumer; correct-as-is | — | -| `cuda_pipeline/gpu_statistics.rs` (`GpuStatistics`) | declared `pub` in `cuda_pipeline/mod.rs`; no call site outside the file | Orphan | `GpuStatistics::compute` never called; `BatchStatistics` struct unused | Plan 2 D.2 audits + decides whether to wire into val path or delete | -| `cuda_pipeline/cublaslt_debug.rs` | `mod cublaslt_debug` (private) inside `cuda_pipeline/mod.rs`; test-only | OUT-of-DQN-scope | Private debug/test module; not a public feature | — | - -## CUDA Kernels (.cu files) - -| Module / kernel | Consumer path | Classification | Notes | Action | -|---|---|---|---|---| -| `epsilon_greedy_kernel.cu` (`epsilon_greedy_select`, `epsilon_greedy_routed`) | `gpu_action_selector.rs` | Wired | Action selection kernel | — | -| `monitoring_kernel.cu` (`monitoring_reduce`) | `gpu_monitoring.rs` | Wired | 12-bin action count + Q stats | — | -| `c51_loss_kernel.cu` | `gpu_dqn_trainer.rs` (C51 loss compute) | Wired | C51 distributional loss | — | -| `c51_grad_kernel.cu` | `gpu_dqn_trainer.rs` (C51 backward) | Wired | C51 gradient backward | — | -| `mse_loss_kernel.cu` | `gpu_dqn_trainer.rs` (MSE warmup loss) | Wired | MSE loss for warmup phase | — | -| `mse_grad_kernel.cu` | `gpu_dqn_trainer.rs` (MSE backward) | Wired | MSE gradient backward | — | -| `iqn_dual_head_kernel.cu` | `gpu_iqn_head.rs` | Wired | IQN dual-head quantile computation | — | -| `iqn_cvar_kernel.cu` | `gpu_iqn_head.rs` | Wired | IQN CVaR risk measure | — | -| `iql_value_kernel.cu` | `gpu_iql_trainer.rs` | Wired | IQL value network kernel | — | -| `cql_grad_kernel.cu` | `gpu_dqn_trainer.rs` (CQL backward) | Wired | Conservative Q-learning gradient | — | -| `experience_kernels.cu` | `gpu_backtest_evaluator.rs` (`BACKTEST_DQN_CUBIN`), `gpu_experience_collector.rs` | Wired | Experience buffer ops + DQN backtest forward | — | -| `reward_shaping_kernel.cu` | `gpu_dqn_trainer.rs` (reward shaping step) | Wired | Per-sample reward shaping | — | -| `nstep_kernel.cu` | `gpu_dqn_trainer.rs` (n-step return) | Wired | N-step Bellman target | — | -| `backward_kernels.cu` | `gpu_dqn_trainer.rs` (weight backward) | Wired | Weight gradient accumulation | — | -| `bias_kernels.cu` | `gpu_dqn_trainer.rs` (bias grad) | Wired | Bias gradient kernels | — | -| `relu_mask_kernel.cu` | `gpu_dqn_trainer.rs` (activation backward) | Wired | ReLU activation mask | — | -| `ema_kernel.cu` | `gpu_dqn_trainer.rs` (target-net soft update) | Wired | EMA target-network update | — | -| `q_stats_kernel.cu` | `gpu_dqn_trainer.rs` (Q-value stats for ISV) | Wired | Per-branch Q-value statistics | — | -| `dqn_utility_kernels.cu` | `gpu_dqn_trainer.rs` (multiple utility ops) | Wired | Misc DQN GPU ops (advantage std, PopArt, etc.) | — | -| `graph_utility_kernels.cu` | `gpu_dqn_trainer.rs` (CUDA graph utility ops) | Wired | Graph capture helper kernels | — | -| `grad_decomp_kernel.cu` | `gpu_dqn_trainer.rs`, called via `fused_training.rs::grad_decomp_launch_c51` | Wired | C51 gradient decomposition (Task 2.0 diagnostic) | — | -| `branch_grad_balance_kernel.cu` | `gpu_dqn_trainer.rs`, called via `fused_training.rs::launch_branch_grad_balance` | Wired | Per-branch gradient L2-norm balancing | — | -| `mamba2_temporal_kernel.cu` (`mamba2_scan_projected_fwd`, `mamba2_scan_projected_bwd`, `isv_temporal_route`, etc.) | `gpu_dqn_trainer.rs::mamba2_forward` + `mamba2_backward`, called in fused training loop | Wired | Mamba2 selective SSM forward + backward (both paths implemented) | — | -| `attention_kernel.cu` | `gpu_attention.rs` | Wired | Scaled dot-product attention forward | — | -| `attention_backward_kernel.cu` | `gpu_attention.rs` | Wired | Attention backward pass | — | -| `training_guard_kernel.cu` (`training_guard_check_and_accumulate`) | `gpu_training_guard.rs` | Wired | NaN / guard check kernel | — | -| `backtest_env_kernel.cu` (`backtest_env_step`) | `gpu_backtest_evaluator.rs` (`ENV_CUBIN`) | Wired | Backtest environment step | — | -| `backtest_metrics_kernel.cu` | `gpu_backtest_evaluator.rs` (`METRICS_CUBIN`) | Wired | Backtest metrics reduction | — | -| `backtest_plan_kernel.cu` (`backtest_plan_diag_reduce`, `backtest_plan_state_isv`) | `gpu_backtest_evaluator.rs` (`BACKTEST_PLAN_CUBIN`) | Wired | Plan-ISV diagnostic reduction in val | — | -| `backtest_forward_ppo_kernel.cu` (`backtest_forward_ppo_kernel`) | `gpu_backtest_evaluator.rs::evaluate_ppo` (`PPO_FORWARD_CUBIN`) | OUT-of-DQN-scope | PPO-path backtest only; correct-as-is | — | -| `backtest_forward_supervised_kernel.cu` (`signal_to_action_kernel`) | `gpu_backtest_evaluator.rs::evaluate_supervised` (`SUPERVISED_SIGNAL_CUBIN`) | OUT-of-DQN-scope | Supervised-model backtest only; correct-as-is | — | -| `signal_adapter_kernel.cu` | `signal_adapter.rs` (loaded at construction) | Wired | Signal-to-action ISV bus adapter | — | -| `statistics_kernel.cu` (`batch_statistics`) | `gpu_statistics.rs` (loaded but `GpuStatistics` has no call sites outside the file) | Orphan | Kernel compiled, wrapper struct exists, but never instantiated by any consumer | Plan 2 D.2 — same resolution as `gpu_statistics.rs` | -| `ppo_experience_kernel.cu` | `gpu_ppo_collector.rs` | OUT-of-DQN-scope | PPO collector; correct-as-is | — | -| `her_episode_kernel.cu` | `gpu_her.rs` | Wired | HER episode boundary kernel | — | -| `her_relabel_kernel.cu` | `gpu_her.rs` | Wired | HER goal relabelling kernel | — | -| `curiosity_training_kernel.cu` | `gpu_curiosity_trainer.rs` | Wired | Curiosity intrinsic reward training | — | -| `curiosity_inference_kernel.cu` | `gpu_curiosity_trainer.rs` | Wired | Curiosity inference forward | — | -| `ensemble_kernels.cu` | `batched_forward.rs` / `batched_backward.rs` (ensemble ops in fused trainer) | Wired | Ensemble head combination | — | -| `dt_kernels.cu` | `decision_transformer.rs` (`DT_CUBIN`) | Wired | Decision Transformer GPU ops | — | -| `trade_stats_kernel.cu` | `gpu_experience_collector.rs` | Wired | Per-trade statistics accumulation | — | -| `backtest_env_kernel.cu` (`backtest_env_step`) | `gpu_backtest_evaluator.rs` | Wired | Already listed above | — | -| `common_device_functions.cuh` | prepended to all kernels at compile time by `build.rs` | Wired | Shared device helpers (BF16 wrappers, warp reduce) | — | -| `state_layout.cuh` | included by kernels that reference state buffer layout | Wired | Named index constants for state buffer | — | -| `trade_physics.cuh` | included by `backtest_env_kernel.cu` and others at compile time | Wired | Kelly / physics helpers shared across kernels | — | - -## Model Modules — Supervised-Only (OUT-of-DQN-scope per Part E) - -| Module / kernel | Consumer path | Classification | Notes | Action | -|---|---|---|---|---| -| `xlstm/mod.rs` + `xlstm/trainable.rs` | `ensemble/adapters/xlstm.rs`, `hyperopt/adapters/xlstm.rs` | OUT-of-DQN-scope | Supervised consumers; DQN does not import this. Part E: keep. | — | -| `kan/mod.rs` + `kan/trainable.rs` | `hyperopt/adapters/kan.rs` only | OUT-of-DQN-scope | KAN splines in DQN trunk (`batched_forward/backward`) are **inline** in `gpu_dqn_trainer.rs` and do not import `crate::kan`. `kan/` wraps ml-supervised KAN for hyperopt. Part E: keep. | — | -| `tgnn/mod.rs` + `tgnn/trainable_adapter.rs` | `hyperopt/adapters/tggn.rs`, `risk/mod.rs` | OUT-of-DQN-scope | TGNN for supervised + risk path | — | -| `tlob/mod.rs` + `tlob/trainable_adapter.rs` | `hyperopt/adapters/tlob.rs`, `data_loaders/tlob_loader.rs` | Partial / OUT-of-DQN-scope | TLOB transformer has hyperopt adapter; DQN trunk wiring scheduled for Plan 2 D.8 | Plan 2 D.8 wires TLOB trunk into DQN | -| `diffusion/mod.rs` + `diffusion/trainable.rs` | `hyperopt/adapters/diffusion.rs`, `trainers/dqn/fused_training.rs` (data aug only) | Partial | Diffusion model used for data augmentation in fused training but lacks a gradient backward path through DQN trunk | Plan 2 tracks full integration | -| `ppo/mod.rs` + `ppo/trainable_adapter.rs` | `trainers/ppo.rs`, `gpu_ppo_collector.rs` | OUT-of-DQN-scope | PPO training path; not DQN | — | - -## Temporal/Recurrent Modules — DQN Integration Status - -| Module / kernel | Consumer path | Classification | Notes | Action | -|---|---|---|---|---| -| `mamba/mod.rs` + `mamba/trainable_adapter.rs` | `trainers/mamba2.rs`, `model_factory.rs`, `inference_validator.rs`; Mamba2 temporal kernel wired fully in `gpu_dqn_trainer.rs` | Wired (DQN trunk) + OUT-of-DQN-scope (supervised `trainers/mamba2.rs`) | Forward and backward both implemented in mamba2_temporal_kernel.cu | — | -| `liquid/mod.rs` + `liquid/adapter.rs` | `trainers/dqn/fused_training.rs` (features), `gpu_dqn_trainer.rs` (`liquid_tau_rk4_kernel`); also `trainers/liquid.rs` supervised path | Partial | Liquid tau ODE modulation wired in DQN kernel; `LiquidTrainableAdapter` from supervised path not called in DQN backward | Plan 2 D.7 audits full backward wiring | -| `tft/mod.rs` + `tft/trainable_adapter.rs` | `trainers/tft/trainer.rs` has full forward + `backward_step()`; `TrainableTFT::backward()` returns error with explicit message that backward is via `TFTTrainer::train()` | Partial | TFT backward available via dedicated trainer; `UnifiedTrainable::backward()` slot returns informational error (not a stub — documented routing) | Plan 4 E.1/E.3 — DQN concept adoption; full backward path tracked in task #76 | - -## Data Pipeline Modules - -| Module / kernel | Consumer path | Classification | Notes | Action | -|---|---|---|---|---| -| `fxcache.rs` (`discover_and_load`, `FxCacheData`) | `train_baseline_rl.rs`, `trainers/dqn/data_loading.rs` | Wired | Primary training data format | — | -| `feature_cache.rs` | `fxcache.rs` (cache key), `hyperopt/adapters/dqn.rs`, smoke tests | Wired | Cache key generation for `.fxcache` auto-discovery | — | -| `data_pipeline/` (`DatasetManager`, `PreparedDataset`) | `lib.rs` prelude re-export; `integration/` modules | Wired | Dataset management abstraction | — | -| `data_loaders/dbn_sequence_loader.rs` | `data_loaders/mod.rs`; internally used by `dbn_tick_adapter.rs` | Wired | DBN sequence loading for supervised models | — | -| `data_loaders/dbn_tick_adapter.rs` | `data_loaders/mod.rs` | Wired | DBN tick-to-bar adapter | — | -| `data_loaders/streaming_dbn_loader.rs` | `data_loaders/mod.rs`; `tests/streaming_pipeline_edge_cases.rs` (test consumer) | Partial | Test-only consumer at tests/streaming_pipeline_edge_cases.rs; if streaming DBN load is genuinely needed for production (memory-efficient data loading), wire into production path in a follow-up task; otherwise schedule for deletion after test utility is moved to dbn_sequence_loader. | Reclassified Partial — test consumer confirmed | -| `data_loaders/tlob_loader.rs` (`TLOBDataLoader`) | `data_loaders/mod.rs`; no external call sites found | Orphan | TLOB data loader built but not called outside its own module and mod.rs | Plan 2 D.8 may wire; surface for user review | -| `training/unified_trainer.rs` (`UnifiedTrainable` trait) | `diffusion/trainable.rs`, `tlob/trainable_adapter.rs`, `mamba/trainable_adapter.rs`, `tft/trainable_adapter.rs`, hyperopt adapters | Wired | Unified training trait for supervised models | — | -| `training/unified_data_loader.rs` | — | — | Deleted (Task 6): zero production + zero test consumers confirmed. `pub mod unified_data_loader` removed from `training.rs`. | DELETED | -| `training/orchestrator.rs` (`TrainingOrchestrator`) | — | — | Deleted (Task 6): zero production + zero test consumers confirmed. `pub mod orchestrator` removed from `training.rs`. | DELETED | -| `training_pipeline.rs` (`ProductionTrainingError`) | `lib.rs` (two From impls: MLError + MLSafetyError); TrainingPipeline struct has no external instantiation | Partial | ProductionTrainingError used via From impl in lib.rs; TrainingPipeline struct itself may be dead — follow-up to extract error enum and delete struct separately. | Reclassified Partial — error type wired, struct possibly unused | -| `training_profile.rs` (`DqnTrainingProfile`) | `train_baseline_rl.rs`, `hyperopt/adapters/dqn.rs`, smoke tests | Wired | TOML-backed hyperparameter profiles | — | -| `walk_forward.rs` (`FoldRange`, `WalkForwardConfig`) | `train_baseline_rl.rs`, `validation/harness.rs` | Wired | Walk-forward fold management | — | - -## Evaluation / Validation Modules - -| Module / kernel | Consumer path | Classification | Notes | Action | -|---|---|---|---|---| -| `validation/mod.rs` (`ValidatableStrategy`) | `validation/harness.rs`, `hyperopt/adapters/dqn.rs` | Wired | Validation trait + entry points | — | -| `validation/harness.rs` | `hyperopt/adapters/dqn.rs`, `trainer/metrics.rs` | Wired | Walk-forward validation harness | — | -| `validation/financial.rs` | `validation/harness.rs` | Wired | Financial metric computation for val | — | -| `validation/regime_analysis.rs` | `validation/harness.rs` | Wired | Regime-conditional eval | — | -| `validation/ppo_adapter.rs` | `validation/harness.rs` | OUT-of-DQN-scope | PPO-specific validation path | — | -| `evaluation/mod.rs` | `hyperopt/adapters/dqn.rs`, `trainer/metrics.rs`, `gpu_backtest_evaluator.rs` | Wired | DQN evaluation engine | — | -| `backtesting/` (`GpuBacktestEvaluator`, `BarrierBacktester`) | `trainer/metrics.rs`, `hyperopt/adapters/*` | Wired | Backtesting framework | — | - -## Supporting Infrastructure Modules - -| Module / kernel | Consumer path | Classification | Notes | Action | -|---|---|---|---|---| -| `checkpoint/mod.rs` | `model_registry.rs` | Wired | Checkpoint save/load | — | -| `inference.rs` | multiple service consumers (67 files by grep) | Wired | Inference entry points | — | -| `inference_validator.rs` (`InferenceValidator`) | — | — | Deleted (Task 6): zero production + zero test consumers confirmed. No `pub mod` declaration found in any file. | DELETED | -| `model_factory.rs` | `mamba` path, `lib.rs` | Wired | Model construction factory | — | -| `model_loader_integration.rs` (`ModelLoaderTrait`) | — | — | Deleted (Task 6): zero production + zero test consumers confirmed. No `pub mod` declaration found in any file. | DELETED | -| `model_registry.rs` | `lib.rs`, various consumers (3 by grep) | Wired | Model lifecycle registry | — | -| `registry/mod.rs` | consumed by integration and service layer (10 consumers) | Wired | Operational model lifecycle | — | -| `hyperopt/` (campaign, adapters) | `trainer/training_loop.rs`, `hyperopt smoke test` | Wired | Bayesian hyperparameter optimisation | — | -| `ensemble/` | integration path, service layer | Wired | Multi-model ensemble | — | -| `integration/` | service layer | Wired | Service integration layer | — | -| `flash_attention/mod.rs` (`FlashAttention3`) | `transformers/hft_transformer.rs`, `trainers/tft/config.rs`, `benchmark/tft_benchmark.rs` | OUT-of-DQN-scope | TFT / transformer path only; DQN uses `gpu_attention.rs` | — | -| `features/` (`FeatureVector`, `extract_ml_features`) | `train_baseline_rl.rs`, DQN trainer | Wired | Feature extraction | — | -| `microstructure/` | `trainers/dqn/features.rs`, OFI pipeline | Wired | VPIN, Kyle lambda, order-flow imbalance | — | -| `regime/mod.rs` | DQN features, data pipeline (50 consumers) | Wired | Regime detection + signal | — | -| `regime_detection/mod.rs` | `lib.rs` declaration only; zero consumers of the facade; zero direct uses of `ml_regime_detection::` anywhere else | Orphan | Thin facade re-exporting ml-regime-detection crate. Zero consumers of the facade, zero direct uses of ml_regime_detection:: anywhere. The underlying ml-regime-detection/ workspace crate exists but appears unused. Deleting a whole workspace crate exceeds Task 6 scope — scheduled for a separate crate-cleanup task. Potentially superseded by ml_regime/ crate (50 consumers). | CRATE-LEVEL-FOLLOWUP | -| `risk/mod.rs` (crate-level) | DQN trainer constructor via `DrawdownMonitor`, `KellyCriterionOptimizer` | Wired | Risk management in training | — | -| `preprocessing.rs` | `features/`, data pipeline | Wired | Log-return normalisation, outlier clipping | — | -| `labeling/mod.rs` | `features/`, supervised training | Wired | Triple-barrier labelling | — | -| `security/mod.rs` | `lib.rs` (1 consumer) | Partial | ML security / prediction validation; no DQN consumer found | Surface for user review | -| `stress_testing/mod.rs` | `dqn/stress_testing.rs`, `ppo/stress_testing.rs` | Wired | Stress-testing framework | — | -| `paper_trading/mod.rs` | `lib.rs` declaration; `tests/paper_trading_integration_test.rs` (test consumer) | Partial | Test-only consumer at tests/paper_trading_integration_test.rs. Production trading-service uses its own paper_trading_executor (distinct module); this facade exists only for ML crate tests. | Reclassified Partial — test consumer confirmed | -| `observability/mod.rs` | `integration/performance_monitor.rs` | Wired | ML observability hooks | — | -| `deployment/` | `lib.rs`, integration layer | Wired | Model deployment + A/B testing | — | -| `universe/mod.rs` | `lib.rs`, integration layer (5 consumers) | Wired | Asset universe management | — | -| `asset_selection/mod.rs` | `lib.rs`, integration layer (referenced in universe path) | Wired | Asset selection logic | — | -| `batch_processing.rs` | `lib.rs`, `integration/` (multiple consumers) | Wired | Batch ML operations | — | -| `bridge.rs` | `lib.rs`, trading-ML bridge | Wired | ML-Financial type bridge | — | -| `data_loader.rs` | `lib.rs`, training service | Wired | Legacy data loader shim | — | -| `data_validation/mod.rs` | `lib.rs`, data pipeline | Wired | Input validation for data pipeline | — | -| `explainability/mod.rs` | `lib.rs` only (1 consumer) | Partial | SHAP / attribution; no DQN call site found | Surface for user review | -| `benchmarks.rs` | `lib.rs`, benchmark harness | OUT-of-DQN-scope | Benchmark entry point; not production training | — | -| `benchmark/` | benchmark harness | OUT-of-DQN-scope | GPU / model benchmarks; not production DQN path | — | -| `portfolio_transformer.rs` | — | — | Deleted (Task 6): zero production + zero test consumers confirmed. `pub mod portfolio_transformer` removed from lib.rs. | DELETED | - -## Summary - -Updated after Task 6 cleanup (2026-04-24): 5 confirmed-orphan files deleted, 3 Orphan rows reclassified Partial, 1 Orphan reclassified with crate-level follow-up action. Plan 1 Task 8 (2026-04-24): added `adaptive_controller.rs` (C.6 trait foundation). - -| Classification | Count | -|---|---| -| Wired | 75 | -| Partial | 10 | -| Orphan (held for follow-up) | 3 | -| Ghost | 0 | -| OUT-of-DQN-scope | 17 | -| **Total** | **105** | - -The 3 remaining Orphan rows are: -- `cuda_pipeline/gpu_statistics.rs` + `statistics_kernel.cu` — held for Plan 2 D.2 wire-or-delete decision. -- `data_loaders/tlob_loader.rs` — held for Plan 2 D.8 TLOB trunk wiring. -- `regime_detection/mod.rs` — crate-level cleanup task (whole workspace crate, exceeds Task 6 scope). - -## Orphan Resolution Index - -| Orphan | Resolution | Status | -|---|---|---| -| `cuda_pipeline/gpu_statistics.rs` + `statistics_kernel.cu` | Wire into val path or delete if superseded by `gpu_monitoring.rs` | Plan 2 D.2 — held | -| `data_loaders/streaming_dbn_loader.rs` | Reclassified Partial: test consumer at tests/streaming_pipeline_edge_cases.rs | RECLASSIFIED | -| `data_loaders/tlob_loader.rs` | Plan 2 D.8 wires TLOB trunk — this loader may be needed then | Plan 2 D.8 — held | -| `training/unified_data_loader.rs` | Deleted (zero consumers) | DELETED | -| `training/orchestrator.rs` | Deleted (zero consumers) | DELETED | -| `training_pipeline.rs` | Reclassified Partial: ProductionTrainingError used via From impl in lib.rs | RECLASSIFIED | -| `inference_validator.rs` | Deleted (zero consumers) | DELETED | -| `model_loader_integration.rs` | Deleted (zero consumers) | DELETED | -| `paper_trading/mod.rs` | Reclassified Partial: test consumer at tests/paper_trading_integration_test.rs | RECLASSIFIED | -| `portfolio_transformer.rs` | Deleted (zero consumers) | DELETED | -| `regime_detection/mod.rs` | Crate-level follow-up scheduled (separate crate-cleanup task) | CRATE-LEVEL-FOLLOWUP | +| Commit | Component | Files | Notes | +|--------|-----------|-------|-------| +| (Task 9) | `AtomsController` — C51 atom-position epoch controller | `adaptive_controller.rs` (trait), `controllers/atoms_controller.rs`, `controllers/mod.rs`, `gpu_dqn_trainer.rs` (`read_isv_v_ranges`), `fused_training.rs`, `training_loop.rs` | Trait + controller struct; `recompute_atom_positions` gated through `AdaptiveController::update()`. Old direct call removed. | +| (Task 10) | `GammaController` — G4 adaptive gamma epoch controller | `controllers/gamma_controller.rs`, `training_loop.rs` | Inline G4 block (~18 lines) removed; replaced by `GammaController::update()`. `adaptive_gamma` field kept in sync. | +| (Task 11) | `KellyCapController` — half-Kelly position-cap epoch controller | `controllers/kelly_cap_controller.rs`, `training_loop.rs` | Inline kelly_f_mean block (~20 lines) removed; replaced by `KellyCapController::update()`. No ISV slot; TradeStats passed directly. | +| (Task 12) | `CqlAlphaController` — CQL alpha scaffolding controller | `controllers/cql_alpha_controller.rs`, `training_loop.rs`, `trainer/mod.rs`, `trainer/constructor.rs` | New scaffolding; static alpha returned. Plan 3 adds seed-phase coupling. No old ad-hoc function existed. |