diff --git a/crates/ml/src/trainers/dqn/mod.rs b/crates/ml/src/trainers/dqn/mod.rs index 171b49517..befee472b 100644 --- a/crates/ml/src/trainers/dqn/mod.rs +++ b/crates/ml/src/trainers/dqn/mod.rs @@ -32,6 +32,8 @@ mod risk; mod statistics; mod trainer; mod smoke_tests; +pub mod state_reset_registry; +pub use state_reset_registry::{ResetCategory, RegistryEntry, StateResetRegistry}; // Re-export all public items for backward compatibility pub use config::{DQNAgentType, DQNHyperparameters}; diff --git a/crates/ml/src/trainers/dqn/state_reset_registry.rs b/crates/ml/src/trainers/dqn/state_reset_registry.rs new file mode 100644 index 000000000..918891bfc --- /dev/null +++ b/crates/ml/src/trainers/dqn/state_reset_registry.rs @@ -0,0 +1,223 @@ +//! Formal classification of every piece of training-time state by reset lifecycle. +//! See `docs/superpowers/specs/2026-04-24-dqn-v2-unified-design.md` §4.A.1. + +/// Reset lifecycle category for a piece of training-time state. +/// +/// See `docs/superpowers/specs/2026-04-24-dqn-v2-unified-design.md` §4.A.1. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ResetCategory { + /// Cleared at fold boundary. + FoldReset, + /// Cleared at validation-window boundary. + WindowReset, + /// Anneals toward bootstrap value over `decay_bars` bars at fold boundary. + SoftReset { decay_bars: u32 }, + /// Survives across folds (learned weights). + TrainingPersist, + /// Version-stable across runs; touched only by explicit migration. + SchemaContract, +} + +/// One entry in the reset registry. +#[derive(Debug, Clone, Copy)] +pub struct RegistryEntry { + pub name: &'static str, + pub category: ResetCategory, + /// Short description — what this state represents. Used in diagnostic dumps. + pub description: &'static str, +} + +/// Typed registry classifying every piece of training-time state. +/// +/// Populated at construction; read-only thereafter. Adding new state requires +/// adding a new entry here in the same commit that introduces the state +/// (per Invariant 2 Wire-It-Up). +#[derive(Debug)] +pub struct StateResetRegistry { + entries: Vec, +} + +impl StateResetRegistry { + /// Build the registry with all currently-known training state. + pub fn new() -> Self { + let entries = vec![ + // ───── Fold-reset state ───────────────────────────────────── + RegistryEntry { + name: "kelly_stats", + category: ResetCategory::FoldReset, + description: "Per-window Kelly win/loss counters (ps[14..18])", + }, + RegistryEntry { + name: "plan_state", + category: ResetCategory::FoldReset, + description: "Val plan_state_buf [N, 7] — plan entry snapshots", + }, + RegistryEntry { + name: "eval_q_mean_ema", + category: ResetCategory::FoldReset, + description: "Per-branch Q-mean EMAs in update_eval_v_range", + }, + RegistryEntry { + name: "eval_q_std_ema", + category: ResetCategory::FoldReset, + description: "Per-branch Q-std EMAs in update_eval_v_range", + }, + RegistryEntry { + name: "isv_v_range_slots", + category: ResetCategory::FoldReset, + description: "ISV[23..31) — v_center/v_half per branch", + }, + RegistryEntry { + name: "isv_learning_health", + category: ResetCategory::FoldReset, + description: "ISV[12] — learning_health EMA", + }, + RegistryEntry { + name: "isv_sharpe_ema", + category: ResetCategory::FoldReset, + description: "ISV[22] — training Sharpe EMA", + }, + RegistryEntry { + name: "isv_q_means", + category: ResetCategory::FoldReset, + description: "ISV[13..22) — per-direction/magnitude Q-mean EMAs + |Q| refs", + }, + // ───── Window-reset state ─────────────────────────────────── + RegistryEntry { + name: "portfolio_state", + category: ResetCategory::WindowReset, + description: "Per-window portfolio (ps[0..8]) — reset at window start", + }, + RegistryEntry { + name: "hold_time", + category: ResetCategory::WindowReset, + description: "ps[5] / ps[10] — consecutive bars in position", + }, + RegistryEntry { + name: "entry_price", + category: ResetCategory::WindowReset, + description: "ps[3] — entry price of current trade", + }, + // ───── Soft-reset state (anneals across folds) ────────────── + RegistryEntry { + name: "adaptive_gamma", + category: ResetCategory::SoftReset { decay_bars: 500 }, + description: "training_loop.rs adaptive_gamma — anneals to 0.905 bootstrap", + }, + RegistryEntry { + name: "isv_grad_balance_targets", + category: ResetCategory::SoftReset { decay_bars: 500 }, + description: "ISV[31..35) — per-branch grad-norm targets", + }, + RegistryEntry { + name: "isv_grad_scale_limit", + category: ResetCategory::SoftReset { decay_bars: 500 }, + description: "ISV[35] — scale-clamp limit EMA", + }, + // ───── Training-persist state ─────────────────────────────── + RegistryEntry { + name: "network_params", + category: ResetCategory::TrainingPersist, + description: "Online network weights (all tensors)", + }, + RegistryEntry { + name: "target_network", + category: ResetCategory::TrainingPersist, + description: "Target network weights — Polyak EMA of online", + }, + RegistryEntry { + name: "adam_momentum", + category: ResetCategory::TrainingPersist, + description: "Adam optimizer first moment (m)", + }, + RegistryEntry { + name: "adam_variance", + category: ResetCategory::TrainingPersist, + description: "Adam optimizer second moment (v)", + }, + RegistryEntry { + name: "adam_step", + category: ResetCategory::TrainingPersist, + description: "Adam step counter t", + }, + RegistryEntry { + name: "replay_buffer", + category: ResetCategory::TrainingPersist, + description: "PER experience buffer — persists across folds", + }, + // ───── Schema-contract state ──────────────────────────────── + RegistryEntry { + name: "ISV_SCHEMA_VERSION", + category: ResetCategory::SchemaContract, + description: "ISV[0] — slot-layout schema version (migrated only)", + }, + RegistryEntry { + name: "isv_iql_branch_scale_floor", + category: ResetCategory::SchemaContract, + description: "ISV[36] — IQL branch_scales floor (safety bound, bootstrap-only)", + }, + ]; + Self { entries } + } + + /// Look up a state's category by name. Returns `None` for unknown names + /// (caller should assert Some before use to catch typos). + pub fn category(&self, name: &str) -> Option { + self.entries.iter().find(|e| e.name == name).map(|e| e.category) + } + + /// Iterator over all entries with `FoldReset` category. + /// Used by the fold-boundary reset logic. + pub fn fold_reset_entries(&self) -> impl Iterator { + self.entries.iter().filter(|e| e.category == ResetCategory::FoldReset) + } + + /// Iterator over all entries with `WindowReset` category. + pub fn window_reset_entries(&self) -> impl Iterator { + self.entries.iter().filter(|e| e.category == ResetCategory::WindowReset) + } + + /// Iterator over all entries with `SoftReset` category. + pub fn soft_reset_entries(&self) -> impl Iterator { + self.entries.iter().filter(|e| matches!(e.category, ResetCategory::SoftReset { .. })) + } + + /// All entries (debug/diagnostic dump). + pub fn all(&self) -> &[RegistryEntry] { + &self.entries + } +} + +impl Default for StateResetRegistry { + fn default() -> Self { + Self::new() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn registry_classifies_known_state() { + let r = StateResetRegistry::new(); + assert_eq!(r.category("kelly_stats"), Some(ResetCategory::FoldReset)); + assert_eq!(r.category("adaptive_gamma"), Some(ResetCategory::SoftReset { decay_bars: 500 })); + assert_eq!(r.category("network_params"), Some(ResetCategory::TrainingPersist)); + assert_eq!(r.category("ISV_SCHEMA_VERSION"), Some(ResetCategory::SchemaContract)); + } + + #[test] + fn registry_fold_reset_iterates_only_fold_reset_entries() { + let r = StateResetRegistry::new(); + let names: Vec<&str> = r.fold_reset_entries().map(|e| e.name).collect(); + assert!(names.contains(&"kelly_stats")); + assert!(!names.contains(&"network_params")); + } + + #[test] + fn registry_unknown_name_returns_none() { + let r = StateResetRegistry::new(); + assert_eq!(r.category("nonexistent_state"), None); + } +} diff --git a/docs/dqn-wire-up-audit.md b/docs/dqn-wire-up-audit.md index f7858b709..4a2c89933 100644 --- a/docs/dqn-wire-up-audit.md +++ b/docs/dqn-wire-up-audit.md @@ -13,3 +13,4 @@ | Module / kernel | Consumer path | Classification | Notes | |---|---|---|---| | (populated during A.5 audit) | | | | +| `crates/ml/src/trainers/dqn/state_reset_registry.rs` | consumers in Plan 1 Task 3+ (reset_fold_state call) | Wired (consumer added in same plan) | A.1 primary implementation |