diff --git a/crates/ml/src/trainers/dqn/trainer/constructor.rs b/crates/ml/src/trainers/dqn/trainer/constructor.rs index 5142bb3f5..7ee62899a 100644 --- a/crates/ml/src/trainers/dqn/trainer/constructor.rs +++ b/crates/ml/src/trainers/dqn/trainer/constructor.rs @@ -555,6 +555,7 @@ impl DQNTrainer { // Capture values before hyperparams is moved into Self let initial_batch_size = hyperparams.batch_size; let base_tau = hyperparams.tau; + let initial_epsilon = hyperparams.epsilon_start as f32; // GPU pipeline: pre-allocate staging buffers on CUDA devices let buffer_pool = matches!(device, MlDevice::Cuda { .. }).then(|| { @@ -771,6 +772,9 @@ impl DQNTrainer { wf_num_windows: 6, wf_purge_bars: 100, wf_min_sharpe: f64::NEG_INFINITY, + enrichment: super::enrichment::EnrichmentState::new( + initial_epsilon, + ), }) } diff --git a/crates/ml/src/trainers/dqn/trainer/enrichment.rs b/crates/ml/src/trainers/dqn/trainer/enrichment.rs new file mode 100644 index 000000000..21d1956f8 --- /dev/null +++ b/crates/ml/src/trainers/dqn/trainer/enrichment.rs @@ -0,0 +1,337 @@ +//! Self-improving enrichment module — 8 enrichment functions. +//! +//! `EnrichmentState` persists across epochs. `EvalTrade` captures per-trade +//! data from the validation backtest. `run_enrichments` dispatches all 8 +//! enrichment functions and returns an `EnrichmentResult` summarising the +//! adjustments for the next training epoch. +//! +//! All pure Rust, no CUDA kernels. + +use tracing::info; + +// --------------------------------------------------------------------------- +// Data structures +// --------------------------------------------------------------------------- + +/// Per-trade data extracted from the validation backtest. +#[derive(Debug, Clone)] +pub(crate) struct EvalTrade { + /// Global bar index where the trade was entered. + pub bar_index: usize, + /// Realised P&L (percentage or dollar, consistent units). + pub pnl: f32, + /// Predicted Q-value at entry. + pub predicted_q: f32, + /// Direction branch output (0=Short, 1=Flat, 2=Long). + pub direction: u8, + /// Magnitude branch output (0=Small, 1=Half, 2=Full). + pub magnitude: u8, + /// How many bars the position was held. + pub holding_bars: u32, + /// Ensemble variance across heads at entry. + pub ensemble_var: f32, +} + +/// Synthetic experience from hindsight replay (E7). +#[derive(Debug, Clone)] +pub(crate) struct HindsightExperience { + /// Bar index of the losing trade. + pub bar_index: usize, + /// Optimal direction (counterfactual). + pub optimal_direction: u8, + /// Optimal magnitude (counterfactual). + pub optimal_magnitude: u8, + /// Estimated counterfactual P&L. + pub counterfactual_pnl: f32, +} + +/// Output of one enrichment cycle. +#[derive(Debug, Clone)] +pub(crate) struct EnrichmentResult { + /// E1: additive correction to Q-targets (clipped ±2.0). + pub q_correction: f32, + /// E2: next-epoch epsilon. + pub epsilon: f32, + /// E3: suggested gamma (None = keep current). + pub gamma: Option, + /// E4: per-branch LR scaling [dir, mag, order, urgency]. + pub branch_lr_scale: [f32; 4], + /// E5: updated ensemble agreement threshold. + pub agreement_threshold: f32, + /// E6: bar indices of top-10% trades (priority replay). + pub winner_indices: Vec, + /// E7: hindsight labels for worst losers. + pub hindsight: Vec, + /// E8: curriculum weights (one per segment). + pub curriculum_weights: Vec, +} + +/// Persistent state across epochs for the enrichment module. +#[derive(Debug, Clone)] +pub(crate) struct EnrichmentState { + /// Current epsilon (updated each enrichment cycle). + pub epsilon: f32, + /// Current ensemble agreement threshold. + pub agreement_threshold: f32, + /// Number of enrichment cycles executed so far. + pub cycle_count: u32, + /// Number of curriculum segments. + pub n_segments: usize, +} + +impl EnrichmentState { + pub(crate) fn new(initial_epsilon: f32) -> Self { + Self { + epsilon: initial_epsilon, + agreement_threshold: 1.0, + cycle_count: 0, + n_segments: 8, + } + } +} + +// --------------------------------------------------------------------------- +// Enrichment functions (E1–E8) +// --------------------------------------------------------------------------- + +/// E1: Q-value reality check — computes additive bias correction. +fn compute_q_correction(trades: &[EvalTrade]) -> f32 { + if trades.is_empty() { + return 0.0; + } + let bias: f32 = trades + .iter() + .map(|t| t.predicted_q - t.pnl) + .sum::() + / trades.len() as f32; + (-bias).clamp(-2.0, 2.0) +} + +/// E2: Adaptive epsilon — performance-driven exploration rate. +fn compute_adaptive_epsilon(current: f32, val_sharpe: f32) -> f32 { + let scale = if val_sharpe > 2.0 { + 0.8 + } else if val_sharpe > 0.5 { + 0.95 + } else if val_sharpe < -0.5 { + 1.2 + } else { + 1.0 + }; + (current * scale).clamp(0.01, 0.15) +} + +/// E3: Dynamic gamma from profitable trade durations. +fn compute_dynamic_gamma(trades: &[EvalTrade]) -> Option { + let holds: Vec = trades + .iter() + .filter(|t| t.pnl > 0.0 && t.holding_bars > 0) + .map(|t| t.holding_bars as f32) + .collect(); + if holds.is_empty() { + return None; + } + let mean = holds.iter().sum::() / holds.len() as f32; + Some((1.0 - 1.0 / mean.max(5.0)).clamp(0.85, 0.98)) +} + +/// E4: Trade autopsy — per-branch error rates to LR scaling. +fn compute_branch_lr_scale(trades: &[EvalTrade]) -> [f32; 4] { + let losers: Vec<&EvalTrade> = trades.iter().filter(|t| t.pnl < 0.0).collect(); + if losers.is_empty() { + return [1.0; 4]; + } + let total = losers.len() as f32; + let mut errors = [0u32; 4]; + for t in &losers { + // Direction errors: went Long and lost, or went Short and lost + if (t.direction == 2 && t.pnl < 0.0) || (t.direction == 0 && t.pnl < 0.0) { + errors[0] += 1; + } + // Magnitude errors: oversized and significant loss + if t.magnitude >= 1 && t.pnl < -0.001 { + errors[1] += 1; + } + // Urgency errors: very short hold and lost + if t.holding_bars < 3 && t.pnl < 0.0 { + errors[3] += 1; + } + } + let rates: [f32; 4] = std::array::from_fn(|i| errors[i] as f32 / total); + let mean_rate = rates.iter().sum::() / 4.0; + std::array::from_fn(|d| (1.0 + 0.5 * (rates[d] - mean_rate)).clamp(0.5, 2.0)) +} + +/// E5: Ensemble agreement threshold tuning. +fn compute_agreement_threshold( + trades: &[EvalTrade], + current: f32, + full_sharpe: f32, +) -> f32 { + if trades.is_empty() { + return current; + } + let agree: Vec<&EvalTrade> = trades.iter().filter(|t| t.ensemble_var < current).collect(); + if agree.is_empty() { + return current * 1.1; + } + let pnls: Vec = agree.iter().map(|t| t.pnl).collect(); + let mean = pnls.iter().sum::() / pnls.len() as f32; + let var = pnls + .iter() + .map(|p| (p - mean).powi(2)) + .sum::() + / pnls.len() as f32; + let agree_sharpe = if var > 1e-12 { mean / var.sqrt() } else { 0.0 }; + let new = if agree_sharpe > full_sharpe * 1.5 { + current * 0.9 + } else if agree_sharpe < full_sharpe { + current * 1.1 + } else { + current + }; + new.clamp(0.01, 10.0) +} + +/// E6: Winner distillation — bar indices of top-10% trades by P&L. +fn compute_winner_indices(trades: &[EvalTrade]) -> Vec { + if trades.is_empty() { + return Vec::new(); + } + let mut sorted: Vec<(usize, f32)> = trades + .iter() + .enumerate() + .map(|(i, t)| (i, t.pnl)) + .collect(); + sorted.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal)); + let top_n = (sorted.len() / 10).max(1); + sorted[..top_n] + .iter() + .map(|(i, _)| trades[*i].bar_index) + .collect() +} + +/// E7: Hindsight optimal labels for losing trades. +fn compute_hindsight_labels(trades: &[EvalTrade]) -> Vec { + let take_n = (trades.len() / 10).max(1); + trades + .iter() + .filter(|t| t.pnl < -0.001) + .take(take_n) + .map(|t| HindsightExperience { + bar_index: t.bar_index, + optimal_direction: if t.direction == 2 { + 0 + } else if t.direction == 0 { + 2 + } else { + 1 + }, + optimal_magnitude: 0, + counterfactual_pnl: (-t.pnl).max(0.0), + }) + .collect() +} + +/// E8: Curriculum weights — inverse-Sharpe per segment. +fn compute_curriculum_weights(trades: &[EvalTrade], n_segments: usize) -> Vec { + if trades.is_empty() || n_segments == 0 { + return Vec::new(); + } + let seg_size = (trades.len() / n_segments).max(1); + let mut weights: Vec = (0..n_segments) + .map(|seg| { + let start = seg * seg_size; + let end = ((seg + 1) * seg_size).min(trades.len()); + if start >= trades.len() { + return 0.1; + } + let slice = &trades[start..end]; + let mean: f32 = + slice.iter().map(|t| t.pnl).sum::() / slice.len().max(1) as f32; + let var: f32 = slice + .iter() + .map(|t| (t.pnl - mean).powi(2)) + .sum::() + / slice.len().max(1) as f32; + let sharpe = mean / var.sqrt().max(1e-6); + (1.0 / sharpe.max(0.1)).clamp(0.1, 10.0) + }) + .collect(); + let total: f32 = weights.iter().sum(); + if total > 1e-6 { + for w in &mut weights { + *w /= total; + } + } + weights +} + +// --------------------------------------------------------------------------- +// Main dispatcher +// --------------------------------------------------------------------------- + +/// Run all 8 enrichment functions, update persistent state, and return results. +pub(crate) fn run_enrichments( + state: &mut EnrichmentState, + eval_trades: &[EvalTrade], + val_sharpe: f32, +) -> EnrichmentResult { + state.cycle_count += 1; + + // E1: Q-value bias correction + let q_correction = compute_q_correction(eval_trades); + + // E2: Adaptive epsilon + let epsilon = compute_adaptive_epsilon(state.epsilon, val_sharpe); + state.epsilon = epsilon; + + // E3: Dynamic gamma + let gamma = compute_dynamic_gamma(eval_trades); + + // E4: Per-branch LR scaling + let branch_lr_scale = compute_branch_lr_scale(eval_trades); + + // E5: Ensemble agreement threshold + let agreement_threshold = compute_agreement_threshold( + eval_trades, + state.agreement_threshold, + val_sharpe, + ); + state.agreement_threshold = agreement_threshold; + + // E6: Winner distillation + let winner_indices = compute_winner_indices(eval_trades); + + // E7: Hindsight labels + let hindsight = compute_hindsight_labels(eval_trades); + + // E8: Curriculum weights + let curriculum_weights = compute_curriculum_weights(eval_trades, state.n_segments); + + info!( + "Enrichment cycle {} complete: q_corr={:.4}, eps={:.4}, gamma={:?}, \ + branch_lr={:?}, agree_thr={:.4}, winners={}, hindsight={}, \ + curriculum_segs={}", + state.cycle_count, + q_correction, + epsilon, + gamma, + branch_lr_scale, + agreement_threshold, + winner_indices.len(), + hindsight.len(), + curriculum_weights.len(), + ); + + EnrichmentResult { + q_correction, + epsilon, + gamma, + branch_lr_scale, + agreement_threshold, + winner_indices, + hindsight, + curriculum_weights, + } +} diff --git a/crates/ml/src/trainers/dqn/trainer/mod.rs b/crates/ml/src/trainers/dqn/trainer/mod.rs index f1c7f19d6..7a65dbce9 100644 --- a/crates/ml/src/trainers/dqn/trainer/mod.rs +++ b/crates/ml/src/trainers/dqn/trainer/mod.rs @@ -39,6 +39,7 @@ mod metrics; mod state; mod constructor; mod training_loop; +pub(crate) mod enrichment; #[cfg(test)] mod tests; @@ -474,6 +475,9 @@ pub struct DQNTrainer { pub(crate) wf_purge_bars: usize, /// G3: Min-Sharpe across walk-forward windows (used for checkpoint selection). pub(crate) wf_min_sharpe: f64, + + /// Self-improving enrichment state (persists across epochs). + pub(crate) enrichment: enrichment::EnrichmentState, } impl std::fmt::Debug for DQNTrainer {