From 7d9808ecf0a46527d3c3dd551fcb2f8265df74bf Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Thu, 5 Mar 2026 21:14:31 +0100 Subject: [PATCH] fix(ml): smooth CVaR penalty, fix clip leakage, align noisy sigma, fix eval_supervised MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes from deep investigation audit (LOW/MEDIUM priority): 1. CVaR penalty: hard cliff (0 or 10) → smooth ramp with gradient signal for PSO. Formula: min(10, max(0, -cvar-0.05)*200). 2. Clip outliers leakage: data_loading.rs now computes clip bounds from training portion only (first 80%), then applies to full series. Log returns and windowed normalize are causal (no leakage). 3. Noisy sigma scheduler: hyperopt now matches conservative() defaults (enabled, initial=0.8, final=0.4) so hyperopt-found params generalize to train_best without scheduler mismatch. 4. evaluate_supervised.rs: NormStats fallback from test data (leakage) replaced with bail! matching evaluate_baseline.rs behavior. 5. Doc comments: stale 27D references updated to 31D (4 locations). Co-Authored-By: Claude Opus 4.6 --- crates/ml/examples/evaluate_supervised.rs | 8 ++-- crates/ml/src/hyperopt/adapters/dqn.rs | 22 +++++----- crates/ml/src/trainers/dqn/data_loading.rs | 48 ++++++++++++++++++++-- 3 files changed, 59 insertions(+), 19 deletions(-) diff --git a/crates/ml/examples/evaluate_supervised.rs b/crates/ml/examples/evaluate_supervised.rs index ae93f43a3..acec57b4d 100644 --- a/crates/ml/examples/evaluate_supervised.rs +++ b/crates/ml/examples/evaluate_supervised.rs @@ -632,13 +632,11 @@ fn main() -> Result<()> { serde_json::from_str(&norm_json) .with_context(|| format!("Failed to parse {}", norm_path.display()))? } else { - warn!( - " NormStats not found at {}, computing from test data (degraded)", + anyhow::bail!( + "NormStats not found at {} - cannot evaluate without training-set statistics. \ + Re-run training to generate norm_stats files.", norm_path.display() ); - let test_feat = extract_ml_features(&window.test) - .context("Feature extraction failed for test bars")?; - NormStats::from_features(&test_feat) }; // Extract and normalize test features diff --git a/crates/ml/src/hyperopt/adapters/dqn.rs b/crates/ml/src/hyperopt/adapters/dqn.rs index 812ac75aa..534f72cc1 100644 --- a/crates/ml/src/hyperopt/adapters/dqn.rs +++ b/crates/ml/src/hyperopt/adapters/dqn.rs @@ -3,7 +3,7 @@ //! This module provides a production-ready adapter for optimizing DQN //! hyperparameters using the generic optimization framework. It implements: //! -//! - 27D continuous parameter space with log-scale handling +//! - 31D continuous parameter space with log-scale handling //! - Training wrapper that integrates with existing DQN pipeline //! - Backtest-based objective using Sharpe ratio (production metric) //! - Prioritized Experience Replay (PER) tuning for Rainbow DQN performance (+25-40% convergence speed) @@ -153,7 +153,7 @@ const TRIAL_VRAM_MB: f64 = 7000.0; /// Corrected: was 0.02 (20KB) — 100x too high const MB_PER_SAMPLE: f64 = 0.0005; -/// DQN hyperparameter space (17D continuous - WAVE 11 with Rainbow booleans hardcoded to TRUE) +/// DQN hyperparameter space (31D continuous - WAVE 11 with Rainbow booleans hardcoded to TRUE) /// /// Defines the hyperparameters to optimize for DQN training: /// **Base Parameters (11D from Wave 1-2)**: @@ -796,7 +796,7 @@ impl ParameterSpace for DQNParams { batch_bound.1 = max_batch; } } - // Cap hidden_dim_base by VRAM (index 23 in 27D space) + // Cap hidden_dim_base by VRAM (index 23 in 31D space) let max_base = budget.max_hidden_dim_base(4, 256, 54, 45); if let Some(dim_bound) = bounds.get_mut(23) { dim_bound.1 = max_base as f64; @@ -2248,7 +2248,7 @@ impl HyperparameterOptimizable for DQNTrainer { // Fix 1: Clamp buffer size to max (4GB GPU constraint) let clamped_buffer_size = params.buffer_size.min(self.buffer_size_max); - info!("Training DQN with parameters (Wave 6.4: 17D search space):"); + info!("Training DQN with parameters (Wave 6.4: 31D search space):"); info!(" Learning rate: {:.6}", params.learning_rate); info!(" Batch size: {}", params.batch_size); info!(" Gamma: {:.3}", params.gamma); @@ -2543,10 +2543,11 @@ impl HyperparameterOptimizable for DQNTrainer { gae_lambda: 0.95, // Default: 0.95 (standard PPO/A2C value) // WAVE 26 P1.11: Noisy Network Sigma Scheduling - enable_noisy_sigma_scheduler: false, // Default: disabled (not in search space yet) - noisy_sigma_initial: 0.6, // Default: 60% initial noise - noisy_sigma_final: 0.4, // Default: 40% final noise - noisy_sigma_anneal_steps: 10000, // Default: 10K steps for annealing + // Must match conservative() defaults so hyperopt params generalize to train_best + enable_noisy_sigma_scheduler: true, // Match production defaults (conservative()) + noisy_sigma_initial: 0.8, // Match conservative(): 80% initial noise for 45-action space + noisy_sigma_final: 0.4, // 40% final noise + noisy_sigma_anneal_steps: 10000, // 10K steps for annealing // QR-DQN (replaces disabled C51) use_qr_dqn: params.use_qr_dqn, @@ -3327,8 +3328,9 @@ impl HyperparameterOptimizable for DQNTrainer { 0.2 * backtest.sharpe_ratio + 0.1 * backtest.omega_ratio; - // Tail risk penalty - let cvar_penalty = if backtest.cvar_95 < -0.05 { 10.0 } else { 0.0 }; + // Tail risk penalty: smooth ramp instead of hard cliff at -5% CVaR. + // Gives PSO gradient signal near the threshold instead of a binary 0/10 jump. + let cvar_penalty = ((-backtest.cvar_95 - 0.05).max(0.0) * 200.0).min(10.0); // Component 2: HFT activity score (25% weight) let hft_activity = calculate_hft_activity_score_wave10(buy_pct, sell_pct, hold_pct); diff --git a/crates/ml/src/trainers/dqn/data_loading.rs b/crates/ml/src/trainers/dqn/data_loading.rs index 0f6d7f967..766eae114 100644 --- a/crates/ml/src/trainers/dqn/data_loading.rs +++ b/crates/ml/src/trainers/dqn/data_loading.rs @@ -10,7 +10,7 @@ use candle_core::{Device, Tensor}; use tracing::{debug, info, warn}; use crate::features::extraction::OHLCVBar; -use crate::preprocessing::{preprocess_prices, PreprocessConfig}; +use crate::preprocessing::{clip_outliers_with_bounds, compute_clip_bounds, compute_log_returns, windowed_normalize, PreprocessConfig}; use crate::training_pipeline::FinancialFeatures; use crate::TrainingMetrics; @@ -299,9 +299,49 @@ impl DQNTrainer { info!(" • Window size: {}", preprocess_config.window_size); info!(" • Clip sigma: ±{:.1}σ", preprocess_config.clip_sigma); - // Apply preprocessing - let preprocessed_tensor = preprocess_prices(&close_tensor, preprocess_config) - .context("Failed to preprocess prices")?; + // Split-aware preprocessing to avoid data leakage in clip bounds. + // Steps 1-2 (log returns, windowed normalize) are causal and safe + // to run on the full series. Only the clip bounds must come from + // the training portion. + let returns = if preprocess_config.use_log_returns { + compute_log_returns(&close_tensor) + .context("Failed to compute log returns")? + } else { + // Simple returns path (mirrors preprocess_prices logic) + let n = close_tensor.dims()[0]; + let prev = close_tensor.narrow(0, 0, n - 1) + .context("Failed to narrow prev prices")?; + let curr = close_tensor.narrow(0, 1, n - 1) + .context("Failed to narrow curr prices")?; + let simple_ret = curr.sub(&prev) + .context("Failed to compute price diff")? + .div(&prev) + .context("Failed to compute simple returns")?; + let first_zero = Tensor::zeros((1,), simple_ret.dtype(), simple_ret.device()) + .context("Failed to create first zero")?; + Tensor::cat(&[&first_zero, &simple_ret], 0) + .context("Failed to concat simple returns")? + }; + + let normalized = windowed_normalize(&returns, preprocess_config.window_size) + .context("Failed windowed normalization")?; + + // Compute clip bounds from training portion only (80/20 split). + // The final sample count after feature extraction is roughly + // (n_bars - warmup), so align the split on the close tensor. + let n_norm = normalized.dims()[0]; + let train_end = (n_norm * 80) / 100; + let train_normalized = normalized.narrow(0, 0, train_end) + .context("Failed to narrow training portion for clip bounds")?; + let clip_bounds = compute_clip_bounds(&train_normalized, preprocess_config.clip_sigma) + .context("Failed to compute clip bounds from training data")?; + + info!(" • Clip bounds (train-only): [{:.4}, {:.4}]", clip_bounds.0, clip_bounds.1); + + // Apply training-derived clip bounds to the full normalized series + let preprocessed_tensor = clip_outliers_with_bounds( + &normalized, clip_bounds.0, clip_bounds.1, + ).context("Failed to clip outliers with training bounds")?; let preprocessed_vec: Vec = preprocessed_tensor .to_vec1()