diff --git a/config/training/dqn-hyperopt.toml b/config/training/dqn-hyperopt.toml index 61218bf0b..8dc2b34ac 100644 --- a/config/training/dqn-hyperopt.toml +++ b/config/training/dqn-hyperopt.toml @@ -70,6 +70,9 @@ dd_threshold = [0.005, 0.03] # HFT: tight drawdown tolerance (0.5%-3% loss_aversion = [1.0, 3.0] time_decay_rate = [0.0001, 0.005] +# Trade conviction filter +q_gap_threshold = [0.0, 0.5] # 0.0=trade every bar, 0.5=high conviction only + [reward] w_dsr = 1.0 w_pnl = 0.3 @@ -78,6 +81,7 @@ w_idle = 0.01 dd_threshold = 0.01 loss_aversion = 1.5 time_decay_rate = 0.0005 +q_gap_threshold = 0.0 [fixed] use_branching = true # GPU pipeline requires branching @@ -109,3 +113,4 @@ w_idle = 0.01 dd_threshold = 0.01 loss_aversion = 1.5 time_decay_rate = 0.0005 +q_gap_threshold = 0.0 diff --git a/config/training/dqn-production.toml b/config/training/dqn-production.toml index 5978c80ef..d4019b3f1 100644 --- a/config/training/dqn-production.toml +++ b/config/training/dqn-production.toml @@ -72,3 +72,4 @@ w_idle = 0.01 dd_threshold = 0.01 loss_aversion = 1.5 time_decay_rate = 0.0005 +q_gap_threshold = 0.0 # disabled by default — hyperopt searches [0.0, 0.5] diff --git a/config/training/dqn-smoketest.toml b/config/training/dqn-smoketest.toml index 753290a10..086f2f148 100644 --- a/config/training/dqn-smoketest.toml +++ b/config/training/dqn-smoketest.toml @@ -36,3 +36,4 @@ w_idle = 0.01 dd_threshold = 0.01 loss_aversion = 1.5 time_decay_rate = 0.0005 +q_gap_threshold = 0.0 diff --git a/crates/ml/src/cuda_pipeline/experience_kernels.cu b/crates/ml/src/cuda_pipeline/experience_kernels.cu index 37e364c35..a77ae8c7b 100644 --- a/crates/ml/src/cuda_pipeline/experience_kernels.cu +++ b/crates/ml/src/cuda_pipeline/experience_kernels.cu @@ -208,29 +208,27 @@ extern "C" __global__ void experience_state_gather( /** * Epsilon-greedy action selection on Q-values produced by cuBLAS. * - * Supports Branching Dueling DQN (use_branching=1) and flat DQN - * (use_branching=0, exposure branch only). - * - * Branching mode — q_values layout: [N, b0_size + b1_size + b2_size] + * Branching Dueling DQN — q_values layout: [N, b0_size + b1_size + b2_size] * Branch 0 (exposure): q_values[i*(b0+b1+b2) + 0 .. +b0) * Branch 1 (order): q_values[i*(b0+b1+b2) + b0 .. +b0+b1) * Branch 2 (urgency): q_values[i*(b0+b1+b2) + b0+b1 .. +b0+b1+b2) * Factored action = a0 * b1_size * b2_size + a1 * b2_size + a2. * - * Flat mode — q_values layout: [N, b0_size] - * Action index = argmax or random in [0, b0_size). + * Q-gap conviction filter: when greedy, if Q(best_exposure) - Q(flat) < + * q_gap_threshold, force exposure to flat (index 2). This reduces trade + * frequency by requiring high conviction for position entry. * * Grid: ceil(N / 256), Block: 256. One thread per episode. * - * @param q_values [N, q_stride] Q-values from cuBLAS - * @param out_actions [N] selected factored action index (output) - * @param rng_states [N] per-episode LCG RNG counter (updated in place) - * @param epsilon exploration probability in [0, 1] - * @param N number of episodes - * @param b0_size exposure branch size (default 5) - * @param b1_size order branch size (default 3) - * @param b2_size urgency branch size (default 3) - * @param use_branching 1 for Branching DQN, 0 for flat + * @param q_values [N, q_stride] Q-values from cuBLAS + * @param out_actions [N] selected factored action index (output) + * @param rng_states [N] per-episode LCG RNG counter (updated in place) + * @param epsilon exploration probability in [0, 1] + * @param N number of episodes + * @param b0_size exposure branch size (default 5) + * @param b1_size order branch size (default 3) + * @param b2_size urgency branch size (default 3) + * @param q_gap_threshold min Q-gap for trade entry (0.0 = disabled) */ extern "C" __global__ void experience_action_select( const float* __restrict__ q_values, @@ -241,66 +239,59 @@ extern "C" __global__ void experience_action_select( int b0_size, int b1_size, int b2_size, - int use_branching + float q_gap_threshold ) { int i = blockIdx.x * blockDim.x + threadIdx.x; if (i >= N) return; unsigned int rng = rng_states[i]; - int action_idx; - if (use_branching) { - /* ---------------------------------------------------------------- */ - /* Branching mode: three independent epsilon-greedy selections. */ - /* ---------------------------------------------------------------- */ - int q_stride = b0_size + b1_size + b2_size; - const float* q_b0 = q_values + (long long)i * q_stride; - const float* q_b1 = q_b0 + b0_size; - const float* q_b2 = q_b1 + b1_size; + int q_stride = b0_size + b1_size + b2_size; + const float* q_b0 = q_values + (long long)i * q_stride; + const float* q_b1 = q_b0 + b0_size; + const float* q_b2 = q_b1 + b1_size; - int a0, a1, a2; - - /* Branch 0: exposure */ - if (lcg_random(&rng) < epsilon) { - int r = (int)(lcg_random(&rng) * (float)b0_size); - a0 = (r >= b0_size) ? b0_size - 1 : r; - } else { - a0 = argmax_n(q_b0, b0_size); - } - - /* Branch 1: order type — all order types always valid, no masking */ - if (lcg_random(&rng) < epsilon) { - int r = (int)(lcg_random(&rng) * (float)b1_size); - a1 = (r >= b1_size) ? b1_size - 1 : r; - } else { - a1 = argmax_n(q_b1, b1_size); - } - - /* Branch 2: urgency — all urgency levels always valid, no masking */ - if (lcg_random(&rng) < epsilon) { - int r = (int)(lcg_random(&rng) * (float)b2_size); - a2 = (r >= b2_size) ? b2_size - 1 : r; - } else { - a2 = argmax_n(q_b2, b2_size); - } - - /* Compose factored action (Tavakoli et al., 2018, §3.1) */ - action_idx = a0 * b1_size * b2_size + a1 * b2_size + a2; + int a0, a1, a2; + /* Branch 0: exposure — with Q-gap conviction filter. + * When greedy (not exploring), only trade if Q(best) - Q(flat) + * exceeds q_gap_threshold. Otherwise default to flat (index 2). + * This teaches the model to trade only with high conviction. */ + if (lcg_random(&rng) < epsilon) { + int r = (int)(lcg_random(&rng) * (float)b0_size); + a0 = (r >= b0_size) ? b0_size - 1 : r; } else { - /* ---------------------------------------------------------------- */ - /* Flat mode: single advantage head = exposure branch only. */ - /* ---------------------------------------------------------------- */ - const float* q_b0 = q_values + (long long)i * b0_size; - - if (lcg_random(&rng) < epsilon) { - int r = (int)(lcg_random(&rng) * (float)b0_size); - action_idx = (r >= b0_size) ? b0_size - 1 : r; - } else { - action_idx = argmax_n(q_b0, b0_size); + a0 = argmax_n(q_b0, b0_size); + /* Q-gap filter: force flat when conviction is low */ + if (q_gap_threshold > 0.0f && b0_size > 2) { + int flat_idx = 2; /* exposure_idx_to_fraction(2) = 0.0 */ + float q_best = q_b0[a0]; + float q_flat = q_b0[flat_idx]; + if (q_best - q_flat < q_gap_threshold) { + a0 = flat_idx; + } } } + /* Branch 1: order type */ + if (lcg_random(&rng) < epsilon) { + int r = (int)(lcg_random(&rng) * (float)b1_size); + a1 = (r >= b1_size) ? b1_size - 1 : r; + } else { + a1 = argmax_n(q_b1, b1_size); + } + + /* Branch 2: urgency */ + if (lcg_random(&rng) < epsilon) { + int r = (int)(lcg_random(&rng) * (float)b2_size); + a2 = (r >= b2_size) ? b2_size - 1 : r; + } else { + a2 = argmax_n(q_b2, b2_size); + } + + /* Compose factored action (Tavakoli et al., 2018, §3.1) */ + int action_idx = a0 * b1_size * b2_size + a1 * b2_size + a2; + out_actions[i] = action_idx; rng_states[i] = rng; } diff --git a/crates/ml/src/cuda_pipeline/gpu_backtest_evaluator.rs b/crates/ml/src/cuda_pipeline/gpu_backtest_evaluator.rs index 04d3fc57d..9cdb1f41d 100644 --- a/crates/ml/src/cuda_pipeline/gpu_backtest_evaluator.rs +++ b/crates/ml/src/cuda_pipeline/gpu_backtest_evaluator.rs @@ -334,6 +334,9 @@ pub struct GpuBacktestEvaluator { /// `experience_action_select` kernel for greedy argmax. action_select_kernel: Option, + /// Q-value gap threshold for trade conviction filter (0.0 = disabled). + q_gap_threshold: f32, + /// RNG states for action selection (greedy mode uses epsilon=0). rng_states: Option>, @@ -618,6 +621,7 @@ impl GpuBacktestEvaluator { cublas_q_values: None, expected_q_kernel: None, action_select_kernel: None, + q_gap_threshold: 0.0, rng_states: None, dqn_graph: None, chunked_cublas_forward: None, @@ -1082,7 +1086,6 @@ impl GpuBacktestEvaluator { let b2 = dqn_cfg.branch_2_size as i32; let v_min_f = dqn_cfg.v_min; let v_max_f = dqn_cfg.v_max; - let use_branching = if dqn_cfg.branch_1_size > 0 && dqn_cfg.branch_2_size > 0 { 1_i32 } else { 0_i32 }; let epsilon = 0.0_f32; let state_row_bytes = n * state_dim * std::mem::size_of::(); let action_row_bytes = n * std::mem::size_of::(); @@ -1177,7 +1180,7 @@ impl GpuBacktestEvaluator { .arg(&b0) .arg(&b1) .arg(&b2) - .arg(&use_branching) + .arg(&self.q_gap_threshold) .launch(launch_cfg) .map_err(|e| MLError::ModelError(format!( "chunked action_select chunk_start={chunk_start}: {e}" diff --git a/crates/ml/src/cuda_pipeline/gpu_experience_collector.rs b/crates/ml/src/cuda_pipeline/gpu_experience_collector.rs index 57981bac3..34c98f4a8 100644 --- a/crates/ml/src/cuda_pipeline/gpu_experience_collector.rs +++ b/crates/ml/src/cuda_pipeline/gpu_experience_collector.rs @@ -201,6 +201,10 @@ pub struct ExperienceCollectorConfig { pub fill_spread_capture_frac: f32, /// Whether fill simulation is enabled (requires median_spread > 0) pub fill_simulation_enabled: bool, + /// Q-value gap threshold for trade conviction filter. + /// When greedy Q(best_exposure) - Q(flat) < threshold, default to flat. + /// 0.0 = disabled (trade on every bar). Typical range [0.0, 0.5]. + pub q_gap_threshold: f32, /// DSR EMA decay rate (e.g. 0.01 = ~100-step window). DSR is always enabled. pub dsr_eta: f32, /// N-step returns lookahead (1 = standard TD, 3-5 typical for Rainbow DQN) @@ -254,6 +258,7 @@ impl Default for ExperienceCollectorConfig { fill_spread_cost_frac: 0.50, fill_spread_capture_frac: 0.50, fill_simulation_enabled: false, + q_gap_threshold: 0.0, dsr_eta: 0.01, n_steps: 1, enable_action_masking: false, @@ -1000,9 +1005,9 @@ impl GpuExperienceCollector { &self.exp_b_logits }; - // ── 4. Action selection: epsilon-greedy ───────────────────── + // ── 4. Action selection: epsilon-greedy + Q-gap conviction filter let epsilon = config.epsilon; - let use_branching = 1_i32; + let q_gap = config.q_gap_threshold; unsafe { self.stream .launch_builder(&self.action_select_kernel) @@ -1014,7 +1019,7 @@ impl GpuExperienceCollector { .arg(&b0) .arg(&b1) .arg(&b2) - .arg(&use_branching) + .arg(&q_gap) .launch(launch_cfg) .map_err(|e| MLError::ModelError(format!( "experience_action_select t={t}: {e}" diff --git a/crates/ml/src/hyperopt/adapters/dqn.rs b/crates/ml/src/hyperopt/adapters/dqn.rs index b5395bc4e..4baafe450 100644 --- a/crates/ml/src/hyperopt/adapters/dqn.rs +++ b/crates/ml/src/hyperopt/adapters/dqn.rs @@ -3,7 +3,7 @@ //! This module provides a production-ready adapter for optimizing DQN //! hyperparameters using the generic optimization framework. It implements: //! -//! - 38D continuous parameter space with log-scale handling +//! - 39D continuous parameter space with log-scale handling //! - Training wrapper that integrates with existing DQN pipeline //! - Backtest-based objective using Sharpe ratio (production metric) //! - Prioritized Experience Replay (PER) tuning for Rainbow DQN performance (+25-40% convergence speed) @@ -144,7 +144,7 @@ const TRIAL_VRAM_MB: f64 = 7000.0; /// Corrected: was 0.02 (20KB) — 100x too high const MB_PER_SAMPLE: f64 = 0.0005; -/// DQN hyperparameter space (38D continuous) +/// DQN hyperparameter space (39D continuous) /// /// Fixed params (not in search space): /// - curiosity_weight: 0.0 (conflicts with NoisyNet) @@ -156,7 +156,7 @@ const MB_PER_SAMPLE: f64 = 0.0005; /// - count_bonus_coefficient: (0.0, 0.3) — UCB exploration bonus, complementary to NoisyNet /// - sharpe_weight: (0.0, 0.5) — tunable Sharpe ratio blending in composite reward /// -/// Composite reward weights (indices 31-37): +/// Composite reward weights (indices 31-38): /// - w_dsr: DSR (Sharpe) weight /// - w_pnl: Normalized PnL weight /// - w_dd: Drawdown penalty weight @@ -384,7 +384,7 @@ pub struct DQNParams { /// H100 with batch=1024, accum=4 → effective batch 4096 (more stable gradients). pub gradient_accumulation_steps: usize, - // === Composite Reward Weights (indices 31-37 in search space) === + // === Composite Reward Weights (indices 31-38 in search space) === /// DSR (Sharpe) weight in composite reward (0.1-2.0) pub w_dsr: f64, /// Normalized PnL weight in composite reward (0.0-1.0) @@ -399,6 +399,8 @@ pub struct DQNParams { pub loss_aversion: f64, /// Position staleness rent per step (0.0001-0.005) pub time_decay_rate: f64, + /// Q-value gap threshold for trade conviction filter (0.0-0.5) + pub q_gap_threshold: f64, } impl Default for DQNParams { @@ -486,13 +488,14 @@ impl Default for DQNParams { dd_threshold: 0.02, loss_aversion: 1.5, time_decay_rate: 0.0005, + q_gap_threshold: 0.0, } } } impl ParameterSpace for DQNParams { fn continuous_bounds() -> Vec<(f64, f64)> { - // 38D search space (C8: added 7 composite reward weights at indices 31-37) + // 39D search space (C8: added 7 composite reward weights at indices 31-38) // All bounds loaded from config/training/dqn-hyperopt.toml [search_space]. // To change search ranges, edit the TOML — no code changes needed. // Log-scale transforms are applied here; TOML stores linear values. @@ -530,7 +533,7 @@ impl ParameterSpace for DQNParams { let ga = b("gradient_accumulation_steps", (1.0, 1.0)); let iq = b("iqn_lambda", (0.0, 2.0)); - // Composite reward weights (indices 31-37) + // Composite reward weights (indices 31-38) let mut rw_dsr = b("w_dsr", (0.1, 2.0)); let mut rw_pnl = b("w_pnl", (0.0, 1.0)); let mut rw_dd = b("w_dd", (0.0, 5.0)); @@ -538,6 +541,7 @@ impl ParameterSpace for DQNParams { let mut rw_ddt = b("dd_threshold", (0.01, 0.10)); let mut rw_la = b("loss_aversion", (1.0, 3.0)); let mut rw_tdr = b("time_decay_rate", (0.0001, 0.005)); + let mut rw_qgt = b("q_gap_threshold", (0.0, 0.5)); // Two-phase hyperopt: fix architecture OR dynamics params to single-point bounds. // Single-point bounds (v, v) make the optimizer trivially converge on those @@ -584,6 +588,7 @@ impl ParameterSpace for DQNParams { .expect("phase_fast.loss_aversion missing from dqn-hyperopt.toml"); let fix_tdr = pf.time_decay_rate .expect("phase_fast.time_decay_rate missing from dqn-hyperopt.toml"); + let fix_qgt = pf.q_gap_threshold.unwrap_or(0.0); rw_dsr = (fix_w_dsr, fix_w_dsr); rw_pnl = (fix_w_pnl, fix_w_pnl); @@ -592,6 +597,7 @@ impl ParameterSpace for DQNParams { rw_ddt = (fix_ddt, fix_ddt); rw_la = (fix_la, fix_la); rw_tdr = (fix_tdr, fix_tdr); + rw_qgt = (fix_qgt, fix_qgt); tracing::info!( "Phase FAST: fixed architecture (hidden_dim={}, num_atoms={}, branch_hidden={}, dueling_hidden={}, v_range={}) + reward weights", @@ -607,7 +613,7 @@ impl ParameterSpace for DQNParams { } HyperoptPhase::Reward(ref best_vec) => { // Phase 3: fix dynamics + architecture from Phase 2 best params. - // Search ONLY reward weights (indices 31-37, 7D). + // Search ONLY reward weights (indices 31-38, 7D). // All other dims fixed to Phase 2's best values. tracing::info!("Phase REWARD: fixing {} dynamics+architecture params, searching 7 reward dims", best_vec.len()); // Bounds will be applied after vec construction below. @@ -646,7 +652,7 @@ impl ParameterSpace for DQNParams { bh, // 28: branch_hidden_dim ga, // 29: gradient_accumulation_steps iq, // 30: iqn_lambda - // Composite reward weights (indices 31-37) + // Composite reward weights (indices 31-38) rw_dsr, // 31: w_dsr rw_pnl, // 32: w_pnl rw_dd, // 33: w_dd @@ -654,6 +660,7 @@ impl ParameterSpace for DQNParams { rw_ddt, // 35: dd_threshold rw_la, // 36: loss_aversion rw_tdr, // 37: time_decay_rate + rw_qgt, // 38: q_gap_threshold ]; // Phase FULL: fix all non-architecture dims to Phase 1 best values. @@ -668,10 +675,10 @@ impl ParameterSpace for DQNParams { } } - // Phase REWARD: fix ALL dims EXCEPT reward weights (indices 31-37). + // Phase REWARD: fix ALL dims EXCEPT reward weights (indices 31-38). // This is the fastest phase — only 7D search, ~10s/trial. if let HyperoptPhase::Reward(ref best_vec) = phase { - let reward_dims: &[usize] = &[31, 32, 33, 34, 35, 36, 37]; + let reward_dims: &[usize] = &[31, 32, 33, 34, 35, 36, 37, 38]; for (i, bv) in best_vec.iter().enumerate() { if i < bounds.len() && !reward_dims.contains(&i) { bounds[i] = (*bv, *bv); @@ -683,7 +690,7 @@ impl ParameterSpace for DQNParams { } fn from_continuous(x: &[f64]) -> Result { - if x.len() != 38 { + if x.len() != 39 { return Err(MLError::ConfigError(format!("Expected 38 continuous parameters, got {}", x.len()))); } @@ -779,7 +786,7 @@ impl ParameterSpace for DQNParams { // C7: IQN dual-head lambda (idx 30) let iqn_lambda = x[30].clamp(0.0, 2.0); - // C8: Composite reward weights (indices 31-37) + // C8: Composite reward weights (indices 31-38) let w_dsr = x[31].clamp(0.1, 2.0); let w_pnl = x[32].clamp(0.0, 1.0); let w_dd = x[33].clamp(0.0, 5.0); @@ -787,6 +794,7 @@ impl ParameterSpace for DQNParams { let dd_threshold = x[35].clamp(0.01, 0.10); let loss_aversion = x[36].clamp(1.0, 3.0); let time_decay_rate = x[37].clamp(0.0001, 0.005); + let q_gap_threshold = x[38].clamp(0.0, 0.5); // WAVE 6 FIX #2: Batch size floor for high learning rates if learning_rate > 2e-4 && batch_size < 120 { @@ -858,7 +866,7 @@ impl ParameterSpace for DQNParams { branch_hidden_dim, use_branching, gradient_accumulation_steps, - // C8: Composite reward weights (indices 31-37) + // C8: Composite reward weights (indices 31-38) w_dsr, w_pnl, w_dd, @@ -866,13 +874,14 @@ impl ParameterSpace for DQNParams { dd_threshold, loss_aversion, time_decay_rate, + q_gap_threshold, }; Ok(params) } fn to_continuous(&self) -> Vec { - // 38D search space (C8: added 7 composite reward weights at indices 31-37) + // 39D search space (C8: added 7 composite reward weights at indices 31-38) vec![ self.learning_rate.ln(), // 0 self.batch_size as f64, // 1 @@ -905,7 +914,7 @@ impl ParameterSpace for DQNParams { self.branch_hidden_dim as f64, // 28: C5 branch_hidden_dim self.gradient_accumulation_steps as f64, // 29: C6 gradient accumulation self.iqn_lambda, // 30: C7 IQN dual-head loss weight - // C8: Composite reward weights (indices 31-37) + // C8: Composite reward weights (indices 31-38) self.w_dsr, // 31 self.w_pnl, // 32 self.w_dd, // 33 @@ -913,6 +922,7 @@ impl ParameterSpace for DQNParams { self.dd_threshold, // 35 self.loss_aversion, // 36 self.time_decay_rate, // 37 + self.q_gap_threshold, // 38 ] } @@ -958,6 +968,7 @@ impl ParameterSpace for DQNParams { "dd_threshold", // 35 "loss_aversion", // 36 "time_decay_rate", // 37 + "q_gap_threshold", // 38 ] } @@ -2545,7 +2556,7 @@ impl HyperparameterOptimizable for DQNTrainer { // Fix 1: Clamp buffer size to max (4GB GPU constraint) let clamped_buffer_size = params.buffer_size.min(self.buffer_size_max); - info!("Training DQN with parameters (C8: 38D search space):"); + info!("Training DQN with parameters (C8: 39D search space):"); info!(" Learning rate: {:.6}", params.learning_rate); info!(" Batch size: {}", params.batch_size); info!(" Gamma: {:.3}", params.gamma); @@ -2837,7 +2848,7 @@ impl HyperparameterOptimizable for DQNTrainer { gradient_collapse_multiplier: 100.0, // Adaptive threshold (LR × 100) gradient_collapse_patience: 5, // 5 consecutive epochs before early stop - // LR scheduling — tunable via hyperopt search space (idx 23 in 38D) + // LR scheduling — tunable via hyperopt search space (idx 23 in 39D) // 0=constant, 1=linear decay, 2=cosine annealing // BUG FIX: total_steps = self.epochs (scheduler steps once per epoch, // NOT per training step). Previous calc used epochs × steps_per_epoch @@ -2973,7 +2984,7 @@ impl HyperparameterOptimizable for DQNTrainer { wf_test_fraction: 0.125, wf_step_fraction: 0.125, - // C8: Composite reward weights — wired from hyperopt search space (indices 31-37) + // C8: Composite reward weights — wired from hyperopt search space (indices 31-38) w_dsr: params.w_dsr, w_pnl: params.w_pnl, w_dd: params.w_dd, @@ -2981,6 +2992,7 @@ impl HyperparameterOptimizable for DQNTrainer { dd_threshold: params.dd_threshold, loss_aversion: params.loss_aversion, time_decay_rate: params.time_decay_rate, + q_gap_threshold: params.q_gap_threshold, ..Default::default() }; @@ -3731,7 +3743,7 @@ mod tests { #[test] fn test_dqn_params_roundtrip() { - // Roundtrip test for the 38D search space (C8: 7 composite reward weights added) + // Roundtrip test for the 39D search space (C8: 7 composite reward weights added) // Only the tuned parameters roundtrip; fixed params get defaults from from_continuous let params = DQNParams { learning_rate: 3.37e-05, @@ -3799,10 +3811,11 @@ mod tests { dd_threshold: 0.04, loss_aversion: 2.0, time_decay_rate: 0.001, + q_gap_threshold: 0.2, }; let continuous = params.to_continuous(); - assert_eq!(continuous.len(), 38, "to_continuous must return 38D vector"); + assert_eq!(continuous.len(), 39, "to_continuous must return 39D vector"); let recovered = DQNParams::from_continuous(&continuous).unwrap(); // Tuned parameters must roundtrip exactly @@ -3845,7 +3858,7 @@ mod tests { // C6: gradient_accumulation_steps roundtrips via index 29 assert_eq!(recovered.gradient_accumulation_steps, params.gradient_accumulation_steps); - // C8: Composite reward weights roundtrip (indices 31-37) + // C8: Composite reward weights roundtrip (indices 31-38) assert!((recovered.w_dsr - params.w_dsr).abs() < 1e-6); assert!((recovered.w_pnl - params.w_pnl).abs() < 1e-6); assert!((recovered.w_dd - params.w_dd).abs() < 1e-6); @@ -3858,7 +3871,7 @@ mod tests { #[test] fn test_dqn_params_bounds() { let bounds = DQNParams::continuous_bounds(); - assert_eq!(bounds.len(), 38); // C8: 38D (added 7 reward weights) + assert_eq!(bounds.len(), 39); // C8: 39D (added 7 reward weights) // Check log-scale bounds are reasonable assert!(bounds[0].0 < bounds[0].1); // learning_rate @@ -3909,15 +3922,16 @@ mod tests { assert_eq!(bounds[32], (0.3, 0.3)); // w_pnl (FIXED in Fast phase) assert_eq!(bounds[33], (1.0, 1.0)); // w_dd (FIXED in Fast phase) assert_eq!(bounds[34], (0.01, 0.01)); // w_idle (FIXED in Fast phase) - assert_eq!(bounds[35], (0.02, 0.02)); // dd_threshold (FIXED in Fast phase) + assert_eq!(bounds[35], (0.01, 0.01)); // dd_threshold (FIXED in Fast phase) assert_eq!(bounds[36], (1.5, 1.5)); // loss_aversion (FIXED in Fast phase) assert_eq!(bounds[37], (0.0005, 0.0005)); // time_decay_rate (FIXED in Fast phase) + assert_eq!(bounds[38], (0.0, 0.0)); // q_gap_threshold (FIXED in Fast phase) } #[test] fn test_param_names() { let names = DQNParams::param_names(); - assert_eq!(names.len(), 38); // C8: 38D (added 7 reward weights) + assert_eq!(names.len(), 39); // C8: 39D (added 7 reward weights) assert_eq!(names[0], "learning_rate"); assert_eq!(names[1], "batch_size"); assert_eq!(names[2], "gamma"); @@ -3953,7 +3967,7 @@ mod tests { #[test] fn test_per_params_always_enabled() { // Test that PER is always enabled with tunable alpha/beta parameters - // 38D search space (C8: added 7 composite reward weights) + // 39D search space (C8: added 7 composite reward weights) let continuous = vec![ 3e-5_f64.ln(), 92.0, 0.92, 97_273_f64.ln(), 4.0, 24.77_f64.ln(), 0.03, 1.0, 0.6, 0.4, // 8-9: per_alpha, per_beta_start @@ -3973,7 +3987,7 @@ mod tests { 128.0, // 28: branch_hidden_dim (C5) 1.0, // 29: gradient_accumulation_steps (C6) 0.25, // 30: iqn_lambda (C7) - // C8: Composite reward weights (indices 31-37) + // C8: Composite reward weights (indices 31-38) 1.0, // 31: w_dsr 0.3, // 32: w_pnl 1.0, // 33: w_dd @@ -3981,6 +3995,7 @@ mod tests { 0.02, // 35: dd_threshold 1.5, // 36: loss_aversion 0.0005, // 37: time_decay_rate + 0.1, // 38: q_gap_threshold ]; let params = DQNParams::from_continuous(&continuous).unwrap(); @@ -3997,7 +4012,7 @@ mod tests { assert!((params.v_min - (-25.0)).abs() < 1e-6, "v_min should be -v_range"); assert!((params.v_max - 25.0).abs() < 1e-6, "v_max should be +v_range"); - // Test PER parameter bounds (min values) -- 38D + // Test PER parameter bounds (min values) -- 39D let continuous_min = vec![ 2e-5_f64.ln(), 64.0, 0.88, 50_000_f64.ln(), 1.0, 10.0_f64.ln(), 0.05, 0.5, 0.4, 0.2, // per_alpha min, per_beta_start min @@ -4025,6 +4040,7 @@ mod tests { 0.01, // 35: dd_threshold min 1.0, // 36: loss_aversion min 0.0001, // 37: time_decay_rate min + 0.0, // 38: q_gap_threshold min ]; let params_min = DQNParams::from_continuous(&continuous_min).unwrap(); assert!((params_min.per_alpha - 0.4).abs() < 1e-6); @@ -4035,7 +4051,7 @@ mod tests { assert!((params_min.v_min - (-10.0)).abs() < 1e-6, "v_min should be -v_range at min"); assert!((params_min.v_max - 10.0).abs() < 1e-6, "v_max should be +v_range at min"); - // Test PER parameter bounds (max values) -- 38D + // Test PER parameter bounds (max values) -- 39D let continuous_max = vec![ 8e-5_f64.ln(), 512.0, 0.99, 100_000_f64.ln(), 4.0, 40.0_f64.ln(), 0.5, 2.0, 0.8, 0.6, // per_alpha max, per_beta_start max @@ -4063,6 +4079,7 @@ mod tests { 0.10, // 35: dd_threshold max 3.0, // 36: loss_aversion max 0.005, // 37: time_decay_rate max + 0.5, // 38: q_gap_threshold max ]; let params_max = DQNParams::from_continuous(&continuous_max).unwrap(); assert!((params_max.per_alpha - 0.8).abs() < 1e-6); @@ -4352,7 +4369,7 @@ mod tests { fn test_qr_dqn_roundtrip_continuous() { let params = DQNParams::default(); let continuous = params.to_continuous(); - assert_eq!(continuous.len(), 38, "Should have 38 continuous dimensions"); + assert_eq!(continuous.len(), 39, "Should have 39 continuous dimensions"); let roundtrip = DQNParams::from_continuous(&continuous).unwrap(); // Default num_atoms=51 <= 100, so QR-DQN disabled, num_quantiles=64 (C51 default) assert!(!roundtrip.use_qr_dqn, "num_atoms=51 should use C51, not QR-DQN"); @@ -4363,7 +4380,7 @@ mod tests { #[test] fn test_qr_dqn_activation_threshold() { let bounds = DQNParams::continuous_bounds(); - let mut params = vec![0.0_f64; 38]; // C8: 38D + let mut params = vec![0.0_f64; 39]; // C8: 39D // Fill with valid defaults params[0] = (1e-4_f64).ln(); // learning_rate params[1] = 128.0; // batch_size @@ -4427,7 +4444,7 @@ mod tests { fn test_batch_size_respects_wide_bounds() { // Simulate PSO choosing batch_size=2048 (within VRAM-aware bounds) let bounds = DQNParams::continuous_bounds(); - let mut params = vec![0.0_f64; 38]; // C8: 38D + let mut params = vec![0.0_f64; 39]; // C8: 39D params[1] = 2048.0; // batch_size (index 1) // Fill other required params with valid defaults params[0] = (1e-4_f64).ln(); // learning_rate @@ -4550,7 +4567,7 @@ mod tests { // eval_softmax_temp removed from search space (backtest uses greedy argmax). // Verify from_continuous always sets it to fixed 1.0 regardless of input. let bounds = DQNParams::continuous_bounds(); - let mut vec = vec![0.0_f64; 38]; // C8: 38D + let mut vec = vec![0.0_f64; 39]; // C8: 39D for i in 0..38 { vec[i] = (bounds[i].0 + bounds[i].1) / 2.0; } diff --git a/crates/ml/src/trainers/dqn/config.rs b/crates/ml/src/trainers/dqn/config.rs index ed4e2804e..96bcb99ce 100644 --- a/crates/ml/src/trainers/dqn/config.rs +++ b/crates/ml/src/trainers/dqn/config.rs @@ -831,6 +831,10 @@ pub struct DQNHyperparameters { pub loss_aversion: f64, /// Position staleness rent per step held pub time_decay_rate: f64, + /// Q-value gap threshold for trade conviction filter. + /// When greedy Q(best_exposure) - Q(flat) < threshold, default to flat. + /// 0.0 = disabled. Typical range [0.0, 0.5]. + pub q_gap_threshold: f64, /// Use Huber loss instead of MSE (more robust to outliers) pub use_huber_loss: bool, /// Huber loss delta threshold (default: 1.0) @@ -1230,6 +1234,7 @@ impl DQNHyperparameters { dd_threshold: 0.02, loss_aversion: 1.5, time_decay_rate: 0.0005, + q_gap_threshold: 0.0, // disabled by default — hyperopt can search [0.0, 0.5] use_huber_loss: true, // Default: Huber loss enabled (more robust) huber_delta: 100.0, // BUG #12 FIX: Scale delta 100x for gradient explosion fix (was 1.0) use_double_dqn: true, // Default: Double DQN enabled (prevents overestimation bias) diff --git a/crates/ml/src/trainers/dqn/trainer/training_loop.rs b/crates/ml/src/trainers/dqn/trainer/training_loop.rs index b1ab210bc..29f0b66b5 100644 --- a/crates/ml/src/trainers/dqn/trainer/training_loop.rs +++ b/crates/ml/src/trainers/dqn/trainer/training_loop.rs @@ -766,6 +766,7 @@ impl DQNTrainer { fill_spread_cost_frac: 0.50, fill_spread_capture_frac: 0.50, fill_simulation_enabled: self.median_vol > 0.0, + q_gap_threshold: self.hyperparams.q_gap_threshold as f32, dsr_eta: self.hyperparams.dsr_eta as f32, n_steps: self.hyperparams.n_steps as i32, ..Default::default() diff --git a/crates/ml/src/training_profile.rs b/crates/ml/src/training_profile.rs index 97af0a765..acec30418 100644 --- a/crates/ml/src/training_profile.rs +++ b/crates/ml/src/training_profile.rs @@ -155,6 +155,8 @@ pub struct RewardSection { pub loss_aversion: Option, /// Position staleness rent per step pub time_decay_rate: Option, + /// Q-value gap threshold for trade conviction (0.0 = disabled) + pub q_gap_threshold: Option, } /// GPU experience collection parameters. @@ -245,6 +247,8 @@ pub struct SearchSpaceSection { pub dd_threshold: Option<[f64; 2]>, pub loss_aversion: Option<[f64; 2]>, pub time_decay_rate: Option<[f64; 2]>, + /// Q-value gap threshold for trade conviction filter + pub q_gap_threshold: Option<[f64; 2]>, } /// PSO optimizer configuration. @@ -275,6 +279,7 @@ pub struct PhaseFastSection { pub dd_threshold: Option, pub loss_aversion: Option, pub time_decay_rate: Option, + pub q_gap_threshold: Option, } /// Three-phase hyperopt configuration. @@ -397,6 +402,7 @@ impl HyperoptProfile { "dd_threshold" => ss.dd_threshold, "loss_aversion" => ss.loss_aversion, "time_decay_rate" => ss.time_decay_rate, + "q_gap_threshold" => ss.q_gap_threshold, _ => None, }; match val { @@ -732,6 +738,9 @@ impl DqnTrainingProfile { if let Some(v) = rw.time_decay_rate { hp.time_decay_rate = v; } + if let Some(v) = rw.q_gap_threshold { + hp.q_gap_threshold = v; + } } // [walk_forward] — no direct mapping to DQNHyperparameters; diff --git a/crates/ml/tests/dqn_action_collapse_fix_test.rs b/crates/ml/tests/dqn_action_collapse_fix_test.rs index 0a86c1f87..f8197cd8c 100644 --- a/crates/ml/tests/dqn_action_collapse_fix_test.rs +++ b/crates/ml/tests/dqn_action_collapse_fix_test.rs @@ -222,16 +222,16 @@ fn test_hyperopt_26d_includes_cql_alpha() { let bounds = DQNParams::continuous_bounds(); assert_eq!( bounds.len(), - 38, - "Search space should be 38D (31 base + 7 composite reward weights), got {}D", + 39, + "Search space should be 39D (31 base + 8 reward/policy weights), got {}D", bounds.len() ); let names = DQNParams::param_names(); assert_eq!( names.len(), - 38, - "param_names should return 38 entries, got {}", + 39, + "param_names should return 39 entries, got {}", names.len() ); @@ -296,7 +296,7 @@ fn test_hyperopt_cql_alpha_round_trip() { params.cql_alpha = 0.25; let continuous = params.to_continuous(); - assert_eq!(continuous.len(), 38); + assert_eq!(continuous.len(), 39); let reconstructed = DQNParams::from_continuous(&continuous).expect("from_continuous should succeed"); diff --git a/crates/ml/tests/ensemble_hyperopt_tests.rs b/crates/ml/tests/ensemble_hyperopt_tests.rs index cccc6ea55..bce97f764 100644 --- a/crates/ml/tests/ensemble_hyperopt_tests.rs +++ b/crates/ml/tests/ensemble_hyperopt_tests.rs @@ -75,7 +75,7 @@ //! TDD Tests for Ensemble Uncertainty Fields in Hyperopt //! //! Validates that ensemble uncertainty parameters are properly set as fixed defaults -//! in the current 38D search space (ensemble params were removed from search space, +//! in the current 39D search space (ensemble params were removed from search space, //! they are now fixed in from_continuous). use ml::hyperopt::adapters::dqn::DQNParams; @@ -128,9 +128,9 @@ fn test_dqn_params_has_ensemble_fields() { fn test_ensemble_search_space_bounds() { // Ensemble params (ensemble_size, beta_variance, etc.) are no longer in the // continuous search space — they are fixed in from_continuous(). Verify that - // the 38D search space is properly sized and ensemble defaults are set. + // the 39D search space is properly sized and ensemble defaults are set. let bounds = DQNParams::continuous_bounds(); - assert_eq!(bounds.len(), 38, "Search space should have 38 continuous parameters total (C8: +7 reward weights)"); + assert_eq!(bounds.len(), 39, "Search space should have 39 continuous parameters total (C8: +8 reward/policy weights)"); // Ensemble params are NOT in bounds — they're fixed in from_continuous. // Verify a DQNParams default still has correct ensemble defaults. @@ -144,11 +144,11 @@ fn test_ensemble_search_space_bounds() { #[test] fn test_from_continuous_validates_ensemble_params() { - // Ensemble params are now fixed in from_continuous() (not in 38D search space). + // Ensemble params are now fixed in from_continuous() (not in 39D search space). // Verify that from_continuous produces correct fixed ensemble defaults. let bounds = DQNParams::continuous_bounds(); let x: Vec = bounds.iter().map(|(lo, hi)| (lo + hi) / 2.0).collect(); - assert_eq!(x.len(), 38, "Should produce 38D midpoint vector"); + assert_eq!(x.len(), 39, "Should produce 39D midpoint vector"); let params = DQNParams::from_continuous(&x).expect("Should convert valid parameters"); @@ -164,7 +164,7 @@ fn test_from_continuous_validates_ensemble_params() { fn test_from_continuous_clamps_ensemble_params() { // Ensemble parameters are now fixed (not in search space), so clamping // is not relevant. Instead verify that from_continuous rejects wrong-length input. - let x = vec![0.0; 40]; // Wrong length (should be 38) + let x = vec![0.0; 40]; // Wrong length (should be 39) let result = DQNParams::from_continuous(&x); assert!(result.is_err(), "Should reject wrong-length (40) input"); @@ -172,10 +172,10 @@ fn test_from_continuous_clamps_ensemble_params() { let result2 = DQNParams::from_continuous(&x2); assert!(result2.is_err(), "Should reject wrong-length (31) input"); - // Correct length (38) with valid midpoints should succeed + // Correct length (39) with valid midpoints should succeed let bounds = DQNParams::continuous_bounds(); let x38: Vec = bounds.iter().map(|(lo, hi)| (lo + hi) / 2.0).collect(); - let params = DQNParams::from_continuous(&x38).expect("Should accept 38D input"); + let params = DQNParams::from_continuous(&x38).expect("Should accept 39D input"); // Ensemble params are fixed regardless of input assert_eq!(params.ensemble_size, 5.0, "Ensemble size fixed to 5.0"); } @@ -195,7 +195,7 @@ fn test_to_continuous_includes_reward_weights() { params.time_decay_rate = 0.001; let continuous = params.to_continuous(); - assert_eq!(continuous.len(), 38, "Continuous representation should have 38 values"); + assert_eq!(continuous.len(), 39, "Continuous representation should have 39 values"); // Verify reward weight positions (indices 31-37) assert_eq!(continuous[31], 1.5, "w_dsr should be at position 31"); @@ -208,7 +208,7 @@ fn test_to_continuous_includes_reward_weights() { // Ensemble params are NOT in the continuous vector (they're fixed) // Verify the vector doesn't include them - assert_eq!(continuous.len(), 38); + assert_eq!(continuous.len(), 39); } #[test]