diff --git a/crates/ml/ml/hyperopt_results/dqn/20260325_232631/best_params.json b/crates/ml/ml/hyperopt_results/dqn/20260325_232631/best_params.json new file mode 100644 index 000000000..971ef1ac6 --- /dev/null +++ b/crates/ml/ml/hyperopt_results/dqn/20260325_232631/best_params.json @@ -0,0 +1,71 @@ +{ + "learning_rate": 0.00005994999354431849, + "batch_size": 307, + "gamma": 0.9572818589229506, + "buffer_size": 66246, + "max_position_absolute": 1.1030284538648683, + "huber_delta": 17.775790174607682, + "entropy_coefficient": 0.38184099247597697, + "transaction_cost_multiplier": 1.7738774066741243, + "per_alpha": 0.45251155566697393, + "per_beta_start": 0.20130083854118397, + "dueling_hidden_dim": 128, + "n_steps": 4, + "tau": 0.008986246204051671, + "num_atoms": 101, + "v_min": -280.91110000212706, + "v_max": 280.91110000212706, + "noisy_sigma_init": 0.24583814763225417, + "minimum_profit_factor": 1.5028035348681215, + "weight_decay": 0.00010597074820791314, + "kelly_fractional": 0.5096243658522589, + "kelly_max_fraction": 0.12010947414322902, + "kelly_min_trades": 20, + "volatility_window": 23, + "use_ensemble_uncertainty": false, + "ensemble_size": 5.0, + "beta_variance": 0.5, + "beta_disagreement": 0.5, + "beta_entropy": 0.2, + "warmup_ratio": 0.0, + "curiosity_weight": 0.0, + "td_error_clamp_max": 10.0, + "batch_diversity_cooldown": 50.0, + "lr_decay_type": 0.0, + "sharpe_weight": 0.10176748614555509, + "gae_lambda": 0.95, + "noisy_sigma_initial": 0.5, + "noisy_sigma_final": 0.3, + "use_spectral_norm": true, + "use_attention": true, + "use_residual": true, + "norm_type": 1.0, + "activation_type": 1.0, + "num_quantiles": 32, + "qr_kappa": 1.0, + "iqn_lambda": 0.8479734309322158, + "spectral_norm_sigma_max": 4.8543579475657985, + "hidden_dim_base": 128, + "noisy_epsilon_floor": 0.1, + "count_bonus_coefficient": 0.23831546372779777, + "cql_alpha": 0.5059776371295834, + "eval_softmax_temp": 1.0, + "dsr_eta": 0.003947815597664894, + "branch_hidden_dim": 64, + "gradient_accumulation_steps": 1, + "w_dsr": 1.0, + "w_pnl": 0.3, + "w_dd": 1.0, + "w_idle": 0.01, + "dd_threshold": 0.01, + "loss_aversion": 1.5, + "time_decay_rate": 0.0005, + "q_gap_threshold": 0.1, + "c51_warmup_epochs": 10, + "dt_pretrain_epochs": 0, + "her_ratio": 0.48591734800375336, + "curiosity_weight_tunable": 0.09052455736721342, + "use_cvar_action_selection": 0.028233223186665324, + "cvar_alpha": 0.03929750758139713, + "min_hold_bars": 9 +} \ No newline at end of file diff --git a/crates/ml/ml/hyperopt_results/dqn/20260325_232631/campaign_summary.json b/crates/ml/ml/hyperopt_results/dqn/20260325_232631/campaign_summary.json new file mode 100644 index 000000000..8d10f4930 --- /dev/null +++ b/crates/ml/ml/hyperopt_results/dqn/20260325_232631/campaign_summary.json @@ -0,0 +1,7 @@ +{ + "trials_completed": 1, + "best_loss": 1000000.0, + "best_params_json": "{\n \"learning_rate\": 0.00005994999354431849,\n \"batch_size\": 307,\n \"gamma\": 0.9572818589229506,\n \"buffer_size\": 66246,\n \"max_position_absolute\": 1.1030284538648683,\n \"huber_delta\": 17.775790174607682,\n \"entropy_coefficient\": 0.38184099247597697,\n \"transaction_cost_multiplier\": 1.7738774066741243,\n \"per_alpha\": 0.45251155566697393,\n \"per_beta_start\": 0.20130083854118397,\n \"dueling_hidden_dim\": 128,\n \"n_steps\": 4,\n \"tau\": 0.008986246204051671,\n \"num_atoms\": 101,\n \"v_min\": -280.91110000212706,\n \"v_max\": 280.91110000212706,\n \"noisy_sigma_init\": 0.24583814763225417,\n \"minimum_profit_factor\": 1.5028035348681215,\n \"weight_decay\": 0.00010597074820791314,\n \"kelly_fractional\": 0.5096243658522589,\n \"kelly_max_fraction\": 0.12010947414322902,\n \"kelly_min_trades\": 20,\n \"volatility_window\": 23,\n \"use_ensemble_uncertainty\": false,\n \"ensemble_size\": 5.0,\n \"beta_variance\": 0.5,\n \"beta_disagreement\": 0.5,\n \"beta_entropy\": 0.2,\n \"warmup_ratio\": 0.0,\n \"curiosity_weight\": 0.0,\n \"td_error_clamp_max\": 10.0,\n \"batch_diversity_cooldown\": 50.0,\n \"lr_decay_type\": 0.0,\n \"sharpe_weight\": 0.10176748614555509,\n \"gae_lambda\": 0.95,\n \"noisy_sigma_initial\": 0.5,\n \"noisy_sigma_final\": 0.3,\n \"use_spectral_norm\": true,\n \"use_attention\": true,\n \"use_residual\": true,\n \"norm_type\": 1.0,\n \"activation_type\": 1.0,\n \"num_quantiles\": 32,\n \"qr_kappa\": 1.0,\n \"iqn_lambda\": 0.8479734309322158,\n \"spectral_norm_sigma_max\": 4.8543579475657985,\n \"hidden_dim_base\": 128,\n \"noisy_epsilon_floor\": 0.1,\n \"count_bonus_coefficient\": 0.23831546372779777,\n \"cql_alpha\": 0.5059776371295834,\n \"eval_softmax_temp\": 1.0,\n \"dsr_eta\": 0.003947815597664894,\n \"branch_hidden_dim\": 64,\n \"gradient_accumulation_steps\": 1,\n \"w_dsr\": 1.0,\n \"w_pnl\": 0.3,\n \"w_dd\": 1.0,\n \"w_idle\": 0.01,\n \"dd_threshold\": 0.01,\n \"loss_aversion\": 1.5,\n \"time_decay_rate\": 0.0005,\n \"q_gap_threshold\": 0.1,\n \"c51_warmup_epochs\": 10,\n \"dt_pretrain_epochs\": 0,\n \"her_ratio\": 0.48591734800375336,\n \"curiosity_weight_tunable\": 0.09052455736721342,\n \"use_cvar_action_selection\": 0.028233223186665324,\n \"cvar_alpha\": 0.03929750758139713,\n \"min_hold_bars\": 9\n}", + "best_checkpoint_path": null, + "total_time_seconds": 593.717371697 +} \ No newline at end of file diff --git a/crates/ml/ml/hyperopt_results/dqn/20260325_235357/best_params.json b/crates/ml/ml/hyperopt_results/dqn/20260325_235357/best_params.json new file mode 100644 index 000000000..971ef1ac6 --- /dev/null +++ b/crates/ml/ml/hyperopt_results/dqn/20260325_235357/best_params.json @@ -0,0 +1,71 @@ +{ + "learning_rate": 0.00005994999354431849, + "batch_size": 307, + "gamma": 0.9572818589229506, + "buffer_size": 66246, + "max_position_absolute": 1.1030284538648683, + "huber_delta": 17.775790174607682, + "entropy_coefficient": 0.38184099247597697, + "transaction_cost_multiplier": 1.7738774066741243, + "per_alpha": 0.45251155566697393, + "per_beta_start": 0.20130083854118397, + "dueling_hidden_dim": 128, + "n_steps": 4, + "tau": 0.008986246204051671, + "num_atoms": 101, + "v_min": -280.91110000212706, + "v_max": 280.91110000212706, + "noisy_sigma_init": 0.24583814763225417, + "minimum_profit_factor": 1.5028035348681215, + "weight_decay": 0.00010597074820791314, + "kelly_fractional": 0.5096243658522589, + "kelly_max_fraction": 0.12010947414322902, + "kelly_min_trades": 20, + "volatility_window": 23, + "use_ensemble_uncertainty": false, + "ensemble_size": 5.0, + "beta_variance": 0.5, + "beta_disagreement": 0.5, + "beta_entropy": 0.2, + "warmup_ratio": 0.0, + "curiosity_weight": 0.0, + "td_error_clamp_max": 10.0, + "batch_diversity_cooldown": 50.0, + "lr_decay_type": 0.0, + "sharpe_weight": 0.10176748614555509, + "gae_lambda": 0.95, + "noisy_sigma_initial": 0.5, + "noisy_sigma_final": 0.3, + "use_spectral_norm": true, + "use_attention": true, + "use_residual": true, + "norm_type": 1.0, + "activation_type": 1.0, + "num_quantiles": 32, + "qr_kappa": 1.0, + "iqn_lambda": 0.8479734309322158, + "spectral_norm_sigma_max": 4.8543579475657985, + "hidden_dim_base": 128, + "noisy_epsilon_floor": 0.1, + "count_bonus_coefficient": 0.23831546372779777, + "cql_alpha": 0.5059776371295834, + "eval_softmax_temp": 1.0, + "dsr_eta": 0.003947815597664894, + "branch_hidden_dim": 64, + "gradient_accumulation_steps": 1, + "w_dsr": 1.0, + "w_pnl": 0.3, + "w_dd": 1.0, + "w_idle": 0.01, + "dd_threshold": 0.01, + "loss_aversion": 1.5, + "time_decay_rate": 0.0005, + "q_gap_threshold": 0.1, + "c51_warmup_epochs": 10, + "dt_pretrain_epochs": 0, + "her_ratio": 0.48591734800375336, + "curiosity_weight_tunable": 0.09052455736721342, + "use_cvar_action_selection": 0.028233223186665324, + "cvar_alpha": 0.03929750758139713, + "min_hold_bars": 9 +} \ No newline at end of file diff --git a/crates/ml/ml/hyperopt_results/dqn/20260325_235357/campaign_summary.json b/crates/ml/ml/hyperopt_results/dqn/20260325_235357/campaign_summary.json new file mode 100644 index 000000000..e2050c66c --- /dev/null +++ b/crates/ml/ml/hyperopt_results/dqn/20260325_235357/campaign_summary.json @@ -0,0 +1,7 @@ +{ + "trials_completed": 1, + "best_loss": 1000000.0, + "best_params_json": "{\n \"learning_rate\": 0.00005994999354431849,\n \"batch_size\": 307,\n \"gamma\": 0.9572818589229506,\n \"buffer_size\": 66246,\n \"max_position_absolute\": 1.1030284538648683,\n \"huber_delta\": 17.775790174607682,\n \"entropy_coefficient\": 0.38184099247597697,\n \"transaction_cost_multiplier\": 1.7738774066741243,\n \"per_alpha\": 0.45251155566697393,\n \"per_beta_start\": 0.20130083854118397,\n \"dueling_hidden_dim\": 128,\n \"n_steps\": 4,\n \"tau\": 0.008986246204051671,\n \"num_atoms\": 101,\n \"v_min\": -280.91110000212706,\n \"v_max\": 280.91110000212706,\n \"noisy_sigma_init\": 0.24583814763225417,\n \"minimum_profit_factor\": 1.5028035348681215,\n \"weight_decay\": 0.00010597074820791314,\n \"kelly_fractional\": 0.5096243658522589,\n \"kelly_max_fraction\": 0.12010947414322902,\n \"kelly_min_trades\": 20,\n \"volatility_window\": 23,\n \"use_ensemble_uncertainty\": false,\n \"ensemble_size\": 5.0,\n \"beta_variance\": 0.5,\n \"beta_disagreement\": 0.5,\n \"beta_entropy\": 0.2,\n \"warmup_ratio\": 0.0,\n \"curiosity_weight\": 0.0,\n \"td_error_clamp_max\": 10.0,\n \"batch_diversity_cooldown\": 50.0,\n \"lr_decay_type\": 0.0,\n \"sharpe_weight\": 0.10176748614555509,\n \"gae_lambda\": 0.95,\n \"noisy_sigma_initial\": 0.5,\n \"noisy_sigma_final\": 0.3,\n \"use_spectral_norm\": true,\n \"use_attention\": true,\n \"use_residual\": true,\n \"norm_type\": 1.0,\n \"activation_type\": 1.0,\n \"num_quantiles\": 32,\n \"qr_kappa\": 1.0,\n \"iqn_lambda\": 0.8479734309322158,\n \"spectral_norm_sigma_max\": 4.8543579475657985,\n \"hidden_dim_base\": 128,\n \"noisy_epsilon_floor\": 0.1,\n \"count_bonus_coefficient\": 0.23831546372779777,\n \"cql_alpha\": 0.5059776371295834,\n \"eval_softmax_temp\": 1.0,\n \"dsr_eta\": 0.003947815597664894,\n \"branch_hidden_dim\": 64,\n \"gradient_accumulation_steps\": 1,\n \"w_dsr\": 1.0,\n \"w_pnl\": 0.3,\n \"w_dd\": 1.0,\n \"w_idle\": 0.01,\n \"dd_threshold\": 0.01,\n \"loss_aversion\": 1.5,\n \"time_decay_rate\": 0.0005,\n \"q_gap_threshold\": 0.1,\n \"c51_warmup_epochs\": 10,\n \"dt_pretrain_epochs\": 0,\n \"her_ratio\": 0.48591734800375336,\n \"curiosity_weight_tunable\": 0.09052455736721342,\n \"use_cvar_action_selection\": 0.028233223186665324,\n \"cvar_alpha\": 0.03929750758139713,\n \"min_hold_bars\": 9\n}", + "best_checkpoint_path": null, + "total_time_seconds": 9.469736839 +} \ No newline at end of file diff --git a/crates/ml/src/cuda_pipeline/gpu_backtest_evaluator.rs b/crates/ml/src/cuda_pipeline/gpu_backtest_evaluator.rs index 75c208e1d..99d4d87c4 100644 --- a/crates/ml/src/cuda_pipeline/gpu_backtest_evaluator.rs +++ b/crates/ml/src/cuda_pipeline/gpu_backtest_evaluator.rs @@ -1138,8 +1138,12 @@ impl GpuBacktestEvaluator { // ── Phase 4: Greedy action selection on chunked Q-values ───────── // Kernel signature: (q_values, out_actions, rng_states, out_q_gaps, epsilon, N, // b0, b1, b2, q_gap_threshold, portfolio_states, min_hold_bars, max_position) - let null_portfolio: u64 = 0; // NULL — backtest has no hold enforcement yet - let eval_min_hold: i32 = 0; // 0 = disabled during evaluation + // Hold enforcement handled by backtest_env_step (Layer 2), not action masking. + // The greedy argmax (epsilon=0) doesn't need masking — any hold-violating + // action is overridden by the env_step kernel. This avoids the stride-8/stride-20 + // portfolio layout incompatibility between backtest and training buffers. + let null_portfolio: u64 = 0; // NULL — Layer 2 in env_step handles holds + let eval_min_hold: i32 = 0; // 0 = disabled here (enforced in env_step instead) let eval_max_pos: f32 = 0.0; unsafe { self.stream diff --git a/crates/ml/src/hyperopt/adapters/dqn.rs b/crates/ml/src/hyperopt/adapters/dqn.rs index b36cee1ae..e5dcb4d03 100644 --- a/crates/ml/src/hyperopt/adapters/dqn.rs +++ b/crates/ml/src/hyperopt/adapters/dqn.rs @@ -2490,12 +2490,14 @@ fn calculate_diversity_penalty(action_distribution: &[f64; 3]) -> f64 { /// /// Returns penalty in [0.0, 10.0] (additive on objective, higher = worse). fn calculate_trade_insufficiency_penalty(total_trades: usize) -> f64 { + /// Minimum trades for non-degenerate trial (below this → full penalty). + const MIN_TRADES_DEGENERATE: usize = 10; const MIN_VIABLE_TRADES: usize = 20; // 20 quality trades >> 100 noise trades if total_trades == 0 { 10.0 // Model does nothing at all - } else if total_trades < 10 { - 5.0 + 5.0 * (1.0 - total_trades as f64 / 10.0) // 5.0-10.0 range + } else if total_trades < MIN_TRADES_DEGENERATE { + 5.0 + 5.0 * (1.0 - total_trades as f64 / MIN_TRADES_DEGENERATE as f64) // 5.0-10.0 range } else if total_trades < MIN_VIABLE_TRADES { 2.0 * (1.0 - total_trades as f64 / MIN_VIABLE_TRADES as f64) // 0.0-2.0 smooth } else { @@ -3736,7 +3738,8 @@ impl HyperparameterOptimizable for DQNTrainer { // when the model is confident in a directional bias. Multi-window backtest // (3 independent windows, mean - 0.5*std) already penalizes lucky single-bucket // flukes. Diversity is a soft signal, not a hard gate. - if backtest.total_trades < 10 { + const MIN_TRADES_DEGENERATE: usize = 10; + if backtest.total_trades < MIN_TRADES_DEGENERATE { info!( "DEGENERATE TRIAL: {} trades → trade_penalty={:.2} (objective={:.2})", backtest.total_trades, trade_penalty, trade_penalty @@ -3770,7 +3773,7 @@ impl HyperparameterOptimizable for DQNTrainer { // CVaR = -0.002 → no penalty (normal) // CVaR = -0.005 → penalty ≈ 2.8 (elevated) // CVaR = -0.010 → penalty ≈ 9.8 (max, dangerous) - let cvar_threshold = 0.003; + let cvar_threshold = 0.05 / (common::thresholds::time::BARS_PER_DAY).sqrt(); let cvar_penalty = ((-backtest.cvar_95 - cvar_threshold).max(0.0) * 1400.0).min(10.0); // Component 2: HFT activity score (25% weight) @@ -4529,7 +4532,8 @@ mod tests { #[test] fn test_qr_dqn_activation_threshold() { let bounds = DQNParams::continuous_bounds(); - let mut params = vec![0.0_f64; 41]; // 41D search space + let dim = DQNParams::continuous_bounds().len(); + let mut params = vec![0.0_f64; dim]; // Fill with valid defaults params[0] = (1e-4_f64).ln(); // learning_rate params[1] = 128.0; // batch_size @@ -4591,7 +4595,8 @@ mod tests { // batch_size is clamped to [64, 1024] in from_continuous — VRAM guard. // Even if PSO suggests 2048, from_continuous caps at 1024. let bounds = DQNParams::continuous_bounds(); - let mut params = vec![0.0_f64; 39]; // C8: 39D + let dim = DQNParams::continuous_bounds().len(); + let mut params = vec![0.0_f64; dim]; params[1] = 2048.0; // batch_size (index 1) — over the clamp // Fill other required params with valid defaults params[0] = (1e-4_f64).ln(); // learning_rate