diff --git a/crates/ml/src/hyperopt/adapters/dqn.rs b/crates/ml/src/hyperopt/adapters/dqn.rs index ffe75d1ad..fa31da51b 100644 --- a/crates/ml/src/hyperopt/adapters/dqn.rs +++ b/crates/ml/src/hyperopt/adapters/dqn.rs @@ -2351,7 +2351,7 @@ impl HyperparameterOptimizable for DQNTrainer { // ALL trials to hit FALLBACK OBJECTIVE. With 8 epochs (~90s per trial), // it's cheaper to run all epochs and get real backtest metrics. // Long training runs (train-best, 50 epochs) still use early stopping. - early_stopping_enabled: self.epochs > 12, + early_stopping_enabled: false, q_value_floor: -5.0, min_loss_improvement_pct: 2.0, plateau_window: (self.epochs / 2).max(5), @@ -3353,21 +3353,11 @@ impl HyperparameterOptimizable for DQNTrainer { // TRADE INSUFFICIENCY PENALTY: gives PSO gradient across degenerate plateau let trade_penalty = calculate_trade_insufficiency_penalty(backtest.total_trades); - // ACTION DIVERSITY PENALTY: scaled for 5-exposure DQN action space. - // - <3/5 unique exposures: degenerate trial, short-circuit with heavy penalty - // (phantom metrics like Sharpe 2317, 100% WR from single exposure bucket) - // - 3-4/5: moderate penalty blended with composite score - // - 5/5: no penalty (healthy diversity — all exposures explored) - if backtest.unique_actions < 3 { - let diversity_ratio = backtest.unique_actions as f64 / 3.0; - // 8.0 for 1 action → ~2.67 for 2 actions (same scale as trade penalty) - let diversity_penalty = 8.0 * (1.0 - diversity_ratio); - info!( - "DEGENERATE TRIAL (low diversity): unique_actions={}/5 → penalty={:.2} (ignoring Sharpe={:.2})", - backtest.unique_actions, diversity_penalty + trade_penalty, backtest.sharpe_ratio - ); - diversity_penalty + trade_penalty - } else if backtest.total_trades < 10 { + // ACTION DIVERSITY: greedy argmax legitimately converges to 1-2 actions + // when the model is confident in a directional bias. Multi-window backtest + // (3 independent windows, mean - 0.5*std) already penalizes lucky single-bucket + // flukes. Diversity is a soft signal, not a hard gate. + if backtest.total_trades < 10 { info!( "DEGENERATE TRIAL: {} trades → trade_penalty={:.2} (objective={:.2})", backtest.total_trades, trade_penalty, trade_penalty @@ -3388,12 +3378,11 @@ impl HyperparameterOptimizable for DQNTrainer { // Component 2: HFT activity score (25% weight) let hft_activity = calculate_hft_activity_score_wave10(buy_pct, sell_pct, hold_pct); - // Graduated diversity penalty for 3-4 unique exposures: - // 3 exposures → 3.0 penalty, 5 exposures → 0.0 penalty (linear ramp) - // 5 exposures: no penalty (all exposures explored) + // Mild diversity bonus: 5/5 unique → 0.0, 1/5 → 0.8 penalty. + // Soft signal for TPE, never dominates real backtest metrics. let mid_diversity_penalty = if backtest.unique_actions < 5 { - let ratio = (backtest.unique_actions as f64 - 3.0) / 2.0; // 0.0 at 3, 1.0 at 5 - 3.0 * (1.0 - ratio.max(0.0)) + let ratio = backtest.unique_actions as f64 / 5.0; + 0.8 * (1.0 - ratio) // 0.64 for 1 action, 0.32 for 3, 0.0 for 5 } else { 0.0 }; @@ -4208,9 +4197,10 @@ mod tests { } #[test] - fn test_diversity_short_circuit_blocks_phantom_sharpe() { - // Verify that <10 unique actions produces a positive penalty objective, - // regardless of how high the Sharpe ratio is (phantom Sharpe=2317 bug) + fn test_low_diversity_adds_mild_penalty_not_hard_gate() { + // With greedy argmax, mono-action policies are valid. The diversity penalty + // should be a mild additive term (≤0.8), NOT a hard gate that overrides Sharpe. + // Multi-window backtest (3 windows, mean - 0.5*std) handles phantom Sharpe. let metrics = DQNMetrics { train_loss: 0.1, val_loss: 0.05, @@ -4224,34 +4214,30 @@ mod tests { gradient_norm: 0.1, q_value_std: 0.1, backtest_metrics: Some(BacktestMetrics { - sharpe_ratio: 2317.0, // Phantom Sharpe! - win_rate: 100.0, - max_drawdown_pct: 0.0, - total_return_pct: 89000.0, - total_trades: 27000, - sortino_ratio: 22000.0, - calmar_ratio: 1_900_000.0, - var_95: 0.03, - cvar_95: 0.03, - beta: 0.0, - alpha: 0.0, - information_ratio: 0.0, - omega_ratio: 232000.0, - unique_actions: 2, // Only 2/5 — degenerate! + sharpe_ratio: 3.0, // Realistic good Sharpe + win_rate: 55.0, + max_drawdown_pct: 5.0, + total_return_pct: 12.0, + total_trades: 200, + sortino_ratio: 4.0, + calmar_ratio: 2.4, + var_95: -0.02, + cvar_95: -0.03, + beta: 0.1, + alpha: 0.05, + information_ratio: 1.5, + omega_ratio: 2.0, + unique_actions: 1, // Mono-action greedy — valid! }), }; let objective = ::extract_objective(&metrics); - // Must be POSITIVE (bad) — the diversity short-circuit should override the composite + // With good Sharpe and 200 trades, objective should be NEGATIVE (good) + // even with mono-action. The 0.64 diversity penalty is far smaller than + // the composite score benefit from Sharpe=3.0. assert!( - objective > 0.0, - "Degenerate trial (2/5 exposures) should have positive objective, got {:.4}", - objective - ); - // Should be in the 5-10 range (graduated penalty), not -369000 - assert!( - objective < 20.0, - "Penalty should be moderate (5-10), got {:.4}", + objective < 0.0, + "Good Sharpe with mono-action should still produce negative (good) objective, got {:.4}", objective ); }