diff --git a/crates/ml/src/dqn/dqn.rs b/crates/ml/src/dqn/dqn.rs index 05e406afd..617f29bc6 100644 --- a/crates/ml/src/dqn/dqn.rs +++ b/crates/ml/src/dqn/dqn.rs @@ -3997,4 +3997,120 @@ mod tests { Ok(()) } + + #[test] + fn test_batch_hierarchical_softmax_actions_smoke() -> anyhow::Result<()> { + // Smoke test: hierarchical softmax produces valid actions with full diversity + let mut config = DQNConfig::emergency_safe_defaults(); + config.state_dim = 8; + config.num_actions = 45; // 5 exposure × 3 order × 3 urgency + config.hidden_dims = vec![32, 32]; + config.use_distributional = false; + config.use_dueling = false; + config.use_iqn = false; + config.use_cql = false; + + let dqn = DQN::new(config)?; + + // Batch of 100 random states + let batch_size = 100; + let states = Tensor::rand(0.0_f32, 1.0_f32, &[batch_size, 8], &dqn.device)?; + + // Temperature 1.0: should produce diverse actions + let actions = dqn.batch_hierarchical_softmax_actions(&states, 1.0)?; + assert_eq!(actions.len(), batch_size, "Should return one action per state"); + + // All actions must be in [0, 45) + for (i, &a) in actions.iter().enumerate() { + assert!(a < 45, "Action {} out of range: {} (expected <45)", i, a); + } + + // With temp=1.0 and 100 samples, expect reasonable diversity (at least 5 unique) + let unique: std::collections::HashSet = actions.iter().copied().collect(); + assert!( + unique.len() >= 5, + "Expected ≥5 unique actions at temp=1.0, got {}/45: {:?}", + unique.len(), + unique + ); + + // Verify exposure coverage: at least 2 of 5 exposure levels used + let exposure_levels: std::collections::HashSet = + actions.iter().map(|&a| a / 9).collect(); + assert!( + exposure_levels.len() >= 2, + "Hierarchical should cover ≥2 exposure levels, got {}/5: {:?}", + exposure_levels.len(), + exposure_levels + ); + + Ok(()) + } + + #[test] + fn test_batch_hierarchical_softmax_low_temp_valid() -> anyhow::Result<()> { + // Even at low temperature, outputs must be valid indices + let mut config = DQNConfig::emergency_safe_defaults(); + config.state_dim = 8; + config.num_actions = 45; + config.hidden_dims = vec![16, 16]; + config.use_distributional = false; + config.use_dueling = false; + config.use_iqn = false; + config.use_cql = false; + + let dqn = DQN::new(config)?; + let states = Tensor::rand(0.0_f32, 1.0_f32, &[50, 8], &dqn.device)?; + + // Very low temp: near-greedy, but still valid + let actions = dqn.batch_hierarchical_softmax_actions(&states, 0.01)?; + assert_eq!(actions.len(), 50); + for &a in &actions { + assert!(a < 45, "Low-temp action out of range: {}", a); + } + + Ok(()) + } + + #[test] + fn test_batch_hierarchical_vs_flat_softmax_diversity() -> anyhow::Result<()> { + // Hierarchical should produce more exposure-level diversity than flat + let mut config = DQNConfig::emergency_safe_defaults(); + config.state_dim = 8; + config.num_actions = 45; + config.hidden_dims = vec![32, 32]; + config.use_distributional = false; + config.use_dueling = false; + config.use_iqn = false; + config.use_cql = false; + + let dqn = DQN::new(config)?; + + // Large batch for statistical power + let states = Tensor::rand(0.0_f32, 1.0_f32, &[500, 8], &dqn.device)?; + + let hierarchical = dqn.batch_hierarchical_softmax_actions(&states, 0.5)?; + let flat = dqn.batch_softmax_actions(&states, 0.5)?; + + // Both must return valid actions + assert_eq!(hierarchical.len(), 500); + assert_eq!(flat.len(), 500); + + // Count exposure-level coverage + let hier_exposures: std::collections::HashSet = + hierarchical.iter().map(|&a| a / 9).collect(); + let flat_exposures: std::collections::HashSet = + flat.iter().map(|&a| a / 9).collect(); + + // Hierarchical should cover at least as many exposure levels + // (this is statistical, not guaranteed, but with 500 samples at temp=0.5 it's reliable) + assert!( + hier_exposures.len() >= flat_exposures.len(), + "Hierarchical ({}/5 exposures) should cover ≥ flat ({}/5)", + hier_exposures.len(), + flat_exposures.len() + ); + + Ok(()) + } } diff --git a/crates/ml/src/hyperopt/adapters/dqn.rs b/crates/ml/src/hyperopt/adapters/dqn.rs index 1d02f1af1..87a545945 100644 --- a/crates/ml/src/hyperopt/adapters/dqn.rs +++ b/crates/ml/src/hyperopt/adapters/dqn.rs @@ -4033,4 +4033,86 @@ mod tests { ); } } + + #[test] + fn test_eval_softmax_temp_floor_is_half() { + // After raising the floor, verify bounds enforce [0.5, 2.0] + let bounds = DQNParams::continuous_bounds(); + let temp_bounds = bounds[26]; // eval_softmax_temp is param 26 + let min_temp = temp_bounds.0.exp(); + let max_temp = temp_bounds.1.exp(); + assert!( + (min_temp - 0.5).abs() < 0.01, + "eval_softmax_temp min should be ~0.5, got {:.4}", + min_temp + ); + assert!( + (max_temp - 2.0).abs() < 0.01, + "eval_softmax_temp max should be ~2.0, got {:.4}", + max_temp + ); + + // from_continuous should clamp to 0.5 even if value is below + let mut low_temp_vec = vec![0.0_f64; 27]; + // Set all to midpoint of bounds + for i in 0..27 { + low_temp_vec[i] = (bounds[i].0 + bounds[i].1) / 2.0; + } + low_temp_vec[26] = 0.01_f64.ln(); // Way below floor + let params = DQNParams::from_continuous(&low_temp_vec).unwrap(); + assert!( + params.eval_softmax_temp >= 0.5, + "eval_softmax_temp should be clamped to ≥0.5, got {}", + params.eval_softmax_temp + ); + } + + #[test] + fn test_diversity_short_circuit_blocks_phantom_sharpe() { + // Verify that <10 unique actions produces a positive penalty objective, + // regardless of how high the Sharpe ratio is (phantom Sharpe=2317 bug) + let metrics = DQNMetrics { + train_loss: 0.1, + val_loss: 0.05, + avg_q_value: 0.5, + final_epsilon: 0.0, + epochs_completed: 8, + avg_episode_reward: 100.0, + buy_action_pct: 0.5, + sell_action_pct: 0.5, + hold_action_pct: 0.0, + gradient_norm: 0.1, + q_value_std: 0.1, + backtest_metrics: Some(BacktestMetrics { + sharpe_ratio: 2317.0, // Phantom Sharpe! + win_rate: 100.0, + max_drawdown_pct: 0.0, + total_return_pct: 89000.0, + total_trades: 27000, + sortino_ratio: 22000.0, + calmar_ratio: 1_900_000.0, + var_95: 0.03, + cvar_95: 0.03, + beta: 0.0, + alpha: 0.0, + information_ratio: 0.0, + omega_ratio: 232000.0, + unique_actions: 2, // Only 2/45 — degenerate! + }), + }; + + let objective = ::extract_objective(&metrics); + // Must be POSITIVE (bad) — the diversity short-circuit should override the composite + assert!( + objective > 0.0, + "Degenerate trial (2/45 actions) should have positive objective, got {:.4}", + objective + ); + // Should be in the 5-10 range (graduated penalty), not -369000 + assert!( + objective < 20.0, + "Penalty should be moderate (5-10), got {:.4}", + objective + ); + } }