diff --git a/crates/ml-dqn/src/evaluation/engine.rs b/crates/ml-dqn/src/evaluation/engine.rs index 1231f2da0..87b6bb3e9 100644 --- a/crates/ml-dqn/src/evaluation/engine.rs +++ b/crates/ml-dqn/src/evaluation/engine.rs @@ -66,7 +66,7 @@ pub struct EvaluationEngine { pub exposure_entry_bar: usize, /// Override fee rate for all trades (None = use action's `transaction_cost()`) fee_rate_override: Option, - /// Running equity (initial_capital + sum of all trade PnLs) + /// Running equity (`initial_capital` + sum of all trade `PnLs`) pub running_equity: f32, /// True when running equity hit zero — no further trades are executed pub margin_called: bool, diff --git a/crates/ml/src/hyperopt/adapters/dqn.rs b/crates/ml/src/hyperopt/adapters/dqn.rs index 5df3fe556..a008e8e75 100644 --- a/crates/ml/src/hyperopt/adapters/dqn.rs +++ b/crates/ml/src/hyperopt/adapters/dqn.rs @@ -2076,53 +2076,32 @@ fn calculate_diversity_penalty(action_distribution: &[f64; 3]) -> f64 { /// Calculate stability penalty from gradient norms and Q-value volatility /// -/// This function detects training instability via two key metrics: -/// 1. **Gradient norm**: Measures gradient explosion risk -/// 2. **Q-value std**: Measures Q-value volatility across epochs +/// Provides smooth PSO gradient signal for training instability detection. +/// Uses log-scale for gradient norm (spans orders of magnitude) and linear +/// ramp for Q-value std. /// -/// # Arguments +/// # Gradient norm thresholds /// -/// * `gradient_norm` - Average gradient norm during training -/// * `q_value_std` - Standard deviation of Q-values across epochs +/// The reported gradient_norm is the **average raw (pre-clip) gradient norm** +/// across all training steps. Gradient clipping is at 10.0, so: +/// - Normal: 1-20 (clip rarely or mildly engaged) +/// - Elevated: 20-200 (clip consistently engaged, training struggling) +/// - Unstable: 200+ (severe gradient pressure, model not converging) /// -/// # Returns +/// Threshold=50 starts the penalty where the clip is engaged 5x on average. +/// Log-scale ramp: provides smooth gradient across 50→5000 range. /// -/// Combined penalty (0.0 = stable, higher = unstable) +/// # Q-value std thresholds /// -/// # Penalty Formula -/// -/// ```text -/// gradient_penalty = if gradient_norm > 50.0: -/// (gradient_norm - 50.0) / 50.0 -/// else: -/// 0.0 -/// -/// q_value_penalty = if q_value_std > 100.0: -/// (q_value_std - 100.0) / 100.0 -/// else: -/// 0.0 -/// -/// stability_penalty = gradient_penalty + q_value_penalty -/// ``` -/// -/// # Why These Thresholds? -/// -/// - **gradient_norm = 20,000.0**: Empirical stability boundary from DQN training -/// - Normal range: 0.1 to 10.0 (healthy gradients) -/// - Warning range: 10.0 to 20,000.0 (elevated but acceptable) -/// - Danger zone: >20,000.0 (gradient explosion likely) -/// - Complements Bug #1 fix (gradient clipping at max_norm=10.0) -/// - Note: Previous threshold (50.0) was overly aggressive; Q-values remain stable at higher gradient norms -/// -/// - **q_value_std = 100.0**: Acceptable Q-value volatility range -/// - Normal range: 0.1 to 10.0 (stable Q-values) -/// - Warning range: 10.0 to 100.0 (elevated volatility) -/// - Danger zone: >100.0 (Q-value oscillation/instability) +/// Q-value std is the standard deviation of per-epoch average Q-values. +/// With DSR and v_range=[10,50], stable training shows std 0.5-5. +/// - Normal: 0-15 +/// - Elevated: 15-100 (Q-values fluctuating significantly across epochs) +/// - Unstable: 100+ (Q-function hasn't converged) /// /// # Edge Cases /// -/// - NaN/Inf gradient_norm: Assigns maximum penalty (gradient_norm = f64::MAX) -/// - NaN/Inf q_value_std: Assigns maximum penalty (q_value_std = f64::MAX) +/// - NaN/Inf: Assigns maximum penalty (f64::MAX triggers cap) /// /// # Wave 10 Helper Functions /// @@ -2141,181 +2120,19 @@ fn calculate_diversity_penalty(action_distribution: &[f64; 3]) -> f64 { /// - Wave 9 analysis: /tmp/ml_training/wave9_production/WAVE9_ANALYSIS_REPORT.md /// - Wave 10 design: /tmp/wave10_incentive_design.md (663 lines, 10 academic citations) -/// Calculate smooth transition weight using tanh function +/// Calculate HFT activity score /// -/// This prevents local minima at tier boundaries by creating smooth gradients -/// across the Sharpe spectrum. Without this, optimizer could get stuck at -/// tier edges (e.g., Sharpe 2.0 exactly). -/// -/// ## Arguments -/// -/// * `value` - Current Sharpe ratio -/// * `threshold` - Tier threshold (e.g., 2.5 for "Excellent") -/// * `steepness` - Transition slope (2.0 = gradual, prevents oscillation) -/// -/// ## Returns -/// -/// Weight in [0, 1]: 0 = below threshold, 0.5 = at threshold, 1 = above threshold -/// -/// ## References -/// -/// - Potential-Based Reward Shaping (arXiv 2025): Smooth transitions prevent local minima -/// - Wave 10 design: Lines 525-531 (smooth_transition implementation) -fn smooth_transition(value: f64, threshold: f64, steepness: f64) -> f64 { - 0.5 * (1.0 + ((value - threshold) * steepness).tanh()) -} - -/// Calculate exponential Sharpe+Drawdown incentive (Wave 10 Component 1) -/// -/// This is the core innovation of Wave 10: massively reward elite performance -/// via exponential bonuses while heavily penalizing wasteful strategies (Sharpe <1.0). -/// -/// ## Tier Structure (Based on Industry Benchmarks) -/// -/// | Tier | Sharpe | Bonus | Drawdown Limit | Example Strategy | -/// |------|--------|-------|----------------|------------------| -/// | **Elite** | ≥3.5 | e^x growth | ≤3% | Renaissance Technologies | -/// | **Excellent** | ≥2.5 | x² growth | ≤5% | HFT market-making | -/// | **Good** | ≥2.0 | Linear | ≤7% | Trend-following AI | -/// | **Mediocre** | ≥1.0 | None (0.0) | ≤10% | Basic HOLD-heavy | -/// | **Wasteful** | <1.0 | Heavy penalty | N/A | Losing money | -/// -/// ## Arguments -/// -/// * `sharpe` - Sharpe ratio from backtest -/// * `drawdown_pct` - Maximum drawdown percentage (e.g., 5.0 = 5%) -/// -/// ## Returns -/// -/// Combined incentive score in [-10.0, 10.0] (higher = better performance) -/// - Positive: Profitable strategies (Sharpe >1.0) -/// - Negative: Wasteful strategies (Sharpe <1.0) or high risk (drawdown >7%) -/// -/// ## Mathematical Formulation -/// -/// ```text -/// sharpe_component = -/// elite_weight * 10.0 * e^(sharpe - 3.5) + -/// excellent_weight * 5.0 * (sharpe - 2.5)² + -/// good_weight * 2.0 * (sharpe - 2.0) + -/// mediocre_weight * 0.0 + -/// (1 - mediocre_weight) * (-10.0 * (1.0 - sharpe)) -/// -/// drawdown_penalty = { -/// 0.0 if DD ≤ 3% (elite) -/// -1.0 * (DD - 3) if DD ≤ 5% (excellent) -/// -2 - 2*(DD - 5) if DD ≤ 7% (good) -/// -6 - 4*(DD - 7) if DD ≤ 10% (mediocre) -/// -18 - 10*(DD-10) if DD > 10% (catastrophic) -/// } -/// -/// combined = clamp(sharpe_component + drawdown_penalty, -10, 10) -/// ``` -/// -/// ## Example Outcomes -/// -/// | Sharpe | Drawdown | Incentive | Tier | Wave 9 (linear) | Improvement | -/// |--------|----------|-----------|------|-----------------|-------------| -/// | 0.5 | 10% | -23.0 → -10.0 | Wasteful | -0.25 | 40x penalty | -/// | 1.0 | 7% | -2.0 | Mediocre | -0.50 | 4x penalty | -/// | 2.0 | 7% | 0.0 | Good | -1.00 | Break-even | -/// | 2.5 | 5% | 3.0 | Excellent | -1.25 | 4.25x bonus | -/// | 3.0 | 3% | 8.0 | Elite | -1.50 | 9.5x bonus | -/// | 4.0 | 2% | 10.0 (capped) | Elite++ | -2.00 | 12x bonus | -/// -/// ## References -/// -/// - Wave 9 result: Sharpe 0.9876 = 47% of industry "Good" standard (2.1) -/// - Wave 10 target: Sharpe ≥2.5 (Excellent tier, 2.5x improvement) -/// - Design doc: /tmp/wave10_incentive_design.md lines 534-580 -fn calculate_exponential_sharpe_incentive(sharpe: f64, drawdown_pct: f64) -> f64 { - // Tier thresholds (based on industry benchmarks) - let elite_threshold = 3.5; - let excellent_threshold = 2.5; - let good_threshold = 2.0; - let mediocre_threshold = 1.0; - - // Smooth transition weights (tanh-based, steepness=2.0) - let elite_weight = smooth_transition(sharpe, elite_threshold, 2.0); - let excellent_weight = smooth_transition(sharpe, excellent_threshold, 2.0); - let good_weight = smooth_transition(sharpe, good_threshold, 2.0); - let mediocre_weight = smooth_transition(sharpe, mediocre_threshold, 2.0); - - // Tier bonuses (exponential for elite, quadratic for excellent) - let elite_bonus = 10.0 * (sharpe - elite_threshold).exp(); // e^x growth - let excellent_bonus = 5.0 * (sharpe - excellent_threshold).powi(2); // x² growth - let good_bonus = 2.0 * (sharpe - good_threshold); // Linear growth - let mediocre_bonus = 0.0; // Baseline (no bonus) - let wasteful_penalty = -10.0 * (mediocre_threshold - sharpe); // Heavy penalty - - // Weighted sum (smooth blending across tiers) - let sharpe_component = elite_weight * elite_bonus - + excellent_weight * excellent_bonus - + good_weight * good_bonus - + mediocre_weight * mediocre_bonus - + (1.0 - mediocre_weight) * wasteful_penalty; - - // Drawdown penalty (exponential for high risk) - let drawdown_penalty = if drawdown_pct <= 3.0 { - 0.0 // Elite tier - } else if drawdown_pct <= 5.0 { - -(drawdown_pct - 3.0) // Excellent tier - } else if drawdown_pct <= 7.0 { - -2.0 - 2.0 * (drawdown_pct - 5.0) // Good tier - } else if drawdown_pct <= 10.0 { - -6.0 - 4.0 * (drawdown_pct - 7.0) // Mediocre tier - } else { - -18.0 - 10.0 * (drawdown_pct - 10.0) // Catastrophic - }; - - // Combined score (Sharpe + Drawdown) - let combined = sharpe_component + drawdown_penalty; - - // Clamp to prevent extreme outliers - combined.clamp(-10.0, 10.0) -} - -/// Calculate HFT activity score (Wave 10 Component 2) -/// -/// Preserved from Wave 9 with standalone extraction for Wave 10 objective. /// Penalizes models that don't actively trade (high HOLD%), rewards balanced BUY/SELL activity. /// /// ## Arguments /// -/// * `buy_pct` - BUY action percentage [0.0, 1.0] -/// * `sell_pct` - SELL action percentage [0.0, 1.0] -/// * `hold_pct` - HOLD action percentage [0.0, 1.0] +/// * `buy_pct` - BUY action percentage [0, 100] (e.g. 27.0 = 27%) +/// * `sell_pct` - SELL action percentage [0, 100] +/// * `hold_pct` - HOLD action percentage [0, 100] /// /// ## Returns /// /// Activity score in [-5.0, 2.0] (higher = more active trading) -/// - Negative: Insufficient action diversity (BUY or SELL <15%) -/// - Positive: High BUY/SELL ratio (rewards active trading) -/// -/// ## Logic -/// -/// ```text -/// if BUY < 15% OR SELL < 15%: -/// score = -5.0 * (15% - min(BUY, SELL)) / 15% // Penalize missing actions -/// else: -/// buy_sell_ratio = (BUY + SELL) / (HOLD + 1e-6) -/// score = 2.0 * min(buy_sell_ratio / 3.0, 1.0) // Cap at 3:1 ratio -/// ``` -/// -/// ## Example Outcomes -/// -/// | BUY | SELL | HOLD | Score | Interpretation | -/// |-----|------|------|-------|----------------| -/// | 27% | 55% | 18% | +2.0 | ⭐ Best (Wave 9 Trial 20) | -/// | 30% | 30% | 40% | +1.0 | Good activity | -/// | 5% | 50% | 45% | -3.3 | Missing BUY action | -/// | 0% | 0% | 100% | -5.0 | Pure HOLD (wasteful) | -/// -/// ## References -/// -/// - "Nothing costs money" principle: Infrastructure runs 24/7, HOLD = idle costs -/// - Wave 9 best trial: 81.8% active (BUY 27.3%, SELL 54.5%, HOLD 18.1%) -/// - Design doc: /tmp/wave10_incentive_design.md lines 582-596 /// Calculate graduated penalty for insufficient trade count. /// @@ -2365,23 +2182,27 @@ fn calculate_stability_penalty(gradient_norm: f64, q_value_std: f64) -> f64 { f64::MAX }; - // Penalty for gradient explosion (threshold: 20,000.0) - // Rationale: Q-values are stable, gradient spikes are false alarms - // Previous threshold (50.0) was overly aggressive - let gradient_penalty = if gradient_norm > 20000.0 { - (gradient_norm - 20000.0) / 20000.0 + // Gradient penalty: log-scale ramp above threshold=50. + // Gradient clipping is at 10.0; avg raw grad norm of 50 means the clip + // is engaged 5x on average — training is struggling to converge. + // Log scale gives smooth PSO gradient across 50→5000 range. + // Capped at 3.0 to prevent catastrophic failures from dominating + // (those are handled by completion_penalty and trade_penalty instead). + let gradient_penalty = if gradient_norm > 50.0 { + (gradient_norm / 50.0).ln().min(3.0) } else { 0.0 }; - // Penalize Q-value std > 100.0 (indicates high volatility) - let q_value_penalty = if q_value_std > 100.0 { - (q_value_std - 100.0) / 100.0 + // Q-value std penalty: linear ramp above threshold=15. + // With DSR and v_range=[10,50], stable Q-value std ≈ 0.5-5. + // Std > 15 means Q-values are fluctuating significantly across epochs. + let q_value_penalty = if q_value_std > 15.0 { + ((q_value_std - 15.0) / 85.0).min(3.0) } else { 0.0 }; - // Return combined penalty (will be weighted 20% in final objective) gradient_penalty + q_value_penalty } diff --git a/crates/ml/src/trainers/dqn/trainer.rs b/crates/ml/src/trainers/dqn/trainer.rs index 1ed1184aa..62f799b58 100644 --- a/crates/ml/src/trainers/dqn/trainer.rs +++ b/crates/ml/src/trainers/dqn/trainer.rs @@ -4544,6 +4544,7 @@ impl DQNTrainer { #[cfg(not(feature = "cuda"))] { let _ = (&batch_q_values, &branching_q_tensors, epsilon, batch_size); + #[allow(clippy::needless_return)] return Err(anyhow::anyhow!("Batch action selection requires CUDA — enable the `cuda` feature")); }