diff --git a/crates/ml/src/dqn/reward.rs b/crates/ml/src/dqn/reward.rs index 15ca28027..90056b564 100644 --- a/crates/ml/src/dqn/reward.rs +++ b/crates/ml/src/dqn/reward.rs @@ -208,7 +208,7 @@ impl Default for RewardConfig { movement_threshold: Decimal::try_from(0.01).unwrap_or(Decimal::ZERO), // 1% matches data distribution hold_penalty_weight: Decimal::try_from(0.01).unwrap_or(Decimal::ZERO), // 1% default penalty diversity_weight: Decimal::try_from(-0.1).unwrap_or(Decimal::ZERO), // -0.1 default (100x stronger than hold_reward) - enable_normalization: true, // Fix #3: Enable normalization (gradient stability) + enable_normalization: false, // Disabled: clipping destroys economic signal magnitude use_percentage_pnl: true, // Bug #17 fix: scale-invariant percentage returns circuit_breaker_config: CircuitBreakerConfig::default(), triple_barrier_profit_bonus: Decimal::try_from(0.5).unwrap_or(Decimal::ZERO), // 50% bonus for hitting profit target @@ -502,8 +502,8 @@ impl RewardFunction { let base_reward = match legacy_action { TradingAction::Buy | TradingAction::Sell => { - // Calculate P&L-based reward - let pnl_reward = self.calculate_pnl_reward(current_state, next_state)?; + // Calculate P&L-based reward scaled by the agent's target exposure + let pnl_reward = self.calculate_pnl_reward(current_state, next_state, action.target_exposure())?; // Calculate risk penalty let risk_penalty = self.calculate_risk_penalty(next_state); @@ -682,6 +682,7 @@ impl RewardFunction { &self, current_state: &TradingState, next_state: &TradingState, + target_exposure: f64, ) -> Result { // Validate portfolio_features length (defensive check) if current_state.portfolio_features.len() < 1 { @@ -724,13 +725,15 @@ impl RewardFunction { ); } - // BUG #40 FIX: Remove 100x PnL scaling for C51 compatibility - // Root cause: 100x multiplier pushes rewards outside C51's V-range [-2, 2] - // This causes Q-value explosions to ±60,000 in distributional RL - // Solution: Use raw percentage returns (±0.02) which stay within bounds - // C51 atoms are calibrated for [-2, 2], and normalization handles any scaling - // Expected: Q-values converge to reward scale, not the other way around - pct_return + // Scale PnL by agent's target exposure so the reward reflects the + // agent's intended position: + // Long100 (+1.0): full market return + // Short100 (-1.0): inverted market return (profit when market drops) + // Flat (0.0): zero PnL (no skin in the game) + // Long50/Short50 (±0.5): half-size position + let exposure_decimal = Decimal::try_from(target_exposure) + .unwrap_or(Decimal::ONE); + pct_return * exposure_decimal } } else { // Absolute dollar change (original implementation) diff --git a/crates/ml/src/evaluation/engine.rs b/crates/ml/src/evaluation/engine.rs index b31d3ad84..c50211426 100644 --- a/crates/ml/src/evaluation/engine.rs +++ b/crates/ml/src/evaluation/engine.rs @@ -64,6 +64,8 @@ pub struct EvaluationEngine { pub exposure_entry_price: f32, /// Bar index where current exposure was first entered pub exposure_entry_bar: usize, + /// Override fee rate for all trades (None = use action's transaction_cost()) + fee_rate_override: Option, } impl EvaluationEngine { @@ -82,6 +84,22 @@ impl EvaluationEngine { current_exposure: 0.0, exposure_entry_price: 0.0, exposure_entry_bar: 0, + fee_rate_override: None, + } + } + + /// Create evaluation engine with explicit fee rate override (in decimal, e.g. 0.00001 for 0.1 bps) + pub fn new_with_fee_rate(initial_capital: f32, kelly_fraction: f64, fee_rate: f64) -> Self { + Self { + current_position: None, + trades: Vec::new(), + initial_capital, + action_counts: [0, 0, 0], + kelly_fraction, + current_exposure: 0.0, + exposure_entry_price: 0.0, + exposure_entry_bar: 0, + fee_rate_override: Some(fee_rate), } } @@ -221,7 +239,7 @@ impl EvaluationEngine { } // Record trade for the exposure change - let fee_rate = action.transaction_cost(); // 0.0015 Market, 0.0005 Limit, 0.001 IoC + let fee_rate = self.fee_rate_override.unwrap_or_else(|| action.transaction_cost()); // PnL from the portion being closed (if reducing or reversing) let closing_size = if self.current_exposure.abs() > EPSILON @@ -289,7 +307,8 @@ impl EvaluationEngine { }; let size = self.current_exposure.abs() * self.kelly_fraction; let gross_pnl = price_diff * direction_sign * size as f32; - let tx_cost = bar.close.abs() as f64 * size * 0.0015; // Market order for forced close + let close_fee = self.fee_rate_override.unwrap_or(0.0015); + let tx_cost = bar.close.abs() as f64 * size * close_fee; self.trades.push(Trade { entry_bar_idx: self.exposure_entry_bar, diff --git a/crates/ml/src/hyperopt/adapters/dqn.rs b/crates/ml/src/hyperopt/adapters/dqn.rs index aa16731dc..ffe75d1ad 100644 --- a/crates/ml/src/hyperopt/adapters/dqn.rs +++ b/crates/ml/src/hyperopt/adapters/dqn.rs @@ -2871,7 +2871,8 @@ impl HyperparameterOptimizable for DQNTrainer { // Reset portfolio state for this window internal_trainer.set_portfolio_for_backtest(0.0, 0.0, 0.0); - let mut engine = EvaluationEngine::new_with_kelly(eval_capital, kelly_fraction); + let backtest_fee_rate = self.tx_cost_bps * 0.0001; // bps → decimal + let mut engine = EvaluationEngine::new_with_fee_rate(eval_capital, kelly_fraction, backtest_fee_rate); let num_chunks = win_len.div_ceil(EVAL_CHUNK_SIZE); let mut ohlcv_bars = Vec::with_capacity(win_len); @@ -2922,7 +2923,7 @@ impl HyperparameterOptimizable for DQNTrainer { })?; let action_indices = - agent_guard.batch_hierarchical_softmax_actions(&batch_tensor, params.eval_softmax_temp)?; + agent_guard.batch_greedy_actions(&batch_tensor)?; // 3. Sequential trade simulation on CPU for (i, &action_idx) in action_indices.iter().enumerate() {