diff --git a/crates/ml/src/cuda_pipeline/c51_grad_kernel.cu b/crates/ml/src/cuda_pipeline/c51_grad_kernel.cu index f9a031f95..6a3f4dacd 100644 --- a/crates/ml/src/cuda_pipeline/c51_grad_kernel.cu +++ b/crates/ml/src/cuda_pipeline/c51_grad_kernel.cu @@ -133,12 +133,10 @@ extern "C" __global__ void c51_grad_kernel( branch_base += batch_size * A_d * num_atoms; } - /* Q-mean drift regularization: ADAPTIVE quadratic penalty. - * Small drift → tiny penalty (don't interfere). Large drift → hard correction. - * lambda=0.1 (scaled up from 0.01 for trade-level reward's smaller Q-values). */ - float q_mean_ema = q_mean_ema_ptr[0]; - float drift_penalty = 0.1f * q_mean_ema * fabsf(q_mean_ema); - d_val_sum += drift_penalty; + /* Q-mean drift: handled adaptively by E1 enrichment (Q-value reality check). + * Hardcoded penalty removed — E1 computes bias correction from actual eval + * performance and adapts to changing reward scales automatically. */ + (void)q_mean_ema_ptr; /* parameter kept for ABI compatibility */ /* Single deterministic write — no atomicAdd */ d_value_logits[b * num_atoms + j] = d_val_sum; diff --git a/crates/ml/src/cuda_pipeline/reward_shaping_kernel.cu b/crates/ml/src/cuda_pipeline/reward_shaping_kernel.cu index 02e2c6d96..9db5491a1 100644 --- a/crates/ml/src/cuda_pipeline/reward_shaping_kernel.cu +++ b/crates/ml/src/cuda_pipeline/reward_shaping_kernel.cu @@ -34,6 +34,10 @@ extern "C" __global__ void reward_rank_normalize( * so high-return trades during volatile periods get lower * rank weight than the same return during calm periods. */ float r_i = (std_ema > 1e-6f) ? (raw_r - return_mean_ema) / std_ema : raw_r; + + /* Skip near-zero rewards (holding bars). Only rank actual trade P&L. */ + int is_trade = (fabsf(r_i) > 0.001f) ? 1 : 0; + float abs_i = fabsf(r_i); __shared__ float tile[TILE_SIZE]; @@ -65,6 +69,13 @@ extern "C" __global__ void reward_rank_normalize( /* Rank in [0, 1], sign of Sharpe contribution preserved */ float rank = (float)count / (float)N; float sign = (r_i > 0.0f) ? 1.0f : (r_i < 0.0f) ? -1.0f : 0.0f; + + /* Holding bars: passthrough zero, don't rank */ + if (!is_trade) { + rewards_out[i] = 0.0f; + return; + } + /* G13: input is already in Sharpe units — no rescale by std_ema needed */ rewards_out[i] = sign * rank; } diff --git a/crates/ml/src/trainers/dqn/financials.rs b/crates/ml/src/trainers/dqn/financials.rs index 9f0b6c77b..44ce2c75b 100644 --- a/crates/ml/src/trainers/dqn/financials.rs +++ b/crates/ml/src/trainers/dqn/financials.rs @@ -7,6 +7,7 @@ use crate::cuda_pipeline::gpu_experience_collector::TradeStats; #[derive(Debug, Clone, Default)] pub(crate) struct EpochFinancials { pub sharpe: f64, + pub sharpe_raw: f64, pub sortino: f64, pub win_rate: f64, pub max_drawdown: f64, @@ -91,7 +92,7 @@ pub(crate) fn compute_epoch_financials( }; let trade_annualization = trades_per_year.max(1.0).sqrt(); - let (sharpe, sortino) = if total_trades > 1 { + let (sharpe, sharpe_raw, sortino) = if total_trades > 1 { // Per-trade mean and variance from GPU-aggregated sums let mean = trade_stats.sum_returns / n_trades; let var = (trade_stats.sum_sq_returns / n_trades) - mean * mean; @@ -103,6 +104,8 @@ pub(crate) fn compute_epoch_financials( 0.0 }; + let sharpe_raw = if std > 1e-10 { mean / std } else { 0.0 }; + // Sortino: downside deviation from losing trades only. // sum_losses is sum of |returns| for losers. We need sum of squared // returns for losers — approximate from loss_count * (avg_loss)^2. @@ -120,9 +123,9 @@ pub(crate) fn compute_epoch_financials( 0.0 }; - (s, sort) + (s, sharpe_raw, sort) } else { - (0.0, 0.0) + (0.0, 0.0, 0.0) }; // MaxDD from step_returns equity curve over last 10K bars, capped at 100%. @@ -182,6 +185,7 @@ pub(crate) fn compute_epoch_financials( EpochFinancials { sharpe, + sharpe_raw, sortino, win_rate, max_drawdown: max_dd, diff --git a/crates/ml/src/trainers/dqn/trainer/training_loop.rs b/crates/ml/src/trainers/dqn/trainer/training_loop.rs index 02fadaf1f..e073722b1 100644 --- a/crates/ml/src/trainers/dqn/trainer/training_loop.rs +++ b/crates/ml/src/trainers/dqn/trainer/training_loop.rs @@ -2017,6 +2017,8 @@ impl DQNTrainer { "Epoch {}/{}: val_Sharpe={:.2} (deterministic backtest on fixed validation window)", epoch + 1, self.hyperparams.epochs, val_sharpe, ); + let val_sharpe_raw = val_sharpe / (self.hyperparams.bars_per_day * 252.0).sqrt(); + info!(" val_Sharpe_raw={:.6} (un-annualized per-bar)", val_sharpe_raw); // Prometheus metrics training_metrics::set_epoch("dqn", "current", (epoch + 1) as f64); @@ -2196,9 +2198,10 @@ impl DQNTrainer { financials.buy_pct, financials.sell_pct, financials.hold_pct, ); info!( - "Epoch {}/{}: Sharpe={:.2} WinRate={:.1}% MaxDD={:.3}% PF={:.2} Return={:+.2}% Trades={}", + "Epoch {}/{}: Sharpe={:.2} Sharpe_raw={:.6} WinRate={:.1}% MaxDD={:.3}% PF={:.2} Return={:+.2}% Trades={}", epoch + 1, self.hyperparams.epochs, - financials.sharpe, financials.win_rate * 100.0, + financials.sharpe, financials.sharpe_raw, + financials.win_rate * 100.0, financials.max_drawdown * 100.0, financials.profit_factor, financials.total_return * 100.0, financials.total_trades, );