diff --git a/config/training/dqn-hyperopt.toml b/config/training/dqn-hyperopt.toml index c83ede1f5..96f04e26e 100644 --- a/config/training/dqn-hyperopt.toml +++ b/config/training/dqn-hyperopt.toml @@ -87,6 +87,11 @@ q_gap_threshold = [0.0, 0.5] # 0.0=trade every bar, 0.5=high conviction cea_weight = [0.1, 1.0] order_credit_weight = [0.0, 0.5] risk_efficiency_weight = [0.0, 0.5] +urgency_credit_weight = [0.0, 0.5] +exit_timing_weight = [0.0, 0.2] +ofi_reward_weight = [0.0, 0.5] +kelly_sizing_weight = [0.0, 0.5] +reward_noise_scale = [0.01, 0.1] [experience] initial_capital = 100000.0 @@ -114,6 +119,11 @@ q_gap_threshold = 0.1 cea_weight = 0.3 order_credit_weight = 0.1 risk_efficiency_weight = 0.1 +urgency_credit_weight = 0.1 +exit_timing_weight = 0.05 +ofi_reward_weight = 0.2 +kelly_sizing_weight = 0.1 +reward_noise_scale = 0.05 [fixed] @@ -145,3 +155,8 @@ q_gap_threshold = 0.1 cea_weight = 0.3 order_credit_weight = 0.1 risk_efficiency_weight = 0.1 +urgency_credit_weight = 0.1 +exit_timing_weight = 0.05 +ofi_reward_weight = 0.2 +kelly_sizing_weight = 0.1 +reward_noise_scale = 0.05 diff --git a/config/training/dqn-localdev.toml b/config/training/dqn-localdev.toml index 7b9e0764b..79e3ec8ea 100644 --- a/config/training/dqn-localdev.toml +++ b/config/training/dqn-localdev.toml @@ -120,3 +120,8 @@ dd_threshold = 0.02 cea_weight = 0.3 order_credit_weight = 0.1 risk_efficiency_weight = 0.1 +urgency_credit_weight = 0.1 +exit_timing_weight = 0.05 +ofi_reward_weight = 0.2 +kelly_sizing_weight = 0.1 +reward_noise_scale = 0.05 diff --git a/config/training/dqn-production.toml b/config/training/dqn-production.toml index 29d65673d..f2395a216 100644 --- a/config/training/dqn-production.toml +++ b/config/training/dqn-production.toml @@ -131,3 +131,8 @@ dd_threshold = 0.02 cea_weight = 0.3 order_credit_weight = 0.1 risk_efficiency_weight = 0.1 +urgency_credit_weight = 0.1 +exit_timing_weight = 0.05 +ofi_reward_weight = 0.2 +kelly_sizing_weight = 0.1 +reward_noise_scale = 0.05 diff --git a/config/training/dqn-smoketest.toml b/config/training/dqn-smoketest.toml index fd2406d70..1923865ce 100644 --- a/config/training/dqn-smoketest.toml +++ b/config/training/dqn-smoketest.toml @@ -127,6 +127,11 @@ dd_threshold = 0.02 cea_weight = 0.3 order_credit_weight = 0.1 risk_efficiency_weight = 0.1 +urgency_credit_weight = 0.1 +exit_timing_weight = 0.05 +ofi_reward_weight = 0.2 +kelly_sizing_weight = 0.1 +reward_noise_scale = 0.05 [walk_forward] step_fraction = 0.25 diff --git a/crates/ml/src/cuda_pipeline/experience_kernels.cu b/crates/ml/src/cuda_pipeline/experience_kernels.cu index 2dbf5bc92..b2a5ade82 100644 --- a/crates/ml/src/cuda_pipeline/experience_kernels.cu +++ b/crates/ml/src/cuda_pipeline/experience_kernels.cu @@ -866,6 +866,11 @@ extern "C" __global__ void experience_env_step( * at episode end (done=1) and added to final reward. */ float* __restrict__ position_histogram, float position_entropy_weight, /* #19: reward += weight * H(histogram). 0=disabled. */ + float urgency_credit_weight, /* v7 gem: urgency branch attribution */ + float exit_timing_weight, /* v7 gem: exit timing quality */ + float ofi_reward_weight, /* v7 gem: OFI-weighted reward confidence */ + float kelly_sizing_weight, /* v7 gem: Kelly-optimal sizing signal */ + float reward_noise_scale, /* v7 gem: reward label smoothing noise */ /* #33 Per-episode saboteur params [N, 3]: (spread_mult, fill_prob, slippage_mult). * NULL = disabled (use global scalars). When non-NULL, overrides * spread_cost, fill_ioc_fill_prob, tx_cost_multiplier per episode. */ @@ -1307,6 +1312,61 @@ extern "C" __global__ void experience_env_step( float risk_eff = segment_return / fabsf(intra_trade_max_dd); reward += risk_efficiency_weight * fminf(risk_eff, 5.0f); } + + /* ── Layer 6: Urgency branch attribution (fill quality) ── */ + if (urgency_credit_weight > 0.0f && fabsf(position) > 0.001f && entry_price > 0.0f) { + float mid_price = (raw_close + raw_next) * 0.5f; + float fill_improvement = (position > 0.0f) + ? (mid_price - entry_price) / fmaxf(entry_price, 1.0f) + : (entry_price - mid_price) / fmaxf(entry_price, 1.0f); + float urgency_credit = fmaxf(fill_improvement / vol_proxy, 0.0f); + reward += urgency_credit_weight * fminf(urgency_credit, 2.0f); + } + + /* ── Layer 7: Exit timing quality ── */ + if (exit_timing_weight > 0.0f && (exiting_trade || reversing_trade)) { + float post_exit_move = (prev_sign > 0) + ? (raw_next - raw_close) / fmaxf(raw_close, 1.0f) + : (raw_close - raw_next) / fmaxf(raw_close, 1.0f); + float timing_quality = -post_exit_move / vol_proxy; + reward += exit_timing_weight * asymmetric_soft_clamp(timing_quality * 10.0f) * 0.1f; + } + + /* ── Layer 8: OFI-weighted reward confidence ── */ + if (ofi_reward_weight > 0.0f) { + float price_move = (raw_next - raw_close) / fmaxf(raw_close, 1.0f); + float move_magnitude = fabsf(price_move) / vol_proxy; + float alignment = 0.0f; + if (position > 0.001f && price_move > 0.0f) alignment = 1.0f; + if (position < -0.001f && price_move < 0.0f) alignment = 1.0f; + float confidence_mult = alignment * fminf(move_magnitude, 1.0f); + reward *= (1.0f + ofi_reward_weight * confidence_mult); + } + + /* ── Layer 9: Kelly-optimal position sizing signal ── */ + if (kelly_sizing_weight > 0.0f && (win_count + loss_count) >= 5.0f) { + float total_trades = win_count + loss_count; + float win_rate_k = win_count / total_trades; + float avg_win = (win_count > 0.0f) ? (sum_wins / win_count) : 0.001f; + float avg_loss = (loss_count > 0.0f) ? (sum_losses / loss_count) : 0.001f; + float payoff_ratio = avg_win / fmaxf(avg_loss, 0.0001f); + float kelly_f = (payoff_ratio * win_rate_k - (1.0f - win_rate_k)) / fmaxf(payoff_ratio, 0.0001f); + kelly_f = fmaxf(fminf(kelly_f, 1.0f), 0.0f); + float actual_fraction = fabsf(position) / fmaxf(max_position, 1.0f); + float kelly_closeness = 1.0f - fabsf(actual_fraction - kelly_f); + kelly_closeness = fmaxf(kelly_closeness, 0.0f); + if (segment_return > 0.0f) { + reward += kelly_sizing_weight * kelly_closeness * 0.5f; + } + } + } + + /* ── Reward label smoothing (deterministic jitter, all non-zero rewards) ── */ + if (reward_noise_scale > 0.0f && reward != 0.0f) { + unsigned int hash = (unsigned int)(i * 31337 + bar_idx * 7919 + current_t * 1013); + hash ^= (hash >> 16); hash *= 0x45d9f3bu; hash ^= (hash >> 16); + float pseudo_noise = ((float)(hash & 0xFFFF) / 65535.0f - 0.5f) * 2.0f; + reward += reward_noise_scale * pseudo_noise; } /* ---- Drawdown penalty (from trade_physics.cuh) ---- */ diff --git a/crates/ml/src/cuda_pipeline/gpu_experience_collector.rs b/crates/ml/src/cuda_pipeline/gpu_experience_collector.rs index 11247af53..13ba79c76 100644 --- a/crates/ml/src/cuda_pipeline/gpu_experience_collector.rs +++ b/crates/ml/src/cuda_pipeline/gpu_experience_collector.rs @@ -193,6 +193,11 @@ pub struct ExperienceCollectorConfig { pub order_credit_weight: f32, /// v7: Intra-trade risk efficiency weight (0.0 = disabled, 0.5 = moderate) pub risk_efficiency_weight: f32, + pub urgency_credit_weight: f32, + pub exit_timing_weight: f32, + pub ofi_reward_weight: f32, + pub kelly_sizing_weight: f32, + pub reward_noise_scale: f32, /// Drawdown fraction before penalty starts (0.02 = 2%) pub dd_threshold: f32, /// Drawdown penalty weight (1.0 = full penalty) @@ -319,6 +324,11 @@ impl Default for ExperienceCollectorConfig { cea_weight: 0.0, order_credit_weight: 0.0, risk_efficiency_weight: 0.0, + urgency_credit_weight: 0.0, + exit_timing_weight: 0.0, + ofi_reward_weight: 0.0, + kelly_sizing_weight: 0.0, + reward_noise_scale: 0.0, dd_threshold: 0.02, w_dd: 1.0, beta_penalty: 0.0, @@ -1841,6 +1851,11 @@ impl GpuExperienceCollector { .arg(&mirror_i32) // #10 mirror universe .arg(&mut self.position_histogram) // #19 position entropy histogram .arg(&config.position_entropy_weight) // #19 position entropy weight + .arg(&config.urgency_credit_weight) + .arg(&config.exit_timing_weight) + .arg(&config.ofi_reward_weight) + .arg(&config.kelly_sizing_weight) + .arg(&config.reward_noise_scale) // #33 Per-episode saboteur params (0 = NULL = disabled) .arg(&{ if self.saboteur_active { diff --git a/crates/ml/src/hyperopt/adapters/dqn.rs b/crates/ml/src/hyperopt/adapters/dqn.rs index bbc53efb0..e3d47c289 100644 --- a/crates/ml/src/hyperopt/adapters/dqn.rs +++ b/crates/ml/src/hyperopt/adapters/dqn.rs @@ -2181,6 +2181,10 @@ impl HyperparameterOptimizable for DQNTrainer { info!(" [v7] cea_weight: {:.3} (TOML base, scaled by loss_shaping_intensity={:.3})", hyperparams.cea_weight, params.loss_shaping_intensity); info!(" [v7] order_credit_weight: {:.3} (TOML base, scaled by loss_shaping_intensity={:.3})", hyperparams.order_credit_weight, params.loss_shaping_intensity); info!(" [v7] risk_efficiency_weight: {:.3} (TOML base, scaled by loss_shaping_intensity={:.3})", hyperparams.risk_efficiency_weight, params.loss_shaping_intensity); + info!(" [v7 gems] urgency={:.3} timing={:.3} ofi={:.3} kelly={:.3} noise={:.3}", + hyperparams.urgency_credit_weight, hyperparams.exit_timing_weight, + hyperparams.ofi_reward_weight, hyperparams.kelly_sizing_weight, + hyperparams.reward_noise_scale); // Derive max_position_absolute from max_leverage using median close price. // This replaces the old hardcoded contract-count cap with a leverage-based diff --git a/crates/ml/src/trainers/dqn/config.rs b/crates/ml/src/trainers/dqn/config.rs index f649b2548..28c9dde06 100644 --- a/crates/ml/src/trainers/dqn/config.rs +++ b/crates/ml/src/trainers/dqn/config.rs @@ -785,6 +785,16 @@ pub struct DQNHyperparameters { /// Reward v7: Intra-trade risk efficiency weight. /// Rewards clean winning trades with minimal intra-trade drawdown. pub risk_efficiency_weight: f64, + /// Reward v7 gem: Urgency branch attribution weight. + pub urgency_credit_weight: f64, + /// Reward v7 gem: Exit timing quality weight. + pub exit_timing_weight: f64, + /// Reward v7 gem: OFI-weighted reward confidence multiplier. + pub ofi_reward_weight: f64, + /// Reward v7 gem: Kelly-optimal sizing signal weight. + pub kelly_sizing_weight: f64, + /// Reward v7 gem: Reward label smoothing noise scale. + pub reward_noise_scale: f64, /// Position staleness rent per step held pub time_decay_rate: f64, /// Q-value gap threshold for trade conviction filter. @@ -1503,6 +1513,11 @@ impl DQNHyperparameters { self.cea_weight = (self.cea_weight * li).clamp(0.0, 2.0); self.order_credit_weight = (self.order_credit_weight * li).clamp(0.0, 1.0); self.risk_efficiency_weight = (self.risk_efficiency_weight * li).clamp(0.0, 1.0); + self.urgency_credit_weight = (self.urgency_credit_weight * li).clamp(0.0, 1.0); + self.exit_timing_weight = (self.exit_timing_weight * li).clamp(0.0, 0.5); + self.ofi_reward_weight = (self.ofi_reward_weight * li).clamp(0.0, 1.0); + self.kelly_sizing_weight = (self.kelly_sizing_weight * li).clamp(0.0, 1.0); + self.reward_noise_scale = (self.reward_noise_scale * li).clamp(0.0, 0.2); self.position_entropy_weight *= li; // v6 legacy (zeroed defaults, no-op) self.asymmetric_dd_weight *= li; @@ -1611,6 +1626,11 @@ impl DQNHyperparameters { cea_weight: 0.3, order_credit_weight: 0.1, risk_efficiency_weight: 0.1, + urgency_credit_weight: 0.1, + exit_timing_weight: 0.05, + ofi_reward_weight: 0.2, + kelly_sizing_weight: 0.1, + reward_noise_scale: 0.05, pruning_epoch: 50, // #20: compute mask at epoch 50 pruning_fraction: 0.7, // #20: prune 70% of smallest weights bottleneck_dim: 2, // #31: 2D information compression (gem of gems) diff --git a/crates/ml/src/trainers/dqn/trainer/training_loop.rs b/crates/ml/src/trainers/dqn/trainer/training_loop.rs index 164c8ec27..c43912b02 100644 --- a/crates/ml/src/trainers/dqn/trainer/training_loop.rs +++ b/crates/ml/src/trainers/dqn/trainer/training_loop.rs @@ -942,9 +942,17 @@ impl DQNTrainer { enable_action_masking: self.enable_action_masking, curiosity_scale: 1.0, loss_aversion: self.hyperparams.loss_aversion as f32, - cea_weight: self.hyperparams.cea_weight as f32, + cea_weight: { + let base = self.hyperparams.cea_weight as f32; + if self.current_epoch < 3 { 1.0_f32.max(base) } else { base } + }, order_credit_weight: self.hyperparams.order_credit_weight as f32, risk_efficiency_weight: self.hyperparams.risk_efficiency_weight as f32, + urgency_credit_weight: self.hyperparams.urgency_credit_weight as f32, + exit_timing_weight: self.hyperparams.exit_timing_weight as f32, + ofi_reward_weight: self.hyperparams.ofi_reward_weight as f32, + kelly_sizing_weight: self.hyperparams.kelly_sizing_weight as f32, + reward_noise_scale: self.hyperparams.reward_noise_scale as f32, tx_cost_multiplier: { let base = self.hyperparams.transaction_cost_multiplier as f32; if adversarial { base * 2.0 } else { base }