gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(reward-v7): implement all gems & pearls

Browse Source 
Layers 6-9 + label smoothing + adaptive CEA warmup:
- Urgency branch attribution (fill price improvement)
- Exit timing quality (market reversal after exit)
- OFI-weighted reward confidence (amplify informed trades)
- Kelly-optimal position sizing signal
- Reward label smoothing (deterministic jitter)
- Adaptive CEA warmup (1.0 for first 3 epochs)

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-04-07 21:08:09 +02:00
parent f05a4c5d44
commit ad083e1c20
9 changed files with 138 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
config/training/dqn-hyperopt.toml
View File

@@ -87,6 +87,11 @@ q_gap_threshold = [0.0, 0.5] # 0.0=trade every bar, 0.5=high conviction
cea_weight = [0.1, 1.0]
order_credit_weight = [0.0, 0.5]
risk_efficiency_weight = [0.0, 0.5]
urgency_credit_weight = [0.0, 0.5]
exit_timing_weight = [0.0, 0.2]
ofi_reward_weight = [0.0, 0.5]
kelly_sizing_weight = [0.0, 0.5]
reward_noise_scale = [0.01, 0.1]
[experience]
initial_capital = 100000.0
@@ -114,6 +119,11 @@ q_gap_threshold = 0.1
cea_weight = 0.3
order_credit_weight = 0.1
risk_efficiency_weight = 0.1
urgency_credit_weight = 0.1
exit_timing_weight = 0.05
ofi_reward_weight = 0.2
kelly_sizing_weight = 0.1
reward_noise_scale = 0.05
[fixed]
@@ -145,3 +155,8 @@ q_gap_threshold = 0.1
cea_weight = 0.3
order_credit_weight = 0.1
risk_efficiency_weight = 0.1
urgency_credit_weight = 0.1
exit_timing_weight = 0.05
ofi_reward_weight = 0.2
kelly_sizing_weight = 0.1
reward_noise_scale = 0.05

5
config/training/dqn-localdev.toml
View File

@@ -120,3 +120,8 @@ dd_threshold = 0.02
cea_weight = 0.3
order_credit_weight = 0.1
risk_efficiency_weight = 0.1
urgency_credit_weight = 0.1
exit_timing_weight = 0.05
ofi_reward_weight = 0.2
kelly_sizing_weight = 0.1
reward_noise_scale = 0.05

5
config/training/dqn-production.toml
View File

@@ -131,3 +131,8 @@ dd_threshold = 0.02
cea_weight = 0.3
order_credit_weight = 0.1
risk_efficiency_weight = 0.1
urgency_credit_weight = 0.1
exit_timing_weight = 0.05
ofi_reward_weight = 0.2
kelly_sizing_weight = 0.1
reward_noise_scale = 0.05

5
config/training/dqn-smoketest.toml
View File

@@ -127,6 +127,11 @@ dd_threshold = 0.02
cea_weight = 0.3
order_credit_weight = 0.1
risk_efficiency_weight = 0.1
urgency_credit_weight = 0.1
exit_timing_weight = 0.05
ofi_reward_weight = 0.2
kelly_sizing_weight = 0.1
reward_noise_scale = 0.05
[walk_forward]
step_fraction = 0.25

60
crates/ml/src/cuda_pipeline/experience_kernels.cu
View File

@@ -866,6 +866,11 @@ extern "C" __global__ void experience_env_step(
* at episode end (done=1) and added to final reward. */
float* __restrict__ position_histogram,
float position_entropy_weight, /* #19: reward += weight * H(histogram). 0=disabled. */
float urgency_credit_weight, /* v7 gem: urgency branch attribution */
float exit_timing_weight, /* v7 gem: exit timing quality */
float ofi_reward_weight, /* v7 gem: OFI-weighted reward confidence */
float kelly_sizing_weight, /* v7 gem: Kelly-optimal sizing signal */
float reward_noise_scale, /* v7 gem: reward label smoothing noise */
/* #33 Per-episode saboteur params [N, 3]: (spread_mult, fill_prob, slippage_mult).
* NULL = disabled (use global scalars). When non-NULL, overrides
* spread_cost, fill_ioc_fill_prob, tx_cost_multiplier per episode. */
@@ -1307,6 +1312,61 @@ extern "C" __global__ void experience_env_step(
float risk_eff = segment_return / fabsf(intra_trade_max_dd);
reward += risk_efficiency_weight * fminf(risk_eff, 5.0f);
}
/* ── Layer 6: Urgency branch attribution (fill quality) ── */
if (urgency_credit_weight > 0.0f && fabsf(position) > 0.001f && entry_price > 0.0f) {
float mid_price = (raw_close + raw_next) * 0.5f;
float fill_improvement = (position > 0.0f)
? (mid_price - entry_price) / fmaxf(entry_price, 1.0f)
: (entry_price - mid_price) / fmaxf(entry_price, 1.0f);
float urgency_credit = fmaxf(fill_improvement / vol_proxy, 0.0f);
reward += urgency_credit_weight * fminf(urgency_credit, 2.0f);
}
/* ── Layer 7: Exit timing quality ── */
if (exit_timing_weight > 0.0f && (exiting_trade || reversing_trade)) {
float post_exit_move = (prev_sign > 0)
? (raw_next - raw_close) / fmaxf(raw_close, 1.0f)
: (raw_close - raw_next) / fmaxf(raw_close, 1.0f);
float timing_quality = -post_exit_move / vol_proxy;
reward += exit_timing_weight * asymmetric_soft_clamp(timing_quality * 10.0f) * 0.1f;
}
/* ── Layer 8: OFI-weighted reward confidence ── */
if (ofi_reward_weight > 0.0f) {
float price_move = (raw_next - raw_close) / fmaxf(raw_close, 1.0f);
float move_magnitude = fabsf(price_move) / vol_proxy;
float alignment = 0.0f;
if (position > 0.001f && price_move > 0.0f) alignment = 1.0f;
if (position < -0.001f && price_move < 0.0f) alignment = 1.0f;
float confidence_mult = alignment * fminf(move_magnitude, 1.0f);
reward *= (1.0f + ofi_reward_weight * confidence_mult);
}
/* ── Layer 9: Kelly-optimal position sizing signal ── */
if (kelly_sizing_weight > 0.0f && (win_count + loss_count) >= 5.0f) {
float total_trades = win_count + loss_count;
float win_rate_k = win_count / total_trades;
float avg_win = (win_count > 0.0f) ? (sum_wins / win_count) : 0.001f;
float avg_loss = (loss_count > 0.0f) ? (sum_losses / loss_count) : 0.001f;
float payoff_ratio = avg_win / fmaxf(avg_loss, 0.0001f);
float kelly_f = (payoff_ratio * win_rate_k - (1.0f - win_rate_k)) / fmaxf(payoff_ratio, 0.0001f);
kelly_f = fmaxf(fminf(kelly_f, 1.0f), 0.0f);
float actual_fraction = fabsf(position) / fmaxf(max_position, 1.0f);
float kelly_closeness = 1.0f - fabsf(actual_fraction - kelly_f);
kelly_closeness = fmaxf(kelly_closeness, 0.0f);
if (segment_return > 0.0f) {
reward += kelly_sizing_weight * kelly_closeness * 0.5f;
}
}
}
/* ── Reward label smoothing (deterministic jitter, all non-zero rewards) ── */
if (reward_noise_scale > 0.0f && reward != 0.0f) {
unsigned int hash = (unsigned int)(i * 31337 + bar_idx * 7919 + current_t * 1013);
hash ^= (hash >> 16); hash *= 0x45d9f3bu; hash ^= (hash >> 16);
float pseudo_noise = ((float)(hash & 0xFFFF) / 65535.0f - 0.5f) * 2.0f;
reward += reward_noise_scale * pseudo_noise;
}
/* ---- Drawdown penalty (from trade_physics.cuh) ---- */

15
crates/ml/src/cuda_pipeline/gpu_experience_collector.rs
View File

@@ -193,6 +193,11 @@ pub struct ExperienceCollectorConfig {
pub order_credit_weight: f32,
/// v7: Intra-trade risk efficiency weight (0.0 = disabled, 0.5 = moderate)
pub risk_efficiency_weight: f32,
pub urgency_credit_weight: f32,
pub exit_timing_weight: f32,
pub ofi_reward_weight: f32,
pub kelly_sizing_weight: f32,
pub reward_noise_scale: f32,
/// Drawdown fraction before penalty starts (0.02 = 2%)
pub dd_threshold: f32,
/// Drawdown penalty weight (1.0 = full penalty)
@@ -319,6 +324,11 @@ impl Default for ExperienceCollectorConfig {
cea_weight: 0.0,
order_credit_weight: 0.0,
risk_efficiency_weight: 0.0,
urgency_credit_weight: 0.0,
exit_timing_weight: 0.0,
ofi_reward_weight: 0.0,
kelly_sizing_weight: 0.0,
reward_noise_scale: 0.0,
dd_threshold: 0.02,
w_dd: 1.0,
beta_penalty: 0.0,
@@ -1841,6 +1851,11 @@ impl GpuExperienceCollector {
.arg(&mirror_i32) // #10 mirror universe
.arg(&mut self.position_histogram) // #19 position entropy histogram
.arg(&config.position_entropy_weight) // #19 position entropy weight
.arg(&config.urgency_credit_weight)
.arg(&config.exit_timing_weight)
.arg(&config.ofi_reward_weight)
.arg(&config.kelly_sizing_weight)
.arg(&config.reward_noise_scale)
// #33 Per-episode saboteur params (0 = NULL = disabled)
.arg(&{
if self.saboteur_active {

4
crates/ml/src/hyperopt/adapters/dqn.rs
View File

@@ -2181,6 +2181,10 @@ impl HyperparameterOptimizable for DQNTrainer {
info!(" [v7] cea_weight: {:.3} (TOML base, scaled by loss_shaping_intensity={:.3})", hyperparams.cea_weight, params.loss_shaping_intensity);
info!(" [v7] order_credit_weight: {:.3} (TOML base, scaled by loss_shaping_intensity={:.3})", hyperparams.order_credit_weight, params.loss_shaping_intensity);
info!(" [v7] risk_efficiency_weight: {:.3} (TOML base, scaled by loss_shaping_intensity={:.3})", hyperparams.risk_efficiency_weight, params.loss_shaping_intensity);
info!(" [v7 gems] urgency={:.3} timing={:.3} ofi={:.3} kelly={:.3} noise={:.3}",
hyperparams.urgency_credit_weight, hyperparams.exit_timing_weight,
hyperparams.ofi_reward_weight, hyperparams.kelly_sizing_weight,
hyperparams.reward_noise_scale);
// Derive max_position_absolute from max_leverage using median close price.
// This replaces the old hardcoded contract-count cap with a leverage-based

20
crates/ml/src/trainers/dqn/config.rs
View File

@@ -785,6 +785,16 @@ pub struct DQNHyperparameters {
/// Reward v7: Intra-trade risk efficiency weight.
/// Rewards clean winning trades with minimal intra-trade drawdown.
pub risk_efficiency_weight: f64,
/// Reward v7 gem: Urgency branch attribution weight.
pub urgency_credit_weight: f64,
/// Reward v7 gem: Exit timing quality weight.
pub exit_timing_weight: f64,
/// Reward v7 gem: OFI-weighted reward confidence multiplier.
pub ofi_reward_weight: f64,
/// Reward v7 gem: Kelly-optimal sizing signal weight.
pub kelly_sizing_weight: f64,
/// Reward v7 gem: Reward label smoothing noise scale.
pub reward_noise_scale: f64,
/// Position staleness rent per step held
pub time_decay_rate: f64,
/// Q-value gap threshold for trade conviction filter.
@@ -1503,6 +1513,11 @@ impl DQNHyperparameters {
self.cea_weight = (self.cea_weight * li).clamp(0.0, 2.0);
self.order_credit_weight = (self.order_credit_weight * li).clamp(0.0, 1.0);
self.risk_efficiency_weight = (self.risk_efficiency_weight * li).clamp(0.0, 1.0);
self.urgency_credit_weight = (self.urgency_credit_weight * li).clamp(0.0, 1.0);
self.exit_timing_weight = (self.exit_timing_weight * li).clamp(0.0, 0.5);
self.ofi_reward_weight = (self.ofi_reward_weight * li).clamp(0.0, 1.0);
self.kelly_sizing_weight = (self.kelly_sizing_weight * li).clamp(0.0, 1.0);
self.reward_noise_scale = (self.reward_noise_scale * li).clamp(0.0, 0.2);
self.position_entropy_weight *= li;
// v6 legacy (zeroed defaults, no-op)
self.asymmetric_dd_weight *= li;
@@ -1611,6 +1626,11 @@ impl DQNHyperparameters {
cea_weight: 0.3,
order_credit_weight: 0.1,
risk_efficiency_weight: 0.1,
urgency_credit_weight: 0.1,
exit_timing_weight: 0.05,
ofi_reward_weight: 0.2,
kelly_sizing_weight: 0.1,
reward_noise_scale: 0.05,
pruning_epoch: 50, // #20: compute mask at epoch 50
pruning_fraction: 0.7, // #20: prune 70% of smallest weights
bottleneck_dim: 2, // #31: 2D information compression (gem of gems)

10
crates/ml/src/trainers/dqn/trainer/training_loop.rs
View File

@@ -942,9 +942,17 @@ impl DQNTrainer {
enable_action_masking: self.enable_action_masking,
curiosity_scale: 1.0,
loss_aversion: self.hyperparams.loss_aversion as f32,
cea_weight: self.hyperparams.cea_weight as f32,
cea_weight: {
let base = self.hyperparams.cea_weight as f32;
if self.current_epoch < 3 { 1.0_f32.max(base) } else { base }
},
order_credit_weight: self.hyperparams.order_credit_weight as f32,
risk_efficiency_weight: self.hyperparams.risk_efficiency_weight as f32,
urgency_credit_weight: self.hyperparams.urgency_credit_weight as f32,
exit_timing_weight: self.hyperparams.exit_timing_weight as f32,
ofi_reward_weight: self.hyperparams.ofi_reward_weight as f32,
kelly_sizing_weight: self.hyperparams.kelly_sizing_weight as f32,
reward_noise_scale: self.hyperparams.reward_noise_scale as f32,
tx_cost_multiplier: {
let base = self.hyperparams.transaction_cost_multiplier as f32;
if adversarial { base * 2.0 } else { base }