From c83ceeda54bb662f5e002750ad7f4b8d0acc193a Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Mon, 30 Mar 2026 09:05:44 +0200 Subject: [PATCH] feat(generalization): #11 time-reversal + #17 counterfactual regret (CUDA) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Time-reversal (#11): 20% of episodes (index % 5 == 0) played backwards in state_gather kernel. Reversed bar indexing forces the model to rely on instantaneous features rather than temporal trajectory patterns. If it can't trade backwards data, it memorized sequence artifacts. Counterfactual regret (#17): at trade completion, compute PnL for all 9 exposure levels and blend reward with regret (taken - best_possible). Default 30% regret / 70% raw PnL. Normalizes rewards across regimes — a bad trade in a bad market has low regret, a bad trade in a good market has high regret. From game theory (CFR). Both fully in CUDA env_step/state_gather kernels. No CPU paths. Co-Authored-By: Claude Opus 4.6 (1M context) --- config/training/dqn-localdev.toml | 2 + config/training/dqn-production.toml | 2 + .../src/cuda_pipeline/experience_kernels.cu | 54 +++++++++++++++++-- .../cuda_pipeline/gpu_experience_collector.rs | 11 ++++ crates/ml/src/trainers/dqn/config.rs | 10 ++++ .../src/trainers/dqn/trainer/training_loop.rs | 2 + crates/ml/src/training_profile.rs | 4 ++ 7 files changed, 82 insertions(+), 3 deletions(-) diff --git a/config/training/dqn-localdev.toml b/config/training/dqn-localdev.toml index 033c70a2a..c49d3d720 100644 --- a/config/training/dqn-localdev.toml +++ b/config/training/dqn-localdev.toml @@ -83,6 +83,8 @@ feature_mask_fraction = 0.3 feature_noise_scale = 0.1 enable_vol_normalization = true asymmetric_dd_weight = 0.5 +time_reversal_mod = 5 +regret_blend = 0.3 enable_mirror_universe = true position_entropy_weight = 0.01 trade_clustering_penalty = 0.05 diff --git a/config/training/dqn-production.toml b/config/training/dqn-production.toml index 2e7e5a871..6df76c954 100644 --- a/config/training/dqn-production.toml +++ b/config/training/dqn-production.toml @@ -88,6 +88,8 @@ feature_mask_fraction = 0.3 feature_noise_scale = 0.1 enable_vol_normalization = true asymmetric_dd_weight = 0.5 +time_reversal_mod = 5 +regret_blend = 0.3 enable_mirror_universe = true position_entropy_weight = 0.01 trade_clustering_penalty = 0.05 diff --git a/crates/ml/src/cuda_pipeline/experience_kernels.cu b/crates/ml/src/cuda_pipeline/experience_kernels.cu index 936c931e6..a56a7281a 100644 --- a/crates/ml/src/cuda_pipeline/experience_kernels.cu +++ b/crates/ml/src/cuda_pipeline/experience_kernels.cu @@ -155,12 +155,32 @@ extern "C" __global__ void experience_state_gather( unsigned int* rng_states, /* #10 Mirror universe: negate return features to flip market direction. * The model must learn direction-invariant structure. 0 = disabled. */ - int mirror_active + int mirror_active, + /* #11 Time-reversal: reverse bar indexing for selected episodes. + * If > 0, episodes with index % time_reversal_mod == 0 are played backwards. + * 5 = 20% of episodes reversed, 3 = 33%, 0 = disabled. */ + int time_reversal_mod ) { int i = blockIdx.x * blockDim.x + threadIdx.x; if (i >= N) return; - int bar_idx = episode_starts[i] + current_timesteps[i]; + int raw_t = current_timesteps[i]; + /* #11 Time-reversal: reverse bar traversal for selected episodes. + * Episode i is reversed when time_reversal_mod > 0 && i % mod == 0. + * Reversed indexing: start + (L-1-t) instead of start + t. */ + int is_reversed = (time_reversal_mod > 0 && (i % time_reversal_mod) == 0); + int bar_idx; + if (is_reversed) { + /* L (episode length) is not available here — approximate as total_bars/N. + * We use episode_starts[i+1]-episode_starts[i] when possible, else 500. */ + int ep_len = (i + 1 < N) ? (episode_starts[i + 1] - episode_starts[i]) : 500; + if (ep_len <= 0) ep_len = 500; + int rev_t = (ep_len - 1 - raw_t); + if (rev_t < 0) rev_t = 0; + bar_idx = episode_starts[i] + rev_t; + } else { + bar_idx = episode_starts[i] + raw_t; + } __nv_bfloat16* out = batch_states + (long long)i * state_dim; @@ -598,7 +618,8 @@ extern "C" __global__ void experience_env_step( * NULL = disabled. Incremented at each timestep. Entropy bonus computed * at episode end (done=1) and added to final reward. */ float* __restrict__ position_histogram, - float position_entropy_weight /* #19: reward += weight * H(histogram). 0=disabled. */ + float position_entropy_weight, /* #19: reward += weight * H(histogram). 0=disabled. */ + float regret_blend /* #17: blend factor for counterfactual regret (0=pure PnL, 1=pure regret) */ ) { int i = blockIdx.x * blockDim.x + threadIdx.x; if (i >= N) return; @@ -1005,6 +1026,33 @@ extern "C" __global__ void experience_env_step( reward = 10.0f * tanhf(reward / 10.0f); } + /* ── #17 Counterfactual regret: blend reward with regret ── */ + /* Regret = reward(taken) - max(reward(all_exposures)). Always <= 0. + * Blend: final = (1-blend) * reward + blend * regret. + * Normalizes rewards across regimes — a bad trade in a bad market + * has low regret (everyone lost), while a bad trade in a good market + * has high regret (you should have been long). */ + if (regret_blend > 0.0f && segment_complete && segment_hold_time > 0.0f) { + /* Compute per-bar PnL for all 9 exposure levels. + * PnL(exposure_k) = position_k * (raw_next - raw_close) * contract_mult + * position_k = compute_target_position(k, b0_size, max_position) + * This is a fast approximation: uses single-bar PnL instead of full + * segment replay. Exact segment-level regret would need 9 parallel + * portfolio sims per episode — not worth the compute. */ + float best_reward = -1e9f; + float price_delta = raw_next - raw_close; + for (int k = 0; k < b0_size; k++) { + float alt_pos = compute_target_position(k, b0_size, max_position); + float alt_pnl = alt_pos * price_delta * contract_multiplier; + /* Simple vol-normalized reward for comparison */ + float alt_return = alt_pnl / fmaxf(prev_equity, 1.0f); + float alt_reward = 10.0f * alt_return; + if (alt_reward > best_reward) best_reward = alt_reward; + } + float regret = reward - best_reward; /* always <= 0 */ + reward = (1.0f - regret_blend) * reward + regret_blend * regret; + } + /* Layer 3: Reward scaling — shorter trades get proportionally less reward. * Smooth gradient toward longer holds (vs binary cliff at min_hold). * Floor at 10% to maintain gradient signal for edge cases. */ diff --git a/crates/ml/src/cuda_pipeline/gpu_experience_collector.rs b/crates/ml/src/cuda_pipeline/gpu_experience_collector.rs index 81e8fea2f..ecc9c13e8 100644 --- a/crates/ml/src/cuda_pipeline/gpu_experience_collector.rs +++ b/crates/ml/src/cuda_pipeline/gpu_experience_collector.rs @@ -262,6 +262,13 @@ pub struct ExperienceCollectorConfig { // ── Gems & Pearls: generalization parameters ──────────────────── + /// #11 Time-reversal modulus: episodes where index % mod == 0 are played backwards. + /// 5 = 20% reversed, 3 = 33%, 0 = disabled. + pub time_reversal_mod: i32, + + /// #17 Counterfactual regret blend (0=pure PnL, 1=pure regret). 0.0 = disabled. + pub regret_blend: f32, + /// #10 Mirror universe: negate return features + invert exposure actions. /// Doubles effective data diversity by training on mirrored price sequences. pub mirror_active: bool, @@ -336,6 +343,8 @@ impl Default for ExperienceCollectorConfig { spread_cost: 0.0, // default: no spread cost (overridden by hyperparams) contract_multiplier: 50.0, margin_pct: 0.06, + time_reversal_mod: 0, + regret_blend: 0.0, mirror_active: false, position_entropy_weight: 0.0, trade_clustering_penalty: 0.0, @@ -1293,6 +1302,7 @@ impl GpuExperienceCollector { .arg(&vol_norm) // #13 vol normalization .arg(&mut self.rng_states) // RNG for noise .arg(&mirror_i32) // #10 mirror universe + .arg(&config.time_reversal_mod) // #11 time-reversal .launch(launch_cfg) .map_err(|e| MLError::ModelError(format!( "experience_state_gather t={t}: {e}" @@ -1448,6 +1458,7 @@ impl GpuExperienceCollector { .arg(&mirror_i32) // #10 mirror universe .arg(&mut self.position_histogram) // #19 position entropy histogram .arg(&config.position_entropy_weight) // #19 position entropy weight + .arg(&config.regret_blend) // #17 counterfactual regret .launch(launch_cfg) .map_err(|e| MLError::ModelError(format!( "experience_env_step t={t}: {e}" diff --git a/crates/ml/src/trainers/dqn/config.rs b/crates/ml/src/trainers/dqn/config.rs index a9936800a..db3990170 100644 --- a/crates/ml/src/trainers/dqn/config.rs +++ b/crates/ml/src/trainers/dqn/config.rs @@ -991,6 +991,14 @@ pub struct DQNHyperparameters { /// Forces the model to visit multiple exposure levels. 0.0 = disabled. pub position_entropy_weight: f64, + /// #11 Time-reversal modulus: episodes where index % mod == 0 are played backwards. + /// 5 = 20% reversed, 0 = disabled. + pub time_reversal_mod: usize, + + /// #17 Counterfactual regret blend: 0.0 = pure PnL reward, 1.0 = pure regret. + /// Regret = reward(taken) - max(reward(all_exposures)). Normalizes across regimes. + pub regret_blend: f64, + /// #25 Trade clustering penalty: penalize temporally clustered trades. /// CV(inter-trade intervals) * this weight is subtracted from reward. /// 0.0 = disabled. @@ -1506,6 +1514,8 @@ impl DQNHyperparameters { feature_mask_fraction: 0.3, // #23: mask 30% of features each epoch feature_noise_scale: 0.1, // #22: add N(0, 0.1*std) noise to features enable_vol_normalization: true, // #13: divide returns by realized vol + time_reversal_mod: 5, // #11: 20% of episodes played backwards + regret_blend: 0.3, // #17: 30% regret + 70% raw PnL enable_mirror_universe: true, // #10: alternate mirrored/normal epochs position_entropy_weight: 0.01, // #19: reward += 0.01 * H(position_histogram) trade_clustering_penalty: 0.05, // #25: penalize temporally clustered trades diff --git a/crates/ml/src/trainers/dqn/trainer/training_loop.rs b/crates/ml/src/trainers/dqn/trainer/training_loop.rs index 18cca8320..bbf97667b 100644 --- a/crates/ml/src/trainers/dqn/trainer/training_loop.rs +++ b/crates/ml/src/trainers/dqn/trainer/training_loop.rs @@ -1191,6 +1191,8 @@ impl DQNTrainer { w_dd: self.hyperparams.w_dd as f32, beta_penalty: self.hyperparams.beta_penalty_strength as f32, // Gems & Pearls generalization params + time_reversal_mod: self.hyperparams.time_reversal_mod as i32, + regret_blend: self.hyperparams.regret_blend as f32, mirror_active: self.hyperparams.enable_mirror_universe && self.current_epoch % 2 == 1, position_entropy_weight: self.hyperparams.position_entropy_weight as f32, trade_clustering_penalty: self.hyperparams.trade_clustering_penalty as f32, diff --git a/crates/ml/src/training_profile.rs b/crates/ml/src/training_profile.rs index ce3e58a00..8f6b72160 100644 --- a/crates/ml/src/training_profile.rs +++ b/crates/ml/src/training_profile.rs @@ -162,6 +162,8 @@ pub struct GeneralizationSection { pub feature_noise_scale: Option, pub enable_vol_normalization: Option, pub asymmetric_dd_weight: Option, + pub time_reversal_mod: Option, + pub regret_blend: Option, pub enable_mirror_universe: Option, pub position_entropy_weight: Option, pub trade_clustering_penalty: Option, @@ -809,6 +811,8 @@ impl DqnTrainingProfile { if let Some(v) = g.feature_noise_scale { hp.feature_noise_scale = v; } if let Some(v) = g.enable_vol_normalization { hp.enable_vol_normalization = v; } if let Some(v) = g.asymmetric_dd_weight { hp.asymmetric_dd_weight = v; } + if let Some(v) = g.time_reversal_mod { hp.time_reversal_mod = v; } + if let Some(v) = g.regret_blend { hp.regret_blend = v; } if let Some(v) = g.enable_mirror_universe { hp.enable_mirror_universe = v; } if let Some(v) = g.position_entropy_weight { hp.position_entropy_weight = v; } if let Some(v) = g.trade_clustering_penalty { hp.trade_clustering_penalty = v; }