gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(generalization): #11 time-reversal + #17 counterfactual regret (CUDA)

Browse Source 
Time-reversal (#11): 20% of episodes (index % 5 == 0) played backwards
in state_gather kernel. Reversed bar indexing forces the model to rely
on instantaneous features rather than temporal trajectory patterns.
If it can't trade backwards data, it memorized sequence artifacts.

Counterfactual regret (#17): at trade completion, compute PnL for all
9 exposure levels and blend reward with regret (taken - best_possible).
Default 30% regret / 70% raw PnL. Normalizes rewards across regimes —
a bad trade in a bad market has low regret, a bad trade in a good
market has high regret. From game theory (CFR).

Both fully in CUDA env_step/state_gather kernels. No CPU paths.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-03-30 09:05:44 +02:00
parent 3c0fbcc5e3
commit c83ceeda54
7 changed files with 82 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
config/training/dqn-localdev.toml
View File

@@ -83,6 +83,8 @@ feature_mask_fraction = 0.3
feature_noise_scale = 0.1
enable_vol_normalization = true
asymmetric_dd_weight = 0.5
time_reversal_mod = 5
regret_blend = 0.3
enable_mirror_universe = true
position_entropy_weight = 0.01
trade_clustering_penalty = 0.05

2
config/training/dqn-production.toml
View File

@@ -88,6 +88,8 @@ feature_mask_fraction = 0.3
feature_noise_scale = 0.1
enable_vol_normalization = true
asymmetric_dd_weight = 0.5
time_reversal_mod = 5
regret_blend = 0.3
enable_mirror_universe = true
position_entropy_weight = 0.01
trade_clustering_penalty = 0.05

54
crates/ml/src/cuda_pipeline/experience_kernels.cu
View File

@@ -155,12 +155,32 @@ extern "C" __global__ void experience_state_gather(
unsigned int* rng_states,
/* #10 Mirror universe: negate return features to flip market direction.
* The model must learn direction-invariant structure. 0 = disabled. */
int mirror_active
int mirror_active,
/* #11 Time-reversal: reverse bar indexing for selected episodes.
* If > 0, episodes with index % time_reversal_mod == 0 are played backwards.
* 5 = 20% of episodes reversed, 3 = 33%, 0 = disabled. */
int time_reversal_mod
) {
int i = blockIdx.x * blockDim.x + threadIdx.x;
if (i >= N) return;
int bar_idx = episode_starts[i] + current_timesteps[i];
int raw_t = current_timesteps[i];
/* #11 Time-reversal: reverse bar traversal for selected episodes.
* Episode i is reversed when time_reversal_mod > 0 && i % mod == 0.
* Reversed indexing: start + (L-1-t) instead of start + t. */
int is_reversed = (time_reversal_mod > 0 && (i % time_reversal_mod) == 0);
int bar_idx;
if (is_reversed) {
/* L (episode length) is not available here — approximate as total_bars/N.
* We use episode_starts[i+1]-episode_starts[i] when possible, else 500. */
int ep_len = (i + 1 < N) ? (episode_starts[i + 1] - episode_starts[i]) : 500;
if (ep_len <= 0) ep_len = 500;
int rev_t = (ep_len - 1 - raw_t);
if (rev_t < 0) rev_t = 0;
bar_idx = episode_starts[i] + rev_t;
} else {
bar_idx = episode_starts[i] + raw_t;
}
__nv_bfloat16* out = batch_states + (long long)i * state_dim;
@@ -598,7 +618,8 @@ extern "C" __global__ void experience_env_step(
* NULL = disabled. Incremented at each timestep. Entropy bonus computed
* at episode end (done=1) and added to final reward. */
float* __restrict__ position_histogram,
float position_entropy_weight /* #19: reward += weight * H(histogram). 0=disabled. */
float position_entropy_weight, /* #19: reward += weight * H(histogram). 0=disabled. */
float regret_blend /* #17: blend factor for counterfactual regret (0=pure PnL, 1=pure regret) */
) {
int i = blockIdx.x * blockDim.x + threadIdx.x;
if (i >= N) return;
@@ -1005,6 +1026,33 @@ extern "C" __global__ void experience_env_step(
reward = 10.0f * tanhf(reward / 10.0f);
}
/* ── #17 Counterfactual regret: blend reward with regret ── */
/* Regret = reward(taken) - max(reward(all_exposures)). Always <= 0.
* Blend: final = (1-blend) * reward + blend * regret.
* Normalizes rewards across regimes — a bad trade in a bad market
* has low regret (everyone lost), while a bad trade in a good market
* has high regret (you should have been long). */
if (regret_blend > 0.0f && segment_complete && segment_hold_time > 0.0f) {
/* Compute per-bar PnL for all 9 exposure levels.
* PnL(exposure_k) = position_k * (raw_next - raw_close) * contract_mult
* position_k = compute_target_position(k, b0_size, max_position)
* This is a fast approximation: uses single-bar PnL instead of full
* segment replay. Exact segment-level regret would need 9 parallel
* portfolio sims per episode — not worth the compute. */
float best_reward = -1e9f;
float price_delta = raw_next - raw_close;
for (int k = 0; k < b0_size; k++) {
float alt_pos = compute_target_position(k, b0_size, max_position);
float alt_pnl = alt_pos * price_delta * contract_multiplier;
/* Simple vol-normalized reward for comparison */
float alt_return = alt_pnl / fmaxf(prev_equity, 1.0f);
float alt_reward = 10.0f * alt_return;
if (alt_reward > best_reward) best_reward = alt_reward;
}
float regret = reward - best_reward; /* always <= 0 */
reward = (1.0f - regret_blend) * reward + regret_blend * regret;
}
/* Layer 3: Reward scaling — shorter trades get proportionally less reward.
* Smooth gradient toward longer holds (vs binary cliff at min_hold).
* Floor at 10% to maintain gradient signal for edge cases. */

11
crates/ml/src/cuda_pipeline/gpu_experience_collector.rs
View File

@@ -262,6 +262,13 @@ pub struct ExperienceCollectorConfig {
// ── Gems & Pearls: generalization parameters ────────────────────
/// #11 Time-reversal modulus: episodes where index % mod == 0 are played backwards.
/// 5 = 20% reversed, 3 = 33%, 0 = disabled.
pub time_reversal_mod: i32,
/// #17 Counterfactual regret blend (0=pure PnL, 1=pure regret). 0.0 = disabled.
pub regret_blend: f32,
/// #10 Mirror universe: negate return features + invert exposure actions.
/// Doubles effective data diversity by training on mirrored price sequences.
pub mirror_active: bool,
@@ -336,6 +343,8 @@ impl Default for ExperienceCollectorConfig {
spread_cost: 0.0, // default: no spread cost (overridden by hyperparams)
contract_multiplier: 50.0,
margin_pct: 0.06,
time_reversal_mod: 0,
regret_blend: 0.0,
mirror_active: false,
position_entropy_weight: 0.0,
trade_clustering_penalty: 0.0,
@@ -1293,6 +1302,7 @@ impl GpuExperienceCollector {
.arg(&vol_norm) // #13 vol normalization
.arg(&mut self.rng_states) // RNG for noise
.arg(&mirror_i32) // #10 mirror universe
.arg(&config.time_reversal_mod) // #11 time-reversal
.launch(launch_cfg)
.map_err(|e| MLError::ModelError(format!(
"experience_state_gather t={t}: {e}"
@@ -1448,6 +1458,7 @@ impl GpuExperienceCollector {
.arg(&mirror_i32) // #10 mirror universe
.arg(&mut self.position_histogram) // #19 position entropy histogram
.arg(&config.position_entropy_weight) // #19 position entropy weight
.arg(&config.regret_blend) // #17 counterfactual regret
.launch(launch_cfg)
.map_err(|e| MLError::ModelError(format!(
"experience_env_step t={t}: {e}"

10
crates/ml/src/trainers/dqn/config.rs
View File

@@ -991,6 +991,14 @@ pub struct DQNHyperparameters {
/// Forces the model to visit multiple exposure levels. 0.0 = disabled.
pub position_entropy_weight: f64,
/// #11 Time-reversal modulus: episodes where index % mod == 0 are played backwards.
/// 5 = 20% reversed, 0 = disabled.
pub time_reversal_mod: usize,
/// #17 Counterfactual regret blend: 0.0 = pure PnL reward, 1.0 = pure regret.
/// Regret = reward(taken) - max(reward(all_exposures)). Normalizes across regimes.
pub regret_blend: f64,
/// #25 Trade clustering penalty: penalize temporally clustered trades.
/// CV(inter-trade intervals) * this weight is subtracted from reward.
/// 0.0 = disabled.
@@ -1506,6 +1514,8 @@ impl DQNHyperparameters {
feature_mask_fraction: 0.3, // #23: mask 30% of features each epoch
feature_noise_scale: 0.1, // #22: add N(0, 0.1*std) noise to features
enable_vol_normalization: true, // #13: divide returns by realized vol
time_reversal_mod: 5, // #11: 20% of episodes played backwards
regret_blend: 0.3, // #17: 30% regret + 70% raw PnL
enable_mirror_universe: true, // #10: alternate mirrored/normal epochs
position_entropy_weight: 0.01, // #19: reward += 0.01 * H(position_histogram)
trade_clustering_penalty: 0.05, // #25: penalize temporally clustered trades

2
crates/ml/src/trainers/dqn/trainer/training_loop.rs
View File

@@ -1191,6 +1191,8 @@ impl DQNTrainer {
w_dd: self.hyperparams.w_dd as f32,
beta_penalty: self.hyperparams.beta_penalty_strength as f32,
// Gems & Pearls generalization params
time_reversal_mod: self.hyperparams.time_reversal_mod as i32,
regret_blend: self.hyperparams.regret_blend as f32,
mirror_active: self.hyperparams.enable_mirror_universe && self.current_epoch % 2 == 1,
position_entropy_weight: self.hyperparams.position_entropy_weight as f32,
trade_clustering_penalty: self.hyperparams.trade_clustering_penalty as f32,

4
crates/ml/src/training_profile.rs
View File

@@ -162,6 +162,8 @@ pub struct GeneralizationSection {
pub feature_noise_scale: Option<f64>,
pub enable_vol_normalization: Option<bool>,
pub asymmetric_dd_weight: Option<f64>,
pub time_reversal_mod: Option<usize>,
pub regret_blend: Option<f64>,
pub enable_mirror_universe: Option<bool>,
pub position_entropy_weight: Option<f64>,
pub trade_clustering_penalty: Option<f64>,
@@ -809,6 +811,8 @@ impl DqnTrainingProfile {
if let Some(v) = g.feature_noise_scale { hp.feature_noise_scale = v; }
if let Some(v) = g.enable_vol_normalization { hp.enable_vol_normalization = v; }
if let Some(v) = g.asymmetric_dd_weight { hp.asymmetric_dd_weight = v; }
if let Some(v) = g.time_reversal_mod { hp.time_reversal_mod = v; }
if let Some(v) = g.regret_blend { hp.regret_blend = v; }
if let Some(v) = g.enable_mirror_universe { hp.enable_mirror_universe = v; }
if let Some(v) = g.position_entropy_weight { hp.position_entropy_weight = v; }
if let Some(v) = g.trade_clustering_penalty { hp.trade_clustering_penalty = v; }