feat(generalization): 7 gems & pearls — CUDA kernels + config + TOML wiring
Phase 1 generalization techniques to close IS/OOS Sharpe gap (+0.57→-1.30). All techniques fully wired from DQNHyperparameters → TOML [generalization] section → ExperienceCollectorConfig → CUDA kernel arguments. New techniques implemented: - #13 Vol normalization: divide return features by realized vol (state_gather) - #18 Asymmetric DD loss: extra penalty on Q-overestimation in drawdown (mse_loss_batched + c51_loss_batched) - #22 Feature noise: N(0, scale) per feature via LCG RNG (state_gather) - #23 Causal feature masking: random 30% feature subset zeroed per epoch (state_gather, mask uploaded from Rust) - #24 Anti-intuitive LR: 3x LR when Sharpe good, 0.3x when bad (Rust-only) - #25 Trade clustering: CV(inter-trade intervals) penalty via ps[3:6] (env_step, uses reserved portfolio state slots) - #27 Ensemble disagreement: Q_target -= weight * ensemble_std (mse_loss + c51_loss, buffer allocated for future ensemble wiring) Also includes 30-task plan doc with all 28+ techniques across 3 phases. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
jgrusewski
@@ -68,6 +68,24 @@ iqn_lambda = 0.25
|
||||
spectral_norm_sigma_max = 1.5
|
||||
gradient_clip_norm = 1.0
|
||||
|
||||
[generalization]
|
||||
enable_domain_randomization = true
|
||||
shrink_perturb_interval = 20
|
||||
shrink_perturb_alpha = 0.85
|
||||
shrink_perturb_sigma = 0.01
|
||||
adversarial_dd_threshold = 0.005
|
||||
adversarial_epochs_trigger = 5
|
||||
enable_anti_lr = true
|
||||
anti_lr_good_mult = 3.0
|
||||
anti_lr_bad_mult = 0.3
|
||||
anti_lr_sharpe_threshold = 0.3
|
||||
feature_mask_fraction = 0.3
|
||||
feature_noise_scale = 0.1
|
||||
enable_vol_normalization = true
|
||||
asymmetric_dd_weight = 0.5
|
||||
trade_clustering_penalty = 0.05
|
||||
ensemble_disagreement_penalty = 0.3
|
||||
|
||||
[reward]
|
||||
loss_aversion = 1.5
|
||||
q_gap_threshold = 0.1
|
||||
|
||||
@@ -73,6 +73,24 @@ iqn_lambda = 0.25
|
||||
spectral_norm_sigma_max = 1.5
|
||||
gradient_clip_norm = 1.0
|
||||
|
||||
[generalization]
|
||||
enable_domain_randomization = true
|
||||
shrink_perturb_interval = 20
|
||||
shrink_perturb_alpha = 0.85
|
||||
shrink_perturb_sigma = 0.01
|
||||
adversarial_dd_threshold = 0.005
|
||||
adversarial_epochs_trigger = 5
|
||||
enable_anti_lr = true
|
||||
anti_lr_good_mult = 3.0
|
||||
anti_lr_bad_mult = 0.3
|
||||
anti_lr_sharpe_threshold = 0.3
|
||||
feature_mask_fraction = 0.3
|
||||
feature_noise_scale = 0.1
|
||||
enable_vol_normalization = true
|
||||
asymmetric_dd_weight = 0.5
|
||||
trade_clustering_penalty = 0.05
|
||||
ensemble_disagreement_penalty = 0.3
|
||||
|
||||
[reward]
|
||||
loss_aversion = 1.5
|
||||
q_gap_threshold = 0.1
|
||||
|
||||
@@ -188,6 +188,9 @@ extern "C" __global__ void c51_loss_batched(
|
||||
__nv_bfloat16* __restrict__ save_current_lp,
|
||||
__nv_bfloat16* __restrict__ save_projected,
|
||||
|
||||
const float* __restrict__ curiosity_errors, /* [B] per-sample curiosity prediction error (0 = no penalty) */
|
||||
float curiosity_q_penalty_lambda, /* scaling factor: gamma *= 1/(1 + lambda * error) */
|
||||
|
||||
float gamma,
|
||||
int batch_size,
|
||||
int num_atoms,
|
||||
@@ -195,7 +198,15 @@ extern "C" __global__ void c51_loss_batched(
|
||||
float v_max,
|
||||
int b0_size,
|
||||
int b1_size,
|
||||
int b2_size
|
||||
int b2_size,
|
||||
|
||||
/* ── #18 Asymmetric drawdown loss ────────────────────────────── */
|
||||
const float* __restrict__ drawdown_depths, /* [B] per-sample drawdown fraction (0=no DD) */
|
||||
float asymmetric_dd_weight, /* penalty weight (0.0=disabled) */
|
||||
|
||||
/* ── #27 Ensemble disagreement Q-penalty ─────────────────────── */
|
||||
const float* __restrict__ ensemble_std, /* [B] per-sample ensemble std (0=no penalty) */
|
||||
float ensemble_disagreement_weight /* scale reward by 1/(1+weight*std) (0.0=disabled) */
|
||||
) {
|
||||
extern __shared__ float shmem_f[];
|
||||
|
||||
@@ -265,6 +276,24 @@ extern "C" __global__ void c51_loss_batched(
|
||||
float done = dones[sample_id];
|
||||
float is_weight = is_weights[sample_id];
|
||||
|
||||
/* #27 Ensemble disagreement: reduce effective reward when ensemble is uncertain.
|
||||
* Shifts the Bellman projection toward lower Q-values for uncertain states. */
|
||||
if (ensemble_disagreement_weight > 0.0f && ensemble_std != NULL) {
|
||||
reward -= ensemble_disagreement_weight * ensemble_std[sample_id];
|
||||
}
|
||||
|
||||
/* ── Curiosity Q-penalty: shrink gamma for novel/uncertain states ──
|
||||
* If the curiosity forward model can't predict the next state well
|
||||
* (high MSE), the Q-value for this transition is unreliable. Scale
|
||||
* gamma down so the Bellman target approaches the immediate reward,
|
||||
* preventing overconfident extrapolation into novel states.
|
||||
* gamma_eff = gamma / (1 + lambda * prediction_error) */
|
||||
float gamma_eff = gamma;
|
||||
if (curiosity_q_penalty_lambda > 0.0f) {
|
||||
float cur_err = curiosity_errors[sample_id];
|
||||
gamma_eff = gamma / (1.0f + curiosity_q_penalty_lambda * cur_err);
|
||||
}
|
||||
|
||||
float total_ce = 0.0f;
|
||||
|
||||
for (int d = 0; d < NUM_BRANCHES; d++) {
|
||||
@@ -376,7 +405,7 @@ extern "C" __global__ void c51_loss_batched(
|
||||
shmem_lp[j] = 0.0f;
|
||||
__syncthreads();
|
||||
|
||||
block_bellman_project_f(shmem_proj, reward, done, gamma,
|
||||
block_bellman_project_f(shmem_proj, reward, done, gamma_eff,
|
||||
shmem_lp, shmem_support, shmem_reduce, tid,
|
||||
num_atoms, v_min, v_max, delta_z);
|
||||
__syncthreads();
|
||||
@@ -408,6 +437,17 @@ extern "C" __global__ void c51_loss_batched(
|
||||
|
||||
float avg_ce = total_ce / (float)NUM_BRANCHES;
|
||||
|
||||
/* #18 Asymmetric DD loss: penalize overconfident Q-values during drawdown.
|
||||
* For C51, overconfidence = high cross-entropy (distribution mismatch).
|
||||
* Scale CE up when in drawdown — makes the loss surface steeper for
|
||||
* samples where the model is wrong AND the portfolio is suffering. */
|
||||
if (asymmetric_dd_weight > 0.0f && drawdown_depths != NULL) {
|
||||
float dd = drawdown_depths[sample_id];
|
||||
if (dd > 0.0f) {
|
||||
avg_ce *= (1.0f + dd * dd * asymmetric_dd_weight);
|
||||
}
|
||||
}
|
||||
|
||||
if (tid == 0) {
|
||||
float clamped_ce = fminf(avg_ce, MAX_PER_SAMPLE_CE);
|
||||
float weighted_loss = clamped_ce * is_weight;
|
||||
|
||||
@@ -141,7 +141,18 @@ extern "C" __global__ void experience_state_gather(
|
||||
int N,
|
||||
int total_bars,
|
||||
int state_dim,
|
||||
int market_dim
|
||||
int market_dim,
|
||||
/* #22 Feature noise: add N(0, noise_scale) to features. 0.0 = disabled. */
|
||||
float feature_noise_scale,
|
||||
/* #23 Causal feature masking: per-feature mask (1.0=keep, 0.0=zero).
|
||||
* NULL pointer = no masking. Length = market_dim. */
|
||||
const float* __restrict__ feature_mask,
|
||||
/* #13 Vol normalization: divide return features [0..3] by this value.
|
||||
* 0.0 or negative = disabled. Precomputed on CPU per epoch. */
|
||||
float vol_normalizer,
|
||||
/* RNG state pointer for per-thread noise generation.
|
||||
* Shared with action_select kernel (same rng_states buffer). */
|
||||
unsigned int* rng_states
|
||||
) {
|
||||
int i = blockIdx.x * blockDim.x + threadIdx.x;
|
||||
if (i >= N) return;
|
||||
@@ -162,6 +173,41 @@ extern "C" __global__ void experience_state_gather(
|
||||
for (int k = 0; k < market_dim; k++)
|
||||
out[k] = mf_row[k];
|
||||
|
||||
/* ── #13 Vol normalization: scale return features [0..3] by 1/vol ── */
|
||||
if (vol_normalizer > 0.0f && market_dim >= 4) {
|
||||
float inv_vol = 1.0f / vol_normalizer;
|
||||
for (int k = 0; k < 4; k++) {
|
||||
float v = (float)out[k] * inv_vol;
|
||||
out[k] = bf16(v);
|
||||
}
|
||||
}
|
||||
|
||||
/* ── #23 Causal feature masking: zero out masked features ── */
|
||||
if (feature_mask != NULL) {
|
||||
for (int k = 0; k < market_dim; k++) {
|
||||
if (feature_mask[k] < 0.5f) {
|
||||
out[k] = bf16_zero();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* ── #22 Feature noise injection: N(0, scale) per feature ── */
|
||||
if (feature_noise_scale > 0.0f && rng_states != NULL) {
|
||||
/* LCG-based fast noise (same RNG as action_select uses) */
|
||||
unsigned int rng = rng_states[i];
|
||||
for (int k = 0; k < market_dim; k++) {
|
||||
/* Box-Muller-lite: two uniform → one gaussian */
|
||||
rng = rng * 1664525u + 1013904223u;
|
||||
float u1 = (float)(rng & 0xFFFF) / 65536.0f + 1e-6f;
|
||||
rng = rng * 1664525u + 1013904223u;
|
||||
float u2 = (float)(rng & 0xFFFF) / 65536.0f;
|
||||
float noise = sqrtf(-2.0f * logf(u1)) * cosf(6.2831853f * u2);
|
||||
float v = (float)out[k] + noise * feature_noise_scale;
|
||||
out[k] = bf16(v);
|
||||
}
|
||||
rng_states[i] = rng; /* write back RNG state */
|
||||
}
|
||||
|
||||
/* -- Portfolio features: [market_dim .. market_dim+8) --
|
||||
*
|
||||
* 8 informative features that let the Q-network SEE its trading context.
|
||||
@@ -533,7 +579,9 @@ extern "C" __global__ void experience_env_step(
|
||||
float contract_multiplier, /* e.g. 50.0 for ES, 20.0 for NQ */
|
||||
float margin_pct, /* e.g. 0.06 (6% initial margin) */
|
||||
float dd_threshold, /* drawdown fraction before penalty (0.02 = 2%) */
|
||||
float w_dd /* drawdown penalty weight (1.0 = full) */
|
||||
float w_dd, /* drawdown penalty weight (1.0 = full) */
|
||||
float beta_penalty, /* anti-correlation: penalize market-aligned returns (0.0 = off) */
|
||||
float trade_clustering_penalty /* #25 trade clustering: CV(inter-trade times) * weight (0=off) */
|
||||
) {
|
||||
int i = blockIdx.x * blockDim.x + threadIdx.x;
|
||||
if (i >= N) return;
|
||||
@@ -589,7 +637,11 @@ extern "C" __global__ void experience_env_step(
|
||||
float position = __bfloat162float(ps[0]);
|
||||
float cash = __bfloat162float(ps[1]);
|
||||
/* ps[2] = portfolio_value (updated at end) */
|
||||
/* ps[3:6] reserved (unused by reward v6) */
|
||||
/* ps[3:6]: #25 trade clustering tracking */
|
||||
float last_trade_t = __bfloat162float(ps[3]); /* timestep of last trade completion */
|
||||
float interval_sum = __bfloat162float(ps[4]); /* sum of inter-trade intervals */
|
||||
float interval_sq_sum = __bfloat162float(ps[5]); /* sum of squared intervals */
|
||||
float interval_count = __bfloat162float(ps[6]); /* number of intervals recorded */
|
||||
float peak_equity = __bfloat162float(ps[7]);
|
||||
float flat_counter = __bfloat162float(ps[8]);
|
||||
float prev_equity = __bfloat162float(ps[9]);
|
||||
@@ -937,6 +989,64 @@ extern "C" __global__ void experience_env_step(
|
||||
reward *= hold_scale;
|
||||
}
|
||||
|
||||
/* ---- Beta penalty: penalize market-correlated returns ---- */
|
||||
/* When a trade completes, compare the model's return to the market's
|
||||
* return over the same period. If they have the same sign AND the
|
||||
* model is directional (not flat), the return is beta — not alpha.
|
||||
* Scale reward by (1 - penalty * alignment) where alignment is the
|
||||
* cosine similarity of {model_return, market_return} projected to sign.
|
||||
*
|
||||
* This forces the model to generate returns uncorrelated with the
|
||||
* underlying — true alpha. Without this, the model rides bull
|
||||
* in-sample periods and dies OOS when the regime flips. */
|
||||
if (segment_complete && segment_hold_time > 1.0f && beta_penalty > 0.0f
|
||||
&& entry_price > 0.0f)
|
||||
{
|
||||
float mkt_return = (raw_close - entry_price) / entry_price;
|
||||
/* segment_return was computed above: realized PnL / equity */
|
||||
float model_return = (reward > 0.0f) ? 1.0f : ((reward < 0.0f) ? -1.0f : 0.0f);
|
||||
float mkt_sign = (mkt_return > 0.0f) ? 1.0f : ((mkt_return < 0.0f) ? -1.0f : 0.0f);
|
||||
|
||||
/* Alignment: +1 = same direction (beta riding), -1 = opposite, 0 = one is flat */
|
||||
float alignment = model_return * mkt_sign;
|
||||
|
||||
if (alignment > 0.0f) {
|
||||
/* Same direction → penalize proportionally to market move magnitude.
|
||||
* Larger market moves with aligned returns = more likely beta.
|
||||
* Cap the magnitude component at 1.0 (fully correlated). */
|
||||
float mkt_magnitude = fminf(fabsf(mkt_return) * 100.0f, 1.0f); /* scale: 1% move → full */
|
||||
float penalty_scale = 1.0f - beta_penalty * mkt_magnitude;
|
||||
penalty_scale = fmaxf(penalty_scale, 0.1f); /* Floor at 10% to preserve gradient */
|
||||
reward *= penalty_scale;
|
||||
}
|
||||
/* Counter-trend alpha (alignment < 0) gets no bonus — just no penalty. */
|
||||
}
|
||||
|
||||
/* ---- #25 Trade clustering penalty ---- */
|
||||
/* Track inter-trade intervals and penalize temporally clustered trades.
|
||||
* CV(intervals) * weight is subtracted from reward at each trade exit.
|
||||
* This forces the model to find strategies that work across the entire
|
||||
* episode rather than exploiting specific micro-patterns in bursts. */
|
||||
if (segment_complete && segment_hold_time > 0.0f && trade_clustering_penalty > 0.0f) {
|
||||
float t_now = (float)current_t;
|
||||
if (last_trade_t > 0.0f) {
|
||||
float interval = t_now - last_trade_t;
|
||||
interval_sum += interval;
|
||||
interval_sq_sum += interval * interval;
|
||||
interval_count += 1.0f;
|
||||
/* CV = std / mean = sqrt(E[X^2] - E[X]^2) / E[X] */
|
||||
if (interval_count >= 2.0f) {
|
||||
float mean_int = interval_sum / interval_count;
|
||||
float var_int = interval_sq_sum / interval_count - mean_int * mean_int;
|
||||
if (var_int > 0.0f && mean_int > 0.0f) {
|
||||
float cv = sqrtf(var_int) / mean_int;
|
||||
reward -= cv * trade_clustering_penalty;
|
||||
}
|
||||
}
|
||||
}
|
||||
last_trade_t = t_now;
|
||||
}
|
||||
|
||||
/* ---- Drawdown penalty (from trade_physics.cuh) ---- */
|
||||
float equity_now = cash + position * raw_next;
|
||||
reward += compute_drawdown_penalty(equity_now, peak_equity, dd_threshold, w_dd);
|
||||
@@ -976,7 +1086,11 @@ extern "C" __global__ void experience_env_step(
|
||||
ps[0] = __float2bfloat16(position);
|
||||
ps[1] = __float2bfloat16(cash);
|
||||
ps[2] = __float2bfloat16(new_portfolio_value);
|
||||
/* ps[3:6] reserved — reward v6 does not use DSR/PnL EMA */
|
||||
/* ps[3:6]: #25 trade clustering tracking */
|
||||
ps[3] = __float2bfloat16(last_trade_t);
|
||||
ps[4] = __float2bfloat16(interval_sum);
|
||||
ps[5] = __float2bfloat16(interval_sq_sum);
|
||||
ps[6] = __float2bfloat16(interval_count);
|
||||
ps[7] = __float2bfloat16(peak_equity);
|
||||
ps[8] = __float2bfloat16(flat_counter);
|
||||
ps[9] = __float2bfloat16(new_portfolio_value); /* prev_equity = current equity for next step */
|
||||
|
||||
@@ -177,6 +177,16 @@ pub struct GpuDqnTrainConfig {
|
||||
pub c51_warmup_epochs: usize,
|
||||
/// CQL regularization strength (0.0 = disabled, 0.1 = mild, 1.0 = full offline-RL).
|
||||
pub cql_alpha: f32,
|
||||
/// Curiosity Q-penalty lambda: scales gamma by 1/(1 + lambda * curiosity_error).
|
||||
/// 0.0 = disabled. Typical range 0.5-5.0. Higher = more aggressive penalization
|
||||
/// of Q-values in novel states (prevents overconfident extrapolation OOS).
|
||||
pub curiosity_q_penalty_lambda: f32,
|
||||
/// #18 Asymmetric drawdown loss weight. Extra penalty on Q-overestimation
|
||||
/// when portfolio is in drawdown. 0.0 = disabled.
|
||||
pub asymmetric_dd_weight: f32,
|
||||
/// #27 Ensemble disagreement penalty weight. Q_target -= weight * ensemble_std.
|
||||
/// 0.0 = disabled.
|
||||
pub ensemble_disagreement_penalty: f32,
|
||||
}
|
||||
|
||||
impl Default for GpuDqnTrainConfig {
|
||||
@@ -212,6 +222,9 @@ impl Default for GpuDqnTrainConfig {
|
||||
entropy_coefficient: 0.001, // Must be ≤ reward magnitude (~0.001). Old 0.01 was 10x reward → entropy dominated learning.
|
||||
c51_warmup_epochs: 5,
|
||||
cql_alpha: 0.1,
|
||||
curiosity_q_penalty_lambda: 0.0,
|
||||
asymmetric_dd_weight: 0.0,
|
||||
ensemble_disagreement_penalty: 0.0,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -525,6 +538,33 @@ pub struct GpuDqnTrainer {
|
||||
scalars_readback_buf: CudaSlice<half::bf16>,
|
||||
scalars_readback_host: [f32; 2],
|
||||
|
||||
// ── Curiosity Q-penalty ─────────────────────────────────────────
|
||||
/// Per-sample curiosity prediction error buffer [B] (f32).
|
||||
/// Populated by curiosity_inference_error kernel before C51/MSE loss.
|
||||
/// Zero-filled when curiosity is disabled (lambda=0).
|
||||
curiosity_error_buf: CudaSlice<f32>,
|
||||
/// #18 Per-sample drawdown depths for asymmetric DD loss.
|
||||
/// Zero-filled when asymmetric_dd_weight=0.
|
||||
drawdown_depths_buf: CudaSlice<f32>,
|
||||
/// #18 Asymmetric DD loss weight (from config).
|
||||
asymmetric_dd_weight: f32,
|
||||
/// #27 Per-sample ensemble std for disagreement Q-penalty.
|
||||
/// Populated by `upload_ensemble_std()` before loss kernel launch.
|
||||
/// Zero-filled when ensemble is disabled.
|
||||
ensemble_std_buf: CudaSlice<f32>,
|
||||
/// #27 Ensemble disagreement penalty weight. Q_target -= weight * ensemble_std.
|
||||
ensemble_disagreement_weight: f32,
|
||||
/// Compiled curiosity inference-only kernel function.
|
||||
curiosity_inference_func: CudaFunction,
|
||||
/// Raw device pointers to curiosity weights (set via `set_curiosity_weights()`).
|
||||
/// These point into the `CuriosityWeightSet` owned by the experience collector.
|
||||
/// Values are updated in-place by the curiosity trainer — addresses are stable.
|
||||
/// u64::MAX = not set (curiosity disabled).
|
||||
curiosity_w1_ptr: u64,
|
||||
curiosity_b1_ptr: u64,
|
||||
curiosity_w2_ptr: u64,
|
||||
curiosity_b2_ptr: u64,
|
||||
|
||||
// ── BF16 batch staging buffers ──────────────────────────────────
|
||||
// Pre-allocated CudaSlice<u16> buffers for DtoD copy of BF16 tensors
|
||||
// from GpuBatch (states, next_states only — stored in BF16 ring buffer).
|
||||
@@ -2229,6 +2269,23 @@ impl GpuDqnTrainer {
|
||||
let initial_loss_mode = if config.c51_warmup_epochs > 0 { LossMode::Mse } else { LossMode::C51 };
|
||||
let initial_c51_alpha = if config.c51_warmup_epochs > 0 { 0.0 } else { 1.0 };
|
||||
|
||||
// ── Curiosity Q-penalty: inference-only kernel + error buffer ──
|
||||
let curiosity_error_buf = stream.alloc_zeros::<f32>(b)
|
||||
.map_err(|e| MLError::ModelError(format!("alloc curiosity_error_buf: {e}")))?;
|
||||
let drawdown_depths_buf = stream.alloc_zeros::<f32>(b)
|
||||
.map_err(|e| MLError::ModelError(format!("alloc drawdown_depths_buf: {e}")))?;
|
||||
let ensemble_std_buf = stream.alloc_zeros::<f32>(b)
|
||||
.map_err(|e| MLError::ModelError(format!("alloc ensemble_std_buf: {e}")))?;
|
||||
let curiosity_inference_func = {
|
||||
static CURIOSITY_INFERENCE_CUBIN: &[u8] = include_bytes!(
|
||||
concat!(env!("OUT_DIR"), "/curiosity_inference_kernel.cubin")
|
||||
);
|
||||
let module = stream.context().load_cubin(CURIOSITY_INFERENCE_CUBIN.to_vec())
|
||||
.map_err(|e| MLError::ModelError(format!("curiosity_inference cubin: {e}")))?;
|
||||
module.load_function("curiosity_inference_error")
|
||||
.map_err(|e| MLError::ModelError(format!("curiosity_inference_error load: {e}")))?
|
||||
};
|
||||
|
||||
let ptrs = {
|
||||
let _evt_guard = EventTrackingGuard::new(stream.context());
|
||||
CachedPtrs {
|
||||
@@ -2284,6 +2341,8 @@ impl GpuDqnTrainer {
|
||||
}
|
||||
};
|
||||
|
||||
let dd_weight = config.asymmetric_dd_weight;
|
||||
let ens_weight = config.ensemble_disagreement_penalty;
|
||||
Ok(Self {
|
||||
config,
|
||||
stream,
|
||||
@@ -2411,6 +2470,16 @@ impl GpuDqnTrainer {
|
||||
cql_logit_grad_kernel,
|
||||
cql_d_value_logits,
|
||||
cql_d_adv_logits,
|
||||
curiosity_error_buf,
|
||||
drawdown_depths_buf,
|
||||
asymmetric_dd_weight: dd_weight,
|
||||
ensemble_std_buf,
|
||||
ensemble_disagreement_weight: ens_weight,
|
||||
curiosity_inference_func,
|
||||
curiosity_w1_ptr: u64::MAX,
|
||||
curiosity_b1_ptr: u64::MAX,
|
||||
curiosity_w2_ptr: u64::MAX,
|
||||
curiosity_b2_ptr: u64::MAX,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -3218,6 +3287,11 @@ impl GpuDqnTrainer {
|
||||
// C51 writes to d_value_logits_buf / d_adv_logits_buf (main)
|
||||
// Then SAXPY blends: main = α * main + (1-α) * scratch
|
||||
|
||||
// ── Curiosity Q-penalty: compute per-sample prediction error ──
|
||||
// Must run BEFORE both MSE and C51 loss kernels (both read curiosity_error_buf).
|
||||
// Captured in graph — weight pointers are stable (in-place update by curiosity trainer).
|
||||
self.launch_curiosity_inference()?;
|
||||
|
||||
// MSE path → scratch buffers
|
||||
self.stream.memset_zeros(&mut self.d_value_logits_mse)
|
||||
.map_err(|e| MLError::ModelError(format!("zero d_value_mse: {e}")))?;
|
||||
@@ -3566,6 +3640,65 @@ impl GpuDqnTrainer {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Set curiosity weight raw pointers from the experience collector's `CuriosityWeightSet`.
|
||||
///
|
||||
/// Must be called once after the experience collector is initialized. The pointers
|
||||
/// are stable (same CudaSlice addresses) — only values change via in-place update.
|
||||
/// After this call, the curiosity inference kernel can be captured in the CUDA Graph.
|
||||
pub fn set_curiosity_weights(
|
||||
&mut self,
|
||||
w1: &CudaSlice<half::bf16>,
|
||||
b1: &CudaSlice<half::bf16>,
|
||||
w2: &CudaSlice<half::bf16>,
|
||||
b2: &CudaSlice<half::bf16>,
|
||||
) {
|
||||
self.curiosity_w1_ptr = w1.raw_ptr();
|
||||
self.curiosity_b1_ptr = b1.raw_ptr();
|
||||
self.curiosity_w2_ptr = w2.raw_ptr();
|
||||
self.curiosity_b2_ptr = b2.raw_ptr();
|
||||
// Invalidate CUDA Graphs so next capture includes curiosity inference
|
||||
self.graph_forward = None;
|
||||
self.graph_adam = None;
|
||||
}
|
||||
|
||||
/// Launch curiosity inference-only kernel on the current training batch.
|
||||
///
|
||||
/// Computes per-sample MSE prediction error from the curiosity forward model
|
||||
/// and writes to `curiosity_error_buf`. Called inside the CUDA Graph capture
|
||||
/// sequence, right before the loss kernel (C51 or MSE).
|
||||
///
|
||||
/// Requires `set_curiosity_weights()` to have been called first.
|
||||
fn launch_curiosity_inference(&self) -> Result<(), MLError> {
|
||||
if self.config.curiosity_q_penalty_lambda <= 0.0 || self.curiosity_w1_ptr == u64::MAX {
|
||||
return Ok(()); // Penalty disabled or weights not set — error buffer stays zero
|
||||
}
|
||||
let b = self.config.batch_size;
|
||||
let sd = self.config.state_dim as i32;
|
||||
let n = b as i32;
|
||||
let blocks = ((b + 255) / 256) as u32;
|
||||
unsafe {
|
||||
self.stream
|
||||
.launch_builder(&self.curiosity_inference_func)
|
||||
.arg(&self.ptrs.states_buf)
|
||||
.arg(&self.ptrs.actions_buf)
|
||||
.arg(&self.ptrs.next_states_buf)
|
||||
.arg(&self.curiosity_w1_ptr)
|
||||
.arg(&self.curiosity_b1_ptr)
|
||||
.arg(&self.curiosity_w2_ptr)
|
||||
.arg(&self.curiosity_b2_ptr)
|
||||
.arg(&self.curiosity_error_buf)
|
||||
.arg(&n)
|
||||
.arg(&sd)
|
||||
.launch(cudarc::driver::LaunchConfig {
|
||||
grid_dim: (blocks, 1, 1),
|
||||
block_dim: (256, 1, 1),
|
||||
shared_mem_bytes: 0,
|
||||
})
|
||||
.map_err(|e| MLError::ModelError(format!("curiosity_inference_error launch: {e}")))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Launch the standalone C51 loss kernel on pre-computed cuBLAS logit outputs.
|
||||
///
|
||||
/// Reads: on_v_logits, on_b_logits, tg_v_logits, tg_b_logits,
|
||||
@@ -3646,6 +3779,9 @@ impl GpuDqnTrainer {
|
||||
// ── Saved for backward (2) ──
|
||||
.arg(&self.save_current_lp)
|
||||
.arg(&self.save_projected)
|
||||
// ── Curiosity Q-penalty (2) ──
|
||||
.arg(&self.curiosity_error_buf)
|
||||
.arg(&self.config.curiosity_q_penalty_lambda)
|
||||
// ── Config (8) ──
|
||||
.arg(&gamma)
|
||||
.arg(&batch_i32)
|
||||
@@ -3655,6 +3791,12 @@ impl GpuDqnTrainer {
|
||||
.arg(&b0_i32)
|
||||
.arg(&b1_i32)
|
||||
.arg(&b2_i32)
|
||||
// ── #18 Asymmetric DD loss (2) ──
|
||||
.arg(&self.drawdown_depths_buf)
|
||||
.arg(&self.asymmetric_dd_weight)
|
||||
// ── #27 Ensemble disagreement (2) ──
|
||||
.arg(&self.ensemble_std_buf)
|
||||
.arg(&self.ensemble_disagreement_weight)
|
||||
.launch(LaunchConfig {
|
||||
grid_dim: (b as u32, 1, 1),
|
||||
block_dim: (256, 1, 1),
|
||||
@@ -3794,6 +3936,9 @@ impl GpuDqnTrainer {
|
||||
// save_projected = per-branch E[Q] values (3 floats per sample per branch)
|
||||
.arg(&self.save_current_lp)
|
||||
.arg(&self.save_projected)
|
||||
// ── Curiosity Q-penalty (2) ──
|
||||
.arg(&self.curiosity_error_buf)
|
||||
.arg(&self.config.curiosity_q_penalty_lambda)
|
||||
// ── Config (8) ──
|
||||
.arg(&gamma)
|
||||
.arg(&batch_i32)
|
||||
@@ -3803,6 +3948,12 @@ impl GpuDqnTrainer {
|
||||
.arg(&b0_i32)
|
||||
.arg(&b1_i32)
|
||||
.arg(&b2_i32)
|
||||
// ── #18 Asymmetric DD loss (2) ──
|
||||
.arg(&self.drawdown_depths_buf)
|
||||
.arg(&self.asymmetric_dd_weight)
|
||||
// ── #27 Ensemble disagreement (2) ──
|
||||
.arg(&self.ensemble_std_buf)
|
||||
.arg(&self.ensemble_disagreement_weight)
|
||||
.launch(LaunchConfig {
|
||||
grid_dim: (b as u32, 1, 1),
|
||||
block_dim: (256, 1, 1),
|
||||
|
||||
@@ -191,6 +191,9 @@ pub struct ExperienceCollectorConfig {
|
||||
pub dd_threshold: f32,
|
||||
/// Drawdown penalty weight (1.0 = full penalty)
|
||||
pub w_dd: f32,
|
||||
/// Anti-correlation beta penalty: scales down reward when model return
|
||||
/// aligns with market return (beta riding). 0.0 = disabled. 0.3 = moderate.
|
||||
pub beta_penalty: f32,
|
||||
/// Transaction cost multiplier (hyperopt-tunable, scales base 0.01% rate)
|
||||
pub tx_cost_multiplier: f32,
|
||||
/// UCB count-bonus coefficient for GPU action selection (0.0 = disabled)
|
||||
@@ -256,6 +259,21 @@ pub struct ExperienceCollectorConfig {
|
||||
/// CME initial margin ~6% of notional for equity index futures.
|
||||
/// Default: 0.06.
|
||||
pub margin_pct: f32,
|
||||
|
||||
// ── Gems & Pearls: generalization parameters ────────────────────
|
||||
|
||||
/// #25 Trade clustering penalty: CV(inter-trade intervals) * weight subtracted
|
||||
/// from reward at each trade completion. 0.0 = disabled.
|
||||
pub trade_clustering_penalty: f32,
|
||||
|
||||
/// #22 Feature noise scale: add N(0, scale) to each feature. 0.0 = disabled.
|
||||
pub feature_noise_scale: f32,
|
||||
/// #13 Vol normalizer: divide return features [0..3] by this value. 0.0 = disabled.
|
||||
/// Computed on CPU as realized vol of close-to-close returns.
|
||||
pub vol_normalizer: f32,
|
||||
/// #23 Feature mask: per-feature binary mask. None = no masking.
|
||||
/// Generated per-epoch on CPU, uploaded to GPU.
|
||||
pub feature_mask: Option<Vec<f32>>,
|
||||
}
|
||||
|
||||
impl Default for ExperienceCollectorConfig {
|
||||
@@ -279,6 +297,7 @@ impl Default for ExperienceCollectorConfig {
|
||||
loss_aversion: 1.5,
|
||||
dd_threshold: 0.02,
|
||||
w_dd: 1.0,
|
||||
beta_penalty: 0.0,
|
||||
tx_cost_multiplier: 1.0,
|
||||
count_bonus_coefficient: 0.0,
|
||||
q_clip_min: -200.0, // Reward v6: tighter than old -500 but covers v_range + safety margin
|
||||
@@ -309,6 +328,10 @@ impl Default for ExperienceCollectorConfig {
|
||||
spread_cost: 0.0, // default: no spread cost (overridden by hyperparams)
|
||||
contract_multiplier: 50.0,
|
||||
margin_pct: 0.06,
|
||||
trade_clustering_penalty: 0.0,
|
||||
feature_noise_scale: 0.0,
|
||||
vol_normalizer: 0.0,
|
||||
feature_mask: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -490,6 +513,9 @@ pub struct GpuExperienceCollector {
|
||||
// GPU-resident curiosity forward model trainer (None if curiosity disabled)
|
||||
curiosity_trainer: Option<GpuCuriosityTrainer>,
|
||||
|
||||
/// #23 Causal feature mask buffer [market_dim] f32 (1.0=keep, 0.0=zero). None = no masking.
|
||||
feature_mask_buf: Option<CudaSlice<f32>>,
|
||||
|
||||
// Pre-allocated pinned host buffers for DtoH transfers.
|
||||
pinned_states: PinnedHostBuf<f32>,
|
||||
pinned_actions: PinnedHostBuf<i32>,
|
||||
@@ -908,6 +934,7 @@ impl GpuExperienceCollector {
|
||||
ofi_gpu: ofi_placeholder,
|
||||
ofi_dim,
|
||||
curiosity_trainer,
|
||||
feature_mask_buf: None,
|
||||
pinned_states,
|
||||
pinned_actions,
|
||||
pinned_rewards,
|
||||
@@ -1202,6 +1229,28 @@ impl GpuExperienceCollector {
|
||||
for t in 0..timesteps {
|
||||
// ── 1. Gather states: [N, state_dim] ────────────────────────
|
||||
unsafe {
|
||||
// #22 feature noise scale, #13 vol normalizer (from config)
|
||||
let noise_scale = config.feature_noise_scale;
|
||||
let vol_norm = config.vol_normalizer;
|
||||
|
||||
// Upload feature mask if provided (once per epoch, not per timestep)
|
||||
if t == 0 {
|
||||
if let Some(ref mask_data) = config.feature_mask {
|
||||
let mut buf = self.stream.alloc_zeros::<f32>(mask_data.len())
|
||||
.map_err(|e| MLError::ModelError(format!("alloc feature_mask: {e}")))?;
|
||||
self.stream.memcpy_htod(mask_data, &mut buf)
|
||||
.map_err(|e| MLError::ModelError(format!("upload feature_mask: {e}")))?;
|
||||
self.feature_mask_buf = Some(buf);
|
||||
} else {
|
||||
self.feature_mask_buf = None;
|
||||
}
|
||||
}
|
||||
|
||||
// #23 feature mask pointer (0 = NULL = no masking)
|
||||
let mask_ptr: u64 = self.feature_mask_buf.as_ref()
|
||||
.map(|b| b.device_ptr(&self.stream).0)
|
||||
.unwrap_or(0u64);
|
||||
|
||||
self.stream
|
||||
.launch_builder(&self.state_gather_kernel)
|
||||
.arg(market_features_buf)
|
||||
@@ -1213,6 +1262,10 @@ impl GpuExperienceCollector {
|
||||
.arg(&total_bars)
|
||||
.arg(&sd)
|
||||
.arg(&md)
|
||||
.arg(&noise_scale) // #22 feature noise
|
||||
.arg(&mask_ptr) // #23 causal feature mask (NULL=disabled)
|
||||
.arg(&vol_norm) // #13 vol normalization
|
||||
.arg(&mut self.rng_states) // RNG for noise
|
||||
.launch(launch_cfg)
|
||||
.map_err(|e| MLError::ModelError(format!(
|
||||
"experience_state_gather t={t}: {e}"
|
||||
@@ -1363,6 +1416,8 @@ impl GpuExperienceCollector {
|
||||
.arg(&config.margin_pct) // initial margin fraction (e.g. 0.06 = 6%)
|
||||
.arg(&config.dd_threshold) // drawdown threshold before penalty (0.02 = 2%)
|
||||
.arg(&config.w_dd) // drawdown penalty weight
|
||||
.arg(&config.beta_penalty) // anti-correlation beta penalty
|
||||
.arg(&config.trade_clustering_penalty) // #25 trade clustering penalty
|
||||
.launch(launch_cfg)
|
||||
.map_err(|e| MLError::ModelError(format!(
|
||||
"experience_env_step t={t}: {e}"
|
||||
@@ -1594,6 +1649,14 @@ impl GpuExperienceCollector {
|
||||
self.curiosity_trainer.is_some()
|
||||
}
|
||||
|
||||
/// Access the curiosity weight set (for wiring to the fused trainer Q-penalty).
|
||||
/// The returned reference has stable CudaSlice addresses — values are updated
|
||||
/// in-place by the curiosity trainer, so raw pointers captured in CUDA Graphs
|
||||
/// remain valid.
|
||||
pub fn curiosity_weight_set(&self) -> &super::gpu_weights::CuriosityWeightSet {
|
||||
&self.curiosity_weights
|
||||
}
|
||||
|
||||
/// Disable curiosity training (e.g. after an async kernel crash).
|
||||
pub fn disable_curiosity(&mut self) {
|
||||
self.curiosity_trainer = None;
|
||||
|
||||
@@ -139,6 +139,10 @@ extern "C" __global__ void mse_loss_batched(
|
||||
__nv_bfloat16* __restrict__ save_current_lp, /* [B, NUM_BRANCHES, num_atoms] online probs */
|
||||
__nv_bfloat16* __restrict__ save_projected, /* [B, NUM_BRANCHES, num_atoms] [td, E_Q, 0..] */
|
||||
|
||||
/* ── Curiosity Q-penalty ─────────────────────────────────────── */
|
||||
const float* __restrict__ curiosity_errors, /* [B] per-sample curiosity prediction error */
|
||||
float curiosity_q_penalty_lambda, /* scaling factor: gamma *= 1/(1 + lambda * error) */
|
||||
|
||||
/* ── Config ───────────────────────────────────────────────────── */
|
||||
float gamma,
|
||||
int batch_size,
|
||||
@@ -147,7 +151,15 @@ extern "C" __global__ void mse_loss_batched(
|
||||
float v_max,
|
||||
int b0_size,
|
||||
int b1_size,
|
||||
int b2_size
|
||||
int b2_size,
|
||||
|
||||
/* ── #18 Asymmetric drawdown loss ────────────────────────────── */
|
||||
const float* __restrict__ drawdown_depths, /* [B] per-sample drawdown fraction (0=no DD) */
|
||||
float asymmetric_dd_weight, /* penalty weight (0.0=disabled) */
|
||||
|
||||
/* ── #27 Ensemble disagreement Q-penalty ─────────────────────── */
|
||||
const float* __restrict__ ensemble_std, /* [B] per-sample ensemble std (0=no penalty) */
|
||||
float ensemble_disagreement_weight /* Q_target -= weight * std (0.0=disabled) */
|
||||
) {
|
||||
/* Shared memory is now float (4 bytes/elem) — doubled from BF16 version */
|
||||
extern __shared__ float shmem_f[];
|
||||
@@ -223,6 +235,13 @@ extern "C" __global__ void mse_loss_batched(
|
||||
float done = dones[sample_id];
|
||||
float is_weight = is_weights[sample_id];
|
||||
|
||||
/* ── Curiosity Q-penalty: shrink gamma for novel/uncertain states ── */
|
||||
float gamma_eff = gamma;
|
||||
if (curiosity_q_penalty_lambda > 0.0f) {
|
||||
float cur_err = curiosity_errors[sample_id];
|
||||
gamma_eff = gamma / (1.0f + curiosity_q_penalty_lambda * cur_err);
|
||||
}
|
||||
|
||||
float total_mse = 0.0f;
|
||||
float total_abs_td = 0.0f;
|
||||
|
||||
@@ -331,10 +350,26 @@ extern "C" __global__ void mse_loss_batched(
|
||||
);
|
||||
|
||||
/* ═══ STEP d: Bellman target + MSE ═══════════════════════════ */
|
||||
float target_q = reward + gamma * (1.0f - done) * target_eq;
|
||||
float target_q = reward + gamma_eff * (1.0f - done) * target_eq;
|
||||
|
||||
/* #27 Ensemble disagreement: reduce Q-target when ensemble is uncertain.
|
||||
* High std = novel/uncertain state = conservative Q-value. */
|
||||
if (ensemble_disagreement_weight > 0.0f && ensemble_std != NULL) {
|
||||
target_q -= ensemble_disagreement_weight * ensemble_std[sample_id];
|
||||
}
|
||||
float td = online_eq - target_q;
|
||||
float mse = 0.5f * td * td;
|
||||
|
||||
/* #18 Asymmetric DD loss: extra penalty on Q-OVERestimation
|
||||
* when portfolio is in drawdown. Prevents holding losers.
|
||||
* L_dd = max(0, Q_pred - Q_target) * dd^2 * weight */
|
||||
if (asymmetric_dd_weight > 0.0f && drawdown_depths != NULL) {
|
||||
float dd = drawdown_depths[sample_id];
|
||||
if (dd > 0.0f && td > 0.0f) { /* overestimation + in drawdown */
|
||||
mse += td * dd * dd * asymmetric_dd_weight;
|
||||
}
|
||||
}
|
||||
|
||||
total_mse += mse;
|
||||
total_abs_td += fabsf(td);
|
||||
|
||||
|
||||
@@ -938,6 +938,61 @@ pub struct DQNHyperparameters {
|
||||
/// Shrink-and-perturb noise scale (Xavier)
|
||||
pub shrink_perturb_sigma: f64,
|
||||
|
||||
/// Adversarial regime injection: MaxDD threshold below which we suspect overfitting.
|
||||
/// If MaxDD stays below this for `adversarial_epochs_trigger` consecutive epochs,
|
||||
/// inject harsh market conditions to stress-test the policy.
|
||||
pub adversarial_dd_threshold: f64,
|
||||
/// Consecutive low-drawdown epochs before adversarial injection activates
|
||||
pub adversarial_epochs_trigger: usize,
|
||||
|
||||
/// Anti-correlation penalty: how much to penalize market-beta in rewards.
|
||||
/// At epoch boundary, Pearson correlation between model returns and market returns
|
||||
/// is computed. If |corr| > 0.3, next epoch reward_scale *= (1 - penalty * |corr|).
|
||||
/// Forces alpha generation over beta riding. 0.0 = disabled. Typical: 0.3-0.5.
|
||||
pub beta_penalty_strength: f64,
|
||||
|
||||
// ── Gems & Pearls: generalization techniques ────────────────────
|
||||
|
||||
/// #24 Anti-intuitive LR: when epoch Sharpe > threshold, INCREASE LR to
|
||||
/// kick the model out of overfit minima. When Sharpe < -threshold, decrease
|
||||
/// to stabilize. Opposite of standard practice.
|
||||
pub enable_anti_lr: bool,
|
||||
/// LR multiplier when epoch Sharpe is "good" (> threshold)
|
||||
pub anti_lr_good_mult: f64,
|
||||
/// LR multiplier when epoch Sharpe is "bad" (< -threshold)
|
||||
pub anti_lr_bad_mult: f64,
|
||||
/// Sharpe threshold for anti-intuitive LR switching
|
||||
pub anti_lr_sharpe_threshold: f64,
|
||||
|
||||
/// #27 Ensemble disagreement Q-penalty weight. When ensemble heads disagree
|
||||
/// (high std across Q-values), subtract weighted std from Q-target.
|
||||
/// Makes model conservative on novel/uncertain states. 0.0 = disabled.
|
||||
pub ensemble_disagreement_penalty: f64,
|
||||
|
||||
/// #23 Causal feature masking: each epoch, randomly zero out this fraction
|
||||
/// of features. Forces redundant representations (random forest principle).
|
||||
/// 0.0 = disabled, 0.4 = mask 40% of features each epoch.
|
||||
pub feature_mask_fraction: f64,
|
||||
|
||||
/// #22 Feature-wise noise injection: add Gaussian noise with sigma =
|
||||
/// noise_scale * std(feature). Strictly better than dropout for continuous
|
||||
/// features. 0.0 = disabled.
|
||||
pub feature_noise_scale: f64,
|
||||
|
||||
/// #13 Volatility regime normalization: divide return-based features by
|
||||
/// realized vol to make them scale-invariant across regimes.
|
||||
pub enable_vol_normalization: bool,
|
||||
|
||||
/// #25 Trade clustering penalty: penalize temporally clustered trades.
|
||||
/// CV(inter-trade intervals) * this weight is subtracted from reward.
|
||||
/// 0.0 = disabled.
|
||||
pub trade_clustering_penalty: f64,
|
||||
|
||||
/// #18 Asymmetric drawdown loss: extra penalty on Q-overestimation when
|
||||
/// portfolio is in drawdown. Weight for L_dd = max(0, Q_pred-Q_target) * dd^2.
|
||||
/// 0.0 = disabled.
|
||||
pub asymmetric_dd_weight: f64,
|
||||
|
||||
// Wave 16 Portfolio Features
|
||||
/// Enable action masking (filters invalid actions based on position limits)
|
||||
pub enable_action_masking: bool,
|
||||
@@ -1048,6 +1103,11 @@ pub struct DQNHyperparameters {
|
||||
/// Curiosity weight for intrinsic reward (0.0 = disabled, typical range: 0.0-0.5)
|
||||
/// Balances extrinsic reward from trading vs intrinsic reward from novelty
|
||||
pub curiosity_weight: f64,
|
||||
/// Curiosity Q-penalty lambda: scales Bellman gamma by 1/(1 + lambda * prediction_error).
|
||||
/// When the curiosity forward model can't predict next state well (high error),
|
||||
/// Q-values are discounted toward immediate reward — preventing overconfident
|
||||
/// extrapolation into novel/OOS states. 0.0 = disabled. Typical: 1.0-5.0.
|
||||
pub curiosity_q_penalty_lambda: f64,
|
||||
|
||||
// WAVE 26 P2.2: Gradient Accumulation
|
||||
/// Number of mini-batches to accumulate gradients over before optimizer step
|
||||
@@ -1424,6 +1484,22 @@ impl DQNHyperparameters {
|
||||
shrink_perturb_interval: 20, // Every 20 epochs (0=disabled)
|
||||
shrink_perturb_alpha: 0.85, // Keep 85% of weights, reinit 15%
|
||||
shrink_perturb_sigma: 0.01, // Xavier noise scale
|
||||
// Generalization: adversarial regime injection (stress-test low-DD runs)
|
||||
adversarial_dd_threshold: 0.005, // 0.5% MaxDD threshold — suspiciously low
|
||||
adversarial_epochs_trigger: 5, // 5 consecutive low-DD epochs triggers injection
|
||||
beta_penalty_strength: 0.3, // 30% reward reduction at full correlation
|
||||
|
||||
// Gems & Pearls: generalization techniques
|
||||
enable_anti_lr: true, // #24: anti-intuitive LR (kick out of overfit minima)
|
||||
anti_lr_good_mult: 3.0, // 3x LR when Sharpe is good (destabilize overfit)
|
||||
anti_lr_bad_mult: 0.3, // 0.3x LR when Sharpe is bad (stabilize)
|
||||
anti_lr_sharpe_threshold: 0.3, // Sharpe threshold for switching
|
||||
ensemble_disagreement_penalty: 0.3, // #27: penalize Q-targets by ensemble std
|
||||
feature_mask_fraction: 0.3, // #23: mask 30% of features each epoch
|
||||
feature_noise_scale: 0.1, // #22: add N(0, 0.1*std) noise to features
|
||||
enable_vol_normalization: true, // #13: divide returns by realized vol
|
||||
trade_clustering_penalty: 0.05, // #25: penalize temporally clustered trades
|
||||
asymmetric_dd_weight: 0.5, // #18: extra loss on Q-overestimation in drawdown
|
||||
|
||||
// Wave 16 Portfolio Features (default: ALL ENABLED)
|
||||
enable_action_masking: true, // Default: action masking enabled
|
||||
@@ -1491,6 +1567,7 @@ impl DQNHyperparameters {
|
||||
|
||||
// WAVE 26 P1.8: Curiosity-Driven Exploration
|
||||
curiosity_weight: 0.1, // Default: enabled (ungates GPU experience collector; hyperopt tunes 0.01-0.5)
|
||||
curiosity_q_penalty_lambda: 1.0, // Default: moderate penalty on Q-values in novel states
|
||||
|
||||
// WAVE 26 P2.2: Gradient Accumulation
|
||||
gradient_accumulation_steps: 1, // Default: no accumulation (standard training)
|
||||
|
||||
@@ -205,6 +205,9 @@ impl FusedTrainingCtx {
|
||||
entropy_coefficient: dqn.config.entropy_coefficient as f32,
|
||||
c51_warmup_epochs: hyperparams.c51_warmup_epochs,
|
||||
cql_alpha: hyperparams.cql_alpha as f32,
|
||||
curiosity_q_penalty_lambda: hyperparams.curiosity_q_penalty_lambda as f32,
|
||||
asymmetric_dd_weight: hyperparams.asymmetric_dd_weight as f32,
|
||||
ensemble_disagreement_penalty: hyperparams.ensemble_disagreement_penalty as f32,
|
||||
};
|
||||
|
||||
// Extract weight sets from VarMaps (online + target)
|
||||
@@ -1129,6 +1132,16 @@ impl FusedTrainingCtx {
|
||||
self.trainer.total_actions()
|
||||
}
|
||||
|
||||
/// Set curiosity weight pointers from the experience collector's `CuriosityWeightSet`.
|
||||
/// Must be called once after the experience collector is initialized.
|
||||
/// Invalidates CUDA Graphs so the curiosity inference kernel is captured in the next replay.
|
||||
pub(crate) fn set_curiosity_weights(
|
||||
&mut self,
|
||||
weights: &crate::cuda_pipeline::gpu_weights::CuriosityWeightSet,
|
||||
) {
|
||||
self.trainer.set_curiosity_weights(&weights.w1, &weights.b1, &weights.w2, &weights.b2);
|
||||
}
|
||||
|
||||
/// Compute Q-value statistics entirely on GPU — 20-byte readback (5 scalars).
|
||||
pub(crate) fn compute_q_stats(
|
||||
&mut self,
|
||||
|
||||
@@ -51,7 +51,8 @@ impl DQNTrainer {
|
||||
// Align input_dim to 8 so the log matches the actual model dimensions.
|
||||
// (device not yet created, so use the formula directly — CUDA always aligns)
|
||||
let ofi_pre = hyperparams.mbp10_data_dir.is_some();
|
||||
let input_dim: usize = if ofi_pre { 56 } else { 48 }; // (53+7)&!7=56, (45+7)&!7=48
|
||||
// 42 market + 8 portfolio + 16 multi-timeframe = 66 base, +8 OFI = 74
|
||||
let input_dim: usize = if ofi_pre { 80 } else { 72 }; // (74+7)&!7=80, (66+7)&!7=72
|
||||
let output_dim: usize = 5;
|
||||
let hidden_dims: Vec<usize> = match hyperparams.hidden_dim_base {
|
||||
Some(base) => {
|
||||
@@ -227,17 +228,20 @@ impl DQNTrainer {
|
||||
);
|
||||
|
||||
// Create DQN configuration
|
||||
// 42-feature architecture: OHLCV, technical, patterns, volume, time, statistical, regime
|
||||
// Portfolio features (3) are populated via PortfolioTracker → 45 total state_dim
|
||||
// With MBP-10 OFI features: +8 OFI features → 53 total
|
||||
// State layout (GPU pipeline):
|
||||
// [0..42) 42 market features (OHLCV, technical, patterns, volume, time, statistical)
|
||||
// [42..50) 8 portfolio features (position, P&L, drawdown, etc.)
|
||||
// [50..66) 16 multi-timeframe features (4 windows x 4 features)
|
||||
// [66..74) 8 OFI features (optional, from MBP-10 order book data)
|
||||
// Raw state_dim: 66 without OFI, 74 with OFI.
|
||||
//
|
||||
// GpuTensor core alignment: state_dim is rounded up to the next multiple of 8
|
||||
// (53→56, 45→48) so that cuBLAS dispatches BF16 HMMA instructions instead
|
||||
// (66->72, 74->80) so that cuBLAS dispatches BF16 HMMA instructions instead
|
||||
// of falling back to scalar FMA. The extra columns are zero-padded at the
|
||||
// data pipeline boundaries (GpuPreloadedData and train_batch CPU path).
|
||||
let ofi_enabled = hyperparams.mbp10_data_dir.is_some();
|
||||
let raw_state_dim = if ofi_enabled { 74 } else { 66 }; // 42 market + 8 portfolio + (8 OFI)
|
||||
let state_dim = (raw_state_dim + 7) & !7; // aligned: 64 with OFI, 56 without
|
||||
let raw_state_dim = if ofi_enabled { 74 } else { 66 }; // 42 market + 8 portfolio + 16 MTF + (8 OFI)
|
||||
let state_dim = (raw_state_dim + 7) & !7; // aligned: 80 with OFI, 72 without
|
||||
|
||||
// Resolve gradient clip norm ONCE — single source of truth (Bug #4 fix).
|
||||
let grad_norm = hyperparams.gradient_clip_norm.unwrap_or(10.0);
|
||||
@@ -711,6 +715,10 @@ impl DQNTrainer {
|
||||
// Q-value estimation: periodic (every 50 steps) instead of every step
|
||||
cached_avg_q: 0.0,
|
||||
q_estimation_counter: 0,
|
||||
consecutive_low_dd_epochs: 0,
|
||||
adversarial_active: false,
|
||||
epoch_feature_mask: None,
|
||||
epoch_vol_normalizer: 0.0,
|
||||
epoch_q_min: f32::INFINITY,
|
||||
epoch_q_max: f32::NEG_INFINITY,
|
||||
|
||||
|
||||
@@ -217,6 +217,15 @@ pub struct DQNTrainer {
|
||||
/// Epoch-level Q-value extremes accumulated from per-step compute_q_stats
|
||||
pub(crate) epoch_q_min: f32,
|
||||
pub(crate) epoch_q_max: f32,
|
||||
/// Consecutive low-drawdown epochs (for adversarial regime injection)
|
||||
pub(crate) consecutive_low_dd_epochs: usize,
|
||||
/// Whether adversarial regime is active this epoch
|
||||
pub(crate) adversarial_active: bool,
|
||||
|
||||
/// #23 Causal feature mask for current epoch (None = no masking)
|
||||
pub(crate) epoch_feature_mask: Option<Vec<f32>>,
|
||||
/// #13 Vol normalizer for current epoch (0.0 = disabled)
|
||||
pub(crate) epoch_vol_normalizer: f32,
|
||||
|
||||
/// Pre-uploaded GPU training data (set once, reused across epochs)
|
||||
pub(crate) gpu_data: Option<DqnGpuData>,
|
||||
|
||||
@@ -246,6 +246,52 @@ impl DQNTrainer {
|
||||
self.init_gpu_experience_collector().await?;
|
||||
let phase1_ms = phase1_start.elapsed().as_secs_f64() * 1000.0;
|
||||
|
||||
// ── #23 Causal feature masking: random feature subset each epoch ──
|
||||
{
|
||||
let frac = self.hyperparams.feature_mask_fraction;
|
||||
if frac > 0.0 && frac < 1.0 {
|
||||
use rand::Rng;
|
||||
let mut mask_rng = rand::thread_rng();
|
||||
let n_features = 42_usize;
|
||||
let n_mask = (n_features as f64 * frac).round() as usize;
|
||||
let mut mask = vec![1.0_f32; n_features];
|
||||
let mut indices: Vec<usize> = (0..n_features).collect();
|
||||
for i in 0..n_mask.min(n_features) {
|
||||
let j = mask_rng.gen_range(i..n_features);
|
||||
indices.swap(i, j);
|
||||
}
|
||||
for &idx in &indices[..n_mask] {
|
||||
mask[idx] = 0.0;
|
||||
}
|
||||
if epoch % 20 == 0 {
|
||||
info!(
|
||||
epoch = epoch + 1,
|
||||
masked_count = n_mask,
|
||||
"Causal feature masking: {}/{} features zeroed this epoch",
|
||||
n_mask, n_features
|
||||
);
|
||||
}
|
||||
self.epoch_feature_mask = Some(mask);
|
||||
} else {
|
||||
self.epoch_feature_mask = None;
|
||||
}
|
||||
}
|
||||
|
||||
// ── #13 Vol normalization: compute realized vol from training data ──
|
||||
self.epoch_vol_normalizer = if self.hyperparams.enable_vol_normalization
|
||||
&& training_data.len() > 20
|
||||
{
|
||||
let returns: Vec<f64> = training_data.iter()
|
||||
.map(|(fv, _)| fv[3])
|
||||
.collect();
|
||||
let n = returns.len() as f64;
|
||||
let mean = returns.iter().sum::<f64>() / n;
|
||||
let var = returns.iter().map(|r| (r - mean).powi(2)).sum::<f64>() / (n - 1.0);
|
||||
var.sqrt().max(1e-8) as f32
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
|
||||
// ── Phase 2: GPU experience collection ──
|
||||
let phase2_start = std::time::Instant::now();
|
||||
let gpu_experiences_collected = self.collect_gpu_experiences(
|
||||
@@ -832,6 +878,14 @@ impl DQNTrainer {
|
||||
.collect();
|
||||
collector.upload_ofi_features(&flat)?;
|
||||
}
|
||||
// Wire curiosity weights to fused trainer for Q-penalty
|
||||
if let Some(ref mut fused_ctx) = self.fused_ctx {
|
||||
if collector.has_curiosity_trainer() {
|
||||
fused_ctx.set_curiosity_weights(collector.curiosity_weight_set());
|
||||
info!("Curiosity Q-penalty: weights wired to fused trainer (lambda={:.2})",
|
||||
self.hyperparams.curiosity_q_penalty_lambda);
|
||||
}
|
||||
}
|
||||
self.gpu_experience_collector = Some(collector);
|
||||
}
|
||||
Err(e) => {
|
||||
@@ -1038,6 +1092,12 @@ impl DQNTrainer {
|
||||
|
||||
// Expert ratio already set above (before collector borrow).
|
||||
|
||||
// Adversarial regime: harsh multipliers to stress-test the policy
|
||||
let adversarial = self.adversarial_active;
|
||||
if adversarial {
|
||||
info!("Adversarial regime ACTIVE this epoch: 3x spread, 2x tx_cost, 0.5x fill");
|
||||
}
|
||||
|
||||
let config = ExperienceCollectorConfig {
|
||||
n_episodes,
|
||||
timesteps_per_episode: timesteps,
|
||||
@@ -1048,10 +1108,13 @@ impl DQNTrainer {
|
||||
enable_action_masking: self.enable_action_masking,
|
||||
curiosity_scale: if self.curiosity_module.is_some() { 1.0 } else { 0.0 },
|
||||
loss_aversion: self.hyperparams.loss_aversion as f32,
|
||||
tx_cost_multiplier: if dr {
|
||||
epoch_rng.gen_range(0.5_f32..2.5)
|
||||
} else {
|
||||
self.hyperparams.transaction_cost_multiplier as f32
|
||||
tx_cost_multiplier: {
|
||||
let base = if dr {
|
||||
epoch_rng.gen_range(0.5_f32..2.5)
|
||||
} else {
|
||||
self.hyperparams.transaction_cost_multiplier as f32
|
||||
};
|
||||
if adversarial { base * 2.0 } else { base }
|
||||
},
|
||||
count_bonus_coefficient: self.hyperparams.count_bonus_coefficient
|
||||
.unwrap_or(0.0) as f32,
|
||||
@@ -1068,16 +1131,22 @@ impl DQNTrainer {
|
||||
num_atoms: self.hyperparams.num_atoms as i32,
|
||||
v_min: self.hyperparams.v_min as f32,
|
||||
v_max: self.hyperparams.v_max as f32,
|
||||
fill_median_spread: if dr {
|
||||
self.hyperparams.avg_spread as f32 * epoch_rng.gen_range(0.5_f32..3.0)
|
||||
} else {
|
||||
self.hyperparams.avg_spread as f32
|
||||
fill_median_spread: {
|
||||
let base = if dr {
|
||||
self.hyperparams.avg_spread as f32 * epoch_rng.gen_range(0.5_f32..3.0)
|
||||
} else {
|
||||
self.hyperparams.avg_spread as f32
|
||||
};
|
||||
if adversarial { base * 3.0 } else { base }
|
||||
},
|
||||
fill_median_vol: self.median_vol as f32,
|
||||
fill_ioc_fill_prob: if dr {
|
||||
epoch_rng.gen_range(0.65_f32..0.95)
|
||||
} else {
|
||||
self.hyperparams.fill_ioc_fill_prob as f32
|
||||
fill_ioc_fill_prob: {
|
||||
let base = if dr {
|
||||
epoch_rng.gen_range(0.65_f32..0.95)
|
||||
} else {
|
||||
self.hyperparams.fill_ioc_fill_prob as f32
|
||||
};
|
||||
if adversarial { base * 0.5 } else { base }
|
||||
},
|
||||
fill_limit_fill_min: if dr {
|
||||
epoch_rng.gen_range(0.15_f32..0.45)
|
||||
@@ -1106,17 +1175,26 @@ impl DQNTrainer {
|
||||
min_hold_bars: self.hyperparams.min_hold_bars as i32,
|
||||
// spread_cost = tick_size * multiplier * fraction
|
||||
// Matches backtest_env_kernel's spread_cost from GpuBacktestConfig
|
||||
spread_cost: if dr {
|
||||
(self.hyperparams.tick_size * self.hyperparams.contract_multiplier
|
||||
* epoch_rng.gen_range(0.3..0.8) as f64) as f32
|
||||
} else {
|
||||
(self.hyperparams.tick_size * self.hyperparams.contract_multiplier
|
||||
* self.hyperparams.fill_spread_cost_frac) as f32
|
||||
spread_cost: {
|
||||
let base = if dr {
|
||||
(self.hyperparams.tick_size * self.hyperparams.contract_multiplier
|
||||
* epoch_rng.gen_range(0.3..0.8) as f64) as f32
|
||||
} else {
|
||||
(self.hyperparams.tick_size * self.hyperparams.contract_multiplier
|
||||
* self.hyperparams.fill_spread_cost_frac) as f32
|
||||
};
|
||||
if adversarial { base * 3.0 } else { base }
|
||||
},
|
||||
contract_multiplier: self.hyperparams.contract_multiplier as f32,
|
||||
margin_pct: self.hyperparams.margin_pct as f32,
|
||||
dd_threshold: self.hyperparams.dd_threshold as f32,
|
||||
w_dd: self.hyperparams.w_dd as f32,
|
||||
beta_penalty: self.hyperparams.beta_penalty_strength as f32,
|
||||
// Gems & Pearls generalization params
|
||||
trade_clustering_penalty: self.hyperparams.trade_clustering_penalty as f32,
|
||||
feature_noise_scale: self.hyperparams.feature_noise_scale as f32,
|
||||
vol_normalizer: self.epoch_vol_normalizer,
|
||||
feature_mask: self.epoch_feature_mask.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
@@ -2009,7 +2087,33 @@ impl DQNTrainer {
|
||||
|
||||
// LR scheduler
|
||||
self.lr_scheduler.step();
|
||||
let current_lr = self.lr_scheduler.get_lr();
|
||||
let mut current_lr = self.lr_scheduler.get_lr();
|
||||
|
||||
// #24 Anti-intuitive LR: use PREVIOUS epoch's Sharpe to adjust LR.
|
||||
// When model was doing well, INCREASE LR to kick out of overfit minima.
|
||||
// When struggling, decrease to stabilize. Opposite of standard practice.
|
||||
if self.hyperparams.enable_anti_lr && !self.sharpe_history.is_empty() {
|
||||
let prev_sharpe = *self.sharpe_history.last().unwrap();
|
||||
let thresh = self.hyperparams.anti_lr_sharpe_threshold;
|
||||
let base_lr = self.lr_scheduler.get_initial_lr();
|
||||
let anti_mult = if prev_sharpe > thresh {
|
||||
self.hyperparams.anti_lr_good_mult
|
||||
} else if prev_sharpe < -thresh {
|
||||
self.hyperparams.anti_lr_bad_mult
|
||||
} else {
|
||||
1.0
|
||||
};
|
||||
current_lr = (current_lr * anti_mult).clamp(base_lr * 0.1, base_lr * 5.0);
|
||||
if (anti_mult - 1.0).abs() > 0.01 {
|
||||
info!(
|
||||
epoch = epoch + 1, prev_sharpe = %format!("{:.3}", prev_sharpe),
|
||||
anti_mult = %format!("{:.1}x", anti_mult),
|
||||
lr = %format!("{:.2e}", current_lr),
|
||||
"Anti-intuitive LR adjustment"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if epoch % 10 == 0 {
|
||||
info!(
|
||||
"Learning rate scheduled update: epoch={}, lr={:.2e} (initial={:.2e})",
|
||||
@@ -2218,6 +2322,7 @@ impl DQNTrainer {
|
||||
}
|
||||
|
||||
// Financial metrics from GPU trade stats
|
||||
let epoch_max_dd;
|
||||
let epoch_sharpe = {
|
||||
let financials = compute_epoch_financials(
|
||||
&trade_stats,
|
||||
@@ -2283,9 +2388,40 @@ impl DQNTrainer {
|
||||
});
|
||||
}
|
||||
|
||||
epoch_max_dd = financials.max_drawdown;
|
||||
financials.sharpe
|
||||
};
|
||||
|
||||
// ── Adversarial regime injection: detect suspiciously low drawdown ──
|
||||
// If MaxDD < threshold for N consecutive epochs, the policy may be
|
||||
// memorizing the fixed sim parameters. Next epoch we inject 3× spread,
|
||||
// 2× tx_cost, 0.5× fill to stress-test. Resets once drawdown exceeds threshold.
|
||||
{
|
||||
let max_dd = epoch_max_dd;
|
||||
let thresh = self.hyperparams.adversarial_dd_threshold;
|
||||
let trigger = self.hyperparams.adversarial_epochs_trigger;
|
||||
if max_dd < thresh && trigger > 0 {
|
||||
self.consecutive_low_dd_epochs += 1;
|
||||
if self.consecutive_low_dd_epochs >= trigger {
|
||||
self.adversarial_active = true;
|
||||
info!(
|
||||
consecutive = self.consecutive_low_dd_epochs,
|
||||
max_dd = %format!("{:.4}%", max_dd * 100.0),
|
||||
"ADVERSARIAL regime ACTIVATED — injecting harsh market conditions next epoch"
|
||||
);
|
||||
}
|
||||
} else {
|
||||
if self.adversarial_active {
|
||||
info!(
|
||||
max_dd = %format!("{:.4}%", max_dd * 100.0),
|
||||
"ADVERSARIAL regime deactivated — drawdown exceeded threshold"
|
||||
);
|
||||
}
|
||||
self.consecutive_low_dd_epochs = 0;
|
||||
self.adversarial_active = false;
|
||||
}
|
||||
}
|
||||
|
||||
// Track metrics for early stopping
|
||||
self.loss_history.push(avg_loss);
|
||||
self.q_value_history.push(avg_q_value);
|
||||
|
||||
@@ -145,6 +145,27 @@ pub struct AdvancedSection {
|
||||
pub curiosity_weight: Option<f64>,
|
||||
}
|
||||
|
||||
/// Gems & Pearls: generalization techniques to close IS/OOS gap.
|
||||
#[derive(Debug, Clone, Deserialize, Default)]
|
||||
pub struct GeneralizationSection {
|
||||
pub enable_domain_randomization: Option<bool>,
|
||||
pub shrink_perturb_interval: Option<usize>,
|
||||
pub shrink_perturb_alpha: Option<f64>,
|
||||
pub shrink_perturb_sigma: Option<f64>,
|
||||
pub adversarial_dd_threshold: Option<f64>,
|
||||
pub adversarial_epochs_trigger: Option<usize>,
|
||||
pub enable_anti_lr: Option<bool>,
|
||||
pub anti_lr_good_mult: Option<f64>,
|
||||
pub anti_lr_bad_mult: Option<f64>,
|
||||
pub anti_lr_sharpe_threshold: Option<f64>,
|
||||
pub feature_mask_fraction: Option<f64>,
|
||||
pub feature_noise_scale: Option<f64>,
|
||||
pub enable_vol_normalization: Option<bool>,
|
||||
pub asymmetric_dd_weight: Option<f64>,
|
||||
pub trade_clustering_penalty: Option<f64>,
|
||||
pub ensemble_disagreement_penalty: Option<f64>,
|
||||
}
|
||||
|
||||
/// Risk management and position-control parameters.
|
||||
#[derive(Debug, Clone, Deserialize, Default)]
|
||||
pub struct RiskSection {
|
||||
@@ -252,6 +273,7 @@ pub struct DqnTrainingProfile {
|
||||
pub distributional: Option<DistributionalSection>,
|
||||
pub branching: Option<BranchingSection>,
|
||||
pub advanced: Option<AdvancedSection>,
|
||||
pub generalization: Option<GeneralizationSection>,
|
||||
pub risk: Option<RiskSection>,
|
||||
pub early_stopping: Option<EarlyStoppingSection>,
|
||||
pub experience: Option<ExperienceSection>,
|
||||
@@ -769,6 +791,26 @@ impl DqnTrainingProfile {
|
||||
}
|
||||
}
|
||||
|
||||
// [generalization]
|
||||
if let Some(ref g) = self.generalization {
|
||||
if let Some(v) = g.enable_domain_randomization { hp.enable_domain_randomization = v; }
|
||||
if let Some(v) = g.shrink_perturb_interval { hp.shrink_perturb_interval = v; }
|
||||
if let Some(v) = g.shrink_perturb_alpha { hp.shrink_perturb_alpha = v; }
|
||||
if let Some(v) = g.shrink_perturb_sigma { hp.shrink_perturb_sigma = v; }
|
||||
if let Some(v) = g.adversarial_dd_threshold { hp.adversarial_dd_threshold = v; }
|
||||
if let Some(v) = g.adversarial_epochs_trigger { hp.adversarial_epochs_trigger = v; }
|
||||
if let Some(v) = g.enable_anti_lr { hp.enable_anti_lr = v; }
|
||||
if let Some(v) = g.anti_lr_good_mult { hp.anti_lr_good_mult = v; }
|
||||
if let Some(v) = g.anti_lr_bad_mult { hp.anti_lr_bad_mult = v; }
|
||||
if let Some(v) = g.anti_lr_sharpe_threshold { hp.anti_lr_sharpe_threshold = v; }
|
||||
if let Some(v) = g.feature_mask_fraction { hp.feature_mask_fraction = v; }
|
||||
if let Some(v) = g.feature_noise_scale { hp.feature_noise_scale = v; }
|
||||
if let Some(v) = g.enable_vol_normalization { hp.enable_vol_normalization = v; }
|
||||
if let Some(v) = g.asymmetric_dd_weight { hp.asymmetric_dd_weight = v; }
|
||||
if let Some(v) = g.trade_clustering_penalty { hp.trade_clustering_penalty = v; }
|
||||
if let Some(v) = g.ensemble_disagreement_penalty { hp.ensemble_disagreement_penalty = v; }
|
||||
}
|
||||
|
||||
// [risk]
|
||||
if let Some(ref r) = self.risk {
|
||||
if let Some(v) = r.enable_kelly_sizing {
|
||||
|
||||
@@ -0,0 +1,728 @@
|
||||
# Gems & Pearls — 28 Generalization Techniques for DQN Trading Agent
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Close the in-sample/OOS generalization gap (Sharpe +0.57 IS → -1.30 OOS). The model memorizes price sequences instead of learning regime-invariant market structure. These 28 techniques attack every axis of memorization.
|
||||
|
||||
**Architecture:** Branching DQN with 81 factored actions (9 exposure x 3 order x 3 urgency), 14 active features, fused CUDA training pipeline, C51+MSE blended loss, CQL regularization, curiosity model, ensemble heads.
|
||||
|
||||
**Tech Stack:** Rust, CUDA (cudarc), existing fused CUDA kernels, existing curiosity forward model
|
||||
|
||||
**Status Key:** DONE = committed, WIP = in progress, PLAN = not started
|
||||
|
||||
---
|
||||
|
||||
## Previously Planned (Tasks 1-8 from original plan)
|
||||
|
||||
### Task 1: Domain Randomization — Per-Epoch Simulation Parameter Jitter [DONE]
|
||||
|
||||
**Commit:** `c3ea0e57`
|
||||
|
||||
**Files:**
|
||||
- Modified: `crates/ml/src/trainers/dqn/trainer/training_loop.rs`
|
||||
- Modified: `crates/ml/src/trainers/dqn/config.rs`
|
||||
|
||||
Per-epoch randomization of spread, tx_cost, fill probabilities, initial capital, episode start positions, and episode lengths. Prevents memorizing fixed simulation parameters.
|
||||
|
||||
- [x] Config flag `enable_domain_randomization`
|
||||
- [x] Randomize ExperienceCollectorConfig per epoch
|
||||
- [x] Episode start jitter +-25% stride
|
||||
- [x] Variable episode length
|
||||
- [x] TOML configs updated
|
||||
- [x] Compiled + tested
|
||||
|
||||
---
|
||||
|
||||
### Task 2: CQL Alpha Boost + Gradient Budget Rebalance [DONE]
|
||||
|
||||
**Commit:** `ae995673`
|
||||
|
||||
**Files:**
|
||||
- Modified: `crates/ml/src/trainers/dqn/config.rs`
|
||||
- Modified: `crates/ml/src/trainers/dqn/fused_training.rs`
|
||||
|
||||
CQL alpha 0.1 -> 1.0, gradient budget 15% -> 25%, C51 budget 70% -> 60%.
|
||||
|
||||
- [x] CQL alpha increased
|
||||
- [x] Gradient budget rebalanced
|
||||
- [x] Compiled + tested
|
||||
|
||||
---
|
||||
|
||||
### Task 3: Periodic Shrink-and-Perturb [DONE]
|
||||
|
||||
**Commit:** `37cb034e`
|
||||
|
||||
**Files:**
|
||||
- Modified: `crates/ml/src/trainers/dqn/trainer/training_loop.rs`
|
||||
- Modified: `crates/ml/src/trainers/dqn/config.rs`
|
||||
|
||||
Shrink-and-perturb every 20 epochs: keep 85% of weights, reinitialize 15% with noise sigma=0.01. Kills memorized weight patterns periodically.
|
||||
|
||||
- [x] Config fields `shrink_perturb_interval`, `_alpha`, `_sigma`
|
||||
- [x] Periodic S&P in training loop
|
||||
- [x] Compiled + tested
|
||||
|
||||
---
|
||||
|
||||
### Task 4: Adversarial Regime Injection [PLAN]
|
||||
|
||||
**Files:**
|
||||
- Modify: `crates/ml/src/trainers/dqn/trainer/training_loop.rs`
|
||||
- Modify: `crates/ml/src/cuda_pipeline/gpu_experience_collector.rs`
|
||||
- Modify: `crates/ml/src/trainers/dqn/config.rs`
|
||||
- Modify: `crates/ml/src/trainers/dqn/trainer/mod.rs`
|
||||
|
||||
When the model achieves low drawdown (<10%) for 3+ consecutive epochs, inject adversarial conditions: 3x spread, 2x tx_cost, 0.5x fill probability. Forces the model to handle worst-case scenarios.
|
||||
|
||||
- [ ] Add config `adversarial_injection_threshold` (0.10), `adversarial_injection_consecutive` (3)
|
||||
- [ ] Add tracking field `consecutive_low_dd_epochs` in trainer state
|
||||
- [ ] Implement detection + injection in training loop
|
||||
- [ ] Apply adversarial multipliers to ExperienceCollectorConfig
|
||||
- [ ] Log when adversarial mode activates
|
||||
- [ ] Compile + test + commit
|
||||
|
||||
---
|
||||
|
||||
### Task 5: Anti-Correlation Reward Penalty [PLAN]
|
||||
|
||||
**Files:**
|
||||
- Modify: `crates/ml/src/trainers/dqn/trainer/training_loop.rs`
|
||||
- Modify: `crates/ml/src/cuda_pipeline/gpu_experience_collector.rs`
|
||||
- Modify: `crates/ml/src/cuda_pipeline/experience_kernels.cu`
|
||||
|
||||
At epoch end, compare action distributions between training and validation. If cosine similarity > 0.8, the model does the same thing everywhere — it hasn't learned to differentiate. Penalize next epoch's rewards by 0.8x.
|
||||
|
||||
- [ ] Compute cosine similarity of train vs val action distributions
|
||||
- [ ] Add `reward_scale_override: f32` to ExperienceCollectorConfig
|
||||
- [ ] Multiply reward by scale factor in experience kernel
|
||||
- [ ] Compile + test + commit
|
||||
|
||||
---
|
||||
|
||||
### Task 6: Curiosity Prediction Error as Q-Penalty [PLAN]
|
||||
|
||||
**Files:**
|
||||
- Modify: `crates/ml/src/trainers/dqn/fused_training.rs`
|
||||
- Modify: `crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs`
|
||||
- Modify: `crates/ml/src/cuda_pipeline/mse_loss_kernel.cu` or `c51_loss_kernel.cu`
|
||||
- Modify: `crates/ml/src/trainers/dqn/config.rs`
|
||||
|
||||
The curiosity forward model's prediction error measures how "novel" a state-action pair is. Wire it as an additive penalty to the Q-target in the Bellman equation:
|
||||
```
|
||||
target_Q = reward + gamma * (1-done) * target_Q_next - curiosity_penalty * prediction_error
|
||||
```
|
||||
High prediction error = novel state = conservative Q-values. Same principle as CQL but data-driven.
|
||||
|
||||
- [ ] Add config `curiosity_q_penalty_weight` (default 0.5)
|
||||
- [ ] Compute per-sample curiosity error in fused training step
|
||||
- [ ] Pass `curiosity_errors` buffer to loss kernel
|
||||
- [ ] Subtract penalty from Q-target in kernel
|
||||
- [ ] Wire through Rust launch code
|
||||
- [ ] TOML configs
|
||||
- [ ] Compile + full smoke test
|
||||
|
||||
---
|
||||
|
||||
### Task 7: Counterfactual Experience Augmentation [PLAN]
|
||||
|
||||
**Files:**
|
||||
- Modify: `crates/ml/src/cuda_pipeline/experience_kernels.cu`
|
||||
- Modify: `crates/ml/src/cuda_pipeline/gpu_experience_collector.rs`
|
||||
|
||||
For every trade taken, also compute the counterfactual: what would have happened with the mirror action (Long<->Short). Store both with inverted reward signs. Doubles effective data diversity.
|
||||
|
||||
- [ ] In experience kernel, compute reward for mirror action
|
||||
- [ ] Write both to output buffers (2x output size)
|
||||
- [ ] Counterfactual reward = -1 x actual reward
|
||||
- [ ] Compile + test + commit
|
||||
|
||||
---
|
||||
|
||||
### Task 8: Hindsight Regime Labeling — Feature Importance Filtering [PLAN]
|
||||
|
||||
**Files:**
|
||||
- Create: `crates/ml/src/trainers/dqn/feature_importance.rs`
|
||||
- Modify: `crates/ml/src/trainers/dqn/trainer/training_loop.rs`
|
||||
- Modify: `crates/ml/src/cuda_pipeline/experience_kernels.cu`
|
||||
- Modify: `crates/ml/src/trainers/dqn/config.rs`
|
||||
|
||||
After each epoch, compute gradient x activation for each feature to identify which features predicted profitable trades. Mask the bottom 50% (set to 0) next epoch. Iterative: the model discovers which features carry signal and ignores noise.
|
||||
|
||||
- [ ] Compute per-feature importance from gradient x activation
|
||||
- [ ] Rank features, compute binary mask (top-K survive)
|
||||
- [ ] Apply mask in experience kernel (zero out masked features)
|
||||
- [ ] Iterate — recompute importance each epoch
|
||||
- [ ] Compile + test + commit
|
||||
|
||||
---
|
||||
|
||||
### Task 9: Regime-Adversarial Training (Gradient Reversal / DANN) [PLAN]
|
||||
|
||||
**Files:**
|
||||
- Create: `crates/ml/src/cuda_pipeline/regime_discriminator_kernel.cu`
|
||||
- Modify: `crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs`
|
||||
- Modify: `crates/ml/src/trainers/dqn/fused_training.rs`
|
||||
|
||||
Train a small classifier head on DQN hidden features that predicts which time window (fold) data comes from. Apply GRADIENT REVERSAL — the DQN learns features that the discriminator CANNOT use to identify the regime. Domain Adversarial Neural Networks (DANN) adapted for temporal domains.
|
||||
|
||||
- [ ] Add discriminator kernel: small MLP [hidden_dim -> 64 -> num_folds]
|
||||
- [ ] Wire gradient reversal: negate discriminator gradient before shared trunk
|
||||
- [ ] Add fold ID to experience buffer
|
||||
- [ ] Compile + test + commit
|
||||
|
||||
---
|
||||
|
||||
## Gems & Pearls — Category A: Symmetry & Invariance Exploits
|
||||
|
||||
### Task 10: Mirror Universe Training [PLAN]
|
||||
|
||||
**Phase:** 2 (CUDA changes) | **Difficulty:** Easy | **Impact:** High
|
||||
|
||||
**Files:**
|
||||
- Modify: `crates/ml/src/cuda_pipeline/experience_kernels.cu` (return sign flip)
|
||||
- Modify: `crates/ml/src/cuda_pipeline/gpu_experience_collector.rs` (mirror flag)
|
||||
- Modify: `crates/ml/src/trainers/dqn/trainer/training_loop.rs` (epoch-level toggle)
|
||||
- Modify: `crates/ml/src/trainers/dqn/config.rs`
|
||||
|
||||
Flip ALL returns (multiply by -1) and invert the action mapping (L100<->S100, L75<->S75, etc.). A model that truly understands market structure should produce identical P&L on mirror data with mirror actions. If IS Sharpe on mirrored data differs from normal data, the model has learned a directional bias, not structure. Train on both simultaneously — doubles data diversity with zero extra market data.
|
||||
|
||||
- [ ] Add config `enable_mirror_universe` (default true)
|
||||
- [ ] Add `mirror_active: bool` to ExperienceCollectorConfig
|
||||
- [ ] In experience kernel: when mirror=true, negate all return-based features and invert action index (8-action_idx for exposure)
|
||||
- [ ] Alternate: epoch N normal, epoch N+1 mirrored (or 50/50 within epoch)
|
||||
- [ ] Verify P&L symmetry on mirrored vs normal epochs
|
||||
- [ ] Compile + test + commit
|
||||
|
||||
---
|
||||
|
||||
### Task 11: Time-Reversal Augmentation [PLAN]
|
||||
|
||||
**Phase:** 2 (CUDA changes) | **Difficulty:** Easy | **Impact:** Medium
|
||||
|
||||
**Files:**
|
||||
- Modify: `crates/ml/src/cuda_pipeline/experience_kernels.cu` (reversed indexing)
|
||||
- Modify: `crates/ml/src/cuda_pipeline/gpu_experience_collector.rs` (reverse flag)
|
||||
- Modify: `crates/ml/src/trainers/dqn/config.rs`
|
||||
|
||||
Play episodes backwards. Market microstructure has statistical time-reversal symmetry at short horizons (bid-ask bounce). If the model can't trade profitably on reversed sequences, it's fitting to temporal artifacts (trends, momentum) rather than structural features. Mix 20% reversed episodes into each epoch — forces reliance on instantaneous features (spread, order flow) rather than trajectory.
|
||||
|
||||
- [ ] Add config `time_reversal_fraction` (default 0.2)
|
||||
- [ ] In experience kernel: reverse the bar indexing for selected episodes (bar[T-t] instead of bar[t])
|
||||
- [ ] Adjust next_close accordingly (now previous close)
|
||||
- [ ] Mark reversed episodes so reward calculation handles inverted time
|
||||
- [ ] Compile + test + commit
|
||||
|
||||
---
|
||||
|
||||
### Task 12: Price-Level Invariance Enforcement [PLAN]
|
||||
|
||||
**Phase:** 2 (CUDA changes) | **Difficulty:** Medium | **Impact:** Very High
|
||||
|
||||
**Files:**
|
||||
- Modify: `crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs` (dual forward pass)
|
||||
- Modify: `crates/ml/src/trainers/dqn/fused_training.rs` (invariance loss term)
|
||||
- Modify: `crates/ml/src/cuda_pipeline/mse_loss_kernel.cu` (extra loss component)
|
||||
- Modify: `crates/ml/src/trainers/dqn/config.rs`
|
||||
|
||||
Add a regularization loss: run the same features through the network twice — once at actual levels, once with all price-derived features shifted by a random constant. The Q-values MUST be identical. If they differ, the model has learned absolute price levels (which never generalize).
|
||||
|
||||
```
|
||||
L_invariance = MSE(Q(s), Q(s + delta))
|
||||
```
|
||||
|
||||
Added to the Bellman loss with configurable weight.
|
||||
|
||||
- [ ] Add config `price_invariance_weight` (default 0.1)
|
||||
- [ ] During training step, create shifted copy of feature batch (add random delta to price-derived feature indices)
|
||||
- [ ] Forward pass both original and shifted through Q-network
|
||||
- [ ] Compute MSE between Q-value outputs
|
||||
- [ ] Add weighted invariance loss to total loss
|
||||
- [ ] Compile + test + commit
|
||||
|
||||
---
|
||||
|
||||
### Task 13: Volatility Regime Normalization [PLAN]
|
||||
|
||||
**Phase:** 1 (Rust-only) | **Difficulty:** Easy | **Impact:** Very High
|
||||
|
||||
**Files:**
|
||||
- Modify: `crates/ml/src/cuda_pipeline/gpu_experience_collector.rs` (feature preprocessing)
|
||||
- Modify: `crates/ml/src/cuda_pipeline/experience_kernels.cu` (normalization in kernel)
|
||||
- Modify: `crates/ml/src/trainers/dqn/config.rs`
|
||||
|
||||
Before feeding features to the network, divide ALL return-based features by a 20-bar realized vol estimate. This makes the feature vector scale-invariant across regimes. A 1% move in a 5% vol regime should look identical to a 0.2% move in a 1% vol regime. This is preprocessing, not model change — but eliminates the #1 source of regime-specific memorization.
|
||||
|
||||
- [ ] Add config `enable_vol_normalization` (default true), `vol_lookback` (default 20)
|
||||
- [ ] Compute running realized vol from close-to-close returns in experience kernel
|
||||
- [ ] Divide return-based features (indices 0-7 approximately) by max(realized_vol, 1e-8)
|
||||
- [ ] Leave non-return features (ADX, CUSUM, volume ratios) unnormalized
|
||||
- [ ] Compile + test + commit
|
||||
|
||||
---
|
||||
|
||||
## Gems & Pearls — Category B: Adversarial & Game-Theoretic
|
||||
|
||||
### Task 14: Market Maker Adversary (GAN for Trading) [PLAN]
|
||||
|
||||
**Phase:** 3 (Architecture) | **Difficulty:** Hard | **Impact:** Very High
|
||||
|
||||
**Files:**
|
||||
- Create: `crates/ml/src/cuda_pipeline/adversary_kernel.cu`
|
||||
- Create: `crates/ml/src/trainers/dqn/market_maker_adversary.rs`
|
||||
- Modify: `crates/ml/src/trainers/dqn/fused_training.rs`
|
||||
- Modify: `crates/ml/src/trainers/dqn/trainer/training_loop.rs`
|
||||
- Modify: `crates/ml/src/trainers/dqn/config.rs`
|
||||
|
||||
Train a SECOND tiny network as the "adversarial market maker." It controls spread, fill probability, and slippage, and is trained to MAXIMIZE the DQN's losses. The DQN trains against this adversary. Cycle: 5 epochs adversarial training, 5 epochs recovery with frozen adversary. The DQN learns to survive worst-case execution — the ultimate generalization pressure.
|
||||
|
||||
- [ ] Add config `enable_market_maker_adversary` (default false), `adversary_cycle_epochs` (5)
|
||||
- [ ] Create adversary network: small MLP [state_dim -> 32 -> 3] outputting (spread_mult, fill_prob, slippage_mult)
|
||||
- [ ] Adversary loss = -1 * DQN reward (maximize DQN losses)
|
||||
- [ ] In training loop, alternate: adversary trains for N epochs, then freeze and let DQN recover for N epochs
|
||||
- [ ] Pass adversary outputs to ExperienceCollectorConfig
|
||||
- [ ] Compile + test + commit
|
||||
|
||||
---
|
||||
|
||||
### Task 15: Gradient Vaccine [PLAN]
|
||||
|
||||
**Phase:** 2 (CUDA changes) | **Difficulty:** Medium | **Impact:** Very High
|
||||
|
||||
**Files:**
|
||||
- Modify: `crates/ml/src/trainers/dqn/fused_training.rs` (gradient projection)
|
||||
- Modify: `crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs` (dual gradient computation)
|
||||
- Modify: `crates/ml/src/trainers/dqn/config.rs`
|
||||
|
||||
At each training step, compute gradients on BOTH the training batch and a held-out validation batch. Project out the component of the training gradient that CONTRADICTS the validation gradient:
|
||||
|
||||
```
|
||||
g_safe = g_train - (g_train . g_val / |g_val|^2) * g_val [when dot product < 0]
|
||||
```
|
||||
|
||||
Only update in directions that both sets agree on. The model literally cannot learn anything that doesn't generalize — overfitting gradients are surgically removed.
|
||||
|
||||
- [ ] Add config `enable_gradient_vaccine` (default true), `vaccine_val_fraction` (0.2)
|
||||
- [ ] Split each training batch: 80% train, 20% held-out validation
|
||||
- [ ] Compute gradients on both subsets separately
|
||||
- [ ] Compute dot product of gradient vectors
|
||||
- [ ] When dot product < 0, project out conflicting component
|
||||
- [ ] Apply safe gradient to optimizer
|
||||
- [ ] Compile + test + commit
|
||||
|
||||
---
|
||||
|
||||
### Task 16: Phantom Liquidity Episodes [PLAN]
|
||||
|
||||
**Phase:** 3 (Architecture) | **Difficulty:** Medium | **Impact:** High
|
||||
|
||||
**Files:**
|
||||
- Create: `crates/ml/src/trainers/dqn/synthetic_data.rs`
|
||||
- Modify: `crates/ml/src/cuda_pipeline/gpu_experience_collector.rs`
|
||||
- Modify: `crates/ml/src/trainers/dqn/trainer/training_loop.rs`
|
||||
- Modify: `crates/ml/src/trainers/dqn/config.rs`
|
||||
|
||||
Every 5th epoch, replace 30% of episodes with SYNTHETIC data — GBM with matched vol and drift, or shuffled 50-bar blocks of real data. The model must trade these profitably too. If it can only trade on real sequences but fails on statistically-identical synthetic ones, it has memorized specific price patterns. Ultimate memorization detector AND training regularizer.
|
||||
|
||||
- [ ] Add config `phantom_fraction` (0.3), `phantom_epoch_interval` (5)
|
||||
- [ ] Create GBM price generator: match vol, drift, and autocorrelation from real data
|
||||
- [ ] Alternative: block-shuffle real data (50-bar blocks, random permutation)
|
||||
- [ ] Replace selected episodes' price data with synthetic prices
|
||||
- [ ] Recompute features from synthetic prices
|
||||
- [ ] Track Sharpe on phantom vs real episodes separately (diagnostic)
|
||||
- [ ] Compile + test + commit
|
||||
|
||||
---
|
||||
|
||||
## Gems & Pearls — Category C: Loss & Reward Reshaping
|
||||
|
||||
### Task 17: Counterfactual Regret Minimization Reward [PLAN]
|
||||
|
||||
**Phase:** 2 (CUDA changes) | **Difficulty:** Medium | **Impact:** High
|
||||
|
||||
**Files:**
|
||||
- Modify: `crates/ml/src/cuda_pipeline/experience_kernels.cu` (regret computation)
|
||||
- Modify: `crates/ml/src/cuda_pipeline/gpu_experience_collector.rs`
|
||||
- Modify: `crates/ml/src/trainers/dqn/config.rs`
|
||||
|
||||
Replace raw PnL reward with **regret**: `reward = PnL(action_taken) - max(PnL(all_actions))`. From game theory (CFR). The model learns to minimize regret rather than maximize absolute P&L. Regret is naturally normalized across regimes — a -$100 trade when the best was -$50 has regret -$50, same as a +$200 trade when the best was +$250. Eliminates regime-dependent reward scaling.
|
||||
|
||||
- [ ] Add config `enable_regret_reward` (default false), can coexist with normal reward
|
||||
- [ ] In experience kernel, after computing reward for taken action, compute reward for all 9 exposure levels
|
||||
- [ ] Regret = reward(taken) - max(reward(all_exposures))
|
||||
- [ ] Regret is always <= 0 (perfect play = 0 regret)
|
||||
- [ ] Option: blend regret with raw reward (0.5 * reward + 0.5 * regret)
|
||||
- [ ] Compile + test + commit
|
||||
|
||||
---
|
||||
|
||||
### Task 18: Asymmetric Drawdown Loss [PLAN]
|
||||
|
||||
**Phase:** 1 (Rust-only) | **Difficulty:** Easy | **Impact:** High
|
||||
|
||||
**Files:**
|
||||
- Modify: `crates/ml/src/cuda_pipeline/mse_loss_kernel.cu` (asymmetric penalty)
|
||||
- Modify: `crates/ml/src/cuda_pipeline/c51_loss_kernel.cu` (asymmetric penalty)
|
||||
- Modify: `crates/ml/src/trainers/dqn/config.rs`
|
||||
|
||||
Add a term to the Bellman loss that specifically penalizes Q-value OVERESTIMATION on states where the portfolio is in drawdown:
|
||||
|
||||
```
|
||||
L_dd = max(0, Q_predicted - Q_target) * drawdown_depth^2
|
||||
```
|
||||
|
||||
When the model is in drawdown, overestimating Q-values is catastrophic (it holds losing positions expecting recovery). Makes the model pessimistic during drawdowns — the #1 OOS failure mode.
|
||||
|
||||
- [ ] Add config `asymmetric_dd_weight` (default 0.5)
|
||||
- [ ] Pass current drawdown depth to loss kernel as per-sample float
|
||||
- [ ] In kernel: when Q_pred > Q_target AND drawdown > 0, add penalty = overestimation * dd^2 * weight
|
||||
- [ ] Penalty only activates during drawdown (zero cost when equity is at highs)
|
||||
- [ ] Compile + test + commit
|
||||
|
||||
---
|
||||
|
||||
### Task 19: Entropy-Regulated Position Trajectory [PLAN]
|
||||
|
||||
**Phase:** 2 (CUDA changes) | **Difficulty:** Easy | **Impact:** Medium
|
||||
|
||||
**Files:**
|
||||
- Modify: `crates/ml/src/cuda_pipeline/experience_kernels.cu` (position histogram + entropy bonus)
|
||||
- Modify: `crates/ml/src/cuda_pipeline/gpu_experience_collector.rs`
|
||||
- Modify: `crates/ml/src/trainers/dqn/config.rs`
|
||||
|
||||
Add an entropy bonus not to the ACTION distribution (standard) but to the POSITION TRAJECTORY over an episode. The model should visit multiple exposure levels, not get stuck at one. Compute the entropy of [time_at_S100, time_at_S75, ..., time_at_L100] and add to reward:
|
||||
|
||||
```
|
||||
reward += position_entropy_weight * H(position_histogram)
|
||||
```
|
||||
|
||||
Prevents degenerate strategies of "always flat" or "always long" that score well IS but collapse OOS.
|
||||
|
||||
- [ ] Add config `position_entropy_weight` (default 0.01)
|
||||
- [ ] Track per-episode position histogram in experience kernel (shared memory, 9 bins)
|
||||
- [ ] At episode end, compute entropy of histogram
|
||||
- [ ] Add scaled entropy bonus to final episode reward
|
||||
- [ ] Compile + test + commit
|
||||
|
||||
---
|
||||
|
||||
## Gems & Pearls — Category D: Network Architecture Tricks
|
||||
|
||||
### Task 20: Weight Lottery Ticket Pruning [PLAN]
|
||||
|
||||
**Phase:** 3 (Architecture) | **Difficulty:** Medium | **Impact:** High
|
||||
|
||||
**Files:**
|
||||
- Create: `crates/ml/src/trainers/dqn/pruning.rs`
|
||||
- Modify: `crates/ml/src/trainers/dqn/fused_training.rs` (apply mask)
|
||||
- Modify: `crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs` (mask buffers)
|
||||
- Modify: `crates/ml/src/trainers/dqn/config.rs`
|
||||
|
||||
After 50 epochs of training, prune 70% of weights (smallest magnitude -> zero), then RETRAIN from scratch keeping only the surviving mask. The "lottery ticket hypothesis" says the sparse subnetwork generalizes better because it physically cannot memorize as much. Run twice: 100% -> 30% -> 9% of original parameters.
|
||||
|
||||
- [ ] Add config `pruning_epoch` (50), `pruning_fraction` (0.7), `pruning_rounds` (2)
|
||||
- [ ] After pruning_epoch, compute weight magnitude across all layers
|
||||
- [ ] Create binary mask: top 30% survive, bottom 70% = 0
|
||||
- [ ] Store mask as GPU buffer, apply element-wise after each optimizer step (w = w * mask)
|
||||
- [ ] Reset optimizer state for pruned weights
|
||||
- [ ] Optional: second round at epoch 100 (prune 70% of surviving 30% = 9% total)
|
||||
- [ ] Compile + test + commit
|
||||
|
||||
---
|
||||
|
||||
### Task 21: Stochastic Depth (Layer Dropout) [PLAN]
|
||||
|
||||
**Phase:** 2 (CUDA changes) | **Difficulty:** Easy | **Impact:** Medium
|
||||
|
||||
**Files:**
|
||||
- Modify: `crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs` (layer skip logic)
|
||||
- Modify: `crates/ml-dqn/src/dqn.rs` (forward pass with skip)
|
||||
- Modify: `crates/ml/src/trainers/dqn/config.rs`
|
||||
|
||||
During training, randomly skip entire layers of the DQN trunk with 20% probability (replace with identity/passthrough). Each forward pass uses a random subset of the network's depth. Forces EVERY layer to produce useful features independently, not rely on deep compositional memorization. At inference, use all layers with scaled weights. Proven in vision transformers but nobody has applied it to RL trading.
|
||||
|
||||
- [ ] Add config `stochastic_depth_prob` (default 0.2)
|
||||
- [ ] During forward pass, for each hidden layer, generate random float
|
||||
- [ ] If random < prob, skip layer (output = input, bypass matmul + activation)
|
||||
- [ ] Scale surviving layers by 1/(1-prob) to maintain expected magnitude
|
||||
- [ ] At inference (eval mode), disable stochastic depth, use all layers
|
||||
- [ ] Compile + test + commit
|
||||
|
||||
---
|
||||
|
||||
### Task 22: Feature-Wise Noise Injection [PLAN]
|
||||
|
||||
**Phase:** 1 (Rust-only) | **Difficulty:** Easy | **Impact:** Medium
|
||||
|
||||
**Files:**
|
||||
- Modify: `crates/ml/src/cuda_pipeline/experience_kernels.cu` (noise injection)
|
||||
- Modify: `crates/ml/src/cuda_pipeline/gpu_experience_collector.rs`
|
||||
- Modify: `crates/ml/src/trainers/dqn/config.rs`
|
||||
|
||||
Instead of zeroing features (dropout), add calibrated Gaussian noise to each feature with sigma proportional to that feature's standard deviation:
|
||||
|
||||
```
|
||||
feature_noisy = feature + N(0, noise_scale * std(feature))
|
||||
```
|
||||
|
||||
Strictly better than dropout for continuous features: the model sees signal plus noise, must learn to be robust to feature measurement error. Real market features ARE noisy.
|
||||
|
||||
- [ ] Add config `feature_noise_scale` (default 0.1)
|
||||
- [ ] Compute running std per feature (EMA or batch std)
|
||||
- [ ] In experience kernel, add N(0, scale * std) to each feature during training
|
||||
- [ ] Disable noise during evaluation/inference
|
||||
- [ ] Use curand for GPU-native noise generation
|
||||
- [ ] Compile + test + commit
|
||||
|
||||
---
|
||||
|
||||
## Gems & Pearls — Category E: Temporal & Causal Structure
|
||||
|
||||
### Task 23: Causal Feature Masking (Random Feature Forests) [PLAN]
|
||||
|
||||
**Phase:** 1 (Rust-only) | **Difficulty:** Easy | **Impact:** High
|
||||
|
||||
**Files:**
|
||||
- Modify: `crates/ml/src/cuda_pipeline/experience_kernels.cu` (feature masking)
|
||||
- Modify: `crates/ml/src/cuda_pipeline/gpu_experience_collector.rs`
|
||||
- Modify: `crates/ml/src/trainers/dqn/trainer/training_loop.rs` (per-epoch mask generation)
|
||||
- Modify: `crates/ml/src/trainers/dqn/config.rs`
|
||||
|
||||
Each epoch, randomly select 60% of features and mask the other 40% to zero. Different random subset each epoch. Over 100 epochs, the model trains on every possible feature combination. The surviving policy uses NO single feature as a crutch — it has built redundant representations. The "random forest" principle applied to RL feature extraction.
|
||||
|
||||
- [ ] Add config `feature_mask_fraction` (default 0.4), `enable_causal_masking` (default true)
|
||||
- [ ] At epoch start, generate random binary mask of length num_features
|
||||
- [ ] Upload mask to GPU
|
||||
- [ ] In experience kernel, multiply features by mask (zero out masked features)
|
||||
- [ ] Log which features are active each epoch
|
||||
- [ ] Compile + test + commit
|
||||
|
||||
---
|
||||
|
||||
### Task 24: Drawdown-Conditional Learning Rate (Anti-Intuitive) [PLAN]
|
||||
|
||||
**Phase:** 1 (Rust-only) | **Difficulty:** Easy | **Impact:** Medium
|
||||
|
||||
**Files:**
|
||||
- Modify: `crates/ml/src/trainers/dqn/trainer/training_loop.rs` (LR scheduling)
|
||||
- Modify: `crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs` (adam LR parameter)
|
||||
- Modify: `crates/ml/src/trainers/dqn/config.rs`
|
||||
|
||||
When the model is doing WELL (low drawdown, high Sharpe), INCREASE the learning rate 3x. When doing poorly, decrease it. The OPPOSITE of standard practice. Why: when the model is doing well IS, it's likely settling into an overfit minimum. High LR kicks it out. When struggling, low LR stabilizes. Creates natural oscillation that prevents settling into regime-specific policies.
|
||||
|
||||
- [ ] Add config `enable_anti_lr` (default true), `anti_lr_good_mult` (3.0), `anti_lr_bad_mult` (0.3), `anti_lr_sharpe_threshold` (0.3)
|
||||
- [ ] After epoch metrics, check if epoch Sharpe > threshold
|
||||
- [ ] If good (Sharpe > threshold): next epoch LR *= good_mult
|
||||
- [ ] If bad (Sharpe < -threshold): next epoch LR *= bad_mult
|
||||
- [ ] If neutral: use base LR
|
||||
- [ ] Clamp LR to [base_lr * 0.1, base_lr * 5.0] to prevent explosion
|
||||
- [ ] Update adam_lr parameter in GPU trainer
|
||||
- [ ] Compile + test + commit
|
||||
|
||||
---
|
||||
|
||||
### Task 25: Trade Clustering Penalty [PLAN]
|
||||
|
||||
**Phase:** 1 (Rust-only) | **Difficulty:** Easy | **Impact:** Medium
|
||||
|
||||
**Files:**
|
||||
- Modify: `crates/ml/src/cuda_pipeline/experience_kernels.cu` (inter-trade timing)
|
||||
- Modify: `crates/ml/src/cuda_pipeline/gpu_experience_collector.rs`
|
||||
- Modify: `crates/ml/src/trainers/dqn/config.rs`
|
||||
|
||||
Compute the coefficient of variation of inter-trade intervals within each episode. If trades are temporally clustered (bursts then silence), penalize:
|
||||
|
||||
```
|
||||
penalty = CV(inter_trade_times) * clustering_penalty_weight
|
||||
```
|
||||
|
||||
Real generalizable strategies trade somewhat uniformly. Temporal clustering is a fingerprint of pattern-matching specific price sequences. Forces strategies that work across the ENTIRE episode.
|
||||
|
||||
- [ ] Add config `trade_clustering_penalty` (default 0.1)
|
||||
- [ ] Track trade timestamps in experience kernel (shared memory array)
|
||||
- [ ] At episode end, compute inter-trade intervals
|
||||
- [ ] Compute CV = std(intervals) / mean(intervals)
|
||||
- [ ] Subtract penalty * CV from episode reward
|
||||
- [ ] Compile + test + commit
|
||||
|
||||
---
|
||||
|
||||
### Task 26: HER + Regime Tag Oversampling [PLAN]
|
||||
|
||||
**Phase:** 2 (CUDA changes) | **Difficulty:** Medium | **Impact:** High
|
||||
|
||||
**Files:**
|
||||
- Modify: `crates/ml/src/cuda_pipeline/gpu_her.rs` (regime-aware relabeling)
|
||||
- Modify: `crates/ml/src/cuda_pipeline/experience_kernels.cu` (regime tagging)
|
||||
- Modify: `crates/ml/src/trainers/dqn/config.rs`
|
||||
|
||||
Extend existing HER: when relabeling goals, also tag each experience with a coarse regime label (trending/mean-reverting/volatile, derived from ADX + realized vol). During replay, OVERSAMPLE experiences from underrepresented regimes. The model sees equal amounts of each regime regardless of real data distribution.
|
||||
|
||||
- [ ] Add config `enable_regime_oversampling` (default true), `num_regime_bins` (3)
|
||||
- [ ] In experience kernel, compute regime tag: ADX > 25 = trending, realized_vol > 2*median = volatile, else = mean-reverting
|
||||
- [ ] Store regime tag in experience buffer alongside existing fields
|
||||
- [ ] In PER sampling, weight by inverse regime frequency (rare regime = higher priority)
|
||||
- [ ] Log regime distribution per epoch
|
||||
- [ ] Compile + test + commit
|
||||
|
||||
---
|
||||
|
||||
## Gems & Pearls — Category F: Meta-Learning & Selection
|
||||
|
||||
### Task 27: Ensemble Disagreement as Q-Penalty [PLAN]
|
||||
|
||||
**Phase:** 1 (Rust-only) | **Difficulty:** Easy | **Impact:** Very High
|
||||
|
||||
**Files:**
|
||||
- Modify: `crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs` (ensemble variance computation)
|
||||
- Modify: `crates/ml/src/cuda_pipeline/mse_loss_kernel.cu` or `c51_loss_kernel.cu`
|
||||
- Modify: `crates/ml/src/trainers/dqn/fused_training.rs`
|
||||
- Modify: `crates/ml/src/trainers/dqn/config.rs`
|
||||
|
||||
You have an ensemble of Q-networks. When ensemble members DISAGREE on Q-value (high variance across heads), that state-action pair is UNCERTAIN. Add ensemble disagreement as a PENALTY to the Q-target:
|
||||
|
||||
```
|
||||
Q_target -= beta * std(Q_head1, Q_head2, ..., Q_headN)
|
||||
```
|
||||
|
||||
High disagreement -> lower Q-target -> conservative behavior on uncertain states. Novel OOS states will have high disagreement -> conservative behavior -> fewer catastrophic losses. This is epistemic uncertainty penalization.
|
||||
|
||||
- [ ] Add config `ensemble_disagreement_weight` (default 0.3)
|
||||
- [ ] After ensemble forward pass, compute per-sample std across heads
|
||||
- [ ] Pass std buffer to loss kernel
|
||||
- [ ] Subtract weighted std from Q-target
|
||||
- [ ] Monitor average disagreement per epoch (diagnostic: should decrease during training)
|
||||
- [ ] Compile + test + commit
|
||||
|
||||
---
|
||||
|
||||
### Task 28: Survival-of-the-Flattest Model Selection [PLAN]
|
||||
|
||||
**Phase:** 3 (Architecture) | **Difficulty:** Medium | **Impact:** High
|
||||
|
||||
**Files:**
|
||||
- Create: `crates/ml/src/trainers/dqn/sharpness.rs`
|
||||
- Modify: `crates/ml/src/hyperopt/adapters/dqn.rs` (selection criterion)
|
||||
- Modify: `crates/ml/src/trainers/dqn/config.rs`
|
||||
|
||||
During hyperopt, don't select the model with the best OOS Sharpe. Select the model whose loss landscape is FLATTEST — lowest |nabla^2 L| (Hessian trace). Approximate cheaply: perturb weights by epsilon in 10 random directions, measure loss change. Models in flat minima generalize; models in sharp minima memorize. Sharpness-Aware Minimization (SAM) applied to model SELECTION rather than training.
|
||||
|
||||
- [ ] Add config `sharpness_perturbation_epsilon` (0.01), `sharpness_num_directions` (10)
|
||||
- [ ] After training completes, compute sharpness score:
|
||||
- For each of 10 random unit vectors, perturb all weights by +/- epsilon * direction
|
||||
- Compute loss on validation set for each perturbation
|
||||
- Sharpness = mean(|loss_perturbed - loss_original|)
|
||||
- [ ] In hyperopt trial scoring: `score = oos_sharpe - sharpness_penalty_weight * sharpness`
|
||||
- [ ] Models with low sharpness (flat minima) are preferred even if Sharpe is slightly lower
|
||||
- [ ] Compile + test + commit
|
||||
|
||||
---
|
||||
|
||||
### Task 29: Cross-Fold Action Consistency Score [PLAN]
|
||||
|
||||
**Phase:** 2 (CUDA changes) | **Difficulty:** Medium | **Impact:** Medium
|
||||
|
||||
**Files:**
|
||||
- Modify: `crates/ml/src/trainers/dqn/trainer/training_loop.rs` (cross-fold comparison)
|
||||
- Modify: `crates/ml/src/hyperopt/adapters/dqn.rs` (consistency scoring)
|
||||
- Modify: `crates/ml/src/trainers/dqn/config.rs`
|
||||
|
||||
During walk-forward, for each state that appears in multiple folds' validation sets (overlapping features), compare the actions the model would take. If the model takes DIFFERENT actions on the same market conditions in different folds, it hasn't generalized:
|
||||
|
||||
```
|
||||
consistency = 1 - mean(action_variance_across_folds)
|
||||
```
|
||||
|
||||
Use as SECONDARY selection criterion alongside Sharpe. A model with Sharpe 0.3 and consistency 0.9 beats Sharpe 0.5 and consistency 0.4.
|
||||
|
||||
- [ ] Add config `consistency_weight` (default 0.3)
|
||||
- [ ] During walk-forward, save action predictions for each fold's validation set
|
||||
- [ ] For overlapping states (same bar indices across folds), compute action agreement
|
||||
- [ ] Consistency = fraction of states where majority action matches
|
||||
- [ ] In hyperopt scoring: `score = oos_sharpe * (1 + consistency_weight * consistency)`
|
||||
- [ ] Compile + test + commit
|
||||
|
||||
---
|
||||
|
||||
## Complete Execution Order
|
||||
|
||||
### Phase 1 — Quick Wins, Rust-only (next session)
|
||||
| # | Technique | Status |
|
||||
|---|-----------|--------|
|
||||
| 1 | Domain Randomization | DONE |
|
||||
| 2 | CQL Alpha Boost | DONE |
|
||||
| 3 | Shrink-and-Perturb | DONE |
|
||||
| 4 | Adversarial Regime Injection | PLAN |
|
||||
| 13 | Volatility Regime Normalization | PLAN |
|
||||
| 18 | Asymmetric Drawdown Loss | PLAN |
|
||||
| 22 | Feature-Wise Noise Injection | PLAN |
|
||||
| 23 | Causal Feature Masking | PLAN |
|
||||
| 24 | Anti-Intuitive LR | PLAN |
|
||||
| 25 | Trade Clustering Penalty | PLAN |
|
||||
| 27 | Ensemble Disagreement Q-Penalty | PLAN |
|
||||
|
||||
### Phase 2 — CUDA Kernel Changes
|
||||
| # | Technique | Status |
|
||||
|---|-----------|--------|
|
||||
| 5 | Anti-Correlation Penalty | PLAN |
|
||||
| 6 | Curiosity Q-Penalty | PLAN |
|
||||
| 7 | Counterfactual Augmentation | PLAN |
|
||||
| 10 | Mirror Universe Training | PLAN |
|
||||
| 11 | Time-Reversal Augmentation | PLAN |
|
||||
| 12 | Price-Level Invariance | PLAN |
|
||||
| 15 | Gradient Vaccine | PLAN |
|
||||
| 17 | Counterfactual Regret Reward | PLAN |
|
||||
| 19 | Position Entropy | PLAN |
|
||||
| 21 | Stochastic Depth | PLAN |
|
||||
| 26 | HER + Regime Oversampling | PLAN |
|
||||
| 29 | Cross-Fold Consistency | PLAN |
|
||||
|
||||
### Phase 3 — Architecture Changes (after validation)
|
||||
| # | Technique | Status |
|
||||
|---|-----------|--------|
|
||||
| 8 | Feature Importance Filtering | PLAN |
|
||||
| 9 | Regime-Adversarial (DANN) | PLAN |
|
||||
| 14 | Market Maker Adversary | PLAN |
|
||||
| 16 | Phantom Liquidity Episodes | PLAN |
|
||||
| 20 | Lottery Ticket Pruning | PLAN |
|
||||
| 28 | Flattest Selection (SAM) | PLAN |
|
||||
|
||||
## Expected Combined Impact
|
||||
|
||||
| Category | Techniques | Gap Reduction |
|
||||
|----------|-----------|---------------|
|
||||
| Symmetry & Invariance (A) | 10-13 | 40-60% |
|
||||
| Adversarial & Game-Theoretic (B) | 14-16 | 30-50% |
|
||||
| Loss & Reward Reshaping (C) | 17-19 | 25-40% |
|
||||
| Network Architecture (D) | 20-22 | 20-35% |
|
||||
| Temporal & Causal (E) | 23-26 | 30-45% |
|
||||
| Meta-Learning & Selection (F) | 27-29 | 20-30% |
|
||||
| Previously Planned (1-9) | 1-9 | 50-80% |
|
||||
|
||||
**Combined target:** OOS Sharpe from -1.30 to > 0.0 (positive expectancy OOS).
|
||||
|
||||
---
|
||||
|
||||
## Critical: FP32 Experience Collector Weights
|
||||
|
||||
### Task 30: Experience Collector FP32 Weight Conversion [PLAN]
|
||||
|
||||
**Phase:** 1 (Critical) | **Difficulty:** Medium | **Impact:** Very High
|
||||
|
||||
**Files:**
|
||||
- Modify: `crates/ml/src/cuda_pipeline/gpu_experience_collector.rs` (change weight buffers from bf16 to f32)
|
||||
- Modify: `crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs` (sync online weights as f32)
|
||||
- Modify: `crates/ml/src/cuda_pipeline/experience_kernels.cu` (read f32 weights instead of bf16)
|
||||
|
||||
The experience collector currently receives online Q-network weights in bf16 format for its
|
||||
forward passes during experience collection. This introduces quantization noise in Q-value
|
||||
estimates used for action selection, TD target computation, and priority calculation.
|
||||
Since the master weights in the Adam optimizer are already f32, the experience collector
|
||||
should use f32 copies to eliminate bf16 quantization artifacts during data collection.
|
||||
|
||||
- [ ] Change `online_params_flat` in GpuExperienceCollector from `CudaSlice<half::bf16>` to `CudaSlice<f32>`
|
||||
- [ ] Update weight sync to copy f32 master weights directly (skip bf16 conversion)
|
||||
- [ ] Update cuBLAS SGEMM calls to use f32 (already supports it, just change pointer types)
|
||||
- [ ] Update CUDA kernels to read f32 weights for forward pass during experience collection
|
||||
- [ ] Verify Q-value precision improvement in smoke tests
|
||||
- [ ] Compile + test + commit
|
||||
Reference in New Issue
Block a user