diff --git a/crates/ml/src/cuda_pipeline/c51_loss_kernel.cu b/crates/ml/src/cuda_pipeline/c51_loss_kernel.cu index e23f13383..05e1fc1a2 100644 --- a/crates/ml/src/cuda_pipeline/c51_loss_kernel.cu +++ b/crates/ml/src/cuda_pipeline/c51_loss_kernel.cu @@ -120,7 +120,19 @@ __device__ float block_expected_q_f( return block_reduce_sum_f(local_sum, shmem_reduce, tid); } -/* Bellman projection in float — critical for numerical stability */ +/* Bellman projection in float — critical for numerical stability. + * + * Task 2.4 (R6 relocation): Huber-style negative-tail compression applied to + * the projected Bellman target `t_z` BEFORE the v_min/v_max clamp. Protects + * the Q-target distribution from catastrophic-loss gradient dominance — + * same invariant previously enforced by reward-layer `asymmetric_soft_clamp` + * at `experience_kernels.cu:78-81`, now structurally co-located with the + * Q-target smoothing step where it belongs per reward-inventory's "wrong- + * level regularization" pattern. + * + * Shape: `if t_z < 0 { t_z = -10 * (1 - exp(t_z / 10)); }` — smooth + * exponential compression; f(-5) ≈ -3.93, f(-15) ≈ -7.77, f(0) = 0, + * positives untouched (upper cap handled by v_max). */ __device__ void block_bellman_project_f( const float* __restrict__ target_probs, float reward, float done, float gamma, @@ -136,6 +148,13 @@ __device__ void block_bellman_project_f( for (int j = tid; j < num_atoms; j += BLOCK_THREADS) { float z_j = support[j]; float t_z = reward + gamma * z_j * (1.0f - done); + /* Task 2.4: Huber-style negative-tail compression (relocated R6). + * Applied before v_min/v_max clamp so the smooth compression is the + * first transform; the hard clamp afterwards still enforces the + * support range. Positives pass through untouched. */ + if (t_z < 0.0f) { + t_z = -10.0f * (1.0f - expf(t_z / 10.0f)); + } t_z = fminf(fmaxf(t_z, v_min), v_max); float b_pos = (t_z - v_min) / delta_z; diff --git a/crates/ml/src/cuda_pipeline/experience_kernels.cu b/crates/ml/src/cuda_pipeline/experience_kernels.cu index 0fd5fae86..fef789d82 100644 --- a/crates/ml/src/cuda_pipeline/experience_kernels.cu +++ b/crates/ml/src/cuda_pipeline/experience_kernels.cu @@ -69,16 +69,14 @@ #define PREV_CLOSE_SLOT 6 /* reserved slot, now stores prev bar's raw_close for DSR */ #define PREV_MID_SLOT 22 /* reserved slot, now stores prev bar's MBP-10 mid-price */ -/* ── Reward v7: Asymmetric soft-clamp ────────────────────────────────── - * Gains: linear, capped at +10 (full gradient for positive rewards) - * Losses: exponential compression (smooth natural risk aversion) - * - * f(+5)=+5.0, f(-5)=-3.93, f(-15)=-7.77, f(+15)=+10.0 - * Replaces loss_aversion + tanh from v6. No separate parameter needed. */ -__device__ __forceinline__ float asymmetric_soft_clamp(float x) { - if (x >= 0.0f) return fminf(x, 10.0f); - return -10.0f * (1.0f - expf(x / 10.0f)); -} +/* Task 2.4: `asymmetric_soft_clamp()` DELETED from the reward layer. The + * negative-tail compression invariant (protect Q-targets from catastrophic- + * loss gradient dominance) was RELOCATED to `c51_loss_kernel.cu:: + * block_bellman_project_f` — Q-target smoothing is the structurally-correct + * home for that protection, per reward-inventory's "wrong-level + * regularization" pattern. The upper +10 cap is preserved inline at each + * former call site as `fminf(reward, 10.0f)`. + * Ref: docs/superpowers/specs/2026-04-21-policy-quality-track2-triage.md §R6. */ /* Shared trade physics (action decoding, hold enforcement, tx costs, * capital floor). Single source of truth — also used by backtest_env_kernel. */ @@ -1142,12 +1140,16 @@ extern "C" __global__ void experience_action_select( * 3. Computes target position = exposure_fraction * max_position. * 4. Applies position adjustment delta with volatility-scaled tx cost. * 5. Runs dynamic trailing stop (regime-adaptive). - * 6. Computes reward v8: - * sparse = trade_return * patience_mult (at trade exit, primary signal) + * 6. Computes reward (post-V7 / Task 0.8 shape — `patience_mult` and the + * V8 segment-patience term were removed when R7 was deleted): + * sparse = 2.0 * vol_normalized_return (at trade exit, primary signal, + * capped inline at +10) * reward = sparse (Sparse Trade-Completion Only) * + CEA bonus (counterfactual exposure advantage) * + order_credit (microstructure credit for order type selection) * + risk_efficiency (intra-trade risk-adjusted return bonus) + * Task 2.4: negative-tail compression (former `asymmetric_soft_clamp`) + * relocated to c51_loss_kernel.cu::block_bellman_project_f. * 7. Writes (batch_states, action, reward, done) to output replay buffer. * 8. Updates portfolio_states[0..19] in place. * 9. Increments current_timesteps[i]. @@ -1202,13 +1204,14 @@ extern "C" __global__ void experience_env_step( * Race-free by construction: each thread writes only its own [i*L+t] (or [cf_off]) slot. * cf_flip_per_sample — 1 iff counterfactual direction-flip fired this (i,t) (written at cf_off) * micro_reward_per_sample — micro-reward additive contribution this step (pre-drawdown, pre-floor) - * loss_aversion_per_sample — asymmetric-clamp delta (pre-clamp − post-clamp base reward) * total_reward_per_sample — final reward value written to out_rewards[out_off] (denominator) * slot_live_per_sample — 1 iff the kernel reached this (i,t) (denominator for firing-rate ratios; - * un-reached slots from early-terminated episodes stay 0 from memset) */ + * un-reached slots from early-terminated episodes stay 0 from memset) + * Task 2.4: loss_aversion_per_sample REMOVED — the R6 clamp was relocated to + * c51_loss_kernel.cu::block_bellman_project_f so the reward-layer diagnostic + * has no source signal. */ int* __restrict__ cf_flip_per_sample, /* [N*L*cf_mult] (written at cf_off = out_off + N*L) */ float* __restrict__ micro_reward_per_sample, /* [N*L] */ - float* __restrict__ loss_aversion_per_sample, /* [N*L] */ float* __restrict__ total_reward_per_sample, /* [N*L] */ int* __restrict__ slot_live_per_sample, /* [N*L] */ /* Task 2.X prerequisite (policy-quality scoping) — per-magnitude win rate @@ -1334,7 +1337,6 @@ extern "C" __global__ void experience_env_step( * total_actions_for_var == 0). Real kernel write happens at line ~1422 * inside the `if (q_variance != NULL)` branch. */ micro_reward_per_sample[out_off] = 0.0f; - loss_aversion_per_sample[out_off] = 0.0f; total_reward_per_sample[out_off] = 0.0f; var_scale_per_sample[out_off] = 0.0f; { @@ -1845,15 +1847,18 @@ extern "C" __global__ void experience_env_step( * * Fixes v6 penalty stacking (95% breakeven win rate → ~52%). * Novel components: - * 1. Asymmetric soft-clamp (natural risk aversion, no param needed) - * 2. Counterfactual Exposure Advantage — per-branch gradient signal - * 3. Order type microstructure credit — execution quality - * 4. Intra-trade risk efficiency — path quality bonus + * 1. Counterfactual Exposure Advantage — per-branch gradient signal + * 2. Order type microstructure credit — execution quality + * 3. Intra-trade risk efficiency — path quality bonus * * Kept: sparse trade-completion, vol-normalization, drawdown penalty, * capital floor, Kelly stats, position entropy. - * Removed: loss_aversion, regret_blend, hold_scale, clustering, - * beta_penalty. + * Removed: loss_aversion (V6 param), regret_blend, hold_scale, + * clustering, beta_penalty. + * Task 2.4: asymmetric soft-clamp (V7 replacement for V6 loss_aversion) + * also removed from reward layer — negative-tail compression + * relocated to c51_loss_kernel.cu::block_bellman_project_f; + * upper +10 cap retained inline. * ================================================================ */ float reward = 0.0f; @@ -1912,17 +1917,13 @@ extern "C" __global__ void experience_env_step( /* (DSR moved to dense per-bar path below — no longer sparse at trade completion) */ - /* ── Layer 2: Asymmetric soft-clamp (replaces loss_aversion + tanh) ── */ + /* ── Layer 2: Upper-cap only (Task 2.4 relocation) ── + * Negative-tail compression moved to c51_loss_kernel.cu:: + * block_bellman_project_f (Huber-style Q-target smoothing). + * Upper +10 cap kept inline for numerical safety against adversarial + * reward explosions from rare large wins. */ float base_reward = 2.0f * vol_normalized_return; - reward = asymmetric_soft_clamp(base_reward); - /* Task 0.8: loss_aversion delta — signed (base_reward - clamped_reward). - * NEGATIVE when negative-tail compression fires (asymmetric_soft_clamp - * returns -10*(1 - exp(x/10)) for x<0, which is LESS NEGATIVE than x, - * so base - clamped < 0). - * POSITIVE when the upper cap fires (rare). - * Host takes |·| before the ratio computation so the diagnostic - * reports clamp MAGNITUDE as fraction of post-transform reward scale. */ - loss_aversion_per_sample[out_off] = base_reward - reward; + reward = fminf(base_reward, 10.0f); /* ── Layer 3: CEA counterfactual loop REMOVED (4-branch: direction(4) replaces exposure(9)). * Dense micro-reward (Gem 1) below provides directional gradient signal. ── */ @@ -2221,9 +2222,11 @@ extern "C" __global__ void experience_env_step( out_dones[out_off] = (float)done; /* Task 0.8: final reward magnitude is the denominator for additive-term - * contribution ratios (micro, loss_aversion). Captured AFTER all shaping - * layers but before CF-slot writes so it reflects the on-policy sample's - * reward that the network actually trains on. */ + * contribution ratios (micro). Captured AFTER all shaping layers but + * before CF-slot writes so it reflects the on-policy sample's reward + * that the network actually trains on. + * Task 2.4: loss_aversion no longer contributes — the R6 clamp relocated + * to c51_loss_kernel.cu::block_bellman_project_f. */ total_reward_per_sample[out_off] = reward; /* ---- Write RAW portfolio return (unshaped) for accurate Sharpe/MaxDD ---- */ @@ -3044,7 +3047,10 @@ extern "C" __global__ void hindsight_relabel_kernel( if (best_pnl > 0.0f) { float opt_return = best_pnl / entry_price; - float opt_reward = asymmetric_soft_clamp(10.0f * opt_return / 0.005f); + /* Task 2.4: upper +10 cap only — negative-tail compression moved to + * c51_loss_kernel.cu::block_bellman_project_f. best_pnl > 0 here so + * opt_reward is always non-negative; the cap enforces numerical safety. */ + float opt_reward = fminf(10.0f * opt_return / 0.005f, 10.0f); rewards[i] = opt_reward; } } diff --git a/crates/ml/src/cuda_pipeline/gpu_experience_collector.rs b/crates/ml/src/cuda_pipeline/gpu_experience_collector.rs index f83560ca5..70d46eb54 100644 --- a/crates/ml/src/cuda_pipeline/gpu_experience_collector.rs +++ b/crates/ml/src/cuda_pipeline/gpu_experience_collector.rs @@ -591,7 +591,8 @@ pub struct GpuExperienceCollector { /// The rest live at `out_off` and are sized `alloc_episodes * alloc_timesteps`. cf_flip_per_sample: CudaSlice, // [alloc_episodes * alloc_timesteps * cf_mult] micro_reward_per_sample: CudaSlice, // [alloc_episodes * alloc_timesteps] - loss_aversion_per_sample: CudaSlice, // [alloc_episodes * alloc_timesteps] + // Task 2.4: loss_aversion_per_sample REMOVED — R6 clamp relocated to + // c51_loss_kernel.cu::block_bellman_project_f (Q-target smoothing). total_reward_per_sample: CudaSlice, // [alloc_episodes * alloc_timesteps] /// Task 2.X prerequisite (policy-quality scoping): per-magnitude win rate + @@ -1030,9 +1031,8 @@ impl GpuExperienceCollector { let micro_reward_per_sample = stream .alloc_zeros::(total_output) .map_err(|e| MLError::ModelError(format!("alloc micro_reward_per_sample: {e}")))?; - let loss_aversion_per_sample = stream - .alloc_zeros::(total_output) - .map_err(|e| MLError::ModelError(format!("alloc loss_aversion_per_sample: {e}")))?; + // Task 2.4: loss_aversion_per_sample allocation removed — R6 clamp + // relocated to c51_loss_kernel.cu::block_bellman_project_f. let total_reward_per_sample = stream .alloc_zeros::(total_output) .map_err(|e| MLError::ModelError(format!("alloc total_reward_per_sample: {e}")))?; @@ -1368,7 +1368,6 @@ impl GpuExperienceCollector { hold_at_exit_per_sample, cf_flip_per_sample, micro_reward_per_sample, - loss_aversion_per_sample, total_reward_per_sample, action_mag_per_sample, step_ret_per_sample, @@ -1825,7 +1824,7 @@ impl GpuExperienceCollector { /// Task 0.8 reward-term contribution audit (Track 2 policy-quality). /// - /// Returns `[popart, cf_flip, trail_r, micro, loss_aversion]`: + /// Returns `[popart, cf_flip, trail_r, micro]`: /// * `popart` — slot 0 is populated by the caller (host-side PopArt state /// lives on the trainer, not the collector). Returned as 0.0. /// * `cf_flip` — firing rate ∈ [0, 1] of the directional counterfactual @@ -1854,20 +1853,20 @@ impl GpuExperienceCollector { /// as a real semantic value ("profile intentionally disables /// R5"), NOT a wiring regression. Phase 2 Task 2.3 kept the /// kernel path live per feedback_no_functionality_removal.md. - /// * `loss_aversion` — |Σ Δclamp| / |Σ post-transform total|. Same caveat - /// as `micro`: numerator is the signed delta - /// (`base_reward - clamped_reward`), kernel writes the signed - /// value and the host takes `.abs()` here. Δ is NEGATIVE when - /// the negative-tail compression fires (the dominant case) - /// and POSITIVE when the upper cap fires (rare). /// - /// Buffers owned by this accessor (cf_flip, micro, loss_aversion, total, - /// slot_live) are reset via memset_zeros. Task 0.5 buffers - /// (trail_triggered_per_sample, traded_per_sample) are LEFT UNTOUCHED so - /// the subsequent per-mag reducer still has data. + /// Task 2.4: the old `loss_aversion` slot (was slot 4) was removed. The R6 + /// reward-layer clamp was relocated to `c51_loss_kernel.cu::block_bellman_ + /// project_f` (Q-target smoothing) so the reward-layer diagnostic has no + /// source signal. Tuple shrinks 5→4; downstream HEALTH_DIAG format string + /// dropped the `la={:.3}` field in the same commit. + /// + /// Buffers owned by this accessor (cf_flip, micro, total, slot_live) are + /// reset via memset_zeros. Task 0.5 buffers (trail_triggered_per_sample, + /// traded_per_sample) are LEFT UNTOUCHED so the subsequent per-mag reducer + /// still has data. /// /// Race-free by construction — kernel writes only its own (i,t) slot. - pub fn reward_contrib_fractions(&mut self) -> Result<[f32; 5], MLError> { + pub fn reward_contrib_fractions(&mut self) -> Result<[f32; 4], MLError> { let n_main = self.alloc_episodes * self.alloc_timesteps; let n_cf = n_main * 2; // cf_mult = 2 (#7 counterfactual doubles output) @@ -1913,40 +1912,33 @@ impl GpuExperienceCollector { }; // ── Additive ratios: Σ|term| / Σ|total| ───────────────────────── + // Task 2.4: `loss_aversion_per_sample` dtoh removed — R6 clamp moved to + // c51_loss_kernel.cu::block_bellman_project_f. let mut h_micro = vec![0.0_f32; n_main]; - let mut h_la = vec![0.0_f32; n_main]; let mut h_total = vec![0.0_f32; n_main]; self.stream.memcpy_dtoh(&self.micro_reward_per_sample, &mut h_micro) .map_err(|e| MLError::ModelError(format!("dtoh micro_reward_per_sample: {e}")))?; - self.stream.memcpy_dtoh(&self.loss_aversion_per_sample, &mut h_la) - .map_err(|e| MLError::ModelError(format!("dtoh loss_aversion_per_sample: {e}")))?; self.stream.memcpy_dtoh(&self.total_reward_per_sample, &mut h_total) .map_err(|e| MLError::ModelError(format!("dtoh total_reward_per_sample: {e}")))?; let mut sum_micro = 0.0_f64; - let mut sum_la = 0.0_f64; let mut sum_total = 0.0_f64; for i in 0..n_main { sum_micro += (h_micro[i].abs()) as f64; - sum_la += (h_la[i].abs()) as f64; sum_total += (h_total[i].abs()) as f64; } - let (micro_r, la_r) = if sum_total > 1e-12 { - ( - (sum_micro / sum_total) as f32, - (sum_la / sum_total) as f32, - ) + let micro_r = if sum_total > 1e-12 { + (sum_micro / sum_total) as f32 } else { - (0.0, 0.0) + 0.0 }; // ── Reset owned buffers ONLY (leave Task 0.5 trail buffers alone) ─ + // Task 2.4: `loss_aversion_per_sample` memset removed with the buffer. self.stream.memset_zeros(&mut self.cf_flip_per_sample) .map_err(|e| MLError::ModelError(format!("memset cf_flip_per_sample: {e}")))?; self.stream.memset_zeros(&mut self.micro_reward_per_sample) .map_err(|e| MLError::ModelError(format!("memset micro_reward_per_sample: {e}")))?; - self.stream.memset_zeros(&mut self.loss_aversion_per_sample) - .map_err(|e| MLError::ModelError(format!("memset loss_aversion_per_sample: {e}")))?; self.stream.memset_zeros(&mut self.total_reward_per_sample) .map_err(|e| MLError::ModelError(format!("memset total_reward_per_sample: {e}")))?; self.stream.memset_zeros(&mut self.slot_live_per_sample) @@ -1955,7 +1947,7 @@ impl GpuExperienceCollector { // PopArt slot is populated by the caller (training_loop) from the // host-side FusedTrainingCtx — it owns `prev_popart_var` + the live // popart_var readback. Return 0.0 here; caller overrides. - Ok([0.0, cf_rate, trail_r, micro_r, la_r]) + Ok([0.0, cf_rate, trail_r, micro_r]) } /// Task 2.X prerequisite (policy-quality scoping): per-magnitude win rate @@ -2166,7 +2158,7 @@ impl GpuExperienceCollector { // Download raw_returns_out for step_returns (once per epoch, cold path). // raw_returns_out contains TRUE per-bar portfolio returns (unshaped): // (equity_t - equity_{t-1}) / equity_{t-1} - // NOT the shaped RL reward (which includes dense*0.1 + sparse*2.0 + loss_aversion). + // NOT the shaped RL reward (which includes dense*0.1 + sparse*2.0 shaping layers). // Total = alloc_episodes * alloc_timesteps floats (~12KB for 3200 experiences). // raw_returns_out is NOT doubled by counterfactual (only states/actions/rewards/dones are) let base_output = self.alloc_episodes * self.alloc_timesteps; @@ -2956,10 +2948,10 @@ impl GpuExperienceCollector { .arg(&mut self.pre_mag_per_sample) .arg(&mut self.traded_per_sample) .arg(&mut self.hold_at_exit_per_sample) - // Task 0.8 reward-term contribution: cf_flip / micro / loss_aversion / total / slot_live + // Task 0.8 reward-term contribution: cf_flip / micro / total / slot_live + // Task 2.4: loss_aversion arg removed (R6 clamp moved to C51 Bellman). .arg(&mut self.cf_flip_per_sample) .arg(&mut self.micro_reward_per_sample) - .arg(&mut self.loss_aversion_per_sample) .arg(&mut self.total_reward_per_sample) .arg(&mut self.slot_live_per_sample) // Task 2.X prerequisite: per-magnitude win-rate + variance instrumentation diff --git a/crates/ml/src/trainers/dqn/smoke_tests/reward_component_audit.rs b/crates/ml/src/trainers/dqn/smoke_tests/reward_component_audit.rs index 4b96baf6d..2b1043082 100644 --- a/crates/ml/src/trainers/dqn/smoke_tests/reward_component_audit.rs +++ b/crates/ml/src/trainers/dqn/smoke_tests/reward_component_audit.rs @@ -6,13 +6,18 @@ //! Phase 1 work performed against logs from a multi-fold L40S run — this //! smoke gates that the diagnostic infrastructure runs cleanly. //! -//! The audit covers 8 reward terms split into 3 measurement classes per -//! spec §4.1: -//! - Additive (R1 step_return, R5 micro-reward, R6 loss-aversion, R7 segment-patience): -//! fraction of |total reward| +//! The audit covers the live reward terms split into 3 measurement classes +//! per spec §4.1: +//! - Additive (R1 step_return, R5 micro-reward): fraction of |total reward| //! - Transform (R2 PopArt): |delta| / |pre| ratio //! - Sample-selector (R3 CF-flip, R4 trailing-stop): firing rate //! +//! History: R7 segment-patience slot was deleted in Task 0.8 (kernel never +//! wired it). R6 loss-aversion slot was deleted in Task 2.4 — the R6 +//! asymmetric-soft-clamp was relocated from the reward layer to the C51 +//! Bellman target smoothing (`c51_loss_kernel.cu::block_bellman_project_f`), +//! so the reward-layer diagnostic no longer has a source signal. +//! //! Run: `FOXHUNT_TEST_DATA=test_data/futures-baseline \ //! cargo test -p ml --release --lib -- reward_component_audit --ignored --nocapture` @@ -42,23 +47,26 @@ fn test_reward_components_contribute() -> Result<()> { // Reward-contribution diagnostic comes from HEALTH_DIAG `reward_contrib` group. // Populated by Task 0.8's kernel-side instrumentation: popart drift from // host-side FusedTrainingCtx, cf_flip/trail firing rates from collector - // per-sample buffers, micro/loss_aversion additive ratios. All finite and - // non-negative by construction (absolute-value sums, rate = a/b with b >= 0). + // per-sample buffers, micro additive ratio. All finite and non-negative + // by construction (absolute-value sums, rate = a/b with b >= 0). + // + // Task 2.4: `loss_aversion` slot removed — R6 clamp relocated to + // c51_loss_kernel.cu::block_bellman_project_f (Q-target smoothing). + // Reward-layer diagnostic no longer has a source signal; tuple shrank 5→4. // // Real triage uses log-grep against a 6-fold × 50-epoch L40S run (Phase 1). // The smoke's job is to ensure the reporting scaffolding never regresses // with NaN/Inf values, which would break log parsing downstream. let summary = trainer.reward_component_audit_summary(); - let [popart, cf_flip, trail_r, micro, loss_aversion] = summary; + let [popart, cf_flip, trail_r, micro] = summary; println!( - "[REWARD_AUDIT] popart={:.4} cf_flip={:.4} trail={:.4} micro={:.4} \ - loss_aversion={:.4}", - popart, cf_flip, trail_r, micro, loss_aversion + "[REWARD_AUDIT] popart={:.4} cf_flip={:.4} trail={:.4} micro={:.4}", + popart, cf_flip, trail_r, micro ); // ── Finite + non-negative gate (structural, unchanged intent) ── // Magnitudes should be unsigned by construction (abs-sums, a/b with b≥0). - let names = ["popart", "cf_flip", "trail", "micro", "loss_aversion"]; + let names = ["popart", "cf_flip", "trail", "micro"]; for (i, &v) in summary.iter().enumerate() { assert!( v.is_finite(), @@ -121,13 +129,14 @@ fn test_reward_components_contribute() -> Result<()> { // No additional floor: these slots legitimately stay near zero in smoke. let _ = (popart, micro); - // loss_aversion fires whenever a trade has a negative return that gets - // clamped by the asymmetric soft-clamp. With 20 epochs and default - // trading, at least some loss clamps should occur. loss_aversion can - // legitimately be small (the bulk of rewards are non-negative PnL), so - // assert ≥ 0 only (covered above). No floor — a run with zero losing - // trades is statistically unlikely but not indicative of a wiring bug. - let _ = loss_aversion; + // Task 2.4: the old `loss_aversion` slot was dropped. R6 was the + // asymmetric-soft-clamp on `base_reward` that fed the reward-layer + // diagnostic. The invariant (protect Q-targets from catastrophic-loss + // gradient dominance) is now enforced one layer deeper at + // `c51_loss_kernel.cu::block_bellman_project_f` — a Huber-style + // compression on the projected Bellman target before the v_min/v_max + // clamp. Same protection, structurally-correct location; no reward-layer + // slot remains to assert against. Ok(()) } diff --git a/crates/ml/src/trainers/dqn/trainer/constructor.rs b/crates/ml/src/trainers/dqn/trainer/constructor.rs index f9833f3c3..13158e001 100644 --- a/crates/ml/src/trainers/dqn/trainer/constructor.rs +++ b/crates/ml/src/trainers/dqn/trainer/constructor.rs @@ -584,7 +584,7 @@ impl DQNTrainer { grad_clip_kicked_this_epoch: false, last_eval_magnitude_dist: [0.0_f32; 3], prev_reward_contrib_popart_var: 0.0, - last_reward_contrib: [0.0_f32; 5], + last_reward_contrib: [0.0_f32; 4], // Wave 16 Portfolio Features (action masking always active) max_position, diff --git a/crates/ml/src/trainers/dqn/trainer/mod.rs b/crates/ml/src/trainers/dqn/trainer/mod.rs index a1dbb4cec..888df1414 100644 --- a/crates/ml/src/trainers/dqn/trainer/mod.rs +++ b/crates/ml/src/trainers/dqn/trainer/mod.rs @@ -321,11 +321,14 @@ pub struct DQNTrainer { pub(crate) prev_reward_contrib_popart_var: f32, /// Task 0.8 — most recent reward-term contribution summary [popart, - /// cf_flip, trail, micro, loss_aversion]. Updated once per HEALTH_DIAG - /// emission; exposed via reward_component_audit_summary for the Track 2 - /// smoke test. `segment_patience` slot removed — the kernel never wired - /// a patience term, so the slot was a stub; schema reduced accordingly. - pub(crate) last_reward_contrib: [f32; 5], + /// cf_flip, trail, micro]. Updated once per HEALTH_DIAG emission; exposed + /// via reward_component_audit_summary for the Track 2 smoke test. + /// `segment_patience` slot removed (Task 0.8) — the kernel never wired a + /// patience term, so the slot was a stub. + /// `loss_aversion` slot removed (Task 2.4) — R6 clamp relocated to + /// c51_loss_kernel.cu::block_bellman_project_f so the reward-layer + /// diagnostic has no source signal. + pub(crate) last_reward_contrib: [f32; 4], // Wave 16 Portfolio Features (action masking is always active). /// Maximum position size for action masking (default: 2.0) @@ -1529,17 +1532,21 @@ impl DQNTrainer { } /// Per-term reward contribution summary (Track 2 audit). - /// Layout: [popart, cf_flip, trail, micro, loss_aversion]. + /// Layout: [popart, cf_flip, trail, micro]. /// Each entry's interpretation depends on the term's measurement class /// per spec §4.1 (additive vs transform vs sample-selector). Used by /// reward_component_audit smoke test. /// /// Task 0.8: populated from HEALTH_DIAG emissions in the epoch-boundary /// block (training_loop.rs). Always finite + non-negative per the smoke - /// test contract. Layout: [popart, cf_flip, trail, micro, loss_aversion]. - /// The old `segment_patience` slot was deleted per the no-stubs rule — - /// the kernel never wired a patience term so the slot was pure stub. - pub fn reward_component_audit_summary(&self) -> [f32; 5] { + /// test contract. + /// + /// History: `segment_patience` slot deleted (Task 0.8) — the kernel never + /// wired a patience term so the slot was pure stub. `loss_aversion` slot + /// deleted (Task 2.4) — R6 clamp relocated to c51_loss_kernel.cu:: + /// block_bellman_project_f (Q-target smoothing) so the reward-layer + /// diagnostic no longer has a source signal. + pub fn reward_component_audit_summary(&self) -> [f32; 4] { self.last_reward_contrib } diff --git a/crates/ml/src/trainers/dqn/trainer/training_loop.rs b/crates/ml/src/trainers/dqn/trainer/training_loop.rs index 1a6c583a0..4ceb98825 100644 --- a/crates/ml/src/trainers/dqn/trainer/training_loop.rs +++ b/crates/ml/src/trainers/dqn/trainer/training_loop.rs @@ -2594,10 +2594,10 @@ impl DQNTrainer { Ok(v) => v, Err(e) => { tracing::warn!("HEALTH_DIAG reward_contrib readback failed: {e}"); - [0.0_f32; 5] + [0.0_f32; 4] } }, - None => [0.0_f32; 5], + None => [0.0_f32; 4], }; // Populate PopArt drift (slot 0): |current_var − prev_var| / |prev_var|. // Uses FusedTrainingCtx::read_popart_variance(). Slot stays at 0.0 @@ -2625,8 +2625,9 @@ impl DQNTrainer { self.prev_reward_contrib_popart_var = cur_var; } // Cache for the smoke-test accessor reward_component_audit_summary. - // Smoke expects all 5 values finite + >= 0 — the readback already + // Smoke expects all 4 values finite + >= 0 — the readback already // enforces that (absolute-value sums, rate = a / b with b >= 0). + // Task 2.4: slot count shrank 5→4 when R6 clamp relocated to C51. self.last_reward_contrib = reward_contrib; // H6: per-magnitude trailing-stop fire rate + hold-at-exit means. @@ -2670,7 +2671,7 @@ impl DQNTrainer { }; tracing::info!( - "HEALTH_DIAG[{}]: health={:.2} components [q_gap={:.2} q_var={:.2} atoms={:.2} grad_stable={:.2} ens_agree={:.2} grad_cos={:.2} spectral={:.2}] effective [cql_alpha={:.4} iqn_budget={:.2} cql_budget={:.2} c51_budget={:.2} tau={:.5} sarsa_tau={:.2} gamma={:.3} cf_ratio={:.2}] novels [distill={} barrier={:.3} plasticity={} ib={:.3} ensemble_collapse={:.2} contrarian={} meta_q_pred={:.2}] diag [sharpe_ema={:.3} action_entropy={:.2}] gems [g12_predictive={:.4}] mag [q_full={:.3} q_half={:.3} q_quarter={:.3} var_scale={:.3} kelly_f={:.3} avg_win_ratio={:.3} grad_ratio_mag_dir={:.4} dist_q={:.3} dist_h={:.3} dist_f={:.3}] grad_split_bwd [iqn={:.4} cql={:.4} c51={:.4} ens={:.4}] grad_split_aux [distill={:.4} rec={:.4} pred={:.4} cql_sx={:.4} c51_bs={:.4}] grad_abs [dir={:.6e} mag={:.6e}] trail [fire_q={:.3} fire_h={:.3} fire_f={:.3} hold_q={:.2} hold_h={:.2} hold_f={:.2}] mag_stats [wr_q={:.3} wr_h={:.3} wr_f={:.3} var_q={:.4e} var_h={:.4e} var_f={:.4e}] noisy [vsn_mag={:.3} vsn_dir={:.3} sigma_mag={:.4} sigma_dir={:.4} drift_mag={:.3} drift_dir={:.3}] eval_dist [eq={:.3} eh={:.3} ef={:.3}] reward_contrib [popart={:.3} cf={:.3} trail_r={:.3} micro={:.3} la={:.3}] controller [anti_lr={} tau={} gamma={} clip={} cql={} cost={} fire_frac={:.2}] explore [ent_mag={:.2} ent_dir={:.2} sigma_mean={:.4}]", + "HEALTH_DIAG[{}]: health={:.2} components [q_gap={:.2} q_var={:.2} atoms={:.2} grad_stable={:.2} ens_agree={:.2} grad_cos={:.2} spectral={:.2}] effective [cql_alpha={:.4} iqn_budget={:.2} cql_budget={:.2} c51_budget={:.2} tau={:.5} sarsa_tau={:.2} gamma={:.3} cf_ratio={:.2}] novels [distill={} barrier={:.3} plasticity={} ib={:.3} ensemble_collapse={:.2} contrarian={} meta_q_pred={:.2}] diag [sharpe_ema={:.3} action_entropy={:.2}] gems [g12_predictive={:.4}] mag [q_full={:.3} q_half={:.3} q_quarter={:.3} var_scale={:.3} kelly_f={:.3} avg_win_ratio={:.3} grad_ratio_mag_dir={:.4} dist_q={:.3} dist_h={:.3} dist_f={:.3}] grad_split_bwd [iqn={:.4} cql={:.4} c51={:.4} ens={:.4}] grad_split_aux [distill={:.4} rec={:.4} pred={:.4} cql_sx={:.4} c51_bs={:.4}] grad_abs [dir={:.6e} mag={:.6e}] trail [fire_q={:.3} fire_h={:.3} fire_f={:.3} hold_q={:.2} hold_h={:.2} hold_f={:.2}] mag_stats [wr_q={:.3} wr_h={:.3} wr_f={:.3} var_q={:.4e} var_h={:.4e} var_f={:.4e}] noisy [vsn_mag={:.3} vsn_dir={:.3} sigma_mag={:.4} sigma_dir={:.4} drift_mag={:.3} drift_dir={:.3}] eval_dist [eq={:.3} eh={:.3} ef={:.3}] reward_contrib [popart={:.3} cf={:.3} trail_r={:.3} micro={:.3}] controller [anti_lr={} tau={} gamma={} clip={} cql={} cost={} fire_frac={:.2}] explore [ent_mag={:.2} ent_dir={:.2} sigma_mean={:.4}]", epoch, health_value, self.learning_health.components.q_gap_norm, @@ -2765,12 +2766,15 @@ impl DQNTrainer { self.last_eval_magnitude_dist[0], self.last_eval_magnitude_dist[1], self.last_eval_magnitude_dist[2], - // Track 2 — reward contrib (5 f32): popart drift, cf_flip rate, - // trail rate, micro additive ratio, loss-aversion additive ratio - // (Task 0.8). `segment_patience` slot deleted — the kernel never - // wired a patience term, so preserving the slot was pure stub. + // Track 2 — reward contrib (4 f32): popart drift, cf_flip rate, + // trail rate, micro additive ratio (Task 0.8). + // History: `segment_patience` slot deleted in Task 0.8 (kernel + // never wired a patience term); `loss_aversion` slot deleted in + // Task 2.4 (R6 clamp relocated to c51_loss_kernel.cu:: + // block_bellman_project_f, reward-layer diagnostic no longer + // has source signal). reward_contrib[0], reward_contrib[1], reward_contrib[2], - reward_contrib[3], reward_contrib[4], + reward_contrib[3], // Track 3 — controllers (6 bool per-epoch fire + 1 f32 max running rate) fire_lr, fire_tau, fire_gamma, fire_clip, fire_cql, fire_cost, fire_frac, // Track 4 — explore (3 f32): ent_mag, ent_dir, sigma_mean. diff --git a/docs/superpowers/specs/2026-04-21-policy-quality-track2-triage.md b/docs/superpowers/specs/2026-04-21-policy-quality-track2-triage.md index 880cae4e2..edf94e569 100644 --- a/docs/superpowers/specs/2026-04-21-policy-quality-track2-triage.md +++ b/docs/superpowers/specs/2026-04-21-policy-quality-track2-triage.md @@ -14,7 +14,7 @@ Preliminary triage against spec §5.2 reward terms R1–R8. At smoke scale: - **3 KEEP** — R1 (step_return, base reward), R3 (CF-flip, 66% contribution is the dominant signal), R8 (raw_returns_out, required by Sharpe / MaxDD / risk-metric pipeline) - **1 KEEP-WITH-CAVEAT** — R4 (trailing-stop, 25 / 60 epochs above the 1 % floor; firing-rate diagnostic only, not a reward additive — re-validate at L40S where Full positions actually enter) -- **1 DELETE** — R6 (loss-aversion asymmetric clamp: 6 / 60 epochs above 1 %, mean 0.26 % — sub-noise) +- **1 DELETED / RELOCATED** — R6 (loss-aversion asymmetric clamp: 6 / 60 epochs above 1 %, mean 0.26 % — sub-noise). **Status:** reward-layer clamp DELETED from `experience_kernels.cu`; negative-tail compression RELOCATED to `c51_loss_kernel.cu::block_bellman_project_f` (Huber-style `if t_z < 0 { t_z = -10 * (1 - exp(t_z / 10)); }` before the `v_min`/`v_max` clamp). Upper +10 cap preserved inline at former call sites as `fminf(reward, 10.0f)`. Same invariant, structurally-correct location per reward-inventory's "wrong-level regularization" pattern. Phase 2 Task 2.4 landed. - **1 FIX-documented-disable** — R5 (micro-reward: `micro_reward_scale=0` in smoke, contribution 0 / 60 epochs — smoke-scale-only disable; production toml has scale=0.1 so the code path is already load-bearing in prod. Per `feedback_no_functionality_removal.md` the DELETE verdict was rejected; Phase 2 Task 2.3 preserved the code path and documented the disable.) - **1 PENDING** — R2 (PopArt drift: 0 / 60 epochs nonzero; warmup-gated by design — requires production-scale run before a final verdict) - **1 ALREADY-REMOVED** — R7 (segment-patience multiplier: never wired in the kernel; slot deleted per Task 0.8 stub-remediation commit `83d524f86`) @@ -179,13 +179,19 @@ Distribution: clamp is silent in the first 33 rows (fold 1 and most of fold 2), The pattern: only segment-complete bars with `base_reward < 0` AND `|base_reward| > ~0` trigger meaningful clamp delta. Fold 1 has low trade-completion count; fold 3 has enough to surface the term. -**Decision-matrix verdict:** **DELETE** — contribution < 1 % in 54 / 60 epochs (majority) AND the invariant (compress catastrophic negative tail) is better expressed at the gradient level (Q-target smoothing / Huber loss) per the reward-inventory pattern for "wrong-level regularization." +**Decision-matrix verdict:** **DELETED / RELOCATED** (Phase 2 Task 2.4 landed) — contribution < 1 % in 54 / 60 epochs (majority) AND the invariant (compress catastrophic negative tail) is better expressed at the gradient level (Q-target smoothing / Huber loss) per the reward-inventory pattern for "wrong-level regularization." With mean 0.26 % and max 3.1 %, even when R6 fires it rarely exceeds 3 % of reward magnitude. The asymmetric compression is doing little work that couldn't be done by Huber loss / Q-target noise in the downstream training kernel. -**Caveat:** the `asymmetric_soft_clamp` call also caps positives at +10 (upper-bound protection against reward explosions from rare large wins). That cap is structurally load-bearing on adversarial inputs even if the negative-tail compression is near-zero. Deletion must preserve the +10 upper clamp (or redirect the protection to gradient clipping downstream). Phase 2 implementation must not just rip out the call. +**Landed relocation:** +- Reward-layer `asymmetric_soft_clamp` function deleted from `experience_kernels.cu:78-81`. +- Both call sites (trade-exit `:~1774` and hindsight-relabel `:~3047`) replaced with inline `fminf(reward, 10.0f)` preserving the +10 upper cap for numerical safety. +- Negative-tail compression relocated to `c51_loss_kernel.cu::block_bellman_project_f` — Huber-style `if (t_z < 0.0f) { t_z = -10.0f * (1.0f - expf(t_z / 10.0f)); }` applied to the projected Bellman target before the existing `v_min`/`v_max` clamp. Same invariant, structurally-correct location (Q-target smoothing is where this protection belongs). +- `loss_aversion_per_sample` buffer removed end-to-end: kernel arg, alloc, struct field, dtoh, memset. Tuple `reward_contrib_fractions` shrank 5→4 (slots now `[popart, cf_flip, trail_r, micro]`). HEALTH_DIAG `la={:.3}` field dropped. Smoke `reward_component_audit` updated. -**Evidence strength:** MEDIUM — sub-noise contribution at smoke scale; preserve the protection mechanism elsewhere when deleting. +**Caveat (historical, now resolved):** the old `asymmetric_soft_clamp` call also capped positives at +10 (upper-bound protection against reward explosions from rare large wins). The relocation preserves that cap inline at each former call site — it was never the load-bearing negative-tail compression, only the upper-bound safety cap. + +**Evidence strength:** MEDIUM — sub-noise contribution at smoke scale; relocation preserves the protection mechanism at the correct layer. --- @@ -259,7 +265,7 @@ Ordered by impact and independence: 1. **FIX-documented-disable R5 (micro-reward)** — DELETE verdict rejected per `feedback_no_functionality_removal.md`. Phase 2 Task 2.3 implementation: preserve the kernel branch at `experience_kernels.cu:1915..2003`, preserve the `micro_reward_scale` config field, preserve the `micro_reward_per_sample` buffer + reducer slot; add code-site comments explaining (a) scale=0 at smoke is intentional test-isolation, (b) production already deploys scale=0.1 so the code path is live in prod, (c) OFI state features are representation signal, micro_reward is gradient-injection — different mechanism, not redundant. Net code delta: +~50 LOC comments, zero deletion. -2. **DELETE R6 (asymmetric soft-clamp as a reward-layer mechanism)** — move the negative-tail compression to Q-target smoothing (same pattern as reward-inventory's `reward_noise_scale` relocation to C51 Bellman). Preserve the +10 upper cap as a separate guard (numerical safety, not loss-aversion). Change surface: `experience_kernels.cu:1774-1783` deletion + equivalent in C51 loss kernel. +2. **DELETED / RELOCATED — R6 (asymmetric soft-clamp)** (Task 2.4 landed) — the negative-tail compression moved to Q-target smoothing (same pattern as reward-inventory's `reward_noise_scale` relocation to C51 Bellman). The +10 upper cap is preserved as a separate guard at each former call site (numerical safety, not loss-aversion). Change surface: `experience_kernels.cu:78-81` (function deleted), `:~1774` + `:~3047` (call sites → `fminf(·, 10.0f)`), `c51_loss_kernel.cu::block_bellman_project_f` (Huber-style compression added before support clamp). Rust tuple `reward_contrib_fractions` + `last_reward_contrib` shrank 5→4; HEALTH_DIAG `la={:.3}` field dropped. 3. **CLEANUP stale R7 docstring** — `experience_kernels.cu:1049` comment mentions `patience_mult` for a removed term. Single-line edit: update to reflect current reward shape or delete the reference.