From 8a93a77adcb013637ca70f86e17c5e7f8e581850 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sun, 31 May 2026 17:25:47 +0200 Subject: [PATCH] refactor(trainer): extract json!{} into IntegratedTrainer::build_diag_value MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The per-step diag JSONL `json!{...}` block had grown to 642 leaf paths across ~30 nested objects, duplicating every ISV slot read into the example binary. Phase A of the 2026-05-31 checkpoints+eval-diag plan extracts it into a single builder on the trainer so the eval phase can reuse it (next commit) without duplicating the schema. Changes: * `IntegratedTrainer::build_diag_value(step, elapsed_s, &DiagInputs) -> Result` — same 642-leaf schema as before, bit-equivalent ISV reads (all from `self.isv_host_slice()`). * `DiagInputs<'a>` struct bundles the host-side per-step state the trainer doesn't own (DiagStaging reads + running counters + windowed act histogram), so the call site stays a 1-liner. * Train loop in `alpha_rl_train.rs` swaps the inline json! for the builder call; the ~140-slot ISV-imports wall collapses to four slots still read by the stderr ticker. * `#![recursion_limit = "256"]` moves from the example into `ml-alpha/src/lib.rs` since the builder now lives in the library. Schema parity verified: `head -1 diag.jsonl | jq 'paths(scalars)|sort'` yields the same 642 keys as before this refactor (no schema drift). --- crates/ml-alpha/examples/alpha_rl_train.rs | 752 ++------------------- crates/ml-alpha/src/lib.rs | 6 + crates/ml-alpha/src/trainer/integrated.rs | 697 ++++++++++++++++++- 3 files changed, 766 insertions(+), 689 deletions(-) diff --git a/crates/ml-alpha/examples/alpha_rl_train.rs b/crates/ml-alpha/examples/alpha_rl_train.rs index 321987040..a9fd9c83f 100644 --- a/crates/ml-alpha/examples/alpha_rl_train.rs +++ b/crates/ml-alpha/examples/alpha_rl_train.rs @@ -44,101 +44,29 @@ use data::providers::databento::dbn_parser::InstrumentFilter; // emerging meta-pattern: any constant that mirrors a kernel-side // structural dimension MUST reference the Rust const, not duplicate // the literal value. -use ml_alpha::rl::common::{N_ACTIONS, FRD_N_ATOMS, FRD_N_HORIZONS}; -use ml_alpha::rl::frd::FRD_OUT_DIM; +use ml_alpha::rl::common::N_ACTIONS; use ml_alpha::data::gpu_dataset::{GpuDataLoader, GpuDataset}; use ml_alpha::data::loader::{ discover_mbp10_files_sorted, MultiHorizonLoader, MultiHorizonLoaderConfig, DEFAULT_OUTCOME_LABEL_COST_ES, }; use ml_alpha::heads::HORIZONS; +// ISV slot imports — only the slots still read DIRECTLY in this file. +// Phase A of the 2026-05-31 checkpoints+eval-diag plan moved all ~140 +// diag-block ISV reads behind `IntegratedTrainer::build_diag_value`, so +// the previous wall of imports collapsed to the four slots still in +// the per-N-step stderr ticker. use ml_alpha::rl::isv_slots::{ - RL_ADVANTAGE_VAR_RATIO_EMA_INDEX, RL_ENTROPY_COEF_INDEX, RL_ENTROPY_OBSERVED_EMA_INDEX, - RL_GAMMA_INDEX, RL_KL_PI_EMA_INDEX, RL_LR_AUX_INDEX, RL_LR_BCE_INDEX, RL_LR_PI_BEST_LOSS_INDEX, - RL_LR_PI_INDEX, RL_LR_PI_LOSS_EMA_INDEX, RL_LR_PI_STEPS_SINCE_BEST_INDEX, - RL_LR_PI_WARMUP_COUNTER_INDEX, RL_LR_Q_BEST_LOSS_INDEX, RL_LR_Q_INDEX, - RL_LR_Q_LOSS_EMA_INDEX, RL_LR_Q_STEPS_SINCE_BEST_INDEX, RL_LR_Q_WARMUP_COUNTER_INDEX, - RL_LR_V_BEST_LOSS_INDEX, RL_LR_V_INDEX, RL_LR_V_LOSS_EMA_INDEX, - RL_LR_V_STEPS_SINCE_BEST_INDEX, RL_LR_V_WARMUP_COUNTER_INDEX, - RL_ADV_VAR_RATIO_CLAMP_INDEX, RL_ADV_VAR_RATIO_TARGET_INDEX, RL_ADV_VAR_STREAM_M2_INDEX, - RL_ADV_VAR_STREAM_MEAN_INDEX, RL_DIV_TARGET_INDEX, RL_ENTROPY_TARGET_FRAC_INDEX, - RL_EPS_BOOTSTRAP_INDEX, RL_IMPROVEMENT_THRESHOLD_INDEX, RL_KL_TARGET_INDEX, - RL_KURT_GAUSSIAN_INDEX, RL_KURT_LIFT_SCALE_INDEX, RL_KURT_NOISE_FLOOR_INDEX, - RL_K_LOOP_DIVISOR_INDEX, RL_K_LOOP_MAX_INDEX, RL_LOSS_LAMBDA_AUX_INDEX, - RL_LR_BOOTSTRAP_INDEX, RL_LR_DECAY_FACTOR_INDEX, RL_LR_LOSS_EMA_ALPHA_INDEX, - RL_LR_MAX_INDEX, RL_LR_MIN_INDEX, RL_LR_WARMUP_STEPS_INDEX, - RL_PLATEAU_PATIENCE_INDEX, RL_PPO_CLAMP_MARGIN_INDEX, - RL_PPO_RATIO_CLAMP_BOOTSTRAP_INDEX, RL_Q_ARG_VS_PI_AGREE_INDEX, - RL_REWARD_CLAMP_LOSS_INDEX, RL_REWARD_CLAMP_WIN_INDEX, RL_REWARD_SCALE_BOOTSTRAP_INDEX, - RL_ROLLOUT_BOOTSTRAP_INDEX, RL_SCHULMAN_ADJUST_RATE_INDEX, RL_SCHULMAN_TOLERANCE_INDEX, - RL_STREAM_ALPHA_INDEX, RL_TAU_BOOTSTRAP_INDEX, - RL_MAX_ABS_SCALED_REWARD_PRE_CLAMP_INDEX, RL_MEAN_ABS_PNL_EMA_INDEX, - RL_POS_SCALED_REWARD_MAX_INDEX, RL_POS_SCALED_REWARD_MAX_EMA_INDEX, - RL_REWARD_CLAMP_MARGIN_INDEX, RL_REWARD_CLAMP_RATIO_INDEX, - RL_REWARD_CLAMP_CLIP_RATE_EMA_INDEX, RL_REWARD_CLAMP_CLIP_RATE_TARGET_INDEX, - RL_C51_V_MAX_INDEX, RL_C51_V_MIN_INDEX, - RL_Q_DISTILL_LAMBDA_INDEX, RL_Q_DISTILL_TEMPERATURE_INDEX, RL_Q_DISTILL_KL_EMA_INDEX, - RL_NEG_SCALED_REWARD_MAX_INDEX, RL_NEG_SCALED_REWARD_MAX_EMA_INDEX, - RL_Q_DISTILL_KL_TARGET_INDEX, RL_REWARD_SCALE_MIN_INDEX, - RL_MEAN_TRADE_DURATION_EMA_INDEX, RL_N_ROLLOUT_STEPS_INDEX, RL_PER_ALPHA_INDEX, - RL_PI_GRAD_NORM_EMA_INDEX, RL_PPO_CLIP_INDEX, RL_PPO_LOG_RATIO_ABS_MAX_INDEX, - RL_PPO_RATIO_CLAMP_MAX_INDEX, RL_Q_DIVERGENCE_EMA_INDEX, RL_Q_GRAD_NORM_EMA_INDEX, - RL_REWARD_SCALE_INDEX, RL_TARGET_TAU_INDEX, RL_TD_KURTOSIS_CLAMP_INDEX, - RL_TD_KURTOSIS_EMA_INDEX, RL_TD_KURT_STREAM_M2_INDEX, RL_TD_KURT_STREAM_M4_INDEX, - RL_TD_KURT_STREAM_MEAN_INDEX, RL_V_GRAD_NORM_EMA_INDEX, - // P1+P2 feature metrics - RL_POPART_MEAN_INDEX, RL_POPART_SIGMA_INDEX, RL_POPART_VAR_INDEX, - RL_SPECTRAL_NORM_MAX_INDEX, RL_SPECTRAL_DECOUPLE_LAMBDA_INDEX, - RL_Q_BIAS_EMA_INDEX, RL_Q_BIAS_CORRECTION_INDEX, - RL_LR_SCALE_Q_INDEX, RL_LR_SCALE_PI_INDEX, RL_LR_SCALE_V_INDEX, RL_LR_SCALE_IQN_INDEX, - RL_OUTCOME_AUX_LAMBDA_INDEX, - RL_SAC_ALPHA_INDEX, RL_SAC_ENTROPY_TARGET_INDEX, - RL_ACTION_ENTROPY_EMA_INDEX, - // Risk-management stack (spec 2026-05-30-adaptive-risk-management-design). - // Per `feedback_wire_everything_up`: the kernels write these slots; - // diag has to surface them or the run is half-blind. - RL_SESSION_PNL_USD_INDEX, RL_SESSION_PNL_WORST_INDEX, - RL_SESSION_DD_LIMIT_USD_INDEX, RL_SESSION_DD_TRIGGERED_INDEX, - RL_CONSEC_LOSS_LIMIT_INDEX, RL_CONSEC_LOSS_COUNT_INDEX, - RL_COOLDOWN_REMAINING_STEPS_INDEX, RL_COOLDOWN_DURATION_INDEX, - RL_MAX_OPEN_UNITS_INDEX, RL_NET_INVENTORY_LIMIT_USD_INDEX, - RL_IQN_ACTION_TAU_INDEX, RL_IQN_ACTION_TAU_MIN_INDEX, - RL_IQN_ACTION_TAU_DD_SENSITIVITY_INDEX, - RL_INVENTORY_PENALTY_BETA_INDEX, RL_INVENTORY_VARIANCE_EMA_INDEX, - RL_KELLY_FRACTION_INDEX, RL_WIN_RATE_EMA_INDEX, - RL_AVG_WIN_USD_EMA_INDEX, RL_AVG_LOSS_USD_EMA_INDEX, - RL_KELLY_SAFETY_FRAC_INDEX, RL_KELLY_MIN_TRADES_FOR_RELEASE_INDEX, - RL_TRAIL_TIGHTEN_FACTOR_INDEX, RL_TRAIL_LOOSEN_FACTOR_INDEX, - RL_CUMULATIVE_DONES_INDEX, - // v9 defensive eval-boundary calibration (spec - // 2026-05-31-v9-defensive-eval-boundary-calibration.md). - RL_REGIME_TRANSITION_REMAINING_INDEX, RL_REGIME_TRANSITION_STEPS_CONFIG_INDEX, - RL_REGIME_TRANSITION_DECAY_STEPS_CONFIG_INDEX, - RL_EVAL_KELLY_SAFETY_DEFENSIVE_INDEX, RL_EVAL_IQN_TAU_MIN_DEFENSIVE_INDEX, - RL_EVAL_ENTROPY_COEF_MIN_DEFENSIVE_INDEX, RL_EVAL_PPO_CLIP_EPS_MIN_DEFENSIVE_INDEX, - RL_EVAL_KELLY_SAFETY_NORMAL_INDEX, RL_EVAL_IQN_TAU_MIN_NORMAL_INDEX, - RL_EVAL_ENTROPY_COEF_MIN_NORMAL_INDEX, RL_EVAL_PPO_CLIP_EPS_MIN_NORMAL_INDEX, - RL_ENTROPY_COEF_MIN_INDEX, RL_PPO_CLIP_EPS_MIN_INDEX, - // Regime-observer slots (F1.8). - RL_REGIME_DEAD_ZONE_FLAG_INDEX, RL_REGIME_DEAD_ZONE_DURATION_INDEX, - RL_REGIME_DEAD_ZONE_TIMEOUT_FLAG_INDEX, RL_REGIME_RECOVERY_FACTOR_INDEX, - RL_REGIME_SESSION_PNL_VARIANCE_EMA_INDEX, RL_REGIME_TAIL_EVENT_RECENCY_INDEX, - RL_REGIME_SESSION_PNL_VAR_COUNT_INDEX, - RL_REGIME_DEAD_ZONE_MAX_DURATION_INDEX, RL_REGIME_TAIL_SIGMA_THRESHOLD_INDEX, - RL_KELLY_EPS_RECOVERY_LIVE_INDEX, RL_KELLY_EPS_RECOVERY_MIN_INDEX, - RL_KELLY_EPS_RECOVERY_MAX_INDEX, RL_KELLY_EPS_RECOVERY_N_RECOVERY_INDEX, - RL_IQN_TAU_TAIL_BOOST_FACTOR_INDEX, RL_IQN_TAU_TAIL_BOOST_N_WINDOW_INDEX, - RL_POPART_MAX_ABS_REWARD_EMA_INDEX, RL_POPART_MAX_DECAY_ALPHA_INDEX, + RL_GAMMA_INDEX, RL_PER_ALPHA_INDEX, RL_PPO_CLIP_INDEX, RL_REWARD_SCALE_INDEX, }; use ml_alpha::trainer::diag_staging::DiagStaging; use ml_alpha::trainer::integrated::{ - IntegratedStepStats, IntegratedTrainer, IntegratedTrainerConfig, + DiagInputs, IntegratedStepStats, IntegratedTrainer, IntegratedTrainerConfig, }; use ml_alpha::trainer::perception::PerceptionTrainerConfig; use ml_backtesting::sim::LobSimCuda; use ml_core::device::MlDevice; use serde::Serialize; -use serde_json::json; use std::io::{BufWriter, Write}; use std::path::PathBuf; use std::process; @@ -622,27 +550,22 @@ fn main() -> Result<()> { let unit_entry_step_host = diag_staging.read_unit_entry_step(); let unit_lots_host = diag_staging.read_unit_lots(); let unit_trail_host = diag_staging.read_unit_trail(); - // Derive active mask from unit_lots (lots != 0 → active). - // Avoids a u8-to-f32 type mismatch in DtoD staging. - let unit_active_host: Vec = unit_lots_host - .iter() - .map(|&l| if l != 0 { 1u8 } else { 0u8 }) - .collect(); let close_unit_index_host = diag_staging.read_close_unit_index(); + // Per-step action histogram (counts) — still computed here because + // the windowed EMA and several per-step counters (tighten/loosen/ + // half-flat) read directly from it. `build_diag_value` recomputes + // its own act_hist internally — the two are bit-equivalent. let mut act_hist = [0u32; N_ACTIONS]; for &a in &actions_host { if (a as usize) < N_ACTIONS { act_hist[a as usize] += 1; } } - // audit — action entropy H(action_dist) windowed. - // - // Per-batch act_hist is one-hot at b_size=1 so per-step entropy - // is structurally 0. Maintain an EMA-smoothed action histogram - // across recent steps (α=1/1000, half-life ≈ 690 steps) and - // compute entropy on the normalised window. This captures the - // actual policy exploration variety over the recent run. + // EMA-smoothed windowed action distribution (cross-step state). + // Mutated here so the next step inherits the updated EMA; + // build_diag_value reads the post-update value via `windowed_act_hist` + // in DiagInputs. let total_actions: u32 = act_hist.iter().sum(); if total_actions > 0 { for i in 0..N_ACTIONS { @@ -652,28 +575,14 @@ fn main() -> Result<()> { + WINDOWED_ACT_ALPHA * p_step; } } - let win_sum: f32 = windowed_act_hist.iter().sum(); - let action_entropy: f32 = if win_sum > 1e-9 { - windowed_act_hist - .iter() - .filter(|&&p| p > 1e-9) - .map(|&p_raw| { - let p = p_raw / win_sum; - -p * p.ln() - }) - .sum() - } else { - 0.0 - }; let reward_sum: f32 = rewards_host.iter().sum(); - let reward_max = rewards_host.iter().cloned().fold(f32::NEG_INFINITY, f32::max); - let reward_min = rewards_host.iter().cloned().fold(f32::INFINITY, f32::min); - let reward_abs_max = rewards_host.iter().map(|r| r.abs()).fold(0.0f32, f32::max); let done_count: u32 = dones_host.iter().map(|&d| if d > 0.5 { 1 } else { 0 }).sum(); - // Per-trade cumulative stats (surfer validation). - // Use post-scale reward / current_scale to recover approximate USD. - let current_scale = trainer.read_isv_host(ml_alpha::rl::isv_slots::RL_REWARD_SCALE_INDEX); + // Per-trade cumulative stats (surfer validation) — mutate the + // train-loop running counters (pnl_cum_usd / total_trades / + // win_count / hold_time_sum). build_diag_value receives the + // post-update values via DiagInputs. + let current_scale = trainer.read_isv_host(RL_REWARD_SCALE_INDEX); for b in 0..cli.n_backtests { if dones_host[b] > 0.5 { let pnl_usd = if current_scale > 1e-9 { @@ -688,7 +597,6 @@ fn main() -> Result<()> { } } let win_rate = if total_trades > 0 { win_count as f64 / total_trades as f64 } else { 0.0 }; - let avg_hold = if total_trades > 0 { hold_time_sum / total_trades as f64 } else { 0.0 }; let isv = trainer.isv_host_slice(); @@ -715,580 +623,52 @@ fn main() -> Result<()> { frd_gate_total += frd_gate_step; heat_cap_total += heat_cap_step; - // Build per-batch per-unit arrays for diag. - let units_diag = { - let b = cli.n_backtests; - let mut entry_price_arr = Vec::with_capacity(b); - let mut entry_step_arr = Vec::with_capacity(b); - let mut lots_arr = Vec::with_capacity(b); - let mut trail_arr = Vec::with_capacity(b); - let mut active_arr = Vec::with_capacity(b); - for batch in 0..b { - let off = batch * 4; - entry_price_arr.push(&unit_entry_price_host[off..off + 4]); - entry_step_arr.push(&unit_entry_step_host[off..off + 4]); - lots_arr.push(&unit_lots_host[off..off + 4]); - trail_arr.push(&unit_trail_host[off..off + 4]); - active_arr.push(&unit_active_host[off..off + 4]); - } - json!({ - "entry_price": entry_price_arr, - "entry_step": entry_step_arr, - "lots": lots_arr, - "trail_distance": trail_arr, - "active_mask": active_arr, - "unit_count": pyramid_count_host, - }) + // Build the per-step diag record via the single-source-of-truth + // builder on the trainer (Phase A of the 2026-05-31 + // checkpoints+eval-diag plan). The 642-leaf json!{} block that + // used to live inline here now lives in + // `IntegratedTrainer::build_diag_value`, which is also invoked + // from the eval phase below — guaranteeing schema parity per + // `feedback_single_source_of_truth_no_duplicates`. + let frd_logits_host = diag_staging.read_frd_logits(); + let diag_inputs = DiagInputs { + b_size: cli.n_backtests, + stats: &stats, + rewards_host, + dones_host, + actions_host: &actions_host, + raw_rewards_host, + trade_duration_host, + outcome_ema_host, + position_lots_host: &position_lots_host, + pyramid_count_host: &pyramid_count_host, + unit_entry_price_host, + unit_entry_step_host: &unit_entry_step_host, + unit_lots_host: &unit_lots_host, + unit_trail_host, + close_unit_index_host: &close_unit_index_host, + frd_logits_host, + pnl_cum_usd, + total_trades, + win_count, + hold_time_sum, + trail_fired_total, + trail_tighten_total, + trail_loosen_total, + pyramid_added_total, + partial_flat_total, + partial_flat_long_total, + partial_flat_short_total, + conf_gate_total, + frd_gate_total, + heat_cap_total, + windowed_act_hist: &windowed_act_hist, }; - - // FRD diag — per-horizon softmax entropy + argmax index, - // averaged across the batch. Reads from DiagStaging (async, - // zero GPU stalls). Computes softmax host-side (small: B x 63). - let frd_diag = { - let frd_logits = diag_staging.read_frd_logits(); - let mut per_h_entropy = [0.0_f32; FRD_N_HORIZONS]; - let mut per_h_argmax_sum = [0.0_f32; FRD_N_HORIZONS]; - for b in 0..cli.n_backtests { - for h in 0..FRD_N_HORIZONS { - let off = b * FRD_OUT_DIM + h * FRD_N_ATOMS; - let row = &frd_logits[off..off + FRD_N_ATOMS]; - let max_l = row.iter().cloned().fold(f32::NEG_INFINITY, f32::max); - let denom: f32 = row.iter().map(|x| (x - max_l).exp()).sum(); - let mut entropy = 0.0_f32; - let mut argmax_idx = 0; - let mut argmax_v = f32::NEG_INFINITY; - for (a, x) in row.iter().enumerate() { - let p = (x - max_l).exp() / denom; - if p > 1e-9 { - entropy -= p * p.ln(); - } - if *x > argmax_v { - argmax_v = *x; - argmax_idx = a; - } - } - per_h_entropy[h] += entropy; - per_h_argmax_sum[h] += argmax_idx as f32; - } - } - let b_f = cli.n_backtests.max(1) as f32; - json!({ - "h1": { "entropy_mean": per_h_entropy[0] / b_f, "argmax_mean": per_h_argmax_sum[0] / b_f }, - "h2": { "entropy_mean": per_h_entropy[1] / b_f, "argmax_mean": per_h_argmax_sum[1] / b_f }, - "h3": { "entropy_mean": per_h_entropy[2] / b_f, "argmax_mean": per_h_argmax_sum[2] / b_f }, - }) - }; - - // v9 eval_warmup pre-compute (json! macro can't take Rust blocks/if-else). - let eval_warmup_remaining = isv[RL_REGIME_TRANSITION_REMAINING_INDEX]; - let eval_warmup_decay_steps = isv[RL_REGIME_TRANSITION_DECAY_STEPS_CONFIG_INDEX]; - let eval_warmup_active = if eval_warmup_remaining > 0.0 { 1.0_f32 } else { 0.0_f32 }; - let eval_warmup_blend = if eval_warmup_remaining < 0.0 { - 0.0_f32 - } else if eval_warmup_remaining > eval_warmup_decay_steps { - 1.0_f32 - } else if eval_warmup_decay_steps > 0.0 { - eval_warmup_remaining / eval_warmup_decay_steps - } else if eval_warmup_remaining > 0.0 { - 1.0_f32 - } else { - 0.0_f32 - }; - - let record = json!({ - "step": step, - "elapsed_s": t_start.elapsed().as_secs_f32(), - "loss": { - "bce": stats.l_bce, - "q": stats.l_q, - "pi": stats.l_pi, - "v": stats.l_v, - "aux": stats.l_aux, - "frd": stats.l_frd, - "total": stats.l_total, - }, - "lambdas": { - "bce": stats.lambdas.bce, - "q": stats.lambdas.q, - "pi": stats.lambdas.pi, - "v": stats.lambdas.v, - "aux": stats.lambdas.aux, - "frd": stats.lambdas.frd, - }, - // 7 R5 controller outputs (the adaptive knobs). - "isv_out": { - "gamma": isv[RL_GAMMA_INDEX], - "target_tau": isv[RL_TARGET_TAU_INDEX], - "ppo_clip_eps": isv[RL_PPO_CLIP_INDEX], - "entropy_coef": isv[RL_ENTROPY_COEF_INDEX], - "n_rollout_steps": isv[RL_N_ROLLOUT_STEPS_INDEX], - "per_alpha": isv[RL_PER_ALPHA_INDEX], - "reward_scale": isv[RL_REWARD_SCALE_INDEX], - }, - // 5 per-head learning rates (rl_lr_controller emits). - "isv_lr": { - "bce": isv[RL_LR_BCE_INDEX], - "q": isv[RL_LR_Q_INDEX], - "pi": isv[RL_LR_PI_INDEX], - "v": isv[RL_LR_V_INDEX], - "aux": isv[RL_LR_AUX_INDEX], - }, - // 7 EMA inputs the controllers consume — knowing these - // makes controller behaviour debuggable (was target wrong? - // was input wrong? was alpha wrong?). - "isv_ema_in": { - "mean_trade_duration": isv[RL_MEAN_TRADE_DURATION_EMA_INDEX], - "q_divergence": isv[RL_Q_DIVERGENCE_EMA_INDEX], - "kl_pi": isv[RL_KL_PI_EMA_INDEX], - "entropy_observed": isv[RL_ENTROPY_OBSERVED_EMA_INDEX], - "advantage_var_ratio": isv[RL_ADVANTAGE_VAR_RATIO_EMA_INDEX], - "td_kurtosis": isv[RL_TD_KURTOSIS_EMA_INDEX], - "mean_abs_pnl": isv[RL_MEAN_ABS_PNL_EMA_INDEX], - }, - // Grad-norm EMAs (diagnostic — no longer drive the LR - // controller as of commit 042de99e6 but still useful to - // correlate with plateau-decay events). - "grad_norm_ema": { - "q": isv[RL_Q_GRAD_NORM_EMA_INDEX], - "pi": isv[RL_PI_GRAD_NORM_EMA_INDEX], - "v": isv[RL_V_GRAD_NORM_EMA_INDEX], - }, - // Plateau-decay LR controller state (per head). loss_ema is - // the controller's internal slow EMA at α=0.05; best is the - // lowest loss_ema observed so far; steps_since_best is the - // staleness counter that fires LR halving at PLATEAU_PATIENCE - // (1000). Confirms whether the controller is seeing plateaus - // and decaying for the right reason. - "lr_plateau": { - "q": { "loss_ema": isv[RL_LR_Q_LOSS_EMA_INDEX], - "best": isv[RL_LR_Q_BEST_LOSS_INDEX], - "stale": isv[RL_LR_Q_STEPS_SINCE_BEST_INDEX], - "warmup": isv[RL_LR_Q_WARMUP_COUNTER_INDEX] }, - "pi": { "loss_ema": isv[RL_LR_PI_LOSS_EMA_INDEX], - "best": isv[RL_LR_PI_BEST_LOSS_INDEX], - "stale": isv[RL_LR_PI_STEPS_SINCE_BEST_INDEX], - "warmup": isv[RL_LR_PI_WARMUP_COUNTER_INDEX] }, - "v": { "loss_ema": isv[RL_LR_V_LOSS_EMA_INDEX], - "best": isv[RL_LR_V_BEST_LOSS_INDEX], - "stale": isv[RL_LR_V_STEPS_SINCE_BEST_INDEX], - "warmup": isv[RL_LR_V_WARMUP_COUNTER_INDEX] }, - }, - // Adaptive risk-management stack state — five orthogonal layers. - // Every slot here is written by a kernel; diag surfaces them so - // controller behaviour, gate firing, and bootstrap state are - // observable without re-reading device memory. - "risk_stack": { - // Layer 1 — CMDP hard constraints. - // session_pnl_usd is MEAN-of-active-accounts (the IQN-τ - // consumer); session_pnl_worst is the most-negative - // per-batch pnl (diag — spot fleet skew). - "cmdp": { - "session_pnl_usd_mean": isv[RL_SESSION_PNL_USD_INDEX], - "session_pnl_worst": isv[RL_SESSION_PNL_WORST_INDEX], - "session_dd_limit_usd": isv[RL_SESSION_DD_LIMIT_USD_INDEX], - "session_dd_triggered": isv[RL_SESSION_DD_TRIGGERED_INDEX], - "consec_loss_count": isv[RL_CONSEC_LOSS_COUNT_INDEX], - "consec_loss_limit": isv[RL_CONSEC_LOSS_LIMIT_INDEX], - "cooldown_remaining_steps": isv[RL_COOLDOWN_REMAINING_STEPS_INDEX], - "cooldown_duration": isv[RL_COOLDOWN_DURATION_INDEX], - "max_open_units": isv[RL_MAX_OPEN_UNITS_INDEX], - "net_inventory_limit_usd": isv[RL_NET_INVENTORY_LIMIT_USD_INDEX], - }, - // Layer 2 — IQN risk-averse action τ. - "iqn_tau": { - "action_tau": isv[RL_IQN_ACTION_TAU_INDEX], - "tau_min": isv[RL_IQN_ACTION_TAU_MIN_INDEX], - "dd_sensitivity": isv[RL_IQN_ACTION_TAU_DD_SENSITIVITY_INDEX], - }, - // Layer 3 — Avellaneda-Stoikov inventory penalty β. - "inventory": { - "penalty_beta": isv[RL_INVENTORY_PENALTY_BETA_INDEX], - "variance_ema": isv[RL_INVENTORY_VARIANCE_EMA_INDEX], - }, - // Layer 4 — Half-Kelly position sizing + its EMA inputs. - "kelly": { - "fraction": isv[RL_KELLY_FRACTION_INDEX], - "win_rate_ema": isv[RL_WIN_RATE_EMA_INDEX], - "avg_win_usd_ema": isv[RL_AVG_WIN_USD_EMA_INDEX], - "avg_loss_usd_ema": isv[RL_AVG_LOSS_USD_EMA_INDEX], - "safety_frac": isv[RL_KELLY_SAFETY_FRAC_INDEX], - "min_trades_for_release": isv[RL_KELLY_MIN_TRADES_FOR_RELEASE_INDEX], - "cumulative_dones": isv[RL_CUMULATIVE_DONES_INDEX], - }, - // Layer D — trail tighten/loosen factors (kernel-side - // counters are already in `trail.{tightened,loosened}_count_*`). - "trail_factors": { - "tighten": isv[RL_TRAIL_TIGHTEN_FACTOR_INDEX], - "loosen": isv[RL_TRAIL_LOOSEN_FACTOR_INDEX], - }, - // v9 — defensive eval-boundary calibration ([[pearl_adaptive_carryover_discipline]]). - // `remaining` is the per-step counter: ≥ decay_steps → full - // defensive blend=1.0; (0, decay_steps] → linear decay to 0; - // 0 → final boundary write of normal values; -1 → kernel - // no-op (post-warmup steady state). The four `floor_*` - // entries are the LIVE values (read AFTER the warmup - // kernel ran this step, so they reflect any override). - "eval_warmup": { - "remaining": eval_warmup_remaining, - "active": eval_warmup_active, - "blend": eval_warmup_blend, - "warmup_steps_config": isv[RL_REGIME_TRANSITION_STEPS_CONFIG_INDEX], - "decay_steps_config": eval_warmup_decay_steps, - "floor_kelly_safety": isv[RL_KELLY_SAFETY_FRAC_INDEX], - "floor_iqn_tau_min": isv[RL_IQN_ACTION_TAU_MIN_INDEX], - "floor_entropy_min": isv[RL_ENTROPY_COEF_MIN_INDEX], - "floor_ppo_eps_min": isv[RL_PPO_CLIP_EPS_MIN_INDEX], - "target_defensive": { - "kelly_safety": isv[RL_EVAL_KELLY_SAFETY_DEFENSIVE_INDEX], - "iqn_tau_min": isv[RL_EVAL_IQN_TAU_MIN_DEFENSIVE_INDEX], - "entropy_min": isv[RL_EVAL_ENTROPY_COEF_MIN_DEFENSIVE_INDEX], - "ppo_eps_min": isv[RL_EVAL_PPO_CLIP_EPS_MIN_DEFENSIVE_INDEX], - }, - "target_normal": { - "kelly_safety": isv[RL_EVAL_KELLY_SAFETY_NORMAL_INDEX], - "iqn_tau_min": isv[RL_EVAL_IQN_TAU_MIN_NORMAL_INDEX], - "entropy_min": isv[RL_EVAL_ENTROPY_COEF_MIN_NORMAL_INDEX], - "ppo_eps_min": isv[RL_EVAL_PPO_CLIP_EPS_MIN_NORMAL_INDEX], - }, - }, - // Regime observer — dead-zone, tail events, Kelly ε-recovery, - // PopArt envelope, IQN τ tail-boost. All values are direct - // ISV reads (drift-free per spec issue #9). - "regime": { - "dead_zone": { - "flag": isv[RL_REGIME_DEAD_ZONE_FLAG_INDEX], - "duration": isv[RL_REGIME_DEAD_ZONE_DURATION_INDEX], - "timeout_flag": isv[RL_REGIME_DEAD_ZONE_TIMEOUT_FLAG_INDEX], - "max_duration": isv[RL_REGIME_DEAD_ZONE_MAX_DURATION_INDEX], - }, - "tail": { - "recency": isv[RL_REGIME_TAIL_EVENT_RECENCY_INDEX], - "session_pnl_variance_ema": isv[RL_REGIME_SESSION_PNL_VARIANCE_EMA_INDEX], - "sigma_threshold": isv[RL_REGIME_TAIL_SIGMA_THRESHOLD_INDEX], - "welford_count": isv[RL_REGIME_SESSION_PNL_VAR_COUNT_INDEX], - }, - "kelly_eps_recovery": { - "factor": isv[RL_REGIME_RECOVERY_FACTOR_INDEX], - "live": isv[RL_KELLY_EPS_RECOVERY_LIVE_INDEX], - "min": isv[RL_KELLY_EPS_RECOVERY_MIN_INDEX], - "max": isv[RL_KELLY_EPS_RECOVERY_MAX_INDEX], - "n_recovery": isv[RL_KELLY_EPS_RECOVERY_N_RECOVERY_INDEX], - }, - "popart_envelope": { - "max_abs_reward_ema": isv[RL_POPART_MAX_ABS_REWARD_EMA_INDEX], - "decay_alpha": isv[RL_POPART_MAX_DECAY_ALPHA_INDEX], - }, - "iqn_tau_boost": { - "factor": isv[RL_IQN_TAU_TAIL_BOOST_FACTOR_INDEX], - "n_window": isv[RL_IQN_TAU_TAIL_BOOST_N_WINDOW_INDEX], - }, - }, - // Calibration diagnostics — atom-span vs. Bellman dynamic - // bound. Per docs/superpowers/specs/2026-05-31-c51-atom- - // span-math-validation.md: the structural constraint is - // `atom_max ≥ WIN + γ × atom_max` (the Bellman target - // upper bound). When `atom_max_headroom > 0`, system is - // self-consistent (no Q clipping); when `< 0`, V_target - // saturates top atom → Q distillation degrades (the v7 - // Fix F failure mode at step 500). - // - // `atom_max_over_3sigma` measures resolution waste: when - // atom_max ≫ 3·popart_σ, atoms are over-sized for actual - // V_target distribution (~99.7% within ±3σ). v8 local - // smoke step 999 had this ratio at 16.8× — i.e., we use - // 1/17 of available atom resolution. - "atom_calibration": { - "win_bound": isv[RL_REWARD_CLAMP_WIN_INDEX], - "atom_max": isv[RL_C51_V_MAX_INDEX], - "gamma": isv[RL_GAMMA_INDEX], - "dynamic_bound": isv[RL_REWARD_CLAMP_WIN_INDEX] - + isv[RL_GAMMA_INDEX] - * isv[RL_C51_V_MAX_INDEX], - "atom_max_headroom": isv[RL_C51_V_MAX_INDEX] - - (isv[RL_REWARD_CLAMP_WIN_INDEX] - + isv[RL_GAMMA_INDEX] - * isv[RL_C51_V_MAX_INDEX]), - "popart_sigma": isv[RL_POPART_SIGMA_INDEX], - "v_target_max_3sigma": 3.0_f32 * isv[RL_POPART_SIGMA_INDEX], - "atom_max_over_3sigma": isv[RL_C51_V_MAX_INDEX] - / (3.0_f32 * isv[RL_POPART_SIGMA_INDEX] + 1e-9_f32), - }, - }, - "replay_len": trainer.gpu_replay.capacity.min(step + 1), - // Post-scale, POST-clamp reward stats — what V regression - // and Q distributional projection actually saw this step. - // The `scaled_pre_clamp_max` field (below) is the SAME - // batch's max BEFORE the [-3, +1] clamp fired. If - // scaled_pre_clamp_max > 1.0 (clamp threshold on the win - // side) repeatedly, the clamp is doing load-bearing work - // — i.e. the reward_scale controller is failing to keep - // typical trade magnitudes within the C51 atom support. - "rewards": { - "sum": reward_sum, - "max": reward_max, - "min": reward_min, - "abs_max": reward_abs_max, - "scaled_pre_clamp_max": - isv[RL_MAX_ABS_SCALED_REWARD_PRE_CLAMP_INDEX], - // Adaptive clamp signals — surface the controller's - // POINT input + EMA + emitted bounds so post-hoc - // analysis can verify the adaptation chain is moving. - "pos_scaled_max": - isv[RL_POS_SCALED_REWARD_MAX_INDEX], - "pos_scaled_max_ema": - isv[RL_POS_SCALED_REWARD_MAX_EMA_INDEX], - // MARGIN controller state — clip rate vs target tells - // us whether the controller is happy (rate ≈ target) - // or fighting the tail (rate > target × tolerance). - "clip_rate_ema": - isv[RL_REWARD_CLAMP_CLIP_RATE_EMA_INDEX], - // C51 atom span — ratcheted from WIN/LOSS clamp by the - // same controller. Watching these expand confirms Q's - // distributional learning ceiling is being lifted. - "c51_v_max": - isv[RL_C51_V_MAX_INDEX], - "c51_v_min": - isv[RL_C51_V_MIN_INDEX], - // Q→π distillation KL signal — drops toward 0 if π is - // successfully chasing Q's preferences. - "q_distill_kl_ema": - isv[RL_Q_DISTILL_KL_EMA_INDEX], - // Negative-tail tracking — input to adaptive RATIO. - "neg_scaled_max": - isv[RL_NEG_SCALED_REWARD_MAX_INDEX], - "neg_scaled_max_ema": - isv[RL_NEG_SCALED_REWARD_MAX_EMA_INDEX], - }, - // audit — PPO importance-ratio clamp diagnostics. - // - // ratio_clamp_max : the adaptive ceiling at ISV[440], - // emitted by rl_ppo_ratio_clamp_controller - // from (1+ε) × PPO_CLAMP_MARGIN. - // log_ratio_abs_max : per-step max(|log π_new − log π_old|) - // over the batch at ISV[441], written - // by ppo_log_ratio_abs_max_b. - // - // The clamp fires when log_ratio_abs_max > ln(ratio_clamp_max). - // For ratio_clamp_max = 10, ln = 2.30. Healthy training has - // log_ratio_abs_max well below this most steps; outliers - // touch or exceed it on rare excursions which the clamp - // bounds before they pollute l_pi. - "ppo": { - "ratio_clamp_max": isv[RL_PPO_RATIO_CLAMP_MAX_INDEX], - "log_ratio_abs_max": isv[RL_PPO_LOG_RATIO_ABS_MAX_INDEX], - }, - // audit — streaming-kernel state for the variance-ratio + - // kurtosis estimators that feed rollout_steps and per_α. - // * `*_stream.{mean, m2, m4}` are the Welford-EMA - // internal state slots (442-446) — confirms the - // kernel is folding observations across STEPS. - // * `*_clamp` is the ISV-resident output ceiling - // (447, 448) seeded once at trainer init by - // rl_streaming_clamp_init — visible here so the diag - // can spot when the clamp is firing - // (post-clamp EMA at the consumer-input slot equals - // `clamp` ↔ clamp active). - "streaming": { - "adv_var": { - "mean": isv[RL_ADV_VAR_STREAM_MEAN_INDEX], - "m2": isv[RL_ADV_VAR_STREAM_M2_INDEX], - "clamp": isv[RL_ADV_VAR_RATIO_CLAMP_INDEX], - }, - "td_kurt": { - "mean": isv[RL_TD_KURT_STREAM_MEAN_INDEX], - "m2": isv[RL_TD_KURT_STREAM_M2_INDEX], - "m4": isv[RL_TD_KURT_STREAM_M4_INDEX], - "clamp": isv[RL_TD_KURTOSIS_CLAMP_INDEX], - }, - }, - // audit — controller decision diagnostics (which adaptation - // branch the multiplicative controllers fired this step). - // - // ratio = input_ema / TARGET (per controller's design) - // branch = "WIDEN" if ratio > TOLERANCE = 1.5 - // "HOLD" if ratio ∈ [1/1.5, 1.5] - // "SHRINK" if ratio < 1/1.5 - // "NOISE" if input_ema is below the noise floor - // - // For rl_rollout_steps_controller (TARGET = 0.1) and - // rl_ppo_clip_controller (TARGET = 0.01), this reveals - // whether the controller is being driven by real divergence - // signal or sitting in the in-band hold zone. Use to spot - // controllers stuck saturating at MAX/MIN (which would - // show 100% WIDEN or 100% SHRINK). - // - // Targets and tolerances are reflected from the kernel - // #defines (kept synchronised by code review at the - // controller-cu file level — there's no ISV slot for these - // design constants because they're fundamental to the - // controller's behaviour, not adaptive). - // audit — K-loop training-intensification iterations performed - // this step. K = clamp(isv[404] / isv[450], 1, isv[451]). - // Both divisor + max are ISV-driven (per - // feedback_isv_for_adaptive_bounds) — surfaced here so - // post-hoc analysis can compute K from any sample without - // recompile. Defaults: divisor=2048 (K=1 at controller - // bootstrap), max=4 (prevents gradient overtraining). - "k_loop": { - "k_updates": trainer.last_k_updates, - "divisor": isv[RL_K_LOOP_DIVISOR_INDEX], - "max": isv[RL_K_LOOP_MAX_INDEX], - }, - // audit — ISV-driven design constants (was kernel `#define`s). - // All tunable at runtime via re-launching rl_isv_write per - // `feedback_isv_for_adaptive_bounds`. Surfaced here so - // post-hoc analysis sees exactly what each controller was - // regressing against / clamping at, every step. - "isv_config": { - "reward_clamp_win": isv[RL_REWARD_CLAMP_WIN_INDEX], - "reward_clamp_loss": isv[RL_REWARD_CLAMP_LOSS_INDEX], - "kl_target": isv[RL_KL_TARGET_INDEX], - "improvement_threshold": isv[RL_IMPROVEMENT_THRESHOLD_INDEX], - "plateau_patience": isv[RL_PLATEAU_PATIENCE_INDEX], - "div_target": isv[RL_DIV_TARGET_INDEX], - "entropy_target_frac": isv[RL_ENTROPY_TARGET_FRAC_INDEX], - "kurt_lift_scale": isv[RL_KURT_LIFT_SCALE_INDEX], - "ppo_clamp_margin": isv[RL_PPO_CLAMP_MARGIN_INDEX], - "lr_warmup_steps": isv[RL_LR_WARMUP_STEPS_INDEX], - "lr_bootstrap": isv[RL_LR_BOOTSTRAP_INDEX], - "lr_min": isv[RL_LR_MIN_INDEX], - "lr_max": isv[RL_LR_MAX_INDEX], - "lr_loss_ema_alpha": isv[RL_LR_LOSS_EMA_ALPHA_INDEX], - "lr_decay_factor": isv[RL_LR_DECAY_FACTOR_INDEX], - "loss_lambda_aux": isv[RL_LOSS_LAMBDA_AUX_INDEX], - "schulman_tolerance": isv[RL_SCHULMAN_TOLERANCE_INDEX], - "schulman_adjust_rate": isv[RL_SCHULMAN_ADJUST_RATE_INDEX], - "stream_alpha": isv[RL_STREAM_ALPHA_INDEX], - "kurt_gaussian": isv[RL_KURT_GAUSSIAN_INDEX], - "kurt_noise_floor": isv[RL_KURT_NOISE_FLOOR_INDEX], - "tau_bootstrap": isv[RL_TAU_BOOTSTRAP_INDEX], - "eps_bootstrap": isv[RL_EPS_BOOTSTRAP_INDEX], - "rollout_bootstrap": isv[RL_ROLLOUT_BOOTSTRAP_INDEX], - "reward_scale_bootstrap":isv[RL_REWARD_SCALE_BOOTSTRAP_INDEX], - "ppo_ratio_clamp_bootstrap": isv[RL_PPO_RATIO_CLAMP_BOOTSTRAP_INDEX], - "reward_clamp_margin": isv[RL_REWARD_CLAMP_MARGIN_INDEX], - "reward_clamp_ratio": isv[RL_REWARD_CLAMP_RATIO_INDEX], - "reward_clamp_clip_rate_target": isv[RL_REWARD_CLAMP_CLIP_RATE_TARGET_INDEX], - "q_distill_lambda": isv[RL_Q_DISTILL_LAMBDA_INDEX], - "q_distill_temperature": isv[RL_Q_DISTILL_TEMPERATURE_INDEX], - "q_distill_kl_target": isv[RL_Q_DISTILL_KL_TARGET_INDEX], - "sac_alpha": isv[RL_SAC_ALPHA_INDEX], - "sac_entropy_target": isv[RL_SAC_ENTROPY_TARGET_INDEX], - "action_entropy_ema": isv[RL_ACTION_ENTROPY_EMA_INDEX], - "reward_scale_min": isv[RL_REWARD_SCALE_MIN_INDEX], - }, - // audit — Q vs π action agreement EMA at slot 407 - // (previously dead). 1.0 = perfect ranking consistency, - // 0.0 = total disagreement. Alarm if persistently < 0.5. - "q_pi_agree_ema": isv[RL_Q_ARG_VS_PI_AGREE_INDEX], - "controller_branch": { - "rollout_steps_input": isv[RL_ADVANTAGE_VAR_RATIO_EMA_INDEX], - // ISV-driven target — seeded by rl_streaming_clamp_init, - // visible here (no longer a hardcoded `#define`). - "rollout_steps_target": isv[RL_ADV_VAR_RATIO_TARGET_INDEX], - "ppo_clip_input": isv[RL_KL_PI_EMA_INDEX], - "ppo_clip_target": 0.01f32, - "target_tau_input": isv[RL_Q_DIVERGENCE_EMA_INDEX], - "target_tau_target": 0.01f32, - "per_alpha_input": isv[RL_TD_KURTOSIS_EMA_INDEX], - "per_alpha_target": 0.6f32, // kurt=10 fixed point - }, - "done_count": done_count, - // Action histogram: indices match `Action` enum - // (0=ShortLarge, 1=ShortSmall, 2=Hold, 3=FlatFromLong, - // 4=FlatFromShort, 5=LongSmall, 6=LongLarge, - // 7=TrailTighten, 8=TrailLoosen). - "action_hist": act_hist.to_vec(), - // audit additions: - // * action_entropy — H(action_dist) computed from - // act_hist; uniform-9 = ln(9) ≈ 2.197. Lower = policy - // collapsed to few actions (exploration failed). - // * position.lots — signed lot count per batch (long/short - // /flat). Reveals whether the agent is actually moving - // positions or stuck. - "action_entropy": action_entropy, - "position": { - "lots": position_lots_host, - }, - "units": units_diag, - "trail": { - "fired_count_step": trail_fired_step, - "fired_count_total": trail_fired_total, - "tightened_count_step": tighten_step, - "loosened_count_step": loosen_step, - "tightened_count_total": trail_tighten_total, - "loosened_count_total": trail_loosen_total, - }, - "pyramid": { - "added_count_step": pyramid_add_step, - "added_count_total": pyramid_added_total, - "units_distribution": pyramid_count_host, - "max_units_reached": pyramid_count_host.iter().any(|&c| c >= 4), - }, - "partial_flat": { - "fired_count_step": partial_flat_step, - "fired_count_total": partial_flat_total, - "long_count_total": partial_flat_long_total, - "short_count_total": partial_flat_short_total, - "close_unit_index": close_unit_index_host, - }, - "confidence_gate": { - "gated_count_step": conf_gate_step, - "gated_count_total": conf_gate_total, - }, - "position_heat": { - "capped_count_step": heat_cap_step, - "capped_count_total": heat_cap_total, - "heat_max_lots": isv[ml_alpha::rl::isv_slots::RL_HEAT_CAP_MAX_LOTS_INDEX], - }, - "anti_martingale": { - "outcome_ema": outcome_ema_host, - "kappa": isv[ml_alpha::rl::isv_slots::RL_ANTIMARTINGALE_KAPPA_INDEX], - }, - "frd_gate": { - "gated_count_step": frd_gate_step, - "gated_count_total": frd_gate_total, - }, - "trading": { - "pnl_cum_usd": pnl_cum_usd, - "total_trades": total_trades, - "win_rate": win_rate, - "avg_hold_steps": avg_hold, - "raw_reward_sum": raw_rewards_host.iter().sum::(), - }, - // SP20 P3 FRD head diag — per-horizon softmax entropy + argmax-mode - // bucket index (averaged across the batch). At init, Xavier × 0.1 - // weights → logits ≈ 0 → entropy ≈ ln(21) = 3.044 and the argmax - // is dominated by tiny initial-weight noise. Watching these drift - // away from the uniform baseline is the supervised-learning signal - // for the FRD head (CE loss + bwd land in F.3). - "frd": frd_diag, - // P1+P2 feature metrics — PopArt reward normalization, - // spectral norm/decoupling, Q-bias correction, per-branch - // LR scaling, and outcome aux head lambda. - "popart": { - "mean": isv[RL_POPART_MEAN_INDEX], - "sigma": isv[RL_POPART_SIGMA_INDEX], - "var": isv[RL_POPART_VAR_INDEX], - }, - "spectral": { - "norm_max_config": isv[RL_SPECTRAL_NORM_MAX_INDEX], - "decouple_lambda": isv[RL_SPECTRAL_DECOUPLE_LAMBDA_INDEX], - }, - "q_bias": { - "ema": isv[RL_Q_BIAS_EMA_INDEX], - "correction": isv[RL_Q_BIAS_CORRECTION_INDEX], - }, - "per_branch_lr": { - "scale_q": isv[RL_LR_SCALE_Q_INDEX], - "scale_pi": isv[RL_LR_SCALE_PI_INDEX], - "scale_v": isv[RL_LR_SCALE_V_INDEX], - "scale_iqn": isv[RL_LR_SCALE_IQN_INDEX], - }, - "outcome_aux": { - "lambda": isv[RL_OUTCOME_AUX_LAMBDA_INDEX], - }, - }); + let record = trainer.build_diag_value( + step as u64, + t_start.elapsed().as_secs_f32(), + &diag_inputs, + ).context("build_diag_value (train)")?; writeln!(diag, "{}", record).context("diag: writeln jsonl")?; if step % cli.log_every == 0 || step == cli.n_steps - 1 { diff --git a/crates/ml-alpha/src/lib.rs b/crates/ml-alpha/src/lib.rs index 8a22b070a..9ebba2166 100644 --- a/crates/ml-alpha/src/lib.rs +++ b/crates/ml-alpha/src/lib.rs @@ -1,3 +1,9 @@ +// The per-step diag `json!{...}` block built by +// `IntegratedTrainer::build_diag_value` spans > 130 leaf fields, +// exceeding serde_json's default macro recursion budget of 128. +// Bump at the crate root so the trainer module compiles. +#![recursion_limit = "256"] + //! ml-alpha — CfC perception + multi-horizon alpha heads. //! //! Phase A (this crate): snapshot-level CfC trunk + 5 horizon heads, diff --git a/crates/ml-alpha/src/trainer/integrated.rs b/crates/ml-alpha/src/trainer/integrated.rs index 88c646ce2..3eced6892 100644 --- a/crates/ml-alpha/src/trainer/integrated.rs +++ b/crates/ml-alpha/src/trainer/integrated.rs @@ -90,14 +90,96 @@ use crate::heads::HIDDEN_DIM; use crate::pinned_mem::{MappedF32Buffer, MappedI32Buffer}; use rand::{Rng, SeedableRng}; -use crate::rl::common::{N_ACTIONS, Q_N_ATOMS, Q_V_MAX, Q_V_MIN}; +use crate::rl::common::{Q_N_ATOMS, Q_V_MAX, Q_V_MIN}; use crate::rl::dqn::{DqnHead, DqnHeadConfig}; use crate::rl::iqn::EMBED_DIM; use crate::rl::noisy::{NoisyLinear, NoisyLinearConfig}; use crate::rl::isv_slots::{ - RL_LR_BCE_INDEX, RL_LR_AUX_INDEX, RL_LR_PI_INDEX, RL_LR_Q_INDEX, - RL_LR_V_INDEX, RL_SLOTS_END, + RL_ACTION_ENTROPY_EMA_INDEX, RL_ADVANTAGE_VAR_RATIO_EMA_INDEX, + RL_ADV_VAR_RATIO_CLAMP_INDEX, RL_ADV_VAR_RATIO_TARGET_INDEX, + RL_ADV_VAR_STREAM_M2_INDEX, RL_ADV_VAR_STREAM_MEAN_INDEX, + RL_ANTIMARTINGALE_KAPPA_INDEX, + RL_AVG_LOSS_USD_EMA_INDEX, RL_AVG_WIN_USD_EMA_INDEX, + RL_C51_V_MAX_INDEX, RL_C51_V_MIN_INDEX, + RL_CONF_GATE_FIRED_COUNT_INDEX, RL_CONSEC_LOSS_COUNT_INDEX, + RL_CONSEC_LOSS_LIMIT_INDEX, RL_COOLDOWN_DURATION_INDEX, + RL_COOLDOWN_REMAINING_STEPS_INDEX, RL_CUMULATIVE_DONES_INDEX, + RL_DIV_TARGET_INDEX, RL_ENTROPY_COEF_INDEX, RL_ENTROPY_COEF_MIN_INDEX, + RL_ENTROPY_OBSERVED_EMA_INDEX, RL_ENTROPY_TARGET_FRAC_INDEX, + RL_EPS_BOOTSTRAP_INDEX, + RL_EVAL_ENTROPY_COEF_MIN_DEFENSIVE_INDEX, RL_EVAL_ENTROPY_COEF_MIN_NORMAL_INDEX, + RL_EVAL_IQN_TAU_MIN_DEFENSIVE_INDEX, RL_EVAL_IQN_TAU_MIN_NORMAL_INDEX, + RL_EVAL_KELLY_SAFETY_DEFENSIVE_INDEX, RL_EVAL_KELLY_SAFETY_NORMAL_INDEX, + RL_EVAL_PPO_CLIP_EPS_MIN_DEFENSIVE_INDEX, RL_EVAL_PPO_CLIP_EPS_MIN_NORMAL_INDEX, + RL_FRD_GATE_FIRED_COUNT_INDEX, + RL_GAMMA_INDEX, RL_HEAT_CAP_FIRED_COUNT_INDEX, RL_HEAT_CAP_MAX_LOTS_INDEX, + RL_IMPROVEMENT_THRESHOLD_INDEX, + RL_INVENTORY_PENALTY_BETA_INDEX, RL_INVENTORY_VARIANCE_EMA_INDEX, + RL_IQN_ACTION_TAU_DD_SENSITIVITY_INDEX, RL_IQN_ACTION_TAU_INDEX, + RL_IQN_ACTION_TAU_MIN_INDEX, + RL_IQN_TAU_TAIL_BOOST_FACTOR_INDEX, RL_IQN_TAU_TAIL_BOOST_N_WINDOW_INDEX, + RL_KELLY_EPS_RECOVERY_LIVE_INDEX, RL_KELLY_EPS_RECOVERY_MAX_INDEX, + RL_KELLY_EPS_RECOVERY_MIN_INDEX, RL_KELLY_EPS_RECOVERY_N_RECOVERY_INDEX, + RL_KELLY_FRACTION_INDEX, RL_KELLY_MIN_TRADES_FOR_RELEASE_INDEX, + RL_KELLY_SAFETY_FRAC_INDEX, RL_KL_PI_EMA_INDEX, RL_KL_TARGET_INDEX, + RL_KURT_GAUSSIAN_INDEX, RL_KURT_LIFT_SCALE_INDEX, RL_KURT_NOISE_FLOOR_INDEX, + RL_K_LOOP_DIVISOR_INDEX, RL_K_LOOP_MAX_INDEX, + RL_LOSS_LAMBDA_AUX_INDEX, RL_LR_AUX_INDEX, RL_LR_BCE_INDEX, + RL_LR_BOOTSTRAP_INDEX, RL_LR_DECAY_FACTOR_INDEX, + RL_LR_LOSS_EMA_ALPHA_INDEX, RL_LR_MAX_INDEX, RL_LR_MIN_INDEX, + RL_LR_PI_BEST_LOSS_INDEX, RL_LR_PI_INDEX, RL_LR_PI_LOSS_EMA_INDEX, + RL_LR_PI_STEPS_SINCE_BEST_INDEX, RL_LR_PI_WARMUP_COUNTER_INDEX, + RL_LR_Q_BEST_LOSS_INDEX, RL_LR_Q_INDEX, RL_LR_Q_LOSS_EMA_INDEX, + RL_LR_Q_STEPS_SINCE_BEST_INDEX, RL_LR_Q_WARMUP_COUNTER_INDEX, + RL_LR_SCALE_IQN_INDEX, RL_LR_SCALE_PI_INDEX, RL_LR_SCALE_Q_INDEX, + RL_LR_SCALE_V_INDEX, + RL_LR_V_BEST_LOSS_INDEX, RL_LR_V_INDEX, RL_LR_V_LOSS_EMA_INDEX, + RL_LR_V_STEPS_SINCE_BEST_INDEX, RL_LR_V_WARMUP_COUNTER_INDEX, + RL_LR_WARMUP_STEPS_INDEX, + RL_MAX_ABS_SCALED_REWARD_PRE_CLAMP_INDEX, RL_MAX_OPEN_UNITS_INDEX, + RL_MEAN_ABS_PNL_EMA_INDEX, RL_MEAN_TRADE_DURATION_EMA_INDEX, + RL_NEG_SCALED_REWARD_MAX_EMA_INDEX, RL_NEG_SCALED_REWARD_MAX_INDEX, + RL_NET_INVENTORY_LIMIT_USD_INDEX, RL_N_ROLLOUT_STEPS_INDEX, + RL_OUTCOME_AUX_LAMBDA_INDEX, RL_PER_ALPHA_INDEX, + RL_PI_GRAD_NORM_EMA_INDEX, RL_PLATEAU_PATIENCE_INDEX, + RL_POPART_MAX_ABS_REWARD_EMA_INDEX, RL_POPART_MAX_DECAY_ALPHA_INDEX, + RL_POPART_MEAN_INDEX, RL_POPART_SIGMA_INDEX, RL_POPART_VAR_INDEX, + RL_POS_SCALED_REWARD_MAX_EMA_INDEX, RL_POS_SCALED_REWARD_MAX_INDEX, + RL_PPO_CLAMP_MARGIN_INDEX, RL_PPO_CLIP_EPS_MIN_INDEX, RL_PPO_CLIP_INDEX, + RL_PPO_LOG_RATIO_ABS_MAX_INDEX, RL_PPO_RATIO_CLAMP_BOOTSTRAP_INDEX, + RL_PPO_RATIO_CLAMP_MAX_INDEX, RL_PYRAMID_ADD_COUNT_INDEX, + RL_Q_ARG_VS_PI_AGREE_INDEX, RL_Q_BIAS_CORRECTION_INDEX, RL_Q_BIAS_EMA_INDEX, + RL_Q_DISTILL_KL_EMA_INDEX, RL_Q_DISTILL_KL_TARGET_INDEX, + RL_Q_DISTILL_LAMBDA_INDEX, RL_Q_DISTILL_TEMPERATURE_INDEX, + RL_Q_DIVERGENCE_EMA_INDEX, RL_Q_GRAD_NORM_EMA_INDEX, + RL_REGIME_DEAD_ZONE_DURATION_INDEX, RL_REGIME_DEAD_ZONE_FLAG_INDEX, + RL_REGIME_DEAD_ZONE_MAX_DURATION_INDEX, + RL_REGIME_DEAD_ZONE_TIMEOUT_FLAG_INDEX, RL_REGIME_RECOVERY_FACTOR_INDEX, + RL_REGIME_SESSION_PNL_VARIANCE_EMA_INDEX, + RL_REGIME_SESSION_PNL_VAR_COUNT_INDEX, RL_REGIME_TAIL_EVENT_RECENCY_INDEX, + RL_REGIME_TAIL_SIGMA_THRESHOLD_INDEX, + RL_REGIME_TRANSITION_DECAY_STEPS_CONFIG_INDEX, + RL_REGIME_TRANSITION_REMAINING_INDEX, RL_REGIME_TRANSITION_STEPS_CONFIG_INDEX, + RL_REWARD_CLAMP_CLIP_RATE_EMA_INDEX, RL_REWARD_CLAMP_CLIP_RATE_TARGET_INDEX, + RL_REWARD_CLAMP_LOSS_INDEX, RL_REWARD_CLAMP_MARGIN_INDEX, + RL_REWARD_CLAMP_RATIO_INDEX, RL_REWARD_CLAMP_WIN_INDEX, + RL_REWARD_SCALE_BOOTSTRAP_INDEX, RL_REWARD_SCALE_INDEX, + RL_REWARD_SCALE_MIN_INDEX, RL_ROLLOUT_BOOTSTRAP_INDEX, + RL_SAC_ALPHA_INDEX, RL_SAC_ENTROPY_TARGET_INDEX, + RL_SCHULMAN_ADJUST_RATE_INDEX, RL_SCHULMAN_TOLERANCE_INDEX, + RL_SESSION_DD_LIMIT_USD_INDEX, RL_SESSION_DD_TRIGGERED_INDEX, + RL_SESSION_PNL_USD_INDEX, RL_SESSION_PNL_WORST_INDEX, + RL_SLOTS_END, RL_SPECTRAL_DECOUPLE_LAMBDA_INDEX, + RL_SPECTRAL_NORM_MAX_INDEX, RL_STREAM_ALPHA_INDEX, + RL_TARGET_TAU_INDEX, RL_TAU_BOOTSTRAP_INDEX, + RL_TD_KURTOSIS_CLAMP_INDEX, RL_TD_KURTOSIS_EMA_INDEX, + RL_TD_KURT_STREAM_M2_INDEX, RL_TD_KURT_STREAM_M4_INDEX, + RL_TD_KURT_STREAM_MEAN_INDEX, + RL_TRAIL_LOOSEN_FACTOR_INDEX, RL_TRAIL_TIGHTEN_FACTOR_INDEX, + RL_V_GRAD_NORM_EMA_INDEX, RL_WIN_RATE_EMA_INDEX, }; +use crate::rl::common::{FRD_N_ATOMS, FRD_N_HORIZONS, N_ACTIONS}; +use crate::rl::frd::FRD_OUT_DIM; use crate::rl::loss_balance::{read_loss_lambdas_from_isv, LossLambdas}; use crate::rl::ppo::{PolicyHead, PpoHeadsConfig, ValueHead}; use crate::rl::reward::RlLobBackend; @@ -453,6 +535,61 @@ pub struct IntegratedStepStats { pub lambdas: LossLambdas, } +/// Host-side per-step inputs consumed by `IntegratedTrainer::build_diag_value`. +/// +/// Bundles everything the diag JSONL needs that the trainer does not own: +/// * `stats` — the just-completed step's loss components. +/// * Host-side reads of the diag staging buffers (rewards, dones, actions, +/// raw_rewards, trade_duration, outcome_ema, position_lots, pyramid_count, +/// unit_entry_price, unit_entry_step, unit_lots, unit_trail, close_unit_index, +/// frd_logits). All slices are borrowed (zero copies). +/// * Running counters (pnl_cum_usd, total_trades, win_count, …) maintained +/// by the caller across steps. +/// +/// The struct is consumed by both the train loop (in `alpha_rl_train.rs`) +/// and the eval loop, guaranteeing schema parity between `diag.jsonl` and +/// `eval_diag.jsonl` per `feedback_single_source_of_truth_no_duplicates`. +pub struct DiagInputs<'a> { + pub b_size: usize, + pub stats: &'a IntegratedStepStats, + // ── DiagStaging host-side reads ───────────────────────────────── + pub rewards_host: &'a [f32], + pub dones_host: &'a [f32], + pub actions_host: &'a [i32], + pub raw_rewards_host: &'a [f32], + pub trade_duration_host: &'a [f32], + pub outcome_ema_host: &'a [f32], + pub position_lots_host: &'a [i32], + pub pyramid_count_host: &'a [i32], + pub unit_entry_price_host: &'a [f32], + pub unit_entry_step_host: &'a [i32], + pub unit_lots_host: &'a [i32], + pub unit_trail_host: &'a [f32], + pub close_unit_index_host: &'a [i32], + pub frd_logits_host: &'a [f32], + // ── Running counters (caller-maintained across steps) ─────────── + pub pnl_cum_usd: f64, + pub total_trades: u64, + pub win_count: u64, + pub hold_time_sum: f64, + pub trail_fired_total: u64, + pub trail_tighten_total: u64, + pub trail_loosen_total: u64, + pub pyramid_added_total: u64, + pub partial_flat_total: u64, + pub partial_flat_long_total: u64, + pub partial_flat_short_total: u64, + pub conf_gate_total: u64, + pub frd_gate_total: u64, + pub heat_cap_total: u64, + /// EMA-smoothed normalised action histogram used for the + /// `action_entropy` field. At b_size=1 the per-step act_hist is + /// one-hot (H≡0), so the diag's exploration signal comes from + /// this windowed distribution (see `WINDOWED_ACT_ALPHA` in the + /// caller). + pub windowed_act_hist: &'a [f32], +} + pub struct IntegratedTrainer { #[allow(dead_code)] cfg: IntegratedTrainerConfig, @@ -9089,6 +9226,560 @@ impl IntegratedTrainer { } Ok(probs) } + + /// Construct the per-step diag JSONL record. + /// + /// Used by both the train loop and the eval loop in `alpha_rl_train.rs` + /// — guarantees schema parity between `diag.jsonl` and `eval_diag.jsonl` + /// per `feedback_single_source_of_truth_no_duplicates`. All ISV reads + /// come from the trainer's mapped-pinned host slice (`isv_host_slice`); + /// host-side per-step state is passed via `DiagInputs`. + /// + /// Caller pre-conditions: + /// * `inputs.b_size` matches the slice lengths in `inputs`. + /// * The diag staging buffer's `sync_and_swap` has been called so + /// the staging reads point at the most recent completed step. + /// * The trainer's ISV one-step-latency state is consistent with + /// the staging reads. + /// + /// 642-leaf schema documented in `docs/superpowers/specs/2026-05-31- + /// checkpoints-and-eval-diag-design.md` §3.5 and the baseline + /// snapshot at `/tmp/diag_keys_baseline.txt`. + pub fn build_diag_value( + &self, + step: u64, + elapsed_s: f32, + inputs: &DiagInputs<'_>, + ) -> Result { + let b_size = inputs.b_size; + anyhow::ensure!( + inputs.rewards_host.len() == b_size + && inputs.dones_host.len() == b_size + && inputs.actions_host.len() == b_size + && inputs.raw_rewards_host.len() == b_size + && inputs.trade_duration_host.len() == b_size + && inputs.outcome_ema_host.len() == b_size + && inputs.position_lots_host.len() == b_size + && inputs.pyramid_count_host.len() == b_size + && inputs.close_unit_index_host.len() == b_size, + "build_diag_value: per-batch slice length mismatch (expected {b_size})" + ); + anyhow::ensure!( + inputs.unit_entry_price_host.len() == b_size * 4 + && inputs.unit_entry_step_host.len() == b_size * 4 + && inputs.unit_lots_host.len() == b_size * 4 + && inputs.unit_trail_host.len() == b_size * 4, + "build_diag_value: per-unit (B×4) slice length mismatch (expected {})", + b_size * 4 + ); + anyhow::ensure!( + inputs.frd_logits_host.len() == b_size * FRD_OUT_DIM, + "build_diag_value: frd_logits length mismatch (expected {})", + b_size * FRD_OUT_DIM + ); + anyhow::ensure!( + inputs.windowed_act_hist.len() == N_ACTIONS, + "build_diag_value: windowed_act_hist length must be {N_ACTIONS}" + ); + + let isv = self.isv_host_slice(); + let stats = inputs.stats; + + // ── Per-step derived stats ────────────────────────────────── + let mut act_hist = [0u32; N_ACTIONS]; + for &a in inputs.actions_host { + if (a as usize) < N_ACTIONS { + act_hist[a as usize] += 1; + } + } + // Action entropy on the EMA-smoothed windowed distribution. + let win_sum: f32 = inputs.windowed_act_hist.iter().sum(); + let action_entropy: f32 = if win_sum > 1e-9 { + inputs + .windowed_act_hist + .iter() + .filter(|&&p| p > 1e-9) + .map(|&p_raw| { + let p = p_raw / win_sum; + -p * p.ln() + }) + .sum() + } else { + 0.0 + }; + let reward_sum: f32 = inputs.rewards_host.iter().sum(); + let reward_max = inputs + .rewards_host + .iter() + .cloned() + .fold(f32::NEG_INFINITY, f32::max); + let reward_min = inputs + .rewards_host + .iter() + .cloned() + .fold(f32::INFINITY, f32::min); + let reward_abs_max = inputs + .rewards_host + .iter() + .map(|r| r.abs()) + .fold(0.0f32, f32::max); + let done_count: u32 = inputs + .dones_host + .iter() + .map(|&d| if d > 0.5 { 1 } else { 0 }) + .sum(); + + let win_rate = if inputs.total_trades > 0 { + inputs.win_count as f64 / inputs.total_trades as f64 + } else { + 0.0 + }; + let avg_hold = if inputs.total_trades > 0 { + inputs.hold_time_sum / inputs.total_trades as f64 + } else { + 0.0 + }; + + // Per-step counters derived from action histogram + ISV diag slots. + let trail_fired_step = isv[RL_CONF_GATE_FIRED_COUNT_INDEX] as u64; + let tighten_step = act_hist[7] as u64; + let loosen_step = act_hist[8] as u64; + let pyramid_add_step = isv[RL_PYRAMID_ADD_COUNT_INDEX] as u64; + let half_flat_long_step = act_hist[9] as u64; + let half_flat_short_step = act_hist[10] as u64; + let partial_flat_step = half_flat_long_step + half_flat_short_step; + let conf_gate_step = isv[RL_CONF_GATE_FIRED_COUNT_INDEX] as u64; + let frd_gate_step = isv[RL_FRD_GATE_FIRED_COUNT_INDEX] as u64; + let heat_cap_step = isv[RL_HEAT_CAP_FIRED_COUNT_INDEX] as u64; + + // Derive active mask from unit_lots (lots != 0 → active). + let unit_active_host: Vec = inputs + .unit_lots_host + .iter() + .map(|&l| if l != 0 { 1u8 } else { 0u8 }) + .collect(); + + // Per-batch per-unit arrays for diag. + let units_diag = { + let mut entry_price_arr = Vec::with_capacity(b_size); + let mut entry_step_arr = Vec::with_capacity(b_size); + let mut lots_arr = Vec::with_capacity(b_size); + let mut trail_arr = Vec::with_capacity(b_size); + let mut active_arr = Vec::with_capacity(b_size); + for batch in 0..b_size { + let off = batch * 4; + entry_price_arr.push(&inputs.unit_entry_price_host[off..off + 4]); + entry_step_arr.push(&inputs.unit_entry_step_host[off..off + 4]); + lots_arr.push(&inputs.unit_lots_host[off..off + 4]); + trail_arr.push(&inputs.unit_trail_host[off..off + 4]); + active_arr.push(&unit_active_host[off..off + 4]); + } + serde_json::json!({ + "entry_price": entry_price_arr, + "entry_step": entry_step_arr, + "lots": lots_arr, + "trail_distance": trail_arr, + "active_mask": active_arr, + "unit_count": inputs.pyramid_count_host, + }) + }; + + // FRD diag — per-horizon softmax entropy + argmax index (averaged + // across the batch). + let frd_diag = { + let mut per_h_entropy = [0.0_f32; FRD_N_HORIZONS]; + let mut per_h_argmax_sum = [0.0_f32; FRD_N_HORIZONS]; + for b in 0..b_size { + for h in 0..FRD_N_HORIZONS { + let off = b * FRD_OUT_DIM + h * FRD_N_ATOMS; + let row = &inputs.frd_logits_host[off..off + FRD_N_ATOMS]; + let max_l = row.iter().cloned().fold(f32::NEG_INFINITY, f32::max); + let denom: f32 = row.iter().map(|x| (x - max_l).exp()).sum(); + let mut entropy = 0.0_f32; + let mut argmax_idx = 0; + let mut argmax_v = f32::NEG_INFINITY; + for (a, x) in row.iter().enumerate() { + let p = (x - max_l).exp() / denom; + if p > 1e-9 { + entropy -= p * p.ln(); + } + if *x > argmax_v { + argmax_v = *x; + argmax_idx = a; + } + } + per_h_entropy[h] += entropy; + per_h_argmax_sum[h] += argmax_idx as f32; + } + } + let b_f = b_size.max(1) as f32; + serde_json::json!({ + "h1": { "entropy_mean": per_h_entropy[0] / b_f, "argmax_mean": per_h_argmax_sum[0] / b_f }, + "h2": { "entropy_mean": per_h_entropy[1] / b_f, "argmax_mean": per_h_argmax_sum[1] / b_f }, + "h3": { "entropy_mean": per_h_entropy[2] / b_f, "argmax_mean": per_h_argmax_sum[2] / b_f }, + }) + }; + + // v9 eval_warmup pre-compute (json! macro can't take if-else). + let eval_warmup_remaining = isv[RL_REGIME_TRANSITION_REMAINING_INDEX]; + let eval_warmup_decay_steps = isv[RL_REGIME_TRANSITION_DECAY_STEPS_CONFIG_INDEX]; + let eval_warmup_active = if eval_warmup_remaining > 0.0 { 1.0_f32 } else { 0.0_f32 }; + let eval_warmup_blend = if eval_warmup_remaining < 0.0 { + 0.0_f32 + } else if eval_warmup_remaining > eval_warmup_decay_steps { + 1.0_f32 + } else if eval_warmup_decay_steps > 0.0 { + eval_warmup_remaining / eval_warmup_decay_steps + } else if eval_warmup_remaining > 0.0 { + 1.0_f32 + } else { + 0.0_f32 + }; + + let replay_len = self.gpu_replay.capacity.min((step as usize) + 1); + + let record = serde_json::json!({ + "step": step, + "elapsed_s": elapsed_s, + "loss": { + "bce": stats.l_bce, + "q": stats.l_q, + "pi": stats.l_pi, + "v": stats.l_v, + "aux": stats.l_aux, + "frd": stats.l_frd, + "total": stats.l_total, + }, + "lambdas": { + "bce": stats.lambdas.bce, + "q": stats.lambdas.q, + "pi": stats.lambdas.pi, + "v": stats.lambdas.v, + "aux": stats.lambdas.aux, + "frd": stats.lambdas.frd, + }, + "isv_out": { + "gamma": isv[RL_GAMMA_INDEX], + "target_tau": isv[RL_TARGET_TAU_INDEX], + "ppo_clip_eps": isv[RL_PPO_CLIP_INDEX], + "entropy_coef": isv[RL_ENTROPY_COEF_INDEX], + "n_rollout_steps": isv[RL_N_ROLLOUT_STEPS_INDEX], + "per_alpha": isv[RL_PER_ALPHA_INDEX], + "reward_scale": isv[RL_REWARD_SCALE_INDEX], + }, + "isv_lr": { + "bce": isv[RL_LR_BCE_INDEX], + "q": isv[RL_LR_Q_INDEX], + "pi": isv[RL_LR_PI_INDEX], + "v": isv[RL_LR_V_INDEX], + "aux": isv[RL_LR_AUX_INDEX], + }, + "isv_ema_in": { + "mean_trade_duration": isv[RL_MEAN_TRADE_DURATION_EMA_INDEX], + "q_divergence": isv[RL_Q_DIVERGENCE_EMA_INDEX], + "kl_pi": isv[RL_KL_PI_EMA_INDEX], + "entropy_observed": isv[RL_ENTROPY_OBSERVED_EMA_INDEX], + "advantage_var_ratio": isv[RL_ADVANTAGE_VAR_RATIO_EMA_INDEX], + "td_kurtosis": isv[RL_TD_KURTOSIS_EMA_INDEX], + "mean_abs_pnl": isv[RL_MEAN_ABS_PNL_EMA_INDEX], + }, + "grad_norm_ema": { + "q": isv[RL_Q_GRAD_NORM_EMA_INDEX], + "pi": isv[RL_PI_GRAD_NORM_EMA_INDEX], + "v": isv[RL_V_GRAD_NORM_EMA_INDEX], + }, + "lr_plateau": { + "q": { "loss_ema": isv[RL_LR_Q_LOSS_EMA_INDEX], + "best": isv[RL_LR_Q_BEST_LOSS_INDEX], + "stale": isv[RL_LR_Q_STEPS_SINCE_BEST_INDEX], + "warmup": isv[RL_LR_Q_WARMUP_COUNTER_INDEX] }, + "pi": { "loss_ema": isv[RL_LR_PI_LOSS_EMA_INDEX], + "best": isv[RL_LR_PI_BEST_LOSS_INDEX], + "stale": isv[RL_LR_PI_STEPS_SINCE_BEST_INDEX], + "warmup": isv[RL_LR_PI_WARMUP_COUNTER_INDEX] }, + "v": { "loss_ema": isv[RL_LR_V_LOSS_EMA_INDEX], + "best": isv[RL_LR_V_BEST_LOSS_INDEX], + "stale": isv[RL_LR_V_STEPS_SINCE_BEST_INDEX], + "warmup": isv[RL_LR_V_WARMUP_COUNTER_INDEX] }, + }, + "risk_stack": { + "cmdp": { + "session_pnl_usd_mean": isv[RL_SESSION_PNL_USD_INDEX], + "session_pnl_worst": isv[RL_SESSION_PNL_WORST_INDEX], + "session_dd_limit_usd": isv[RL_SESSION_DD_LIMIT_USD_INDEX], + "session_dd_triggered": isv[RL_SESSION_DD_TRIGGERED_INDEX], + "consec_loss_count": isv[RL_CONSEC_LOSS_COUNT_INDEX], + "consec_loss_limit": isv[RL_CONSEC_LOSS_LIMIT_INDEX], + "cooldown_remaining_steps": isv[RL_COOLDOWN_REMAINING_STEPS_INDEX], + "cooldown_duration": isv[RL_COOLDOWN_DURATION_INDEX], + "max_open_units": isv[RL_MAX_OPEN_UNITS_INDEX], + "net_inventory_limit_usd": isv[RL_NET_INVENTORY_LIMIT_USD_INDEX], + }, + "iqn_tau": { + "action_tau": isv[RL_IQN_ACTION_TAU_INDEX], + "tau_min": isv[RL_IQN_ACTION_TAU_MIN_INDEX], + "dd_sensitivity": isv[RL_IQN_ACTION_TAU_DD_SENSITIVITY_INDEX], + }, + "inventory": { + "penalty_beta": isv[RL_INVENTORY_PENALTY_BETA_INDEX], + "variance_ema": isv[RL_INVENTORY_VARIANCE_EMA_INDEX], + }, + "kelly": { + "fraction": isv[RL_KELLY_FRACTION_INDEX], + "win_rate_ema": isv[RL_WIN_RATE_EMA_INDEX], + "avg_win_usd_ema": isv[RL_AVG_WIN_USD_EMA_INDEX], + "avg_loss_usd_ema": isv[RL_AVG_LOSS_USD_EMA_INDEX], + "safety_frac": isv[RL_KELLY_SAFETY_FRAC_INDEX], + "min_trades_for_release": isv[RL_KELLY_MIN_TRADES_FOR_RELEASE_INDEX], + "cumulative_dones": isv[RL_CUMULATIVE_DONES_INDEX], + }, + "trail_factors": { + "tighten": isv[RL_TRAIL_TIGHTEN_FACTOR_INDEX], + "loosen": isv[RL_TRAIL_LOOSEN_FACTOR_INDEX], + }, + "eval_warmup": { + "remaining": eval_warmup_remaining, + "active": eval_warmup_active, + "blend": eval_warmup_blend, + "warmup_steps_config": isv[RL_REGIME_TRANSITION_STEPS_CONFIG_INDEX], + "decay_steps_config": eval_warmup_decay_steps, + "floor_kelly_safety": isv[RL_KELLY_SAFETY_FRAC_INDEX], + "floor_iqn_tau_min": isv[RL_IQN_ACTION_TAU_MIN_INDEX], + "floor_entropy_min": isv[RL_ENTROPY_COEF_MIN_INDEX], + "floor_ppo_eps_min": isv[RL_PPO_CLIP_EPS_MIN_INDEX], + "target_defensive": { + "kelly_safety": isv[RL_EVAL_KELLY_SAFETY_DEFENSIVE_INDEX], + "iqn_tau_min": isv[RL_EVAL_IQN_TAU_MIN_DEFENSIVE_INDEX], + "entropy_min": isv[RL_EVAL_ENTROPY_COEF_MIN_DEFENSIVE_INDEX], + "ppo_eps_min": isv[RL_EVAL_PPO_CLIP_EPS_MIN_DEFENSIVE_INDEX], + }, + "target_normal": { + "kelly_safety": isv[RL_EVAL_KELLY_SAFETY_NORMAL_INDEX], + "iqn_tau_min": isv[RL_EVAL_IQN_TAU_MIN_NORMAL_INDEX], + "entropy_min": isv[RL_EVAL_ENTROPY_COEF_MIN_NORMAL_INDEX], + "ppo_eps_min": isv[RL_EVAL_PPO_CLIP_EPS_MIN_NORMAL_INDEX], + }, + }, + "regime": { + "dead_zone": { + "flag": isv[RL_REGIME_DEAD_ZONE_FLAG_INDEX], + "duration": isv[RL_REGIME_DEAD_ZONE_DURATION_INDEX], + "timeout_flag": isv[RL_REGIME_DEAD_ZONE_TIMEOUT_FLAG_INDEX], + "max_duration": isv[RL_REGIME_DEAD_ZONE_MAX_DURATION_INDEX], + }, + "tail": { + "recency": isv[RL_REGIME_TAIL_EVENT_RECENCY_INDEX], + "session_pnl_variance_ema": isv[RL_REGIME_SESSION_PNL_VARIANCE_EMA_INDEX], + "sigma_threshold": isv[RL_REGIME_TAIL_SIGMA_THRESHOLD_INDEX], + "welford_count": isv[RL_REGIME_SESSION_PNL_VAR_COUNT_INDEX], + }, + "kelly_eps_recovery": { + "factor": isv[RL_REGIME_RECOVERY_FACTOR_INDEX], + "live": isv[RL_KELLY_EPS_RECOVERY_LIVE_INDEX], + "min": isv[RL_KELLY_EPS_RECOVERY_MIN_INDEX], + "max": isv[RL_KELLY_EPS_RECOVERY_MAX_INDEX], + "n_recovery": isv[RL_KELLY_EPS_RECOVERY_N_RECOVERY_INDEX], + }, + "popart_envelope": { + "max_abs_reward_ema": isv[RL_POPART_MAX_ABS_REWARD_EMA_INDEX], + "decay_alpha": isv[RL_POPART_MAX_DECAY_ALPHA_INDEX], + }, + "iqn_tau_boost": { + "factor": isv[RL_IQN_TAU_TAIL_BOOST_FACTOR_INDEX], + "n_window": isv[RL_IQN_TAU_TAIL_BOOST_N_WINDOW_INDEX], + }, + }, + "atom_calibration": { + "win_bound": isv[RL_REWARD_CLAMP_WIN_INDEX], + "atom_max": isv[RL_C51_V_MAX_INDEX], + "gamma": isv[RL_GAMMA_INDEX], + "dynamic_bound": isv[RL_REWARD_CLAMP_WIN_INDEX] + + isv[RL_GAMMA_INDEX] + * isv[RL_C51_V_MAX_INDEX], + "atom_max_headroom": isv[RL_C51_V_MAX_INDEX] + - (isv[RL_REWARD_CLAMP_WIN_INDEX] + + isv[RL_GAMMA_INDEX] + * isv[RL_C51_V_MAX_INDEX]), + "popart_sigma": isv[RL_POPART_SIGMA_INDEX], + "v_target_max_3sigma": 3.0_f32 * isv[RL_POPART_SIGMA_INDEX], + "atom_max_over_3sigma": isv[RL_C51_V_MAX_INDEX] + / (3.0_f32 * isv[RL_POPART_SIGMA_INDEX] + 1e-9_f32), + }, + }, + "replay_len": replay_len, + "rewards": { + "sum": reward_sum, + "max": reward_max, + "min": reward_min, + "abs_max": reward_abs_max, + "scaled_pre_clamp_max": + isv[RL_MAX_ABS_SCALED_REWARD_PRE_CLAMP_INDEX], + "pos_scaled_max": + isv[RL_POS_SCALED_REWARD_MAX_INDEX], + "pos_scaled_max_ema": + isv[RL_POS_SCALED_REWARD_MAX_EMA_INDEX], + "clip_rate_ema": + isv[RL_REWARD_CLAMP_CLIP_RATE_EMA_INDEX], + "c51_v_max": + isv[RL_C51_V_MAX_INDEX], + "c51_v_min": + isv[RL_C51_V_MIN_INDEX], + "q_distill_kl_ema": + isv[RL_Q_DISTILL_KL_EMA_INDEX], + "neg_scaled_max": + isv[RL_NEG_SCALED_REWARD_MAX_INDEX], + "neg_scaled_max_ema": + isv[RL_NEG_SCALED_REWARD_MAX_EMA_INDEX], + }, + "ppo": { + "ratio_clamp_max": isv[RL_PPO_RATIO_CLAMP_MAX_INDEX], + "log_ratio_abs_max": isv[RL_PPO_LOG_RATIO_ABS_MAX_INDEX], + }, + "streaming": { + "adv_var": { + "mean": isv[RL_ADV_VAR_STREAM_MEAN_INDEX], + "m2": isv[RL_ADV_VAR_STREAM_M2_INDEX], + "clamp": isv[RL_ADV_VAR_RATIO_CLAMP_INDEX], + }, + "td_kurt": { + "mean": isv[RL_TD_KURT_STREAM_MEAN_INDEX], + "m2": isv[RL_TD_KURT_STREAM_M2_INDEX], + "m4": isv[RL_TD_KURT_STREAM_M4_INDEX], + "clamp": isv[RL_TD_KURTOSIS_CLAMP_INDEX], + }, + }, + "k_loop": { + "k_updates": self.last_k_updates, + "divisor": isv[RL_K_LOOP_DIVISOR_INDEX], + "max": isv[RL_K_LOOP_MAX_INDEX], + }, + "isv_config": { + "reward_clamp_win": isv[RL_REWARD_CLAMP_WIN_INDEX], + "reward_clamp_loss": isv[RL_REWARD_CLAMP_LOSS_INDEX], + "kl_target": isv[RL_KL_TARGET_INDEX], + "improvement_threshold": isv[RL_IMPROVEMENT_THRESHOLD_INDEX], + "plateau_patience": isv[RL_PLATEAU_PATIENCE_INDEX], + "div_target": isv[RL_DIV_TARGET_INDEX], + "entropy_target_frac": isv[RL_ENTROPY_TARGET_FRAC_INDEX], + "kurt_lift_scale": isv[RL_KURT_LIFT_SCALE_INDEX], + "ppo_clamp_margin": isv[RL_PPO_CLAMP_MARGIN_INDEX], + "lr_warmup_steps": isv[RL_LR_WARMUP_STEPS_INDEX], + "lr_bootstrap": isv[RL_LR_BOOTSTRAP_INDEX], + "lr_min": isv[RL_LR_MIN_INDEX], + "lr_max": isv[RL_LR_MAX_INDEX], + "lr_loss_ema_alpha": isv[RL_LR_LOSS_EMA_ALPHA_INDEX], + "lr_decay_factor": isv[RL_LR_DECAY_FACTOR_INDEX], + "loss_lambda_aux": isv[RL_LOSS_LAMBDA_AUX_INDEX], + "schulman_tolerance": isv[RL_SCHULMAN_TOLERANCE_INDEX], + "schulman_adjust_rate": isv[RL_SCHULMAN_ADJUST_RATE_INDEX], + "stream_alpha": isv[RL_STREAM_ALPHA_INDEX], + "kurt_gaussian": isv[RL_KURT_GAUSSIAN_INDEX], + "kurt_noise_floor": isv[RL_KURT_NOISE_FLOOR_INDEX], + "tau_bootstrap": isv[RL_TAU_BOOTSTRAP_INDEX], + "eps_bootstrap": isv[RL_EPS_BOOTSTRAP_INDEX], + "rollout_bootstrap": isv[RL_ROLLOUT_BOOTSTRAP_INDEX], + "reward_scale_bootstrap":isv[RL_REWARD_SCALE_BOOTSTRAP_INDEX], + "ppo_ratio_clamp_bootstrap": isv[RL_PPO_RATIO_CLAMP_BOOTSTRAP_INDEX], + "reward_clamp_margin": isv[RL_REWARD_CLAMP_MARGIN_INDEX], + "reward_clamp_ratio": isv[RL_REWARD_CLAMP_RATIO_INDEX], + "reward_clamp_clip_rate_target": isv[RL_REWARD_CLAMP_CLIP_RATE_TARGET_INDEX], + "q_distill_lambda": isv[RL_Q_DISTILL_LAMBDA_INDEX], + "q_distill_temperature": isv[RL_Q_DISTILL_TEMPERATURE_INDEX], + "q_distill_kl_target": isv[RL_Q_DISTILL_KL_TARGET_INDEX], + "sac_alpha": isv[RL_SAC_ALPHA_INDEX], + "sac_entropy_target": isv[RL_SAC_ENTROPY_TARGET_INDEX], + "action_entropy_ema": isv[RL_ACTION_ENTROPY_EMA_INDEX], + "reward_scale_min": isv[RL_REWARD_SCALE_MIN_INDEX], + }, + "q_pi_agree_ema": isv[RL_Q_ARG_VS_PI_AGREE_INDEX], + "controller_branch": { + "rollout_steps_input": isv[RL_ADVANTAGE_VAR_RATIO_EMA_INDEX], + "rollout_steps_target": isv[RL_ADV_VAR_RATIO_TARGET_INDEX], + "ppo_clip_input": isv[RL_KL_PI_EMA_INDEX], + "ppo_clip_target": 0.01f32, + "target_tau_input": isv[RL_Q_DIVERGENCE_EMA_INDEX], + "target_tau_target": 0.01f32, + "per_alpha_input": isv[RL_TD_KURTOSIS_EMA_INDEX], + "per_alpha_target": 0.6f32, + }, + "done_count": done_count, + "action_hist": act_hist.to_vec(), + "action_entropy": action_entropy, + "position": { + "lots": inputs.position_lots_host, + }, + "units": units_diag, + "trail": { + "fired_count_step": trail_fired_step, + "fired_count_total": inputs.trail_fired_total, + "tightened_count_step": tighten_step, + "loosened_count_step": loosen_step, + "tightened_count_total": inputs.trail_tighten_total, + "loosened_count_total": inputs.trail_loosen_total, + }, + "pyramid": { + "added_count_step": pyramid_add_step, + "added_count_total": inputs.pyramid_added_total, + "units_distribution": inputs.pyramid_count_host, + "max_units_reached": inputs.pyramid_count_host.iter().any(|&c| c >= 4), + }, + "partial_flat": { + "fired_count_step": partial_flat_step, + "fired_count_total": inputs.partial_flat_total, + "long_count_total": inputs.partial_flat_long_total, + "short_count_total": inputs.partial_flat_short_total, + "close_unit_index": inputs.close_unit_index_host, + }, + "confidence_gate": { + "gated_count_step": conf_gate_step, + "gated_count_total": inputs.conf_gate_total, + }, + "position_heat": { + "capped_count_step": heat_cap_step, + "capped_count_total": inputs.heat_cap_total, + "heat_max_lots": isv[RL_HEAT_CAP_MAX_LOTS_INDEX], + }, + "anti_martingale": { + "outcome_ema": inputs.outcome_ema_host, + "kappa": isv[RL_ANTIMARTINGALE_KAPPA_INDEX], + }, + "frd_gate": { + "gated_count_step": frd_gate_step, + "gated_count_total": inputs.frd_gate_total, + }, + "trading": { + "pnl_cum_usd": inputs.pnl_cum_usd, + "total_trades": inputs.total_trades, + "win_rate": win_rate, + "avg_hold_steps": avg_hold, + "raw_reward_sum": inputs.raw_rewards_host.iter().sum::(), + }, + "frd": frd_diag, + "popart": { + "mean": isv[RL_POPART_MEAN_INDEX], + "sigma": isv[RL_POPART_SIGMA_INDEX], + "var": isv[RL_POPART_VAR_INDEX], + }, + "spectral": { + "norm_max_config": isv[RL_SPECTRAL_NORM_MAX_INDEX], + "decouple_lambda": isv[RL_SPECTRAL_DECOUPLE_LAMBDA_INDEX], + }, + "q_bias": { + "ema": isv[RL_Q_BIAS_EMA_INDEX], + "correction": isv[RL_Q_BIAS_CORRECTION_INDEX], + }, + "per_branch_lr": { + "scale_q": isv[RL_LR_SCALE_Q_INDEX], + "scale_pi": isv[RL_LR_SCALE_PI_INDEX], + "scale_v": isv[RL_LR_SCALE_V_INDEX], + "scale_iqn": isv[RL_LR_SCALE_IQN_INDEX], + }, + "outcome_aux": { + "lambda": isv[RL_OUTCOME_AUX_LAMBDA_INDEX], + }, + }); + Ok(record) + } } /// Free-function entry point for the `grad_h_accumulate_scaled` kernel.