gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor(trainer): extract json!{} into IntegratedTrainer::build_diag_value

Browse Source 
The per-step diag JSONL `json!{...}` block had grown to 642 leaf paths
across ~30 nested objects, duplicating every ISV slot read into the
example binary. Phase A of the 2026-05-31 checkpoints+eval-diag plan
extracts it into a single builder on the trainer so the eval phase
can reuse it (next commit) without duplicating the schema.

Changes:
  * `IntegratedTrainer::build_diag_value(step, elapsed_s, &DiagInputs)
    -> Result<serde_json::Value>` — same 642-leaf schema as before,
    bit-equivalent ISV reads (all from `self.isv_host_slice()`).
  * `DiagInputs<'a>` struct bundles the host-side per-step state the
    trainer doesn't own (DiagStaging reads + running counters +
    windowed act histogram), so the call site stays a 1-liner.
  * Train loop in `alpha_rl_train.rs` swaps the inline json! for the
    builder call; the ~140-slot ISV-imports wall collapses to four
    slots still read by the stderr ticker.
  * `#![recursion_limit = "256"]` moves from the example into
    `ml-alpha/src/lib.rs` since the builder now lives in the library.

Schema parity verified: `head -1 diag.jsonl | jq 'paths(scalars)|sort'`
yields the same 642 keys as before this refactor (no schema drift).
This commit is contained in:
jgrusewski
2026-05-31 17:25:47 +02:00
parent 13d8ed76da
commit 8a93a77adc
3 changed files with 766 additions and 689 deletions
Download Patch File Download Diff File Expand all files Collapse all files

752
crates/ml-alpha/examples/alpha_rl_train.rs
View File

@@ -44,101 +44,29 @@ use data::providers::databento::dbn_parser::InstrumentFilter;
// emerging meta-pattern: any constant that mirrors a kernel-side
// structural dimension MUST reference the Rust const, not duplicate
// the literal value.
use ml_alpha::rl::common::{N_ACTIONS, FRD_N_ATOMS, FRD_N_HORIZONS};
use ml_alpha::rl::frd::FRD_OUT_DIM;
use ml_alpha::rl::common::N_ACTIONS;
use ml_alpha::data::gpu_dataset::{GpuDataLoader, GpuDataset};
use ml_alpha::data::loader::{
discover_mbp10_files_sorted, MultiHorizonLoader, MultiHorizonLoaderConfig,
DEFAULT_OUTCOME_LABEL_COST_ES,
};
use ml_alpha::heads::HORIZONS;
// ISV slot imports — only the slots still read DIRECTLY in this file.
// Phase A of the 2026-05-31 checkpoints+eval-diag plan moved all ~140
// diag-block ISV reads behind `IntegratedTrainer::build_diag_value`, so
// the previous wall of imports collapsed to the four slots still in
// the per-N-step stderr ticker.
use ml_alpha::rl::isv_slots::{
RL_ADVANTAGE_VAR_RATIO_EMA_INDEX, RL_ENTROPY_COEF_INDEX, RL_ENTROPY_OBSERVED_EMA_INDEX,
RL_GAMMA_INDEX, RL_KL_PI_EMA_INDEX, RL_LR_AUX_INDEX, RL_LR_BCE_INDEX, RL_LR_PI_BEST_LOSS_INDEX,
RL_LR_PI_INDEX, RL_LR_PI_LOSS_EMA_INDEX, RL_LR_PI_STEPS_SINCE_BEST_INDEX,
RL_LR_PI_WARMUP_COUNTER_INDEX, RL_LR_Q_BEST_LOSS_INDEX, RL_LR_Q_INDEX,
RL_LR_Q_LOSS_EMA_INDEX, RL_LR_Q_STEPS_SINCE_BEST_INDEX, RL_LR_Q_WARMUP_COUNTER_INDEX,
RL_LR_V_BEST_LOSS_INDEX, RL_LR_V_INDEX, RL_LR_V_LOSS_EMA_INDEX,
RL_LR_V_STEPS_SINCE_BEST_INDEX, RL_LR_V_WARMUP_COUNTER_INDEX,
RL_ADV_VAR_RATIO_CLAMP_INDEX, RL_ADV_VAR_RATIO_TARGET_INDEX, RL_ADV_VAR_STREAM_M2_INDEX,
RL_ADV_VAR_STREAM_MEAN_INDEX, RL_DIV_TARGET_INDEX, RL_ENTROPY_TARGET_FRAC_INDEX,
RL_EPS_BOOTSTRAP_INDEX, RL_IMPROVEMENT_THRESHOLD_INDEX, RL_KL_TARGET_INDEX,
RL_KURT_GAUSSIAN_INDEX, RL_KURT_LIFT_SCALE_INDEX, RL_KURT_NOISE_FLOOR_INDEX,
RL_K_LOOP_DIVISOR_INDEX, RL_K_LOOP_MAX_INDEX, RL_LOSS_LAMBDA_AUX_INDEX,
RL_LR_BOOTSTRAP_INDEX, RL_LR_DECAY_FACTOR_INDEX, RL_LR_LOSS_EMA_ALPHA_INDEX,
RL_LR_MAX_INDEX, RL_LR_MIN_INDEX, RL_LR_WARMUP_STEPS_INDEX,
RL_PLATEAU_PATIENCE_INDEX, RL_PPO_CLAMP_MARGIN_INDEX,
RL_PPO_RATIO_CLAMP_BOOTSTRAP_INDEX, RL_Q_ARG_VS_PI_AGREE_INDEX,
RL_REWARD_CLAMP_LOSS_INDEX, RL_REWARD_CLAMP_WIN_INDEX, RL_REWARD_SCALE_BOOTSTRAP_INDEX,
RL_ROLLOUT_BOOTSTRAP_INDEX, RL_SCHULMAN_ADJUST_RATE_INDEX, RL_SCHULMAN_TOLERANCE_INDEX,
RL_STREAM_ALPHA_INDEX, RL_TAU_BOOTSTRAP_INDEX,
RL_MAX_ABS_SCALED_REWARD_PRE_CLAMP_INDEX, RL_MEAN_ABS_PNL_EMA_INDEX,
RL_POS_SCALED_REWARD_MAX_INDEX, RL_POS_SCALED_REWARD_MAX_EMA_INDEX,
RL_REWARD_CLAMP_MARGIN_INDEX, RL_REWARD_CLAMP_RATIO_INDEX,
RL_REWARD_CLAMP_CLIP_RATE_EMA_INDEX, RL_REWARD_CLAMP_CLIP_RATE_TARGET_INDEX,
RL_C51_V_MAX_INDEX, RL_C51_V_MIN_INDEX,
RL_Q_DISTILL_LAMBDA_INDEX, RL_Q_DISTILL_TEMPERATURE_INDEX, RL_Q_DISTILL_KL_EMA_INDEX,
RL_NEG_SCALED_REWARD_MAX_INDEX, RL_NEG_SCALED_REWARD_MAX_EMA_INDEX,
RL_Q_DISTILL_KL_TARGET_INDEX, RL_REWARD_SCALE_MIN_INDEX,
RL_MEAN_TRADE_DURATION_EMA_INDEX, RL_N_ROLLOUT_STEPS_INDEX, RL_PER_ALPHA_INDEX,
RL_PI_GRAD_NORM_EMA_INDEX, RL_PPO_CLIP_INDEX, RL_PPO_LOG_RATIO_ABS_MAX_INDEX,
RL_PPO_RATIO_CLAMP_MAX_INDEX, RL_Q_DIVERGENCE_EMA_INDEX, RL_Q_GRAD_NORM_EMA_INDEX,
RL_REWARD_SCALE_INDEX, RL_TARGET_TAU_INDEX, RL_TD_KURTOSIS_CLAMP_INDEX,
RL_TD_KURTOSIS_EMA_INDEX, RL_TD_KURT_STREAM_M2_INDEX, RL_TD_KURT_STREAM_M4_INDEX,
RL_TD_KURT_STREAM_MEAN_INDEX, RL_V_GRAD_NORM_EMA_INDEX,
// P1+P2 feature metrics
RL_POPART_MEAN_INDEX, RL_POPART_SIGMA_INDEX, RL_POPART_VAR_INDEX,
RL_SPECTRAL_NORM_MAX_INDEX, RL_SPECTRAL_DECOUPLE_LAMBDA_INDEX,
RL_Q_BIAS_EMA_INDEX, RL_Q_BIAS_CORRECTION_INDEX,
RL_LR_SCALE_Q_INDEX, RL_LR_SCALE_PI_INDEX, RL_LR_SCALE_V_INDEX, RL_LR_SCALE_IQN_INDEX,
RL_OUTCOME_AUX_LAMBDA_INDEX,
RL_SAC_ALPHA_INDEX, RL_SAC_ENTROPY_TARGET_INDEX,
RL_ACTION_ENTROPY_EMA_INDEX,
// Risk-management stack (spec 2026-05-30-adaptive-risk-management-design).
// Per `feedback_wire_everything_up`: the kernels write these slots;
// diag has to surface them or the run is half-blind.
RL_SESSION_PNL_USD_INDEX, RL_SESSION_PNL_WORST_INDEX,
RL_SESSION_DD_LIMIT_USD_INDEX, RL_SESSION_DD_TRIGGERED_INDEX,
RL_CONSEC_LOSS_LIMIT_INDEX, RL_CONSEC_LOSS_COUNT_INDEX,
RL_COOLDOWN_REMAINING_STEPS_INDEX, RL_COOLDOWN_DURATION_INDEX,
RL_MAX_OPEN_UNITS_INDEX, RL_NET_INVENTORY_LIMIT_USD_INDEX,
RL_IQN_ACTION_TAU_INDEX, RL_IQN_ACTION_TAU_MIN_INDEX,
RL_IQN_ACTION_TAU_DD_SENSITIVITY_INDEX,
RL_INVENTORY_PENALTY_BETA_INDEX, RL_INVENTORY_VARIANCE_EMA_INDEX,
RL_KELLY_FRACTION_INDEX, RL_WIN_RATE_EMA_INDEX,
RL_AVG_WIN_USD_EMA_INDEX, RL_AVG_LOSS_USD_EMA_INDEX,
RL_KELLY_SAFETY_FRAC_INDEX, RL_KELLY_MIN_TRADES_FOR_RELEASE_INDEX,
RL_TRAIL_TIGHTEN_FACTOR_INDEX, RL_TRAIL_LOOSEN_FACTOR_INDEX,
RL_CUMULATIVE_DONES_INDEX,
// v9 defensive eval-boundary calibration (spec
// 2026-05-31-v9-defensive-eval-boundary-calibration.md).
RL_REGIME_TRANSITION_REMAINING_INDEX, RL_REGIME_TRANSITION_STEPS_CONFIG_INDEX,
RL_REGIME_TRANSITION_DECAY_STEPS_CONFIG_INDEX,
RL_EVAL_KELLY_SAFETY_DEFENSIVE_INDEX, RL_EVAL_IQN_TAU_MIN_DEFENSIVE_INDEX,
RL_EVAL_ENTROPY_COEF_MIN_DEFENSIVE_INDEX, RL_EVAL_PPO_CLIP_EPS_MIN_DEFENSIVE_INDEX,
RL_EVAL_KELLY_SAFETY_NORMAL_INDEX, RL_EVAL_IQN_TAU_MIN_NORMAL_INDEX,
RL_EVAL_ENTROPY_COEF_MIN_NORMAL_INDEX, RL_EVAL_PPO_CLIP_EPS_MIN_NORMAL_INDEX,
RL_ENTROPY_COEF_MIN_INDEX, RL_PPO_CLIP_EPS_MIN_INDEX,
// Regime-observer slots (F1.8).
RL_REGIME_DEAD_ZONE_FLAG_INDEX, RL_REGIME_DEAD_ZONE_DURATION_INDEX,
RL_REGIME_DEAD_ZONE_TIMEOUT_FLAG_INDEX, RL_REGIME_RECOVERY_FACTOR_INDEX,
RL_REGIME_SESSION_PNL_VARIANCE_EMA_INDEX, RL_REGIME_TAIL_EVENT_RECENCY_INDEX,
RL_REGIME_SESSION_PNL_VAR_COUNT_INDEX,
RL_REGIME_DEAD_ZONE_MAX_DURATION_INDEX, RL_REGIME_TAIL_SIGMA_THRESHOLD_INDEX,
RL_KELLY_EPS_RECOVERY_LIVE_INDEX, RL_KELLY_EPS_RECOVERY_MIN_INDEX,
RL_KELLY_EPS_RECOVERY_MAX_INDEX, RL_KELLY_EPS_RECOVERY_N_RECOVERY_INDEX,
RL_IQN_TAU_TAIL_BOOST_FACTOR_INDEX, RL_IQN_TAU_TAIL_BOOST_N_WINDOW_INDEX,
RL_POPART_MAX_ABS_REWARD_EMA_INDEX, RL_POPART_MAX_DECAY_ALPHA_INDEX,
RL_GAMMA_INDEX, RL_PER_ALPHA_INDEX, RL_PPO_CLIP_INDEX, RL_REWARD_SCALE_INDEX,
};
use ml_alpha::trainer::diag_staging::DiagStaging;
use ml_alpha::trainer::integrated::{
IntegratedStepStats, IntegratedTrainer, IntegratedTrainerConfig,
DiagInputs, IntegratedStepStats, IntegratedTrainer, IntegratedTrainerConfig,
};
use ml_alpha::trainer::perception::PerceptionTrainerConfig;
use ml_backtesting::sim::LobSimCuda;
use ml_core::device::MlDevice;
use serde::Serialize;
use serde_json::json;
use std::io::{BufWriter, Write};
use std::path::PathBuf;
use std::process;
@@ -622,27 +550,22 @@ fn main() -> Result<()> {
let unit_entry_step_host = diag_staging.read_unit_entry_step();
let unit_lots_host = diag_staging.read_unit_lots();
let unit_trail_host = diag_staging.read_unit_trail();
// Derive active mask from unit_lots (lots != 0 → active).
// Avoids a u8-to-f32 type mismatch in DtoD staging.
let unit_active_host: Vec<u8> = unit_lots_host
.iter()
.map(|&l| if l != 0 { 1u8 } else { 0u8 })
.collect();
let close_unit_index_host = diag_staging.read_close_unit_index();
// Per-step action histogram (counts) — still computed here because
// the windowed EMA and several per-step counters (tighten/loosen/
// half-flat) read directly from it. `build_diag_value` recomputes
// its own act_hist internally — the two are bit-equivalent.
let mut act_hist = [0u32; N_ACTIONS];
for &a in &actions_host {
if (a as usize) < N_ACTIONS {
act_hist[a as usize] += 1;
}
}
// audit — action entropy H(action_dist) windowed.
//
// Per-batch act_hist is one-hot at b_size=1 so per-step entropy
// is structurally 0. Maintain an EMA-smoothed action histogram
// across recent steps (α=1/1000, half-life ≈ 690 steps) and
// compute entropy on the normalised window. This captures the
// actual policy exploration variety over the recent run.
// EMA-smoothed windowed action distribution (cross-step state).
// Mutated here so the next step inherits the updated EMA;
// build_diag_value reads the post-update value via `windowed_act_hist`
// in DiagInputs.
let total_actions: u32 = act_hist.iter().sum();
if total_actions > 0 {
for i in 0..N_ACTIONS {
@@ -652,28 +575,14 @@ fn main() -> Result<()> {
+ WINDOWED_ACT_ALPHA * p_step;
}
}
let win_sum: f32 = windowed_act_hist.iter().sum();
let action_entropy: f32 = if win_sum > 1e-9 {
windowed_act_hist
.iter()
.filter(|&&p| p > 1e-9)
.map(|&p_raw| {
let p = p_raw / win_sum;
-p * p.ln()
})
.sum()
} else {
0.0
};
let reward_sum: f32 = rewards_host.iter().sum();
let reward_max = rewards_host.iter().cloned().fold(f32::NEG_INFINITY, f32::max);
let reward_min = rewards_host.iter().cloned().fold(f32::INFINITY, f32::min);
let reward_abs_max = rewards_host.iter().map(|r| r.abs()).fold(0.0f32, f32::max);
let done_count: u32 = dones_host.iter().map(|&d| if d > 0.5 { 1 } else { 0 }).sum();
// Per-trade cumulative stats (surfer validation).
// Use post-scale reward / current_scale to recover approximate USD.
let current_scale = trainer.read_isv_host(ml_alpha::rl::isv_slots::RL_REWARD_SCALE_INDEX);
// Per-trade cumulative stats (surfer validation) — mutate the
// train-loop running counters (pnl_cum_usd / total_trades /
// win_count / hold_time_sum). build_diag_value receives the
// post-update values via DiagInputs.
let current_scale = trainer.read_isv_host(RL_REWARD_SCALE_INDEX);
for b in 0..cli.n_backtests {
if dones_host[b] > 0.5 {
let pnl_usd = if current_scale > 1e-9 {
@@ -688,7 +597,6 @@ fn main() -> Result<()> {
}
}
let win_rate = if total_trades > 0 { win_count as f64 / total_trades as f64 } else { 0.0 };
let avg_hold = if total_trades > 0 { hold_time_sum / total_trades as f64 } else { 0.0 };
let isv = trainer.isv_host_slice();
@@ -715,580 +623,52 @@ fn main() -> Result<()> {
frd_gate_total += frd_gate_step;
heat_cap_total += heat_cap_step;
// Build per-batch per-unit arrays for diag.
let units_diag = {
let b = cli.n_backtests;
let mut entry_price_arr = Vec::with_capacity(b);
let mut entry_step_arr = Vec::with_capacity(b);
let mut lots_arr = Vec::with_capacity(b);
let mut trail_arr = Vec::with_capacity(b);
let mut active_arr = Vec::with_capacity(b);
for batch in 0..b {
let off = batch * 4;
entry_price_arr.push(&unit_entry_price_host[off..off + 4]);
entry_step_arr.push(&unit_entry_step_host[off..off + 4]);
lots_arr.push(&unit_lots_host[off..off + 4]);
trail_arr.push(&unit_trail_host[off..off + 4]);
active_arr.push(&unit_active_host[off..off + 4]);
}
json!({
"entry_price": entry_price_arr,
"entry_step": entry_step_arr,
"lots": lots_arr,
"trail_distance": trail_arr,
"active_mask": active_arr,
"unit_count": pyramid_count_host,
})
// Build the per-step diag record via the single-source-of-truth
// builder on the trainer (Phase A of the 2026-05-31
// checkpoints+eval-diag plan). The 642-leaf json!{} block that
// used to live inline here now lives in
// `IntegratedTrainer::build_diag_value`, which is also invoked
// from the eval phase below — guaranteeing schema parity per
// `feedback_single_source_of_truth_no_duplicates`.
let frd_logits_host = diag_staging.read_frd_logits();
let diag_inputs = DiagInputs {
b_size: cli.n_backtests,
stats: &stats,
rewards_host,
dones_host,
actions_host: &actions_host,
raw_rewards_host,
trade_duration_host,
outcome_ema_host,
position_lots_host: &position_lots_host,
pyramid_count_host: &pyramid_count_host,
unit_entry_price_host,
unit_entry_step_host: &unit_entry_step_host,
unit_lots_host: &unit_lots_host,
unit_trail_host,
close_unit_index_host: &close_unit_index_host,
frd_logits_host,
pnl_cum_usd,
total_trades,
win_count,
hold_time_sum,
trail_fired_total,
trail_tighten_total,
trail_loosen_total,
pyramid_added_total,
partial_flat_total,
partial_flat_long_total,
partial_flat_short_total,
conf_gate_total,
frd_gate_total,
heat_cap_total,
windowed_act_hist: &windowed_act_hist,
};
// FRD diag — per-horizon softmax entropy + argmax index,
// averaged across the batch. Reads from DiagStaging (async,
// zero GPU stalls). Computes softmax host-side (small: B x 63).
let frd_diag = {
let frd_logits = diag_staging.read_frd_logits();
let mut per_h_entropy = [0.0_f32; FRD_N_HORIZONS];
let mut per_h_argmax_sum = [0.0_f32; FRD_N_HORIZONS];
for b in 0..cli.n_backtests {
for h in 0..FRD_N_HORIZONS {
let off = b * FRD_OUT_DIM + h * FRD_N_ATOMS;
let row = &frd_logits[off..off + FRD_N_ATOMS];
let max_l = row.iter().cloned().fold(f32::NEG_INFINITY, f32::max);
let denom: f32 = row.iter().map(|x| (x - max_l).exp()).sum();
let mut entropy = 0.0_f32;
let mut argmax_idx = 0;
let mut argmax_v = f32::NEG_INFINITY;
for (a, x) in row.iter().enumerate() {
let p = (x - max_l).exp() / denom;
if p > 1e-9 {
entropy -= p * p.ln();
}
if *x > argmax_v {
argmax_v = *x;
argmax_idx = a;
}
}
per_h_entropy[h] += entropy;
per_h_argmax_sum[h] += argmax_idx as f32;
}
}
let b_f = cli.n_backtests.max(1) as f32;
json!({
"h1": { "entropy_mean": per_h_entropy[0] / b_f, "argmax_mean": per_h_argmax_sum[0] / b_f },
"h2": { "entropy_mean": per_h_entropy[1] / b_f, "argmax_mean": per_h_argmax_sum[1] / b_f },
"h3": { "entropy_mean": per_h_entropy[2] / b_f, "argmax_mean": per_h_argmax_sum[2] / b_f },
})
};
// v9 eval_warmup pre-compute (json! macro can't take Rust blocks/if-else).
let eval_warmup_remaining = isv[RL_REGIME_TRANSITION_REMAINING_INDEX];
let eval_warmup_decay_steps = isv[RL_REGIME_TRANSITION_DECAY_STEPS_CONFIG_INDEX];
let eval_warmup_active = if eval_warmup_remaining > 0.0 { 1.0_f32 } else { 0.0_f32 };
let eval_warmup_blend = if eval_warmup_remaining < 0.0 {
0.0_f32
} else if eval_warmup_remaining > eval_warmup_decay_steps {
1.0_f32
} else if eval_warmup_decay_steps > 0.0 {
eval_warmup_remaining / eval_warmup_decay_steps
} else if eval_warmup_remaining > 0.0 {
1.0_f32
} else {
0.0_f32
};
let record = json!({
"step": step,
"elapsed_s": t_start.elapsed().as_secs_f32(),
"loss": {
"bce": stats.l_bce,
"q": stats.l_q,
"pi": stats.l_pi,
"v": stats.l_v,
"aux": stats.l_aux,
"frd": stats.l_frd,
"total": stats.l_total,
},
"lambdas": {
"bce": stats.lambdas.bce,
"q": stats.lambdas.q,
"pi": stats.lambdas.pi,
"v": stats.lambdas.v,
"aux": stats.lambdas.aux,
"frd": stats.lambdas.frd,
},
// 7 R5 controller outputs (the adaptive knobs).
"isv_out": {
"gamma": isv[RL_GAMMA_INDEX],
"target_tau": isv[RL_TARGET_TAU_INDEX],
"ppo_clip_eps": isv[RL_PPO_CLIP_INDEX],
"entropy_coef": isv[RL_ENTROPY_COEF_INDEX],
"n_rollout_steps": isv[RL_N_ROLLOUT_STEPS_INDEX],
"per_alpha": isv[RL_PER_ALPHA_INDEX],
"reward_scale": isv[RL_REWARD_SCALE_INDEX],
},
// 5 per-head learning rates (rl_lr_controller emits).
"isv_lr": {
"bce": isv[RL_LR_BCE_INDEX],
"q": isv[RL_LR_Q_INDEX],
"pi": isv[RL_LR_PI_INDEX],
"v": isv[RL_LR_V_INDEX],
"aux": isv[RL_LR_AUX_INDEX],
},
// 7 EMA inputs the controllers consume — knowing these
// makes controller behaviour debuggable (was target wrong?
// was input wrong? was alpha wrong?).
"isv_ema_in": {
"mean_trade_duration": isv[RL_MEAN_TRADE_DURATION_EMA_INDEX],
"q_divergence": isv[RL_Q_DIVERGENCE_EMA_INDEX],
"kl_pi": isv[RL_KL_PI_EMA_INDEX],
"entropy_observed": isv[RL_ENTROPY_OBSERVED_EMA_INDEX],
"advantage_var_ratio": isv[RL_ADVANTAGE_VAR_RATIO_EMA_INDEX],
"td_kurtosis": isv[RL_TD_KURTOSIS_EMA_INDEX],
"mean_abs_pnl": isv[RL_MEAN_ABS_PNL_EMA_INDEX],
},
// Grad-norm EMAs (diagnostic — no longer drive the LR
// controller as of commit 042de99e6 but still useful to
// correlate with plateau-decay events).
"grad_norm_ema": {
"q": isv[RL_Q_GRAD_NORM_EMA_INDEX],
"pi": isv[RL_PI_GRAD_NORM_EMA_INDEX],
"v": isv[RL_V_GRAD_NORM_EMA_INDEX],
},
// Plateau-decay LR controller state (per head). loss_ema is
// the controller's internal slow EMA at α=0.05; best is the
// lowest loss_ema observed so far; steps_since_best is the
// staleness counter that fires LR halving at PLATEAU_PATIENCE
// (1000). Confirms whether the controller is seeing plateaus
// and decaying for the right reason.
"lr_plateau": {
"q": { "loss_ema": isv[RL_LR_Q_LOSS_EMA_INDEX],
"best": isv[RL_LR_Q_BEST_LOSS_INDEX],
"stale": isv[RL_LR_Q_STEPS_SINCE_BEST_INDEX],
"warmup": isv[RL_LR_Q_WARMUP_COUNTER_INDEX] },
"pi": { "loss_ema": isv[RL_LR_PI_LOSS_EMA_INDEX],
"best": isv[RL_LR_PI_BEST_LOSS_INDEX],
"stale": isv[RL_LR_PI_STEPS_SINCE_BEST_INDEX],
"warmup": isv[RL_LR_PI_WARMUP_COUNTER_INDEX] },
"v": { "loss_ema": isv[RL_LR_V_LOSS_EMA_INDEX],
"best": isv[RL_LR_V_BEST_LOSS_INDEX],
"stale": isv[RL_LR_V_STEPS_SINCE_BEST_INDEX],
"warmup": isv[RL_LR_V_WARMUP_COUNTER_INDEX] },
},
// Adaptive risk-management stack state — five orthogonal layers.
// Every slot here is written by a kernel; diag surfaces them so
// controller behaviour, gate firing, and bootstrap state are
// observable without re-reading device memory.
"risk_stack": {
// Layer 1 — CMDP hard constraints.
// session_pnl_usd is MEAN-of-active-accounts (the IQN-τ
// consumer); session_pnl_worst is the most-negative
// per-batch pnl (diag — spot fleet skew).
"cmdp": {
"session_pnl_usd_mean": isv[RL_SESSION_PNL_USD_INDEX],
"session_pnl_worst": isv[RL_SESSION_PNL_WORST_INDEX],
"session_dd_limit_usd": isv[RL_SESSION_DD_LIMIT_USD_INDEX],
"session_dd_triggered": isv[RL_SESSION_DD_TRIGGERED_INDEX],
"consec_loss_count": isv[RL_CONSEC_LOSS_COUNT_INDEX],
"consec_loss_limit": isv[RL_CONSEC_LOSS_LIMIT_INDEX],
"cooldown_remaining_steps": isv[RL_COOLDOWN_REMAINING_STEPS_INDEX],
"cooldown_duration": isv[RL_COOLDOWN_DURATION_INDEX],
"max_open_units": isv[RL_MAX_OPEN_UNITS_INDEX],
"net_inventory_limit_usd": isv[RL_NET_INVENTORY_LIMIT_USD_INDEX],
},
// Layer 2 — IQN risk-averse action τ.
"iqn_tau": {
"action_tau": isv[RL_IQN_ACTION_TAU_INDEX],
"tau_min": isv[RL_IQN_ACTION_TAU_MIN_INDEX],
"dd_sensitivity": isv[RL_IQN_ACTION_TAU_DD_SENSITIVITY_INDEX],
},
// Layer 3 — Avellaneda-Stoikov inventory penalty β.
"inventory": {
"penalty_beta": isv[RL_INVENTORY_PENALTY_BETA_INDEX],
"variance_ema": isv[RL_INVENTORY_VARIANCE_EMA_INDEX],
},
// Layer 4 — Half-Kelly position sizing + its EMA inputs.
"kelly": {
"fraction": isv[RL_KELLY_FRACTION_INDEX],
"win_rate_ema": isv[RL_WIN_RATE_EMA_INDEX],
"avg_win_usd_ema": isv[RL_AVG_WIN_USD_EMA_INDEX],
"avg_loss_usd_ema": isv[RL_AVG_LOSS_USD_EMA_INDEX],
"safety_frac": isv[RL_KELLY_SAFETY_FRAC_INDEX],
"min_trades_for_release": isv[RL_KELLY_MIN_TRADES_FOR_RELEASE_INDEX],
"cumulative_dones": isv[RL_CUMULATIVE_DONES_INDEX],
},
// Layer D — trail tighten/loosen factors (kernel-side
// counters are already in `trail.{tightened,loosened}_count_*`).
"trail_factors": {
"tighten": isv[RL_TRAIL_TIGHTEN_FACTOR_INDEX],
"loosen": isv[RL_TRAIL_LOOSEN_FACTOR_INDEX],
},
// v9 — defensive eval-boundary calibration ([[pearl_adaptive_carryover_discipline]]).
// `remaining` is the per-step counter: ≥ decay_steps → full
// defensive blend=1.0; (0, decay_steps] → linear decay to 0;
// 0 → final boundary write of normal values; -1 → kernel
// no-op (post-warmup steady state). The four `floor_*`
// entries are the LIVE values (read AFTER the warmup
// kernel ran this step, so they reflect any override).
"eval_warmup": {
"remaining": eval_warmup_remaining,
"active": eval_warmup_active,
"blend": eval_warmup_blend,
"warmup_steps_config": isv[RL_REGIME_TRANSITION_STEPS_CONFIG_INDEX],
"decay_steps_config": eval_warmup_decay_steps,
"floor_kelly_safety": isv[RL_KELLY_SAFETY_FRAC_INDEX],
"floor_iqn_tau_min": isv[RL_IQN_ACTION_TAU_MIN_INDEX],
"floor_entropy_min": isv[RL_ENTROPY_COEF_MIN_INDEX],
"floor_ppo_eps_min": isv[RL_PPO_CLIP_EPS_MIN_INDEX],
"target_defensive": {
"kelly_safety": isv[RL_EVAL_KELLY_SAFETY_DEFENSIVE_INDEX],
"iqn_tau_min": isv[RL_EVAL_IQN_TAU_MIN_DEFENSIVE_INDEX],
"entropy_min": isv[RL_EVAL_ENTROPY_COEF_MIN_DEFENSIVE_INDEX],
"ppo_eps_min": isv[RL_EVAL_PPO_CLIP_EPS_MIN_DEFENSIVE_INDEX],
},
"target_normal": {
"kelly_safety": isv[RL_EVAL_KELLY_SAFETY_NORMAL_INDEX],
"iqn_tau_min": isv[RL_EVAL_IQN_TAU_MIN_NORMAL_INDEX],
"entropy_min": isv[RL_EVAL_ENTROPY_COEF_MIN_NORMAL_INDEX],
"ppo_eps_min": isv[RL_EVAL_PPO_CLIP_EPS_MIN_NORMAL_INDEX],
},
},
// Regime observer — dead-zone, tail events, Kelly ε-recovery,
// PopArt envelope, IQN τ tail-boost. All values are direct
// ISV reads (drift-free per spec issue #9).
"regime": {
"dead_zone": {
"flag": isv[RL_REGIME_DEAD_ZONE_FLAG_INDEX],
"duration": isv[RL_REGIME_DEAD_ZONE_DURATION_INDEX],
"timeout_flag": isv[RL_REGIME_DEAD_ZONE_TIMEOUT_FLAG_INDEX],
"max_duration": isv[RL_REGIME_DEAD_ZONE_MAX_DURATION_INDEX],
},
"tail": {
"recency": isv[RL_REGIME_TAIL_EVENT_RECENCY_INDEX],
"session_pnl_variance_ema": isv[RL_REGIME_SESSION_PNL_VARIANCE_EMA_INDEX],
"sigma_threshold": isv[RL_REGIME_TAIL_SIGMA_THRESHOLD_INDEX],
"welford_count": isv[RL_REGIME_SESSION_PNL_VAR_COUNT_INDEX],
},
"kelly_eps_recovery": {
"factor": isv[RL_REGIME_RECOVERY_FACTOR_INDEX],
"live": isv[RL_KELLY_EPS_RECOVERY_LIVE_INDEX],
"min": isv[RL_KELLY_EPS_RECOVERY_MIN_INDEX],
"max": isv[RL_KELLY_EPS_RECOVERY_MAX_INDEX],
"n_recovery": isv[RL_KELLY_EPS_RECOVERY_N_RECOVERY_INDEX],
},
"popart_envelope": {
"max_abs_reward_ema": isv[RL_POPART_MAX_ABS_REWARD_EMA_INDEX],
"decay_alpha": isv[RL_POPART_MAX_DECAY_ALPHA_INDEX],
},
"iqn_tau_boost": {
"factor": isv[RL_IQN_TAU_TAIL_BOOST_FACTOR_INDEX],
"n_window": isv[RL_IQN_TAU_TAIL_BOOST_N_WINDOW_INDEX],
},
},
// Calibration diagnostics — atom-span vs. Bellman dynamic
// bound. Per docs/superpowers/specs/2026-05-31-c51-atom-
// span-math-validation.md: the structural constraint is
// `atom_max ≥ WIN + γ × atom_max` (the Bellman target
// upper bound). When `atom_max_headroom > 0`, system is
// self-consistent (no Q clipping); when `< 0`, V_target
// saturates top atom → Q distillation degrades (the v7
// Fix F failure mode at step 500).
//
// `atom_max_over_3sigma` measures resolution waste: when
// atom_max ≫ 3·popart_σ, atoms are over-sized for actual
// V_target distribution (~99.7% within ±3σ). v8 local
// smoke step 999 had this ratio at 16.8× — i.e., we use
// 1/17 of available atom resolution.
"atom_calibration": {
"win_bound": isv[RL_REWARD_CLAMP_WIN_INDEX],
"atom_max": isv[RL_C51_V_MAX_INDEX],
"gamma": isv[RL_GAMMA_INDEX],
"dynamic_bound": isv[RL_REWARD_CLAMP_WIN_INDEX]
+ isv[RL_GAMMA_INDEX]
* isv[RL_C51_V_MAX_INDEX],
"atom_max_headroom": isv[RL_C51_V_MAX_INDEX]
- (isv[RL_REWARD_CLAMP_WIN_INDEX]
+ isv[RL_GAMMA_INDEX]
* isv[RL_C51_V_MAX_INDEX]),
"popart_sigma": isv[RL_POPART_SIGMA_INDEX],
"v_target_max_3sigma": 3.0_f32 * isv[RL_POPART_SIGMA_INDEX],
"atom_max_over_3sigma": isv[RL_C51_V_MAX_INDEX]
/ (3.0_f32 * isv[RL_POPART_SIGMA_INDEX] + 1e-9_f32),
},
},
"replay_len": trainer.gpu_replay.capacity.min(step + 1),
// Post-scale, POST-clamp reward stats — what V regression
// and Q distributional projection actually saw this step.
// The `scaled_pre_clamp_max` field (below) is the SAME
// batch's max BEFORE the [-3, +1] clamp fired. If
// scaled_pre_clamp_max > 1.0 (clamp threshold on the win
// side) repeatedly, the clamp is doing load-bearing work
// — i.e. the reward_scale controller is failing to keep
// typical trade magnitudes within the C51 atom support.
"rewards": {
"sum": reward_sum,
"max": reward_max,
"min": reward_min,
"abs_max": reward_abs_max,
"scaled_pre_clamp_max":
isv[RL_MAX_ABS_SCALED_REWARD_PRE_CLAMP_INDEX],
// Adaptive clamp signals — surface the controller's
// POINT input + EMA + emitted bounds so post-hoc
// analysis can verify the adaptation chain is moving.
"pos_scaled_max":
isv[RL_POS_SCALED_REWARD_MAX_INDEX],
"pos_scaled_max_ema":
isv[RL_POS_SCALED_REWARD_MAX_EMA_INDEX],
// MARGIN controller state — clip rate vs target tells
// us whether the controller is happy (rate ≈ target)
// or fighting the tail (rate > target × tolerance).
"clip_rate_ema":
isv[RL_REWARD_CLAMP_CLIP_RATE_EMA_INDEX],
// C51 atom span — ratcheted from WIN/LOSS clamp by the
// same controller. Watching these expand confirms Q's
// distributional learning ceiling is being lifted.
"c51_v_max":
isv[RL_C51_V_MAX_INDEX],
"c51_v_min":
isv[RL_C51_V_MIN_INDEX],
// Q→π distillation KL signal — drops toward 0 if π is
// successfully chasing Q's preferences.
"q_distill_kl_ema":
isv[RL_Q_DISTILL_KL_EMA_INDEX],
// Negative-tail tracking — input to adaptive RATIO.
"neg_scaled_max":
isv[RL_NEG_SCALED_REWARD_MAX_INDEX],
"neg_scaled_max_ema":
isv[RL_NEG_SCALED_REWARD_MAX_EMA_INDEX],
},
// audit — PPO importance-ratio clamp diagnostics.
//
// ratio_clamp_max : the adaptive ceiling at ISV[440],
// emitted by rl_ppo_ratio_clamp_controller
// from (1+ε) × PPO_CLAMP_MARGIN.
// log_ratio_abs_max : per-step max(|log π_new − log π_old|)
// over the batch at ISV[441], written
// by ppo_log_ratio_abs_max_b.
//
// The clamp fires when log_ratio_abs_max > ln(ratio_clamp_max).
// For ratio_clamp_max = 10, ln = 2.30. Healthy training has
// log_ratio_abs_max well below this most steps; outliers
// touch or exceed it on rare excursions which the clamp
// bounds before they pollute l_pi.
"ppo": {
"ratio_clamp_max": isv[RL_PPO_RATIO_CLAMP_MAX_INDEX],
"log_ratio_abs_max": isv[RL_PPO_LOG_RATIO_ABS_MAX_INDEX],
},
// audit — streaming-kernel state for the variance-ratio +
// kurtosis estimators that feed rollout_steps and per_α.
// * `*_stream.{mean, m2, m4}` are the Welford-EMA
// internal state slots (442-446) — confirms the
// kernel is folding observations across STEPS.
// * `*_clamp` is the ISV-resident output ceiling
// (447, 448) seeded once at trainer init by
// rl_streaming_clamp_init — visible here so the diag
// can spot when the clamp is firing
// (post-clamp EMA at the consumer-input slot equals
// `clamp` ↔ clamp active).
"streaming": {
"adv_var": {
"mean": isv[RL_ADV_VAR_STREAM_MEAN_INDEX],
"m2": isv[RL_ADV_VAR_STREAM_M2_INDEX],
"clamp": isv[RL_ADV_VAR_RATIO_CLAMP_INDEX],
},
"td_kurt": {
"mean": isv[RL_TD_KURT_STREAM_MEAN_INDEX],
"m2": isv[RL_TD_KURT_STREAM_M2_INDEX],
"m4": isv[RL_TD_KURT_STREAM_M4_INDEX],
"clamp": isv[RL_TD_KURTOSIS_CLAMP_INDEX],
},
},
// audit — controller decision diagnostics (which adaptation
// branch the multiplicative controllers fired this step).
//
// ratio = input_ema / TARGET (per controller's design)
// branch = "WIDEN" if ratio > TOLERANCE = 1.5
// "HOLD" if ratio ∈ [1/1.5, 1.5]
// "SHRINK" if ratio < 1/1.5
// "NOISE" if input_ema is below the noise floor
//
// For rl_rollout_steps_controller (TARGET = 0.1) and
// rl_ppo_clip_controller (TARGET = 0.01), this reveals
// whether the controller is being driven by real divergence
// signal or sitting in the in-band hold zone. Use to spot
// controllers stuck saturating at MAX/MIN (which would
// show 100% WIDEN or 100% SHRINK).
//
// Targets and tolerances are reflected from the kernel
// #defines (kept synchronised by code review at the
// controller-cu file level — there's no ISV slot for these
// design constants because they're fundamental to the
// controller's behaviour, not adaptive).
// audit — K-loop training-intensification iterations performed
// this step. K = clamp(isv[404] / isv[450], 1, isv[451]).
// Both divisor + max are ISV-driven (per
// feedback_isv_for_adaptive_bounds) — surfaced here so
// post-hoc analysis can compute K from any sample without
// recompile. Defaults: divisor=2048 (K=1 at controller
// bootstrap), max=4 (prevents gradient overtraining).
"k_loop": {
"k_updates": trainer.last_k_updates,
"divisor": isv[RL_K_LOOP_DIVISOR_INDEX],
"max": isv[RL_K_LOOP_MAX_INDEX],
},
// audit — ISV-driven design constants (was kernel `#define`s).
// All tunable at runtime via re-launching rl_isv_write per
// `feedback_isv_for_adaptive_bounds`. Surfaced here so
// post-hoc analysis sees exactly what each controller was
// regressing against / clamping at, every step.
"isv_config": {
"reward_clamp_win": isv[RL_REWARD_CLAMP_WIN_INDEX],
"reward_clamp_loss": isv[RL_REWARD_CLAMP_LOSS_INDEX],
"kl_target": isv[RL_KL_TARGET_INDEX],
"improvement_threshold": isv[RL_IMPROVEMENT_THRESHOLD_INDEX],
"plateau_patience": isv[RL_PLATEAU_PATIENCE_INDEX],
"div_target": isv[RL_DIV_TARGET_INDEX],
"entropy_target_frac": isv[RL_ENTROPY_TARGET_FRAC_INDEX],
"kurt_lift_scale": isv[RL_KURT_LIFT_SCALE_INDEX],
"ppo_clamp_margin": isv[RL_PPO_CLAMP_MARGIN_INDEX],
"lr_warmup_steps": isv[RL_LR_WARMUP_STEPS_INDEX],
"lr_bootstrap": isv[RL_LR_BOOTSTRAP_INDEX],
"lr_min": isv[RL_LR_MIN_INDEX],
"lr_max": isv[RL_LR_MAX_INDEX],
"lr_loss_ema_alpha": isv[RL_LR_LOSS_EMA_ALPHA_INDEX],
"lr_decay_factor": isv[RL_LR_DECAY_FACTOR_INDEX],
"loss_lambda_aux": isv[RL_LOSS_LAMBDA_AUX_INDEX],
"schulman_tolerance": isv[RL_SCHULMAN_TOLERANCE_INDEX],
"schulman_adjust_rate": isv[RL_SCHULMAN_ADJUST_RATE_INDEX],
"stream_alpha": isv[RL_STREAM_ALPHA_INDEX],
"kurt_gaussian": isv[RL_KURT_GAUSSIAN_INDEX],
"kurt_noise_floor": isv[RL_KURT_NOISE_FLOOR_INDEX],
"tau_bootstrap": isv[RL_TAU_BOOTSTRAP_INDEX],
"eps_bootstrap": isv[RL_EPS_BOOTSTRAP_INDEX],
"rollout_bootstrap": isv[RL_ROLLOUT_BOOTSTRAP_INDEX],
"reward_scale_bootstrap":isv[RL_REWARD_SCALE_BOOTSTRAP_INDEX],
"ppo_ratio_clamp_bootstrap": isv[RL_PPO_RATIO_CLAMP_BOOTSTRAP_INDEX],
"reward_clamp_margin": isv[RL_REWARD_CLAMP_MARGIN_INDEX],
"reward_clamp_ratio": isv[RL_REWARD_CLAMP_RATIO_INDEX],
"reward_clamp_clip_rate_target": isv[RL_REWARD_CLAMP_CLIP_RATE_TARGET_INDEX],
"q_distill_lambda": isv[RL_Q_DISTILL_LAMBDA_INDEX],
"q_distill_temperature": isv[RL_Q_DISTILL_TEMPERATURE_INDEX],
"q_distill_kl_target": isv[RL_Q_DISTILL_KL_TARGET_INDEX],
"sac_alpha": isv[RL_SAC_ALPHA_INDEX],
"sac_entropy_target": isv[RL_SAC_ENTROPY_TARGET_INDEX],
"action_entropy_ema": isv[RL_ACTION_ENTROPY_EMA_INDEX],
"reward_scale_min": isv[RL_REWARD_SCALE_MIN_INDEX],
},
// audit — Q vs π action agreement EMA at slot 407
// (previously dead). 1.0 = perfect ranking consistency,
// 0.0 = total disagreement. Alarm if persistently < 0.5.
"q_pi_agree_ema": isv[RL_Q_ARG_VS_PI_AGREE_INDEX],
"controller_branch": {
"rollout_steps_input": isv[RL_ADVANTAGE_VAR_RATIO_EMA_INDEX],
// ISV-driven target — seeded by rl_streaming_clamp_init,
// visible here (no longer a hardcoded `#define`).
"rollout_steps_target": isv[RL_ADV_VAR_RATIO_TARGET_INDEX],
"ppo_clip_input": isv[RL_KL_PI_EMA_INDEX],
"ppo_clip_target": 0.01f32,
"target_tau_input": isv[RL_Q_DIVERGENCE_EMA_INDEX],
"target_tau_target": 0.01f32,
"per_alpha_input": isv[RL_TD_KURTOSIS_EMA_INDEX],
"per_alpha_target": 0.6f32, // kurt=10 fixed point
},
"done_count": done_count,
// Action histogram: indices match `Action` enum
// (0=ShortLarge, 1=ShortSmall, 2=Hold, 3=FlatFromLong,
// 4=FlatFromShort, 5=LongSmall, 6=LongLarge,
// 7=TrailTighten, 8=TrailLoosen).
"action_hist": act_hist.to_vec(),
// audit additions:
// * action_entropy — H(action_dist) computed from
// act_hist; uniform-9 = ln(9) ≈ 2.197. Lower = policy
// collapsed to few actions (exploration failed).
// * position.lots — signed lot count per batch (long/short
// /flat). Reveals whether the agent is actually moving
// positions or stuck.
"action_entropy": action_entropy,
"position": {
"lots": position_lots_host,
},
"units": units_diag,
"trail": {
"fired_count_step": trail_fired_step,
"fired_count_total": trail_fired_total,
"tightened_count_step": tighten_step,
"loosened_count_step": loosen_step,
"tightened_count_total": trail_tighten_total,
"loosened_count_total": trail_loosen_total,
},
"pyramid": {
"added_count_step": pyramid_add_step,
"added_count_total": pyramid_added_total,
"units_distribution": pyramid_count_host,
"max_units_reached": pyramid_count_host.iter().any(|&c| c >= 4),
},
"partial_flat": {
"fired_count_step": partial_flat_step,
"fired_count_total": partial_flat_total,
"long_count_total": partial_flat_long_total,
"short_count_total": partial_flat_short_total,
"close_unit_index": close_unit_index_host,
},
"confidence_gate": {
"gated_count_step": conf_gate_step,
"gated_count_total": conf_gate_total,
},
"position_heat": {
"capped_count_step": heat_cap_step,
"capped_count_total": heat_cap_total,
"heat_max_lots": isv[ml_alpha::rl::isv_slots::RL_HEAT_CAP_MAX_LOTS_INDEX],
},
"anti_martingale": {
"outcome_ema": outcome_ema_host,
"kappa": isv[ml_alpha::rl::isv_slots::RL_ANTIMARTINGALE_KAPPA_INDEX],
},
"frd_gate": {
"gated_count_step": frd_gate_step,
"gated_count_total": frd_gate_total,
},
"trading": {
"pnl_cum_usd": pnl_cum_usd,
"total_trades": total_trades,
"win_rate": win_rate,
"avg_hold_steps": avg_hold,
"raw_reward_sum": raw_rewards_host.iter().sum::<f32>(),
},
// SP20 P3 FRD head diag — per-horizon softmax entropy + argmax-mode
// bucket index (averaged across the batch). At init, Xavier × 0.1
// weights → logits ≈ 0 → entropy ≈ ln(21) = 3.044 and the argmax
// is dominated by tiny initial-weight noise. Watching these drift
// away from the uniform baseline is the supervised-learning signal
// for the FRD head (CE loss + bwd land in F.3).
"frd": frd_diag,
// P1+P2 feature metrics — PopArt reward normalization,
// spectral norm/decoupling, Q-bias correction, per-branch
// LR scaling, and outcome aux head lambda.
"popart": {
"mean": isv[RL_POPART_MEAN_INDEX],
"sigma": isv[RL_POPART_SIGMA_INDEX],
"var": isv[RL_POPART_VAR_INDEX],
},
"spectral": {
"norm_max_config": isv[RL_SPECTRAL_NORM_MAX_INDEX],
"decouple_lambda": isv[RL_SPECTRAL_DECOUPLE_LAMBDA_INDEX],
},
"q_bias": {
"ema": isv[RL_Q_BIAS_EMA_INDEX],
"correction": isv[RL_Q_BIAS_CORRECTION_INDEX],
},
"per_branch_lr": {
"scale_q": isv[RL_LR_SCALE_Q_INDEX],
"scale_pi": isv[RL_LR_SCALE_PI_INDEX],
"scale_v": isv[RL_LR_SCALE_V_INDEX],
"scale_iqn": isv[RL_LR_SCALE_IQN_INDEX],
},
"outcome_aux": {
"lambda": isv[RL_OUTCOME_AUX_LAMBDA_INDEX],
},
});
let record = trainer.build_diag_value(
step as u64,
t_start.elapsed().as_secs_f32(),
&diag_inputs,
).context("build_diag_value (train)")?;
writeln!(diag, "{}", record).context("diag: writeln jsonl")?;
if step % cli.log_every == 0 || step == cli.n_steps - 1 {

6
crates/ml-alpha/src/lib.rs
View File

@@ -1,3 +1,9 @@
// The per-step diag `json!{...}` block built by
// `IntegratedTrainer::build_diag_value` spans > 130 leaf fields,
// exceeding serde_json's default macro recursion budget of 128.
// Bump at the crate root so the trainer module compiles.
#![recursion_limit = "256"]
//! ml-alpha — CfC perception + multi-horizon alpha heads.
//!
//! Phase A (this crate): snapshot-level CfC trunk + 5 horizon heads,

697
crates/ml-alpha/src/trainer/integrated.rs
View File

@@ -90,14 +90,96 @@ use crate::heads::HIDDEN_DIM;
use crate::pinned_mem::{MappedF32Buffer, MappedI32Buffer};
use rand::{Rng, SeedableRng};
use crate::rl::common::{N_ACTIONS, Q_N_ATOMS, Q_V_MAX, Q_V_MIN};
use crate::rl::common::{Q_N_ATOMS, Q_V_MAX, Q_V_MIN};
use crate::rl::dqn::{DqnHead, DqnHeadConfig};
use crate::rl::iqn::EMBED_DIM;
use crate::rl::noisy::{NoisyLinear, NoisyLinearConfig};
use crate::rl::isv_slots::{
RL_LR_BCE_INDEX, RL_LR_AUX_INDEX, RL_LR_PI_INDEX, RL_LR_Q_INDEX,
RL_LR_V_INDEX, RL_SLOTS_END,
RL_ACTION_ENTROPY_EMA_INDEX, RL_ADVANTAGE_VAR_RATIO_EMA_INDEX,
RL_ADV_VAR_RATIO_CLAMP_INDEX, RL_ADV_VAR_RATIO_TARGET_INDEX,
RL_ADV_VAR_STREAM_M2_INDEX, RL_ADV_VAR_STREAM_MEAN_INDEX,
RL_ANTIMARTINGALE_KAPPA_INDEX,
RL_AVG_LOSS_USD_EMA_INDEX, RL_AVG_WIN_USD_EMA_INDEX,
RL_C51_V_MAX_INDEX, RL_C51_V_MIN_INDEX,
RL_CONF_GATE_FIRED_COUNT_INDEX, RL_CONSEC_LOSS_COUNT_INDEX,
RL_CONSEC_LOSS_LIMIT_INDEX, RL_COOLDOWN_DURATION_INDEX,
RL_COOLDOWN_REMAINING_STEPS_INDEX, RL_CUMULATIVE_DONES_INDEX,
RL_DIV_TARGET_INDEX, RL_ENTROPY_COEF_INDEX, RL_ENTROPY_COEF_MIN_INDEX,
RL_ENTROPY_OBSERVED_EMA_INDEX, RL_ENTROPY_TARGET_FRAC_INDEX,
RL_EPS_BOOTSTRAP_INDEX,
RL_EVAL_ENTROPY_COEF_MIN_DEFENSIVE_INDEX, RL_EVAL_ENTROPY_COEF_MIN_NORMAL_INDEX,
RL_EVAL_IQN_TAU_MIN_DEFENSIVE_INDEX, RL_EVAL_IQN_TAU_MIN_NORMAL_INDEX,
RL_EVAL_KELLY_SAFETY_DEFENSIVE_INDEX, RL_EVAL_KELLY_SAFETY_NORMAL_INDEX,
RL_EVAL_PPO_CLIP_EPS_MIN_DEFENSIVE_INDEX, RL_EVAL_PPO_CLIP_EPS_MIN_NORMAL_INDEX,
RL_FRD_GATE_FIRED_COUNT_INDEX,
RL_GAMMA_INDEX, RL_HEAT_CAP_FIRED_COUNT_INDEX, RL_HEAT_CAP_MAX_LOTS_INDEX,
RL_IMPROVEMENT_THRESHOLD_INDEX,
RL_INVENTORY_PENALTY_BETA_INDEX, RL_INVENTORY_VARIANCE_EMA_INDEX,
RL_IQN_ACTION_TAU_DD_SENSITIVITY_INDEX, RL_IQN_ACTION_TAU_INDEX,
RL_IQN_ACTION_TAU_MIN_INDEX,
RL_IQN_TAU_TAIL_BOOST_FACTOR_INDEX, RL_IQN_TAU_TAIL_BOOST_N_WINDOW_INDEX,
RL_KELLY_EPS_RECOVERY_LIVE_INDEX, RL_KELLY_EPS_RECOVERY_MAX_INDEX,
RL_KELLY_EPS_RECOVERY_MIN_INDEX, RL_KELLY_EPS_RECOVERY_N_RECOVERY_INDEX,
RL_KELLY_FRACTION_INDEX, RL_KELLY_MIN_TRADES_FOR_RELEASE_INDEX,
RL_KELLY_SAFETY_FRAC_INDEX, RL_KL_PI_EMA_INDEX, RL_KL_TARGET_INDEX,
RL_KURT_GAUSSIAN_INDEX, RL_KURT_LIFT_SCALE_INDEX, RL_KURT_NOISE_FLOOR_INDEX,
RL_K_LOOP_DIVISOR_INDEX, RL_K_LOOP_MAX_INDEX,
RL_LOSS_LAMBDA_AUX_INDEX, RL_LR_AUX_INDEX, RL_LR_BCE_INDEX,
RL_LR_BOOTSTRAP_INDEX, RL_LR_DECAY_FACTOR_INDEX,
RL_LR_LOSS_EMA_ALPHA_INDEX, RL_LR_MAX_INDEX, RL_LR_MIN_INDEX,
RL_LR_PI_BEST_LOSS_INDEX, RL_LR_PI_INDEX, RL_LR_PI_LOSS_EMA_INDEX,
RL_LR_PI_STEPS_SINCE_BEST_INDEX, RL_LR_PI_WARMUP_COUNTER_INDEX,
RL_LR_Q_BEST_LOSS_INDEX, RL_LR_Q_INDEX, RL_LR_Q_LOSS_EMA_INDEX,
RL_LR_Q_STEPS_SINCE_BEST_INDEX, RL_LR_Q_WARMUP_COUNTER_INDEX,
RL_LR_SCALE_IQN_INDEX, RL_LR_SCALE_PI_INDEX, RL_LR_SCALE_Q_INDEX,
RL_LR_SCALE_V_INDEX,
RL_LR_V_BEST_LOSS_INDEX, RL_LR_V_INDEX, RL_LR_V_LOSS_EMA_INDEX,
RL_LR_V_STEPS_SINCE_BEST_INDEX, RL_LR_V_WARMUP_COUNTER_INDEX,
RL_LR_WARMUP_STEPS_INDEX,
RL_MAX_ABS_SCALED_REWARD_PRE_CLAMP_INDEX, RL_MAX_OPEN_UNITS_INDEX,
RL_MEAN_ABS_PNL_EMA_INDEX, RL_MEAN_TRADE_DURATION_EMA_INDEX,
RL_NEG_SCALED_REWARD_MAX_EMA_INDEX, RL_NEG_SCALED_REWARD_MAX_INDEX,
RL_NET_INVENTORY_LIMIT_USD_INDEX, RL_N_ROLLOUT_STEPS_INDEX,
RL_OUTCOME_AUX_LAMBDA_INDEX, RL_PER_ALPHA_INDEX,
RL_PI_GRAD_NORM_EMA_INDEX, RL_PLATEAU_PATIENCE_INDEX,
RL_POPART_MAX_ABS_REWARD_EMA_INDEX, RL_POPART_MAX_DECAY_ALPHA_INDEX,
RL_POPART_MEAN_INDEX, RL_POPART_SIGMA_INDEX, RL_POPART_VAR_INDEX,
RL_POS_SCALED_REWARD_MAX_EMA_INDEX, RL_POS_SCALED_REWARD_MAX_INDEX,
RL_PPO_CLAMP_MARGIN_INDEX, RL_PPO_CLIP_EPS_MIN_INDEX, RL_PPO_CLIP_INDEX,
RL_PPO_LOG_RATIO_ABS_MAX_INDEX, RL_PPO_RATIO_CLAMP_BOOTSTRAP_INDEX,
RL_PPO_RATIO_CLAMP_MAX_INDEX, RL_PYRAMID_ADD_COUNT_INDEX,
RL_Q_ARG_VS_PI_AGREE_INDEX, RL_Q_BIAS_CORRECTION_INDEX, RL_Q_BIAS_EMA_INDEX,
RL_Q_DISTILL_KL_EMA_INDEX, RL_Q_DISTILL_KL_TARGET_INDEX,
RL_Q_DISTILL_LAMBDA_INDEX, RL_Q_DISTILL_TEMPERATURE_INDEX,
RL_Q_DIVERGENCE_EMA_INDEX, RL_Q_GRAD_NORM_EMA_INDEX,
RL_REGIME_DEAD_ZONE_DURATION_INDEX, RL_REGIME_DEAD_ZONE_FLAG_INDEX,
RL_REGIME_DEAD_ZONE_MAX_DURATION_INDEX,
RL_REGIME_DEAD_ZONE_TIMEOUT_FLAG_INDEX, RL_REGIME_RECOVERY_FACTOR_INDEX,
RL_REGIME_SESSION_PNL_VARIANCE_EMA_INDEX,
RL_REGIME_SESSION_PNL_VAR_COUNT_INDEX, RL_REGIME_TAIL_EVENT_RECENCY_INDEX,
RL_REGIME_TAIL_SIGMA_THRESHOLD_INDEX,
RL_REGIME_TRANSITION_DECAY_STEPS_CONFIG_INDEX,
RL_REGIME_TRANSITION_REMAINING_INDEX, RL_REGIME_TRANSITION_STEPS_CONFIG_INDEX,
RL_REWARD_CLAMP_CLIP_RATE_EMA_INDEX, RL_REWARD_CLAMP_CLIP_RATE_TARGET_INDEX,
RL_REWARD_CLAMP_LOSS_INDEX, RL_REWARD_CLAMP_MARGIN_INDEX,
RL_REWARD_CLAMP_RATIO_INDEX, RL_REWARD_CLAMP_WIN_INDEX,
RL_REWARD_SCALE_BOOTSTRAP_INDEX, RL_REWARD_SCALE_INDEX,
RL_REWARD_SCALE_MIN_INDEX, RL_ROLLOUT_BOOTSTRAP_INDEX,
RL_SAC_ALPHA_INDEX, RL_SAC_ENTROPY_TARGET_INDEX,
RL_SCHULMAN_ADJUST_RATE_INDEX, RL_SCHULMAN_TOLERANCE_INDEX,
RL_SESSION_DD_LIMIT_USD_INDEX, RL_SESSION_DD_TRIGGERED_INDEX,
RL_SESSION_PNL_USD_INDEX, RL_SESSION_PNL_WORST_INDEX,
RL_SLOTS_END, RL_SPECTRAL_DECOUPLE_LAMBDA_INDEX,
RL_SPECTRAL_NORM_MAX_INDEX, RL_STREAM_ALPHA_INDEX,
RL_TARGET_TAU_INDEX, RL_TAU_BOOTSTRAP_INDEX,
RL_TD_KURTOSIS_CLAMP_INDEX, RL_TD_KURTOSIS_EMA_INDEX,
RL_TD_KURT_STREAM_M2_INDEX, RL_TD_KURT_STREAM_M4_INDEX,
RL_TD_KURT_STREAM_MEAN_INDEX,
RL_TRAIL_LOOSEN_FACTOR_INDEX, RL_TRAIL_TIGHTEN_FACTOR_INDEX,
RL_V_GRAD_NORM_EMA_INDEX, RL_WIN_RATE_EMA_INDEX,
};
use crate::rl::common::{FRD_N_ATOMS, FRD_N_HORIZONS, N_ACTIONS};
use crate::rl::frd::FRD_OUT_DIM;
use crate::rl::loss_balance::{read_loss_lambdas_from_isv, LossLambdas};
use crate::rl::ppo::{PolicyHead, PpoHeadsConfig, ValueHead};
use crate::rl::reward::RlLobBackend;
@@ -453,6 +535,61 @@ pub struct IntegratedStepStats {
pub lambdas: LossLambdas,
}
/// Host-side per-step inputs consumed by `IntegratedTrainer::build_diag_value`.
///
/// Bundles everything the diag JSONL needs that the trainer does not own:
/// * `stats` — the just-completed step's loss components.
/// * Host-side reads of the diag staging buffers (rewards, dones, actions,
/// raw_rewards, trade_duration, outcome_ema, position_lots, pyramid_count,
/// unit_entry_price, unit_entry_step, unit_lots, unit_trail, close_unit_index,
/// frd_logits). All slices are borrowed (zero copies).
/// * Running counters (pnl_cum_usd, total_trades, win_count, …) maintained
/// by the caller across steps.
///
/// The struct is consumed by both the train loop (in `alpha_rl_train.rs`)
/// and the eval loop, guaranteeing schema parity between `diag.jsonl` and
/// `eval_diag.jsonl` per `feedback_single_source_of_truth_no_duplicates`.
pub struct DiagInputs<'a> {
pub b_size: usize,
pub stats: &'a IntegratedStepStats,
// ── DiagStaging host-side reads ─────────────────────────────────
pub rewards_host: &'a [f32],
pub dones_host: &'a [f32],
pub actions_host: &'a [i32],
pub raw_rewards_host: &'a [f32],
pub trade_duration_host: &'a [f32],
pub outcome_ema_host: &'a [f32],
pub position_lots_host: &'a [i32],
pub pyramid_count_host: &'a [i32],
pub unit_entry_price_host: &'a [f32],
pub unit_entry_step_host: &'a [i32],
pub unit_lots_host: &'a [i32],
pub unit_trail_host: &'a [f32],
pub close_unit_index_host: &'a [i32],
pub frd_logits_host: &'a [f32],
// ── Running counters (caller-maintained across steps) ───────────
pub pnl_cum_usd: f64,
pub total_trades: u64,
pub win_count: u64,
pub hold_time_sum: f64,
pub trail_fired_total: u64,
pub trail_tighten_total: u64,
pub trail_loosen_total: u64,
pub pyramid_added_total: u64,
pub partial_flat_total: u64,
pub partial_flat_long_total: u64,
pub partial_flat_short_total: u64,
pub conf_gate_total: u64,
pub frd_gate_total: u64,
pub heat_cap_total: u64,
/// EMA-smoothed normalised action histogram used for the
/// `action_entropy` field. At b_size=1 the per-step act_hist is
/// one-hot (H≡0), so the diag's exploration signal comes from
/// this windowed distribution (see `WINDOWED_ACT_ALPHA` in the
/// caller).
pub windowed_act_hist: &'a [f32],
}
pub struct IntegratedTrainer {
#[allow(dead_code)]
cfg: IntegratedTrainerConfig,
@@ -9089,6 +9226,560 @@ impl IntegratedTrainer {
}
Ok(probs)
}
/// Construct the per-step diag JSONL record.
///
/// Used by both the train loop and the eval loop in `alpha_rl_train.rs`
/// — guarantees schema parity between `diag.jsonl` and `eval_diag.jsonl`
/// per `feedback_single_source_of_truth_no_duplicates`. All ISV reads
/// come from the trainer's mapped-pinned host slice (`isv_host_slice`);
/// host-side per-step state is passed via `DiagInputs`.
///
/// Caller pre-conditions:
/// * `inputs.b_size` matches the slice lengths in `inputs`.
/// * The diag staging buffer's `sync_and_swap` has been called so
/// the staging reads point at the most recent completed step.
/// * The trainer's ISV one-step-latency state is consistent with
/// the staging reads.
///
/// 642-leaf schema documented in `docs/superpowers/specs/2026-05-31-
/// checkpoints-and-eval-diag-design.md` §3.5 and the baseline
/// snapshot at `/tmp/diag_keys_baseline.txt`.
pub fn build_diag_value(
&self,
step: u64,
elapsed_s: f32,
inputs: &DiagInputs<'_>,
) -> Result<serde_json::Value> {
let b_size = inputs.b_size;
anyhow::ensure!(
inputs.rewards_host.len() == b_size
&& inputs.dones_host.len() == b_size
&& inputs.actions_host.len() == b_size
&& inputs.raw_rewards_host.len() == b_size
&& inputs.trade_duration_host.len() == b_size
&& inputs.outcome_ema_host.len() == b_size
&& inputs.position_lots_host.len() == b_size
&& inputs.pyramid_count_host.len() == b_size
&& inputs.close_unit_index_host.len() == b_size,
"build_diag_value: per-batch slice length mismatch (expected {b_size})"
);
anyhow::ensure!(
inputs.unit_entry_price_host.len() == b_size * 4
&& inputs.unit_entry_step_host.len() == b_size * 4
&& inputs.unit_lots_host.len() == b_size * 4
&& inputs.unit_trail_host.len() == b_size * 4,
"build_diag_value: per-unit (B×4) slice length mismatch (expected {})",
b_size * 4
);
anyhow::ensure!(
inputs.frd_logits_host.len() == b_size * FRD_OUT_DIM,
"build_diag_value: frd_logits length mismatch (expected {})",
b_size * FRD_OUT_DIM
);
anyhow::ensure!(
inputs.windowed_act_hist.len() == N_ACTIONS,
"build_diag_value: windowed_act_hist length must be {N_ACTIONS}"
);
let isv = self.isv_host_slice();
let stats = inputs.stats;
// ── Per-step derived stats ──────────────────────────────────
let mut act_hist = [0u32; N_ACTIONS];
for &a in inputs.actions_host {
if (a as usize) < N_ACTIONS {
act_hist[a as usize] += 1;
}
}
// Action entropy on the EMA-smoothed windowed distribution.
let win_sum: f32 = inputs.windowed_act_hist.iter().sum();
let action_entropy: f32 = if win_sum > 1e-9 {
inputs
.windowed_act_hist
.iter()
.filter(|&&p| p > 1e-9)
.map(|&p_raw| {
let p = p_raw / win_sum;
-p * p.ln()
})
.sum()
} else {
0.0
};
let reward_sum: f32 = inputs.rewards_host.iter().sum();
let reward_max = inputs
.rewards_host
.iter()
.cloned()
.fold(f32::NEG_INFINITY, f32::max);
let reward_min = inputs
.rewards_host
.iter()
.cloned()
.fold(f32::INFINITY, f32::min);
let reward_abs_max = inputs
.rewards_host
.iter()
.map(|r| r.abs())
.fold(0.0f32, f32::max);
let done_count: u32 = inputs
.dones_host
.iter()
.map(|&d| if d > 0.5 { 1 } else { 0 })
.sum();
let win_rate = if inputs.total_trades > 0 {
inputs.win_count as f64 / inputs.total_trades as f64
} else {
0.0
};
let avg_hold = if inputs.total_trades > 0 {
inputs.hold_time_sum / inputs.total_trades as f64
} else {
0.0
};
// Per-step counters derived from action histogram + ISV diag slots.
let trail_fired_step = isv[RL_CONF_GATE_FIRED_COUNT_INDEX] as u64;
let tighten_step = act_hist[7] as u64;
let loosen_step = act_hist[8] as u64;
let pyramid_add_step = isv[RL_PYRAMID_ADD_COUNT_INDEX] as u64;
let half_flat_long_step = act_hist[9] as u64;
let half_flat_short_step = act_hist[10] as u64;
let partial_flat_step = half_flat_long_step + half_flat_short_step;
let conf_gate_step = isv[RL_CONF_GATE_FIRED_COUNT_INDEX] as u64;
let frd_gate_step = isv[RL_FRD_GATE_FIRED_COUNT_INDEX] as u64;
let heat_cap_step = isv[RL_HEAT_CAP_FIRED_COUNT_INDEX] as u64;
// Derive active mask from unit_lots (lots != 0 → active).
let unit_active_host: Vec<u8> = inputs
.unit_lots_host
.iter()
.map(|&l| if l != 0 { 1u8 } else { 0u8 })
.collect();
// Per-batch per-unit arrays for diag.
let units_diag = {
let mut entry_price_arr = Vec::with_capacity(b_size);
let mut entry_step_arr = Vec::with_capacity(b_size);
let mut lots_arr = Vec::with_capacity(b_size);
let mut trail_arr = Vec::with_capacity(b_size);
let mut active_arr = Vec::with_capacity(b_size);
for batch in 0..b_size {
let off = batch * 4;
entry_price_arr.push(&inputs.unit_entry_price_host[off..off + 4]);
entry_step_arr.push(&inputs.unit_entry_step_host[off..off + 4]);
lots_arr.push(&inputs.unit_lots_host[off..off + 4]);
trail_arr.push(&inputs.unit_trail_host[off..off + 4]);
active_arr.push(&unit_active_host[off..off + 4]);
}
serde_json::json!({
"entry_price": entry_price_arr,
"entry_step": entry_step_arr,
"lots": lots_arr,
"trail_distance": trail_arr,
"active_mask": active_arr,
"unit_count": inputs.pyramid_count_host,
})
};
// FRD diag — per-horizon softmax entropy + argmax index (averaged
// across the batch).
let frd_diag = {
let mut per_h_entropy = [0.0_f32; FRD_N_HORIZONS];
let mut per_h_argmax_sum = [0.0_f32; FRD_N_HORIZONS];
for b in 0..b_size {
for h in 0..FRD_N_HORIZONS {
let off = b * FRD_OUT_DIM + h * FRD_N_ATOMS;
let row = &inputs.frd_logits_host[off..off + FRD_N_ATOMS];
let max_l = row.iter().cloned().fold(f32::NEG_INFINITY, f32::max);
let denom: f32 = row.iter().map(|x| (x - max_l).exp()).sum();
let mut entropy = 0.0_f32;
let mut argmax_idx = 0;
let mut argmax_v = f32::NEG_INFINITY;
for (a, x) in row.iter().enumerate() {
let p = (x - max_l).exp() / denom;
if p > 1e-9 {
entropy -= p * p.ln();
}
if *x > argmax_v {
argmax_v = *x;
argmax_idx = a;
}
}
per_h_entropy[h] += entropy;
per_h_argmax_sum[h] += argmax_idx as f32;
}
}
let b_f = b_size.max(1) as f32;
serde_json::json!({
"h1": { "entropy_mean": per_h_entropy[0] / b_f, "argmax_mean": per_h_argmax_sum[0] / b_f },
"h2": { "entropy_mean": per_h_entropy[1] / b_f, "argmax_mean": per_h_argmax_sum[1] / b_f },
"h3": { "entropy_mean": per_h_entropy[2] / b_f, "argmax_mean": per_h_argmax_sum[2] / b_f },
})
};
// v9 eval_warmup pre-compute (json! macro can't take if-else).
let eval_warmup_remaining = isv[RL_REGIME_TRANSITION_REMAINING_INDEX];
let eval_warmup_decay_steps = isv[RL_REGIME_TRANSITION_DECAY_STEPS_CONFIG_INDEX];
let eval_warmup_active = if eval_warmup_remaining > 0.0 { 1.0_f32 } else { 0.0_f32 };
let eval_warmup_blend = if eval_warmup_remaining < 0.0 {
0.0_f32
} else if eval_warmup_remaining > eval_warmup_decay_steps {
1.0_f32
} else if eval_warmup_decay_steps > 0.0 {
eval_warmup_remaining / eval_warmup_decay_steps
} else if eval_warmup_remaining > 0.0 {
1.0_f32
} else {
0.0_f32
};
let replay_len = self.gpu_replay.capacity.min((step as usize) + 1);
let record = serde_json::json!({
"step": step,
"elapsed_s": elapsed_s,
"loss": {
"bce": stats.l_bce,
"q": stats.l_q,
"pi": stats.l_pi,
"v": stats.l_v,
"aux": stats.l_aux,
"frd": stats.l_frd,
"total": stats.l_total,
},
"lambdas": {
"bce": stats.lambdas.bce,
"q": stats.lambdas.q,
"pi": stats.lambdas.pi,
"v": stats.lambdas.v,
"aux": stats.lambdas.aux,
"frd": stats.lambdas.frd,
},
"isv_out": {
"gamma": isv[RL_GAMMA_INDEX],
"target_tau": isv[RL_TARGET_TAU_INDEX],
"ppo_clip_eps": isv[RL_PPO_CLIP_INDEX],
"entropy_coef": isv[RL_ENTROPY_COEF_INDEX],
"n_rollout_steps": isv[RL_N_ROLLOUT_STEPS_INDEX],
"per_alpha": isv[RL_PER_ALPHA_INDEX],
"reward_scale": isv[RL_REWARD_SCALE_INDEX],
},
"isv_lr": {
"bce": isv[RL_LR_BCE_INDEX],
"q": isv[RL_LR_Q_INDEX],
"pi": isv[RL_LR_PI_INDEX],
"v": isv[RL_LR_V_INDEX],
"aux": isv[RL_LR_AUX_INDEX],
},
"isv_ema_in": {
"mean_trade_duration": isv[RL_MEAN_TRADE_DURATION_EMA_INDEX],
"q_divergence": isv[RL_Q_DIVERGENCE_EMA_INDEX],
"kl_pi": isv[RL_KL_PI_EMA_INDEX],
"entropy_observed": isv[RL_ENTROPY_OBSERVED_EMA_INDEX],
"advantage_var_ratio": isv[RL_ADVANTAGE_VAR_RATIO_EMA_INDEX],
"td_kurtosis": isv[RL_TD_KURTOSIS_EMA_INDEX],
"mean_abs_pnl": isv[RL_MEAN_ABS_PNL_EMA_INDEX],
},
"grad_norm_ema": {
"q": isv[RL_Q_GRAD_NORM_EMA_INDEX],
"pi": isv[RL_PI_GRAD_NORM_EMA_INDEX],
"v": isv[RL_V_GRAD_NORM_EMA_INDEX],
},
"lr_plateau": {
"q": { "loss_ema": isv[RL_LR_Q_LOSS_EMA_INDEX],
"best": isv[RL_LR_Q_BEST_LOSS_INDEX],
"stale": isv[RL_LR_Q_STEPS_SINCE_BEST_INDEX],
"warmup": isv[RL_LR_Q_WARMUP_COUNTER_INDEX] },
"pi": { "loss_ema": isv[RL_LR_PI_LOSS_EMA_INDEX],
"best": isv[RL_LR_PI_BEST_LOSS_INDEX],
"stale": isv[RL_LR_PI_STEPS_SINCE_BEST_INDEX],
"warmup": isv[RL_LR_PI_WARMUP_COUNTER_INDEX] },
"v": { "loss_ema": isv[RL_LR_V_LOSS_EMA_INDEX],
"best": isv[RL_LR_V_BEST_LOSS_INDEX],
"stale": isv[RL_LR_V_STEPS_SINCE_BEST_INDEX],
"warmup": isv[RL_LR_V_WARMUP_COUNTER_INDEX] },
},
"risk_stack": {
"cmdp": {
"session_pnl_usd_mean": isv[RL_SESSION_PNL_USD_INDEX],
"session_pnl_worst": isv[RL_SESSION_PNL_WORST_INDEX],
"session_dd_limit_usd": isv[RL_SESSION_DD_LIMIT_USD_INDEX],
"session_dd_triggered": isv[RL_SESSION_DD_TRIGGERED_INDEX],
"consec_loss_count": isv[RL_CONSEC_LOSS_COUNT_INDEX],
"consec_loss_limit": isv[RL_CONSEC_LOSS_LIMIT_INDEX],
"cooldown_remaining_steps": isv[RL_COOLDOWN_REMAINING_STEPS_INDEX],
"cooldown_duration": isv[RL_COOLDOWN_DURATION_INDEX],
"max_open_units": isv[RL_MAX_OPEN_UNITS_INDEX],
"net_inventory_limit_usd": isv[RL_NET_INVENTORY_LIMIT_USD_INDEX],
},
"iqn_tau": {
"action_tau": isv[RL_IQN_ACTION_TAU_INDEX],
"tau_min": isv[RL_IQN_ACTION_TAU_MIN_INDEX],
"dd_sensitivity": isv[RL_IQN_ACTION_TAU_DD_SENSITIVITY_INDEX],
},
"inventory": {
"penalty_beta": isv[RL_INVENTORY_PENALTY_BETA_INDEX],
"variance_ema": isv[RL_INVENTORY_VARIANCE_EMA_INDEX],
},
"kelly": {
"fraction": isv[RL_KELLY_FRACTION_INDEX],
"win_rate_ema": isv[RL_WIN_RATE_EMA_INDEX],
"avg_win_usd_ema": isv[RL_AVG_WIN_USD_EMA_INDEX],
"avg_loss_usd_ema": isv[RL_AVG_LOSS_USD_EMA_INDEX],
"safety_frac": isv[RL_KELLY_SAFETY_FRAC_INDEX],
"min_trades_for_release": isv[RL_KELLY_MIN_TRADES_FOR_RELEASE_INDEX],
"cumulative_dones": isv[RL_CUMULATIVE_DONES_INDEX],
},
"trail_factors": {
"tighten": isv[RL_TRAIL_TIGHTEN_FACTOR_INDEX],
"loosen": isv[RL_TRAIL_LOOSEN_FACTOR_INDEX],
},
"eval_warmup": {
"remaining": eval_warmup_remaining,
"active": eval_warmup_active,
"blend": eval_warmup_blend,
"warmup_steps_config": isv[RL_REGIME_TRANSITION_STEPS_CONFIG_INDEX],
"decay_steps_config": eval_warmup_decay_steps,
"floor_kelly_safety": isv[RL_KELLY_SAFETY_FRAC_INDEX],
"floor_iqn_tau_min": isv[RL_IQN_ACTION_TAU_MIN_INDEX],
"floor_entropy_min": isv[RL_ENTROPY_COEF_MIN_INDEX],
"floor_ppo_eps_min": isv[RL_PPO_CLIP_EPS_MIN_INDEX],
"target_defensive": {
"kelly_safety": isv[RL_EVAL_KELLY_SAFETY_DEFENSIVE_INDEX],
"iqn_tau_min": isv[RL_EVAL_IQN_TAU_MIN_DEFENSIVE_INDEX],
"entropy_min": isv[RL_EVAL_ENTROPY_COEF_MIN_DEFENSIVE_INDEX],
"ppo_eps_min": isv[RL_EVAL_PPO_CLIP_EPS_MIN_DEFENSIVE_INDEX],
},
"target_normal": {
"kelly_safety": isv[RL_EVAL_KELLY_SAFETY_NORMAL_INDEX],
"iqn_tau_min": isv[RL_EVAL_IQN_TAU_MIN_NORMAL_INDEX],
"entropy_min": isv[RL_EVAL_ENTROPY_COEF_MIN_NORMAL_INDEX],
"ppo_eps_min": isv[RL_EVAL_PPO_CLIP_EPS_MIN_NORMAL_INDEX],
},
},
"regime": {
"dead_zone": {
"flag": isv[RL_REGIME_DEAD_ZONE_FLAG_INDEX],
"duration": isv[RL_REGIME_DEAD_ZONE_DURATION_INDEX],
"timeout_flag": isv[RL_REGIME_DEAD_ZONE_TIMEOUT_FLAG_INDEX],
"max_duration": isv[RL_REGIME_DEAD_ZONE_MAX_DURATION_INDEX],
},
"tail": {
"recency": isv[RL_REGIME_TAIL_EVENT_RECENCY_INDEX],
"session_pnl_variance_ema": isv[RL_REGIME_SESSION_PNL_VARIANCE_EMA_INDEX],
"sigma_threshold": isv[RL_REGIME_TAIL_SIGMA_THRESHOLD_INDEX],
"welford_count": isv[RL_REGIME_SESSION_PNL_VAR_COUNT_INDEX],
},
"kelly_eps_recovery": {
"factor": isv[RL_REGIME_RECOVERY_FACTOR_INDEX],
"live": isv[RL_KELLY_EPS_RECOVERY_LIVE_INDEX],
"min": isv[RL_KELLY_EPS_RECOVERY_MIN_INDEX],
"max": isv[RL_KELLY_EPS_RECOVERY_MAX_INDEX],
"n_recovery": isv[RL_KELLY_EPS_RECOVERY_N_RECOVERY_INDEX],
},
"popart_envelope": {
"max_abs_reward_ema": isv[RL_POPART_MAX_ABS_REWARD_EMA_INDEX],
"decay_alpha": isv[RL_POPART_MAX_DECAY_ALPHA_INDEX],
},
"iqn_tau_boost": {
"factor": isv[RL_IQN_TAU_TAIL_BOOST_FACTOR_INDEX],
"n_window": isv[RL_IQN_TAU_TAIL_BOOST_N_WINDOW_INDEX],
},
},
"atom_calibration": {
"win_bound": isv[RL_REWARD_CLAMP_WIN_INDEX],
"atom_max": isv[RL_C51_V_MAX_INDEX],
"gamma": isv[RL_GAMMA_INDEX],
"dynamic_bound": isv[RL_REWARD_CLAMP_WIN_INDEX]
+ isv[RL_GAMMA_INDEX]
* isv[RL_C51_V_MAX_INDEX],
"atom_max_headroom": isv[RL_C51_V_MAX_INDEX]
- (isv[RL_REWARD_CLAMP_WIN_INDEX]
+ isv[RL_GAMMA_INDEX]
* isv[RL_C51_V_MAX_INDEX]),
"popart_sigma": isv[RL_POPART_SIGMA_INDEX],
"v_target_max_3sigma": 3.0_f32 * isv[RL_POPART_SIGMA_INDEX],
"atom_max_over_3sigma": isv[RL_C51_V_MAX_INDEX]
/ (3.0_f32 * isv[RL_POPART_SIGMA_INDEX] + 1e-9_f32),
},
},
"replay_len": replay_len,
"rewards": {
"sum": reward_sum,
"max": reward_max,
"min": reward_min,
"abs_max": reward_abs_max,
"scaled_pre_clamp_max":
isv[RL_MAX_ABS_SCALED_REWARD_PRE_CLAMP_INDEX],
"pos_scaled_max":
isv[RL_POS_SCALED_REWARD_MAX_INDEX],
"pos_scaled_max_ema":
isv[RL_POS_SCALED_REWARD_MAX_EMA_INDEX],
"clip_rate_ema":
isv[RL_REWARD_CLAMP_CLIP_RATE_EMA_INDEX],
"c51_v_max":
isv[RL_C51_V_MAX_INDEX],
"c51_v_min":
isv[RL_C51_V_MIN_INDEX],
"q_distill_kl_ema":
isv[RL_Q_DISTILL_KL_EMA_INDEX],
"neg_scaled_max":
isv[RL_NEG_SCALED_REWARD_MAX_INDEX],
"neg_scaled_max_ema":
isv[RL_NEG_SCALED_REWARD_MAX_EMA_INDEX],
},
"ppo": {
"ratio_clamp_max": isv[RL_PPO_RATIO_CLAMP_MAX_INDEX],
"log_ratio_abs_max": isv[RL_PPO_LOG_RATIO_ABS_MAX_INDEX],
},
"streaming": {
"adv_var": {
"mean": isv[RL_ADV_VAR_STREAM_MEAN_INDEX],
"m2": isv[RL_ADV_VAR_STREAM_M2_INDEX],
"clamp": isv[RL_ADV_VAR_RATIO_CLAMP_INDEX],
},
"td_kurt": {
"mean": isv[RL_TD_KURT_STREAM_MEAN_INDEX],
"m2": isv[RL_TD_KURT_STREAM_M2_INDEX],
"m4": isv[RL_TD_KURT_STREAM_M4_INDEX],
"clamp": isv[RL_TD_KURTOSIS_CLAMP_INDEX],
},
},
"k_loop": {
"k_updates": self.last_k_updates,
"divisor": isv[RL_K_LOOP_DIVISOR_INDEX],
"max": isv[RL_K_LOOP_MAX_INDEX],
},
"isv_config": {
"reward_clamp_win": isv[RL_REWARD_CLAMP_WIN_INDEX],
"reward_clamp_loss": isv[RL_REWARD_CLAMP_LOSS_INDEX],
"kl_target": isv[RL_KL_TARGET_INDEX],
"improvement_threshold": isv[RL_IMPROVEMENT_THRESHOLD_INDEX],
"plateau_patience": isv[RL_PLATEAU_PATIENCE_INDEX],
"div_target": isv[RL_DIV_TARGET_INDEX],
"entropy_target_frac": isv[RL_ENTROPY_TARGET_FRAC_INDEX],
"kurt_lift_scale": isv[RL_KURT_LIFT_SCALE_INDEX],
"ppo_clamp_margin": isv[RL_PPO_CLAMP_MARGIN_INDEX],
"lr_warmup_steps": isv[RL_LR_WARMUP_STEPS_INDEX],
"lr_bootstrap": isv[RL_LR_BOOTSTRAP_INDEX],
"lr_min": isv[RL_LR_MIN_INDEX],
"lr_max": isv[RL_LR_MAX_INDEX],
"lr_loss_ema_alpha": isv[RL_LR_LOSS_EMA_ALPHA_INDEX],
"lr_decay_factor": isv[RL_LR_DECAY_FACTOR_INDEX],
"loss_lambda_aux": isv[RL_LOSS_LAMBDA_AUX_INDEX],
"schulman_tolerance": isv[RL_SCHULMAN_TOLERANCE_INDEX],
"schulman_adjust_rate": isv[RL_SCHULMAN_ADJUST_RATE_INDEX],
"stream_alpha": isv[RL_STREAM_ALPHA_INDEX],
"kurt_gaussian": isv[RL_KURT_GAUSSIAN_INDEX],
"kurt_noise_floor": isv[RL_KURT_NOISE_FLOOR_INDEX],
"tau_bootstrap": isv[RL_TAU_BOOTSTRAP_INDEX],
"eps_bootstrap": isv[RL_EPS_BOOTSTRAP_INDEX],
"rollout_bootstrap": isv[RL_ROLLOUT_BOOTSTRAP_INDEX],
"reward_scale_bootstrap":isv[RL_REWARD_SCALE_BOOTSTRAP_INDEX],
"ppo_ratio_clamp_bootstrap": isv[RL_PPO_RATIO_CLAMP_BOOTSTRAP_INDEX],
"reward_clamp_margin": isv[RL_REWARD_CLAMP_MARGIN_INDEX],
"reward_clamp_ratio": isv[RL_REWARD_CLAMP_RATIO_INDEX],
"reward_clamp_clip_rate_target": isv[RL_REWARD_CLAMP_CLIP_RATE_TARGET_INDEX],
"q_distill_lambda": isv[RL_Q_DISTILL_LAMBDA_INDEX],
"q_distill_temperature": isv[RL_Q_DISTILL_TEMPERATURE_INDEX],
"q_distill_kl_target": isv[RL_Q_DISTILL_KL_TARGET_INDEX],
"sac_alpha": isv[RL_SAC_ALPHA_INDEX],
"sac_entropy_target": isv[RL_SAC_ENTROPY_TARGET_INDEX],
"action_entropy_ema": isv[RL_ACTION_ENTROPY_EMA_INDEX],
"reward_scale_min": isv[RL_REWARD_SCALE_MIN_INDEX],
},
"q_pi_agree_ema": isv[RL_Q_ARG_VS_PI_AGREE_INDEX],
"controller_branch": {
"rollout_steps_input": isv[RL_ADVANTAGE_VAR_RATIO_EMA_INDEX],
"rollout_steps_target": isv[RL_ADV_VAR_RATIO_TARGET_INDEX],
"ppo_clip_input": isv[RL_KL_PI_EMA_INDEX],
"ppo_clip_target": 0.01f32,
"target_tau_input": isv[RL_Q_DIVERGENCE_EMA_INDEX],
"target_tau_target": 0.01f32,
"per_alpha_input": isv[RL_TD_KURTOSIS_EMA_INDEX],
"per_alpha_target": 0.6f32,
},
"done_count": done_count,
"action_hist": act_hist.to_vec(),
"action_entropy": action_entropy,
"position": {
"lots": inputs.position_lots_host,
},
"units": units_diag,
"trail": {
"fired_count_step": trail_fired_step,
"fired_count_total": inputs.trail_fired_total,
"tightened_count_step": tighten_step,
"loosened_count_step": loosen_step,
"tightened_count_total": inputs.trail_tighten_total,
"loosened_count_total": inputs.trail_loosen_total,
},
"pyramid": {
"added_count_step": pyramid_add_step,
"added_count_total": inputs.pyramid_added_total,
"units_distribution": inputs.pyramid_count_host,
"max_units_reached": inputs.pyramid_count_host.iter().any(|&c| c >= 4),
},
"partial_flat": {
"fired_count_step": partial_flat_step,
"fired_count_total": inputs.partial_flat_total,
"long_count_total": inputs.partial_flat_long_total,
"short_count_total": inputs.partial_flat_short_total,
"close_unit_index": inputs.close_unit_index_host,
},
"confidence_gate": {
"gated_count_step": conf_gate_step,
"gated_count_total": inputs.conf_gate_total,
},
"position_heat": {
"capped_count_step": heat_cap_step,
"capped_count_total": inputs.heat_cap_total,
"heat_max_lots": isv[RL_HEAT_CAP_MAX_LOTS_INDEX],
},
"anti_martingale": {
"outcome_ema": inputs.outcome_ema_host,
"kappa": isv[RL_ANTIMARTINGALE_KAPPA_INDEX],
},
"frd_gate": {
"gated_count_step": frd_gate_step,
"gated_count_total": inputs.frd_gate_total,
},
"trading": {
"pnl_cum_usd": inputs.pnl_cum_usd,
"total_trades": inputs.total_trades,
"win_rate": win_rate,
"avg_hold_steps": avg_hold,
"raw_reward_sum": inputs.raw_rewards_host.iter().sum::<f32>(),
},
"frd": frd_diag,
"popart": {
"mean": isv[RL_POPART_MEAN_INDEX],
"sigma": isv[RL_POPART_SIGMA_INDEX],
"var": isv[RL_POPART_VAR_INDEX],
},
"spectral": {
"norm_max_config": isv[RL_SPECTRAL_NORM_MAX_INDEX],
"decouple_lambda": isv[RL_SPECTRAL_DECOUPLE_LAMBDA_INDEX],
},
"q_bias": {
"ema": isv[RL_Q_BIAS_EMA_INDEX],
"correction": isv[RL_Q_BIAS_CORRECTION_INDEX],
},
"per_branch_lr": {
"scale_q": isv[RL_LR_SCALE_Q_INDEX],
"scale_pi": isv[RL_LR_SCALE_PI_INDEX],
"scale_v": isv[RL_LR_SCALE_V_INDEX],
"scale_iqn": isv[RL_LR_SCALE_IQN_INDEX],
},
"outcome_aux": {
"lambda": isv[RL_OUTCOME_AUX_LAMBDA_INDEX],
},
});
Ok(record)
}
}
/// Free-function entry point for the `grad_h_accumulate_scaled` kernel.