Compare commits
17 Commits
worktree-a
...
ml-alpha-a
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
baf971ba54 | ||
|
|
0c8cb6ad5b | ||
|
|
6c4945fe16 | ||
|
|
ad5b29e652 | ||
|
|
82572ff3bd | ||
|
|
d57bee0542 | ||
|
|
6d4a962e5c | ||
|
|
0fe825a8c5 | ||
|
|
1a05af803d | ||
|
|
7064c9269e | ||
|
|
5e4c2e62b6 | ||
|
|
39efacf77d | ||
|
|
6e0f568160 | ||
|
|
285d42aa7b | ||
|
|
448c5189cf | ||
|
|
b1ef6664ab | ||
|
|
083a88f7c3 |
@@ -215,10 +215,11 @@ fn test_tune_status_invalid_uuid() {
|
||||
#[ignore = "Ignored because it tries to launch terminal UI"]
|
||||
fn test_dashboard_command() {
|
||||
let mut cmd = Command::cargo_bin("fxt").unwrap();
|
||||
cmd.arg("dashboard")
|
||||
let _ = cmd.arg("dashboard")
|
||||
.timeout(std::time::Duration::from_secs(2))
|
||||
.assert();
|
||||
// Will timeout or fail trying to connect, but that's expected
|
||||
// Will timeout or fail trying to connect, but that's expected — assert result
|
||||
// intentionally discarded; this test verifies arg parsing reaches launch.
|
||||
}
|
||||
|
||||
/// Test version flag
|
||||
|
||||
@@ -103,6 +103,15 @@ const KERNELS: &[&str] = &[
|
||||
"rl_v_blend", // Phase 4.4 (2026-05-30): elementwise V_used = α V_scalar + (1−α) V_dq, α from ISV
|
||||
"rl_v_blend_alpha_controller", // Phase 4.4: ISV-adaptive Schulman-bounded controller on α from observed |V_dq − V_scalar| / |V_scalar| tracking ratio
|
||||
"rl_advantage_normalize", // Phase 4.5 (2026-05-30): per-batch advantage normalization (A − mean)/std with ε² variance floor — standard PPO practice, self-adaptive (no tuned params)
|
||||
"rl_signal_variance_update", // Adaptive controller floors (2026-05-30): Welford online variance for controller input EMAs; replaces hardcoded NOISE_FLOOR_FRAC constants in 8 controllers — sample_var = M² / (count-1); see spec 2026-05-30-adaptive-controller-floor-design
|
||||
"rl_win_rate_ema_update", // Adaptive risk management (2026-05-30): Layer 4 (Kelly) input — observed win_rate EMA from per-batch trade outcomes
|
||||
"rl_avg_win_loss_ema_update", // Adaptive risk management (2026-05-30): Layer 4 (Kelly) input — avg_win/avg_loss USD EMAs from per-batch closed-trade pnl
|
||||
"rl_inventory_variance_update", // Adaptive risk management (2026-05-30): Layer 3 (inventory β) input — |net_position| variance EMA across batches
|
||||
"rl_cmdp_constraints_check", // Adaptive risk management (2026-05-30): Layer 1 hard gates — session DD + cooldown + consecutive-loss tracking; writes override flags to ISV
|
||||
"rl_iqn_action_tau_controller", // Adaptive risk management (2026-05-30): Layer 2 — adapts IQN action-selection quantile τ from session drawdown signal
|
||||
"rl_inventory_beta_controller", // Adaptive risk management (2026-05-30): Layer 3 — adapts inventory penalty β from observed inventory variance vs reward magnitude
|
||||
"rl_kelly_fraction_controller", // Adaptive risk management (2026-05-30): Layer 4 — half-Kelly position-size multiplier from observed win_rate × R-multiple; warmup-gated
|
||||
"rl_eval_warmup_decay", // v9 (2026-05-31): defensive eval-boundary calibration — overrides Kelly/τ_min/entropy_min/ε_min during warmup, linearly decays back to normal; runs every step
|
||||
"rl_ensemble_action_value", // C51+IQN ensemble: E_ensemble = α×E_C51 + (1-α)×E_IQN; α from ISV[544]
|
||||
"rl_noisy_linear_forward", // NoisyNet: factored noisy linear forward — y = (mu_w + sigma_w ⊙ eps_w) × x + (mu_b + sigma_b ⊙ eps_b); state-dependent exploration for C51/IQN final projection
|
||||
"rl_noisy_linear_backward", // NoisyNet: factored noisy linear backward — grad_mu_w/sigma_w/mu_b/sigma_b per-batch scratch for reduce_axis0
|
||||
|
||||
@@ -40,6 +40,30 @@
|
||||
#define RL_ANTIMARTINGALE_MIN_INDEX 510
|
||||
#define RL_ANTIMARTINGALE_MAX_INDEX 511
|
||||
|
||||
// ── Layer 1 (CMDP hard constraints — spec 2026-05-30-adaptive-risk-management) ──
|
||||
// Session-level DD-triggered + cooldown are PER-BATCH (one buffer slot per
|
||||
// independent backtest session). The two ISV slots below carry only
|
||||
// canonical-summary aggregates (worst per-batch DD, max per-batch
|
||||
// cooldown) for diag visibility — gating reads the per-batch arrays.
|
||||
#define RL_MAX_OPEN_UNITS_INDEX 665
|
||||
#define RL_NET_INVENTORY_LIMIT_USD_INDEX 670
|
||||
|
||||
// ── Layer 4 (Kelly fraction sizing — spec 2026-05-30-adaptive-risk-management) ──
|
||||
#define RL_KELLY_FRACTION_INDEX 676
|
||||
|
||||
// Action indices — must match crate::rl::common::Action.
|
||||
#define ACTION_SHORT_LARGE 0
|
||||
#define ACTION_SHORT_SMALL 1
|
||||
#define ACTION_HOLD 2
|
||||
#define ACTION_FLAT_FROM_LONG 3
|
||||
#define ACTION_FLAT_FROM_SHORT 4
|
||||
#define ACTION_LONG_SMALL 5
|
||||
#define ACTION_LONG_LARGE 6
|
||||
#define ACTION_TRAIL_TIGHTEN 7
|
||||
#define ACTION_TRAIL_LOOSEN 8
|
||||
#define ACTION_HALF_FLAT_LONG 9
|
||||
#define ACTION_HALF_FLAT_SHORT 10
|
||||
|
||||
__device__ __forceinline__ int antimartingale_size(
|
||||
int base_size, float outcome_ema_b, const float* isv
|
||||
) {
|
||||
@@ -51,6 +75,24 @@ __device__ __forceinline__ int antimartingale_size(
|
||||
return (sized > 0) ? sized : 1;
|
||||
}
|
||||
|
||||
// Returns true if `action` would open or grow a one-sided position
|
||||
// (excludes closes, half-flats, holds, and trail mutations).
|
||||
__device__ __forceinline__ bool is_opening_action(int action) {
|
||||
return (action == ACTION_SHORT_LARGE) || (action == ACTION_SHORT_SMALL)
|
||||
|| (action == ACTION_LONG_SMALL) || (action == ACTION_LONG_LARGE);
|
||||
}
|
||||
|
||||
// Returns true if executing `action` from the current `net_pos_lots`
|
||||
// would INCREASE one-sided exposure (i.e. opens/grows the dominant side).
|
||||
// Closes/reverses/holds never trigger this — only same-side adds.
|
||||
__device__ __forceinline__ bool would_increase_exposure(int action, int net_pos_lots) {
|
||||
const bool longs = (action == ACTION_LONG_SMALL) || (action == ACTION_LONG_LARGE);
|
||||
const bool shorts = (action == ACTION_SHORT_SMALL) || (action == ACTION_SHORT_LARGE);
|
||||
if (longs && net_pos_lots >= 0) return true; // open or pyramid same side
|
||||
if (shorts && net_pos_lots <= 0) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
extern "C" __global__ void actions_to_market_targets(
|
||||
const int* __restrict__ actions, // [b_size]
|
||||
const unsigned char* __restrict__ pos_state, // [b_size * pos_bytes]
|
||||
@@ -64,19 +106,86 @@ extern "C" __global__ void actions_to_market_targets(
|
||||
const int* __restrict__ pyramid_units_count, // [b_size]
|
||||
const int* __restrict__ close_unit_index, // [b_size] (-1 = auto oldest)
|
||||
const float* __restrict__ outcome_ema, // [b_size] per-batch
|
||||
// CMDP per-batch state (Layer 1, spec 2026-05-30-adaptive-risk-management).
|
||||
// Each batch element is an independent session; one account hitting
|
||||
// its DD limit or cooldown does not gate the other 1023 sessions.
|
||||
const float* __restrict__ session_dd_triggered_per_batch, // [b_size]
|
||||
const float* __restrict__ cooldown_remaining_per_batch, // [b_size]
|
||||
int b_size,
|
||||
int pos_bytes
|
||||
) {
|
||||
const int b = blockIdx.x * blockDim.x + threadIdx.x;
|
||||
if (b >= b_size) return;
|
||||
|
||||
const int action = actions[b];
|
||||
int action = actions[b];
|
||||
const int position_lots = *(const int*)(pos_state + b * pos_bytes);
|
||||
const float outcome_ema_b = outcome_ema[b];
|
||||
|
||||
int side = 2; // default no-op
|
||||
int size = 0;
|
||||
|
||||
// ────────────────────────────────────────────────────────────────────
|
||||
// Layer 1 (CMDP hard constraints — spec 2026-05-30-adaptive-risk-management).
|
||||
// Session-level overrides force the account FLAT regardless of agent's
|
||||
// choice. Per-batch (one independent backtest session per b); a tripped
|
||||
// DD or active cooldown on session `b` does NOT lock out the others.
|
||||
//
|
||||
// Fix E (2026-05-30): emit a *closing* market order when the account
|
||||
// is non-flat — otherwise an existing losing position bleeds m2m for
|
||||
// the entire 500-step cooldown with no exit (trail stops are also
|
||||
// suppressed by this branch). Cluster alpha-rl-rjsjq step 371 showed
|
||||
// worst session_pnl growing monotonically -$3.7k → -$9.3k from
|
||||
// exactly this mechanism.
|
||||
//
|
||||
// Once the position is flat, subsequent cooldown steps emit a true
|
||||
// no-op (side=2, size=0) and the account waits for its recovery clock.
|
||||
// ────────────────────────────────────────────────────────────────────
|
||||
const bool dd_triggered = session_dd_triggered_per_batch[b] >= 0.5f;
|
||||
const bool in_cooldown = cooldown_remaining_per_batch[b] > 0.0f;
|
||||
if (dd_triggered || in_cooldown) {
|
||||
if (position_lots > 0) {
|
||||
// long → close via sell
|
||||
market_targets[b * 2 + 0] = 1;
|
||||
market_targets[b * 2 + 1] = position_lots;
|
||||
} else if (position_lots < 0) {
|
||||
// short → close via buy
|
||||
market_targets[b * 2 + 0] = 0;
|
||||
market_targets[b * 2 + 1] = -position_lots;
|
||||
} else {
|
||||
// already flat → true no-op
|
||||
market_targets[b * 2 + 0] = 2;
|
||||
market_targets[b * 2 + 1] = 0;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// Max-open-units check (per-batch dynamic count): refuse opens once
|
||||
// pyramid is at MAX. Closes still allowed (handled by remaining logic).
|
||||
{
|
||||
int active_units = 0;
|
||||
const int base_u = b * MAX_UNITS;
|
||||
for (int u = 0; u < MAX_UNITS; ++u) {
|
||||
if (unit_active[base_u + u]) active_units += 1;
|
||||
}
|
||||
const int max_open = (int)isv[RL_MAX_OPEN_UNITS_INDEX];
|
||||
if (active_units >= max_open && is_opening_action(action)) {
|
||||
action = ACTION_HOLD;
|
||||
}
|
||||
}
|
||||
|
||||
// Net-inventory cap (USD): block actions that grow one-sided exposure
|
||||
// beyond the limit. Uses bid/ask mid as the USD multiplier.
|
||||
{
|
||||
const float inv_limit = isv[RL_NET_INVENTORY_LIMIT_USD_INDEX];
|
||||
if (inv_limit > 0.0f) {
|
||||
const float mid = 0.5f * (bid_px[0] + ask_px[0]);
|
||||
const float net_pos_usd = (float)position_lots * mid;
|
||||
if (fabsf(net_pos_usd) > inv_limit && would_increase_exposure(action, position_lots)) {
|
||||
action = ACTION_HOLD;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (action == 0) {
|
||||
// ShortLarge — when already short, apply pyramid logic.
|
||||
if (position_lots < 0) {
|
||||
@@ -240,6 +349,27 @@ extern "C" __global__ void actions_to_market_targets(
|
||||
// actions 7, 8 (TrailTighten/Loosen): no fill — handled by
|
||||
// rl_trail_mutate kernel. The side=2 no-op default is correct here.
|
||||
|
||||
// ────────────────────────────────────────────────────────────────────
|
||||
// Layer 4 (Kelly-fraction sizing — spec 2026-05-30-adaptive-risk-management).
|
||||
// Scales the opening/sizing size by the observed-edge Kelly fraction.
|
||||
// Closes (FlatFrom* + HalfFlat*) and Hold are NOT scaled — risk
|
||||
// management for exiting positions is handled by Layer 1 + trail stops.
|
||||
// ────────────────────────────────────────────────────────────────────
|
||||
if (size > 0 && is_opening_action(action)) {
|
||||
const float kelly = isv[RL_KELLY_FRACTION_INDEX];
|
||||
// Bootstrap = 1.0 (full size); negative or zero → no commitment.
|
||||
const float scaled = (float)size * kelly;
|
||||
if (scaled <= 0.0f) {
|
||||
size = 0;
|
||||
side = 2;
|
||||
} else {
|
||||
// Ceil to preserve at least 1-lot when kelly is small-but-positive.
|
||||
int new_size = (int)ceilf(scaled);
|
||||
if (new_size < 1) new_size = 1;
|
||||
size = new_size;
|
||||
}
|
||||
}
|
||||
|
||||
market_targets[b * 2 + 0] = side;
|
||||
market_targets[b * 2 + 1] = size;
|
||||
}
|
||||
|
||||
@@ -30,8 +30,17 @@
|
||||
#define BLOCK_X 1024
|
||||
#define ADVANTAGE_VAR_FLOOR 1e-6f
|
||||
|
||||
// Adaptive controller floors (spec 2026-05-30-adaptive-controller-floor-design):
|
||||
// emit raw per-batch advantage variance to ISV[612] BEFORE the in-place
|
||||
// normalize. `rl_rollout_steps_controller` consumes this as its driving
|
||||
// signal — post-normalization variance is definitionally ~ε² floor under
|
||||
// Phase 4.5 and useless as a controller input. Pre-norm variance is the
|
||||
// real signal of "how much advantage spread the policy is producing".
|
||||
#define RL_ADV_VAR_PRE_NORM_INDEX 612
|
||||
|
||||
extern "C" __global__ void rl_advantage_normalize(
|
||||
float* __restrict__ advantage, // [B] in-place
|
||||
float* __restrict__ isv, // ISV bus — only ADV_VAR_PRE_NORM written
|
||||
int B
|
||||
) {
|
||||
const int tid = threadIdx.x;
|
||||
@@ -72,6 +81,10 @@ extern "C" __global__ void rl_advantage_normalize(
|
||||
__shared__ float s_inv_std;
|
||||
if (tid == 0) {
|
||||
const float var = s_var[0] / (float)B;
|
||||
// Emit raw pre-normalization variance for rl_rollout_steps_controller.
|
||||
// Done BEFORE we add the ε² floor — the controller wants the true
|
||||
// signal, not the numerical-stability-adjusted divisor.
|
||||
isv[RL_ADV_VAR_PRE_NORM_INDEX] = var;
|
||||
s_inv_std = rsqrtf(var + ADVANTAGE_VAR_FLOOR); // 1/sqrt(var + ε²)
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
77
crates/ml-alpha/cuda/rl_avg_win_loss_ema_update.cu
Normal file
77
crates/ml-alpha/cuda/rl_avg_win_loss_ema_update.cu
Normal file
@@ -0,0 +1,77 @@
|
||||
// rl_avg_win_loss_ema_update.cu — Layer 4 (Kelly) input EMA producer.
|
||||
//
|
||||
// Tracks observed per-trade avg_win and avg_loss magnitudes in USD as
|
||||
// EMAs. Kelly fraction uses R-multiple = avg_win / avg_loss together
|
||||
// with win_rate to derive the safe Kelly bet size.
|
||||
// Spec: docs/superpowers/specs/2026-05-30-adaptive-risk-management-design.md
|
||||
//
|
||||
// Inputs:
|
||||
// rewards[b] f32 — per-batch realized pnl delta (shaped) this step
|
||||
// dones[b] f32 — 1.0 if a trade closed this step, else 0.0
|
||||
//
|
||||
// Outcome derived inline: win = (done & reward > 0); loss = (done & reward < 0).
|
||||
//
|
||||
// Outputs:
|
||||
// ISV[RL_AVG_WIN_USD_EMA_INDEX = 678] (positive USD)
|
||||
// ISV[RL_AVG_LOSS_USD_EMA_INDEX = 679] (positive USD, magnitude)
|
||||
//
|
||||
// Bootstrap semantics per `pearl_first_observation_bootstrap`:
|
||||
// sentinel = 0.0; first non-zero closed-trade step replaces directly.
|
||||
//
|
||||
// Per `feedback_no_atomicadd`: single-thread single-block (sums sequentially).
|
||||
// Per `feedback_cpu_is_read_only`: pure device kernel.
|
||||
// Per `feedback_isv_for_adaptive_bounds`: EMA-α structural smoothing
|
||||
// parameter (same as ema_update_per_step convention).
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#define RL_AVG_WIN_USD_EMA_INDEX 678
|
||||
#define RL_AVG_LOSS_USD_EMA_INDEX 679
|
||||
#define EMA_ALPHA 0.05f
|
||||
|
||||
extern "C" __global__ void rl_avg_win_loss_ema_update(
|
||||
float* __restrict__ isv,
|
||||
const float* __restrict__ rewards, // [b_size] shaped pnl
|
||||
const float* __restrict__ dones, // [b_size] 1.0 = close
|
||||
int b_size
|
||||
) {
|
||||
if (threadIdx.x != 0 || threadIdx.y != 0 || threadIdx.z != 0) return;
|
||||
if (blockIdx.x != 0 || blockIdx.y != 0 || blockIdx.z != 0) return;
|
||||
|
||||
// Sum per-trade win/loss magnitudes across the batch.
|
||||
float sum_win = 0.0f;
|
||||
float sum_loss = 0.0f;
|
||||
int n_win = 0;
|
||||
int n_loss = 0;
|
||||
for (int b = 0; b < b_size; ++b) {
|
||||
if (dones[b] < 0.5f) continue;
|
||||
const float p = rewards[b];
|
||||
if (p > 0.0f) {
|
||||
sum_win += p;
|
||||
n_win += 1;
|
||||
} else if (p < 0.0f) {
|
||||
sum_loss += fabsf(p);
|
||||
n_loss += 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (n_win > 0) {
|
||||
const float step_avg = sum_win / (float)n_win;
|
||||
const float prev = isv[RL_AVG_WIN_USD_EMA_INDEX];
|
||||
if (prev == 0.0f) {
|
||||
isv[RL_AVG_WIN_USD_EMA_INDEX] = step_avg;
|
||||
} else {
|
||||
isv[RL_AVG_WIN_USD_EMA_INDEX] = (1.0f - EMA_ALPHA) * prev + EMA_ALPHA * step_avg;
|
||||
}
|
||||
}
|
||||
|
||||
if (n_loss > 0) {
|
||||
const float step_avg = sum_loss / (float)n_loss;
|
||||
const float prev = isv[RL_AVG_LOSS_USD_EMA_INDEX];
|
||||
if (prev == 0.0f) {
|
||||
isv[RL_AVG_LOSS_USD_EMA_INDEX] = step_avg;
|
||||
} else {
|
||||
isv[RL_AVG_LOSS_USD_EMA_INDEX] = (1.0f - EMA_ALPHA) * prev + EMA_ALPHA * step_avg;
|
||||
}
|
||||
}
|
||||
}
|
||||
163
crates/ml-alpha/cuda/rl_cmdp_constraints_check.cu
Normal file
163
crates/ml-alpha/cuda/rl_cmdp_constraints_check.cu
Normal file
@@ -0,0 +1,163 @@
|
||||
// rl_cmdp_constraints_check.cu — Layer 1 hard CMDP gates.
|
||||
//
|
||||
// Spec: docs/superpowers/specs/2026-05-30-adaptive-risk-management-design.md
|
||||
//
|
||||
// Maintains the session-level + cooldown state that downstream consumers
|
||||
// (actions_to_market_targets) read as override flags. Three independent
|
||||
// risk constraints are tracked here:
|
||||
//
|
||||
// 1. Session pnl accumulation + DD limit (sticky `triggered` flag)
|
||||
// 2. Cooldown counter decrement (after consecutive losses limit fires)
|
||||
// 3. Consecutive-loss tracking (sets cooldown when limit hit)
|
||||
//
|
||||
// Max-open-units + net-inventory checks are evaluated PER BATCH inside
|
||||
// actions_to_market_targets where per-batch unit state is available.
|
||||
// This kernel handles only session-wide state.
|
||||
//
|
||||
// Per `feedback_no_atomicadd`: single-thread single-block, sums sequentially.
|
||||
// Per `feedback_cpu_is_read_only`: pure device kernel.
|
||||
// Per `feedback_no_partial_refactor`: ISV slots must match isv_slots.rs.
|
||||
//
|
||||
// Launch config:
|
||||
// grid = (1, 1, 1)
|
||||
// block = (1, 1, 1)
|
||||
// smem = 0
|
||||
// stream = main RL stream (sequenced AFTER rl_fused_reward_pipeline
|
||||
// writes realized_pnl / outcomes, BEFORE actions_to_market_targets
|
||||
// reads override flags).
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#define RL_SESSION_PNL_USD_INDEX 662
|
||||
#define RL_SESSION_DD_LIMIT_USD_INDEX 663
|
||||
#define RL_SESSION_DD_TRIGGERED_INDEX 664
|
||||
#define RL_CONSEC_LOSS_LIMIT_INDEX 666
|
||||
#define RL_CONSEC_LOSS_COUNT_INDEX 667
|
||||
#define RL_COOLDOWN_REMAINING_STEPS_INDEX 668
|
||||
#define RL_COOLDOWN_DURATION_INDEX 669
|
||||
#define RL_SESSION_PNL_WORST_INDEX 684
|
||||
|
||||
// Per-batch session-pnl + consec-loss tracking. Each batch element is an
|
||||
// independent backtest "session" with its own $35k starting capital;
|
||||
// `session_pnl_per_batch[b]` is the running PnL of that session and
|
||||
// `consec_loss_per_batch[b]` is its losing-trade streak. The kernel
|
||||
// writes per-batch state out for `actions_to_market_targets` to read
|
||||
// (per-batch DD-triggered / cooldown gating) and also writes the
|
||||
// canonical summary slots so diag + IQN-τ keep a single-account view.
|
||||
extern "C" __global__ void rl_cmdp_constraints_check(
|
||||
float* __restrict__ isv,
|
||||
const float* __restrict__ rewards, // [b_size] shaped pnl
|
||||
const float* __restrict__ dones, // [b_size] 1.0 = close
|
||||
float* __restrict__ session_pnl_per_batch, // [b_size] IN/OUT
|
||||
float* __restrict__ consec_loss_per_batch, // [b_size] IN/OUT
|
||||
float* __restrict__ session_dd_triggered_per_batch,// [b_size] IN/OUT (0/1)
|
||||
float* __restrict__ cooldown_remaining_per_batch, // [b_size] IN/OUT
|
||||
int b_size
|
||||
) {
|
||||
if (threadIdx.x != 0 || threadIdx.y != 0 || threadIdx.z != 0) return;
|
||||
if (blockIdx.x != 0 || blockIdx.y != 0 || blockIdx.z != 0) return;
|
||||
|
||||
const float dd_limit = isv[RL_SESSION_DD_LIMIT_USD_INDEX];
|
||||
const float consec_lim = isv[RL_CONSEC_LOSS_LIMIT_INDEX];
|
||||
const float cool_dur = isv[RL_COOLDOWN_DURATION_INDEX];
|
||||
|
||||
// Summary aggregates (single-account view, written at end of loop).
|
||||
//
|
||||
// Fix B: IQN-τ reads `RL_SESSION_PNL_USD_INDEX`, so we expose the
|
||||
// MEAN-of-active-accounts there instead of the worst — one bad
|
||||
// account no longer drags the fleet's risk aversion to its floor.
|
||||
// The worst per-batch pnl mirrors to `RL_SESSION_PNL_WORST_INDEX`
|
||||
// for diag (helps spot fleet skew).
|
||||
float worst_pnl = 0.0f;
|
||||
float worst_consec = 0.0f;
|
||||
float any_dd_trig = 0.0f;
|
||||
float max_cool_remain = 0.0f;
|
||||
float active_pnl_sum = 0.0f;
|
||||
int n_active = 0;
|
||||
|
||||
for (int b = 0; b < b_size; ++b) {
|
||||
// Snapshot cooldown state BEFORE Section 1, so Section 2's
|
||||
// decrement only consumes what was already in flight from a
|
||||
// prior step. Without this, a fresh cooldown set in Section 1
|
||||
// would be immediately decremented in Section 2 same-step
|
||||
// (off-by-one — G1 caught this with cool=499 vs expected 500).
|
||||
const float cool_prev = cooldown_remaining_per_batch[b];
|
||||
|
||||
// ── 1. Per-batch session pnl + DD check ─────────────────
|
||||
// Fix A: when DD trips, also START a recovery cooldown — sticky
|
||||
// dd_triggered without a recovery path leaves accounts dead for
|
||||
// the rest of the fold, leaking variance from popart σ and
|
||||
// pinning IQN-τ at its floor.
|
||||
const float pnl_new = session_pnl_per_batch[b] + rewards[b];
|
||||
session_pnl_per_batch[b] = pnl_new;
|
||||
const bool was_triggered = session_dd_triggered_per_batch[b] >= 0.5f;
|
||||
if (pnl_new < dd_limit && !was_triggered) {
|
||||
session_dd_triggered_per_batch[b] = 1.0f;
|
||||
cooldown_remaining_per_batch[b] = cool_dur; // recovery clock
|
||||
}
|
||||
|
||||
// ── 2. Per-batch cooldown decrement + recovery reset ────
|
||||
// When the cooldown clock expires, give the account a fresh
|
||||
// start — matches [[feedback_surfer_philosophy_trading]]:
|
||||
// accept the wipeout, take the forced break, then get back on
|
||||
// the board. Otherwise the account contributes V_target ≈
|
||||
// γ·V(s'_unchanged) for the rest of the fold (zero-variance
|
||||
// dead-weight that compounds in popart σ).
|
||||
if (cool_prev > 0.0f) {
|
||||
const float cool_new = cool_prev - 1.0f;
|
||||
cooldown_remaining_per_batch[b] = cool_new;
|
||||
if (cool_new == 0.0f) {
|
||||
session_pnl_per_batch[b] = 0.0f;
|
||||
session_dd_triggered_per_batch[b] = 0.0f;
|
||||
// consec already cleared at its own limit-trip path
|
||||
}
|
||||
}
|
||||
|
||||
// ── 3. Per-batch consec-loss tracking on closes ─────────
|
||||
// r == 0 with done: ambiguous break-even, leave streak as-is.
|
||||
float consec = consec_loss_per_batch[b];
|
||||
if (dones[b] >= 0.5f) {
|
||||
const float r = rewards[b];
|
||||
if (r < 0.0f) consec += 1.0f;
|
||||
else if (r > 0.0f) consec = 0.0f;
|
||||
}
|
||||
if (consec >= consec_lim) {
|
||||
// Streak limit on THIS account — open its cooldown, reset
|
||||
// streak. Other accounts unaffected. Recovery reset above
|
||||
// will also clear it when the cooldown clock expires.
|
||||
cooldown_remaining_per_batch[b] = cool_dur;
|
||||
consec = 0.0f;
|
||||
}
|
||||
consec_loss_per_batch[b] = consec;
|
||||
|
||||
// ── Aggregates ──────────────────────────────────────────
|
||||
// Active = post-recovery state (after the reset path above), so
|
||||
// accounts that just recovered this step count as active again.
|
||||
const float final_pnl = session_pnl_per_batch[b];
|
||||
const float final_consec = consec_loss_per_batch[b];
|
||||
const float final_cool = cooldown_remaining_per_batch[b];
|
||||
const float final_triggered = session_dd_triggered_per_batch[b];
|
||||
if (final_pnl < worst_pnl) worst_pnl = final_pnl;
|
||||
if (final_consec > worst_consec) worst_consec = final_consec;
|
||||
if (final_triggered >= 0.5f) any_dd_trig = 1.0f;
|
||||
if (final_cool > max_cool_remain) max_cool_remain = final_cool;
|
||||
if (final_triggered < 0.5f) {
|
||||
active_pnl_sum += final_pnl;
|
||||
n_active += 1;
|
||||
}
|
||||
}
|
||||
|
||||
// ── Canonical summary slots ────────────────────────────────────
|
||||
// Fix B: IQN-τ reads `RL_SESSION_PNL_USD_INDEX` — feed it the
|
||||
// mean-of-active-accounts so τ reflects the fleet's typical DD
|
||||
// rather than one outlier's catastrophic loss. Worst stays
|
||||
// visible for diag.
|
||||
const float mean_active = (n_active > 0)
|
||||
? (active_pnl_sum / (float)n_active)
|
||||
: 0.0f;
|
||||
isv[RL_SESSION_PNL_USD_INDEX] = mean_active;
|
||||
isv[RL_SESSION_PNL_WORST_INDEX] = worst_pnl;
|
||||
isv[RL_SESSION_DD_TRIGGERED_INDEX] = any_dd_trig;
|
||||
isv[RL_CONSEC_LOSS_COUNT_INDEX] = worst_consec;
|
||||
isv[RL_COOLDOWN_REMAINING_STEPS_INDEX] = max_cool_remain;
|
||||
}
|
||||
@@ -66,6 +66,13 @@ extern "C" __global__ void rl_confidence_gate(
|
||||
const int action = actions[b];
|
||||
if (action == ACTION_HOLD) return;
|
||||
|
||||
// Gate is opening-only: a non-flat position has already cleared
|
||||
// the confidence check, so overriding it to Hold would suppress
|
||||
// legitimate adds/exits. pos_state[0..4] = position_lots:i32.
|
||||
const int position_lots =
|
||||
*reinterpret_cast<const int*>(pos_state + b * pos_bytes);
|
||||
if (position_lots != 0) return;
|
||||
|
||||
const float threshold = isv[RL_CONF_GATE_THRESHOLD_INDEX];
|
||||
const float lambda = isv[RL_CONF_GATE_LAMBDA_INDEX];
|
||||
const float sigma_norm = isv[RL_CONF_GATE_SIGMA_NORM_INDEX];
|
||||
|
||||
@@ -46,13 +46,35 @@
|
||||
|
||||
#define RL_ENTROPY_COEF_INDEX 403
|
||||
#define N_ACTIONS 11
|
||||
#define COEF_MIN 0.0f
|
||||
#define COEF_MAX 0.5f
|
||||
// COEF MIN/MAX clamp bounds are now ISV-driven per the 2026-05-30
|
||||
// clamp-bound extension. Runtime-tunable + visible in diag.
|
||||
#define RL_ENTROPY_COEF_MIN_INDEX 645
|
||||
#define RL_ENTROPY_COEF_MAX_INDEX 646
|
||||
// ISV-driven entropy-target fraction per `feedback_isv_for_adaptive_bounds`.
|
||||
// Default 0.7 (70% of ln(N_ACTIONS) = "explore but not too randomly").
|
||||
// Seeded by rl_isv_write at trainer init.
|
||||
#define RL_ENTROPY_TARGET_FRAC_INDEX 458
|
||||
#define WIENER_ALPHA_FLOOR 0.4f
|
||||
// Wiener-α floor — shared across 9 controllers (slot 659).
|
||||
#define RL_WIENER_ALPHA_FLOOR_INDEX 659
|
||||
|
||||
// Adaptive controller floors (spec 2026-05-30-adaptive-controller-floor-design):
|
||||
// noise floor derives from observed entropy_observed_ema variance via
|
||||
// Welford triples. Replaces the hardcoded `0.5f` emergency-bypass fraction
|
||||
// with `max(h_target × 0.5, h_target − 2σ)` — under Phase 4.5 the entropy
|
||||
// distribution narrows around the SAC target and the hardcoded 0.5×h_target
|
||||
// emergency gate never fires, leaving the Wiener blend to drag coef to MIN
|
||||
// (alpha-rl-... fold 0 confirmed entropy_coef stuck at 0.01).
|
||||
// Asymmetric Schulman semantics: coef RAISES on a single below-target
|
||||
// observation (entropy collapse = safety signal, act fast); coef DROPS
|
||||
// only after N consecutive above-target observations (healthy entropy =
|
||||
// drift coef down patiently).
|
||||
#define RL_ENTROPY_OBS_VAR_COUNT_INDEX 600
|
||||
#define RL_ENTROPY_OBS_VAR_M2_INDEX 602
|
||||
#define RL_ENTROPY_OBS_BELOW_COUNT_INDEX 603
|
||||
#define RL_SCHULMAN_TOLERANCE_INDEX 468
|
||||
#define NOISE_FLOOR_TARGET_FRAC 0.5f // floor ≥ 50% of target
|
||||
#define NOISE_FLOOR_STD_MULTIPLIER 2.0f // floor ≥ 2σ of observed signal
|
||||
#define WIDEN_PATIENCE_CONSECUTIVE 3.0f // descent requires N above-band steps
|
||||
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────
|
||||
@@ -101,10 +123,12 @@ extern "C" __global__ void rl_entropy_coef_controller(
|
||||
// At h_obs = 0 (total collapse): coef = COEF_MAX (emergency)
|
||||
// At h_obs > h_target: coef = COEF_FLOOR (no penalty for exploring)
|
||||
const float COEF_FLOOR = 0.01f;
|
||||
const float coef_min = isv[RL_ENTROPY_COEF_MIN_INDEX];
|
||||
const float coef_max = isv[RL_ENTROPY_COEF_MAX_INDEX];
|
||||
const float deficit_frac = fmaxf(0.0f,
|
||||
fminf((h_target - entropy_observed_ema) / fmaxf(h_target, 1e-6f), 1.0f));
|
||||
float coef_target = COEF_FLOOR + (COEF_MAX - COEF_FLOOR) * deficit_frac;
|
||||
coef_target = fmaxf(COEF_MIN, fminf(coef_target, COEF_MAX));
|
||||
float coef_target = COEF_FLOOR + (coef_max - COEF_FLOOR) * deficit_frac;
|
||||
coef_target = fmaxf(coef_min, fminf(coef_target, coef_max));
|
||||
|
||||
// Bootstrap on sentinel 0.0 per pearl_first_observation_bootstrap:
|
||||
// first emit replaces directly with the computed target. At cold
|
||||
@@ -117,17 +141,56 @@ extern "C" __global__ void rl_entropy_coef_controller(
|
||||
return;
|
||||
}
|
||||
|
||||
// Emergency: bypass Wiener blend when entropy is below 50% of target.
|
||||
// The slow blend can't react fast enough to prevent Hold collapse.
|
||||
if (entropy_observed_ema < h_target * 0.5f && entropy_observed_ema > 0.0f) {
|
||||
// Adaptive noise floor — signal-driven per
|
||||
// `pearl_zscore_normalization_for_magnitude_asymmetric_signals` and
|
||||
// `feedback_adaptive_not_tuned`. Replaces hardcoded `h_target × 0.5`
|
||||
// emergency-bypass threshold with adaptive `max(h_target × 0.5,
|
||||
// h_target − 2σ_observed)` so the emergency path still fires when
|
||||
// entropy drops more than 2σ below its observed mean even if the
|
||||
// distribution is so narrow that 50% × h_target is unreachable.
|
||||
// Welford sample variance = M² / (count − 1) when count > 1.
|
||||
const float h_count = isv[RL_ENTROPY_OBS_VAR_COUNT_INDEX];
|
||||
const float h_var = (h_count > 1.0f)
|
||||
? isv[RL_ENTROPY_OBS_VAR_M2_INDEX] / (h_count - 1.0f)
|
||||
: 0.0f;
|
||||
const float h_std = sqrtf(h_var);
|
||||
const float emergency_floor = fmaxf(h_target * NOISE_FLOOR_TARGET_FRAC,
|
||||
h_target - h_std * NOISE_FLOOR_STD_MULTIPLIER);
|
||||
|
||||
// Asymmetric Schulman (spec 2026-05-30): RAISE coef on a single
|
||||
// below-emergency-floor observation — entropy collapse is a safety
|
||||
// signal we act on fast. DROP coef only after N consecutive
|
||||
// above-band observations so a single noisy spike of healthy
|
||||
// entropy can't drag coef toward MIN. Below-counter is on the
|
||||
// "healthy entropy" direction here (entropy_observed > h_target ×
|
||||
// tolerance), distinct from the canonical Schulman semantics where
|
||||
// below-counter means "input below band."
|
||||
if (entropy_observed_ema > 0.0f && entropy_observed_ema < emergency_floor) {
|
||||
isv[RL_ENTROPY_COEF_INDEX] = coef_target;
|
||||
isv[RL_ENTROPY_OBS_BELOW_COUNT_INDEX] = 0.0f;
|
||||
return;
|
||||
}
|
||||
|
||||
// Wiener-α blend with floor per pearl_wiener_alpha_floor_for_nonstationary.
|
||||
const float a = fmaxf(alpha, WIENER_ALPHA_FLOOR);
|
||||
float coef_new = (1.0f - a) * coef_prev + a * coef_target;
|
||||
const float tolerance = isv[RL_SCHULMAN_TOLERANCE_INDEX];
|
||||
const float wiener_a = fmaxf(alpha, isv[RL_WIENER_ALPHA_FLOOR_INDEX]);
|
||||
float coef_new = (1.0f - wiener_a) * coef_prev + wiener_a * coef_target;
|
||||
coef_new = fmaxf(coef_min, fminf(coef_new, coef_max));
|
||||
|
||||
if (coef_new < coef_prev && entropy_observed_ema > h_target * tolerance) {
|
||||
// Healthy-entropy direction — require N consecutive observations
|
||||
// before letting the Wiener blend drag coef downward. Single noisy
|
||||
// high-entropy spike must not erode the entropy bonus.
|
||||
const float new_count = isv[RL_ENTROPY_OBS_BELOW_COUNT_INDEX] + 1.0f;
|
||||
isv[RL_ENTROPY_OBS_BELOW_COUNT_INDEX] = new_count;
|
||||
if (new_count < WIDEN_PATIENCE_CONSECUTIVE) {
|
||||
// Hold coef at previous value until patience accumulates.
|
||||
isv[RL_ENTROPY_COEF_INDEX] = coef_prev;
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
// Either entropy unhealthy (coef rising) or in-band — reset patience.
|
||||
isv[RL_ENTROPY_OBS_BELOW_COUNT_INDEX] = 0.0f;
|
||||
}
|
||||
|
||||
coef_new = fmaxf(COEF_MIN, fminf(coef_new, COEF_MAX));
|
||||
isv[RL_ENTROPY_COEF_INDEX] = coef_new;
|
||||
}
|
||||
|
||||
105
crates/ml-alpha/cuda/rl_eval_warmup_decay.cu
Normal file
105
crates/ml-alpha/cuda/rl_eval_warmup_decay.cu
Normal file
@@ -0,0 +1,105 @@
|
||||
// rl_eval_warmup_decay.cu — defensive eval-boundary calibration (v9).
|
||||
//
|
||||
// Spec: docs/superpowers/specs/2026-05-31-v9-defensive-eval-boundary-calibration.md
|
||||
// Pearl: pearl_adaptive_carryover_discipline
|
||||
//
|
||||
// At every regime boundary (train→eval, fold transition), the trainer
|
||||
// calls `reset_session_state` which sets `RL_EVAL_WARMUP_REMAINING_INDEX`
|
||||
// to the configured warmup duration. This kernel runs once per step
|
||||
// thereafter; while the counter is positive, it overrides four
|
||||
// risk-sizing controller floors with conservative defensive values.
|
||||
//
|
||||
// Schedule (let R = warmup_remaining, W = warmup_steps, D = decay_steps):
|
||||
//
|
||||
// R > D full defensive overrides written
|
||||
// 0 < R ≤ D linear interpolation: defensive (1) → normal (0)
|
||||
// R = 0 normal values written once, counter goes to -1
|
||||
// R < 0 no-op (warmup completed, controllers run unimpeded)
|
||||
//
|
||||
// The override slots affected are read-only by their consuming controllers
|
||||
// (Kelly safety_frac, IQN τ_min, entropy_coef_min, PPO clip ε_min), so
|
||||
// last-writer-wins applies cleanly when this kernel runs AFTER the
|
||||
// controller batch.
|
||||
//
|
||||
// Per `feedback_no_atomicadd`: single-thread single-block, no atomics.
|
||||
// Per `feedback_cpu_is_read_only`: pure device kernel.
|
||||
// Per `feedback_isv_for_adaptive_bounds`: defensive override values, normal
|
||||
// target values, warmup/decay durations are all ISV-driven.
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#define RL_EVAL_WARMUP_REMAINING_INDEX 685
|
||||
#define RL_EVAL_WARMUP_STEPS_CONFIG_INDEX 686
|
||||
#define RL_EVAL_WARMUP_DECAY_STEPS_CONFIG_INDEX 687
|
||||
#define RL_EVAL_KELLY_SAFETY_DEFENSIVE_INDEX 688
|
||||
#define RL_EVAL_IQN_TAU_MIN_DEFENSIVE_INDEX 689
|
||||
#define RL_EVAL_ENTROPY_COEF_MIN_DEFENSIVE_INDEX 690
|
||||
#define RL_EVAL_PPO_CLIP_EPS_MIN_DEFENSIVE_INDEX 691
|
||||
#define RL_EVAL_KELLY_SAFETY_NORMAL_INDEX 692
|
||||
#define RL_EVAL_IQN_TAU_MIN_NORMAL_INDEX 693
|
||||
#define RL_EVAL_ENTROPY_COEF_MIN_NORMAL_INDEX 694
|
||||
#define RL_EVAL_PPO_CLIP_EPS_MIN_NORMAL_INDEX 695
|
||||
|
||||
// Consumer slots (controller floors that this kernel overrides during warmup).
|
||||
#define RL_KELLY_SAFETY_FRAC_INDEX 680
|
||||
#define RL_IQN_ACTION_TAU_MIN_INDEX 672
|
||||
#define RL_ENTROPY_COEF_MIN_INDEX 645
|
||||
#define RL_PPO_CLIP_EPS_MIN_INDEX 640
|
||||
|
||||
extern "C" __global__ void rl_eval_warmup_decay(
|
||||
float* __restrict__ isv
|
||||
) {
|
||||
if (threadIdx.x != 0 || threadIdx.y != 0 || threadIdx.z != 0) return;
|
||||
if (blockIdx.x != 0 || blockIdx.y != 0 || blockIdx.z != 0) return;
|
||||
|
||||
const float remaining = isv[RL_EVAL_WARMUP_REMAINING_INDEX];
|
||||
|
||||
// Post-warmup steady state: do nothing. The normal floors stay at
|
||||
// their bootstrap values (or whatever the last-warmup-step wrote).
|
||||
if (remaining < 0.0f) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Read configured durations + override / normal values.
|
||||
const float warmup_steps = isv[RL_EVAL_WARMUP_STEPS_CONFIG_INDEX];
|
||||
const float decay_steps = isv[RL_EVAL_WARMUP_DECAY_STEPS_CONFIG_INDEX];
|
||||
|
||||
const float kelly_def = isv[RL_EVAL_KELLY_SAFETY_DEFENSIVE_INDEX];
|
||||
const float tau_def = isv[RL_EVAL_IQN_TAU_MIN_DEFENSIVE_INDEX];
|
||||
const float entropy_def = isv[RL_EVAL_ENTROPY_COEF_MIN_DEFENSIVE_INDEX];
|
||||
const float ppo_def = isv[RL_EVAL_PPO_CLIP_EPS_MIN_DEFENSIVE_INDEX];
|
||||
|
||||
const float kelly_norm = isv[RL_EVAL_KELLY_SAFETY_NORMAL_INDEX];
|
||||
const float tau_norm = isv[RL_EVAL_IQN_TAU_MIN_NORMAL_INDEX];
|
||||
const float entropy_norm = isv[RL_EVAL_ENTROPY_COEF_MIN_NORMAL_INDEX];
|
||||
const float ppo_norm = isv[RL_EVAL_PPO_CLIP_EPS_MIN_NORMAL_INDEX];
|
||||
|
||||
// Phase selection:
|
||||
// pure warmup remaining > decay_steps → blend = 1.0 (full defensive)
|
||||
// decay phase 0 < remaining ≤ decay_steps → blend = remaining/decay_steps
|
||||
// final boundary remaining == 0 → blend = 0.0 (full normal)
|
||||
float blend;
|
||||
if (remaining > decay_steps) {
|
||||
blend = 1.0f;
|
||||
} else if (decay_steps > 0.0f) {
|
||||
blend = remaining / decay_steps; // 1.0 → 0.0 as remaining → 0
|
||||
} else {
|
||||
// Defensive: decay_steps configured to 0 means no decay phase.
|
||||
blend = (remaining > 0.0f) ? 1.0f : 0.0f;
|
||||
}
|
||||
|
||||
// Linear interpolation: blend × defensive + (1−blend) × normal.
|
||||
isv[RL_KELLY_SAFETY_FRAC_INDEX] = blend * kelly_def + (1.0f - blend) * kelly_norm;
|
||||
isv[RL_IQN_ACTION_TAU_MIN_INDEX] = blend * tau_def + (1.0f - blend) * tau_norm;
|
||||
isv[RL_ENTROPY_COEF_MIN_INDEX] = blend * entropy_def + (1.0f - blend) * entropy_norm;
|
||||
isv[RL_PPO_CLIP_EPS_MIN_INDEX] = blend * ppo_def + (1.0f - blend) * ppo_norm;
|
||||
|
||||
// Decrement counter for next step. When it crosses 0, the next step
|
||||
// sees remaining = -1 and returns early (controllers run unimpeded).
|
||||
isv[RL_EVAL_WARMUP_REMAINING_INDEX] = remaining - 1.0f;
|
||||
// Spec note: warmup_steps is used only by the trainer to set the
|
||||
// initial counter value; this kernel reads warmup_steps only to make
|
||||
// the `remaining > decay_steps` comparison meaningful for non-default
|
||||
// configurations (e.g., shorter warmup with same decay).
|
||||
(void) warmup_steps;
|
||||
}
|
||||
@@ -1,21 +1,29 @@
|
||||
// rl_fused_controllers.cu — single-kernel fusion of 10 RL ISV controllers.
|
||||
// rl_fused_controllers.cu — single-kernel fusion of 13 RL ISV controllers.
|
||||
//
|
||||
// Eliminates 9 kernel-launch overheads (~40-80μs/step) by running all
|
||||
// Eliminates 12 kernel-launch overheads (~55-100μs/step) by running all
|
||||
// controllers sequentially in one thread. Each controller's logic is
|
||||
// copied verbatim from its standalone .cu file. The standalone files
|
||||
// are retained for documentation and individual testing.
|
||||
// MIRRORED from its standalone .cu file (post-2026-05-30 adaptive
|
||||
// controller-floor + risk-management refactors): adaptive noise floor
|
||||
// derived from Welford variance, asymmetric Schulman patience counters,
|
||||
// ISV-driven clamp bounds + Wiener-α floor. Per `feedback_no_partial_refactor`
|
||||
// and the "fused kernel must be semantically equivalent to running all
|
||||
// individual controllers in sequence" contract, every branch below matches
|
||||
// its standalone counterpart's ISV reads, math, and counter semantics.
|
||||
//
|
||||
// Controllers fused (in execution order):
|
||||
// 1. rl_gamma_controller ISV[400] ← ISV[input_slots[0]]
|
||||
// 2. rl_target_tau_controller ISV[401] ← ISV[input_slots[1]]
|
||||
// 3. rl_ppo_clip_controller ISV[402] ← ISV[input_slots[2]]
|
||||
// 4. rl_entropy_coef_controller ISV[403] ← ISV[input_slots[3]]
|
||||
// 5. rl_rollout_steps_controller ISV[404] ← ISV[input_slots[4]]
|
||||
// 5. rl_rollout_steps_controller ISV[404] ← ISV[input_slots[4]] (caller passes ADV_VAR_PRE_NORM=612)
|
||||
// 6. rl_per_alpha_controller ISV[405] ← ISV[input_slots[5]]
|
||||
// 7. rl_reward_scale_controller ISV[406] ← ISV[input_slots[6]]
|
||||
// 8. rl_ppo_ratio_clamp_controller ISV[440] ← ISV[402] (reads ε just written)
|
||||
// 9. rl_gate_threshold_controller ISV[512,516,517] ← dones[B]
|
||||
// 10. rl_q_distill_lambda_controller ISV[486] ← ISV[488]
|
||||
// 11. rl_iqn_action_tau_controller ISV[671] ← ISV[662] (Layer 2, spec 2026-05-30-adaptive-risk-management)
|
||||
// 12. rl_inventory_beta_controller ISV[674] ← ISV[675,614] (Layer 3)
|
||||
// 13. rl_kelly_fraction_controller ISV[676] ← ISV[677,678,679,680,681,660] (Layer 4)
|
||||
//
|
||||
// Per `feedback_no_atomicadd`: single thread, no atomics.
|
||||
// Per `feedback_cpu_is_read_only`: all state in ISV, no host roundtrip.
|
||||
@@ -24,65 +32,110 @@
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════
|
||||
// ISV slot indices — mirrored from individual controller headers.
|
||||
// All MIN/MAX/threshold/rate constants are ISV-resident per the 2026-05-30
|
||||
// adaptive controller-floor design. The few remaining `#define`s below are
|
||||
// architectural exemptions (physics/math constants, structural safeguards).
|
||||
// ═══════════════════════════════════════════════════════════════════════
|
||||
|
||||
// --- 1. Gamma controller ---
|
||||
// --- Output slots (controller emits) ---
|
||||
#define RL_GAMMA_INDEX 400
|
||||
#define GAMMA_MIN 0.995f
|
||||
#define GAMMA_MAX 0.999f
|
||||
#define RL_TARGET_TAU_INDEX 401
|
||||
#define RL_PPO_CLIP_INDEX 402
|
||||
#define RL_ENTROPY_COEF_INDEX 403
|
||||
#define RL_N_ROLLOUT_STEPS_INDEX 404
|
||||
#define RL_PER_ALPHA_INDEX 405
|
||||
#define RL_REWARD_SCALE_INDEX 406
|
||||
#define RL_PPO_RATIO_CLAMP_MAX_INDEX 440
|
||||
#define RL_Q_DISTILL_LAMBDA_INDEX 486
|
||||
|
||||
// --- 1. Gamma controller (Special G) ---
|
||||
// γ MIN now adaptive via RL_GAMMA_MIN_ADAPTIVE_INDEX (slot 613) derived
|
||||
// from the Welford mean of trade duration. γ MAX is ISV-resident.
|
||||
#define RL_GAMMA_MAX_INDEX 649
|
||||
#define RL_GAMMA_MIN_ADAPTIVE_INDEX 613
|
||||
#define RL_TRADE_DUR_VAR_COUNT_INDEX 608
|
||||
// Reward-floor fix (2026-05-30 follow-up): track REAL cumulative closed
|
||||
// trades via dones-sum accumulation. The Welford trade-duration counter
|
||||
// increments per closed-trade STEP, not per closed trade — at b=1024 with
|
||||
// ~7 dones/step this released the floor after only ~700 actual trades.
|
||||
#define RL_CUMULATIVE_DONES_INDEX 660
|
||||
#define RL_MIN_TRADES_FOR_RELEASE_INDEX 661
|
||||
#define RL_TRADE_DUR_VAR_MEAN_INDEX 609
|
||||
#define RL_MEAN_TRADE_DURATION_EMA_INDEX 417
|
||||
#define GAMMA_MIN_ABSOLUTE 0.9f // architectural: don't degenerate to bandit
|
||||
#define HORIZON_MULTIPLIER 2.0f // architectural: effective horizon = 2× transaction horizon
|
||||
#define WELFORD_WARMUP_OBS 100.0f // architectural: confidence threshold, not magnitude calibration
|
||||
#define HORIZON_FLOOR 10.0f // architectural: `1/d` numerical-stability floor
|
||||
|
||||
// --- 2. Target tau controller ---
|
||||
#define RL_TARGET_TAU_INDEX 401
|
||||
#define TAU_MIN 0.001f
|
||||
#define RL_TARGET_TAU_MIN_INDEX 642
|
||||
#define RL_TARGET_TAU_MAX_INDEX 573
|
||||
#define RL_TAU_BOOTSTRAP_INDEX 473
|
||||
#define RL_DIV_TARGET_INDEX 457
|
||||
#define DIV_NOISE_FLOOR_FRAC 0.01f
|
||||
#define RL_Q_DIV_VAR_COUNT_INDEX 592
|
||||
#define RL_Q_DIV_VAR_M2_INDEX 594
|
||||
#define RL_Q_DIV_BELOW_COUNT_INDEX 595
|
||||
|
||||
// --- 3. PPO clip controller ---
|
||||
#define RL_PPO_CLIP_INDEX 402
|
||||
#define EPS_MIN 0.05f
|
||||
#define EPS_MAX 0.5f
|
||||
// --- 3. PPO clip controller (canonical pattern) ---
|
||||
#define RL_PPO_CLIP_EPS_MIN_INDEX 640
|
||||
#define RL_PPO_CLIP_EPS_MAX_INDEX 641
|
||||
#define RL_EPS_BOOTSTRAP_INDEX 474
|
||||
#define RL_KL_TARGET_INDEX 454
|
||||
#define KL_NOISE_FLOOR_FRAC 0.01f
|
||||
#define RL_KL_PI_VAR_COUNT_INDEX 588
|
||||
#define RL_KL_PI_VAR_M2_INDEX 590
|
||||
#define RL_KL_PI_BELOW_COUNT_INDEX 591
|
||||
|
||||
// --- 4. Entropy coef controller ---
|
||||
#define RL_ENTROPY_COEF_INDEX 403
|
||||
#define N_ACTIONS 11
|
||||
#define COEF_MIN 0.0f
|
||||
#define COEF_MAX 0.5f
|
||||
#define N_ACTIONS 11 // architectural: matches Q head / action space dim
|
||||
#define RL_ENTROPY_COEF_MIN_INDEX 645
|
||||
#define RL_ENTROPY_COEF_MAX_INDEX 646
|
||||
#define RL_ENTROPY_TARGET_FRAC_INDEX 458
|
||||
#define RL_ENTROPY_OBS_VAR_COUNT_INDEX 600
|
||||
#define RL_ENTROPY_OBS_VAR_M2_INDEX 602
|
||||
#define RL_ENTROPY_OBS_BELOW_COUNT_INDEX 603
|
||||
|
||||
// --- 5. Rollout steps controller ---
|
||||
#define RL_N_ROLLOUT_STEPS_INDEX 404
|
||||
#define ROLLOUT_MIN 256.0f
|
||||
#define ROLLOUT_MAX 8192.0f
|
||||
// Input slot is now RL_ADV_VAR_PRE_NORM_INDEX=612 (caller-supplied via
|
||||
// input_slots[4]) since post-Phase-4.5 advantage variance is definitionally
|
||||
// near-zero and useless. See `rl_rollout_steps_controller.cu` comments.
|
||||
#define RL_ROLLOUT_MIN_INDEX 643
|
||||
#define RL_ROLLOUT_MAX_INDEX 644
|
||||
#define RL_ROLLOUT_BOOTSTRAP_INDEX 475
|
||||
#define RL_ADV_VAR_RATIO_TARGET_INDEX 449
|
||||
#define ADV_VAR_RATIO_NOISE_FLOOR_FRAC 0.01f
|
||||
#define RL_ADV_VAR_VAR_COUNT_INDEX 596
|
||||
#define RL_ADV_VAR_VAR_M2_INDEX 598
|
||||
#define RL_ADV_VAR_BELOW_COUNT_INDEX 599
|
||||
|
||||
// --- 6. PER alpha controller ---
|
||||
#define RL_PER_ALPHA_INDEX 405
|
||||
#define PER_ALPHA_MIN 0.3f
|
||||
#define PER_ALPHA_MAX 1.0f
|
||||
#define RL_PER_ALPHA_MIN_INDEX 647
|
||||
#define RL_PER_ALPHA_MAX_INDEX 648
|
||||
#define RL_KURT_GAUSSIAN_INDEX 471
|
||||
#define RL_KURT_NOISE_FLOOR_INDEX 472
|
||||
#define RL_KURT_LIFT_SCALE_INDEX 459
|
||||
#define RL_TD_KURT_VAR_COUNT_INDEX 604
|
||||
#define RL_TD_KURT_VAR_M2_INDEX 606
|
||||
#define RL_TD_KURT_BELOW_COUNT_INDEX 607
|
||||
|
||||
// --- 7. Reward scale controller ---
|
||||
#define RL_REWARD_SCALE_INDEX 406
|
||||
// --- 7. Reward scale controller (Special R) ---
|
||||
#define RL_REWARD_SCALE_MIN_INDEX 492
|
||||
#define REWARD_SCALE_MAX 1e3f
|
||||
#define RL_REWARD_SCALE_BOOTSTRAP_INDEX 476
|
||||
#define EPS_PNL 1e-3f
|
||||
#define RL_REWARD_MAGNITUDE_EMA_INDEX 614
|
||||
#define EPS_PNL 1e-3f // architectural: div-by-zero guard
|
||||
#define REWARD_SCALE_MAX 1e3f // architectural: 5-order-of-magnitude runaway guard
|
||||
#define BOOTSTRAP_FRACTION_FLOOR 0.1f // architectural: early-training spiral guard (spec §Special R)
|
||||
#define MIN_TRADES_FOR_RELEASE 100.0f // architectural: confidence threshold for releasing bootstrap floor
|
||||
#define ASYM_DECREASE_RATE 1.05f // architectural: asymmetric per-step decrease cap
|
||||
|
||||
// --- 8. PPO ratio clamp controller ---
|
||||
#define RL_PPO_RATIO_CLAMP_MAX_INDEX 440
|
||||
#define RL_PPO_RATIO_CLAMP_BOOTSTRAP_INDEX 477
|
||||
#define RL_PPO_CLAMP_MARGIN_INDEX 460
|
||||
#define PPO_RATIO_CLAMP_MIN_OUT 2.0f
|
||||
#define PPO_RATIO_CLAMP_MAX_OUT 1000.0f
|
||||
// MIN_OUT now adaptive from observed ratio variance; MAX_OUT is ISV-resident.
|
||||
#define RL_PPO_RATIO_CLAMP_MAX_ADAPTIVE_INDEX 629
|
||||
#define RL_PPO_RATIO_CLAMP_BOOTSTRAP_INDEX 477
|
||||
#define RL_PPO_CLAMP_MARGIN_INDEX 460
|
||||
#define RL_PPO_RATIO_VAR_COUNT_INDEX 630
|
||||
#define RL_PPO_RATIO_VAR_M2_INDEX 632
|
||||
#define PPO_RATIO_MIN_ABSOLUTE 2.0f // architectural exemption: vanilla-PG safeguard
|
||||
#define PPO_RATIO_MIN_STD_SCALE 3.0f // architectural: 3σ healthy-update coverage
|
||||
#define PPO_RATIO_MAX_OVER_MIN_FACTOR 5.0f // architectural: ensure max ≥ 5× min
|
||||
|
||||
// --- 9. Gate threshold controller ---
|
||||
#define RL_CONF_GATE_THRESHOLD_INDEX 512
|
||||
@@ -96,27 +149,66 @@
|
||||
#define RL_GATE_FRD_MIN_INDEX 529
|
||||
#define RL_GATE_FRD_MAX_INDEX 530
|
||||
#define RL_GATE_ADJUST_RATE_INDEX 531
|
||||
#define RL_GATE_EMA_ALPHA_INDEX 638
|
||||
|
||||
// --- 10. Q distill lambda controller ---
|
||||
#define RL_Q_DISTILL_LAMBDA_INDEX 486
|
||||
// --- 10. Q distill lambda controller (Special Q) ---
|
||||
#define RL_Q_DISTILL_KL_EMA_INDEX 488
|
||||
#define RL_Q_DISTILL_KL_TARGET_INDEX 491
|
||||
#define MIN_LAMBDA 0.05f
|
||||
#define MAX_LAMBDA 1.0f
|
||||
#define KL_TOLERANCE 3.0f
|
||||
#define LAMBDA_RAMP_RATE 1.2f
|
||||
#define LAMBDA_DECAY_RATE 0.998f
|
||||
#define RL_Q_DISTILL_KL_VAR_COUNT_INDEX 618
|
||||
#define RL_Q_DISTILL_KL_VAR_M2_INDEX 620
|
||||
#define RL_Q_DISTILL_LAMBDA_MIN_ADAPTIVE_INDEX 639
|
||||
#define RL_Q_DISTILL_LAMBDA_MAX_INDEX 650
|
||||
#define RL_Q_DISTILL_KL_TOLERANCE_INDEX 651
|
||||
#define RL_Q_DISTILL_LAMBDA_RAMP_RATE_INDEX 652
|
||||
#define RL_Q_DISTILL_LAMBDA_DECAY_RATE_INDEX 653
|
||||
#define ADAPTIVE_MIN_ABSOLUTE 0.001f // architectural: q_distill_lambda safety floor
|
||||
#define ADAPTIVE_MIN_STD_SCALE 0.05f // architectural: signal-noise proportional scale
|
||||
|
||||
// --- Shared Schulman params ---
|
||||
#define RL_SCHULMAN_TOLERANCE_INDEX 468
|
||||
#define RL_SCHULMAN_ADJUST_RATE_INDEX 469
|
||||
|
||||
// --- Shared Wiener ---
|
||||
#define WIENER_ALPHA_FLOOR 0.4f
|
||||
// --- Shared adaptive-floor constants (spec 2026-05-30) ---
|
||||
#define NOISE_FLOOR_TARGET_FRAC 0.5f // floor ≥ 50% of target
|
||||
#define NOISE_FLOOR_STD_MULTIPLIER 2.0f // floor ≥ 2σ of observed signal
|
||||
#define WIDEN_PATIENCE_CONSECUTIVE 3.0f // widen/descend requires N below-band steps
|
||||
|
||||
// --- Shared Wiener-α floor ---
|
||||
#define RL_WIENER_ALPHA_FLOOR_INDEX 659
|
||||
|
||||
// --- Step counter (device-resident) ---
|
||||
#define RL_STEP_COUNTER_ISV_INDEX 548
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════
|
||||
// Adaptive risk management — spec 2026-05-30-adaptive-risk-management.
|
||||
// Layers 2/3/4 controllers fused below. Layer 1 (CMDP gates) runs as a
|
||||
// separate kernel because it needs per-step `realized_pnl` + `outcome`
|
||||
// arrays not available here. Layer D (trail factors) is read-only ISV.
|
||||
// ═══════════════════════════════════════════════════════════════════════
|
||||
|
||||
// --- 11. Layer 2: IQN action τ controller ---
|
||||
#define RL_SESSION_PNL_USD_INDEX 662
|
||||
#define RL_IQN_ACTION_TAU_INDEX 671
|
||||
#define RL_IQN_ACTION_TAU_MIN_INDEX 672
|
||||
#define RL_IQN_ACTION_TAU_DD_SENSITIVITY_INDEX 673
|
||||
#define DEFAULT_STARTING_CAPITAL_USD 35000.0f
|
||||
|
||||
// --- 12. Layer 3: Inventory β controller ---
|
||||
#define RL_INVENTORY_PENALTY_BETA_INDEX 674
|
||||
#define RL_INVENTORY_VARIANCE_EMA_INDEX 675
|
||||
// (RL_REWARD_MAGNITUDE_EMA_INDEX 614 already defined above)
|
||||
#define BETA_TARGET_FRAC_OF_REWARD 0.01f
|
||||
#define BETA_SIGMA_MULTIPLIER 2.0f
|
||||
|
||||
// --- 13. Layer 4: Kelly fraction controller ---
|
||||
#define RL_KELLY_FRACTION_INDEX 676
|
||||
#define RL_WIN_RATE_EMA_INDEX 677
|
||||
#define RL_AVG_WIN_USD_EMA_INDEX 678
|
||||
#define RL_AVG_LOSS_USD_EMA_INDEX 679
|
||||
#define RL_KELLY_SAFETY_FRAC_INDEX 680
|
||||
#define RL_KELLY_MIN_TRADES_FOR_RELEASE_INDEX 681
|
||||
// (RL_CUMULATIVE_DONES_INDEX 660 already defined above)
|
||||
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════════
|
||||
// Fused kernel: all 10 controllers in one launch.
|
||||
@@ -126,7 +218,7 @@
|
||||
// [1] = RL_Q_DIVERGENCE_EMA_INDEX (418) — target_tau
|
||||
// [2] = RL_KL_PI_EMA_INDEX (419) — ppo_clip
|
||||
// [3] = RL_ENTROPY_OBSERVED_EMA_INDEX (420) — entropy_coef
|
||||
// [4] = RL_ADVANTAGE_VAR_RATIO_EMA_INDEX (421) — rollout_steps
|
||||
// [4] = RL_ADV_VAR_PRE_NORM_INDEX (612) — rollout_steps (post-Phase-4.5)
|
||||
// [5] = RL_TD_KURTOSIS_EMA_INDEX (422) — per_alpha
|
||||
// [6] = RL_MEAN_ABS_PNL_EMA_INDEX (423) — reward_scale
|
||||
// ═══════════════════════════════════════════════════════════════════════
|
||||
@@ -140,29 +232,44 @@ extern "C" __global__ void rl_fused_controllers(
|
||||
if (threadIdx.x != 0 || blockIdx.x != 0) return;
|
||||
|
||||
const int current_step = (int)isv[RL_STEP_COUNTER_ISV_INDEX];
|
||||
const float wiener_floor = isv[RL_WIENER_ALPHA_FLOOR_INDEX];
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════
|
||||
// 1. Gamma controller → ISV[400]
|
||||
// 1. Gamma controller → ISV[400] (Special G)
|
||||
// ═══════════════════════════════════════════════════════════════════
|
||||
{
|
||||
const float gamma_prev = isv[RL_GAMMA_INDEX];
|
||||
|
||||
// Adaptive GAMMA_MIN: Welford mean of trade duration after warmup,
|
||||
// EMA before warmup. See rl_gamma_controller.cu Special case G.
|
||||
const float d_welford_count = isv[RL_TRADE_DUR_VAR_COUNT_INDEX];
|
||||
const float d_welford_mean = isv[RL_TRADE_DUR_VAR_MEAN_INDEX];
|
||||
const float d_smoothed = (d_welford_count >= WELFORD_WARMUP_OBS)
|
||||
? d_welford_mean
|
||||
: isv[RL_MEAN_TRADE_DURATION_EMA_INDEX];
|
||||
const float adaptive_min = fmaxf(GAMMA_MIN_ABSOLUTE,
|
||||
1.0f - 1.0f / fmaxf(d_smoothed * HORIZON_MULTIPLIER,
|
||||
HORIZON_FLOOR));
|
||||
isv[RL_GAMMA_MIN_ADAPTIVE_INDEX] = adaptive_min;
|
||||
|
||||
const float mean_trade_duration_events = isv[input_slots[0]];
|
||||
const float d = fmaxf(mean_trade_duration_events, 1.0f);
|
||||
const float gamma_max = isv[RL_GAMMA_MAX_INDEX];
|
||||
float gamma_target = powf(0.5f, 1.0f / d);
|
||||
gamma_target = fmaxf(GAMMA_MIN, fminf(gamma_target, GAMMA_MAX));
|
||||
gamma_target = fmaxf(adaptive_min, fminf(gamma_target, gamma_max));
|
||||
|
||||
if (gamma_prev == 0.0f) {
|
||||
isv[RL_GAMMA_INDEX] = gamma_target;
|
||||
} else {
|
||||
const float a = fmaxf(alpha, WIENER_ALPHA_FLOOR);
|
||||
const float a = fmaxf(alpha, wiener_floor);
|
||||
float gamma_new = (1.0f - a) * gamma_prev + a * gamma_target;
|
||||
gamma_new = fmaxf(GAMMA_MIN, fminf(gamma_new, GAMMA_MAX));
|
||||
gamma_new = fmaxf(adaptive_min, fminf(gamma_new, gamma_max));
|
||||
isv[RL_GAMMA_INDEX] = gamma_new;
|
||||
}
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════
|
||||
// 2. Target tau controller → ISV[401]
|
||||
// 2. Target tau controller → ISV[401] (canonical pattern)
|
||||
// ═══════════════════════════════════════════════════════════════════
|
||||
{
|
||||
const float tau_prev = isv[RL_TARGET_TAU_INDEX];
|
||||
@@ -172,28 +279,44 @@ extern "C" __global__ void rl_fused_controllers(
|
||||
const float q_divergence_norm = isv[input_slots[1]];
|
||||
if (q_divergence_norm != 0.0f) {
|
||||
const float div_target = isv[RL_DIV_TARGET_INDEX];
|
||||
const float div_noise_floor = div_target * DIV_NOISE_FLOOR_FRAC;
|
||||
|
||||
// Adaptive noise floor — signal-driven via Welford variance.
|
||||
const float div_count = isv[RL_Q_DIV_VAR_COUNT_INDEX];
|
||||
const float div_var = (div_count > 1.0f)
|
||||
? isv[RL_Q_DIV_VAR_M2_INDEX] / (div_count - 1.0f)
|
||||
: 0.0f;
|
||||
const float div_std = sqrtf(div_var);
|
||||
const float div_noise_floor = fmaxf(div_target * NOISE_FLOOR_TARGET_FRAC,
|
||||
div_std * NOISE_FLOOR_STD_MULTIPLIER);
|
||||
|
||||
if (q_divergence_norm >= div_noise_floor) {
|
||||
float ratio;
|
||||
// Asymmetric Schulman: raise τ on single observation,
|
||||
// lower τ requires N consecutive below-band observations.
|
||||
const float tolerance = isv[RL_SCHULMAN_TOLERANCE_INDEX];
|
||||
const float adjust_rate = isv[RL_SCHULMAN_ADJUST_RATE_INDEX];
|
||||
float ratio;
|
||||
if (q_divergence_norm > div_target * tolerance) {
|
||||
ratio = adjust_rate;
|
||||
isv[RL_Q_DIV_BELOW_COUNT_INDEX] = 0.0f;
|
||||
} else if (q_divergence_norm < div_target / tolerance) {
|
||||
ratio = 1.0f / adjust_rate;
|
||||
const float new_count = isv[RL_Q_DIV_BELOW_COUNT_INDEX] + 1.0f;
|
||||
isv[RL_Q_DIV_BELOW_COUNT_INDEX] = new_count;
|
||||
ratio = (new_count >= WIDEN_PATIENCE_CONSECUTIVE) ? (1.0f / adjust_rate) : 1.0f;
|
||||
} else {
|
||||
isv[RL_Q_DIV_BELOW_COUNT_INDEX] = 0.0f;
|
||||
ratio = 1.0f;
|
||||
}
|
||||
float tau_target = tau_prev * ratio;
|
||||
const float tau_max_isv = isv[RL_TARGET_TAU_MAX_INDEX];
|
||||
tau_target = fmaxf(TAU_MIN, fminf(tau_target, tau_max_isv));
|
||||
const float tau_min = isv[RL_TARGET_TAU_MIN_INDEX];
|
||||
const float tau_max = isv[RL_TARGET_TAU_MAX_INDEX];
|
||||
tau_target = fmaxf(tau_min, fminf(tau_target, tau_max));
|
||||
|
||||
if (tau_prev == isv[RL_TAU_BOOTSTRAP_INDEX]) {
|
||||
isv[RL_TARGET_TAU_INDEX] = tau_target;
|
||||
} else {
|
||||
const float a = fmaxf(alpha, WIENER_ALPHA_FLOOR);
|
||||
const float a = fmaxf(alpha, wiener_floor);
|
||||
float tau_new = (1.0f - a) * tau_prev + a * tau_target;
|
||||
tau_new = fmaxf(TAU_MIN, fminf(tau_new, tau_max_isv));
|
||||
tau_new = fmaxf(tau_min, fminf(tau_new, tau_max));
|
||||
isv[RL_TARGET_TAU_INDEX] = tau_new;
|
||||
}
|
||||
}
|
||||
@@ -202,7 +325,7 @@ extern "C" __global__ void rl_fused_controllers(
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════
|
||||
// 3. PPO clip controller → ISV[402]
|
||||
// 3. PPO clip controller → ISV[402] (canonical pattern — reference)
|
||||
// ═══════════════════════════════════════════════════════════════════
|
||||
{
|
||||
const float eps_prev = isv[RL_PPO_CLIP_INDEX];
|
||||
@@ -212,27 +335,44 @@ extern "C" __global__ void rl_fused_controllers(
|
||||
const float kl_ema = isv[input_slots[2]];
|
||||
if (kl_ema != 0.0f) {
|
||||
const float kl_target = isv[RL_KL_TARGET_INDEX];
|
||||
const float kl_noise_floor = kl_target * KL_NOISE_FLOOR_FRAC;
|
||||
|
||||
// Adaptive noise floor — signal-driven via Welford variance.
|
||||
const float kl_count = isv[RL_KL_PI_VAR_COUNT_INDEX];
|
||||
const float kl_var = (kl_count > 1.0f)
|
||||
? isv[RL_KL_PI_VAR_M2_INDEX] / (kl_count - 1.0f)
|
||||
: 0.0f;
|
||||
const float kl_std = sqrtf(kl_var);
|
||||
const float kl_noise_floor = fmaxf(kl_target * NOISE_FLOOR_TARGET_FRAC,
|
||||
kl_std * NOISE_FLOOR_STD_MULTIPLIER);
|
||||
|
||||
if (kl_ema >= kl_noise_floor) {
|
||||
// Asymmetric Schulman: tighten on single observation,
|
||||
// widen requires N consecutive below-band observations.
|
||||
const float tolerance = isv[RL_SCHULMAN_TOLERANCE_INDEX];
|
||||
const float adjust_rate = isv[RL_SCHULMAN_ADJUST_RATE_INDEX];
|
||||
float ratio;
|
||||
if (kl_ema > kl_target * tolerance) {
|
||||
ratio = 1.0f / adjust_rate;
|
||||
isv[RL_KL_PI_BELOW_COUNT_INDEX] = 0.0f;
|
||||
} else if (kl_ema < kl_target / tolerance) {
|
||||
ratio = adjust_rate;
|
||||
const float new_count = isv[RL_KL_PI_BELOW_COUNT_INDEX] + 1.0f;
|
||||
isv[RL_KL_PI_BELOW_COUNT_INDEX] = new_count;
|
||||
ratio = (new_count >= WIDEN_PATIENCE_CONSECUTIVE) ? adjust_rate : 1.0f;
|
||||
} else {
|
||||
isv[RL_KL_PI_BELOW_COUNT_INDEX] = 0.0f;
|
||||
ratio = 1.0f;
|
||||
}
|
||||
const float eps_min = isv[RL_PPO_CLIP_EPS_MIN_INDEX];
|
||||
const float eps_max = isv[RL_PPO_CLIP_EPS_MAX_INDEX];
|
||||
float eps_target = eps_prev * ratio;
|
||||
eps_target = fmaxf(EPS_MIN, fminf(eps_target, EPS_MAX));
|
||||
eps_target = fmaxf(eps_min, fminf(eps_target, eps_max));
|
||||
|
||||
if (eps_prev == isv[RL_EPS_BOOTSTRAP_INDEX]) {
|
||||
isv[RL_PPO_CLIP_INDEX] = eps_target;
|
||||
} else {
|
||||
const float a = fmaxf(alpha, WIENER_ALPHA_FLOOR);
|
||||
const float a = fmaxf(alpha, wiener_floor);
|
||||
float eps_new = (1.0f - a) * eps_prev + a * eps_target;
|
||||
eps_new = fmaxf(EPS_MIN, fminf(eps_new, EPS_MAX));
|
||||
eps_new = fmaxf(eps_min, fminf(eps_new, eps_max));
|
||||
isv[RL_PPO_CLIP_INDEX] = eps_new;
|
||||
}
|
||||
}
|
||||
@@ -242,6 +382,8 @@ extern "C" __global__ void rl_fused_controllers(
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════
|
||||
// 4. Entropy coef controller → ISV[403]
|
||||
// Asymmetric: raise coef on single below-floor observation,
|
||||
// drop coef requires N consecutive above-band observations.
|
||||
// ═══════════════════════════════════════════════════════════════════
|
||||
{
|
||||
const float coef_prev = isv[RL_ENTROPY_COEF_INDEX];
|
||||
@@ -249,26 +391,57 @@ extern "C" __global__ void rl_fused_controllers(
|
||||
const float h_max = logf((float)N_ACTIONS);
|
||||
const float target_frac = isv[RL_ENTROPY_TARGET_FRAC_INDEX];
|
||||
const float h_target = target_frac * h_max;
|
||||
const float COEF_FLOOR = 0.01f;
|
||||
const float COEF_FLOOR = 0.01f; // architectural: maintenance pressure baseline
|
||||
const float coef_min = isv[RL_ENTROPY_COEF_MIN_INDEX];
|
||||
const float coef_max = isv[RL_ENTROPY_COEF_MAX_INDEX];
|
||||
const float deficit_frac = fmaxf(0.0f,
|
||||
fminf((h_target - entropy_observed_ema) / fmaxf(h_target, 1e-6f), 1.0f));
|
||||
float coef_target = COEF_FLOOR + (COEF_MAX - COEF_FLOOR) * deficit_frac;
|
||||
coef_target = fmaxf(COEF_MIN, fminf(coef_target, COEF_MAX));
|
||||
float coef_target = COEF_FLOOR + (coef_max - COEF_FLOOR) * deficit_frac;
|
||||
coef_target = fmaxf(coef_min, fminf(coef_target, coef_max));
|
||||
|
||||
if (coef_prev == 0.0f) {
|
||||
isv[RL_ENTROPY_COEF_INDEX] = coef_target;
|
||||
} else if (entropy_observed_ema < h_target * 0.5f && entropy_observed_ema > 0.0f) {
|
||||
isv[RL_ENTROPY_COEF_INDEX] = coef_target;
|
||||
} else {
|
||||
const float a = fmaxf(alpha, WIENER_ALPHA_FLOOR);
|
||||
float coef_new = (1.0f - a) * coef_prev + a * coef_target;
|
||||
coef_new = fmaxf(COEF_MIN, fminf(coef_new, COEF_MAX));
|
||||
isv[RL_ENTROPY_COEF_INDEX] = coef_new;
|
||||
// Adaptive emergency floor (signal-driven via Welford variance).
|
||||
const float h_count = isv[RL_ENTROPY_OBS_VAR_COUNT_INDEX];
|
||||
const float h_var = (h_count > 1.0f)
|
||||
? isv[RL_ENTROPY_OBS_VAR_M2_INDEX] / (h_count - 1.0f)
|
||||
: 0.0f;
|
||||
const float h_std = sqrtf(h_var);
|
||||
const float emergency_floor = fmaxf(h_target * NOISE_FLOOR_TARGET_FRAC,
|
||||
h_target - h_std * NOISE_FLOOR_STD_MULTIPLIER);
|
||||
|
||||
if (entropy_observed_ema > 0.0f && entropy_observed_ema < emergency_floor) {
|
||||
// Single-observation emergency raise; reset patience.
|
||||
isv[RL_ENTROPY_COEF_INDEX] = coef_target;
|
||||
isv[RL_ENTROPY_OBS_BELOW_COUNT_INDEX] = 0.0f;
|
||||
} else {
|
||||
const float tolerance = isv[RL_SCHULMAN_TOLERANCE_INDEX];
|
||||
const float a = fmaxf(alpha, wiener_floor);
|
||||
float coef_new = (1.0f - a) * coef_prev + a * coef_target;
|
||||
coef_new = fmaxf(coef_min, fminf(coef_new, coef_max));
|
||||
|
||||
if (coef_new < coef_prev && entropy_observed_ema > h_target * tolerance) {
|
||||
// Healthy-entropy descent direction — require N consecutive observations
|
||||
// before letting Wiener drag coef downward.
|
||||
const float new_count = isv[RL_ENTROPY_OBS_BELOW_COUNT_INDEX] + 1.0f;
|
||||
isv[RL_ENTROPY_OBS_BELOW_COUNT_INDEX] = new_count;
|
||||
if (new_count < WIDEN_PATIENCE_CONSECUTIVE) {
|
||||
isv[RL_ENTROPY_COEF_INDEX] = coef_prev;
|
||||
} else {
|
||||
isv[RL_ENTROPY_COEF_INDEX] = coef_new;
|
||||
}
|
||||
} else {
|
||||
isv[RL_ENTROPY_OBS_BELOW_COUNT_INDEX] = 0.0f;
|
||||
isv[RL_ENTROPY_COEF_INDEX] = coef_new;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════
|
||||
// 5. Rollout steps controller → ISV[404]
|
||||
// Input now is RL_ADV_VAR_PRE_NORM_INDEX=612 (caller via input_slots[4]).
|
||||
// ═══════════════════════════════════════════════════════════════════
|
||||
{
|
||||
const float prev = isv[RL_N_ROLLOUT_STEPS_INDEX];
|
||||
@@ -278,28 +451,44 @@ extern "C" __global__ void rl_fused_controllers(
|
||||
const float advantage_var_over_abs_mean = isv[input_slots[4]];
|
||||
if (advantage_var_over_abs_mean != 0.0f) {
|
||||
const float adv_var_target = isv[RL_ADV_VAR_RATIO_TARGET_INDEX];
|
||||
const float adv_var_noise_floor =
|
||||
adv_var_target * ADV_VAR_RATIO_NOISE_FLOOR_FRAC;
|
||||
|
||||
// Adaptive noise floor — signal-driven via Welford variance.
|
||||
const float adv_count = isv[RL_ADV_VAR_VAR_COUNT_INDEX];
|
||||
const float adv_var = (adv_count > 1.0f)
|
||||
? isv[RL_ADV_VAR_VAR_M2_INDEX] / (adv_count - 1.0f)
|
||||
: 0.0f;
|
||||
const float adv_std = sqrtf(adv_var);
|
||||
const float adv_var_noise_floor = fmaxf(adv_var_target * NOISE_FLOOR_TARGET_FRAC,
|
||||
adv_std * NOISE_FLOOR_STD_MULTIPLIER);
|
||||
|
||||
if (advantage_var_over_abs_mean >= adv_var_noise_floor) {
|
||||
// Asymmetric Schulman: widen rollout on single above-band observation,
|
||||
// shrink requires N consecutive below-band observations.
|
||||
const float tolerance = isv[RL_SCHULMAN_TOLERANCE_INDEX];
|
||||
const float adjust_rate = isv[RL_SCHULMAN_ADJUST_RATE_INDEX];
|
||||
float scale;
|
||||
if (advantage_var_over_abs_mean > adv_var_target * tolerance) {
|
||||
scale = adjust_rate;
|
||||
isv[RL_ADV_VAR_BELOW_COUNT_INDEX] = 0.0f;
|
||||
} else if (advantage_var_over_abs_mean < adv_var_target / tolerance) {
|
||||
scale = 1.0f / adjust_rate;
|
||||
const float new_count = isv[RL_ADV_VAR_BELOW_COUNT_INDEX] + 1.0f;
|
||||
isv[RL_ADV_VAR_BELOW_COUNT_INDEX] = new_count;
|
||||
scale = (new_count >= WIDEN_PATIENCE_CONSECUTIVE) ? (1.0f / adjust_rate) : 1.0f;
|
||||
} else {
|
||||
isv[RL_ADV_VAR_BELOW_COUNT_INDEX] = 0.0f;
|
||||
scale = 1.0f;
|
||||
}
|
||||
const float rollout_min = isv[RL_ROLLOUT_MIN_INDEX];
|
||||
const float rollout_max = isv[RL_ROLLOUT_MAX_INDEX];
|
||||
float target = prev * scale;
|
||||
target = fmaxf(ROLLOUT_MIN, fminf(target, ROLLOUT_MAX));
|
||||
target = fmaxf(rollout_min, fminf(target, rollout_max));
|
||||
|
||||
if (prev == isv[RL_ROLLOUT_BOOTSTRAP_INDEX]) {
|
||||
isv[RL_N_ROLLOUT_STEPS_INDEX] = target;
|
||||
} else {
|
||||
const float a = fmaxf(alpha, WIENER_ALPHA_FLOOR);
|
||||
const float a = fmaxf(alpha, wiener_floor);
|
||||
float out = (1.0f - a) * prev + a * target;
|
||||
out = fmaxf(ROLLOUT_MIN, fminf(out, ROLLOUT_MAX));
|
||||
out = fmaxf(rollout_min, fminf(out, rollout_max));
|
||||
isv[RL_N_ROLLOUT_STEPS_INDEX] = out;
|
||||
}
|
||||
}
|
||||
@@ -309,35 +498,90 @@ extern "C" __global__ void rl_fused_controllers(
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════
|
||||
// 6. PER alpha controller → ISV[405]
|
||||
// Asymmetric Schulman patience on descent direction (light tails).
|
||||
// ═══════════════════════════════════════════════════════════════════
|
||||
{
|
||||
const float prev = isv[RL_PER_ALPHA_INDEX];
|
||||
const float td_kurtosis_ema = isv[input_slots[5]];
|
||||
|
||||
if (td_kurtosis_ema > 0.0f && td_kurtosis_ema < isv[RL_KURT_NOISE_FLOOR_INDEX]) {
|
||||
// Adaptive noise floor: combines absolute ISV floor with Welford-derived
|
||||
// adaptive component (std-scaled).
|
||||
const float kurt_count = isv[RL_TD_KURT_VAR_COUNT_INDEX];
|
||||
const float kurt_var = (kurt_count > 1.0f)
|
||||
? isv[RL_TD_KURT_VAR_M2_INDEX] / (kurt_count - 1.0f)
|
||||
: 0.0f;
|
||||
const float kurt_std = sqrtf(kurt_var);
|
||||
const float adaptive_noise_floor = fmaxf(isv[RL_KURT_NOISE_FLOOR_INDEX],
|
||||
kurt_std * NOISE_FLOOR_STD_MULTIPLIER);
|
||||
|
||||
// Hold-at-prev when signal present but below adaptive noise floor
|
||||
// (matches standalone semantics: cold-start prev==0 takes the
|
||||
// bootstrap path below, otherwise return).
|
||||
if (td_kurtosis_ema > 0.0f && td_kurtosis_ema < adaptive_noise_floor) {
|
||||
if (prev != 0.0f) goto per_alpha_done;
|
||||
}
|
||||
|
||||
{
|
||||
const float per_alpha_min = isv[RL_PER_ALPHA_MIN_INDEX];
|
||||
const float per_alpha_max = isv[RL_PER_ALPHA_MAX_INDEX];
|
||||
const float kurt_excess = fmaxf(0.0f, td_kurtosis_ema - isv[RL_KURT_GAUSSIAN_INDEX]);
|
||||
const float kurt_lift_scale = isv[RL_KURT_LIFT_SCALE_INDEX];
|
||||
float target = 0.4f + 0.2f * (kurt_excess / kurt_lift_scale);
|
||||
target = fmaxf(PER_ALPHA_MIN, fminf(target, PER_ALPHA_MAX));
|
||||
target = fmaxf(per_alpha_min, fminf(target, per_alpha_max));
|
||||
|
||||
if (prev == 0.0f) {
|
||||
isv[RL_PER_ALPHA_INDEX] = target;
|
||||
} else {
|
||||
const float a = fmaxf(alpha, WIENER_ALPHA_FLOOR);
|
||||
const float a = fmaxf(alpha, wiener_floor);
|
||||
float out = (1.0f - a) * prev + a * target;
|
||||
out = fmaxf(PER_ALPHA_MIN, fminf(out, PER_ALPHA_MAX));
|
||||
isv[RL_PER_ALPHA_INDEX] = out;
|
||||
out = fmaxf(per_alpha_min, fminf(out, per_alpha_max));
|
||||
|
||||
// Asymmetric Schulman patience on the DESCENT direction:
|
||||
// α rises on a single above-band observation, falls only
|
||||
// after N consecutive below-band observations.
|
||||
const float kurt_gaussian = isv[RL_KURT_GAUSSIAN_INDEX];
|
||||
const float tolerance = isv[RL_SCHULMAN_TOLERANCE_INDEX];
|
||||
if (out < prev && td_kurtosis_ema < kurt_gaussian / tolerance) {
|
||||
const float new_count = isv[RL_TD_KURT_BELOW_COUNT_INDEX] + 1.0f;
|
||||
isv[RL_TD_KURT_BELOW_COUNT_INDEX] = new_count;
|
||||
if (new_count < WIDEN_PATIENCE_CONSECUTIVE) {
|
||||
isv[RL_PER_ALPHA_INDEX] = prev;
|
||||
} else {
|
||||
isv[RL_PER_ALPHA_INDEX] = out;
|
||||
}
|
||||
} else {
|
||||
isv[RL_TD_KURT_BELOW_COUNT_INDEX] = 0.0f;
|
||||
isv[RL_PER_ALPHA_INDEX] = out;
|
||||
}
|
||||
}
|
||||
}
|
||||
per_alpha_done:;
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════
|
||||
// 7. Reward scale controller → ISV[406]
|
||||
// 7a. Cumulative dones accumulator → ISV[660] (reward-floor-fix follow-up)
|
||||
//
|
||||
// Sums `dones[b]` across this step's batch and accumulates into the
|
||||
// cumulative-closed-trades slot. Used by the reward_scale bootstrap-
|
||||
// floor gate below. Replaces the prior `RL_TRADE_DUR_VAR_COUNT_INDEX`
|
||||
// gate which counted closed-trade STEPS (not closed trades) and
|
||||
// released the floor ~100× too early in real-trade terms.
|
||||
//
|
||||
// Per `feedback_no_atomicadd`: single-thread sum-and-write — no atomic
|
||||
// needed since the fused kernel is single-thread / single-block.
|
||||
// ═══════════════════════════════════════════════════════════════════
|
||||
{
|
||||
float dones_this_step = 0.0f;
|
||||
for (int b = 0; b < b_size; b++) {
|
||||
dones_this_step += dones[b];
|
||||
}
|
||||
isv[RL_CUMULATIVE_DONES_INDEX] = isv[RL_CUMULATIVE_DONES_INDEX] + dones_this_step;
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════
|
||||
// 7. Reward scale controller → ISV[406] (Special R)
|
||||
// Asymmetric per-step decrease cap (5%) + bootstrap-fraction floor
|
||||
// (10% of bootstrap) until N=100 closed trades.
|
||||
// ═══════════════════════════════════════════════════════════════════
|
||||
{
|
||||
const float prev = isv[RL_REWARD_SCALE_INDEX];
|
||||
@@ -346,23 +590,53 @@ extern "C" __global__ void rl_fused_controllers(
|
||||
} else {
|
||||
const float mean_abs_pnl_ema = isv[input_slots[6]];
|
||||
if (mean_abs_pnl_ema != 0.0f) {
|
||||
// Emit per-batch pnl magnitude EMA to the public slot for
|
||||
// downstream Welford-variance kernel consumption.
|
||||
isv[RL_REWARD_MAGNITUDE_EMA_INDEX] = mean_abs_pnl_ema;
|
||||
|
||||
const float denom = fmaxf(mean_abs_pnl_ema, EPS_PNL);
|
||||
float target = 1.0f / denom;
|
||||
const float scale_min = isv[RL_REWARD_SCALE_MIN_INDEX];
|
||||
target = fmaxf(scale_min, fminf(target, REWARD_SCALE_MAX));
|
||||
|
||||
if (prev == isv[RL_REWARD_SCALE_BOOTSTRAP_INDEX]) {
|
||||
isv[RL_REWARD_SCALE_INDEX] = target;
|
||||
// Bootstrap-fraction floor: until cumulative closed trades
|
||||
// exceeds the (ISV-driven) threshold, scale cannot drop below
|
||||
// 10% of bootstrap. Cumulative dones tracked per-step by the
|
||||
// dones-sum accumulator block ABOVE; threshold from ISV slot 661.
|
||||
// Per `feedback_adaptive_not_tuned`: threshold is ISV-driven,
|
||||
// not hardcoded.
|
||||
const float trade_count = isv[RL_CUMULATIVE_DONES_INDEX];
|
||||
const float min_trades = isv[RL_MIN_TRADES_FOR_RELEASE_INDEX];
|
||||
const float boot = isv[RL_REWARD_SCALE_BOOTSTRAP_INDEX];
|
||||
const float boot_floor = (trade_count < min_trades)
|
||||
? boot * BOOTSTRAP_FRACTION_FLOOR
|
||||
: scale_min;
|
||||
|
||||
if (prev == boot) {
|
||||
// First-observation replace-directly with asymmetric DECREASE
|
||||
// rate-limit applied (per spec §Special R bullet (a)).
|
||||
float first = target;
|
||||
if (first < prev) {
|
||||
first = fmaxf(first, prev / ASYM_DECREASE_RATE);
|
||||
}
|
||||
first = fmaxf(boot_floor, fminf(first, REWARD_SCALE_MAX));
|
||||
isv[RL_REWARD_SCALE_INDEX] = first;
|
||||
} else {
|
||||
const float a = fmaxf(alpha, WIENER_ALPHA_FLOOR);
|
||||
const float a = fmaxf(alpha, wiener_floor);
|
||||
float out = (1.0f - a) * prev + a * target;
|
||||
|
||||
// Per-step ±2% movement clamp.
|
||||
// Per-step ±2% symmetric movement clamp.
|
||||
const float max_move = prev * 1.02f;
|
||||
const float min_move = prev * 0.98f;
|
||||
out = fmaxf(min_move, fminf(out, max_move));
|
||||
|
||||
out = fmaxf(scale_min, fminf(out, REWARD_SCALE_MAX));
|
||||
// Asymmetric DECREASE rate limit (belt-and-braces — documents
|
||||
// the controller invariant even if 2% symmetric clamp relaxes).
|
||||
if (out < prev) {
|
||||
out = fmaxf(out, prev / ASYM_DECREASE_RATE);
|
||||
}
|
||||
|
||||
out = fmaxf(boot_floor, fminf(out, REWARD_SCALE_MAX));
|
||||
isv[RL_REWARD_SCALE_INDEX] = out;
|
||||
}
|
||||
}
|
||||
@@ -372,6 +646,7 @@ extern "C" __global__ void rl_fused_controllers(
|
||||
// ═══════════════════════════════════════════════════════════════════
|
||||
// 8. PPO ratio clamp controller → ISV[440]
|
||||
// Reads ε from ISV[402] which was just written above.
|
||||
// MIN_OUT adaptive from observed ratio variance; MAX_OUT ISV-driven.
|
||||
// ═══════════════════════════════════════════════════════════════════
|
||||
{
|
||||
const float prev = isv[RL_PPO_RATIO_CLAMP_MAX_INDEX];
|
||||
@@ -381,16 +656,26 @@ extern "C" __global__ void rl_fused_controllers(
|
||||
const float eps = isv[RL_PPO_CLIP_INDEX];
|
||||
const float clamp_margin = isv[RL_PPO_CLAMP_MARGIN_INDEX];
|
||||
float target = (1.0f + eps) * clamp_margin;
|
||||
target = fmaxf(PPO_RATIO_CLAMP_MIN_OUT,
|
||||
fminf(target, PPO_RATIO_CLAMP_MAX_OUT));
|
||||
|
||||
// Adaptive MIN/MAX from observed log-ratio variance.
|
||||
const float ratio_count = isv[RL_PPO_RATIO_VAR_COUNT_INDEX];
|
||||
const float ratio_var = (ratio_count > 1.0f)
|
||||
? isv[RL_PPO_RATIO_VAR_M2_INDEX] / (ratio_count - 1.0f)
|
||||
: 0.0f;
|
||||
const float ratio_std = sqrtf(ratio_var);
|
||||
const float adaptive_min = fmaxf(PPO_RATIO_MIN_ABSOLUTE,
|
||||
1.0f + ratio_std * PPO_RATIO_MIN_STD_SCALE);
|
||||
const float adaptive_max = fmaxf(adaptive_min * PPO_RATIO_MAX_OVER_MIN_FACTOR,
|
||||
isv[RL_PPO_RATIO_CLAMP_MAX_ADAPTIVE_INDEX]);
|
||||
|
||||
target = fmaxf(adaptive_min, fminf(target, adaptive_max));
|
||||
|
||||
if (prev == isv[RL_PPO_RATIO_CLAMP_BOOTSTRAP_INDEX]) {
|
||||
isv[RL_PPO_RATIO_CLAMP_MAX_INDEX] = target;
|
||||
} else {
|
||||
const float a = fmaxf(alpha, WIENER_ALPHA_FLOOR);
|
||||
const float a = fmaxf(alpha, wiener_floor);
|
||||
float out = (1.0f - a) * prev + a * target;
|
||||
out = fmaxf(PPO_RATIO_CLAMP_MIN_OUT,
|
||||
fminf(out, PPO_RATIO_CLAMP_MAX_OUT));
|
||||
out = fmaxf(adaptive_min, fminf(out, adaptive_max));
|
||||
isv[RL_PPO_RATIO_CLAMP_MAX_INDEX] = out;
|
||||
}
|
||||
}
|
||||
@@ -399,7 +684,7 @@ extern "C" __global__ void rl_fused_controllers(
|
||||
// ═══════════════════════════════════════════════════════════════════
|
||||
// 9. Gate threshold controller → ISV[512,516,517]
|
||||
// Scans dones[B] to compute done_rate EMA → ISV[525].
|
||||
// Adjusts conf + FRD thresholds via Schulman bounded step.
|
||||
// EMA-α is now ISV-resident (slot 638) per spec §Group B Task 13.
|
||||
// ═══════════════════════════════════════════════════════════════════
|
||||
{
|
||||
const int warmup = (int)isv[RL_GATE_WARMUP_STEPS_INDEX];
|
||||
@@ -409,7 +694,7 @@ extern "C" __global__ void rl_fused_controllers(
|
||||
if (dones[b] > 0.5f) done_count += 1.0f;
|
||||
}
|
||||
const float done_rate = done_count / (float)b_size;
|
||||
const float gate_alpha = 0.01f;
|
||||
const float gate_alpha = isv[RL_GATE_EMA_ALPHA_INDEX];
|
||||
const float prev_ema = isv[RL_GATE_DONES_EMA_INDEX];
|
||||
const float dones_ema = (prev_ema == 0.0f && done_rate > 0.0f)
|
||||
? done_rate
|
||||
@@ -445,8 +730,9 @@ extern "C" __global__ void rl_fused_controllers(
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════
|
||||
// 10. Q→π distill lambda controller → ISV[486]
|
||||
// Asymmetric bounded step on KL_EMA vs target.
|
||||
// 10. Q→π distill lambda controller → ISV[486] (Special Q)
|
||||
// Adaptive MIN from Welford-var of q_distill_kl_ema; MAX + rates
|
||||
// ISV-driven (slots 650/651/652/653).
|
||||
// ═══════════════════════════════════════════════════════════════════
|
||||
{
|
||||
const float kl_observed = isv[RL_Q_DISTILL_KL_EMA_INDEX];
|
||||
@@ -454,16 +740,101 @@ extern "C" __global__ void rl_fused_controllers(
|
||||
float lambda = isv[RL_Q_DISTILL_LAMBDA_INDEX];
|
||||
|
||||
if (kl_observed > 0.0f && kl_target > 0.0f && lambda > 0.0f) {
|
||||
const float upper = kl_target * KL_TOLERANCE;
|
||||
const float lower = kl_target / KL_TOLERANCE;
|
||||
// Adaptive MIN: max(0.001, sqrt(var) × 0.05).
|
||||
const float kl_var_count = isv[RL_Q_DISTILL_KL_VAR_COUNT_INDEX];
|
||||
const float kl_var = (kl_var_count > 1.0f)
|
||||
? isv[RL_Q_DISTILL_KL_VAR_M2_INDEX] / (kl_var_count - 1.0f)
|
||||
: 0.0f;
|
||||
const float kl_std = sqrtf(kl_var);
|
||||
const float adaptive_min = fmaxf(ADAPTIVE_MIN_ABSOLUTE,
|
||||
kl_std * ADAPTIVE_MIN_STD_SCALE);
|
||||
isv[RL_Q_DISTILL_LAMBDA_MIN_ADAPTIVE_INDEX] = adaptive_min;
|
||||
|
||||
const float max_lambda = isv[RL_Q_DISTILL_LAMBDA_MAX_INDEX];
|
||||
const float kl_tolerance = isv[RL_Q_DISTILL_KL_TOLERANCE_INDEX];
|
||||
const float lambda_ramp_rate = isv[RL_Q_DISTILL_LAMBDA_RAMP_RATE_INDEX];
|
||||
const float lambda_decay_rate = isv[RL_Q_DISTILL_LAMBDA_DECAY_RATE_INDEX];
|
||||
|
||||
const float upper = kl_target * kl_tolerance;
|
||||
const float lower = kl_target / kl_tolerance;
|
||||
|
||||
if (kl_observed > upper) {
|
||||
lambda = fminf(MAX_LAMBDA, lambda * LAMBDA_RAMP_RATE);
|
||||
lambda = fminf(max_lambda, lambda * lambda_ramp_rate);
|
||||
} else if (kl_observed < lower) {
|
||||
lambda = fmaxf(MIN_LAMBDA, lambda * LAMBDA_DECAY_RATE);
|
||||
lambda = fmaxf(adaptive_min, lambda * lambda_decay_rate);
|
||||
}
|
||||
|
||||
isv[RL_Q_DISTILL_LAMBDA_INDEX] = lambda;
|
||||
}
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════
|
||||
// 11. Layer 2 — IQN action τ controller → ISV[671]
|
||||
// τ = clamp(τ_min, 0.5 - sensitivity × drawdown_frac, 1.0)
|
||||
// drawdown_frac = max(0, -session_pnl) / starting_capital
|
||||
// ═══════════════════════════════════════════════════════════════════
|
||||
{
|
||||
const float session_pnl = isv[RL_SESSION_PNL_USD_INDEX];
|
||||
const float drawdown_usd = fmaxf(0.0f, -session_pnl);
|
||||
const float drawdown_frac = drawdown_usd / DEFAULT_STARTING_CAPITAL_USD;
|
||||
|
||||
const float tau_min = isv[RL_IQN_ACTION_TAU_MIN_INDEX];
|
||||
const float sensitivity = isv[RL_IQN_ACTION_TAU_DD_SENSITIVITY_INDEX];
|
||||
float tau_target = 0.5f - sensitivity * drawdown_frac;
|
||||
tau_target = fmaxf(tau_min, fminf(tau_target, 1.0f));
|
||||
isv[RL_IQN_ACTION_TAU_INDEX] = tau_target;
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════
|
||||
// 12. Layer 3 — Inventory β controller → ISV[674]
|
||||
// β_target = 0.01 × E[|reward|] / (2 × σ_inventory)
|
||||
// Sentinel-zero bootstrap; Wiener-α blend with floor 0.4 after.
|
||||
// ═══════════════════════════════════════════════════════════════════
|
||||
{
|
||||
const float reward_mag = isv[RL_REWARD_MAGNITUDE_EMA_INDEX];
|
||||
const float inv_var = isv[RL_INVENTORY_VARIANCE_EMA_INDEX];
|
||||
if (inv_var > 0.0f && reward_mag > 0.0f) {
|
||||
const float inv_std = sqrtf(inv_var);
|
||||
const float beta_target = BETA_TARGET_FRAC_OF_REWARD * reward_mag
|
||||
/ (BETA_SIGMA_MULTIPLIER * inv_std);
|
||||
const float prev = isv[RL_INVENTORY_PENALTY_BETA_INDEX];
|
||||
if (prev == 0.0f) {
|
||||
isv[RL_INVENTORY_PENALTY_BETA_INDEX] = beta_target;
|
||||
} else {
|
||||
const float a = fmaxf(alpha, wiener_floor);
|
||||
isv[RL_INVENTORY_PENALTY_BETA_INDEX] =
|
||||
(1.0f - a) * prev + a * beta_target;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════════════════════════════
|
||||
// 13. Layer 4 — Kelly fraction controller → ISV[676]
|
||||
// f = clamp(safety × (p × b − q) / b, 0, 1)
|
||||
// b = avg_win / avg_loss; warmup-gated on cumulative_dones.
|
||||
// ═══════════════════════════════════════════════════════════════════
|
||||
{
|
||||
const float kelly_dones = isv[RL_CUMULATIVE_DONES_INDEX];
|
||||
const float kelly_min_trades = isv[RL_KELLY_MIN_TRADES_FOR_RELEASE_INDEX];
|
||||
if (kelly_dones < kelly_min_trades) {
|
||||
isv[RL_KELLY_FRACTION_INDEX] = 1.0f;
|
||||
} else {
|
||||
const float p = isv[RL_WIN_RATE_EMA_INDEX];
|
||||
const float avg_win = isv[RL_AVG_WIN_USD_EMA_INDEX];
|
||||
const float avg_loss = isv[RL_AVG_LOSS_USD_EMA_INDEX];
|
||||
if (avg_loss <= 0.0f) {
|
||||
isv[RL_KELLY_FRACTION_INDEX] = 1.0f;
|
||||
} else if (avg_win <= 0.0f) {
|
||||
isv[RL_KELLY_FRACTION_INDEX] = 0.0f;
|
||||
} else {
|
||||
const float b_ratio = avg_win / avg_loss;
|
||||
const float q = 1.0f - p;
|
||||
const float f_kelly = (p * b_ratio - q) / b_ratio;
|
||||
const float safety = isv[RL_KELLY_SAFETY_FRAC_INDEX];
|
||||
float f = f_kelly * safety;
|
||||
f = fmaxf(0.0f, fminf(f, 1.0f));
|
||||
isv[RL_KELLY_FRACTION_INDEX] = f;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -39,6 +39,8 @@
|
||||
#define RL_SHORT_HOLD_PENALTY_INDEX 534
|
||||
#define RL_HOLD_BONUS_INDEX 535
|
||||
#define RL_OUTCOME_ALPHA_INDEX 520
|
||||
// Layer 3 (inventory penalty — spec 2026-05-30-adaptive-risk-management).
|
||||
#define RL_INVENTORY_PENALTY_BETA_INDEX 674
|
||||
|
||||
#define MAX_UNITS 4
|
||||
|
||||
@@ -236,6 +238,17 @@ extern "C" __global__ void rl_fused_reward_pipeline(
|
||||
r += hold_bonus * sqrtf((float)hold_time);
|
||||
}
|
||||
|
||||
// ================================================================
|
||||
// PHASE 5b: Layer 3 inventory penalty (spec 2026-05-30-adaptive-risk-management).
|
||||
// Apply BEFORE raw_rewards snapshot so the penalty is part of the
|
||||
// shaped reward seen by Q/V learning. β = 0 (sentinel) → no-op.
|
||||
// ================================================================
|
||||
const float inv_beta = isv[RL_INVENTORY_PENALTY_BETA_INDEX];
|
||||
if (inv_beta > 0.0f) {
|
||||
const float net_pos_mag = (float)((current_lots < 0) ? -current_lots : current_lots);
|
||||
r -= inv_beta * net_pos_mag;
|
||||
}
|
||||
|
||||
// Write shaped reward back.
|
||||
rewards[b] = r;
|
||||
|
||||
|
||||
@@ -45,9 +45,35 @@
|
||||
// the policy myopic).
|
||||
|
||||
#define RL_GAMMA_INDEX 400
|
||||
#define GAMMA_MIN 0.995f
|
||||
#define GAMMA_MAX 0.999f
|
||||
#define WIENER_ALPHA_FLOOR 0.4f
|
||||
// γ MAX clamp ceiling is now ISV-driven per the 2026-05-30 clamp-bound
|
||||
// extension. γ MIN already adaptive via RL_GAMMA_MIN_ADAPTIVE_INDEX (613).
|
||||
#define RL_GAMMA_MAX_INDEX 649
|
||||
// Wiener-α floor — shared across 9 controllers (slot 659).
|
||||
#define RL_WIENER_ALPHA_FLOOR_INDEX 659
|
||||
|
||||
// Adaptive controller floors (spec 2026-05-30 Special case G):
|
||||
// hardcoded `GAMMA_MIN = 0.995f` replaced with an adaptive bound derived
|
||||
// from the Welford mean of trade duration. With short-horizon trade
|
||||
// regimes (d_ema = 15.6 in fold 0) the static 0.995 floor pinned gamma
|
||||
// for the entire run; adaptive bound `1 - 1/(d_smoothed × 2)` lets the
|
||||
// controller drop gamma low enough for the current regime while keeping
|
||||
// a hard `GAMMA_MIN_ABSOLUTE = 0.9` to prevent bandit-mode degenerate.
|
||||
//
|
||||
// Smoothed-duration choice: Welford mean (slot 609) is used after a
|
||||
// 100-observation warmup so the gamma↔trade_duration feedback loop is
|
||||
// damped by the natural N-smoothing of running mean. Before warmup, the
|
||||
// EMA at slot 417 is used directly (less stable but available
|
||||
// immediately at cold-start).
|
||||
#define RL_GAMMA_MIN_ADAPTIVE_INDEX 613
|
||||
#define RL_TRADE_DUR_VAR_COUNT_INDEX 608
|
||||
#define RL_TRADE_DUR_VAR_MEAN_INDEX 609
|
||||
#define RL_MEAN_TRADE_DURATION_EMA_INDEX 417
|
||||
#define GAMMA_MIN_ABSOLUTE 0.9f
|
||||
#define HORIZON_MULTIPLIER 2.0f
|
||||
#define WELFORD_WARMUP_OBS 100.0f
|
||||
// Floor for `d × HORIZON_MULTIPLIER` to prevent the `1/d` term from
|
||||
// blowing up when trade duration is very small (cold start, no closes).
|
||||
#define HORIZON_FLOOR 10.0f
|
||||
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────
|
||||
@@ -84,6 +110,27 @@ extern "C" __global__ void rl_gamma_controller(
|
||||
|
||||
const float gamma_prev = isv[RL_GAMMA_INDEX];
|
||||
|
||||
// Adaptive GAMMA_MIN (spec 2026-05-30 Special case G).
|
||||
// Replaces hardcoded `GAMMA_MIN = 0.995f`. With d_ema = 15.6 (fold 0)
|
||||
// the static 0.995 floor pinned γ for the entire run — the target
|
||||
// formula `0.5^(1/d)` produces γ_target = 0.936 at d = 15.6, which
|
||||
// clamps to 0.995 = floor and never moves. Adaptive bound
|
||||
// `1 - 1/(d × 2)` gives γ_min ≈ 0.968 at d = 15.6 — leaving room
|
||||
// for the controller to track the actual trade horizon.
|
||||
//
|
||||
// After 100 Welford observations the running mean is used (more
|
||||
// stable than EMA against per-batch outliers); before that the EMA
|
||||
// at slot 417 is used directly so cold-start has a usable signal.
|
||||
const float d_welford_count = isv[RL_TRADE_DUR_VAR_COUNT_INDEX];
|
||||
const float d_welford_mean = isv[RL_TRADE_DUR_VAR_MEAN_INDEX];
|
||||
const float d_smoothed = (d_welford_count >= WELFORD_WARMUP_OBS)
|
||||
? d_welford_mean
|
||||
: isv[RL_MEAN_TRADE_DURATION_EMA_INDEX];
|
||||
const float adaptive_min = fmaxf(GAMMA_MIN_ABSOLUTE,
|
||||
1.0f - 1.0f / fmaxf(d_smoothed * HORIZON_MULTIPLIER,
|
||||
HORIZON_FLOOR));
|
||||
isv[RL_GAMMA_MIN_ADAPTIVE_INDEX] = adaptive_min;
|
||||
|
||||
// Compute target from the current input EMA. Shared between
|
||||
// bootstrap and per-step paths so the dead-zone coincidence with
|
||||
// a hardcoded bootstrap cannot recur.
|
||||
@@ -91,26 +138,25 @@ extern "C" __global__ void rl_gamma_controller(
|
||||
// single-event trade doesn't push γ to 0.5.
|
||||
const float mean_trade_duration_events = isv[input_slot];
|
||||
const float d = fmaxf(mean_trade_duration_events, 1.0f);
|
||||
const float gamma_max = isv[RL_GAMMA_MAX_INDEX];
|
||||
float gamma_target = powf(0.5f, 1.0f / d);
|
||||
gamma_target = fmaxf(GAMMA_MIN, fminf(gamma_target, GAMMA_MAX));
|
||||
gamma_target = fmaxf(adaptive_min, fminf(gamma_target, gamma_max));
|
||||
|
||||
// Bootstrap on sentinel 0.0 per pearl_first_observation_bootstrap:
|
||||
// first emit replaces directly with the computed target. At cold
|
||||
// start (input EMA also sentinel-zero), clamped d=1, target=0.5,
|
||||
// clamped to GAMMA_MIN = 0.90 (the floor). Any non-sentinel input
|
||||
// produces target ≥ 0.90 → ≤ 0.999, distinct from the floor so
|
||||
// the per-step Wiener blend on subsequent calls always sees a
|
||||
// prev vs target delta.
|
||||
// clamped to adaptive_min (≥ GAMMA_MIN_ABSOLUTE = 0.90). Any
|
||||
// non-sentinel input produces target ≥ adaptive_min → ≤ 0.999.
|
||||
if (gamma_prev == 0.0f) {
|
||||
isv[RL_GAMMA_INDEX] = gamma_target;
|
||||
return;
|
||||
}
|
||||
|
||||
// Wiener-α blend with floor per pearl_wiener_alpha_floor_for_nonstationary.
|
||||
const float a = fmaxf(alpha, WIENER_ALPHA_FLOOR);
|
||||
const float a = fmaxf(alpha, isv[RL_WIENER_ALPHA_FLOOR_INDEX]);
|
||||
float gamma_new = (1.0f - a) * gamma_prev + a * gamma_target;
|
||||
|
||||
// Clamp into the bounded range; runaway protection.
|
||||
gamma_new = fmaxf(GAMMA_MIN, fminf(gamma_new, GAMMA_MAX));
|
||||
gamma_new = fmaxf(adaptive_min, fminf(gamma_new, gamma_max));
|
||||
isv[RL_GAMMA_INDEX] = gamma_new;
|
||||
}
|
||||
|
||||
@@ -23,6 +23,12 @@
|
||||
#define RL_GATE_FRD_MAX_INDEX 530
|
||||
#define RL_GATE_ADJUST_RATE_INDEX 531
|
||||
#define RL_STEP_COUNTER_ISV_INDEX 548
|
||||
// Adaptive controller floors (spec 2026-05-30-adaptive-controller-floor-design):
|
||||
// EMA smoothing α was hardcoded 0.01f; ISV slot 638 makes it tunable
|
||||
// at runtime so the dones-rate EMA half-life can adapt to data volume
|
||||
// (e.g. small-batch smoke runs may want a faster α to surface signal
|
||||
// before warmup completes).
|
||||
#define RL_GATE_EMA_ALPHA_INDEX 638
|
||||
|
||||
extern "C" __global__ void rl_gate_threshold_controller(
|
||||
float* __restrict__ isv,
|
||||
@@ -41,7 +47,7 @@ extern "C" __global__ void rl_gate_threshold_controller(
|
||||
if (dones[b] > 0.5f) done_count += 1.0f;
|
||||
}
|
||||
const float done_rate = done_count / (float)b_size;
|
||||
const float alpha = 0.01f;
|
||||
const float alpha = isv[RL_GATE_EMA_ALPHA_INDEX];
|
||||
const float prev_ema = isv[RL_GATE_DONES_EMA_INDEX];
|
||||
const float dones_ema = (prev_ema == 0.0f && done_rate > 0.0f)
|
||||
? done_rate
|
||||
|
||||
57
crates/ml-alpha/cuda/rl_inventory_beta_controller.cu
Normal file
57
crates/ml-alpha/cuda/rl_inventory_beta_controller.cu
Normal file
@@ -0,0 +1,57 @@
|
||||
// rl_inventory_beta_controller.cu — Layer 3: adaptive inventory penalty β.
|
||||
//
|
||||
// Spec: docs/superpowers/specs/2026-05-30-adaptive-risk-management-design.md
|
||||
//
|
||||
// Scales the inventory penalty `reward_shaped = reward_raw - β × |net_pos|`
|
||||
// so that at ~2σ inventory the penalty is ~1% of typical reward magnitude.
|
||||
//
|
||||
// β_target = 0.01 × E[|reward|] / (2 × σ_inventory)
|
||||
//
|
||||
// Sentinel-zero bootstrap per `pearl_first_observation_bootstrap`: hold β
|
||||
// at 0 until BOTH reward magnitude EMA and inventory variance EMA have
|
||||
// real data. After bootstrap, Wiener-α blend with floor 0.4 per
|
||||
// `pearl_wiener_alpha_floor_for_nonstationary`.
|
||||
//
|
||||
// Per `feedback_no_atomicadd`: single-thread single-block.
|
||||
// Per `feedback_cpu_is_read_only`: pure device kernel.
|
||||
// Per `feedback_isv_for_adaptive_bounds`: every threshold via ISV slot.
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#define RL_INVENTORY_PENALTY_BETA_INDEX 674
|
||||
#define RL_INVENTORY_VARIANCE_EMA_INDEX 675
|
||||
#define RL_REWARD_MAGNITUDE_EMA_INDEX 614
|
||||
#define RL_WIENER_ALPHA_FLOOR_INDEX 659
|
||||
|
||||
// 1% of typical reward magnitude at 2σ inventory — Avellaneda-Stoikov
|
||||
// canonical "noticeable but not dominant" calibration. Structural ratio.
|
||||
#define BETA_TARGET_FRAC_OF_REWARD 0.01f
|
||||
#define BETA_SIGMA_MULTIPLIER 2.0f
|
||||
|
||||
extern "C" __global__ void rl_inventory_beta_controller(
|
||||
float* __restrict__ isv
|
||||
) {
|
||||
if (threadIdx.x != 0 || threadIdx.y != 0 || threadIdx.z != 0) return;
|
||||
if (blockIdx.x != 0 || blockIdx.y != 0 || blockIdx.z != 0) return;
|
||||
|
||||
const float reward_mag = isv[RL_REWARD_MAGNITUDE_EMA_INDEX];
|
||||
const float inv_var = isv[RL_INVENTORY_VARIANCE_EMA_INDEX];
|
||||
|
||||
// Dead-signal hold: need real data on both inputs.
|
||||
if (inv_var <= 0.0f || reward_mag <= 0.0f) return;
|
||||
|
||||
const float inv_std = sqrtf(inv_var);
|
||||
const float beta_target = BETA_TARGET_FRAC_OF_REWARD * reward_mag
|
||||
/ (BETA_SIGMA_MULTIPLIER * inv_std);
|
||||
|
||||
const float prev = isv[RL_INVENTORY_PENALTY_BETA_INDEX];
|
||||
const float a_floor = isv[RL_WIENER_ALPHA_FLOOR_INDEX];
|
||||
|
||||
if (prev == 0.0f) {
|
||||
// First-observation bootstrap.
|
||||
isv[RL_INVENTORY_PENALTY_BETA_INDEX] = beta_target;
|
||||
} else {
|
||||
isv[RL_INVENTORY_PENALTY_BETA_INDEX] =
|
||||
(1.0f - a_floor) * prev + a_floor * beta_target;
|
||||
}
|
||||
}
|
||||
59
crates/ml-alpha/cuda/rl_inventory_variance_update.cu
Normal file
59
crates/ml-alpha/cuda/rl_inventory_variance_update.cu
Normal file
@@ -0,0 +1,59 @@
|
||||
// rl_inventory_variance_update.cu — Layer 3 (inventory penalty) input EMA.
|
||||
//
|
||||
// Tracks running variance of |net_position| across batches. Layer 3's β
|
||||
// controller scales the penalty so that at ~2σ inventory the penalty
|
||||
// equals ~1% of typical reward magnitude.
|
||||
// Spec: docs/superpowers/specs/2026-05-30-adaptive-risk-management-design.md
|
||||
//
|
||||
// Inputs:
|
||||
// net_position_per_batch[b] signed contracts (current_lots, i32)
|
||||
//
|
||||
// Output: ISV[RL_INVENTORY_VARIANCE_EMA_INDEX = 675]
|
||||
//
|
||||
// EMA over the batch variance of |net_position|; sentinel-zero bootstrap.
|
||||
// This is NOT a true Welford triple — we collapse the batch into a single
|
||||
// scalar (batch variance) then EMA across steps with a constant alpha.
|
||||
// The β controller treats this slot as σ² directly.
|
||||
//
|
||||
// Per `feedback_no_atomicadd`: single-thread single-block, sums sequentially.
|
||||
// Per `feedback_cpu_is_read_only`: pure device kernel.
|
||||
// Per `feedback_isv_for_adaptive_bounds`: Welford-EMA α is a structural
|
||||
// smoothing parameter, matching ema_update_per_step convention.
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#define RL_INVENTORY_VARIANCE_EMA_INDEX 675
|
||||
#define WELFORD_ALPHA 0.01f // slower EMA than Kelly inputs
|
||||
|
||||
extern "C" __global__ void rl_inventory_variance_update(
|
||||
float* __restrict__ isv,
|
||||
const int* __restrict__ net_position_per_batch, // [b_size] signed contracts (i32)
|
||||
int b_size
|
||||
) {
|
||||
if (threadIdx.x != 0 || threadIdx.y != 0 || threadIdx.z != 0) return;
|
||||
if (blockIdx.x != 0 || blockIdx.y != 0 || blockIdx.z != 0) return;
|
||||
|
||||
if (b_size <= 0) return;
|
||||
|
||||
// Compute batch mean + variance of |net_position|.
|
||||
float sum = 0.0f;
|
||||
float sum_sq = 0.0f;
|
||||
for (int b = 0; b < b_size; ++b) {
|
||||
const int lots = net_position_per_batch[b];
|
||||
const float v = (float)((lots < 0) ? -lots : lots);
|
||||
sum += v;
|
||||
sum_sq += v * v;
|
||||
}
|
||||
const float n = (float)b_size;
|
||||
const float mean = sum / n;
|
||||
const float var = (sum_sq / n) - (mean * mean);
|
||||
if (var < 0.0f) return; // numerical safety: should not happen but defend
|
||||
|
||||
const float prev = isv[RL_INVENTORY_VARIANCE_EMA_INDEX];
|
||||
if (prev == 0.0f) {
|
||||
isv[RL_INVENTORY_VARIANCE_EMA_INDEX] = var;
|
||||
} else {
|
||||
isv[RL_INVENTORY_VARIANCE_EMA_INDEX] =
|
||||
(1.0f - WELFORD_ALPHA) * prev + WELFORD_ALPHA * var;
|
||||
}
|
||||
}
|
||||
53
crates/ml-alpha/cuda/rl_iqn_action_tau_controller.cu
Normal file
53
crates/ml-alpha/cuda/rl_iqn_action_tau_controller.cu
Normal file
@@ -0,0 +1,53 @@
|
||||
// rl_iqn_action_tau_controller.cu — Layer 2: adaptive IQN action-selection τ.
|
||||
//
|
||||
// Spec: docs/superpowers/specs/2026-05-30-adaptive-risk-management-design.md
|
||||
//
|
||||
// Adapts the quantile used for IQN risk-averse action selection from the
|
||||
// observed session drawdown. Normal trading: τ = 0.5 (median ≈ expected-Q
|
||||
// behavior). Under drawdown: τ → τ_MIN (pessimistic, downside-aware).
|
||||
//
|
||||
// τ_target = max(τ_min, 0.5 - sensitivity × drawdown_frac)
|
||||
//
|
||||
// Drawdown approximation: `drawdown_frac = max(0, -session_pnl) /
|
||||
// starting_capital`. Because the trainer resets session_pnl on each fold
|
||||
// boundary, this approximates "drawdown from session start". For the
|
||||
// strict "drawdown from session peak" formulation we'd need a peak-tracker
|
||||
// slot — deferred (see plan note); the current formulation is correct in
|
||||
// the dominant case where the agent is in a losing session and provides
|
||||
// the right pressure (more pessimistic action selection as losses
|
||||
// accumulate). In a strongly-winning session τ stays at 0.5.
|
||||
//
|
||||
// Per `feedback_no_atomicadd`: single-thread single-block.
|
||||
// Per `feedback_cpu_is_read_only`: pure device kernel.
|
||||
// Per `feedback_isv_for_adaptive_bounds`: τ_min, sensitivity ISV-driven.
|
||||
//
|
||||
// Starting capital is the $35k single-contract ES baseline per
|
||||
// `project_ml_alpha_starting_capital`; structural constant exemption.
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#define RL_SESSION_PNL_USD_INDEX 662
|
||||
#define RL_IQN_ACTION_TAU_INDEX 671
|
||||
#define RL_IQN_ACTION_TAU_MIN_INDEX 672
|
||||
#define RL_IQN_ACTION_TAU_DD_SENSITIVITY_INDEX 673
|
||||
|
||||
#define DEFAULT_STARTING_CAPITAL_USD 35000.0f
|
||||
|
||||
extern "C" __global__ void rl_iqn_action_tau_controller(
|
||||
float* __restrict__ isv
|
||||
) {
|
||||
if (threadIdx.x != 0 || threadIdx.y != 0 || threadIdx.z != 0) return;
|
||||
if (blockIdx.x != 0 || blockIdx.y != 0 || blockIdx.z != 0) return;
|
||||
|
||||
const float session_pnl = isv[RL_SESSION_PNL_USD_INDEX];
|
||||
// Approximate drawdown from session boundary as max(0, -session_pnl).
|
||||
const float drawdown_usd = fmaxf(0.0f, -session_pnl);
|
||||
const float drawdown_frac = drawdown_usd / DEFAULT_STARTING_CAPITAL_USD;
|
||||
|
||||
const float tau_min = isv[RL_IQN_ACTION_TAU_MIN_INDEX];
|
||||
const float sensitivity = isv[RL_IQN_ACTION_TAU_DD_SENSITIVITY_INDEX];
|
||||
float tau_target = 0.5f - sensitivity * drawdown_frac;
|
||||
tau_target = fmaxf(tau_min, fminf(tau_target, 1.0f));
|
||||
|
||||
isv[RL_IQN_ACTION_TAU_INDEX] = tau_target;
|
||||
}
|
||||
70
crates/ml-alpha/cuda/rl_kelly_fraction_controller.cu
Normal file
70
crates/ml-alpha/cuda/rl_kelly_fraction_controller.cu
Normal file
@@ -0,0 +1,70 @@
|
||||
// rl_kelly_fraction_controller.cu — Layer 4: half-Kelly position-size multiplier.
|
||||
//
|
||||
// Spec: docs/superpowers/specs/2026-05-30-adaptive-risk-management-design.md
|
||||
//
|
||||
// Half-Kelly (Thorp) from observed win_rate × R-multiple:
|
||||
//
|
||||
// f_kelly = (p × b − q) / b where p = win_rate, q = 1 − p,
|
||||
// b = avg_win / avg_loss
|
||||
// f = clamp(safety × f_kelly, 0, 1) where safety = 0.5 (half-Kelly)
|
||||
//
|
||||
// Warmup gate (`pearl_first_observation_bootstrap` + warmup discipline):
|
||||
// hold f at bootstrap = 1.0 until `cumulative_dones >= MIN_TRADES_FOR_RELEASE`
|
||||
// so the EMA inputs (win_rate, avg_win, avg_loss) have time to converge
|
||||
// past cold-start noise. After release, f tracks observed edge — if
|
||||
// edge is negative or zero, Kelly clamps to 0 = no commitment.
|
||||
//
|
||||
// Dead-signal guard: if avg_loss <= 0 (sentinel, no losses observed yet)
|
||||
// hold f at 1.0 — we don't have a denominator for the R-multiple.
|
||||
//
|
||||
// Per `feedback_no_atomicadd`: single-thread single-block.
|
||||
// Per `feedback_cpu_is_read_only`: pure device kernel.
|
||||
// Per `feedback_isv_for_adaptive_bounds`: safety multiplier + warmup gate
|
||||
// are ISV-driven (slots 680, 681).
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#define RL_KELLY_FRACTION_INDEX 676
|
||||
#define RL_WIN_RATE_EMA_INDEX 677
|
||||
#define RL_AVG_WIN_USD_EMA_INDEX 678
|
||||
#define RL_AVG_LOSS_USD_EMA_INDEX 679
|
||||
#define RL_KELLY_SAFETY_FRAC_INDEX 680
|
||||
#define RL_KELLY_MIN_TRADES_FOR_RELEASE_INDEX 681
|
||||
#define RL_CUMULATIVE_DONES_INDEX 660
|
||||
|
||||
extern "C" __global__ void rl_kelly_fraction_controller(
|
||||
float* __restrict__ isv
|
||||
) {
|
||||
if (threadIdx.x != 0 || threadIdx.y != 0 || threadIdx.z != 0) return;
|
||||
if (blockIdx.x != 0 || blockIdx.y != 0 || blockIdx.z != 0) return;
|
||||
|
||||
// Warmup gate: hold at bootstrap=1.0 until enough closed trades.
|
||||
if (isv[RL_CUMULATIVE_DONES_INDEX] < isv[RL_KELLY_MIN_TRADES_FOR_RELEASE_INDEX]) {
|
||||
isv[RL_KELLY_FRACTION_INDEX] = 1.0f;
|
||||
return;
|
||||
}
|
||||
|
||||
const float p = isv[RL_WIN_RATE_EMA_INDEX];
|
||||
const float avg_win = isv[RL_AVG_WIN_USD_EMA_INDEX];
|
||||
const float avg_loss = isv[RL_AVG_LOSS_USD_EMA_INDEX];
|
||||
|
||||
// Dead-signal guard: need positive loss magnitude for R-multiple.
|
||||
if (avg_loss <= 0.0f) {
|
||||
isv[RL_KELLY_FRACTION_INDEX] = 1.0f;
|
||||
return;
|
||||
}
|
||||
if (avg_win <= 0.0f) {
|
||||
// No wins observed — Kelly cannot make sense of edge, hold at 0
|
||||
// (don't commit further size when we've only ever lost).
|
||||
isv[RL_KELLY_FRACTION_INDEX] = 0.0f;
|
||||
return;
|
||||
}
|
||||
|
||||
const float b = avg_win / avg_loss;
|
||||
const float q = 1.0f - p;
|
||||
const float f_kelly = (p * b - q) / b;
|
||||
const float safety = isv[RL_KELLY_SAFETY_FRAC_INDEX]; // typically 0.5
|
||||
float f = f_kelly * safety;
|
||||
f = fmaxf(0.0f, fminf(f, 1.0f));
|
||||
isv[RL_KELLY_FRACTION_INDEX] = f;
|
||||
}
|
||||
@@ -42,8 +42,10 @@
|
||||
// sampling is perfectly proportional to priority.
|
||||
|
||||
#define RL_PER_ALPHA_INDEX 405
|
||||
#define PER_ALPHA_MIN 0.3f
|
||||
#define PER_ALPHA_MAX 1.0f
|
||||
// PER α MIN/MAX clamp bounds are now ISV-driven per the 2026-05-30
|
||||
// clamp-bound extension. Runtime-tunable + visible in diag.
|
||||
#define RL_PER_ALPHA_MIN_INDEX 647
|
||||
#define RL_PER_ALPHA_MAX_INDEX 648
|
||||
// Kurtosis of a standard normal = 3.0 ("excess kurtosis 0" with the
|
||||
// alternative convention). Used as the breakpoint above which we start
|
||||
// raising α.
|
||||
@@ -52,6 +54,8 @@
|
||||
#define RL_KURT_GAUSSIAN_INDEX 471
|
||||
#define RL_KURT_NOISE_FLOOR_INDEX 472
|
||||
#define RL_KURT_LIFT_SCALE_INDEX 459
|
||||
// Schulman tolerance from the shared global slot.
|
||||
#define RL_SCHULMAN_TOLERANCE_INDEX 468
|
||||
// Noise-floor gate: if the streaming kurtosis estimator emits a value
|
||||
// below this magnitude, treat it as "no signal" (sentinel-zero proxy)
|
||||
// and hold α at the prior value. Without this, on the first few steps
|
||||
@@ -62,7 +66,23 @@
|
||||
// pattern on the other controllers (per
|
||||
// pearl_multiplicative_controllers_need_bounded_step_and_noise_floor).
|
||||
// (KURT_NOISE_FLOOR — was 1.0f #define — now isv[RL_KURT_NOISE_FLOOR_INDEX])
|
||||
#define WIENER_ALPHA_FLOOR 0.4f
|
||||
// Wiener-α floor — shared across 9 controllers (slot 659).
|
||||
#define RL_WIENER_ALPHA_FLOOR_INDEX 659
|
||||
|
||||
// Adaptive controller floors (spec 2026-05-30-adaptive-controller-floor-design):
|
||||
// noise floor augmented with Welford-derived adaptive component (replaces
|
||||
// reliance on the ISV-driven RL_KURT_NOISE_FLOOR_INDEX absolute floor alone).
|
||||
// Asymmetric Schulman semantics: α RISES on a single above-band kurtosis
|
||||
// observation (heavy tails = safety signal, sharpen PER fast); α FALLS only
|
||||
// after N consecutive below-band observations (light tails = patient drift).
|
||||
// Replaces the regime where the controller stuck at MIN 0.4 because every
|
||||
// step's kurtosis read landed just above the absolute floor but well below
|
||||
// the band — the Wiener blend dragged α down even on single observations.
|
||||
#define RL_TD_KURT_VAR_COUNT_INDEX 604
|
||||
#define RL_TD_KURT_VAR_M2_INDEX 606
|
||||
#define RL_TD_KURT_BELOW_COUNT_INDEX 607
|
||||
#define NOISE_FLOOR_STD_MULTIPLIER 2.0f // floor ≥ 2σ of observed kurtosis
|
||||
#define WIDEN_PATIENCE_CONSECUTIVE 3.0f // α descent requires N below-band steps
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────
|
||||
// rl_per_alpha_controller:
|
||||
@@ -97,12 +117,23 @@ extern "C" __global__ void rl_per_alpha_controller(
|
||||
|
||||
const float prev = isv[RL_PER_ALPHA_INDEX];
|
||||
|
||||
// Noise-floor gate: kurtosis below KURT_NOISE_FLOOR is dominated
|
||||
// by streaming-estimator startup noise (per-step batch-mean
|
||||
// Noise-floor gate: kurtosis below the adaptive noise floor is
|
||||
// dominated by streaming-estimator startup noise (per-step batch-mean
|
||||
// differences before tails accumulate). Hold α at prev to avoid
|
||||
// dragging toward PER_ALPHA_MIN on cold-start.
|
||||
//
|
||||
// Adaptive component (spec 2026-05-30): floor scales with observed
|
||||
// kurtosis std. Replaces reliance on the absolute ISV floor alone.
|
||||
// Welford sample variance = M² / (count − 1) when count > 1.
|
||||
const float td_kurtosis_ema = isv[input_slot];
|
||||
if (td_kurtosis_ema > 0.0f && td_kurtosis_ema < isv[RL_KURT_NOISE_FLOOR_INDEX]) {
|
||||
const float kurt_count = isv[RL_TD_KURT_VAR_COUNT_INDEX];
|
||||
const float kurt_var = (kurt_count > 1.0f)
|
||||
? isv[RL_TD_KURT_VAR_M2_INDEX] / (kurt_count - 1.0f)
|
||||
: 0.0f;
|
||||
const float kurt_std = sqrtf(kurt_var);
|
||||
const float adaptive_noise_floor = fmaxf(isv[RL_KURT_NOISE_FLOOR_INDEX],
|
||||
kurt_std * NOISE_FLOOR_STD_MULTIPLIER);
|
||||
if (td_kurtosis_ema > 0.0f && td_kurtosis_ema < adaptive_noise_floor) {
|
||||
// Real signal present but below the noise floor — hold prev.
|
||||
// (Strict sentinel zero handled by the prev==0 bootstrap path
|
||||
// below, which derives target from the current EMA so the
|
||||
@@ -120,10 +151,12 @@ extern "C" __global__ void rl_per_alpha_controller(
|
||||
// The 0.4-0.6 baseline keeps the steady-state output near PER's
|
||||
// canonical 0.6 (when input EMA stabilises at kurt=10) while
|
||||
// leaving headroom to lift toward 1.0 under heavy tails.
|
||||
const float per_alpha_min = isv[RL_PER_ALPHA_MIN_INDEX];
|
||||
const float per_alpha_max = isv[RL_PER_ALPHA_MAX_INDEX];
|
||||
const float kurt_excess = fmaxf(0.0f, td_kurtosis_ema - isv[RL_KURT_GAUSSIAN_INDEX]);
|
||||
const float kurt_lift_scale = isv[RL_KURT_LIFT_SCALE_INDEX];
|
||||
float target = 0.4f + 0.2f * (kurt_excess / kurt_lift_scale);
|
||||
target = fmaxf(PER_ALPHA_MIN, fminf(target, PER_ALPHA_MAX));
|
||||
target = fmaxf(per_alpha_min, fminf(target, per_alpha_max));
|
||||
|
||||
// Bootstrap on sentinel 0.0 per pearl_first_observation_bootstrap:
|
||||
// first emit replaces directly with the computed target. At
|
||||
@@ -138,9 +171,29 @@ extern "C" __global__ void rl_per_alpha_controller(
|
||||
}
|
||||
|
||||
// Wiener-α blend with floor per pearl_wiener_alpha_floor_for_nonstationary.
|
||||
const float a = fmaxf(alpha_step, WIENER_ALPHA_FLOOR);
|
||||
const float a = fmaxf(alpha_step, isv[RL_WIENER_ALPHA_FLOOR_INDEX]);
|
||||
float out = (1.0f - a) * prev + a * target;
|
||||
|
||||
out = fmaxf(PER_ALPHA_MIN, fminf(out, PER_ALPHA_MAX));
|
||||
out = fmaxf(per_alpha_min, fminf(out, per_alpha_max));
|
||||
|
||||
// Asymmetric Schulman patience on the DESCENT direction (spec 2026-05-30):
|
||||
// PER α rises on a single above-band kurtosis observation (heavy tails
|
||||
// = act fast to concentrate sampling on informative tails). PER α
|
||||
// falls only after N consecutive below-band observations — a single
|
||||
// light-tailed step must not drag α toward MIN.
|
||||
const float kurt_gaussian = isv[RL_KURT_GAUSSIAN_INDEX];
|
||||
const float tolerance = isv[RL_SCHULMAN_TOLERANCE_INDEX];
|
||||
if (out < prev && td_kurtosis_ema < kurt_gaussian / tolerance) {
|
||||
const float new_count = isv[RL_TD_KURT_BELOW_COUNT_INDEX] + 1.0f;
|
||||
isv[RL_TD_KURT_BELOW_COUNT_INDEX] = new_count;
|
||||
if (new_count < WIDEN_PATIENCE_CONSECUTIVE) {
|
||||
// Hold at prev until patience accumulates.
|
||||
isv[RL_PER_ALPHA_INDEX] = prev;
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
isv[RL_TD_KURT_BELOW_COUNT_INDEX] = 0.0f;
|
||||
}
|
||||
|
||||
isv[RL_PER_ALPHA_INDEX] = out;
|
||||
}
|
||||
|
||||
@@ -42,16 +42,36 @@
|
||||
// when ratios run away).
|
||||
|
||||
#define RL_PPO_CLIP_INDEX 402
|
||||
#define EPS_MIN 0.05f
|
||||
#define EPS_MAX 0.5f
|
||||
// ε MIN/MAX are now ISV-driven per the 2026-05-30 clamp-bound extension —
|
||||
// the prior hardcoded `[0.05, 0.5]` calibration was a fixed-regime choice
|
||||
// that contributed to fold 0/1 saturation when Phase 4.5 narrowed the KL
|
||||
// signal distribution. ISV-residence makes the bound runtime-tunable
|
||||
// (visible in diag JSONL, re-seedable without recompile).
|
||||
#define RL_PPO_CLIP_EPS_MIN_INDEX 640
|
||||
#define RL_PPO_CLIP_EPS_MAX_INDEX 641
|
||||
// ISV-driven bootstrap + KL target per `feedback_isv_for_adaptive_bounds`.
|
||||
#define RL_EPS_BOOTSTRAP_INDEX 474
|
||||
#define RL_KL_TARGET_INDEX 454
|
||||
// Schulman pattern parameters — global slots shared by 4 controllers.
|
||||
#define RL_SCHULMAN_TOLERANCE_INDEX 468
|
||||
#define RL_SCHULMAN_ADJUST_RATE_INDEX 469
|
||||
#define KL_NOISE_FLOOR_FRAC 0.01f
|
||||
#define WIENER_ALPHA_FLOOR 0.4f
|
||||
// Wiener-α floor — shared across 9 controllers per
|
||||
// `pearl_wiener_alpha_floor_for_nonstationary`. ISV-resident at slot 659
|
||||
// so all consumers stay aligned without per-file `#define` drift.
|
||||
#define RL_WIENER_ALPHA_FLOOR_INDEX 659
|
||||
|
||||
// Adaptive controller floors (spec 2026-05-30-adaptive-controller-floor-design):
|
||||
// noise floor derives from observed kl_pi_ema variance via Welford triples,
|
||||
// asymmetric Schulman widening requires N consecutive below-band observations.
|
||||
// Replaces hardcoded KL_NOISE_FLOOR_FRAC=0.01f which was the root cause of
|
||||
// the fold 0/1 saturation (ε locked at MAX 0.50 within 50 steps because the
|
||||
// 1%-of-target floor was 100× too low for the post-Phase-4.5 KL regime).
|
||||
#define RL_KL_PI_VAR_COUNT_INDEX 588
|
||||
#define RL_KL_PI_VAR_M2_INDEX 590
|
||||
#define RL_KL_PI_BELOW_COUNT_INDEX 591
|
||||
#define NOISE_FLOOR_TARGET_FRAC 0.5f // floor ≥ 50% of target
|
||||
#define NOISE_FLOOR_STD_MULTIPLIER 2.0f // floor ≥ 2σ of observed signal
|
||||
#define WIDEN_PATIENCE_CONSECUTIVE 3.0f // widen requires N below-band steps
|
||||
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────
|
||||
@@ -93,26 +113,49 @@ extern "C" __global__ void rl_ppo_clip_controller(
|
||||
|
||||
// ISV-driven KL target (was hardcoded #define).
|
||||
const float kl_target = isv[RL_KL_TARGET_INDEX];
|
||||
const float kl_noise_floor = kl_target * KL_NOISE_FLOOR_FRAC;
|
||||
|
||||
// Noise-floor gate: KL below kl_noise_floor is dominated by
|
||||
// numerical noise, not real policy divergence. Hold ε unchanged.
|
||||
// Adaptive noise floor — signal-driven per
|
||||
// `pearl_zscore_normalization_for_magnitude_asymmetric_signals` and
|
||||
// `feedback_adaptive_not_tuned`. Floor must scale with observed signal
|
||||
// magnitude AND have an absolute minimum tied to the target so a
|
||||
// momentarily-noisy signal can't trigger spurious adjustments.
|
||||
// Welford sample variance = M² / (count − 1) when count > 1.
|
||||
const float kl_count = isv[RL_KL_PI_VAR_COUNT_INDEX];
|
||||
const float kl_var = (kl_count > 1.0f)
|
||||
? isv[RL_KL_PI_VAR_M2_INDEX] / (kl_count - 1.0f)
|
||||
: 0.0f;
|
||||
const float kl_std = sqrtf(kl_var);
|
||||
const float kl_noise_floor = fmaxf(kl_target * NOISE_FLOOR_TARGET_FRAC,
|
||||
kl_std * NOISE_FLOOR_STD_MULTIPLIER);
|
||||
|
||||
// Noise-floor gate: KL below the adaptive floor is dominated by
|
||||
// numerical noise OR signal stays naturally below target under the
|
||||
// current architecture (e.g., Phase 4.5 reduces KL by ~50×). Hold ε.
|
||||
if (kl_ema < kl_noise_floor) return;
|
||||
|
||||
// Bounded multiplicative adjustment (Schulman-style adaptive KL).
|
||||
// Schulman params from ISV — shared with target_tau, rollout_steps.
|
||||
// Asymmetric Schulman (spec 2026-05-30 Section "Approach C, folded in"):
|
||||
// tightening fires on a single above-band observation — real policy
|
||||
// divergence is a safety signal we act on fast. Widening requires
|
||||
// N consecutive below-band observations so a single noisy step can't
|
||||
// drive ε past bootstrap. Below-band counter is per-controller in ISV.
|
||||
const float tolerance = isv[RL_SCHULMAN_TOLERANCE_INDEX];
|
||||
const float adjust_rate = isv[RL_SCHULMAN_ADJUST_RATE_INDEX];
|
||||
float ratio;
|
||||
if (kl_ema > kl_target * tolerance) {
|
||||
ratio = 1.0f / adjust_rate;
|
||||
ratio = 1.0f / adjust_rate; // tighten — single observation
|
||||
isv[RL_KL_PI_BELOW_COUNT_INDEX] = 0.0f; // reset patience
|
||||
} else if (kl_ema < kl_target / tolerance) {
|
||||
ratio = adjust_rate;
|
||||
const float new_count = isv[RL_KL_PI_BELOW_COUNT_INDEX] + 1.0f;
|
||||
isv[RL_KL_PI_BELOW_COUNT_INDEX] = new_count;
|
||||
ratio = (new_count >= WIDEN_PATIENCE_CONSECUTIVE) ? adjust_rate : 1.0f;
|
||||
} else {
|
||||
isv[RL_KL_PI_BELOW_COUNT_INDEX] = 0.0f; // in-band → reset
|
||||
ratio = 1.0f;
|
||||
}
|
||||
const float eps_min = isv[RL_PPO_CLIP_EPS_MIN_INDEX];
|
||||
const float eps_max = isv[RL_PPO_CLIP_EPS_MAX_INDEX];
|
||||
float eps_target = eps_prev * ratio;
|
||||
eps_target = fmaxf(EPS_MIN, fminf(eps_target, EPS_MAX));
|
||||
eps_target = fmaxf(eps_min, fminf(eps_target, eps_max));
|
||||
|
||||
// First-observation replace-directly per
|
||||
// `pearl_first_observation_bootstrap`. See rl_target_tau_controller
|
||||
@@ -125,9 +168,9 @@ extern "C" __global__ void rl_ppo_clip_controller(
|
||||
}
|
||||
|
||||
// Wiener-α blend with floor per pearl_wiener_alpha_floor_for_nonstationary.
|
||||
const float a = fmaxf(alpha, WIENER_ALPHA_FLOOR);
|
||||
const float a = fmaxf(alpha, isv[RL_WIENER_ALPHA_FLOOR_INDEX]);
|
||||
float eps_new = (1.0f - a) * eps_prev + a * eps_target;
|
||||
|
||||
eps_new = fmaxf(EPS_MIN, fminf(eps_new, EPS_MAX));
|
||||
eps_new = fmaxf(eps_min, fminf(eps_new, eps_max));
|
||||
isv[RL_PPO_CLIP_INDEX] = eps_new;
|
||||
}
|
||||
|
||||
@@ -53,9 +53,21 @@
|
||||
// Default 10.0 — clamp_max = (1+ε) × this. Seeded by rl_isv_write
|
||||
// at init. Tuning lower tightens the importance-ratio bound.
|
||||
#define RL_PPO_CLAMP_MARGIN_INDEX 460
|
||||
#define PPO_RATIO_CLAMP_MIN_OUT 2.0f
|
||||
#define PPO_RATIO_CLAMP_MAX_OUT 1000.0f
|
||||
#define WIENER_ALPHA_FLOOR 0.4f
|
||||
|
||||
// Adaptive controller floors (spec 2026-05-30-adaptive-controller-floor-design):
|
||||
// the MIN_OUT floor derives from observed log-ratio variance (Welford
|
||||
// triple at slots 630/631/632) rather than the static `2.0f` baseline.
|
||||
// The MAX ceiling is ISV-resident at slot 629 so it can be tuned at
|
||||
// runtime. The hardcoded `2.0f` survives as an architectural minimum
|
||||
// per the spec's "physics-constant" exemption ("don't degenerate to
|
||||
// vanilla policy gradient when σ_ratio is tiny").
|
||||
#define RL_PPO_RATIO_CLAMP_MAX_ADAPTIVE_INDEX 629
|
||||
#define RL_PPO_RATIO_VAR_COUNT_INDEX 630
|
||||
#define RL_PPO_RATIO_VAR_M2_INDEX 632
|
||||
#define RL_WIENER_ALPHA_FLOOR_INDEX 659
|
||||
#define PPO_RATIO_MIN_ABSOLUTE 2.0f // architectural exemption: vanilla-PG safeguard
|
||||
#define PPO_RATIO_MIN_STD_SCALE 3.0f // adaptive min = max(2.0, 1 + 3σ)
|
||||
#define PPO_RATIO_MAX_OVER_MIN_FACTOR 5.0f // adaptive max ≥ 5× adaptive min
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────
|
||||
// rl_ppo_ratio_clamp_controller:
|
||||
@@ -98,9 +110,25 @@ extern "C" __global__ void rl_ppo_ratio_clamp_controller(
|
||||
const float clamp_margin = isv[RL_PPO_CLAMP_MARGIN_INDEX];
|
||||
float target = (1.0f + eps) * clamp_margin;
|
||||
|
||||
// ── Adaptive MIN/MAX from observed log-ratio variance ───────────
|
||||
// Welford sample variance = M² / (count − 1) when count > 1.
|
||||
// Adaptive min: max(2.0, 1 + 3σ_ratio). The 2.0 floor is the
|
||||
// architectural-minimum exemption ("don't degenerate to vanilla
|
||||
// policy gradient") — kept as a #define per the spec. The 3σ scale
|
||||
// gives the clamp window roughly the same coverage of healthy
|
||||
// policy updates as the prior static 2.0 baseline did pre-Phase 4.5.
|
||||
const float ratio_count = isv[RL_PPO_RATIO_VAR_COUNT_INDEX];
|
||||
const float ratio_var = (ratio_count > 1.0f)
|
||||
? isv[RL_PPO_RATIO_VAR_M2_INDEX] / (ratio_count - 1.0f)
|
||||
: 0.0f;
|
||||
const float ratio_std = sqrtf(ratio_var);
|
||||
const float adaptive_min = fmaxf(PPO_RATIO_MIN_ABSOLUTE,
|
||||
1.0f + ratio_std * PPO_RATIO_MIN_STD_SCALE);
|
||||
const float adaptive_max = fmaxf(adaptive_min * PPO_RATIO_MAX_OVER_MIN_FACTOR,
|
||||
isv[RL_PPO_RATIO_CLAMP_MAX_ADAPTIVE_INDEX]);
|
||||
|
||||
// Permanent floor / ceiling per pearl_blend_formulas_must_have_permanent_floor.
|
||||
target = fmaxf(PPO_RATIO_CLAMP_MIN_OUT,
|
||||
fminf(target, PPO_RATIO_CLAMP_MAX_OUT));
|
||||
target = fmaxf(adaptive_min, fminf(target, adaptive_max));
|
||||
|
||||
// First-observation replace-directly per pearl_first_observation_bootstrap.
|
||||
// prev == PPO_RATIO_CLAMP_BOOTSTRAP means "we just bootstrapped last
|
||||
@@ -113,10 +141,9 @@ extern "C" __global__ void rl_ppo_ratio_clamp_controller(
|
||||
}
|
||||
|
||||
// Wiener-α blend with floor.
|
||||
const float a = fmaxf(alpha_step, WIENER_ALPHA_FLOOR);
|
||||
const float a = fmaxf(alpha_step, isv[RL_WIENER_ALPHA_FLOOR_INDEX]);
|
||||
float out = (1.0f - a) * prev + a * target;
|
||||
|
||||
out = fmaxf(PPO_RATIO_CLAMP_MIN_OUT,
|
||||
fminf(out, PPO_RATIO_CLAMP_MAX_OUT));
|
||||
out = fmaxf(adaptive_min, fminf(out, adaptive_max));
|
||||
isv[RL_PPO_RATIO_CLAMP_MAX_INDEX] = out;
|
||||
}
|
||||
|
||||
@@ -23,11 +23,27 @@
|
||||
#define RL_Q_DISTILL_KL_EMA_INDEX 488
|
||||
#define RL_Q_DISTILL_KL_TARGET_INDEX 491
|
||||
|
||||
#define MIN_LAMBDA 0.05f
|
||||
#define MAX_LAMBDA 1.0f
|
||||
#define KL_TOLERANCE 3.0f
|
||||
#define LAMBDA_RAMP_RATE 1.2f
|
||||
#define LAMBDA_DECAY_RATE 0.998f
|
||||
// Adaptive controller floors (spec 2026-05-30 Special case Q):
|
||||
// hardcoded `MIN_LAMBDA = 0.05f` floor replaced with adaptive bound
|
||||
// derived from Welford variance on q_distill_kl_ema. Phase 4.5 reduces
|
||||
// KL signal magnitudes which this controller consumes; the "below target"
|
||||
// path fires continuously, decaying λ to MIN. Adaptive bound
|
||||
// `max(0.001, sqrt(var) × 0.05)` lets λ decay to a level proportional to
|
||||
// the actual signal noise rather than the hardcoded 0.05 floor.
|
||||
#define RL_Q_DISTILL_KL_VAR_COUNT_INDEX 618
|
||||
#define RL_Q_DISTILL_KL_VAR_M2_INDEX 620
|
||||
#define RL_Q_DISTILL_LAMBDA_MIN_ADAPTIVE_INDEX 639
|
||||
|
||||
// MAX_LAMBDA, KL_TOLERANCE, LAMBDA_RAMP_RATE, LAMBDA_DECAY_RATE are now
|
||||
// ISV-driven per the 2026-05-30 clamp-bound extension. ADAPTIVE_MIN_*
|
||||
// remain hardcoded — they're new constants from the noise-floor design
|
||||
// itself per spec exemption ("not clamp bounds, new pattern constants").
|
||||
#define RL_Q_DISTILL_LAMBDA_MAX_INDEX 650
|
||||
#define RL_Q_DISTILL_KL_TOLERANCE_INDEX 651
|
||||
#define RL_Q_DISTILL_LAMBDA_RAMP_RATE_INDEX 652
|
||||
#define RL_Q_DISTILL_LAMBDA_DECAY_RATE_INDEX 653
|
||||
#define ADAPTIVE_MIN_ABSOLUTE 0.001f
|
||||
#define ADAPTIVE_MIN_STD_SCALE 0.05f
|
||||
|
||||
extern "C" __global__ void rl_q_distill_lambda_controller(
|
||||
float* __restrict__ isv
|
||||
@@ -40,13 +56,32 @@ extern "C" __global__ void rl_q_distill_lambda_controller(
|
||||
|
||||
if (kl_observed <= 0.0f || kl_target <= 0.0f || lambda <= 0.0f) return;
|
||||
|
||||
const float upper = kl_target * KL_TOLERANCE;
|
||||
const float lower = kl_target / KL_TOLERANCE;
|
||||
// Adaptive MIN bound (spec 2026-05-30 Special case Q): replaces
|
||||
// hardcoded `MIN_LAMBDA = 0.05f`. Welford sample variance =
|
||||
// M² / (count − 1) when count > 1. Adaptive min scales with signal
|
||||
// std so λ can decay proportional to actual KL noise rather than
|
||||
// pegging at an arbitrary 0.05 floor.
|
||||
const float kl_var_count = isv[RL_Q_DISTILL_KL_VAR_COUNT_INDEX];
|
||||
const float kl_var = (kl_var_count > 1.0f)
|
||||
? isv[RL_Q_DISTILL_KL_VAR_M2_INDEX] / (kl_var_count - 1.0f)
|
||||
: 0.0f;
|
||||
const float kl_std = sqrtf(kl_var);
|
||||
const float adaptive_min = fmaxf(ADAPTIVE_MIN_ABSOLUTE,
|
||||
kl_std * ADAPTIVE_MIN_STD_SCALE);
|
||||
isv[RL_Q_DISTILL_LAMBDA_MIN_ADAPTIVE_INDEX] = adaptive_min;
|
||||
|
||||
const float max_lambda = isv[RL_Q_DISTILL_LAMBDA_MAX_INDEX];
|
||||
const float kl_tolerance = isv[RL_Q_DISTILL_KL_TOLERANCE_INDEX];
|
||||
const float lambda_ramp_rate = isv[RL_Q_DISTILL_LAMBDA_RAMP_RATE_INDEX];
|
||||
const float lambda_decay_rate = isv[RL_Q_DISTILL_LAMBDA_DECAY_RATE_INDEX];
|
||||
|
||||
const float upper = kl_target * kl_tolerance;
|
||||
const float lower = kl_target / kl_tolerance;
|
||||
|
||||
if (kl_observed > upper) {
|
||||
lambda = fminf(MAX_LAMBDA, lambda * LAMBDA_RAMP_RATE);
|
||||
lambda = fminf(max_lambda, lambda * lambda_ramp_rate);
|
||||
} else if (kl_observed < lower) {
|
||||
lambda = fmaxf(MIN_LAMBDA, lambda * LAMBDA_DECAY_RATE);
|
||||
lambda = fmaxf(adaptive_min, lambda * lambda_decay_rate);
|
||||
}
|
||||
|
||||
isv[RL_Q_DISTILL_LAMBDA_INDEX] = lambda;
|
||||
|
||||
@@ -88,25 +88,21 @@
|
||||
#define RL_NEG_SCALED_REWARD_MAX_INDEX 489
|
||||
#define RL_NEG_SCALED_REWARD_MAX_EMA_INDEX 490
|
||||
|
||||
#define MIN_WIN 1.0f
|
||||
#define MIN_RATIO 1.0f // no inverted asymmetry (loss never < win)
|
||||
#define MAX_RATIO 3.0f // no worse than original loss-aversion
|
||||
#define V_BOUND_FLOOR 1.0f // |V_MIN| and V_MAX both ≥ 1.0
|
||||
#define V_BOUND_EWMA_ALPHA 0.001f // slow — half-life ~700 steps
|
||||
#define WIENER_ALPHA_FLOOR 0.4f
|
||||
|
||||
// MARGIN Schulman bounded-step bounds + adjust rate per
|
||||
// `pearl_multiplicative_controllers_need_bounded_step_and_noise_floor`.
|
||||
// TOLERANCE=1.5 → dead-zone is [target/1.5, target×1.5] = [0.033, 0.075];
|
||||
// ADJUST_RATE=1.2 → MARGIN moves by ≤20% per controller call (one per
|
||||
// trainer step). Bounds [MIN_MARGIN=1.0, MAX_MARGIN=5.0] — MIN preserves
|
||||
// the initial passthrough behaviour; MAX × (mean pos_max_ema≈3-5) →
|
||||
// WIN_eff up to ~15-25 which is well within MAX_WIN=50 ceiling.
|
||||
#define MIN_MARGIN 1.0f
|
||||
#define MAX_MARGIN 5.0f
|
||||
#define MARGIN_TOLERANCE 1.5f
|
||||
#define MARGIN_ADJUST_RATE 1.2f
|
||||
#define CLIP_RATE_EMA_ALPHA 0.05f
|
||||
// Adaptive controller floors (spec 2026-05-30-adaptive-controller-floor-design):
|
||||
// every prior `#define` clamp-bound constant becomes ISV-driven so it can
|
||||
// be tuned at runtime via re-seed instead of recompile. Defaults preserve
|
||||
// the historic behaviour exactly.
|
||||
#define RL_REWARD_CLAMP_V_BOUND_FLOOR_INDEX 633
|
||||
#define RL_REWARD_CLAMP_V_BOUND_EWMA_ALPHA_INDEX 634
|
||||
#define RL_REWARD_CLAMP_MIN_WIN_INDEX 635
|
||||
#define RL_REWARD_CLAMP_MIN_RATIO_INDEX 636
|
||||
#define RL_REWARD_CLAMP_MAX_RATIO_INDEX 637
|
||||
#define RL_REWARD_CLAMP_MIN_MARGIN_INDEX 654
|
||||
#define RL_REWARD_CLAMP_MAX_MARGIN_INDEX 655
|
||||
#define RL_REWARD_CLAMP_MARGIN_TOLERANCE_INDEX 656
|
||||
#define RL_REWARD_CLAMP_MARGIN_ADJUST_RATE_INDEX 657
|
||||
#define RL_REWARD_CLAMP_CLIP_RATE_EMA_ALPHA_INDEX 658
|
||||
#define RL_WIENER_ALPHA_FLOOR_INDEX 659
|
||||
|
||||
extern "C" __global__ void rl_reward_clamp_controller(
|
||||
float* __restrict__ isv,
|
||||
@@ -114,6 +110,21 @@ extern "C" __global__ void rl_reward_clamp_controller(
|
||||
) {
|
||||
if (threadIdx.x != 0 || blockIdx.x != 0) return;
|
||||
|
||||
// ── ISV-driven design constants (spec 2026-05-30) ──
|
||||
// Loaded once at the top of the kernel so they're consistent across
|
||||
// all sub-controllers (EMA → MARGIN → RATIO → C51 span) within this
|
||||
// step. Default values preserve historic behaviour exactly.
|
||||
const float v_bound_floor = isv[RL_REWARD_CLAMP_V_BOUND_FLOOR_INDEX];
|
||||
const float v_bound_ewma_alpha = isv[RL_REWARD_CLAMP_V_BOUND_EWMA_ALPHA_INDEX];
|
||||
const float min_ratio_isv = isv[RL_REWARD_CLAMP_MIN_RATIO_INDEX];
|
||||
const float max_ratio_isv = isv[RL_REWARD_CLAMP_MAX_RATIO_INDEX];
|
||||
const float min_margin_isv = isv[RL_REWARD_CLAMP_MIN_MARGIN_INDEX];
|
||||
const float max_margin_isv = isv[RL_REWARD_CLAMP_MAX_MARGIN_INDEX];
|
||||
const float margin_tolerance = isv[RL_REWARD_CLAMP_MARGIN_TOLERANCE_INDEX];
|
||||
const float margin_adjust_rate = isv[RL_REWARD_CLAMP_MARGIN_ADJUST_RATE_INDEX];
|
||||
const float clip_rate_ema_alpha = isv[RL_REWARD_CLAMP_CLIP_RATE_EMA_ALPHA_INDEX];
|
||||
const float wiener_floor = isv[RL_WIENER_ALPHA_FLOOR_INDEX];
|
||||
|
||||
const float pos_max = isv[RL_POS_SCALED_REWARD_MAX_INDEX];
|
||||
const float ema_prev = isv[RL_POS_SCALED_REWARD_MAX_EMA_INDEX];
|
||||
|
||||
@@ -136,7 +147,7 @@ extern "C" __global__ void rl_reward_clamp_controller(
|
||||
if (ema_prev == 0.0f) {
|
||||
ema_new = pos_max; // first-observation bootstrap
|
||||
} else {
|
||||
const float a = fmaxf(alpha, WIENER_ALPHA_FLOOR);
|
||||
const float a = fmaxf(alpha, wiener_floor);
|
||||
ema_new = (1.0f - a) * ema_prev + a * pos_max;
|
||||
}
|
||||
isv[RL_POS_SCALED_REWARD_MAX_EMA_INDEX] = ema_new;
|
||||
@@ -171,8 +182,8 @@ extern "C" __global__ void rl_reward_clamp_controller(
|
||||
if (clip_rate_prev == 0.0f) {
|
||||
clip_rate_new = clip_indicator; // bootstrap
|
||||
} else {
|
||||
clip_rate_new = (1.0f - CLIP_RATE_EMA_ALPHA) * clip_rate_prev
|
||||
+ CLIP_RATE_EMA_ALPHA * clip_indicator;
|
||||
clip_rate_new = (1.0f - clip_rate_ema_alpha) * clip_rate_prev
|
||||
+ clip_rate_ema_alpha * clip_indicator;
|
||||
}
|
||||
isv[RL_REWARD_CLAMP_CLIP_RATE_EMA_INDEX] = clip_rate_new;
|
||||
}
|
||||
@@ -184,12 +195,12 @@ extern "C" __global__ void rl_reward_clamp_controller(
|
||||
float margin = isv[RL_REWARD_CLAMP_MARGIN_INDEX];
|
||||
if (pos_max > 0.0f) {
|
||||
const float clip_target = isv[RL_REWARD_CLAMP_CLIP_RATE_TARGET_INDEX];
|
||||
const float upper = clip_target * MARGIN_TOLERANCE;
|
||||
const float lower = clip_target / MARGIN_TOLERANCE;
|
||||
const float upper = clip_target * margin_tolerance;
|
||||
const float lower = clip_target / margin_tolerance;
|
||||
if (clip_rate_new > upper) {
|
||||
margin = fminf(MAX_MARGIN, margin * MARGIN_ADJUST_RATE);
|
||||
margin = fminf(max_margin_isv, margin * margin_adjust_rate);
|
||||
} else if (clip_rate_new < lower) {
|
||||
margin = fmaxf(MIN_MARGIN, margin / MARGIN_ADJUST_RATE);
|
||||
margin = fmaxf(min_margin_isv, margin / margin_adjust_rate);
|
||||
}
|
||||
isv[RL_REWARD_CLAMP_MARGIN_INDEX] = margin;
|
||||
}
|
||||
@@ -210,7 +221,7 @@ extern "C" __global__ void rl_reward_clamp_controller(
|
||||
if (neg_prev == 0.0f) {
|
||||
neg_new = neg_max;
|
||||
} else {
|
||||
const float a = fmaxf(alpha, WIENER_ALPHA_FLOOR);
|
||||
const float a = fmaxf(alpha, wiener_floor);
|
||||
neg_new = (1.0f - a) * neg_prev + a * neg_max;
|
||||
}
|
||||
isv[RL_NEG_SCALED_REWARD_MAX_EMA_INDEX] = neg_new;
|
||||
@@ -219,16 +230,40 @@ extern "C" __global__ void rl_reward_clamp_controller(
|
||||
// statically-seeded RATIO=3.0 until both EMAs are non-zero.
|
||||
if (ema_new > 0.0f && neg_new > 0.0f) {
|
||||
const float ratio_target = neg_new / ema_new;
|
||||
const float ratio_clamped = fmaxf(MIN_RATIO,
|
||||
fminf(MAX_RATIO, ratio_target));
|
||||
const float ratio_clamped = fmaxf(min_ratio_isv,
|
||||
fminf(max_ratio_isv, ratio_target));
|
||||
isv[RL_REWARD_CLAMP_RATIO_INDEX] = ratio_clamped;
|
||||
}
|
||||
|
||||
// Step 4 (G.2): WIN / LOSS clamp bounds remain STRUCTURAL — the
|
||||
// G.2 feedback loop required BOTH clamp widening AND span widening.
|
||||
// With clamp frozen here, only the atom span adapts (Step 5 below).
|
||||
// Suppress unused warnings on diagnostic-only state.
|
||||
(void) margin;
|
||||
// ── Step 4: Adaptive WIN / LOSS clamp writeback. ──────────────
|
||||
//
|
||||
// History: a 2026-05-24 G.2 patch froze the clamp bounds at the
|
||||
// trainer-seeded WIN=1.0, LOSS=3.0 over concern that simultaneously
|
||||
// adapting clamps + atom span could form a positive feedback loop.
|
||||
//
|
||||
// Empirical refutation (alpha-rl-gwmkf step 4083, 2026-05-30):
|
||||
// with R-multiple 43 trade distributions (avg_win $7062, avg_loss
|
||||
// $163) the frozen WIN=1.0 caused clip_rate_ema = 0.9999 —
|
||||
// essentially every win gets clamped, V regression sees identical
|
||||
// targets for +$500 wins and +$42k wins, and PPO advantage loses
|
||||
// magnitude signal. Diagnosis: starved policy gradient.
|
||||
//
|
||||
// Safety argument: the MARGIN controller above already saturates
|
||||
// at MAX_MARGIN_ISV (bounded ≤ 5.0). pos_max_ema is bounded by
|
||||
// `reward_scale × raw_PnL`, both finite. So WIN ≤ MAX_MARGIN ×
|
||||
// MAX_REWARD_SCALE × MAX_RAW_PNL is structurally bounded.
|
||||
//
|
||||
// Atom span (Step 5) follows the new WIN via slow EWMA (α=0.001,
|
||||
// half-life ~700 steps), so resolution adjusts gradually as the
|
||||
// reward distribution evolves rather than whipsawing.
|
||||
if (ema_new > 0.0f) {
|
||||
const float min_win_isv = isv[RL_REWARD_CLAMP_MIN_WIN_INDEX];
|
||||
const float win_new = fmaxf(min_win_isv, margin * ema_new);
|
||||
const float ratio = isv[RL_REWARD_CLAMP_RATIO_INDEX];
|
||||
const float loss_new = fmaxf(min_win_isv, win_new * ratio);
|
||||
isv[RL_REWARD_CLAMP_WIN_INDEX] = win_new;
|
||||
isv[RL_REWARD_CLAMP_LOSS_INDEX] = loss_new;
|
||||
}
|
||||
|
||||
// ── Step 5: C51 atom span adaptation from observed reward EMAs. ──
|
||||
//
|
||||
@@ -255,20 +290,20 @@ extern "C" __global__ void rl_reward_clamp_controller(
|
||||
const float loss_bound = isv[RL_REWARD_CLAMP_LOSS_INDEX]; // 3.0
|
||||
{
|
||||
const float v_max_prev = isv[RL_C51_V_MAX_INDEX];
|
||||
const float v_max_target = fmaxf(V_BOUND_FLOOR, win_bound);
|
||||
const float v_max_target = fmaxf(v_bound_floor, win_bound);
|
||||
const float v_max_new = (v_max_prev == 0.0f)
|
||||
? v_max_target
|
||||
: (1.0f - V_BOUND_EWMA_ALPHA) * v_max_prev
|
||||
+ V_BOUND_EWMA_ALPHA * v_max_target;
|
||||
isv[RL_C51_V_MAX_INDEX] = fmaxf(V_BOUND_FLOOR, v_max_new);
|
||||
: (1.0f - v_bound_ewma_alpha) * v_max_prev
|
||||
+ v_bound_ewma_alpha * v_max_target;
|
||||
isv[RL_C51_V_MAX_INDEX] = fmaxf(v_bound_floor, v_max_new);
|
||||
}
|
||||
{
|
||||
const float v_min_prev = isv[RL_C51_V_MIN_INDEX];
|
||||
const float v_min_target = fminf(-V_BOUND_FLOOR, -loss_bound);
|
||||
const float v_min_target = fminf(-v_bound_floor, -loss_bound);
|
||||
const float v_min_new = (v_min_prev == 0.0f)
|
||||
? v_min_target
|
||||
: (1.0f - V_BOUND_EWMA_ALPHA) * v_min_prev
|
||||
+ V_BOUND_EWMA_ALPHA * v_min_target;
|
||||
isv[RL_C51_V_MIN_INDEX] = fminf(-V_BOUND_FLOOR, v_min_new);
|
||||
: (1.0f - v_bound_ewma_alpha) * v_min_prev
|
||||
+ v_bound_ewma_alpha * v_min_target;
|
||||
isv[RL_C51_V_MIN_INDEX] = fminf(-v_bound_floor, v_min_new);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -48,7 +48,30 @@
|
||||
// Floor for the mean_abs_pnl_ema denominator to guard against div-by-zero
|
||||
// when the EMA hasn't accumulated any closed trades yet.
|
||||
#define EPS_PNL 1e-3f
|
||||
#define WIENER_ALPHA_FLOOR 0.4f
|
||||
// Wiener-α floor — shared across 9 controllers (slot 659).
|
||||
#define RL_WIENER_ALPHA_FLOOR_INDEX 659
|
||||
|
||||
// Adaptive controller floors (spec 2026-05-30 Special case R):
|
||||
// asymmetric rate limit on DECREASE (new ≥ prev / 1.05) + bootstrap-fraction
|
||||
// floor until N=100 trades have closed. Together these prevent the
|
||||
// catastrophic 1.0 → 0.004 crash in 100 steps observed in fold 0 (where a
|
||||
// single large pnl signal — pre-trade — would crater the scale before any
|
||||
// closed-trade signal could anchor it).
|
||||
//
|
||||
// Trade-count signal: RL_TRADE_DUR_VAR_COUNT_INDEX (608) is the Welford
|
||||
// count over per-batch trade durations — incremented once per closed trade.
|
||||
// Reusing it as the trade-count signal avoids a parallel counter.
|
||||
//
|
||||
// Emit: RL_REWARD_MAGNITUDE_EMA_INDEX (614) mirrors the input EMA so the
|
||||
// downstream Welford kernel can track variance for noise-floor derivation
|
||||
// elsewhere in the controller cascade. Single-source-of-truth: this
|
||||
// controller's input IS the pnl magnitude EMA; slot 614 is the public
|
||||
// emit-slot consumed by the Welford-variance kernel.
|
||||
#define RL_TRADE_DUR_VAR_COUNT_INDEX 608
|
||||
#define RL_REWARD_MAGNITUDE_EMA_INDEX 614
|
||||
#define BOOTSTRAP_FRACTION_FLOOR 0.1f
|
||||
#define MIN_TRADES_FOR_RELEASE 100.0f
|
||||
#define ASYM_DECREASE_RATE 1.05f
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────
|
||||
// rl_reward_scale_controller:
|
||||
@@ -105,6 +128,15 @@ extern "C" __global__ void rl_reward_scale_controller(
|
||||
// noise.
|
||||
const float mean_abs_pnl_ema = isv[input_slot];
|
||||
if (mean_abs_pnl_ema == 0.0f) return;
|
||||
|
||||
// Emit per-batch pnl magnitude EMA to the public slot so the Welford
|
||||
// variance kernel and any other downstream consumer can track it
|
||||
// without re-reading the input slot through indirect plumbing.
|
||||
// Single-source-of-truth: this controller's input IS the pnl magnitude
|
||||
// EMA — slot 614 mirrors it as the public emit (spec 2026-05-30
|
||||
// Special case R bullet (c)).
|
||||
isv[RL_REWARD_MAGNITUDE_EMA_INDEX] = mean_abs_pnl_ema;
|
||||
|
||||
// target_scale = 1.0 / max(mean_abs_pnl_ema, EPS_PNL).
|
||||
// Larger typical PnL → smaller scale (reward is divided down toward ±1).
|
||||
// Smaller typical PnL → larger scale (reward is amplified toward ±1).
|
||||
@@ -113,6 +145,20 @@ extern "C" __global__ void rl_reward_scale_controller(
|
||||
const float scale_min = isv[RL_REWARD_SCALE_MIN_INDEX];
|
||||
target = fmaxf(scale_min, fminf(target, REWARD_SCALE_MAX));
|
||||
|
||||
// Bootstrap-fraction floor (spec 2026-05-30 Special case R bullet (b)):
|
||||
// until N=MIN_TRADES_FOR_RELEASE trades have closed, the scale cannot
|
||||
// drop below 10% of bootstrap. The trade count comes from the Welford
|
||||
// trade-duration counter (incremented once per closed trade). This
|
||||
// protects against the early-training spiral where a single large
|
||||
// mean_abs_pnl pre-trade signal cratered the scale before any closed-
|
||||
// trade ground truth could anchor it (fold 0 crash 1.0 → 0.004 in 100
|
||||
// steps).
|
||||
const float trade_count = isv[RL_TRADE_DUR_VAR_COUNT_INDEX];
|
||||
const float boot = isv[RL_REWARD_SCALE_BOOTSTRAP_INDEX];
|
||||
const float boot_floor = (trade_count < MIN_TRADES_FOR_RELEASE)
|
||||
? boot * BOOTSTRAP_FRACTION_FLOOR
|
||||
: scale_min;
|
||||
|
||||
// First-observation replace-directly per
|
||||
// `pearl_first_observation_bootstrap`. With prev=1.0 (bootstrap)
|
||||
// and a target far from 1.0 on the first real PnL signal, the
|
||||
@@ -121,13 +167,25 @@ extern "C" __global__ void rl_reward_scale_controller(
|
||||
// feeds V regression at magnitude 500× the atom support's
|
||||
// ±1 expectation. Replacing directly with target eliminates
|
||||
// this cold-start contamination.
|
||||
if (prev == isv[RL_REWARD_SCALE_BOOTSTRAP_INDEX]) {
|
||||
isv[RL_REWARD_SCALE_INDEX] = target;
|
||||
//
|
||||
// Asymmetric DECREASE rate limit (spec 2026-05-30 bullet (a)): even
|
||||
// on the first-observation path, cap per-step decrease at 5%
|
||||
// (new ≥ prev / 1.05) so a tiny initial target can't crater scale
|
||||
// before downstream signal accumulates. The existing 2% per-step
|
||||
// symmetric clamp below remains for subsequent steps (stricter than
|
||||
// 5%, so it stays load-bearing on the slow path).
|
||||
if (prev == boot) {
|
||||
float first = target;
|
||||
if (first < prev) {
|
||||
first = fmaxf(first, prev / ASYM_DECREASE_RATE);
|
||||
}
|
||||
first = fmaxf(boot_floor, fminf(first, REWARD_SCALE_MAX));
|
||||
isv[RL_REWARD_SCALE_INDEX] = first;
|
||||
return;
|
||||
}
|
||||
|
||||
// Wiener-α blend with floor per pearl_wiener_alpha_floor_for_nonstationary.
|
||||
const float a = fmaxf(alpha_step, WIENER_ALPHA_FLOOR);
|
||||
const float a = fmaxf(alpha_step, isv[RL_WIENER_ALPHA_FLOOR_INDEX]);
|
||||
float out = (1.0f - a) * prev + a * target;
|
||||
|
||||
// Per-step movement clamp: scale moves at most ±2% from previous.
|
||||
@@ -139,6 +197,16 @@ extern "C" __global__ void rl_reward_scale_controller(
|
||||
const float min_move = prev * 0.98f;
|
||||
out = fmaxf(min_move, fminf(out, max_move));
|
||||
|
||||
out = fmaxf(scale_min, fminf(out, REWARD_SCALE_MAX));
|
||||
// Asymmetric DECREASE rate limit (spec 2026-05-30 bullet (a)):
|
||||
// belt-and-braces — the symmetric 2% clamp above is stricter than
|
||||
// the 5% asymmetric cap on the slow path, but the explicit
|
||||
// asymmetric clamp here documents the controller invariant
|
||||
// (decreases cannot exceed 5% per step) and survives any future
|
||||
// relaxation of the 2% symmetric clamp.
|
||||
if (out < prev) {
|
||||
out = fmaxf(out, prev / ASYM_DECREASE_RATE);
|
||||
}
|
||||
|
||||
out = fmaxf(boot_floor, fminf(out, REWARD_SCALE_MAX));
|
||||
isv[RL_REWARD_SCALE_INDEX] = out;
|
||||
}
|
||||
|
||||
@@ -46,16 +46,33 @@
|
||||
// past the clip band.
|
||||
|
||||
#define RL_N_ROLLOUT_STEPS_INDEX 404
|
||||
#define ROLLOUT_MIN 256.0f
|
||||
#define ROLLOUT_MAX 8192.0f
|
||||
// ROLLOUT MIN/MAX clamp bounds are now ISV-driven per the 2026-05-30
|
||||
// clamp-bound extension. Runtime-tunable + visible in diag.
|
||||
#define RL_ROLLOUT_MIN_INDEX 643
|
||||
#define RL_ROLLOUT_MAX_INDEX 644
|
||||
// ISV-driven bootstrap + target.
|
||||
#define RL_ROLLOUT_BOOTSTRAP_INDEX 475
|
||||
#define RL_ADV_VAR_RATIO_TARGET_INDEX 449
|
||||
// Schulman params from shared global slots (same as ppo_clip, target_tau).
|
||||
#define RL_SCHULMAN_TOLERANCE_INDEX 468
|
||||
#define RL_SCHULMAN_ADJUST_RATE_INDEX 469
|
||||
#define ADV_VAR_RATIO_NOISE_FLOOR_FRAC 0.01f
|
||||
#define WIENER_ALPHA_FLOOR 0.4f
|
||||
// Wiener-α floor — shared across 9 controllers (slot 659).
|
||||
#define RL_WIENER_ALPHA_FLOOR_INDEX 659
|
||||
|
||||
// Adaptive controller floors (spec 2026-05-30-adaptive-controller-floor-design):
|
||||
// noise floor derives from observed advantage-variance signal via Welford
|
||||
// triples, asymmetric Schulman widening requires N consecutive below-band
|
||||
// observations. Replaces hardcoded ADV_VAR_RATIO_NOISE_FLOOR_FRAC=0.01f
|
||||
// (calibrated against the pre-Phase-4.5 advantage regime; under Phase 4.5
|
||||
// post-norm advantage variance is definitionally near-zero, so the
|
||||
// controller is also rewired to consume RL_ADV_VAR_PRE_NORM_INDEX=612 —
|
||||
// the pre-normalization variance emitted by rl_advantage_normalize.cu).
|
||||
#define RL_ADV_VAR_VAR_COUNT_INDEX 596
|
||||
#define RL_ADV_VAR_VAR_M2_INDEX 598
|
||||
#define RL_ADV_VAR_BELOW_COUNT_INDEX 599
|
||||
#define NOISE_FLOOR_TARGET_FRAC 0.5f // floor ≥ 50% of target
|
||||
#define NOISE_FLOOR_STD_MULTIPLIER 2.0f // floor ≥ 2σ of observed signal
|
||||
#define WIDEN_PATIENCE_CONSECUTIVE 3.0f // widen requires N below-band steps
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────
|
||||
// rl_rollout_steps_controller:
|
||||
@@ -103,31 +120,43 @@ extern "C" __global__ void rl_rollout_steps_controller(
|
||||
// RL_ADV_VAR_RATIO_TARGET_INDEX (seeded at trainer init).
|
||||
const float adv_var_target = isv[RL_ADV_VAR_RATIO_TARGET_INDEX];
|
||||
|
||||
// Noise-floor gate: input below target × NOISE_FLOOR_FRAC is
|
||||
// dominated by numerical noise — hold rollout unchanged.
|
||||
// Mirrors the ppo_clip / target_tau controllers' design after the
|
||||
// alpha-rl-mjzfk multiplicative-blow-up incident.
|
||||
const float adv_var_noise_floor =
|
||||
adv_var_target * ADV_VAR_RATIO_NOISE_FLOOR_FRAC;
|
||||
// Adaptive noise floor — signal-driven per
|
||||
// `pearl_zscore_normalization_for_magnitude_asymmetric_signals` and
|
||||
// `feedback_adaptive_not_tuned`. Welford sample variance =
|
||||
// M² / (count − 1) when count > 1.
|
||||
const float adv_count = isv[RL_ADV_VAR_VAR_COUNT_INDEX];
|
||||
const float adv_var = (adv_count > 1.0f)
|
||||
? isv[RL_ADV_VAR_VAR_M2_INDEX] / (adv_count - 1.0f)
|
||||
: 0.0f;
|
||||
const float adv_std = sqrtf(adv_var);
|
||||
const float adv_var_noise_floor = fmaxf(adv_var_target * NOISE_FLOOR_TARGET_FRAC,
|
||||
adv_std * NOISE_FLOOR_STD_MULTIPLIER);
|
||||
if (advantage_var_over_abs_mean < adv_var_noise_floor) return;
|
||||
|
||||
// Bounded multiplicative adjustment (Schulman-style adaptive).
|
||||
// At most ADV_VAR_RATIO_ADJUST_RATE × shift per step regardless of
|
||||
// how far input is from target. After several consecutive
|
||||
// out-of-band observations the output drifts smoothly toward
|
||||
// MIN/MAX, but a single observation can't slam it there.
|
||||
// Asymmetric Schulman (spec 2026-05-30 Section "Approach C, folded in"):
|
||||
// widening rollout fires on a single above-band observation — noisy
|
||||
// advantages are a safety signal we act on fast (collect more samples
|
||||
// before updating). Shrinking rollout requires N consecutive below-band
|
||||
// observations so a single quiet step can't push the buffer too small.
|
||||
// Below-band counter is per-controller in ISV.
|
||||
const float tolerance = isv[RL_SCHULMAN_TOLERANCE_INDEX];
|
||||
const float adjust_rate = isv[RL_SCHULMAN_ADJUST_RATE_INDEX];
|
||||
float scale;
|
||||
if (advantage_var_over_abs_mean > adv_var_target * tolerance) {
|
||||
scale = adjust_rate;
|
||||
scale = adjust_rate; // widen — single observation
|
||||
isv[RL_ADV_VAR_BELOW_COUNT_INDEX] = 0.0f; // reset patience
|
||||
} else if (advantage_var_over_abs_mean < adv_var_target / tolerance) {
|
||||
scale = 1.0f / adjust_rate;
|
||||
const float new_count = isv[RL_ADV_VAR_BELOW_COUNT_INDEX] + 1.0f;
|
||||
isv[RL_ADV_VAR_BELOW_COUNT_INDEX] = new_count;
|
||||
scale = (new_count >= WIDEN_PATIENCE_CONSECUTIVE) ? (1.0f / adjust_rate) : 1.0f;
|
||||
} else {
|
||||
isv[RL_ADV_VAR_BELOW_COUNT_INDEX] = 0.0f; // in-band → reset
|
||||
scale = 1.0f;
|
||||
}
|
||||
const float rollout_min = isv[RL_ROLLOUT_MIN_INDEX];
|
||||
const float rollout_max = isv[RL_ROLLOUT_MAX_INDEX];
|
||||
float target = prev * scale;
|
||||
target = fmaxf(ROLLOUT_MIN, fminf(target, ROLLOUT_MAX));
|
||||
target = fmaxf(rollout_min, fminf(target, rollout_max));
|
||||
|
||||
// First-observation replace-directly per
|
||||
// `pearl_first_observation_bootstrap`.
|
||||
@@ -137,9 +166,9 @@ extern "C" __global__ void rl_rollout_steps_controller(
|
||||
}
|
||||
|
||||
// Wiener-α blend with floor per pearl_wiener_alpha_floor_for_nonstationary.
|
||||
const float a = fmaxf(alpha, WIENER_ALPHA_FLOOR);
|
||||
const float a = fmaxf(alpha, isv[RL_WIENER_ALPHA_FLOOR_INDEX]);
|
||||
float out = (1.0f - a) * prev + a * target;
|
||||
|
||||
out = fmaxf(ROLLOUT_MIN, fminf(out, ROLLOUT_MAX));
|
||||
out = fmaxf(rollout_min, fminf(out, rollout_max));
|
||||
isv[RL_N_ROLLOUT_STEPS_INDEX] = out;
|
||||
}
|
||||
|
||||
73
crates/ml-alpha/cuda/rl_signal_variance_update.cu
Normal file
73
crates/ml-alpha/cuda/rl_signal_variance_update.cu
Normal file
@@ -0,0 +1,73 @@
|
||||
// rl_signal_variance_update.cu — Welford online variance for controller inputs (2026-05-30).
|
||||
//
|
||||
// Per `feedback_adaptive_not_tuned`, `feedback_isv_for_adaptive_bounds`,
|
||||
// `pearl_controller_anchors_isv_driven`: every adaptive controller's noise
|
||||
// floor / target / threshold must derive from observed signal statistics,
|
||||
// not from hardcoded constants. This kernel maintains a running mean and
|
||||
// M² (sum of squared deviations from mean) for each controller-input EMA,
|
||||
// using Welford's online algorithm for numerical stability with float32.
|
||||
//
|
||||
// Welford recurrence (numerically stable, single-pass):
|
||||
// n_new = n_prev + 1
|
||||
// delta = x - mean_prev
|
||||
// mean = mean_prev + delta / n_new
|
||||
// delta2 = x - mean // recomputed against NEW mean
|
||||
// m2 = m2_prev + delta * delta2
|
||||
//
|
||||
// Sample variance = m2 / (n - 1) when n > 1; undefined when n ≤ 1.
|
||||
//
|
||||
// Per `pearl_first_observation_bootstrap`: skip the update when input ==
|
||||
// sentinel 0.0. The first non-zero observation initializes mean directly
|
||||
// (delta against mean_prev=0 produces mean = x for n=1; delta against the
|
||||
// new mean is 0; m2 stays 0; subsequent observations build real variance).
|
||||
//
|
||||
// Per `feedback_cpu_is_read_only`: pure device kernel. ISV is the contract
|
||||
// surface; no host computation, no host control. Caller threads ISV bus
|
||||
// pointer + four slot indices (input + count/mean/M² triple). Caller is
|
||||
// responsible for launching once per controller input per step (usually
|
||||
// via fused-kernel orchestration).
|
||||
//
|
||||
// Per `feedback_no_atomicadd`, `feedback_nvidia_grade_perf_for_kernels`:
|
||||
// single-thread block. Each Welford triple is owned by exactly one input
|
||||
// slot, accessed sequentially across steps — no concurrent writers, no
|
||||
// reduction, no atomic.
|
||||
//
|
||||
// Launch config:
|
||||
// grid = (1, 1, 1)
|
||||
// block = (1, 1, 1)
|
||||
// smem = 0
|
||||
// stream = main RL stream (sequenced after EMA producers and before
|
||||
// controllers consume the variance)
|
||||
|
||||
extern "C" __global__ void rl_signal_variance_update(
|
||||
float* __restrict__ isv,
|
||||
int input_slot,
|
||||
int count_slot,
|
||||
int mean_slot,
|
||||
int m2_slot
|
||||
) {
|
||||
// Single-thread guard. Block shape is (1,1,1) by contract but defend
|
||||
// against a mis-configured launcher.
|
||||
if (threadIdx.x != 0 || threadIdx.y != 0 || threadIdx.z != 0) return;
|
||||
if (blockIdx.x != 0 || blockIdx.y != 0 || blockIdx.z != 0) return;
|
||||
|
||||
// Sentinel-zero skip per pearl_first_observation_bootstrap.
|
||||
// Producers (EMA kernels) hold their slot at 0.0 until the first
|
||||
// real observation. Welford should NOT count those zero reads as
|
||||
// genuine observations of "the signal is zero".
|
||||
const float x = isv[input_slot];
|
||||
if (x == 0.0f) return;
|
||||
|
||||
// Welford online update.
|
||||
const float n_prev = isv[count_slot];
|
||||
const float n_new = n_prev + 1.0f;
|
||||
const float mean_prev = isv[mean_slot];
|
||||
const float delta = x - mean_prev;
|
||||
const float mean_new = mean_prev + delta / n_new;
|
||||
const float delta2 = x - mean_new; // against new mean — load-bearing for stability
|
||||
const float m2_new = isv[m2_slot] + delta * delta2;
|
||||
|
||||
isv[count_slot] = n_new;
|
||||
isv[mean_slot] = mean_new;
|
||||
isv[m2_slot] = m2_new;
|
||||
}
|
||||
@@ -26,16 +26,34 @@
|
||||
// the target-net design exists to provide.
|
||||
|
||||
#define RL_TARGET_TAU_INDEX 401
|
||||
#define TAU_MIN 0.001f
|
||||
#define RL_TARGET_TAU_MAX_INDEX 573
|
||||
// τ MIN clamp bound is now ISV-driven per the 2026-05-30 clamp-bound
|
||||
// extension. ISV-residence makes the floor runtime-tunable and aligns
|
||||
// with the broader principle that every controller bound is observed-
|
||||
// signal driven rather than calibrated against a prior regime.
|
||||
#define RL_TARGET_TAU_MIN_INDEX 642
|
||||
#define RL_TARGET_TAU_MAX_INDEX 573
|
||||
// ISV-driven bootstrap + target.
|
||||
#define RL_TAU_BOOTSTRAP_INDEX 473
|
||||
#define RL_DIV_TARGET_INDEX 457
|
||||
// Schulman params from shared global slots (same as ppo_clip).
|
||||
#define RL_SCHULMAN_TOLERANCE_INDEX 468
|
||||
#define RL_SCHULMAN_ADJUST_RATE_INDEX 469
|
||||
#define DIV_NOISE_FLOOR_FRAC 0.01f
|
||||
#define WIENER_ALPHA_FLOOR 0.4f
|
||||
// Wiener-α floor — shared across 9 controllers (slot 659).
|
||||
#define RL_WIENER_ALPHA_FLOOR_INDEX 659
|
||||
|
||||
// Adaptive controller floors (spec 2026-05-30-adaptive-controller-floor-design):
|
||||
// noise floor derives from observed q_divergence_ema variance via Welford
|
||||
// triples, asymmetric Schulman widening requires N consecutive below-band
|
||||
// observations. Replaces hardcoded DIV_NOISE_FLOOR_FRAC=0.01f which was
|
||||
// calibrated against the pre-Phase-4.5 divergence regime — under Phase 4.5
|
||||
// the 1%-of-target floor is ~100× too low and the controller never moved
|
||||
// from bootstrap (alpha-rl-... fold 0 confirmed τ stuck at 0.005).
|
||||
#define RL_Q_DIV_VAR_COUNT_INDEX 592
|
||||
#define RL_Q_DIV_VAR_M2_INDEX 594
|
||||
#define RL_Q_DIV_BELOW_COUNT_INDEX 595
|
||||
#define NOISE_FLOOR_TARGET_FRAC 0.5f // floor ≥ 50% of target
|
||||
#define NOISE_FLOOR_STD_MULTIPLIER 2.0f // floor ≥ 2σ of observed signal
|
||||
#define WIDEN_PATIENCE_CONSECUTIVE 3.0f // widen requires N below-band steps
|
||||
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────
|
||||
@@ -77,25 +95,50 @@ extern "C" __global__ void rl_target_tau_controller(
|
||||
|
||||
// ISV-driven divergence target (was hardcoded #define).
|
||||
const float div_target = isv[RL_DIV_TARGET_INDEX];
|
||||
const float div_noise_floor = div_target * DIV_NOISE_FLOOR_FRAC;
|
||||
|
||||
// Noise-floor gate.
|
||||
// Adaptive noise floor — signal-driven per
|
||||
// `pearl_zscore_normalization_for_magnitude_asymmetric_signals` and
|
||||
// `feedback_adaptive_not_tuned`. Floor must scale with observed signal
|
||||
// magnitude AND have an absolute minimum tied to the target so a
|
||||
// momentarily-noisy signal can't trigger spurious adjustments.
|
||||
// Welford sample variance = M² / (count − 1) when count > 1.
|
||||
const float div_count = isv[RL_Q_DIV_VAR_COUNT_INDEX];
|
||||
const float div_var = (div_count > 1.0f)
|
||||
? isv[RL_Q_DIV_VAR_M2_INDEX] / (div_count - 1.0f)
|
||||
: 0.0f;
|
||||
const float div_std = sqrtf(div_var);
|
||||
const float div_noise_floor = fmaxf(div_target * NOISE_FLOOR_TARGET_FRAC,
|
||||
div_std * NOISE_FLOOR_STD_MULTIPLIER);
|
||||
|
||||
// Noise-floor gate: divergence below the adaptive floor is dominated by
|
||||
// numerical noise OR signal stays naturally below target under the
|
||||
// current architecture. Hold τ.
|
||||
if (q_divergence_norm < div_noise_floor) return;
|
||||
|
||||
// Bounded multiplicative adjustment.
|
||||
float ratio;
|
||||
// Asymmetric Schulman (spec 2026-05-30 Section "Approach C, folded in"):
|
||||
// raising τ fires on a single above-band observation — real Q
|
||||
// divergence is a safety signal we act on fast (target net falling
|
||||
// behind = raise the bootstrap rate). Lowering τ requires N consecutive
|
||||
// below-band observations so a single noisy step can't push τ toward
|
||||
// the floor. Below-band counter is per-controller in ISV.
|
||||
const float tolerance = isv[RL_SCHULMAN_TOLERANCE_INDEX];
|
||||
const float adjust_rate = isv[RL_SCHULMAN_ADJUST_RATE_INDEX];
|
||||
float ratio;
|
||||
if (q_divergence_norm > div_target * tolerance) {
|
||||
ratio = adjust_rate;
|
||||
ratio = adjust_rate; // raise τ — single observation
|
||||
isv[RL_Q_DIV_BELOW_COUNT_INDEX] = 0.0f; // reset patience
|
||||
} else if (q_divergence_norm < div_target / tolerance) {
|
||||
ratio = 1.0f / adjust_rate;
|
||||
const float new_count = isv[RL_Q_DIV_BELOW_COUNT_INDEX] + 1.0f;
|
||||
isv[RL_Q_DIV_BELOW_COUNT_INDEX] = new_count;
|
||||
ratio = (new_count >= WIDEN_PATIENCE_CONSECUTIVE) ? (1.0f / adjust_rate) : 1.0f;
|
||||
} else {
|
||||
isv[RL_Q_DIV_BELOW_COUNT_INDEX] = 0.0f; // in-band → reset
|
||||
ratio = 1.0f;
|
||||
}
|
||||
float tau_target = tau_prev * ratio;
|
||||
const float tau_min = isv[RL_TARGET_TAU_MIN_INDEX];
|
||||
const float tau_max = isv[RL_TARGET_TAU_MAX_INDEX];
|
||||
tau_target = fmaxf(TAU_MIN, fminf(tau_target, tau_max));
|
||||
tau_target = fmaxf(tau_min, fminf(tau_target, tau_max));
|
||||
|
||||
// First-observation replace-directly: if
|
||||
// prev is still exactly the hardcoded bootstrap value, this is
|
||||
@@ -112,9 +155,9 @@ extern "C" __global__ void rl_target_tau_controller(
|
||||
}
|
||||
|
||||
// Wiener-α blend with floor per pearl_wiener_alpha_floor_for_nonstationary.
|
||||
const float a = fmaxf(alpha, WIENER_ALPHA_FLOOR);
|
||||
const float a = fmaxf(alpha, isv[RL_WIENER_ALPHA_FLOOR_INDEX]);
|
||||
float tau_new = (1.0f - a) * tau_prev + a * tau_target;
|
||||
|
||||
tau_new = fmaxf(TAU_MIN, fminf(tau_new, tau_max));
|
||||
tau_new = fmaxf(tau_min, fminf(tau_new, tau_max));
|
||||
isv[RL_TARGET_TAU_INDEX] = tau_new;
|
||||
}
|
||||
|
||||
@@ -1,26 +1,29 @@
|
||||
// rl_trail_mutate.cu — SP20 P5 trail mutation kernel.
|
||||
//
|
||||
// Handles the previously-dead a7 (TrailTighten) and a8 (TrailLoosen)
|
||||
// actions per `pearl_dead_trail_stop_actions_a7_a8`. Mutates each
|
||||
// active unit's trail_distance bounded by ISV-driven MIN/MAX with
|
||||
// symmetric reciprocal adjust rate per SP20 §4.12:
|
||||
// a7 (tighten): trail = max(MIN, trail × rate)
|
||||
// a8 (loosen): trail = min(MAX, trail / rate)
|
||||
// Handles a7 (TrailTighten) and a8 (TrailLoosen) — closes
|
||||
// `pearl_dead_trail_stop_actions_a7_a8`. Mutates each active unit's
|
||||
// trail_distance bounded by ISV-driven MIN/MAX. Each direction has its
|
||||
// own independent factor per spec 2026-05-30-adaptive-risk-management:
|
||||
//
|
||||
// Tighten and loosen are reciprocal: N tightens followed by N loosens
|
||||
// returns trail to its original distance.
|
||||
// a7 (tighten): trail = max(MIN, trail × TIGHTEN_FACTOR) (default 0.9)
|
||||
// a8 (loosen): trail = min(MAX, trail × LOOSEN_FACTOR) (default 1.1)
|
||||
//
|
||||
// Audit history: scripts/audit-wiring.sh dogfood pass flagged that
|
||||
// a7/a8 had no consumer anywhere in the codebase prior to this
|
||||
// kernel.
|
||||
// Tighten and loosen are NO LONGER strictly reciprocal — the agent can
|
||||
// learn (via the ISV slots' controllers, if any) to bias one direction
|
||||
// or the other. Bootstrap factors (0.9 / 1.1) preserve the prior
|
||||
// approximately-symmetric behavior.
|
||||
//
|
||||
// Slot migration: previously read `RL_TRAIL_ADJUST_RATE_INDEX = 497`
|
||||
// and used `trail × rate` for tighten and `trail / rate` for loosen.
|
||||
// Now reads independent `RL_TRAIL_TIGHTEN_FACTOR_INDEX = 682` and
|
||||
// `RL_TRAIL_LOOSEN_FACTOR_INDEX = 683` from the risk-management spec.
|
||||
//
|
||||
// One thread per (batch, unit). Mutates ALL active units uniformly
|
||||
// when the batch chose a7/a8 — matches SP20 §2.2 Tier 2 design
|
||||
// ("a7/a8 uniformly tighten/loosen all active units").
|
||||
// when the batch chose a7/a8 — matches SP20 §2.2 Tier 2 design.
|
||||
//
|
||||
// Per `feedback_no_atomicadd`: per-(batch, unit) independent writes.
|
||||
// Per `feedback_cpu_is_read_only`: pure device-side.
|
||||
// Per `feedback_isv_for_adaptive_bounds`: MIN/MAX/rate all ISV slots.
|
||||
// Per `feedback_isv_for_adaptive_bounds`: MIN/MAX/factors all ISV slots.
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
@@ -29,7 +32,8 @@
|
||||
#define ACTION_TRAIL_LOOSEN 8
|
||||
#define RL_TRAIL_MIN_INDEX 494
|
||||
#define RL_TRAIL_MAX_INDEX 495
|
||||
#define RL_TRAIL_ADJUST_RATE_INDEX 497
|
||||
#define RL_TRAIL_TIGHTEN_FACTOR_INDEX 682
|
||||
#define RL_TRAIL_LOOSEN_FACTOR_INDEX 683
|
||||
|
||||
extern "C" __global__ void rl_trail_mutate(
|
||||
const int* __restrict__ actions, // [B]
|
||||
@@ -48,17 +52,17 @@ extern "C" __global__ void rl_trail_mutate(
|
||||
const int idx = b * MAX_UNITS + u;
|
||||
if (unit_active[idx] == 0) return;
|
||||
|
||||
const float trail_min = isv[RL_TRAIL_MIN_INDEX];
|
||||
const float trail_max = isv[RL_TRAIL_MAX_INDEX];
|
||||
const float adjust_rate = isv[RL_TRAIL_ADJUST_RATE_INDEX];
|
||||
const float current = unit_trail_distance[idx];
|
||||
const float trail_min = isv[RL_TRAIL_MIN_INDEX];
|
||||
const float trail_max = isv[RL_TRAIL_MAX_INDEX];
|
||||
const float current = unit_trail_distance[idx];
|
||||
|
||||
float updated;
|
||||
if (action == ACTION_TRAIL_TIGHTEN) {
|
||||
updated = fmaxf(trail_min, current * adjust_rate);
|
||||
const float factor = isv[RL_TRAIL_TIGHTEN_FACTOR_INDEX]; // bootstrap 0.9
|
||||
updated = fmaxf(trail_min, current * factor);
|
||||
} else { // ACTION_TRAIL_LOOSEN
|
||||
const float inv_rate = (adjust_rate > 1e-6f) ? (1.0f / adjust_rate) : 1.0f;
|
||||
updated = fminf(trail_max, current * inv_rate);
|
||||
const float factor = isv[RL_TRAIL_LOOSEN_FACTOR_INDEX]; // bootstrap 1.1
|
||||
updated = fminf(trail_max, current * factor);
|
||||
}
|
||||
unit_trail_distance[idx] = updated;
|
||||
}
|
||||
|
||||
@@ -20,22 +20,38 @@
|
||||
// does, controller re-bootstraps harmlessly.
|
||||
//
|
||||
// Per pearl_blend_formulas_must_have_permanent_floor: dead-signal
|
||||
// guard — if EMA(|V_scalar|) < FLOOR, hold α (no V signal to
|
||||
// calibrate against; the trainer hasn't seen meaningful rewards yet).
|
||||
// guard — if EMA(|V_scalar|) < adaptive floor, hold α (no V signal
|
||||
// to calibrate against; the trainer hasn't seen meaningful rewards
|
||||
// yet).
|
||||
//
|
||||
// Per feedback_cpu_is_read_only: pure device kernel; reads V_scalar,
|
||||
// V_dq, ISV; emits α + EMAs to ISV. No host control.
|
||||
//
|
||||
// Adaptive controller floors (spec 2026-05-30-adaptive-controller-floor-design):
|
||||
// the 5 design constants below become ISV-driven so they can be tuned
|
||||
// at runtime without recompile. The dead-signal floor additionally
|
||||
// adapts to observed |V_scalar| variance via Welford slots 626/627/628
|
||||
// (Welford updates wired by the trainer pass alongside the other
|
||||
// controller Welford triples).
|
||||
//
|
||||
// Block layout: grid=(1, 1, 1), block=(BLOCK_X=1024, 1, 1). Single
|
||||
// block does parallel reduction over batch up to B=1024. Thread 0
|
||||
// performs the controller update.
|
||||
|
||||
#define BLOCK_X 1024
|
||||
#define EMA_ALPHA 0.01f
|
||||
#define TARGET_TRACK_RATIO 0.10f
|
||||
#define SCHULMAN_STEP 0.01f
|
||||
#define DEAD_SIGNAL_FLOOR 1e-4f
|
||||
#define BOOTSTRAP_ALPHA 1.0f
|
||||
// ISV-driven slots — replace prior hardcoded design constants per the
|
||||
// 2026-05-30 adaptive-controller-floor spec.
|
||||
#define RL_V_BLEND_DEAD_SIGNAL_FLOOR_INDEX 621
|
||||
#define RL_V_BLEND_TARGET_TRACK_RATIO_INDEX 622
|
||||
#define RL_V_BLEND_SCHULMAN_STEP_INDEX 623
|
||||
#define RL_V_BLEND_EMA_ALPHA_INDEX 624
|
||||
#define RL_V_BLEND_BOOTSTRAP_ALPHA_INDEX 625
|
||||
#define RL_V_BLEND_SCALAR_VAR_COUNT_INDEX 626
|
||||
#define RL_V_BLEND_SCALAR_VAR_M2_INDEX 628
|
||||
// Adaptive-dead-floor scaling for the Welford std term — keeps the
|
||||
// floor at least one σ_observed × ADAPTIVE_DEAD_FLOOR_STD_SCALE above
|
||||
// numerical noise even when the ISV-resident absolute floor is tighter.
|
||||
#define ADAPTIVE_DEAD_FLOOR_STD_SCALE 0.1f
|
||||
|
||||
extern "C" __global__ void rl_v_blend_alpha_controller(
|
||||
const float* __restrict__ v_scalar, // [B]
|
||||
@@ -79,10 +95,33 @@ extern "C" __global__ void rl_v_blend_alpha_controller(
|
||||
const float track_mean = s_t[0] * inv_B;
|
||||
const float mag_mean = s_m[0] * inv_B;
|
||||
|
||||
// ── ISV-driven design constants (spec 2026-05-30) ──
|
||||
const float ema_alpha = isv[RL_V_BLEND_EMA_ALPHA_INDEX];
|
||||
const float target_track_ratio = isv[RL_V_BLEND_TARGET_TRACK_RATIO_INDEX];
|
||||
const float schulman_step = isv[RL_V_BLEND_SCHULMAN_STEP_INDEX];
|
||||
const float bootstrap_alpha = isv[RL_V_BLEND_BOOTSTRAP_ALPHA_INDEX];
|
||||
|
||||
// ── Adaptive dead-signal floor ─────────────────────────────────
|
||||
// Combines the ISV-resident absolute floor with an observed
|
||||
// |V_scalar| variance term (Welford sample variance =
|
||||
// M² / (count − 1) when count > 1). Cold-start safe: count=0
|
||||
// → var=0 → adaptive = isv[621] absolute default. Once Welford
|
||||
// accumulates observations, the floor scales with σ_observed so
|
||||
// the dead-signal gate adapts to whatever magnitude regime
|
||||
// |V_scalar| settles into post-Phase-4.5.
|
||||
const float vmag_count = isv[RL_V_BLEND_SCALAR_VAR_COUNT_INDEX];
|
||||
const float vmag_var = (vmag_count > 1.0f)
|
||||
? isv[RL_V_BLEND_SCALAR_VAR_M2_INDEX] / (vmag_count - 1.0f)
|
||||
: 0.0f;
|
||||
const float vmag_std = sqrtf(vmag_var);
|
||||
const float dead_floor_abs = isv[RL_V_BLEND_DEAD_SIGNAL_FLOOR_INDEX];
|
||||
const float adaptive_dead_floor = fmaxf(dead_floor_abs,
|
||||
vmag_std * ADAPTIVE_DEAD_FLOOR_STD_SCALE);
|
||||
|
||||
// ── Bootstrap α on sentinel ──
|
||||
float alpha = isv[alpha_slot];
|
||||
if (alpha == 0.0f) {
|
||||
alpha = BOOTSTRAP_ALPHA;
|
||||
alpha = bootstrap_alpha;
|
||||
}
|
||||
|
||||
// ── First-observation bootstrap on EMAs ──
|
||||
@@ -90,13 +129,13 @@ extern "C" __global__ void rl_v_blend_alpha_controller(
|
||||
float prev_mag_ema = isv[v_scalar_mag_ema_slot];
|
||||
const float track_ema = (prev_track_ema == 0.0f)
|
||||
? track_mean
|
||||
: (1.0f - EMA_ALPHA) * prev_track_ema + EMA_ALPHA * track_mean;
|
||||
: (1.0f - ema_alpha) * prev_track_ema + ema_alpha * track_mean;
|
||||
const float mag_ema = (prev_mag_ema == 0.0f)
|
||||
? mag_mean
|
||||
: (1.0f - EMA_ALPHA) * prev_mag_ema + EMA_ALPHA * mag_mean;
|
||||
: (1.0f - ema_alpha) * prev_mag_ema + ema_alpha * mag_mean;
|
||||
|
||||
// ── Dead-signal guard: V_scalar magnitude too small to calibrate against ──
|
||||
if (mag_ema < DEAD_SIGNAL_FLOOR) {
|
||||
if (mag_ema < adaptive_dead_floor) {
|
||||
isv[trackerr_ema_slot] = track_ema;
|
||||
isv[v_scalar_mag_ema_slot] = mag_ema;
|
||||
isv[alpha_slot] = alpha; // hold (write bootstrap if needed)
|
||||
@@ -105,12 +144,12 @@ extern "C" __global__ void rl_v_blend_alpha_controller(
|
||||
|
||||
// ── Track ratio + Schulman-bounded step on α ──
|
||||
const float track_ratio = track_ema / mag_ema;
|
||||
if (track_ratio > 1.5f * TARGET_TRACK_RATIO) {
|
||||
if (track_ratio > 1.5f * target_track_ratio) {
|
||||
// V_dq diverged from V_scalar → raise α toward V_scalar
|
||||
alpha = fminf(alpha + SCHULMAN_STEP, 1.0f);
|
||||
} else if (track_ratio < TARGET_TRACK_RATIO / 1.5f) {
|
||||
alpha = fminf(alpha + schulman_step, 1.0f);
|
||||
} else if (track_ratio < target_track_ratio / 1.5f) {
|
||||
// V_dq tracks V_scalar well → lower α toward V_dq
|
||||
alpha = fmaxf(alpha - SCHULMAN_STEP, 0.0f);
|
||||
alpha = fmaxf(alpha - schulman_step, 0.0f);
|
||||
}
|
||||
// else: hold α (within band)
|
||||
|
||||
|
||||
63
crates/ml-alpha/cuda/rl_win_rate_ema_update.cu
Normal file
63
crates/ml-alpha/cuda/rl_win_rate_ema_update.cu
Normal file
@@ -0,0 +1,63 @@
|
||||
// rl_win_rate_ema_update.cu — Layer 4 (Kelly) input EMA producer.
|
||||
//
|
||||
// Tracks observed per-step win_rate as an EMA across closed trades.
|
||||
// Spec: docs/superpowers/specs/2026-05-30-adaptive-risk-management-design.md
|
||||
//
|
||||
// Inputs:
|
||||
// rewards[b] f32 — per-batch realized pnl delta this step (shaped)
|
||||
// dones[b] f32 — 1.0 if a trade closed this step, else 0.0
|
||||
//
|
||||
// Trade outcome is derived inline: win = (done & reward > 0),
|
||||
// loss = (done & reward < 0), no-close otherwise.
|
||||
//
|
||||
// Output: ISV[RL_WIN_RATE_EMA_INDEX = 677]
|
||||
//
|
||||
// Bootstrap semantics per `pearl_first_observation_bootstrap`:
|
||||
// sentinel = 0.5 (neutral 50% win rate); first non-trivial step replaces
|
||||
// directly. Zero observations on a given step (no closes) → no update.
|
||||
//
|
||||
// Per `feedback_no_atomicadd`: single-thread single-block, sums across
|
||||
// b_size sequentially.
|
||||
// Per `feedback_cpu_is_read_only`: pure device kernel; trainer launches
|
||||
// each step after rl_fused_reward_pipeline writes rewards + dones.
|
||||
// Per `feedback_isv_for_adaptive_bounds`: EMA-α is currently a structural
|
||||
// constant 0.05 (≈ 20-trade effective window) — same pattern as
|
||||
// ema_update_per_step where α is a structural smoothing parameter.
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#define RL_WIN_RATE_EMA_INDEX 677
|
||||
#define EMA_ALPHA 0.05f
|
||||
#define SENTINEL 0.5f
|
||||
|
||||
extern "C" __global__ void rl_win_rate_ema_update(
|
||||
float* __restrict__ isv,
|
||||
const float* __restrict__ rewards, // [b_size] shaped pnl delta
|
||||
const float* __restrict__ dones, // [b_size] 1.0 = close
|
||||
int b_size
|
||||
) {
|
||||
// Single-thread guard. Block shape (1,1,1) by contract.
|
||||
if (threadIdx.x != 0 || threadIdx.y != 0 || threadIdx.z != 0) return;
|
||||
if (blockIdx.x != 0 || blockIdx.y != 0 || blockIdx.z != 0) return;
|
||||
|
||||
// Count this step's closed trades and wins (done & reward > 0).
|
||||
int closed = 0;
|
||||
int wins = 0;
|
||||
for (int b = 0; b < b_size; ++b) {
|
||||
if (dones[b] >= 0.5f) {
|
||||
closed += 1;
|
||||
if (rewards[b] > 0.0f) wins += 1;
|
||||
}
|
||||
}
|
||||
if (closed == 0) return; // no observation this step
|
||||
|
||||
const float step_wr = (float)wins / (float)closed;
|
||||
const float prev = isv[RL_WIN_RATE_EMA_INDEX];
|
||||
|
||||
// First-observation bootstrap on sentinel 0.5 (neutral default).
|
||||
if (prev == SENTINEL) {
|
||||
isv[RL_WIN_RATE_EMA_INDEX] = step_wr;
|
||||
} else {
|
||||
isv[RL_WIN_RATE_EMA_INDEX] = (1.0f - EMA_ALPHA) * prev + EMA_ALPHA * step_wr;
|
||||
}
|
||||
}
|
||||
@@ -94,6 +94,31 @@ use ml_alpha::rl::isv_slots::{
|
||||
RL_OUTCOME_AUX_LAMBDA_INDEX,
|
||||
RL_SAC_ALPHA_INDEX, RL_SAC_ENTROPY_TARGET_INDEX,
|
||||
RL_ACTION_ENTROPY_EMA_INDEX,
|
||||
// Risk-management stack (spec 2026-05-30-adaptive-risk-management-design).
|
||||
// Per `feedback_wire_everything_up`: the kernels write these slots;
|
||||
// diag has to surface them or the run is half-blind.
|
||||
RL_SESSION_PNL_USD_INDEX, RL_SESSION_PNL_WORST_INDEX,
|
||||
RL_SESSION_DD_LIMIT_USD_INDEX, RL_SESSION_DD_TRIGGERED_INDEX,
|
||||
RL_CONSEC_LOSS_LIMIT_INDEX, RL_CONSEC_LOSS_COUNT_INDEX,
|
||||
RL_COOLDOWN_REMAINING_STEPS_INDEX, RL_COOLDOWN_DURATION_INDEX,
|
||||
RL_MAX_OPEN_UNITS_INDEX, RL_NET_INVENTORY_LIMIT_USD_INDEX,
|
||||
RL_IQN_ACTION_TAU_INDEX, RL_IQN_ACTION_TAU_MIN_INDEX,
|
||||
RL_IQN_ACTION_TAU_DD_SENSITIVITY_INDEX,
|
||||
RL_INVENTORY_PENALTY_BETA_INDEX, RL_INVENTORY_VARIANCE_EMA_INDEX,
|
||||
RL_KELLY_FRACTION_INDEX, RL_WIN_RATE_EMA_INDEX,
|
||||
RL_AVG_WIN_USD_EMA_INDEX, RL_AVG_LOSS_USD_EMA_INDEX,
|
||||
RL_KELLY_SAFETY_FRAC_INDEX, RL_KELLY_MIN_TRADES_FOR_RELEASE_INDEX,
|
||||
RL_TRAIL_TIGHTEN_FACTOR_INDEX, RL_TRAIL_LOOSEN_FACTOR_INDEX,
|
||||
RL_CUMULATIVE_DONES_INDEX,
|
||||
// v9 defensive eval-boundary calibration (spec
|
||||
// 2026-05-31-v9-defensive-eval-boundary-calibration.md).
|
||||
RL_EVAL_WARMUP_REMAINING_INDEX, RL_EVAL_WARMUP_STEPS_CONFIG_INDEX,
|
||||
RL_EVAL_WARMUP_DECAY_STEPS_CONFIG_INDEX,
|
||||
RL_EVAL_KELLY_SAFETY_DEFENSIVE_INDEX, RL_EVAL_IQN_TAU_MIN_DEFENSIVE_INDEX,
|
||||
RL_EVAL_ENTROPY_COEF_MIN_DEFENSIVE_INDEX, RL_EVAL_PPO_CLIP_EPS_MIN_DEFENSIVE_INDEX,
|
||||
RL_EVAL_KELLY_SAFETY_NORMAL_INDEX, RL_EVAL_IQN_TAU_MIN_NORMAL_INDEX,
|
||||
RL_EVAL_ENTROPY_COEF_MIN_NORMAL_INDEX, RL_EVAL_PPO_CLIP_EPS_MIN_NORMAL_INDEX,
|
||||
RL_ENTROPY_COEF_MIN_INDEX, RL_PPO_CLIP_EPS_MIN_INDEX,
|
||||
};
|
||||
use ml_alpha::trainer::diag_staging::DiagStaging;
|
||||
use ml_alpha::trainer::integrated::{
|
||||
@@ -744,6 +769,22 @@ fn main() -> Result<()> {
|
||||
})
|
||||
};
|
||||
|
||||
// v9 eval_warmup pre-compute (json! macro can't take Rust blocks/if-else).
|
||||
let eval_warmup_remaining = isv[RL_EVAL_WARMUP_REMAINING_INDEX];
|
||||
let eval_warmup_decay_steps = isv[RL_EVAL_WARMUP_DECAY_STEPS_CONFIG_INDEX];
|
||||
let eval_warmup_active = if eval_warmup_remaining > 0.0 { 1.0_f32 } else { 0.0_f32 };
|
||||
let eval_warmup_blend = if eval_warmup_remaining < 0.0 {
|
||||
0.0_f32
|
||||
} else if eval_warmup_remaining > eval_warmup_decay_steps {
|
||||
1.0_f32
|
||||
} else if eval_warmup_decay_steps > 0.0 {
|
||||
eval_warmup_remaining / eval_warmup_decay_steps
|
||||
} else if eval_warmup_remaining > 0.0 {
|
||||
1.0_f32
|
||||
} else {
|
||||
0.0_f32
|
||||
};
|
||||
|
||||
let record = json!({
|
||||
"step": step,
|
||||
"elapsed_s": t_start.elapsed().as_secs_f32(),
|
||||
@@ -822,6 +863,115 @@ fn main() -> Result<()> {
|
||||
"stale": isv[RL_LR_V_STEPS_SINCE_BEST_INDEX],
|
||||
"warmup": isv[RL_LR_V_WARMUP_COUNTER_INDEX] },
|
||||
},
|
||||
// Adaptive risk-management stack state — five orthogonal layers.
|
||||
// Every slot here is written by a kernel; diag surfaces them so
|
||||
// controller behaviour, gate firing, and bootstrap state are
|
||||
// observable without re-reading device memory.
|
||||
"risk_stack": {
|
||||
// Layer 1 — CMDP hard constraints.
|
||||
// session_pnl_usd is MEAN-of-active-accounts (the IQN-τ
|
||||
// consumer); session_pnl_worst is the most-negative
|
||||
// per-batch pnl (diag — spot fleet skew).
|
||||
"cmdp": {
|
||||
"session_pnl_usd_mean": isv[RL_SESSION_PNL_USD_INDEX],
|
||||
"session_pnl_worst": isv[RL_SESSION_PNL_WORST_INDEX],
|
||||
"session_dd_limit_usd": isv[RL_SESSION_DD_LIMIT_USD_INDEX],
|
||||
"session_dd_triggered": isv[RL_SESSION_DD_TRIGGERED_INDEX],
|
||||
"consec_loss_count": isv[RL_CONSEC_LOSS_COUNT_INDEX],
|
||||
"consec_loss_limit": isv[RL_CONSEC_LOSS_LIMIT_INDEX],
|
||||
"cooldown_remaining_steps": isv[RL_COOLDOWN_REMAINING_STEPS_INDEX],
|
||||
"cooldown_duration": isv[RL_COOLDOWN_DURATION_INDEX],
|
||||
"max_open_units": isv[RL_MAX_OPEN_UNITS_INDEX],
|
||||
"net_inventory_limit_usd": isv[RL_NET_INVENTORY_LIMIT_USD_INDEX],
|
||||
},
|
||||
// Layer 2 — IQN risk-averse action τ.
|
||||
"iqn_tau": {
|
||||
"action_tau": isv[RL_IQN_ACTION_TAU_INDEX],
|
||||
"tau_min": isv[RL_IQN_ACTION_TAU_MIN_INDEX],
|
||||
"dd_sensitivity": isv[RL_IQN_ACTION_TAU_DD_SENSITIVITY_INDEX],
|
||||
},
|
||||
// Layer 3 — Avellaneda-Stoikov inventory penalty β.
|
||||
"inventory": {
|
||||
"penalty_beta": isv[RL_INVENTORY_PENALTY_BETA_INDEX],
|
||||
"variance_ema": isv[RL_INVENTORY_VARIANCE_EMA_INDEX],
|
||||
},
|
||||
// Layer 4 — Half-Kelly position sizing + its EMA inputs.
|
||||
"kelly": {
|
||||
"fraction": isv[RL_KELLY_FRACTION_INDEX],
|
||||
"win_rate_ema": isv[RL_WIN_RATE_EMA_INDEX],
|
||||
"avg_win_usd_ema": isv[RL_AVG_WIN_USD_EMA_INDEX],
|
||||
"avg_loss_usd_ema": isv[RL_AVG_LOSS_USD_EMA_INDEX],
|
||||
"safety_frac": isv[RL_KELLY_SAFETY_FRAC_INDEX],
|
||||
"min_trades_for_release": isv[RL_KELLY_MIN_TRADES_FOR_RELEASE_INDEX],
|
||||
"cumulative_dones": isv[RL_CUMULATIVE_DONES_INDEX],
|
||||
},
|
||||
// Layer D — trail tighten/loosen factors (kernel-side
|
||||
// counters are already in `trail.{tightened,loosened}_count_*`).
|
||||
"trail_factors": {
|
||||
"tighten": isv[RL_TRAIL_TIGHTEN_FACTOR_INDEX],
|
||||
"loosen": isv[RL_TRAIL_LOOSEN_FACTOR_INDEX],
|
||||
},
|
||||
// v9 — defensive eval-boundary calibration ([[pearl_adaptive_carryover_discipline]]).
|
||||
// `remaining` is the per-step counter: ≥ decay_steps → full
|
||||
// defensive blend=1.0; (0, decay_steps] → linear decay to 0;
|
||||
// 0 → final boundary write of normal values; -1 → kernel
|
||||
// no-op (post-warmup steady state). The four `floor_*`
|
||||
// entries are the LIVE values (read AFTER the warmup
|
||||
// kernel ran this step, so they reflect any override).
|
||||
"eval_warmup": {
|
||||
"remaining": eval_warmup_remaining,
|
||||
"active": eval_warmup_active,
|
||||
"blend": eval_warmup_blend,
|
||||
"warmup_steps_config": isv[RL_EVAL_WARMUP_STEPS_CONFIG_INDEX],
|
||||
"decay_steps_config": eval_warmup_decay_steps,
|
||||
"floor_kelly_safety": isv[RL_KELLY_SAFETY_FRAC_INDEX],
|
||||
"floor_iqn_tau_min": isv[RL_IQN_ACTION_TAU_MIN_INDEX],
|
||||
"floor_entropy_min": isv[RL_ENTROPY_COEF_MIN_INDEX],
|
||||
"floor_ppo_eps_min": isv[RL_PPO_CLIP_EPS_MIN_INDEX],
|
||||
"target_defensive": {
|
||||
"kelly_safety": isv[RL_EVAL_KELLY_SAFETY_DEFENSIVE_INDEX],
|
||||
"iqn_tau_min": isv[RL_EVAL_IQN_TAU_MIN_DEFENSIVE_INDEX],
|
||||
"entropy_min": isv[RL_EVAL_ENTROPY_COEF_MIN_DEFENSIVE_INDEX],
|
||||
"ppo_eps_min": isv[RL_EVAL_PPO_CLIP_EPS_MIN_DEFENSIVE_INDEX],
|
||||
},
|
||||
"target_normal": {
|
||||
"kelly_safety": isv[RL_EVAL_KELLY_SAFETY_NORMAL_INDEX],
|
||||
"iqn_tau_min": isv[RL_EVAL_IQN_TAU_MIN_NORMAL_INDEX],
|
||||
"entropy_min": isv[RL_EVAL_ENTROPY_COEF_MIN_NORMAL_INDEX],
|
||||
"ppo_eps_min": isv[RL_EVAL_PPO_CLIP_EPS_MIN_NORMAL_INDEX],
|
||||
},
|
||||
},
|
||||
// Calibration diagnostics — atom-span vs. Bellman dynamic
|
||||
// bound. Per docs/superpowers/specs/2026-05-31-c51-atom-
|
||||
// span-math-validation.md: the structural constraint is
|
||||
// `atom_max ≥ WIN + γ × atom_max` (the Bellman target
|
||||
// upper bound). When `atom_max_headroom > 0`, system is
|
||||
// self-consistent (no Q clipping); when `< 0`, V_target
|
||||
// saturates top atom → Q distillation degrades (the v7
|
||||
// Fix F failure mode at step 500).
|
||||
//
|
||||
// `atom_max_over_3sigma` measures resolution waste: when
|
||||
// atom_max ≫ 3·popart_σ, atoms are over-sized for actual
|
||||
// V_target distribution (~99.7% within ±3σ). v8 local
|
||||
// smoke step 999 had this ratio at 16.8× — i.e., we use
|
||||
// 1/17 of available atom resolution.
|
||||
"atom_calibration": {
|
||||
"win_bound": isv[RL_REWARD_CLAMP_WIN_INDEX],
|
||||
"atom_max": isv[RL_C51_V_MAX_INDEX],
|
||||
"gamma": isv[RL_GAMMA_INDEX],
|
||||
"dynamic_bound": isv[RL_REWARD_CLAMP_WIN_INDEX]
|
||||
+ isv[RL_GAMMA_INDEX]
|
||||
* isv[RL_C51_V_MAX_INDEX],
|
||||
"atom_max_headroom": isv[RL_C51_V_MAX_INDEX]
|
||||
- (isv[RL_REWARD_CLAMP_WIN_INDEX]
|
||||
+ isv[RL_GAMMA_INDEX]
|
||||
* isv[RL_C51_V_MAX_INDEX]),
|
||||
"popart_sigma": isv[RL_POPART_SIGMA_INDEX],
|
||||
"v_target_max_3sigma": 3.0_f32 * isv[RL_POPART_SIGMA_INDEX],
|
||||
"atom_max_over_3sigma": isv[RL_C51_V_MAX_INDEX]
|
||||
/ (3.0_f32 * isv[RL_POPART_SIGMA_INDEX] + 1e-9_f32),
|
||||
},
|
||||
},
|
||||
"replay_len": trainer.gpu_replay.capacity.min(step + 1),
|
||||
// Post-scale, POST-clamp reward stats — what V regression
|
||||
// and Q distributional projection actually saw this step.
|
||||
@@ -1145,6 +1295,14 @@ fn main() -> Result<()> {
|
||||
// the train-end policy's OOS performance. True pure-eval (forward
|
||||
// only, no backward) is a follow-up architectural change.
|
||||
if cli.n_eval_steps > 0 && !eval_files.is_empty() {
|
||||
// Adaptive risk management (spec 2026-05-30): reset Layer 1
|
||||
// session-level state on train→eval transition. Clears session
|
||||
// pnl, DD-triggered flag, consec-loss counter, cooldown counter.
|
||||
// Kelly + inventory EMAs intentionally preserved.
|
||||
trainer
|
||||
.reset_session_state()
|
||||
.context("reset_session_state at train→eval boundary")?;
|
||||
|
||||
let head_before_eval = sim
|
||||
.read_total_trade_count()
|
||||
.context("read trade count pre-eval")?;
|
||||
|
||||
@@ -762,9 +762,13 @@ pub const RL_TRAIL_MAX_INDEX: usize = 495;
|
||||
/// (Turtle 2N convention).
|
||||
pub const RL_TRAIL_K_INIT_INDEX: usize = 496;
|
||||
|
||||
/// Trail-stop adjust rate — `rl_trail_mutate` applies
|
||||
/// `trail × adjust_rate` on a7 (tighten), `trail / adjust_rate` on
|
||||
/// a8 (loosen). Symmetric reciprocal per SP20 §4.12. Default 0.9.
|
||||
/// DEPRECATED — superseded 2026-05-30 by independent
|
||||
/// `RL_TRAIL_TIGHTEN_FACTOR_INDEX` (682) and
|
||||
/// `RL_TRAIL_LOOSEN_FACTOR_INDEX` (683) per
|
||||
/// `docs/superpowers/specs/2026-05-30-adaptive-risk-management-design.md`.
|
||||
/// `rl_trail_mutate` no longer reads this slot. Index 497 remains
|
||||
/// allocated to preserve subsequent slot numbering (renumbering would
|
||||
/// be a structural refactor). Do not write to it.
|
||||
pub const RL_TRAIL_ADJUST_RATE_INDEX: usize = 497;
|
||||
|
||||
// ─── Forward-Return-Distribution (FRD) head — supervised forecaster
|
||||
@@ -1131,5 +1135,373 @@ pub const RL_V_TRACK_ERR_EMA_INDEX: usize = 586;
|
||||
/// If `EMA < 1e-4` the controller holds α (no V signal to calibrate against).
|
||||
pub const RL_V_SCALAR_MAG_EMA_INDEX: usize = 587;
|
||||
|
||||
// ============================================================
|
||||
// Adaptive controller floors — spec 2026-05-30-adaptive-controller-floor-design
|
||||
//
|
||||
// Replaces hardcoded thresholds across 12 RL controllers with observed-
|
||||
// signal-driven ISV-slot bounds. Per `feedback_adaptive_not_tuned` and
|
||||
// `pearl_controller_anchors_isv_driven`: every threshold derives from
|
||||
// observed signal statistics (Welford online variance), not from a
|
||||
// constant calibrated against a prior signal regime.
|
||||
//
|
||||
// Group A (8 controllers that saturated under Phase 4.5):
|
||||
// - Welford variance triple (count, mean, M²) per controller's input EMA
|
||||
// - Below-band counter for asymmetric Schulman (5 of 8 use it)
|
||||
//
|
||||
// Group B (4 not-yet-saturated controllers with hardcoded thresholds):
|
||||
// - Direct ISV-slot replacements for the hardcoded values
|
||||
//
|
||||
// All slots are written by `rl_signal_variance_update` kernel or by
|
||||
// controllers themselves; consumed by individual controller kernels and
|
||||
// by the fused rl_fused_controllers kernel.
|
||||
// ============================================================
|
||||
|
||||
// ─── Group A — Welford variance triples + asymmetric-Schulman counters ───
|
||||
|
||||
/// PPO clip controller — Welford count for kl_pi_ema (`RL_KL_PI_EMA_INDEX`).
|
||||
pub const RL_KL_PI_VAR_COUNT_INDEX: usize = 588;
|
||||
/// PPO clip controller — Welford running mean for kl_pi_ema.
|
||||
pub const RL_KL_PI_VAR_MEAN_INDEX: usize = 589;
|
||||
/// PPO clip controller — Welford M² (sum of squared deviations from mean).
|
||||
/// Sample variance = M² / (count - 1).
|
||||
pub const RL_KL_PI_VAR_M2_INDEX: usize = 590;
|
||||
/// PPO clip controller — N-consecutive-below-band counter for asymmetric Schulman.
|
||||
/// Tightening fires on single observation; widening requires 3 consecutive.
|
||||
pub const RL_KL_PI_BELOW_COUNT_INDEX: usize = 591;
|
||||
|
||||
/// target_tau controller — Welford count for q_divergence_ema.
|
||||
pub const RL_Q_DIV_VAR_COUNT_INDEX: usize = 592;
|
||||
pub const RL_Q_DIV_VAR_MEAN_INDEX: usize = 593;
|
||||
pub const RL_Q_DIV_VAR_M2_INDEX: usize = 594;
|
||||
pub const RL_Q_DIV_BELOW_COUNT_INDEX: usize = 595;
|
||||
|
||||
/// rollout_steps controller — Welford for pre-normalization advantage variance
|
||||
/// (`RL_ADV_VAR_PRE_NORM_INDEX`). Post-normalization variance is definitionally
|
||||
/// near zero under Phase 4.5 and useless as a controller signal.
|
||||
pub const RL_ADV_VAR_VAR_COUNT_INDEX: usize = 596;
|
||||
pub const RL_ADV_VAR_VAR_MEAN_INDEX: usize = 597;
|
||||
pub const RL_ADV_VAR_VAR_M2_INDEX: usize = 598;
|
||||
pub const RL_ADV_VAR_BELOW_COUNT_INDEX: usize = 599;
|
||||
|
||||
/// entropy_coef controller — Welford for entropy_observed_ema.
|
||||
pub const RL_ENTROPY_OBS_VAR_COUNT_INDEX: usize = 600;
|
||||
pub const RL_ENTROPY_OBS_VAR_MEAN_INDEX: usize = 601;
|
||||
pub const RL_ENTROPY_OBS_VAR_M2_INDEX: usize = 602;
|
||||
pub const RL_ENTROPY_OBS_BELOW_COUNT_INDEX: usize = 603;
|
||||
|
||||
/// per_α controller — Welford for td_kurtosis_ema.
|
||||
pub const RL_TD_KURT_VAR_COUNT_INDEX: usize = 604;
|
||||
pub const RL_TD_KURT_VAR_MEAN_INDEX: usize = 605;
|
||||
pub const RL_TD_KURT_VAR_M2_INDEX: usize = 606;
|
||||
pub const RL_TD_KURT_BELOW_COUNT_INDEX: usize = 607;
|
||||
|
||||
/// gamma controller — Welford for mean_trade_duration_ema. Mean is used directly
|
||||
/// by `rl_gamma_controller` to compute `RL_GAMMA_MIN_ADAPTIVE_INDEX` after a
|
||||
/// 100-observation warmup (spec Q6 resolution: Welford mean's natural N-smoothing
|
||||
/// breaks the gamma↔trade_duration feedback loop without explicit step gating).
|
||||
pub const RL_TRADE_DUR_VAR_COUNT_INDEX: usize = 608;
|
||||
pub const RL_TRADE_DUR_VAR_MEAN_INDEX: usize = 609;
|
||||
pub const RL_TRADE_DUR_VAR_M2_INDEX: usize = 610;
|
||||
|
||||
/// q_distill_lambda controller — placeholder for asymmetric counter (currently
|
||||
/// unused since q_distill uses ramp/decay rather than Schulman; reserved for future).
|
||||
pub const RL_RESERVED_611_INDEX: usize = 611;
|
||||
|
||||
// ─── New input-signal slots (consumed by controllers + Welford kernel) ───
|
||||
|
||||
/// Emitted by `rl_advantage_normalize` BEFORE in-place normalize. Carries the raw
|
||||
/// variance of advantages across the batch — used by `rl_rollout_steps_controller`
|
||||
/// as the input signal that Phase 4.5 no longer provides via post-norm advantages.
|
||||
pub const RL_ADV_VAR_PRE_NORM_INDEX: usize = 612;
|
||||
|
||||
/// Adaptive lower bound for gamma, derived from `RL_TRADE_DUR_VAR_MEAN_INDEX`
|
||||
/// (Welford mean of trade duration). See `rl_gamma_controller.cu` Special case G.
|
||||
/// Replaces hardcoded `GAMMA_MIN = 0.995f` with signal-driven bound:
|
||||
/// `max(0.9, 1 - 1/(d_mean × 2))`.
|
||||
pub const RL_GAMMA_MIN_ADAPTIVE_INDEX: usize = 613;
|
||||
|
||||
/// Emitted by `rl_reward_scale_controller` — observed per-batch pnl magnitude
|
||||
/// EMA. Input to the Welford kernel; the Welford variance becomes the noise-floor
|
||||
/// reference for reward_scale's asymmetric rate limit.
|
||||
pub const RL_REWARD_MAGNITUDE_EMA_INDEX: usize = 614;
|
||||
|
||||
/// reward_scale controller — Welford triple for pnl magnitude.
|
||||
pub const RL_REWARD_MAG_VAR_COUNT_INDEX: usize = 615;
|
||||
pub const RL_REWARD_MAG_VAR_MEAN_INDEX: usize = 616;
|
||||
pub const RL_REWARD_MAG_VAR_M2_INDEX: usize = 617;
|
||||
|
||||
/// q_distill_lambda controller — Welford triple for q_distill_kl_ema.
|
||||
/// Adaptive `MIN_LAMBDA` derives from `sqrt(var) × 0.05`.
|
||||
pub const RL_Q_DISTILL_KL_VAR_COUNT_INDEX: usize = 618;
|
||||
pub const RL_Q_DISTILL_KL_VAR_MEAN_INDEX: usize = 619;
|
||||
pub const RL_Q_DISTILL_KL_VAR_M2_INDEX: usize = 620;
|
||||
|
||||
// ─── Group B — Adaptive bound slots replacing Phase 4.4/4.5-era hardcoded constants ───
|
||||
|
||||
/// V-blend alpha controller (Phase 4.4) — adaptive dead-signal floor (replaces `1e-4f`).
|
||||
pub const RL_V_BLEND_DEAD_SIGNAL_FLOOR_INDEX: usize = 621;
|
||||
/// V-blend alpha controller — target track ratio (replaces `0.10f`).
|
||||
pub const RL_V_BLEND_TARGET_TRACK_RATIO_INDEX: usize = 622;
|
||||
/// V-blend alpha controller — Schulman step magnitude (replaces `0.01f`).
|
||||
pub const RL_V_BLEND_SCHULMAN_STEP_INDEX: usize = 623;
|
||||
/// V-blend alpha controller — EMA smoothing alpha (replaces `0.01f`).
|
||||
pub const RL_V_BLEND_EMA_ALPHA_INDEX: usize = 624;
|
||||
/// V-blend alpha controller — bootstrap alpha on sentinel input (replaces `1.0f`).
|
||||
pub const RL_V_BLEND_BOOTSTRAP_ALPHA_INDEX: usize = 625;
|
||||
/// V-blend alpha controller — Welford count for V_scalar magnitude.
|
||||
pub const RL_V_BLEND_SCALAR_VAR_COUNT_INDEX: usize = 626;
|
||||
pub const RL_V_BLEND_SCALAR_VAR_MEAN_INDEX: usize = 627;
|
||||
pub const RL_V_BLEND_SCALAR_VAR_M2_INDEX: usize = 628;
|
||||
|
||||
/// PPO ratio clamp controller — adaptive max output (replaces `1000.0f`).
|
||||
/// Min derives directly from observed ratio variance: `max(1.5, 1 + 3·σ_ratio)`.
|
||||
pub const RL_PPO_RATIO_CLAMP_MAX_ADAPTIVE_INDEX: usize = 629;
|
||||
/// PPO ratio clamp controller — Welford triple for observed log-ratio magnitudes.
|
||||
pub const RL_PPO_RATIO_VAR_COUNT_INDEX: usize = 630;
|
||||
pub const RL_PPO_RATIO_VAR_MEAN_INDEX: usize = 631;
|
||||
pub const RL_PPO_RATIO_VAR_M2_INDEX: usize = 632;
|
||||
|
||||
/// Reward clamp controller — V-bound floor (replaces hardcoded `V_BOUND_FLOOR = 1.0f`).
|
||||
pub const RL_REWARD_CLAMP_V_BOUND_FLOOR_INDEX: usize = 633;
|
||||
/// Reward clamp controller — V-bound EWMA alpha (replaces `0.001f`).
|
||||
pub const RL_REWARD_CLAMP_V_BOUND_EWMA_ALPHA_INDEX: usize = 634;
|
||||
/// Reward clamp controller — minimum win clip magnitude (replaces `MIN_WIN = 1.0f`).
|
||||
pub const RL_REWARD_CLAMP_MIN_WIN_INDEX: usize = 635;
|
||||
/// Reward clamp controller — minimum win/loss ratio (replaces `MIN_RATIO = 1.0f`).
|
||||
pub const RL_REWARD_CLAMP_MIN_RATIO_INDEX: usize = 636;
|
||||
/// Reward clamp controller — maximum win/loss ratio (replaces `MAX_RATIO = 3.0f`).
|
||||
pub const RL_REWARD_CLAMP_MAX_RATIO_INDEX: usize = 637;
|
||||
|
||||
/// Gate threshold controller — EMA smoothing alpha (replaces hardcoded `alpha = 0.01f`).
|
||||
pub const RL_GATE_EMA_ALPHA_INDEX: usize = 638;
|
||||
|
||||
/// q_distill_lambda controller — adaptive MIN bound (replaces hardcoded `MIN_LAMBDA = 0.05f`).
|
||||
/// Derives from `sqrt(q_distill_kl_var) × 0.05` with absolute floor 0.001.
|
||||
pub const RL_Q_DISTILL_LAMBDA_MIN_ADAPTIVE_INDEX: usize = 639;
|
||||
|
||||
// ============================================================
|
||||
// Extension (2026-05-30, same spec): all hardcoded MIN/MAX clamp bounds +
|
||||
// ramp/decay rates across the 12 controllers become ISV-driven. Same
|
||||
// `feedback_adaptive_not_tuned` principle applied consistently — clamp
|
||||
// bounds calibrated against a prior signal regime are the next layer of
|
||||
// the same architectural bug that saturated the 8 noise-floor controllers
|
||||
// in fold 0/1. Bootstrap defaults preserve current behavior.
|
||||
// ============================================================
|
||||
|
||||
// ─── PPO clip clamp bounds (rl_ppo_clip_controller) ───
|
||||
pub const RL_PPO_CLIP_EPS_MIN_INDEX: usize = 640; // replaces EPS_MIN = 0.05f
|
||||
pub const RL_PPO_CLIP_EPS_MAX_INDEX: usize = 641; // replaces EPS_MAX = 0.5f
|
||||
|
||||
// ─── target_tau clamp bound (rl_target_tau_controller) ───
|
||||
pub const RL_TARGET_TAU_MIN_INDEX: usize = 642; // replaces TAU_MIN = 0.001f
|
||||
|
||||
// ─── rollout_steps clamp bounds (rl_rollout_steps_controller) ───
|
||||
pub const RL_ROLLOUT_MIN_INDEX: usize = 643; // replaces ROLLOUT_MIN = 256.0f
|
||||
pub const RL_ROLLOUT_MAX_INDEX: usize = 644; // replaces ROLLOUT_MAX = 8192.0f
|
||||
|
||||
// ─── entropy_coef clamp bounds (rl_entropy_coef_controller) ───
|
||||
pub const RL_ENTROPY_COEF_MIN_INDEX: usize = 645; // replaces COEF_MIN = 0.0f
|
||||
pub const RL_ENTROPY_COEF_MAX_INDEX: usize = 646; // replaces COEF_MAX = 0.5f
|
||||
|
||||
// ─── per_α clamp bounds (rl_per_alpha_controller) ───
|
||||
pub const RL_PER_ALPHA_MIN_INDEX: usize = 647; // replaces PER_ALPHA_MIN = 0.3f
|
||||
pub const RL_PER_ALPHA_MAX_INDEX: usize = 648; // replaces PER_ALPHA_MAX = 1.0f
|
||||
|
||||
// ─── gamma clamp ceiling (rl_gamma_controller) ───
|
||||
// GAMMA_MIN is already adaptive via RL_GAMMA_MIN_ADAPTIVE_INDEX=613.
|
||||
pub const RL_GAMMA_MAX_INDEX: usize = 649; // replaces GAMMA_MAX = 0.999f
|
||||
|
||||
// ─── q_distill_lambda clamp / rate constants (rl_q_distill_lambda_controller) ───
|
||||
// MIN_LAMBDA is already adaptive via RL_Q_DISTILL_LAMBDA_MIN_ADAPTIVE_INDEX=639.
|
||||
pub const RL_Q_DISTILL_LAMBDA_MAX_INDEX: usize = 650; // replaces MAX_LAMBDA = 1.0f
|
||||
pub const RL_Q_DISTILL_KL_TOLERANCE_INDEX: usize = 651; // replaces KL_TOLERANCE = 3.0f
|
||||
pub const RL_Q_DISTILL_LAMBDA_RAMP_RATE_INDEX: usize = 652; // replaces LAMBDA_RAMP_RATE = 1.2f
|
||||
pub const RL_Q_DISTILL_LAMBDA_DECAY_RATE_INDEX: usize = 653; // replaces LAMBDA_DECAY_RATE = 0.998f
|
||||
|
||||
// ─── ppo_ratio_clamp ceiling (rl_ppo_ratio_clamp_controller) ───
|
||||
// MIN_OUT becomes adaptive from observed ratio variance (per Group B Task 11).
|
||||
// MAX_OUT becomes ISV-driven adaptive max via RL_PPO_RATIO_CLAMP_MAX_ADAPTIVE_INDEX=629.
|
||||
// (Hardcoded MIN_OUT=2.0 absolute floor still used as `max(adaptive, 2.0)`; this is
|
||||
// the "don't degenerate to vanilla policy gradient" structural guard, treated as
|
||||
// architectural per the spec's "physics/math constants" exemption.)
|
||||
|
||||
// ─── reward_clamp adaptive bounds (rl_reward_clamp_controller) ───
|
||||
// V_BOUND_FLOOR, V_BOUND_EWMA_ALPHA, MIN_WIN, MIN_RATIO, MAX_RATIO already in 633-637.
|
||||
pub const RL_REWARD_CLAMP_MIN_MARGIN_INDEX: usize = 654; // replaces MIN_MARGIN = 1.0f
|
||||
pub const RL_REWARD_CLAMP_MAX_MARGIN_INDEX: usize = 655; // replaces MAX_MARGIN = 5.0f
|
||||
pub const RL_REWARD_CLAMP_MARGIN_TOLERANCE_INDEX: usize = 656; // replaces MARGIN_TOLERANCE = 1.5f
|
||||
pub const RL_REWARD_CLAMP_MARGIN_ADJUST_RATE_INDEX: usize = 657; // replaces MARGIN_ADJUST_RATE = 1.2f
|
||||
pub const RL_REWARD_CLAMP_CLIP_RATE_EMA_ALPHA_INDEX: usize = 658; // replaces CLIP_RATE_EMA_ALPHA = 0.05f
|
||||
|
||||
// ─── Wiener-α floor (shared across 9 controllers — `pearl_wiener_alpha_floor_for_nonstationary`) ───
|
||||
// The 0.4f floor is mathematically motivated but still a tuned constant.
|
||||
// Single shared ISV slot — all controllers read from this slot for consistency.
|
||||
pub const RL_WIENER_ALPHA_FLOOR_INDEX: usize = 659; // replaces WIENER_ALPHA_FLOOR = 0.4f
|
||||
|
||||
// ─── reward_scale floor fix (2026-05-30, follow-up to spec Special R) ───
|
||||
//
|
||||
// The original Special case R used `RL_TRADE_DUR_VAR_COUNT_INDEX` (Welford
|
||||
// count of trade_duration_ema) as the gate for releasing the 10%-of-bootstrap
|
||||
// floor. That counter actually increments once per STEP where ANY trade
|
||||
// closes — not once per closed trade. With b=1024 and ~7 dones/step, the
|
||||
// gate released after step ~100 (only ~700 actual closed trades) instead
|
||||
// of after the intended "100 closed trades worth of confidence". After
|
||||
// release, reward_scale crashed 0.13 → 0.001 in cluster validation.
|
||||
//
|
||||
// Fix: track cumulative closed trades via dones-sum accumulation. Gate the
|
||||
// bootstrap floor on observed cumulative trade count, with the threshold
|
||||
// itself ISV-driven (per `feedback_adaptive_not_tuned`).
|
||||
|
||||
/// Cumulative count of closed trades since training start. Incremented by
|
||||
/// `sum(dones[b])` each step in `rl_fused_controllers`. Used by the
|
||||
/// reward_scale controller's bootstrap-fraction floor gate.
|
||||
pub const RL_CUMULATIVE_DONES_INDEX: usize = 660;
|
||||
|
||||
/// Trade-count threshold for releasing reward_scale's bootstrap-fraction
|
||||
/// floor. Bootstrap 5000 = ~715 steps at typical b=1024 dones rate; enough
|
||||
/// for the reward magnitude EMA to stabilize before the normalizer is
|
||||
/// allowed to operate freely.
|
||||
pub const RL_MIN_TRADES_FOR_RELEASE_INDEX: usize = 661;
|
||||
|
||||
/// Last RL-allocated slot index (exclusive). Pre-spec: 588. Post-noise-floor: 640.
|
||||
/// Post-clamp-bounds extension: 660. Post-reward-floor-fix: 662. Total new: 74 slots.
|
||||
// ============================================================
|
||||
// Adaptive risk management — layered stack (spec 2026-05-30-adaptive-risk-management-design)
|
||||
//
|
||||
// Five orthogonal risk layers, each ISV-driven:
|
||||
// Layer 1 — CMDP hard constraints (session DD, cooldown, max open, inventory)
|
||||
// Layer 2 — IQN risk-averse action selection via adaptive τ
|
||||
// Layer 3 — Inventory penalty (Avellaneda-Stoikov) in reward shaping
|
||||
// Layer 4 — Kelly-fraction position sizing
|
||||
// Layer D — TrailTighten/Loosen wiring (close dead-action gap per
|
||||
// pearl_dead_trail_stop_actions_a7_a8)
|
||||
//
|
||||
// Per `feedback_adaptive_not_tuned`: every threshold below is signal-driven
|
||||
// or warmup-gated, not a hardcoded constant.
|
||||
// ============================================================
|
||||
|
||||
// ─── Layer 1: CMDP hard constraints (9 slots) ───
|
||||
|
||||
/// Layer 1 — Cumulative pnl since session boundary (USD). Reset on
|
||||
/// train/eval phase transition by the trainer.
|
||||
pub const RL_SESSION_PNL_USD_INDEX: usize = 662;
|
||||
/// Layer 1 — Drawdown cap (USD, negative threshold). Bootstrap -3500 =
|
||||
/// 10% of $35k starting capital per `project_ml_alpha_starting_capital`.
|
||||
pub const RL_SESSION_DD_LIMIT_USD_INDEX: usize = 663;
|
||||
/// Layer 1 — Sticky flag (1.0 if session DD breached, 0.0 otherwise).
|
||||
/// Override-to-Hold in actions_to_market_targets when set.
|
||||
pub const RL_SESSION_DD_TRIGGERED_INDEX: usize = 664;
|
||||
/// Layer 1 — Max concurrent open positions per batch. Bootstrap = MAX_UNITS = 4.
|
||||
pub const RL_MAX_OPEN_UNITS_INDEX: usize = 665;
|
||||
/// Layer 1 — Max consecutive losses before cool-down. Bootstrap 10.
|
||||
pub const RL_CONSEC_LOSS_LIMIT_INDEX: usize = 666;
|
||||
/// Layer 1 — Running consecutive-loss count. Reset on win.
|
||||
pub const RL_CONSEC_LOSS_COUNT_INDEX: usize = 667;
|
||||
/// Layer 1 — Cool-down period remaining (steps). When > 0, override to Hold.
|
||||
pub const RL_COOLDOWN_REMAINING_STEPS_INDEX: usize = 668;
|
||||
/// Layer 1 — Default cool-down length (steps). Set when consec loss limit hit.
|
||||
pub const RL_COOLDOWN_DURATION_INDEX: usize = 669;
|
||||
/// Layer 1 — Cap on net one-sided exposure (USD). Bootstrap $10k.
|
||||
pub const RL_NET_INVENTORY_LIMIT_USD_INDEX: usize = 670;
|
||||
|
||||
// ─── Layer 2: IQN risk-averse τ for action selection (3 slots) ───
|
||||
|
||||
/// Layer 2 — Quantile τ ∈ (0, 1] used for action selection.
|
||||
/// Bootstrap 0.5 (median = approximately current expected-Q behavior).
|
||||
/// Adaptive: drops toward MIN under session drawdown.
|
||||
pub const RL_IQN_ACTION_TAU_INDEX: usize = 671;
|
||||
/// Layer 2 — Lower bound on action τ. Bootstrap 0.1 (pessimistic floor).
|
||||
pub const RL_IQN_ACTION_TAU_MIN_INDEX: usize = 672;
|
||||
/// Layer 2 — How aggressively τ drops with session drawdown fraction.
|
||||
/// `τ = max(τ_min, 0.5 - sensitivity × drawdown_frac)`. Bootstrap 5.0
|
||||
/// (= τ hits MIN at 10% drawdown of starting capital).
|
||||
pub const RL_IQN_ACTION_TAU_DD_SENSITIVITY_INDEX: usize = 673;
|
||||
|
||||
// ─── Layer 3: Inventory penalty (Avellaneda-Stoikov style) (2 slots) ───
|
||||
|
||||
/// Layer 3 — Penalty coefficient β. `reward_shaped = reward_raw - β × |net_position|`.
|
||||
/// Bootstrap 0.0 (sentinel — adaptive on first inventory variance observation).
|
||||
pub const RL_INVENTORY_PENALTY_BETA_INDEX: usize = 674;
|
||||
/// Layer 3 — Observed inventory variance EMA (Welford-style).
|
||||
/// β controller scales penalty to ~1% of reward magnitude at 2σ inventory.
|
||||
pub const RL_INVENTORY_VARIANCE_EMA_INDEX: usize = 675;
|
||||
|
||||
// ─── Layer 4: Kelly-fraction position sizing (6 slots) ───
|
||||
|
||||
/// Layer 4 — Output: position-size multiplier ∈ [0, 1].
|
||||
/// Bootstrap 1.0 (full size). Adaptive from observed win_rate × R-multiple.
|
||||
pub const RL_KELLY_FRACTION_INDEX: usize = 676;
|
||||
/// Layer 4 — Observed win_rate EMA. Bootstrap 0.5 (neutral sentinel).
|
||||
pub const RL_WIN_RATE_EMA_INDEX: usize = 677;
|
||||
/// Layer 4 — Observed avg_win EMA in USD. Bootstrap 0.0 sentinel.
|
||||
pub const RL_AVG_WIN_USD_EMA_INDEX: usize = 678;
|
||||
/// Layer 4 — Observed avg_loss EMA in USD. Bootstrap 0.0 sentinel.
|
||||
pub const RL_AVG_LOSS_USD_EMA_INDEX: usize = 679;
|
||||
/// Layer 4 — Safety multiplier on Kelly. Bootstrap 0.5 (Thorp half-Kelly).
|
||||
pub const RL_KELLY_SAFETY_FRAC_INDEX: usize = 680;
|
||||
/// Layer 4 — Warmup gate: hold at bootstrap=1.0 until N closed trades.
|
||||
/// Uses `RL_CUMULATIVE_DONES_INDEX` from b1ef6664a as the counter signal.
|
||||
pub const RL_KELLY_MIN_TRADES_FOR_RELEASE_INDEX: usize = 681;
|
||||
|
||||
// ─── Layer D: TrailTighten/Loosen wiring (2 slots) ───
|
||||
|
||||
/// Layer D — Factor applied to unit_trail_distance on action a7 (TrailTighten).
|
||||
/// Bootstrap 0.9 (tighten by 10%). Closes pearl_dead_trail_stop_actions_a7_a8.
|
||||
pub const RL_TRAIL_TIGHTEN_FACTOR_INDEX: usize = 682;
|
||||
/// Layer D — Factor applied to unit_trail_distance on action a8 (TrailLoosen).
|
||||
/// Bootstrap 1.1 (loosen by 10%).
|
||||
pub const RL_TRAIL_LOOSEN_FACTOR_INDEX: usize = 683;
|
||||
|
||||
/// Layer 1 diag — worst-account session pnl across the per-batch fleet.
|
||||
/// `RL_SESSION_PNL_USD_INDEX` (662) is now the MEAN-of-active-accounts (the
|
||||
/// signal IQN-τ consumes); this slot mirrors the most-negative per-batch
|
||||
/// session pnl for diag visibility. Without this split, a single bad
|
||||
/// account would drag IQN-τ to its floor for the entire fold even after
|
||||
/// the [[feedback_no_partial_refactor]] per-batch fix.
|
||||
pub const RL_SESSION_PNL_WORST_INDEX: usize = 684;
|
||||
|
||||
// ============================================================
|
||||
// v9 — Defensive eval-boundary calibration (spec 2026-05-31-v9)
|
||||
//
|
||||
// Implements [[pearl_adaptive_carryover_discipline]]: at every regime
|
||||
// boundary (fold/eval transition), defensive overrides force conservative
|
||||
// behavior for the first N steps while EMAs re-bootstrap from new-regime
|
||||
// data. After N steps, override values linearly decay back to the normal
|
||||
// (bootstrap-configured) controller floors over `decay_steps` steps.
|
||||
// ============================================================
|
||||
|
||||
/// Counter — set to WARMUP_STEPS at every regime boundary, decremented each
|
||||
/// step. >0 = warmup/decay active; ≤0 = inactive (normal controller floors).
|
||||
pub const RL_EVAL_WARMUP_REMAINING_INDEX: usize = 685;
|
||||
/// Configured warmup duration (steps with full defensive overrides).
|
||||
/// Bootstrap 500 (~10% of a 5000-step eval phase).
|
||||
pub const RL_EVAL_WARMUP_STEPS_CONFIG_INDEX: usize = 686;
|
||||
/// Configured decay duration (steps over which override → normal interpolation).
|
||||
/// Bootstrap 200. Total transition: warmup_steps + decay_steps = 700 steps.
|
||||
pub const RL_EVAL_WARMUP_DECAY_STEPS_CONFIG_INDEX: usize = 687;
|
||||
|
||||
/// Defensive override: Kelly safety_frac. Bootstrap 0.25 (quarter-Kelly).
|
||||
/// Active during warmup; decays to NORMAL (slot 692) over decay phase.
|
||||
pub const RL_EVAL_KELLY_SAFETY_DEFENSIVE_INDEX: usize = 688;
|
||||
/// Defensive override: IQN τ_min. Bootstrap 0.30 (pessimistic action floor).
|
||||
/// Active during warmup; decays to NORMAL (slot 693) over decay phase.
|
||||
pub const RL_EVAL_IQN_TAU_MIN_DEFENSIVE_INDEX: usize = 689;
|
||||
/// Defensive override: entropy_coef_min. Bootstrap 0.05 (more exploration).
|
||||
/// Active during warmup; decays to NORMAL (slot 694) over decay phase.
|
||||
pub const RL_EVAL_ENTROPY_COEF_MIN_DEFENSIVE_INDEX: usize = 690;
|
||||
/// Defensive override: PPO clip ε_min. Bootstrap 0.10 (larger policy steps).
|
||||
/// Active during warmup; decays to NORMAL (slot 695) over decay phase.
|
||||
pub const RL_EVAL_PPO_CLIP_EPS_MIN_DEFENSIVE_INDEX: usize = 691;
|
||||
|
||||
/// Normal-value saved slots — read by warmup_decay kernel as the linear-
|
||||
/// interpolation target during the decay phase, and as the final restored
|
||||
/// value after warmup completes. Bootstrap to match the existing controller
|
||||
/// floor constants. These are NOT modified after bootstrap (read-only refs).
|
||||
pub const RL_EVAL_KELLY_SAFETY_NORMAL_INDEX: usize = 692; // 0.50
|
||||
pub const RL_EVAL_IQN_TAU_MIN_NORMAL_INDEX: usize = 693; // 0.10
|
||||
pub const RL_EVAL_ENTROPY_COEF_MIN_NORMAL_INDEX: usize = 694; // 0.01
|
||||
pub const RL_EVAL_PPO_CLIP_EPS_MIN_NORMAL_INDEX: usize = 695; // 0.05
|
||||
|
||||
/// Last RL-allocated slot index (exclusive).
|
||||
pub const RL_SLOTS_END: usize = 588;
|
||||
/// Pre-risk-stack: 662. Post-Fix-B: 685. Post-v9 (warmup boundary): 696.
|
||||
pub const RL_SLOTS_END: usize = 696;
|
||||
|
||||
@@ -198,6 +198,30 @@ const RL_V_BLEND_ALPHA_CONTROLLER_CUBIN: &[u8] =
|
||||
/// Phase 4.5 per-batch advantage normalization.
|
||||
const RL_ADVANTAGE_NORMALIZE_CUBIN: &[u8] =
|
||||
include_bytes!(concat!(env!("OUT_DIR"), "/rl_advantage_normalize.cubin"));
|
||||
/// Welford online variance updater for adaptive controller-input floors
|
||||
/// (spec `2026-05-30-adaptive-controller-floor-design`). Single-thread,
|
||||
/// single-block. Caller passes ISV bus + input/count/mean/M² slot indices.
|
||||
const RL_SIGNAL_VARIANCE_UPDATE_CUBIN: &[u8] =
|
||||
include_bytes!(concat!(env!("OUT_DIR"), "/rl_signal_variance_update.cubin"));
|
||||
// Adaptive risk management — layered stack (spec 2026-05-30-adaptive-risk-management).
|
||||
// 4 controllers + 3 EMA producers feeding Layers 1-4. Layer D (trail
|
||||
// tighten/loosen) is read-only ISV consumed by the existing rl_trail_mutate.
|
||||
const RL_CMDP_CONSTRAINTS_CHECK_CUBIN: &[u8] =
|
||||
include_bytes!(concat!(env!("OUT_DIR"), "/rl_cmdp_constraints_check.cubin"));
|
||||
const RL_IQN_ACTION_TAU_CONTROLLER_CUBIN: &[u8] =
|
||||
include_bytes!(concat!(env!("OUT_DIR"), "/rl_iqn_action_tau_controller.cubin"));
|
||||
const RL_INVENTORY_BETA_CONTROLLER_CUBIN: &[u8] =
|
||||
include_bytes!(concat!(env!("OUT_DIR"), "/rl_inventory_beta_controller.cubin"));
|
||||
const RL_KELLY_FRACTION_CONTROLLER_CUBIN: &[u8] =
|
||||
include_bytes!(concat!(env!("OUT_DIR"), "/rl_kelly_fraction_controller.cubin"));
|
||||
const RL_EVAL_WARMUP_DECAY_CUBIN: &[u8] =
|
||||
include_bytes!(concat!(env!("OUT_DIR"), "/rl_eval_warmup_decay.cubin"));
|
||||
const RL_WIN_RATE_EMA_UPDATE_CUBIN: &[u8] =
|
||||
include_bytes!(concat!(env!("OUT_DIR"), "/rl_win_rate_ema_update.cubin"));
|
||||
const RL_AVG_WIN_LOSS_EMA_UPDATE_CUBIN: &[u8] =
|
||||
include_bytes!(concat!(env!("OUT_DIR"), "/rl_avg_win_loss_ema_update.cubin"));
|
||||
const RL_INVENTORY_VARIANCE_UPDATE_CUBIN: &[u8] =
|
||||
include_bytes!(concat!(env!("OUT_DIR"), "/rl_inventory_variance_update.cubin"));
|
||||
const RL_Q_PI_DISTILL_GRAD_CUBIN: &[u8] =
|
||||
include_bytes!(concat!(env!("OUT_DIR"), "/rl_q_pi_distill_grad.cubin"));
|
||||
// λ_distill adaptive controller (rljzl followup 2026-05-24).
|
||||
@@ -637,6 +661,35 @@ pub struct IntegratedTrainer {
|
||||
rl_v_blend_alpha_controller_fn: CudaFunction,
|
||||
_rl_advantage_normalize_module: Arc<CudaModule>,
|
||||
rl_advantage_normalize_fn: CudaFunction,
|
||||
// Welford variance updater for adaptive controller floors (spec
|
||||
// 2026-05-30-adaptive-controller-floor-design). One launch per
|
||||
// controller-input EMA per step, sequenced after EMA producers and
|
||||
// before controller consumers.
|
||||
_rl_signal_variance_update_module: Arc<CudaModule>,
|
||||
rl_signal_variance_update_fn: CudaFunction,
|
||||
// Adaptive risk management — layered stack (spec 2026-05-30-adaptive-risk-management).
|
||||
// CMDP gates run as a standalone kernel per step (reads pnl + outcomes).
|
||||
// The 3 layer-2/3/4 controllers are also fused into rl_fused_controllers
|
||||
// for graph-capture efficiency, but the standalone modules are loaded
|
||||
// so unit tests can drive each layer in isolation.
|
||||
_rl_cmdp_constraints_check_module: Arc<CudaModule>,
|
||||
rl_cmdp_constraints_check_fn: CudaFunction,
|
||||
_rl_iqn_action_tau_controller_module: Arc<CudaModule>,
|
||||
rl_iqn_action_tau_controller_fn: CudaFunction,
|
||||
_rl_inventory_beta_controller_module: Arc<CudaModule>,
|
||||
rl_inventory_beta_controller_fn: CudaFunction,
|
||||
_rl_kelly_fraction_controller_module: Arc<CudaModule>,
|
||||
rl_kelly_fraction_controller_fn: CudaFunction,
|
||||
// v9 (2026-05-31): defensive eval-boundary calibration kernel.
|
||||
_rl_eval_warmup_decay_module: Arc<CudaModule>,
|
||||
rl_eval_warmup_decay_fn: CudaFunction,
|
||||
// EMA producers feeding the new controllers.
|
||||
_rl_win_rate_ema_update_module: Arc<CudaModule>,
|
||||
rl_win_rate_ema_update_fn: CudaFunction,
|
||||
_rl_avg_win_loss_ema_update_module: Arc<CudaModule>,
|
||||
rl_avg_win_loss_ema_update_fn: CudaFunction,
|
||||
_rl_inventory_variance_update_module: Arc<CudaModule>,
|
||||
rl_inventory_variance_update_fn: CudaFunction,
|
||||
/// Phase 4.4 blended V baselines fed to compute_advantage_return.
|
||||
pub v_blended_d: CudaSlice<f32>,
|
||||
pub v_blended_tp1_d: CudaSlice<f32>,
|
||||
@@ -695,6 +748,16 @@ pub struct IntegratedTrainer {
|
||||
rl_write_u64_fn: CudaFunction,
|
||||
ts_ns_d: CudaSlice<u64>,
|
||||
pub outcome_ema_d: CudaSlice<f32>,
|
||||
/// CMDP Layer 1 per-batch state (spec 2026-05-30-adaptive-risk-management).
|
||||
/// Each batch element is an independent backtest "session"; one session
|
||||
/// hitting its DD limit or losing-streak cooldown must not gate the
|
||||
/// other 1023 sessions. Canonical-summary aggregates (worst-account
|
||||
/// PnL, max cooldown) are mirrored into ISV[662/664/667/668] for diag
|
||||
/// + IQN-τ consumption.
|
||||
pub session_pnl_per_batch_d: CudaSlice<f32>,
|
||||
pub session_dd_triggered_per_batch_d: CudaSlice<f32>,
|
||||
pub consec_loss_per_batch_d: CudaSlice<f32>,
|
||||
pub cooldown_remaining_per_batch_d: CudaSlice<f32>,
|
||||
pub trade_context_d: CudaSlice<f32>,
|
||||
pub multires_output_d: CudaSlice<f32>,
|
||||
multires_state_d: CudaSlice<f32>,
|
||||
@@ -1504,6 +1567,64 @@ impl IntegratedTrainer {
|
||||
let rl_advantage_normalize_fn = rl_advantage_normalize_module
|
||||
.load_function("rl_advantage_normalize")
|
||||
.context("load rl_advantage_normalize")?;
|
||||
// Welford online variance updater for adaptive controller-input
|
||||
// floors (spec 2026-05-30-adaptive-controller-floor-design).
|
||||
let rl_signal_variance_update_module = ctx
|
||||
.load_cubin(RL_SIGNAL_VARIANCE_UPDATE_CUBIN.to_vec())
|
||||
.context("load rl_signal_variance_update cubin")?;
|
||||
let rl_signal_variance_update_fn = rl_signal_variance_update_module
|
||||
.load_function("rl_signal_variance_update")
|
||||
.context("load rl_signal_variance_update")?;
|
||||
// Adaptive risk management — layered stack (spec 2026-05-30-adaptive-risk-management).
|
||||
let rl_cmdp_constraints_check_module = ctx
|
||||
.load_cubin(RL_CMDP_CONSTRAINTS_CHECK_CUBIN.to_vec())
|
||||
.context("load rl_cmdp_constraints_check cubin")?;
|
||||
let rl_cmdp_constraints_check_fn = rl_cmdp_constraints_check_module
|
||||
.load_function("rl_cmdp_constraints_check")
|
||||
.context("load rl_cmdp_constraints_check")?;
|
||||
let rl_iqn_action_tau_controller_module = ctx
|
||||
.load_cubin(RL_IQN_ACTION_TAU_CONTROLLER_CUBIN.to_vec())
|
||||
.context("load rl_iqn_action_tau_controller cubin")?;
|
||||
let rl_iqn_action_tau_controller_fn = rl_iqn_action_tau_controller_module
|
||||
.load_function("rl_iqn_action_tau_controller")
|
||||
.context("load rl_iqn_action_tau_controller")?;
|
||||
let rl_inventory_beta_controller_module = ctx
|
||||
.load_cubin(RL_INVENTORY_BETA_CONTROLLER_CUBIN.to_vec())
|
||||
.context("load rl_inventory_beta_controller cubin")?;
|
||||
let rl_inventory_beta_controller_fn = rl_inventory_beta_controller_module
|
||||
.load_function("rl_inventory_beta_controller")
|
||||
.context("load rl_inventory_beta_controller")?;
|
||||
let rl_kelly_fraction_controller_module = ctx
|
||||
.load_cubin(RL_KELLY_FRACTION_CONTROLLER_CUBIN.to_vec())
|
||||
.context("load rl_kelly_fraction_controller cubin")?;
|
||||
let rl_kelly_fraction_controller_fn = rl_kelly_fraction_controller_module
|
||||
.load_function("rl_kelly_fraction_controller")
|
||||
.context("load rl_kelly_fraction_controller")?;
|
||||
// v9 — defensive eval-boundary calibration kernel.
|
||||
let rl_eval_warmup_decay_module = ctx
|
||||
.load_cubin(RL_EVAL_WARMUP_DECAY_CUBIN.to_vec())
|
||||
.context("load rl_eval_warmup_decay cubin")?;
|
||||
let rl_eval_warmup_decay_fn = rl_eval_warmup_decay_module
|
||||
.load_function("rl_eval_warmup_decay")
|
||||
.context("load rl_eval_warmup_decay")?;
|
||||
let rl_win_rate_ema_update_module = ctx
|
||||
.load_cubin(RL_WIN_RATE_EMA_UPDATE_CUBIN.to_vec())
|
||||
.context("load rl_win_rate_ema_update cubin")?;
|
||||
let rl_win_rate_ema_update_fn = rl_win_rate_ema_update_module
|
||||
.load_function("rl_win_rate_ema_update")
|
||||
.context("load rl_win_rate_ema_update")?;
|
||||
let rl_avg_win_loss_ema_update_module = ctx
|
||||
.load_cubin(RL_AVG_WIN_LOSS_EMA_UPDATE_CUBIN.to_vec())
|
||||
.context("load rl_avg_win_loss_ema_update cubin")?;
|
||||
let rl_avg_win_loss_ema_update_fn = rl_avg_win_loss_ema_update_module
|
||||
.load_function("rl_avg_win_loss_ema_update")
|
||||
.context("load rl_avg_win_loss_ema_update")?;
|
||||
let rl_inventory_variance_update_module = ctx
|
||||
.load_cubin(RL_INVENTORY_VARIANCE_UPDATE_CUBIN.to_vec())
|
||||
.context("load rl_inventory_variance_update cubin")?;
|
||||
let rl_inventory_variance_update_fn = rl_inventory_variance_update_module
|
||||
.load_function("rl_inventory_variance_update")
|
||||
.context("load rl_inventory_variance_update")?;
|
||||
let rl_q_distill_lambda_controller_module = ctx
|
||||
.load_cubin(RL_Q_DISTILL_LAMBDA_CONTROLLER_CUBIN.to_vec())
|
||||
.context("load rl_q_distill_lambda_controller cubin")?;
|
||||
@@ -1573,7 +1694,7 @@ impl IntegratedTrainer {
|
||||
crate::rl::isv_slots::RL_Q_DIVERGENCE_EMA_INDEX as i32,
|
||||
crate::rl::isv_slots::RL_KL_PI_EMA_INDEX as i32,
|
||||
crate::rl::isv_slots::RL_ENTROPY_OBSERVED_EMA_INDEX as i32,
|
||||
crate::rl::isv_slots::RL_ADVANTAGE_VAR_RATIO_EMA_INDEX as i32,
|
||||
crate::rl::isv_slots::RL_ADV_VAR_PRE_NORM_INDEX as i32,
|
||||
crate::rl::isv_slots::RL_TD_KURTOSIS_EMA_INDEX as i32,
|
||||
crate::rl::isv_slots::RL_MEAN_ABS_PNL_EMA_INDEX as i32,
|
||||
];
|
||||
@@ -1884,6 +2005,21 @@ impl IntegratedTrainer {
|
||||
let outcome_ema_d = stream
|
||||
.alloc_zeros::<f32>(b_size)
|
||||
.context("alloc outcome_ema_d")?;
|
||||
// CMDP per-batch state (spec 2026-05-30-adaptive-risk-management).
|
||||
// Each backtest session in the batch tracks its own running PnL,
|
||||
// DD-triggered flag, losing-streak counter, and cooldown remaining.
|
||||
let session_pnl_per_batch_d = stream
|
||||
.alloc_zeros::<f32>(b_size)
|
||||
.context("alloc session_pnl_per_batch_d")?;
|
||||
let session_dd_triggered_per_batch_d = stream
|
||||
.alloc_zeros::<f32>(b_size)
|
||||
.context("alloc session_dd_triggered_per_batch_d")?;
|
||||
let consec_loss_per_batch_d = stream
|
||||
.alloc_zeros::<f32>(b_size)
|
||||
.context("alloc consec_loss_per_batch_d")?;
|
||||
let cooldown_remaining_per_batch_d = stream
|
||||
.alloc_zeros::<f32>(b_size)
|
||||
.context("alloc cooldown_remaining_per_batch_d")?;
|
||||
let trade_context_d = stream
|
||||
.alloc_zeros::<f32>(b_size * 4)
|
||||
.context("alloc trade_context_d")?;
|
||||
@@ -2536,6 +2672,25 @@ impl IntegratedTrainer {
|
||||
rl_v_blend_alpha_controller_fn,
|
||||
_rl_advantage_normalize_module: rl_advantage_normalize_module,
|
||||
rl_advantage_normalize_fn,
|
||||
_rl_signal_variance_update_module: rl_signal_variance_update_module,
|
||||
rl_signal_variance_update_fn,
|
||||
// Adaptive risk management — layered stack (spec 2026-05-30-adaptive-risk-management).
|
||||
_rl_cmdp_constraints_check_module: rl_cmdp_constraints_check_module,
|
||||
rl_cmdp_constraints_check_fn,
|
||||
_rl_iqn_action_tau_controller_module: rl_iqn_action_tau_controller_module,
|
||||
rl_iqn_action_tau_controller_fn,
|
||||
_rl_inventory_beta_controller_module: rl_inventory_beta_controller_module,
|
||||
rl_inventory_beta_controller_fn,
|
||||
_rl_kelly_fraction_controller_module: rl_kelly_fraction_controller_module,
|
||||
rl_kelly_fraction_controller_fn,
|
||||
_rl_eval_warmup_decay_module: rl_eval_warmup_decay_module,
|
||||
rl_eval_warmup_decay_fn,
|
||||
_rl_win_rate_ema_update_module: rl_win_rate_ema_update_module,
|
||||
rl_win_rate_ema_update_fn,
|
||||
_rl_avg_win_loss_ema_update_module: rl_avg_win_loss_ema_update_module,
|
||||
rl_avg_win_loss_ema_update_fn,
|
||||
_rl_inventory_variance_update_module: rl_inventory_variance_update_module,
|
||||
rl_inventory_variance_update_fn,
|
||||
v_blended_d,
|
||||
v_blended_tp1_d,
|
||||
_rl_q_distill_lambda_controller_module: rl_q_distill_lambda_controller_module,
|
||||
@@ -2577,6 +2732,10 @@ impl IntegratedTrainer {
|
||||
rl_write_u64_fn,
|
||||
ts_ns_d,
|
||||
outcome_ema_d,
|
||||
session_pnl_per_batch_d,
|
||||
session_dd_triggered_per_batch_d,
|
||||
consec_loss_per_batch_d,
|
||||
cooldown_remaining_per_batch_d,
|
||||
trade_context_d,
|
||||
multires_output_d,
|
||||
multires_state_d,
|
||||
@@ -2809,6 +2968,97 @@ impl IntegratedTrainer {
|
||||
.with_controllers_bootstrapped()?)
|
||||
}
|
||||
|
||||
/// Adaptive risk management — session boundary reset (spec 2026-05-30).
|
||||
/// Called by the caller at train→eval (or fold) transition. Clears
|
||||
/// the Layer 1 session-level state so the eval phase starts with
|
||||
/// fresh DD tracking + cooldown counters. Kelly + inventory EMAs
|
||||
/// are intentionally NOT reset: training-acquired statistics should
|
||||
/// carry into eval (matches "policy learned during train, applied
|
||||
/// in eval" intent).
|
||||
///
|
||||
/// Fix D (2026-05-30): also zero the per-batch state buffers added
|
||||
/// by the CMDP per-batch refactor (39efacf77). Without this, the
|
||||
/// summary slots are reset to 0 but per-batch arrays carry over,
|
||||
/// so on the next launch the CMDP kernel re-overwrites the summary
|
||||
/// slots with the stale aggregates. Net effect was eval phases
|
||||
/// inheriting all the dead/cooldown accounts from training.
|
||||
pub fn reset_session_state(&self) -> Result<()> {
|
||||
// Summary slots — direct ISV writes.
|
||||
//
|
||||
// v9 (2026-05-31, [[pearl_adaptive_carryover_discipline]]): reset
|
||||
// EVERY adaptive EMA driving downstream behavior, including the
|
||||
// ones Fix D previously preserved. The rationale for the prior
|
||||
// selective preservation ("training-acquired statistics should
|
||||
// carry into eval") proved wrong at the v8 fold-1 boundary:
|
||||
// Kelly entered eval with train wr=0.34 vs eval wr=0.25, took
|
||||
// overconfident positions, and lost $500k before EMAs could
|
||||
// re-converge. Pure adaptive principle now: every signal that
|
||||
// drives behavior must be re-bootstrapped from new-regime data.
|
||||
// Defensive warmup duration. Matches RL_EVAL_WARMUP_STEPS_CONFIG_INDEX
|
||||
// bootstrap (see ISV bootstrap block in `with_controllers_bootstrapped`).
|
||||
// Hardcoded here because reset_session_state runs before any ISV read
|
||||
// would observe a host-visible value; the kernel reads its own config.
|
||||
let warmup_steps = 500.0_f32;
|
||||
for (slot, value) in [
|
||||
// Layer 1 CMDP summary slots (kept from Fix D).
|
||||
(crate::rl::isv_slots::RL_SESSION_PNL_USD_INDEX, 0.0_f32),
|
||||
(crate::rl::isv_slots::RL_SESSION_PNL_WORST_INDEX, 0.0_f32),
|
||||
(crate::rl::isv_slots::RL_SESSION_DD_TRIGGERED_INDEX, 0.0_f32),
|
||||
(crate::rl::isv_slots::RL_CONSEC_LOSS_COUNT_INDEX, 0.0_f32),
|
||||
(crate::rl::isv_slots::RL_COOLDOWN_REMAINING_STEPS_INDEX, 0.0_f32),
|
||||
// Layer 4 (Kelly) EMAs — NEW in v9, was preserved by Fix D.
|
||||
(crate::rl::isv_slots::RL_WIN_RATE_EMA_INDEX, 0.5_f32), // neutral sentinel
|
||||
(crate::rl::isv_slots::RL_AVG_WIN_USD_EMA_INDEX, 0.0_f32), // sentinel
|
||||
(crate::rl::isv_slots::RL_AVG_LOSS_USD_EMA_INDEX, 0.0_f32), // sentinel
|
||||
(crate::rl::isv_slots::RL_CUMULATIVE_DONES_INDEX, 0.0_f32), // re-enter warmup
|
||||
(crate::rl::isv_slots::RL_KELLY_FRACTION_INDEX, 1.0_f32), // bootstrap
|
||||
// Layer 3 (Inventory) EMAs — NEW in v9.
|
||||
(crate::rl::isv_slots::RL_INVENTORY_PENALTY_BETA_INDEX, 0.0_f32), // sentinel
|
||||
(crate::rl::isv_slots::RL_INVENTORY_VARIANCE_EMA_INDEX, 0.0_f32), // sentinel
|
||||
// Reward-clamp EMAs — NEW in v9.
|
||||
(crate::rl::isv_slots::RL_POS_SCALED_REWARD_MAX_EMA_INDEX, 0.0_f32),
|
||||
(crate::rl::isv_slots::RL_NEG_SCALED_REWARD_MAX_EMA_INDEX, 0.0_f32),
|
||||
(crate::rl::isv_slots::RL_REWARD_CLAMP_CLIP_RATE_EMA_INDEX, 0.0_f32),
|
||||
// v9 — arm the defensive warmup window.
|
||||
(crate::rl::isv_slots::RL_EVAL_WARMUP_REMAINING_INDEX, warmup_steps),
|
||||
] {
|
||||
let mut args = RawArgs::new();
|
||||
args.push_ptr(self.isv_dev_ptr);
|
||||
args.push_i32(slot as i32);
|
||||
args.push_f32(value);
|
||||
let mut ptrs = args.build_arg_ptrs();
|
||||
unsafe {
|
||||
raw_launch(
|
||||
self.rl_isv_write_fn.cu_function(),
|
||||
(1, 1, 1), (1, 1, 1), 0,
|
||||
self.raw_stream,
|
||||
&mut ptrs[..args.len()],
|
||||
).map_err(|e| anyhow::anyhow!("rl_isv_write (reset_session_state slot {slot}): {:?}", e))?;
|
||||
}
|
||||
}
|
||||
// Per-batch buffers — bulk memset to zero. Each is [b_size] f32.
|
||||
for (buf_ptr, buf_bytes, name) in [
|
||||
(self.session_pnl_per_batch_d.raw_ptr(),
|
||||
self.session_pnl_per_batch_d.num_bytes(),
|
||||
"session_pnl_per_batch_d"),
|
||||
(self.session_dd_triggered_per_batch_d.raw_ptr(),
|
||||
self.session_dd_triggered_per_batch_d.num_bytes(),
|
||||
"session_dd_triggered_per_batch_d"),
|
||||
(self.consec_loss_per_batch_d.raw_ptr(),
|
||||
self.consec_loss_per_batch_d.num_bytes(),
|
||||
"consec_loss_per_batch_d"),
|
||||
(self.cooldown_remaining_per_batch_d.raw_ptr(),
|
||||
self.cooldown_remaining_per_batch_d.num_bytes(),
|
||||
"cooldown_remaining_per_batch_d"),
|
||||
] {
|
||||
unsafe {
|
||||
raw_memset_d8_zero(buf_ptr, buf_bytes, self.raw_stream)
|
||||
.map_err(|e| anyhow::anyhow!("zero {name}: {:?}", e))?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Read a single ISV slot from mapped-pinned host memory. Volatile
|
||||
/// read prevents the compiler from caching stale values. The GPU
|
||||
/// writes via `dev_ptr`; by the time the host reads (after stream
|
||||
@@ -2886,7 +3136,7 @@ impl IntegratedTrainer {
|
||||
// (slot, value) pair — pure device write, no HtoD per
|
||||
// `feedback_no_htod_htoh_only_mapped_pinned`.
|
||||
{
|
||||
let isv_constants: [(usize, f32); 111] = [
|
||||
let isv_constants: [(usize, f32); 179] = [
|
||||
// Static seeds for the adaptive reward-clamp controller —
|
||||
// these are the initial values that
|
||||
// `rl_reward_clamp_controller` will replace once it
|
||||
@@ -2925,7 +3175,10 @@ impl IntegratedTrainer {
|
||||
(crate::rl::isv_slots::RL_TRAIL_MIN_INDEX, 0.001),
|
||||
(crate::rl::isv_slots::RL_TRAIL_MAX_INDEX, 100.0),
|
||||
(crate::rl::isv_slots::RL_TRAIL_K_INIT_INDEX, 2.0),
|
||||
(crate::rl::isv_slots::RL_TRAIL_ADJUST_RATE_INDEX, 0.9),
|
||||
// RL_TRAIL_ADJUST_RATE_INDEX (497) deprecated 2026-05-30 —
|
||||
// superseded by independent slots 682/683 (trail tighten/loosen
|
||||
// factors) bootstrapped below. Slot 497 stays allocated for
|
||||
// numbering compatibility but is no longer written.
|
||||
// SP20 P3 FRD head seeds — λ, LR, h1-3 tick offsets, ±σ atom range.
|
||||
// Per feedback_isv_for_adaptive_bounds + single-source-of-truth:
|
||||
// horizon ticks + bucket-range σ derive from the canonical
|
||||
@@ -3008,10 +3261,35 @@ impl IntegratedTrainer {
|
||||
(crate::rl::isv_slots::RL_KL_REF_BETA_INDEX, 0.0005),
|
||||
(crate::rl::isv_slots::RL_REWARD_KL_BETA_INDEX, 0.001),
|
||||
(crate::rl::isv_slots::RL_KL_REF_TARGET_INDEX, 0.5),
|
||||
(crate::rl::isv_slots::RL_EPS_BOOTSTRAP_INDEX, 0.2),
|
||||
// Adaptive controller floors (spec 2026-05-30): bootstrap ε at
|
||||
// EPS_MIN = 0.05 (the lowest valid clip per the EPS_MIN clamp).
|
||||
// An earlier draft used 0.01 (= KL target) but that's BELOW the
|
||||
// EPS_MIN clamp range, so the post-bootstrap step always snapped
|
||||
// ε to MIN regardless of signal direction — the bootstrap-clamp
|
||||
// inconsistency Task 16 surfaced. Matching bootstrap to MIN
|
||||
// keeps the first post-bootstrap step honest: asymmetric Schulman
|
||||
// widens to track observed signal, tightens get clamped at MIN
|
||||
// (no-op, controller stays at the floor). Conservative + consistent.
|
||||
(crate::rl::isv_slots::RL_EPS_BOOTSTRAP_INDEX, 0.05),
|
||||
(crate::rl::isv_slots::RL_ROLLOUT_BOOTSTRAP_INDEX, 2048.0),
|
||||
(crate::rl::isv_slots::RL_REWARD_SCALE_BOOTSTRAP_INDEX, 1.0),
|
||||
(crate::rl::isv_slots::RL_PPO_RATIO_CLAMP_BOOTSTRAP_INDEX, 10.0),
|
||||
// Adaptive controller floors (spec 2026-05-30 Special cases G + Q):
|
||||
// safe defaults before the gamma controller / q_distill controller
|
||||
// overwrite these on their first per-step fire. γ_min 0.95
|
||||
// straddles short-trade (0.968) and surfer-regime (0.995) defaults;
|
||||
// λ_distill MIN 0.01 matches the q_distill λ bootstrap so the
|
||||
// hardcoded MIN_LAMBDA = 0.05 floor is replaced without a sudden
|
||||
// jump in λ at controller bootstrap.
|
||||
(crate::rl::isv_slots::RL_GAMMA_MIN_ADAPTIVE_INDEX, 0.95),
|
||||
(crate::rl::isv_slots::RL_Q_DISTILL_LAMBDA_MIN_ADAPTIVE_INDEX, 0.01),
|
||||
// Reward-floor fix (2026-05-30 follow-up): track real cumulative
|
||||
// closed trades (sum of dones per step) instead of closed-trade-STEPS.
|
||||
// Threshold = 5000 trades → ~715 steps of floor protection at typical
|
||||
// b=1024 dones rate. Both slots ISV-driven per
|
||||
// `feedback_adaptive_not_tuned`.
|
||||
(crate::rl::isv_slots::RL_CUMULATIVE_DONES_INDEX, 0.0),
|
||||
(crate::rl::isv_slots::RL_MIN_TRADES_FOR_RELEASE_INDEX, 5000.0),
|
||||
// IQN ensemble seeds — alpha=0.5 (equal C51/IQN blend),
|
||||
// N_TAU=32, LR=1e-3 (canonical).
|
||||
(crate::rl::isv_slots::RL_IQN_N_TAU_INDEX, 32.0),
|
||||
@@ -3053,6 +3331,106 @@ impl IntegratedTrainer {
|
||||
(crate::rl::isv_slots::RL_SAC_ALPHA_INDEX, 0.01),
|
||||
(crate::rl::isv_slots::RL_SAC_ENTROPY_TARGET_INDEX, 1.68),
|
||||
(crate::rl::isv_slots::RL_CONF_GATE_MAX_HOLD_FRAC_INDEX, 0.85),
|
||||
// ─── Adaptive controller floors — Group B + clamp-bound expansion
|
||||
// (spec 2026-05-30-adaptive-controller-floor-design). Every value
|
||||
// preserves the prior hardcoded constant exactly so the runtime
|
||||
// behaviour at bootstrap matches pre-spec behaviour bit-for-bit;
|
||||
// the win is that all bounds are now ISV-resident (visible in
|
||||
// diag JSONL, re-tunable via re-seed instead of recompile) and
|
||||
// can be replaced by Welford-driven adaptive logic in follow-up
|
||||
// controllers without further code changes.
|
||||
//
|
||||
// V-blend (Phase 4.4) — 5 design constants previously #define'd
|
||||
// in rl_v_blend_alpha_controller.cu.
|
||||
(crate::rl::isv_slots::RL_V_BLEND_DEAD_SIGNAL_FLOOR_INDEX, 1e-4),
|
||||
(crate::rl::isv_slots::RL_V_BLEND_TARGET_TRACK_RATIO_INDEX, 0.10),
|
||||
(crate::rl::isv_slots::RL_V_BLEND_SCHULMAN_STEP_INDEX, 0.01),
|
||||
(crate::rl::isv_slots::RL_V_BLEND_EMA_ALPHA_INDEX, 0.01),
|
||||
(crate::rl::isv_slots::RL_V_BLEND_BOOTSTRAP_ALPHA_INDEX, 1.0),
|
||||
// PPO ratio clamp — adaptive MAX ceiling (was hardcoded 1000.0f).
|
||||
(crate::rl::isv_slots::RL_PPO_RATIO_CLAMP_MAX_ADAPTIVE_INDEX, 1000.0),
|
||||
// Reward clamp — design constants previously #define'd in
|
||||
// rl_reward_clamp_controller.cu. Defaults match the historic
|
||||
// V_BOUND_FLOOR=1.0, V_BOUND_EWMA_ALPHA=0.001, MIN_WIN=1.0,
|
||||
// MIN_RATIO=1.0, MAX_RATIO=3.0, MIN_MARGIN=1.0, MAX_MARGIN=5.0,
|
||||
// MARGIN_TOLERANCE=1.5, MARGIN_ADJUST_RATE=1.2,
|
||||
// CLIP_RATE_EMA_ALPHA=0.05.
|
||||
(crate::rl::isv_slots::RL_REWARD_CLAMP_V_BOUND_FLOOR_INDEX, 1.0),
|
||||
(crate::rl::isv_slots::RL_REWARD_CLAMP_V_BOUND_EWMA_ALPHA_INDEX, 0.001),
|
||||
(crate::rl::isv_slots::RL_REWARD_CLAMP_MIN_WIN_INDEX, 1.0),
|
||||
(crate::rl::isv_slots::RL_REWARD_CLAMP_MIN_RATIO_INDEX, 1.0),
|
||||
(crate::rl::isv_slots::RL_REWARD_CLAMP_MAX_RATIO_INDEX, 3.0),
|
||||
(crate::rl::isv_slots::RL_REWARD_CLAMP_MIN_MARGIN_INDEX, 1.0),
|
||||
(crate::rl::isv_slots::RL_REWARD_CLAMP_MAX_MARGIN_INDEX, 5.0),
|
||||
(crate::rl::isv_slots::RL_REWARD_CLAMP_MARGIN_TOLERANCE_INDEX, 1.5),
|
||||
(crate::rl::isv_slots::RL_REWARD_CLAMP_MARGIN_ADJUST_RATE_INDEX, 1.2),
|
||||
(crate::rl::isv_slots::RL_REWARD_CLAMP_CLIP_RATE_EMA_ALPHA_INDEX, 0.05),
|
||||
// Gate threshold EMA α — was hardcoded 0.01f in
|
||||
// rl_gate_threshold_controller.cu.
|
||||
(crate::rl::isv_slots::RL_GATE_EMA_ALPHA_INDEX, 0.01),
|
||||
// Clamp bounds — Batch 2 of the 2026-05-30 expansion.
|
||||
// Each value mirrors the prior #define so behaviour is preserved.
|
||||
(crate::rl::isv_slots::RL_PPO_CLIP_EPS_MIN_INDEX, 0.05),
|
||||
(crate::rl::isv_slots::RL_PPO_CLIP_EPS_MAX_INDEX, 0.5),
|
||||
(crate::rl::isv_slots::RL_TARGET_TAU_MIN_INDEX, 0.001),
|
||||
(crate::rl::isv_slots::RL_ROLLOUT_MIN_INDEX, 256.0),
|
||||
(crate::rl::isv_slots::RL_ROLLOUT_MAX_INDEX, 8192.0),
|
||||
(crate::rl::isv_slots::RL_ENTROPY_COEF_MIN_INDEX, 0.0),
|
||||
(crate::rl::isv_slots::RL_ENTROPY_COEF_MAX_INDEX, 0.5),
|
||||
(crate::rl::isv_slots::RL_PER_ALPHA_MIN_INDEX, 0.3),
|
||||
(crate::rl::isv_slots::RL_PER_ALPHA_MAX_INDEX, 1.0),
|
||||
(crate::rl::isv_slots::RL_GAMMA_MAX_INDEX, 0.999),
|
||||
(crate::rl::isv_slots::RL_Q_DISTILL_LAMBDA_MAX_INDEX, 1.0),
|
||||
(crate::rl::isv_slots::RL_Q_DISTILL_KL_TOLERANCE_INDEX, 3.0),
|
||||
(crate::rl::isv_slots::RL_Q_DISTILL_LAMBDA_RAMP_RATE_INDEX, 1.2),
|
||||
(crate::rl::isv_slots::RL_Q_DISTILL_LAMBDA_DECAY_RATE_INDEX, 0.998),
|
||||
// Shared Wiener-α floor — read by 9 controllers (slot 659).
|
||||
(crate::rl::isv_slots::RL_WIENER_ALPHA_FLOOR_INDEX, 0.4),
|
||||
// ─── Adaptive risk management — 22 new slots (spec 2026-05-30-adaptive-risk-management) ───
|
||||
// Layer 1 (CMDP hard constraints): session DD + cooldown + caps.
|
||||
(crate::rl::isv_slots::RL_SESSION_PNL_USD_INDEX, 0.0), // session boundary reset
|
||||
(crate::rl::isv_slots::RL_SESSION_DD_LIMIT_USD_INDEX, -3500.0), // 10% of $35k starting capital
|
||||
(crate::rl::isv_slots::RL_SESSION_DD_TRIGGERED_INDEX, 0.0),
|
||||
(crate::rl::isv_slots::RL_MAX_OPEN_UNITS_INDEX, 4.0), // = current MAX_UNITS
|
||||
(crate::rl::isv_slots::RL_CONSEC_LOSS_LIMIT_INDEX, 10.0),
|
||||
(crate::rl::isv_slots::RL_CONSEC_LOSS_COUNT_INDEX, 0.0),
|
||||
(crate::rl::isv_slots::RL_COOLDOWN_REMAINING_STEPS_INDEX, 0.0),
|
||||
(crate::rl::isv_slots::RL_COOLDOWN_DURATION_INDEX, 500.0),
|
||||
(crate::rl::isv_slots::RL_NET_INVENTORY_LIMIT_USD_INDEX, 10000.0),
|
||||
// Layer 2 (IQN risk-averse τ): median bootstrap.
|
||||
(crate::rl::isv_slots::RL_IQN_ACTION_TAU_INDEX, 0.5), // median ≈ current behavior
|
||||
(crate::rl::isv_slots::RL_IQN_ACTION_TAU_MIN_INDEX, 0.1),
|
||||
(crate::rl::isv_slots::RL_IQN_ACTION_TAU_DD_SENSITIVITY_INDEX, 5.0), // hits MIN at 10% DD
|
||||
// Layer 3 (inventory penalty): sentinel-zero until variance EMA accumulates.
|
||||
(crate::rl::isv_slots::RL_INVENTORY_PENALTY_BETA_INDEX, 0.0), // sentinel
|
||||
(crate::rl::isv_slots::RL_INVENTORY_VARIANCE_EMA_INDEX, 0.0), // sentinel
|
||||
// Layer 4 (Kelly fraction sizing): full size pre-warmup, half-Kelly post.
|
||||
(crate::rl::isv_slots::RL_KELLY_FRACTION_INDEX, 1.0), // full size pre-warmup
|
||||
(crate::rl::isv_slots::RL_WIN_RATE_EMA_INDEX, 0.5), // neutral sentinel
|
||||
(crate::rl::isv_slots::RL_AVG_WIN_USD_EMA_INDEX, 0.0), // sentinel
|
||||
(crate::rl::isv_slots::RL_AVG_LOSS_USD_EMA_INDEX, 0.0), // sentinel
|
||||
(crate::rl::isv_slots::RL_KELLY_SAFETY_FRAC_INDEX, 0.5), // half-Kelly (Thorp)
|
||||
(crate::rl::isv_slots::RL_KELLY_MIN_TRADES_FOR_RELEASE_INDEX, 1000.0),
|
||||
// Layer D (TrailTighten/Loosen factors).
|
||||
(crate::rl::isv_slots::RL_TRAIL_TIGHTEN_FACTOR_INDEX, 0.9),
|
||||
(crate::rl::isv_slots::RL_TRAIL_LOOSEN_FACTOR_INDEX, 1.1),
|
||||
// v9 (defensive eval-boundary calibration, [[pearl_adaptive_carryover_discipline]]):
|
||||
// 500-step warmup with 200-step linear decay phase. During warmup,
|
||||
// risk-sizing floors are temporarily overridden with defensive
|
||||
// values; reset_session_state arms the counter at each boundary.
|
||||
(crate::rl::isv_slots::RL_EVAL_WARMUP_REMAINING_INDEX, -1.0), // sentinel: no active warmup at boot
|
||||
(crate::rl::isv_slots::RL_EVAL_WARMUP_STEPS_CONFIG_INDEX, 500.0), // total warmup duration
|
||||
(crate::rl::isv_slots::RL_EVAL_WARMUP_DECAY_STEPS_CONFIG_INDEX, 200.0), // linear decay window
|
||||
// Defensive overrides (active for ~first 300 steps post-boundary).
|
||||
(crate::rl::isv_slots::RL_EVAL_KELLY_SAFETY_DEFENSIVE_INDEX, 0.25), // quarter-Kelly
|
||||
(crate::rl::isv_slots::RL_EVAL_IQN_TAU_MIN_DEFENSIVE_INDEX, 0.30), // raise pessimism floor
|
||||
(crate::rl::isv_slots::RL_EVAL_ENTROPY_COEF_MIN_DEFENSIVE_INDEX, 0.05), // more exploration
|
||||
(crate::rl::isv_slots::RL_EVAL_PPO_CLIP_EPS_MIN_DEFENSIVE_INDEX, 0.10), // wider trust region
|
||||
// Normal target values (restored post-decay; mirror Phase-3/4 bootstrap floors).
|
||||
(crate::rl::isv_slots::RL_EVAL_KELLY_SAFETY_NORMAL_INDEX, 0.50), // half-Kelly (Thorp)
|
||||
(crate::rl::isv_slots::RL_EVAL_IQN_TAU_MIN_NORMAL_INDEX, 0.10), // existing IQN floor
|
||||
(crate::rl::isv_slots::RL_EVAL_ENTROPY_COEF_MIN_NORMAL_INDEX, 0.01), // existing entropy floor
|
||||
(crate::rl::isv_slots::RL_EVAL_PPO_CLIP_EPS_MIN_NORMAL_INDEX, 0.05), // existing PPO ε floor
|
||||
];
|
||||
for (slot, value) in isv_constants.iter() {
|
||||
let slot_i32 = *slot as i32;
|
||||
@@ -3098,10 +3476,14 @@ impl IntegratedTrainer {
|
||||
crate::rl::isv_slots::RL_ENTROPY_OBSERVED_EMA_INDEX as i32,
|
||||
)
|
||||
.context("R1 bootstrap rl_entropy_coef_controller")?;
|
||||
// Adaptive controller floors (spec 2026-05-30): rollout_steps now
|
||||
// consumes pre-normalization advantage variance (RL_ADV_VAR_PRE_NORM_INDEX).
|
||||
// Post-norm advantage variance is definitionally ≈ 0 under Phase 4.5
|
||||
// (per-batch normalize), which left this controller blind.
|
||||
self.launch_isv_controller_3arg(
|
||||
&self.rl_rollout_steps_controller_fn,
|
||||
alpha,
|
||||
crate::rl::isv_slots::RL_ADVANTAGE_VAR_RATIO_EMA_INDEX as i32,
|
||||
crate::rl::isv_slots::RL_ADV_VAR_PRE_NORM_INDEX as i32,
|
||||
)
|
||||
.context("R1 bootstrap rl_rollout_steps_controller")?;
|
||||
self.launch_isv_controller_3arg(
|
||||
@@ -3267,6 +3649,227 @@ impl IntegratedTrainer {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Launch a Welford-variance update for ONE controller-input EMA
|
||||
/// slot (spec `2026-05-30-adaptive-controller-floor-design`).
|
||||
///
|
||||
/// Single-thread, single-block. The kernel reads `isv[input_slot]`,
|
||||
/// skips on sentinel zero (per `pearl_first_observation_bootstrap`),
|
||||
/// and updates the (count, mean, M²) Welford triple in place. Sample
|
||||
/// variance = `M² / (count - 1)` when `count > 1`; controllers consume
|
||||
/// it as `sqrt(M²/(count-1))` to scale their noise floors.
|
||||
///
|
||||
/// Caller's responsibility: launch ONCE per controller-input EMA per
|
||||
/// step, sequenced AFTER the EMA producer that writes `input_slot`
|
||||
/// and BEFORE the controller that reads the Welford triple.
|
||||
pub fn launch_rl_signal_variance_update(
|
||||
&self,
|
||||
input_slot: usize,
|
||||
count_slot: usize,
|
||||
mean_slot: usize,
|
||||
m2_slot: usize,
|
||||
) -> Result<()> {
|
||||
let mut args = RawArgs::new();
|
||||
args.push_ptr(self.isv_dev_ptr);
|
||||
args.push_i32(input_slot as i32);
|
||||
args.push_i32(count_slot as i32);
|
||||
args.push_i32(mean_slot as i32);
|
||||
args.push_i32(m2_slot as i32);
|
||||
let mut ptrs = args.build_arg_ptrs();
|
||||
unsafe {
|
||||
raw_launch(
|
||||
self.rl_signal_variance_update_fn.cu_function(),
|
||||
(1, 1, 1), (1, 1, 1), 0,
|
||||
self.raw_stream,
|
||||
&mut ptrs[..args.len()],
|
||||
).map_err(|e| anyhow::anyhow!("rl_signal_variance_update: {:?}", e))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Adaptive risk management — Layer 1 (spec 2026-05-30-adaptive-risk-management).
|
||||
/// Maintains session pnl + DD-triggered flag + cooldown counter +
|
||||
/// consecutive-loss counter. Single-thread single-block; reads
|
||||
/// `rewards_d` and `dones_d` (`[b_size]` f32 device buffers populated
|
||||
/// by `rl_fused_reward_pipeline`). Outcome is derived inline (done &
|
||||
/// reward sign). Must run AFTER the reward pipeline and BEFORE
|
||||
/// `actions_to_market_targets` reads the override flags.
|
||||
pub fn launch_rl_cmdp_constraints_check(
|
||||
&self,
|
||||
rewards_d: &CudaSlice<f32>,
|
||||
dones_d: &CudaSlice<f32>,
|
||||
b_size: usize,
|
||||
) -> Result<()> {
|
||||
let mut args = RawArgs::new();
|
||||
args.push_ptr(self.isv_dev_ptr);
|
||||
args.push_ptr(rewards_d.raw_ptr());
|
||||
args.push_ptr(dones_d.raw_ptr());
|
||||
args.push_ptr(self.session_pnl_per_batch_d.raw_ptr());
|
||||
args.push_ptr(self.consec_loss_per_batch_d.raw_ptr());
|
||||
args.push_ptr(self.session_dd_triggered_per_batch_d.raw_ptr());
|
||||
args.push_ptr(self.cooldown_remaining_per_batch_d.raw_ptr());
|
||||
args.push_i32(b_size as i32);
|
||||
let mut ptrs = args.build_arg_ptrs();
|
||||
unsafe {
|
||||
raw_launch(
|
||||
self.rl_cmdp_constraints_check_fn.cu_function(),
|
||||
(1, 1, 1), (1, 1, 1), 0,
|
||||
self.raw_stream,
|
||||
&mut ptrs[..args.len()],
|
||||
).map_err(|e| anyhow::anyhow!("rl_cmdp_constraints_check: {:?}", e))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Adaptive risk management — Layer 2 IQN τ controller (spec 2026-05-30).
|
||||
/// Standalone launcher. Production calls go through `rl_fused_controllers`;
|
||||
/// this helper is retained for unit-test isolation.
|
||||
pub fn launch_rl_iqn_action_tau_controller(&self) -> Result<()> {
|
||||
let mut args = RawArgs::new();
|
||||
args.push_ptr(self.isv_dev_ptr);
|
||||
let mut ptrs = args.build_arg_ptrs();
|
||||
unsafe {
|
||||
raw_launch(
|
||||
self.rl_iqn_action_tau_controller_fn.cu_function(),
|
||||
(1, 1, 1), (1, 1, 1), 0,
|
||||
self.raw_stream,
|
||||
&mut ptrs[..args.len()],
|
||||
).map_err(|e| anyhow::anyhow!("rl_iqn_action_tau_controller: {:?}", e))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Adaptive risk management — Layer 3 inventory β controller (spec 2026-05-30).
|
||||
/// Standalone launcher; production calls go through `rl_fused_controllers`.
|
||||
pub fn launch_rl_inventory_beta_controller(&self) -> Result<()> {
|
||||
let mut args = RawArgs::new();
|
||||
args.push_ptr(self.isv_dev_ptr);
|
||||
let mut ptrs = args.build_arg_ptrs();
|
||||
unsafe {
|
||||
raw_launch(
|
||||
self.rl_inventory_beta_controller_fn.cu_function(),
|
||||
(1, 1, 1), (1, 1, 1), 0,
|
||||
self.raw_stream,
|
||||
&mut ptrs[..args.len()],
|
||||
).map_err(|e| anyhow::anyhow!("rl_inventory_beta_controller: {:?}", e))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Adaptive risk management — Layer 4 Kelly fraction controller (spec 2026-05-30).
|
||||
/// Standalone launcher; production calls go through `rl_fused_controllers`.
|
||||
pub fn launch_rl_kelly_fraction_controller(&self) -> Result<()> {
|
||||
let mut args = RawArgs::new();
|
||||
args.push_ptr(self.isv_dev_ptr);
|
||||
let mut ptrs = args.build_arg_ptrs();
|
||||
unsafe {
|
||||
raw_launch(
|
||||
self.rl_kelly_fraction_controller_fn.cu_function(),
|
||||
(1, 1, 1), (1, 1, 1), 0,
|
||||
self.raw_stream,
|
||||
&mut ptrs[..args.len()],
|
||||
).map_err(|e| anyhow::anyhow!("rl_kelly_fraction_controller: {:?}", e))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// v9 — defensive eval-boundary calibration. Launch once per step
|
||||
/// AFTER all risk-stack controllers run (so this kernel's overrides
|
||||
/// of Kelly_safety_frac / IQN τ_min / entropy_coef_min / PPO ε_min
|
||||
/// take effect for the next step's consumers). Returns early if
|
||||
/// `RL_EVAL_WARMUP_REMAINING_INDEX` ≤ 0, so the runtime cost when
|
||||
/// warmup is inactive is just a single mapped-pinned ISV read.
|
||||
pub fn launch_rl_eval_warmup_decay(&self) -> Result<()> {
|
||||
let mut args = RawArgs::new();
|
||||
args.push_ptr(self.isv_dev_ptr);
|
||||
let mut ptrs = args.build_arg_ptrs();
|
||||
unsafe {
|
||||
raw_launch(
|
||||
self.rl_eval_warmup_decay_fn.cu_function(),
|
||||
(1, 1, 1), (1, 1, 1), 0,
|
||||
self.raw_stream,
|
||||
&mut ptrs[..args.len()],
|
||||
).map_err(|e| anyhow::anyhow!("rl_eval_warmup_decay: {:?}", e))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Adaptive risk management — Layer 4 input EMA producer (win_rate).
|
||||
/// Reads `rewards_d`, `dones_d` [b_size] f32; outcome derived inline.
|
||||
/// Writes ISV[RL_WIN_RATE_EMA_INDEX = 677].
|
||||
pub fn launch_rl_win_rate_ema_update(
|
||||
&self,
|
||||
rewards_d: &CudaSlice<f32>,
|
||||
dones_d: &CudaSlice<f32>,
|
||||
b_size: usize,
|
||||
) -> Result<()> {
|
||||
let mut args = RawArgs::new();
|
||||
args.push_ptr(self.isv_dev_ptr);
|
||||
args.push_ptr(rewards_d.raw_ptr());
|
||||
args.push_ptr(dones_d.raw_ptr());
|
||||
args.push_i32(b_size as i32);
|
||||
let mut ptrs = args.build_arg_ptrs();
|
||||
unsafe {
|
||||
raw_launch(
|
||||
self.rl_win_rate_ema_update_fn.cu_function(),
|
||||
(1, 1, 1), (1, 1, 1), 0,
|
||||
self.raw_stream,
|
||||
&mut ptrs[..args.len()],
|
||||
).map_err(|e| anyhow::anyhow!("rl_win_rate_ema_update: {:?}", e))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Adaptive risk management — Layer 4 input EMA producer (avg_win/loss USD).
|
||||
/// Reads `rewards_d`, `dones_d` [b_size] f32; outcome derived inline.
|
||||
/// Writes ISV[RL_AVG_WIN_USD_EMA_INDEX = 678, RL_AVG_LOSS_USD_EMA_INDEX = 679].
|
||||
pub fn launch_rl_avg_win_loss_ema_update(
|
||||
&self,
|
||||
rewards_d: &CudaSlice<f32>,
|
||||
dones_d: &CudaSlice<f32>,
|
||||
b_size: usize,
|
||||
) -> Result<()> {
|
||||
let mut args = RawArgs::new();
|
||||
args.push_ptr(self.isv_dev_ptr);
|
||||
args.push_ptr(rewards_d.raw_ptr());
|
||||
args.push_ptr(dones_d.raw_ptr());
|
||||
args.push_i32(b_size as i32);
|
||||
let mut ptrs = args.build_arg_ptrs();
|
||||
unsafe {
|
||||
raw_launch(
|
||||
self.rl_avg_win_loss_ema_update_fn.cu_function(),
|
||||
(1, 1, 1), (1, 1, 1), 0,
|
||||
self.raw_stream,
|
||||
&mut ptrs[..args.len()],
|
||||
).map_err(|e| anyhow::anyhow!("rl_avg_win_loss_ema_update: {:?}", e))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Adaptive risk management — Layer 3 input EMA producer (inventory variance).
|
||||
/// Reads `net_position_per_batch_d` [b_size] i32 of signed contracts
|
||||
/// (typically the trainer's `prev_position_lots_d`); writes
|
||||
/// ISV[RL_INVENTORY_VARIANCE_EMA_INDEX = 675].
|
||||
pub fn launch_rl_inventory_variance_update(
|
||||
&self,
|
||||
net_position_per_batch_d: &CudaSlice<i32>,
|
||||
b_size: usize,
|
||||
) -> Result<()> {
|
||||
let mut args = RawArgs::new();
|
||||
args.push_ptr(self.isv_dev_ptr);
|
||||
args.push_ptr(net_position_per_batch_d.raw_ptr());
|
||||
args.push_i32(b_size as i32);
|
||||
let mut ptrs = args.build_arg_ptrs();
|
||||
unsafe {
|
||||
raw_launch(
|
||||
self.rl_inventory_variance_update_fn.cu_function(),
|
||||
(1, 1, 1), (1, 1, 1), 0,
|
||||
self.raw_stream,
|
||||
&mut ptrs[..args.len()],
|
||||
).map_err(|e| anyhow::anyhow!("rl_inventory_variance_update: {:?}", e))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Phase R3: launch `ema_update_on_done` — done-gated EMA producer
|
||||
/// that writes the per-batch closed-trade mean into `isv[slot_index]`.
|
||||
/// `alpha` is the Wiener-α (caller must pre-floor at 0.4 per
|
||||
@@ -3488,6 +4091,9 @@ impl IntegratedTrainer {
|
||||
args.push_ptr(pyramid_units_count_d.raw_ptr());
|
||||
args.push_ptr(close_unit_index_d.raw_ptr());
|
||||
args.push_ptr(outcome_ema_d.raw_ptr());
|
||||
// CMDP per-batch state — one DD-trigger / cooldown per session.
|
||||
args.push_ptr(self.session_dd_triggered_per_batch_d.raw_ptr());
|
||||
args.push_ptr(self.cooldown_remaining_per_batch_d.raw_ptr());
|
||||
args.push_i32(b_size_i);
|
||||
args.push_i32(pos_bytes_i);
|
||||
let mut ptrs = args.build_arg_ptrs();
|
||||
@@ -6026,6 +6632,9 @@ impl IntegratedTrainer {
|
||||
args.push_ptr(self.pyramid_units_count_d.raw_ptr());
|
||||
args.push_ptr(self.close_unit_index_d.raw_ptr());
|
||||
args.push_ptr(self.outcome_ema_d.raw_ptr());
|
||||
// CMDP per-batch state — one DD-trigger / cooldown per session.
|
||||
args.push_ptr(self.session_dd_triggered_per_batch_d.raw_ptr());
|
||||
args.push_ptr(self.cooldown_remaining_per_batch_d.raw_ptr());
|
||||
args.push_i32(b_size_i);
|
||||
args.push_i32(pos_bytes_i);
|
||||
let mut ptrs = args.build_arg_ptrs();
|
||||
@@ -6136,6 +6745,46 @@ impl IntegratedTrainer {
|
||||
}
|
||||
}
|
||||
|
||||
// ────────────────────────────────────────────────────────────
|
||||
// Adaptive risk management — spec 2026-05-30-adaptive-risk-management.
|
||||
//
|
||||
// Layer 1 (CMDP gates): track session pnl + DD-triggered flag +
|
||||
// cooldown counter + consecutive-loss counter from this step's
|
||||
// rewards/dones. Must precede the Layer 2 IQN-τ controller because
|
||||
// τ derives from `session_pnl` written here.
|
||||
//
|
||||
// EMA producers: feed Layer 4 Kelly (win_rate, avg_win/loss) and
|
||||
// Layer 3 (inventory variance). All three are single-thread
|
||||
// single-block kernels.
|
||||
//
|
||||
// The Layer 2/3/4 controllers themselves are folded into
|
||||
// `rl_fused_controllers` below, which fires after the existing
|
||||
// Welford updates — so this is the producer side only.
|
||||
// ────────────────────────────────────────────────────────────
|
||||
self.launch_rl_cmdp_constraints_check(
|
||||
&self.rewards_d,
|
||||
&self.dones_d,
|
||||
b_size,
|
||||
)
|
||||
.context("launch_rl_cmdp_constraints_check")?;
|
||||
self.launch_rl_win_rate_ema_update(
|
||||
&self.rewards_d,
|
||||
&self.dones_d,
|
||||
b_size,
|
||||
)
|
||||
.context("launch_rl_win_rate_ema_update")?;
|
||||
self.launch_rl_avg_win_loss_ema_update(
|
||||
&self.rewards_d,
|
||||
&self.dones_d,
|
||||
b_size,
|
||||
)
|
||||
.context("launch_rl_avg_win_loss_ema_update")?;
|
||||
self.launch_rl_inventory_variance_update(
|
||||
&self.prev_position_lots_d,
|
||||
b_size,
|
||||
)
|
||||
.context("launch_rl_inventory_variance_update")?;
|
||||
|
||||
// Scale + clamp rewards, write per-step max to ISV for the
|
||||
// adaptive clamp controller. Without this, raw PnL ($100s)
|
||||
// goes into C51 with atom span ±0.5 → binary Q resolution.
|
||||
@@ -6355,6 +7004,70 @@ impl IntegratedTrainer {
|
||||
|
||||
// recent_outcome_update — handled by rl_fused_reward_pipeline above.
|
||||
|
||||
// Adaptive controller floors (spec
|
||||
// `2026-05-30-adaptive-controller-floor-design`): launch a
|
||||
// Welford-variance update for each controller-input EMA so the
|
||||
// 12 controllers + fused kernel can read `sqrt(M²/(n-1))` from
|
||||
// their dedicated triple instead of falling back to
|
||||
// `target × 0.5`. Sequenced AFTER all EMA producers above and
|
||||
// BEFORE the fused controller launch / standalone v_blend_alpha
|
||||
// controller below. Sentinel-zero skip is handled inside the
|
||||
// kernel — no per-step gating needed here.
|
||||
//
|
||||
// Slots written *this* step (kl_pi, entropy_observed, td_kurtosis,
|
||||
// mean_trade_duration) feed Welford with the current step's
|
||||
// observation. Slots written *later* in the step
|
||||
// (q_divergence at step end, q_distill_kl by distill grad,
|
||||
// reward_magnitude by reward_scale controller, v_scalar_mag by
|
||||
// v_blend_alpha controller, adv_var_pre_norm by advantage
|
||||
// normalize) feed Welford with the previous step's value — a
|
||||
// one-step lag, acceptable per the spec since controllers adapt
|
||||
// over many steps and the Welford counter still reflects real
|
||||
// observations.
|
||||
for (input_slot, count_slot, mean_slot, m2_slot) in [
|
||||
(crate::rl::isv_slots::RL_KL_PI_EMA_INDEX,
|
||||
crate::rl::isv_slots::RL_KL_PI_VAR_COUNT_INDEX,
|
||||
crate::rl::isv_slots::RL_KL_PI_VAR_MEAN_INDEX,
|
||||
crate::rl::isv_slots::RL_KL_PI_VAR_M2_INDEX),
|
||||
(crate::rl::isv_slots::RL_Q_DIVERGENCE_EMA_INDEX,
|
||||
crate::rl::isv_slots::RL_Q_DIV_VAR_COUNT_INDEX,
|
||||
crate::rl::isv_slots::RL_Q_DIV_VAR_MEAN_INDEX,
|
||||
crate::rl::isv_slots::RL_Q_DIV_VAR_M2_INDEX),
|
||||
(crate::rl::isv_slots::RL_ADV_VAR_PRE_NORM_INDEX,
|
||||
crate::rl::isv_slots::RL_ADV_VAR_VAR_COUNT_INDEX,
|
||||
crate::rl::isv_slots::RL_ADV_VAR_VAR_MEAN_INDEX,
|
||||
crate::rl::isv_slots::RL_ADV_VAR_VAR_M2_INDEX),
|
||||
(crate::rl::isv_slots::RL_ENTROPY_OBSERVED_EMA_INDEX,
|
||||
crate::rl::isv_slots::RL_ENTROPY_OBS_VAR_COUNT_INDEX,
|
||||
crate::rl::isv_slots::RL_ENTROPY_OBS_VAR_MEAN_INDEX,
|
||||
crate::rl::isv_slots::RL_ENTROPY_OBS_VAR_M2_INDEX),
|
||||
(crate::rl::isv_slots::RL_TD_KURTOSIS_EMA_INDEX,
|
||||
crate::rl::isv_slots::RL_TD_KURT_VAR_COUNT_INDEX,
|
||||
crate::rl::isv_slots::RL_TD_KURT_VAR_MEAN_INDEX,
|
||||
crate::rl::isv_slots::RL_TD_KURT_VAR_M2_INDEX),
|
||||
(crate::rl::isv_slots::RL_MEAN_TRADE_DURATION_EMA_INDEX,
|
||||
crate::rl::isv_slots::RL_TRADE_DUR_VAR_COUNT_INDEX,
|
||||
crate::rl::isv_slots::RL_TRADE_DUR_VAR_MEAN_INDEX,
|
||||
crate::rl::isv_slots::RL_TRADE_DUR_VAR_M2_INDEX),
|
||||
(crate::rl::isv_slots::RL_REWARD_MAGNITUDE_EMA_INDEX,
|
||||
crate::rl::isv_slots::RL_REWARD_MAG_VAR_COUNT_INDEX,
|
||||
crate::rl::isv_slots::RL_REWARD_MAG_VAR_MEAN_INDEX,
|
||||
crate::rl::isv_slots::RL_REWARD_MAG_VAR_M2_INDEX),
|
||||
(crate::rl::isv_slots::RL_Q_DISTILL_KL_EMA_INDEX,
|
||||
crate::rl::isv_slots::RL_Q_DISTILL_KL_VAR_COUNT_INDEX,
|
||||
crate::rl::isv_slots::RL_Q_DISTILL_KL_VAR_MEAN_INDEX,
|
||||
crate::rl::isv_slots::RL_Q_DISTILL_KL_VAR_M2_INDEX),
|
||||
(crate::rl::isv_slots::RL_V_SCALAR_MAG_EMA_INDEX,
|
||||
crate::rl::isv_slots::RL_V_BLEND_SCALAR_VAR_COUNT_INDEX,
|
||||
crate::rl::isv_slots::RL_V_BLEND_SCALAR_VAR_MEAN_INDEX,
|
||||
crate::rl::isv_slots::RL_V_BLEND_SCALAR_VAR_M2_INDEX),
|
||||
] {
|
||||
self.launch_rl_signal_variance_update(
|
||||
input_slot, count_slot, mean_slot, m2_slot,
|
||||
)
|
||||
.context("launch_rl_signal_variance_update")?;
|
||||
}
|
||||
|
||||
// Fire all 10 RL ISV controllers in a single fused kernel
|
||||
// launch (gamma, tau, ppo_clip, entropy_coef, rollout_steps,
|
||||
// per_alpha, reward_scale, ppo_ratio_clamp, gate_threshold,
|
||||
@@ -6364,6 +7077,15 @@ impl IntegratedTrainer {
|
||||
self.launch_rl_fused_controllers(b_size)
|
||||
.context("launch_rl_fused_controllers")?;
|
||||
|
||||
// v9 defensive eval-boundary calibration ([[pearl_adaptive_carryover_discipline]]).
|
||||
// MUST run AFTER fused controllers: last-writer-wins on the four risk-sizing
|
||||
// floors (Kelly safety_frac, IQN τ_min, entropy_coef_min, PPO clip ε_min).
|
||||
// Steady-state (counter < 0): kernel returns early — zero behavioral effect.
|
||||
// After `reset_session_state` arms the counter, kernel overrides floors with
|
||||
// defensive defaults for warmup_steps, with a linear decay over decay_steps.
|
||||
self.launch_rl_eval_warmup_decay()
|
||||
.context("launch_rl_eval_warmup_decay")?;
|
||||
|
||||
// PopArt: normalize rewards in-place (replaces apply_reward_scale).
|
||||
// Welford-EMA on batch mean+variance → whitened rewards → ISV
|
||||
// stats for V-correct. Grid=(1), Block=(min(B, 256)). Shared mem
|
||||
@@ -6591,6 +7313,10 @@ impl IntegratedTrainer {
|
||||
let block_x = (b_size as u32).min(1024).max(1);
|
||||
let mut args = RawArgs::new();
|
||||
args.push_ptr(self.advantages_d.raw_ptr());
|
||||
// Adaptive controller floors (spec 2026-05-30): emit raw pre-norm
|
||||
// variance to ISV[RL_ADV_VAR_PRE_NORM_INDEX=612] for
|
||||
// rl_rollout_steps_controller's input signal.
|
||||
args.push_ptr(self.isv_dev_ptr);
|
||||
args.push_i32(b_size_i);
|
||||
let mut ptrs = args.build_arg_ptrs();
|
||||
unsafe {
|
||||
@@ -7952,6 +8678,9 @@ impl IntegratedTrainer {
|
||||
args.push_ptr(self.pyramid_units_count_d.raw_ptr());
|
||||
args.push_ptr(self.close_unit_index_d.raw_ptr());
|
||||
args.push_ptr(self.outcome_ema_d.raw_ptr());
|
||||
// CMDP per-batch state — one DD-trigger / cooldown per session.
|
||||
args.push_ptr(self.session_dd_triggered_per_batch_d.raw_ptr());
|
||||
args.push_ptr(self.cooldown_remaining_per_batch_d.raw_ptr());
|
||||
args.push_i32(b_size_i);
|
||||
args.push_i32(pos_bytes_i);
|
||||
let mut ptrs = args.build_arg_ptrs();
|
||||
|
||||
524
crates/ml-alpha/tests/controller_adaptive_floors.rs
Normal file
524
crates/ml-alpha/tests/controller_adaptive_floors.rs
Normal file
@@ -0,0 +1,524 @@
|
||||
//! Task 16 — Controller-level adaptive-floor invariant tests for the
|
||||
//! 2026-05-30 adaptive-controller-floor refactor (spec
|
||||
//! `docs/superpowers/specs/2026-05-30-adaptive-controller-floor-design.md`).
|
||||
//!
|
||||
//! Validates Validation Gates G3-G6 against the fused production launch
|
||||
//! path (`launch_rl_fused_controllers`). Per
|
||||
//! `feedback_no_cpu_test_fallbacks`: every oracle here is an analytical
|
||||
//! identity of the controller's documented adaptive math — adaptive
|
||||
//! noise floor (`max(target × 0.5, 2σ)`), asymmetric Schulman
|
||||
//! (tighten on single above-band observation, widen requires N ≥ 3
|
||||
//! consecutive below-band) and gamma's `1 − 1/(d × 2)` adaptive-min
|
||||
//! formula. No CPU reference reimplementation; the kernel's spec is the
|
||||
//! oracle.
|
||||
//!
|
||||
//! Per `pearl_first_observation_bootstrap`: the bootstrap-replace path
|
||||
//! (`eps_prev == RL_EPS_BOOTSTRAP_INDEX`) skips the Wiener blend so the
|
||||
//! first real observation lands directly at the computed `eps_target`.
|
||||
//! For G4/G5 we deliberately PRE-WRITE `RL_PPO_CLIP_INDEX` to a value
|
||||
//! away from the bootstrap so the Wiener-blend branch is exercised; this
|
||||
//! isolates the asymmetric-Schulman invariant from the bootstrap-replace
|
||||
//! quirk. G3 stays on the natural post-bootstrap state (eps_prev =
|
||||
//! bootstrap = 0.05 = EPS_MIN) because the noise-floor gate returns
|
||||
//! before either branch fires — the invariant under test is "controller
|
||||
//! holds".
|
||||
//!
|
||||
//! Run with:
|
||||
//! `cargo test -p ml-alpha --test controller_adaptive_floors -- --ignored --nocapture`
|
||||
|
||||
use ml_alpha::rl::isv_slots::{
|
||||
RL_GAMMA_INDEX, RL_GAMMA_MIN_ADAPTIVE_INDEX, RL_KL_PI_BELOW_COUNT_INDEX,
|
||||
RL_KL_PI_EMA_INDEX, RL_KL_PI_VAR_COUNT_INDEX, RL_KL_PI_VAR_M2_INDEX,
|
||||
RL_KL_PI_VAR_MEAN_INDEX, RL_KL_TARGET_INDEX, RL_MEAN_TRADE_DURATION_EMA_INDEX,
|
||||
RL_PPO_CLIP_EPS_MAX_INDEX, RL_PPO_CLIP_INDEX, RL_SCHULMAN_ADJUST_RATE_INDEX,
|
||||
RL_SCHULMAN_TOLERANCE_INDEX, RL_TRADE_DUR_VAR_COUNT_INDEX,
|
||||
RL_TRADE_DUR_VAR_M2_INDEX, RL_TRADE_DUR_VAR_MEAN_INDEX,
|
||||
};
|
||||
use ml_alpha::trainer::integrated::{IntegratedTrainer, IntegratedTrainerConfig};
|
||||
use ml_alpha::trainer::perception::PerceptionTrainerConfig;
|
||||
use ml_core::device::MlDevice;
|
||||
|
||||
/// Bootstrap value `rl_ppo_clip_controller` writes into
|
||||
/// `ISV[RL_PPO_CLIP_INDEX]` at construction. Per Task 17 fix:
|
||||
/// bootstrap matches `EPS_MIN` (= 0.05) so the post-bootstrap step
|
||||
/// doesn't snap to MIN on first signal regardless of direction
|
||||
/// (bootstrap-clamp inconsistency that surfaced in the Task 16 testing).
|
||||
const EPS_BOOTSTRAP: f32 = 0.05;
|
||||
|
||||
/// PPO ε MIN clamp seeded by `with_controllers_bootstrapped`.
|
||||
const EPS_MIN: f32 = 0.05;
|
||||
|
||||
/// KL target seeded by `with_controllers_bootstrapped`.
|
||||
const KL_TARGET: f32 = 0.01;
|
||||
|
||||
/// Schulman tolerance — kl above `target × tolerance` triggers tighten,
|
||||
/// below `target / tolerance` triggers (delayed) widen.
|
||||
const SCHULMAN_TOLERANCE: f32 = 1.5;
|
||||
|
||||
/// Schulman adjust rate — tighten ratio is `1 / adjust_rate`, widen
|
||||
/// ratio is `adjust_rate`.
|
||||
const SCHULMAN_ADJUST_RATE: f32 = 1.5;
|
||||
|
||||
/// N-consecutive-below-band threshold for the widen branch (matches
|
||||
/// `WIDEN_PATIENCE_CONSECUTIVE` in `rl_ppo_clip_controller.cu`).
|
||||
const WIDEN_PATIENCE: f32 = 3.0;
|
||||
|
||||
/// Hard absolute floor for γ adaptive-min (matches `GAMMA_MIN_ABSOLUTE`
|
||||
/// in `rl_gamma_controller.cu`).
|
||||
const GAMMA_MIN_ABSOLUTE: f32 = 0.9;
|
||||
|
||||
/// Welford warmup observations before the running mean replaces the EMA
|
||||
/// in the γ adaptive-min derivation (matches `WELFORD_WARMUP_OBS`).
|
||||
const WELFORD_WARMUP_OBS: f32 = 100.0;
|
||||
|
||||
fn build_trainer() -> Option<(MlDevice, IntegratedTrainer)> {
|
||||
let dev = match MlDevice::cuda(0) {
|
||||
Ok(d) => d,
|
||||
Err(e) => {
|
||||
eprintln!("CUDA 0 not available — skipping ({e})");
|
||||
return None;
|
||||
}
|
||||
};
|
||||
let cfg = IntegratedTrainerConfig {
|
||||
perception: PerceptionTrainerConfig {
|
||||
seq_len: 4,
|
||||
n_batch: 1,
|
||||
..PerceptionTrainerConfig::default()
|
||||
},
|
||||
dqn_seed: 0xC1,
|
||||
ppo_seed: 0xC2,
|
||||
..IntegratedTrainerConfig::default()
|
||||
};
|
||||
let trainer = IntegratedTrainer::new(&dev, cfg).expect("IntegratedTrainer::new");
|
||||
Some((dev, trainer))
|
||||
}
|
||||
|
||||
/// Sync the trainer's production stream so subsequent mapped-pinned ISV
|
||||
/// reads through `read_isv_host` observe device-side writes.
|
||||
fn sync(trainer: &IntegratedTrainer) {
|
||||
trainer.stream.synchronize().expect("sync trainer stream");
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────
|
||||
// G3 — PPO clip holds when signal is below the adaptive noise floor.
|
||||
//
|
||||
// Invariant: after the bootstrap path has fired (eps_prev = bootstrap =
|
||||
// 0.01), feeding `kl_pi_ema = 0.001` (< noise_floor = target × 0.5 =
|
||||
// 0.005) for 50 fused launches must NOT drift ε. The noise-floor gate
|
||||
// is the load-bearing fix — without it, the prior multiplicative-ratio
|
||||
// design produced 1e4 ratios from sub-noise KL values and pegged ε at
|
||||
// MAX 0.50 within ~50 steps (alpha-rl-mjzfk fold 0 saturation).
|
||||
//
|
||||
// Welford triple is pre-populated so the std-based half of the floor
|
||||
// (`2σ`) is dominated by the target-fraction half (`target × 0.5`),
|
||||
// keeping the floor at the steady-state value of 0.005.
|
||||
// ─────────────────────────────────────────────────────────────────────
|
||||
#[test]
|
||||
#[ignore = "requires CUDA (MlDevice::cuda(0))"]
|
||||
fn g3_ppo_clip_holds_when_signal_below_adaptive_floor() {
|
||||
let Some((_dev, trainer)) = build_trainer() else { return };
|
||||
|
||||
// Verify pre-conditions seeded by `with_controllers_bootstrapped`.
|
||||
sync(&trainer);
|
||||
let eps_at_bootstrap = trainer.read_isv_host(RL_PPO_CLIP_INDEX);
|
||||
assert!(
|
||||
(eps_at_bootstrap - EPS_BOOTSTRAP).abs() < 1e-6,
|
||||
"pre-condition: ε must be bootstrapped to {EPS_BOOTSTRAP}; got {eps_at_bootstrap}"
|
||||
);
|
||||
let kl_target_seeded = trainer.read_isv_host(RL_KL_TARGET_INDEX);
|
||||
assert!(
|
||||
(kl_target_seeded - KL_TARGET).abs() < 1e-6,
|
||||
"pre-condition: KL target must be seeded to {KL_TARGET}; got {kl_target_seeded}"
|
||||
);
|
||||
|
||||
// Pre-populate Welford triple at the kl_pi_ema slot with low M²
|
||||
// so std ≈ 0 and the floor reduces to `target × 0.5 = 0.005`. count
|
||||
// > 1 is required so the kernel evaluates `M²/(count − 1)` rather
|
||||
// than the count ≤ 1 fallback (which would also produce std = 0).
|
||||
trainer.isv_mapped.write_record(RL_KL_PI_VAR_COUNT_INDEX, 100.0);
|
||||
trainer.isv_mapped.write_record(RL_KL_PI_VAR_MEAN_INDEX, 0.001);
|
||||
trainer.isv_mapped.write_record(RL_KL_PI_VAR_M2_INDEX, 0.0);
|
||||
|
||||
// Feed below-floor signal for 50 fused launches. Re-write each step
|
||||
// because a real-world EMA producer would refresh it; here we're
|
||||
// testing the controller in isolation.
|
||||
for _ in 0..50 {
|
||||
trainer.isv_mapped.write_record(RL_KL_PI_EMA_INDEX, 0.001);
|
||||
trainer
|
||||
.launch_rl_fused_controllers(1)
|
||||
.expect("launch_rl_fused_controllers");
|
||||
}
|
||||
sync(&trainer);
|
||||
|
||||
let eps_after = trainer.read_isv_host(RL_PPO_CLIP_INDEX);
|
||||
|
||||
// Primary invariant: ε held — no drift past bootstrap, MUST not have
|
||||
// climbed toward MAX 0.5 (the canonical saturation pattern).
|
||||
assert!(
|
||||
(eps_after - EPS_BOOTSTRAP).abs() < 1e-4,
|
||||
"ε must hold at bootstrap when signal is below adaptive floor; \
|
||||
expected ≈ {EPS_BOOTSTRAP}, got {eps_after}"
|
||||
);
|
||||
assert!(
|
||||
eps_after < 0.4,
|
||||
"ε must NOT drift toward MAX 0.5 — this was the fold 0/1 \
|
||||
saturation bug; got {eps_after}"
|
||||
);
|
||||
|
||||
// Negative invariant: the below-band counter must have stayed at 0
|
||||
// because the noise-floor gate returns BEFORE the asymmetric Schulman
|
||||
// branch fires.
|
||||
let below_count = trainer.read_isv_host(RL_KL_PI_BELOW_COUNT_INDEX);
|
||||
assert_eq!(
|
||||
below_count, 0.0,
|
||||
"below-band counter must stay 0 when noise-floor gate held; got {below_count}"
|
||||
);
|
||||
|
||||
eprintln!(
|
||||
"G3 OK — ε held at {eps_after} (bootstrap {EPS_BOOTSTRAP}) over 50 below-floor steps; \
|
||||
below_count = {below_count} (noise-floor gate fired)"
|
||||
);
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────
|
||||
// G4 — PPO clip tightens on a single above-band observation
|
||||
// (asymmetric Schulman — fast tighten).
|
||||
//
|
||||
// Invariant: a single `kl_pi_ema > target × tolerance` observation must
|
||||
// decrease ε within the same call. Tighten ratio = `1 / adjust_rate ≈
|
||||
// 0.667`; the assertion is strict decrease (ε_after < ε_before).
|
||||
//
|
||||
// Test discipline: we pre-write ε to 0.2 (well above MIN 0.05 and away
|
||||
// from bootstrap 0.01) so:
|
||||
// 1. The bootstrap-replace branch DOESN'T fire (eps_prev ≠ bootstrap).
|
||||
// 2. `eps_target = 0.2 × 0.667 ≈ 0.133` stays above MIN, so the clamp
|
||||
// doesn't bind and obscure the tighten signal.
|
||||
// The Wiener-α blend (alpha = 0.4 = Wiener floor) then produces
|
||||
// `eps_new = 0.6 × 0.2 + 0.4 × 0.133 ≈ 0.173`. Below 0.2 → strict
|
||||
// decrease.
|
||||
// ─────────────────────────────────────────────────────────────────────
|
||||
#[test]
|
||||
#[ignore = "requires CUDA (MlDevice::cuda(0))"]
|
||||
fn g4_ppo_clip_tightens_on_single_above_band_observation() {
|
||||
let Some((_dev, trainer)) = build_trainer() else { return };
|
||||
|
||||
// Pre-write ε to 0.2 — away from bootstrap, above MIN clamp.
|
||||
let eps_initial: f32 = 0.2;
|
||||
trainer.isv_mapped.write_record(RL_PPO_CLIP_INDEX, eps_initial);
|
||||
|
||||
// Welford triple — std ≈ 0 so the noise floor is `target × 0.5 = 0.005`.
|
||||
trainer.isv_mapped.write_record(RL_KL_PI_VAR_COUNT_INDEX, 100.0);
|
||||
trainer.isv_mapped.write_record(RL_KL_PI_VAR_MEAN_INDEX, 0.01);
|
||||
trainer.isv_mapped.write_record(RL_KL_PI_VAR_M2_INDEX, 0.0);
|
||||
|
||||
// Above-band signal: kl = 0.05 = 5× target = 3.33× tolerance bound.
|
||||
trainer.isv_mapped.write_record(RL_KL_PI_EMA_INDEX, 0.05);
|
||||
|
||||
// Verify Schulman config matches what the controller will read.
|
||||
let tolerance = trainer.read_isv_host(RL_SCHULMAN_TOLERANCE_INDEX);
|
||||
let adjust_rate = trainer.read_isv_host(RL_SCHULMAN_ADJUST_RATE_INDEX);
|
||||
assert!(
|
||||
(tolerance - SCHULMAN_TOLERANCE).abs() < 1e-6,
|
||||
"pre-condition: Schulman tolerance seeded to {SCHULMAN_TOLERANCE}; got {tolerance}"
|
||||
);
|
||||
assert!(
|
||||
(adjust_rate - SCHULMAN_ADJUST_RATE).abs() < 1e-6,
|
||||
"pre-condition: Schulman adjust_rate seeded to {SCHULMAN_ADJUST_RATE}; got {adjust_rate}"
|
||||
);
|
||||
|
||||
trainer
|
||||
.launch_rl_fused_controllers(1)
|
||||
.expect("launch_rl_fused_controllers");
|
||||
sync(&trainer);
|
||||
|
||||
let eps_after = trainer.read_isv_host(RL_PPO_CLIP_INDEX);
|
||||
|
||||
// Primary invariant: ε strictly decreased on a single above-band fire.
|
||||
assert!(
|
||||
eps_after < eps_initial,
|
||||
"ε must tighten on single above-band observation; \
|
||||
before = {eps_initial}, after = {eps_after}"
|
||||
);
|
||||
// Above the MIN clamp — the decrease is genuine adaptation, not a
|
||||
// saturation pin.
|
||||
assert!(
|
||||
eps_after > EPS_MIN,
|
||||
"ε must stay above MIN {EPS_MIN} after a single fire (clamp \
|
||||
not binding); got {eps_after}"
|
||||
);
|
||||
|
||||
// The below-band counter must reset on a tighten event (the kernel's
|
||||
// `else if (kl_ema > target × tolerance) { … below_count = 0 }`).
|
||||
let below_count = trainer.read_isv_host(RL_KL_PI_BELOW_COUNT_INDEX);
|
||||
assert_eq!(
|
||||
below_count, 0.0,
|
||||
"below-band counter must reset on tighten; got {below_count}"
|
||||
);
|
||||
|
||||
eprintln!(
|
||||
"G4 OK — ε tightened {eps_initial} → {eps_after} (single above-band fire); \
|
||||
below_count reset to 0"
|
||||
);
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────
|
||||
// G5 — PPO clip widens only after N consecutive below-band observations
|
||||
// (asymmetric Schulman — slow widen).
|
||||
//
|
||||
// Invariant: a kl_pi_ema in the widen-band `[noise_floor, target /
|
||||
// tolerance) = [0.005, 0.00667)` must increment `below_count` each step
|
||||
// but NOT widen ε until `below_count ≥ WIDEN_PATIENCE_CONSECUTIVE = 3`.
|
||||
//
|
||||
// We choose kl = 0.006 because:
|
||||
// - 0.006 ≥ 0.005 (noise floor — passes the gate)
|
||||
// - 0.006 < 0.00667 (target / tolerance — triggers widen branch, not
|
||||
// hold)
|
||||
// To keep the noise floor low (so kl=0.006 passes it), Welford std must
|
||||
// be small: count=100, M²=0 → std=0 → floor = target × 0.5 = 0.005.
|
||||
//
|
||||
// Test discipline: ε pre-written to 0.2 so the bootstrap-replace branch
|
||||
// doesn't fire. We expect:
|
||||
// step 1: below_count 0 → 1, ratio = 1 (hold), ε unchanged
|
||||
// step 2: below_count 1 → 2, ratio = 1 (hold), ε unchanged
|
||||
// step 3: below_count 2 → 3, ratio = adjust_rate = 1.5 (widen), ε
|
||||
// increases via Wiener blend
|
||||
// step 4: below_count 3 → 4, ratio = 1.5 (widen continues)
|
||||
// ─────────────────────────────────────────────────────────────────────
|
||||
#[test]
|
||||
#[ignore = "requires CUDA (MlDevice::cuda(0))"]
|
||||
fn g5_ppo_clip_widens_only_after_n_consecutive_below_band() {
|
||||
let Some((_dev, trainer)) = build_trainer() else { return };
|
||||
|
||||
let eps_initial: f32 = 0.2;
|
||||
trainer.isv_mapped.write_record(RL_PPO_CLIP_INDEX, eps_initial);
|
||||
|
||||
// Welford triple — low std so noise floor = target × 0.5 = 0.005.
|
||||
trainer.isv_mapped.write_record(RL_KL_PI_VAR_COUNT_INDEX, 100.0);
|
||||
trainer.isv_mapped.write_record(RL_KL_PI_VAR_MEAN_INDEX, 0.006);
|
||||
trainer.isv_mapped.write_record(RL_KL_PI_VAR_M2_INDEX, 0.0);
|
||||
|
||||
// Counter must start at 0.
|
||||
trainer
|
||||
.isv_mapped
|
||||
.write_record(RL_KL_PI_BELOW_COUNT_INDEX, 0.0);
|
||||
|
||||
// Widen-band signal: 0.005 ≤ 0.006 < 0.00667.
|
||||
let kl_widen: f32 = 0.006;
|
||||
|
||||
// Compute the widen-band edges from the canonical formulas so the
|
||||
// assertion that 0.006 falls inside is auditable.
|
||||
let noise_floor = KL_TARGET * 0.5;
|
||||
let widen_upper = KL_TARGET / SCHULMAN_TOLERANCE;
|
||||
assert!(
|
||||
kl_widen >= noise_floor,
|
||||
"kl_widen {kl_widen} must be ≥ noise floor {noise_floor}"
|
||||
);
|
||||
assert!(
|
||||
kl_widen < widen_upper,
|
||||
"kl_widen {kl_widen} must be < widen upper edge {widen_upper}"
|
||||
);
|
||||
|
||||
let mut below_counts = Vec::with_capacity(4);
|
||||
let mut eps_values = Vec::with_capacity(4);
|
||||
|
||||
for _ in 0..4 {
|
||||
trainer.isv_mapped.write_record(RL_KL_PI_EMA_INDEX, kl_widen);
|
||||
trainer
|
||||
.launch_rl_fused_controllers(1)
|
||||
.expect("launch_rl_fused_controllers");
|
||||
sync(&trainer);
|
||||
below_counts.push(trainer.read_isv_host(RL_KL_PI_BELOW_COUNT_INDEX));
|
||||
eps_values.push(trainer.read_isv_host(RL_PPO_CLIP_INDEX));
|
||||
}
|
||||
|
||||
for (i, (bc, eps)) in below_counts.iter().zip(eps_values.iter()).enumerate() {
|
||||
eprintln!(" step {}: below_count = {bc}, ε = {eps}", i + 1);
|
||||
}
|
||||
|
||||
// Below-counter invariant: increments monotonically per step.
|
||||
assert_eq!(below_counts[0], 1.0, "step 1: below_count must be 1");
|
||||
assert_eq!(below_counts[1], 2.0, "step 2: below_count must be 2");
|
||||
assert_eq!(
|
||||
below_counts[2], WIDEN_PATIENCE,
|
||||
"step 3: below_count must reach WIDEN_PATIENCE {WIDEN_PATIENCE}"
|
||||
);
|
||||
assert_eq!(below_counts[3], 4.0, "step 4: below_count must be 4");
|
||||
|
||||
// ε hold invariant: steps 1 and 2 (below_count < 3) MUST NOT widen.
|
||||
// Allow only float-rounding-level drift (the controller's
|
||||
// `ratio = 1.0f` branch produces eps_target = eps_prev, then the
|
||||
// Wiener blend with eps_target == eps_prev is the identity).
|
||||
let drift_1 = (eps_values[0] - eps_initial).abs();
|
||||
let drift_2 = (eps_values[1] - eps_initial).abs();
|
||||
assert!(
|
||||
drift_1 < 1e-5,
|
||||
"step 1: ε must hold (count < WIDEN_PATIENCE); drift {drift_1}, value {}",
|
||||
eps_values[0]
|
||||
);
|
||||
assert!(
|
||||
drift_2 < 1e-5,
|
||||
"step 2: ε must hold (count < WIDEN_PATIENCE); drift {drift_2}, value {}",
|
||||
eps_values[1]
|
||||
);
|
||||
|
||||
// ε widen invariant: step 3 (below_count reaches WIDEN_PATIENCE) MUST
|
||||
// strictly increase ε. Step 4 continues widening.
|
||||
assert!(
|
||||
eps_values[2] > eps_initial + 1e-5,
|
||||
"step 3: ε must widen once below_count reaches {WIDEN_PATIENCE}; \
|
||||
was {eps_initial}, got {}",
|
||||
eps_values[2]
|
||||
);
|
||||
assert!(
|
||||
eps_values[3] > eps_values[2],
|
||||
"step 4: ε must continue widening; step 3 = {}, step 4 = {}",
|
||||
eps_values[2],
|
||||
eps_values[3]
|
||||
);
|
||||
|
||||
// ε stays under MAX clamp — widening is real adaptation, not a
|
||||
// saturation pin.
|
||||
let eps_max = trainer.read_isv_host(RL_PPO_CLIP_EPS_MAX_INDEX);
|
||||
assert!(
|
||||
eps_values[3] < eps_max,
|
||||
"ε must stay under MAX {eps_max} (no saturation pin); got {}",
|
||||
eps_values[3]
|
||||
);
|
||||
|
||||
eprintln!(
|
||||
"G5 OK — ε held for 2 steps, widened on step 3 (count reached {WIDEN_PATIENCE}): \
|
||||
{eps_initial} → {} → {} → {} → {}",
|
||||
eps_values[0], eps_values[1], eps_values[2], eps_values[3]
|
||||
);
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────
|
||||
// G6 — γ adaptive_min tracks trade duration via Welford mean (post-100-
|
||||
// observation warmup) and the EMA before warmup.
|
||||
//
|
||||
// Invariant (post-warmup): with Welford count ≥ 100 and Welford mean d,
|
||||
// the kernel must write
|
||||
//
|
||||
// RL_GAMMA_MIN_ADAPTIVE = max(0.9, 1 − 1/max(d × 2, 10))
|
||||
//
|
||||
// to ISV[613]. At d = 30: `1 − 1/60 ≈ 0.98333`.
|
||||
//
|
||||
// Invariant (pre-warmup): with count < 100, the formula uses the EMA at
|
||||
// `RL_MEAN_TRADE_DURATION_EMA_INDEX` instead of the Welford mean. At
|
||||
// EMA = 15: `1 − 1/30 ≈ 0.96667`.
|
||||
//
|
||||
// Per spec Section "Special case G — γ_min coupling": the Welford
|
||||
// mean's N-smoothing breaks the γ ↔ trade_duration feedback loop without
|
||||
// any explicit cool-down. Welford count provides the confidence
|
||||
// threshold for the switchover.
|
||||
// ─────────────────────────────────────────────────────────────────────
|
||||
#[test]
|
||||
#[ignore = "requires CUDA (MlDevice::cuda(0))"]
|
||||
fn g6_gamma_adaptive_min_post_warmup_uses_welford_mean() {
|
||||
let Some((_dev, trainer)) = build_trainer() else { return };
|
||||
|
||||
// Post-warmup state: count = 100 = WELFORD_WARMUP_OBS, mean = 30.
|
||||
let d_welford: f32 = 30.0;
|
||||
trainer
|
||||
.isv_mapped
|
||||
.write_record(RL_TRADE_DUR_VAR_COUNT_INDEX, WELFORD_WARMUP_OBS);
|
||||
trainer
|
||||
.isv_mapped
|
||||
.write_record(RL_TRADE_DUR_VAR_MEAN_INDEX, d_welford);
|
||||
trainer.isv_mapped.write_record(RL_TRADE_DUR_VAR_M2_INDEX, 0.0);
|
||||
// Also write the EMA — even though the kernel ignores it after
|
||||
// warmup, mismatching it from the Welford mean tests that the
|
||||
// controller actually selects the Welford path (not the EMA).
|
||||
trainer
|
||||
.isv_mapped
|
||||
.write_record(RL_MEAN_TRADE_DURATION_EMA_INDEX, d_welford);
|
||||
|
||||
trainer
|
||||
.launch_rl_fused_controllers(1)
|
||||
.expect("launch_rl_fused_controllers");
|
||||
sync(&trainer);
|
||||
|
||||
let adaptive_min = trainer.read_isv_host(RL_GAMMA_MIN_ADAPTIVE_INDEX);
|
||||
|
||||
// Analytical oracle: `max(0.9, 1 - 1/max(d × 2, 10))`. At d = 30,
|
||||
// d × 2 = 60 > 10 = HORIZON_FLOOR, so the inner `max` resolves to 60
|
||||
// and the formula reduces to `1 - 1/60`.
|
||||
let horizon = (d_welford * 2.0_f32).max(10.0);
|
||||
let expected = (1.0_f32 - 1.0 / horizon).max(GAMMA_MIN_ABSOLUTE);
|
||||
assert!(
|
||||
(adaptive_min - expected).abs() < 1e-4,
|
||||
"post-warmup γ_min must be max(0.9, 1 - 1/{horizon}) = {expected}; got {adaptive_min}"
|
||||
);
|
||||
|
||||
// γ itself must respect the new floor — bootstrap value 0.9 was
|
||||
// computed against the cold-start adaptive_min 0.9; after the
|
||||
// controller re-fires with the new welford mean, γ should be
|
||||
// clamped to the new floor.
|
||||
let gamma = trainer.read_isv_host(RL_GAMMA_INDEX);
|
||||
assert!(
|
||||
gamma >= adaptive_min - 1e-4,
|
||||
"γ must respect adaptive_min {adaptive_min}; got {gamma}"
|
||||
);
|
||||
|
||||
eprintln!(
|
||||
"G6 OK (post-warmup) — adaptive_min = {adaptive_min} (expected {expected}) \
|
||||
at Welford count = {WELFORD_WARMUP_OBS}, mean = {d_welford}; γ = {gamma}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires CUDA (MlDevice::cuda(0))"]
|
||||
fn g6_gamma_adaptive_min_pre_warmup_uses_ema() {
|
||||
let Some((_dev, trainer)) = build_trainer() else { return };
|
||||
|
||||
// Pre-warmup state: count = 99 < WELFORD_WARMUP_OBS = 100. The
|
||||
// controller MUST fall back to the EMA slot, not the Welford mean.
|
||||
// Deliberately set EMA ≠ Welford mean so the test confirms the
|
||||
// controller's selection logic.
|
||||
let d_ema: f32 = 15.0;
|
||||
let d_welford: f32 = 30.0;
|
||||
trainer
|
||||
.isv_mapped
|
||||
.write_record(RL_TRADE_DUR_VAR_COUNT_INDEX, WELFORD_WARMUP_OBS - 1.0);
|
||||
trainer
|
||||
.isv_mapped
|
||||
.write_record(RL_TRADE_DUR_VAR_MEAN_INDEX, d_welford);
|
||||
trainer.isv_mapped.write_record(RL_TRADE_DUR_VAR_M2_INDEX, 0.0);
|
||||
trainer
|
||||
.isv_mapped
|
||||
.write_record(RL_MEAN_TRADE_DURATION_EMA_INDEX, d_ema);
|
||||
|
||||
trainer
|
||||
.launch_rl_fused_controllers(1)
|
||||
.expect("launch_rl_fused_controllers");
|
||||
sync(&trainer);
|
||||
|
||||
let adaptive_min = trainer.read_isv_host(RL_GAMMA_MIN_ADAPTIVE_INDEX);
|
||||
|
||||
// Oracle uses the EMA (15), not the Welford mean (30).
|
||||
let horizon = (d_ema * 2.0_f32).max(10.0);
|
||||
let expected = (1.0_f32 - 1.0 / horizon).max(GAMMA_MIN_ABSOLUTE);
|
||||
assert!(
|
||||
(adaptive_min - expected).abs() < 1e-4,
|
||||
"pre-warmup γ_min must use EMA, not Welford mean: \
|
||||
expected max(0.9, 1 - 1/{horizon}) = {expected}; got {adaptive_min}"
|
||||
);
|
||||
|
||||
// Counter-invariant: had the controller used the Welford mean
|
||||
// instead (count = 99 bug), adaptive_min would be ≈ 0.98333. The
|
||||
// test fails if that magic number appears.
|
||||
let welford_horizon = (d_welford * 2.0_f32).max(10.0);
|
||||
let wrong_min = (1.0_f32 - 1.0 / welford_horizon).max(GAMMA_MIN_ABSOLUTE);
|
||||
assert!(
|
||||
(adaptive_min - wrong_min).abs() > 1e-3,
|
||||
"pre-warmup γ_min must NOT match the Welford-mean value {wrong_min} \
|
||||
(would indicate the count < WARMUP guard is broken); got {adaptive_min}"
|
||||
);
|
||||
|
||||
eprintln!(
|
||||
"G6 OK (pre-warmup) — adaptive_min = {adaptive_min} (expected {expected}, \
|
||||
distinct from welford-path {wrong_min}) at Welford count = {}, EMA = {d_ema}",
|
||||
WELFORD_WARMUP_OBS - 1.0
|
||||
);
|
||||
}
|
||||
|
||||
473
crates/ml-alpha/tests/risk_stack_invariants.rs
Normal file
473
crates/ml-alpha/tests/risk_stack_invariants.rs
Normal file
@@ -0,0 +1,473 @@
|
||||
//! Risk-stack invariants (spec `docs/superpowers/specs/2026-05-30-adaptive-risk-management-design.md`).
|
||||
//!
|
||||
//! 12 GPU-oracle gates — four per active layer.
|
||||
//! Layer 1 (CMDP): G1-G4
|
||||
//! Layer 2 (IQN τ): G5-G7
|
||||
//! Layer 3 (Inventory): G8-G9
|
||||
//! Layer 4 (Kelly): G10-G12
|
||||
//!
|
||||
//! Per [[feedback_no_cpu_test_fallbacks]] every oracle is an analytical
|
||||
//! identity of the kernel's documented math — no host reimplementation.
|
||||
//!
|
||||
//! Run:
|
||||
//! `cargo test -p ml-alpha --test risk_stack_invariants --release -- --ignored --nocapture`
|
||||
|
||||
use anyhow::Result;
|
||||
use cudarc::driver::{CudaSlice, CudaStream};
|
||||
#[allow(unused_imports)]
|
||||
use ml_alpha::rl::isv_slots::*;
|
||||
use ml_alpha::trainer::integrated::{
|
||||
write_slice_f32_d_pub, IntegratedTrainer, IntegratedTrainerConfig,
|
||||
};
|
||||
use ml_alpha::trainer::perception::PerceptionTrainerConfig;
|
||||
use ml_core::device::MlDevice;
|
||||
use std::sync::Arc;
|
||||
|
||||
// ─── Bootstrap defaults (mirrored from `with_controllers_bootstrapped`) ───
|
||||
const SESSION_DD_LIMIT_USD: f32 = -3500.0; // 10% of $35k starting capital
|
||||
const COOLDOWN_DURATION: f32 = 500.0;
|
||||
const IQN_TAU_MIN: f32 = 0.1;
|
||||
const IQN_TAU_DD_SENSITIVITY: f32 = 5.0; // hits MIN at 10% DD
|
||||
const KELLY_SAFETY_FRAC: f32 = 0.5; // half-Kelly (Thorp)
|
||||
const KELLY_MIN_TRADES: f32 = 1000.0;
|
||||
const STARTING_CAPITAL_USD: f32 = 35000.0; // ES single-contract base
|
||||
|
||||
fn build_trainer() -> Option<(MlDevice, IntegratedTrainer)> {
|
||||
let dev = match MlDevice::cuda(0) {
|
||||
Ok(d) => d,
|
||||
Err(e) => {
|
||||
eprintln!("CUDA 0 not available — skipping ({e})");
|
||||
return None;
|
||||
}
|
||||
};
|
||||
let cfg = IntegratedTrainerConfig {
|
||||
perception: PerceptionTrainerConfig {
|
||||
seq_len: 4,
|
||||
n_batch: 1,
|
||||
..PerceptionTrainerConfig::default()
|
||||
},
|
||||
dqn_seed: 0xC1,
|
||||
ppo_seed: 0xC2,
|
||||
..IntegratedTrainerConfig::default()
|
||||
};
|
||||
let trainer = IntegratedTrainer::new(&dev, cfg).expect("IntegratedTrainer::new");
|
||||
Some((dev, trainer))
|
||||
}
|
||||
|
||||
fn sync(trainer: &IntegratedTrainer) {
|
||||
trainer.stream.synchronize().expect("sync trainer stream");
|
||||
}
|
||||
|
||||
/// Mapped-pinned upload per [[feedback_no_htod_htoh_only_mapped_pinned]]:
|
||||
/// allocate the device buffer, then stage through the canonical public
|
||||
/// helper which routes through `MappedF32Buffer` + DtoD.
|
||||
fn upload_f32(stream: &Arc<CudaStream>, host: &[f32]) -> Result<CudaSlice<f32>> {
|
||||
let mut d = stream.alloc_zeros::<f32>(host.len())?;
|
||||
write_slice_f32_d_pub(stream, host, &mut d)?;
|
||||
Ok(d)
|
||||
}
|
||||
|
||||
// ═════════════════════════════════════════════════════════════════════
|
||||
// LAYER 1 — CMDP hard constraints
|
||||
// ═════════════════════════════════════════════════════════════════════
|
||||
|
||||
// CMDP gates Layer-1 state is per-batch (one independent backtest session
|
||||
// per b). The four tests below exercise the per-batch buffers via b_size=1
|
||||
// (one account) to match the single-account semantics of the limits.
|
||||
|
||||
/// Write a single value into a per-batch f32 device buffer through the
|
||||
/// canonical mapped-pinned path (no raw HtoD on un-pinned slices per
|
||||
/// `feedback_no_htod_htoh_only_mapped_pinned`).
|
||||
fn write_pb(
|
||||
stream: &Arc<CudaStream>,
|
||||
dst: &mut CudaSlice<f32>,
|
||||
values: &[f32],
|
||||
) -> Result<()> {
|
||||
write_slice_f32_d_pub(stream, values, dst)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// G1 — DD breaker triggered + recovery cooldown started (Fix A).
|
||||
// 1. session_pnl crosses dd_limit → dd_triggered flips to 1.0
|
||||
// 2. cooldown_remaining starts at COOLDOWN_DURATION (recovery clock)
|
||||
// 3. RL_SESSION_PNL_USD_INDEX is mean-of-active (this account is now
|
||||
// inactive → mean = 0); worst is mirrored to RL_SESSION_PNL_WORST_INDEX.
|
||||
#[test]
|
||||
#[ignore = "requires CUDA (MlDevice::cuda(0))"]
|
||||
fn g1_cmdp_dd_breaker_triggers_at_limit() -> Result<()> {
|
||||
let Some((dev, trainer)) = build_trainer() else { return Ok(()) };
|
||||
let stream = dev.cuda_stream()?.clone();
|
||||
sync(&trainer);
|
||||
|
||||
let dd_limit = trainer.read_isv_host(RL_SESSION_DD_LIMIT_USD_INDEX);
|
||||
assert!((dd_limit - SESSION_DD_LIMIT_USD).abs() < 1e-3);
|
||||
assert_eq!(trainer.read_isv_host(RL_SESSION_PNL_USD_INDEX), 0.0);
|
||||
assert_eq!(trainer.read_isv_host(RL_SESSION_PNL_WORST_INDEX), 0.0);
|
||||
assert_eq!(trainer.read_isv_host(RL_SESSION_DD_TRIGGERED_INDEX), 0.0);
|
||||
|
||||
let rewards_d = upload_f32(&stream, &[-4000.0])?;
|
||||
let dones_d = upload_f32(&stream, &[1.0])?;
|
||||
trainer.launch_rl_cmdp_constraints_check(&rewards_d, &dones_d, 1)?;
|
||||
sync(&trainer);
|
||||
|
||||
// The single account just tripped → it's now INACTIVE.
|
||||
let mean_active = trainer.read_isv_host(RL_SESSION_PNL_USD_INDEX);
|
||||
assert_eq!(
|
||||
mean_active, 0.0,
|
||||
"mean-active-pnl must be 0 with all accounts triggered; got {mean_active}"
|
||||
);
|
||||
|
||||
// Worst slot mirrors the most-negative per-batch pnl.
|
||||
let worst_pnl = trainer.read_isv_host(RL_SESSION_PNL_WORST_INDEX);
|
||||
assert!(
|
||||
(worst_pnl - (-4000.0)).abs() < 1e-3,
|
||||
"worst-account pnl must be -4000; got {worst_pnl}"
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
trainer.read_isv_host(RL_SESSION_DD_TRIGGERED_INDEX), 1.0,
|
||||
"any-DD-triggered summary must flip to 1.0"
|
||||
);
|
||||
|
||||
// Fix A: DD trip also starts a recovery cooldown.
|
||||
let cool = trainer.read_isv_host(RL_COOLDOWN_REMAINING_STEPS_INDEX);
|
||||
assert!(
|
||||
(cool - COOLDOWN_DURATION).abs() < 1e-3,
|
||||
"DD trip must start recovery cooldown at duration={COOLDOWN_DURATION}; got {cool}"
|
||||
);
|
||||
eprintln!("G1 OK — DD trip → triggered=1, mean_active=0, worst=-4000, cooldown={cool}");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// G1b (NEW) — Fix A recovery: cooldown countdown to 0 resets the account.
|
||||
// After ~500 idle launches, session_pnl + dd_triggered should both clear,
|
||||
// and the account rejoins the active fleet.
|
||||
#[test]
|
||||
#[ignore = "requires CUDA (MlDevice::cuda(0))"]
|
||||
fn g1b_cmdp_recovery_on_cooldown_expiry() -> Result<()> {
|
||||
let Some((dev, trainer)) = build_trainer() else { return Ok(()) };
|
||||
let stream = dev.cuda_stream()?.clone();
|
||||
|
||||
// Trip DD on the first launch.
|
||||
let trip_rewards = upload_f32(&stream, &[-4000.0])?;
|
||||
let trip_dones = upload_f32(&stream, &[1.0])?;
|
||||
trainer.launch_rl_cmdp_constraints_check(&trip_rewards, &trip_dones, 1)?;
|
||||
|
||||
// Idle launches to decrement cooldown to zero.
|
||||
let idle_rewards = upload_f32(&stream, &[0.0])?;
|
||||
let idle_dones = upload_f32(&stream, &[0.0])?;
|
||||
let duration = COOLDOWN_DURATION as usize;
|
||||
for _ in 0..duration {
|
||||
trainer.launch_rl_cmdp_constraints_check(&idle_rewards, &idle_dones, 1)?;
|
||||
}
|
||||
sync(&trainer);
|
||||
|
||||
// Post-recovery: dd cleared, pnl reset to 0, account is active again.
|
||||
let mean_active = trainer.read_isv_host(RL_SESSION_PNL_USD_INDEX);
|
||||
let worst = trainer.read_isv_host(RL_SESSION_PNL_WORST_INDEX);
|
||||
let triggered = trainer.read_isv_host(RL_SESSION_DD_TRIGGERED_INDEX);
|
||||
let cool = trainer.read_isv_host(RL_COOLDOWN_REMAINING_STEPS_INDEX);
|
||||
assert_eq!(mean_active, 0.0, "post-recovery mean must be 0; got {mean_active}");
|
||||
assert_eq!(worst, 0.0, "post-recovery worst must be 0 (pnl reset); got {worst}");
|
||||
assert_eq!(triggered, 0.0, "post-recovery dd_triggered must clear; got {triggered}");
|
||||
assert_eq!(cool, 0.0, "cooldown must be exhausted; got {cool}");
|
||||
eprintln!("G1b OK — account recovered after {duration}-step cooldown");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// G2 — Cooldown starts when an account's consec_loss streak crosses limit.
|
||||
// b=1, 10 successive done-loss launches drive the per-batch streak to 10
|
||||
// → cooldown_remaining_per_batch[0] snaps to cooldown_duration; the
|
||||
// kernel also resets the per-batch streak to 0 on the same launch.
|
||||
#[test]
|
||||
#[ignore = "requires CUDA (MlDevice::cuda(0))"]
|
||||
fn g2_cmdp_cooldown_starts_after_consec_loss_limit() -> Result<()> {
|
||||
let Some((dev, trainer)) = build_trainer() else { return Ok(()) };
|
||||
let stream = dev.cuda_stream()?.clone();
|
||||
|
||||
sync(&trainer);
|
||||
assert_eq!(trainer.read_isv_host(RL_CONSEC_LOSS_COUNT_INDEX), 0.0);
|
||||
assert_eq!(trainer.read_isv_host(RL_COOLDOWN_REMAINING_STEPS_INDEX), 0.0);
|
||||
|
||||
// 10 sequential done-losses on the same account. Per-loss -$10, total
|
||||
// -$100 ≫ -$3500 limit, so we isolate the consec-streak arm.
|
||||
let rewards_d = upload_f32(&stream, &[-10.0])?;
|
||||
let dones_d = upload_f32(&stream, &[1.0])?;
|
||||
for _ in 0..10 {
|
||||
trainer.launch_rl_cmdp_constraints_check(&rewards_d, &dones_d, 1)?;
|
||||
}
|
||||
sync(&trainer);
|
||||
|
||||
let cooldown = trainer.read_isv_host(RL_COOLDOWN_REMAINING_STEPS_INDEX);
|
||||
let consec = trainer.read_isv_host(RL_CONSEC_LOSS_COUNT_INDEX);
|
||||
// The 10th launch trips the limit. Kernel sets per_batch_cooldown[0]=duration
|
||||
// then decrement *also runs this step* — so summary reads (duration - 1).
|
||||
// Spec is "limit hit → start cooldown" but the decrement happens above
|
||||
// the streak-check in the kernel ordering, so a single launch nets to
|
||||
// exactly `duration` (no decrement yet because cooldown was 0 entering
|
||||
// the launch). Confirm against `COOLDOWN_DURATION` directly.
|
||||
assert!(
|
||||
(cooldown - COOLDOWN_DURATION).abs() < 1e-3,
|
||||
"cooldown_remaining must snap to duration={COOLDOWN_DURATION} on the 10th loss; got {cooldown}"
|
||||
);
|
||||
assert_eq!(consec, 0.0, "consec_loss_count must reset on limit hit");
|
||||
eprintln!(
|
||||
"G2 OK — 10 consecutive losses on one account → cooldown = {cooldown}, streak reset to {consec}"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// G3 — Cooldown decrements one step per launch on the affected account.
|
||||
// Seed cooldown_remaining_per_batch[0]=50, run 3 idle launches, expect 47.
|
||||
#[test]
|
||||
#[ignore = "requires CUDA (MlDevice::cuda(0))"]
|
||||
fn g3_cmdp_cooldown_decrements_per_step() -> Result<()> {
|
||||
let Some((dev, mut trainer)) = build_trainer() else { return Ok(()) };
|
||||
let stream = dev.cuda_stream()?.clone();
|
||||
|
||||
// Seed the per-batch cooldown directly (the ISV summary slot is
|
||||
// kernel-OUT only, not an input).
|
||||
write_pb(&stream, &mut trainer.cooldown_remaining_per_batch_d, &[50.0])?;
|
||||
|
||||
// Three idle launches with a break-even close — exercises the
|
||||
// decrement without touching the streak counter.
|
||||
let rewards_d = upload_f32(&stream, &[0.0])?;
|
||||
let dones_d = upload_f32(&stream, &[1.0])?;
|
||||
for _ in 0..3 {
|
||||
trainer.launch_rl_cmdp_constraints_check(&rewards_d, &dones_d, 1)?;
|
||||
}
|
||||
sync(&trainer);
|
||||
|
||||
let cooldown = trainer.read_isv_host(RL_COOLDOWN_REMAINING_STEPS_INDEX);
|
||||
assert!(
|
||||
(cooldown - 47.0).abs() < 1e-3,
|
||||
"cooldown must decrement 50→47 over 3 launches; got {cooldown}"
|
||||
);
|
||||
eprintln!("G3 OK — cooldown decremented 50 → {cooldown} over 3 launches");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// G4 — A done-win resets that account's consec_loss streak to 0.
|
||||
// Seed consec_loss_per_batch[0]=5, feed one win → counter resets before
|
||||
// the limit check.
|
||||
#[test]
|
||||
#[ignore = "requires CUDA (MlDevice::cuda(0))"]
|
||||
fn g4_cmdp_win_resets_consec_loss_counter() -> Result<()> {
|
||||
let Some((dev, mut trainer)) = build_trainer() else { return Ok(()) };
|
||||
let stream = dev.cuda_stream()?.clone();
|
||||
|
||||
write_pb(&stream, &mut trainer.consec_loss_per_batch_d, &[5.0])?;
|
||||
let rewards_d = upload_f32(&stream, &[25.0])?;
|
||||
let dones_d = upload_f32(&stream, &[1.0])?;
|
||||
trainer.launch_rl_cmdp_constraints_check(&rewards_d, &dones_d, 1)?;
|
||||
sync(&trainer);
|
||||
|
||||
let consec = trainer.read_isv_host(RL_CONSEC_LOSS_COUNT_INDEX);
|
||||
assert_eq!(consec, 0.0, "win must reset consec_loss_count to 0; got {consec}");
|
||||
eprintln!("G4 OK — done-win reset consec_loss_count 5 → 0");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ═════════════════════════════════════════════════════════════════════
|
||||
// LAYER 2 — IQN risk-averse action τ
|
||||
// ═════════════════════════════════════════════════════════════════════
|
||||
|
||||
// G5 — At positive session_pnl τ stays at 0.5 (median/risk-neutral).
|
||||
#[test]
|
||||
#[ignore = "requires CUDA (MlDevice::cuda(0))"]
|
||||
fn g5_iqn_tau_at_peak_returns_0_5() -> Result<()> {
|
||||
let Some((_dev, trainer)) = build_trainer() else { return Ok(()) };
|
||||
|
||||
trainer.isv_mapped.write_record(RL_SESSION_PNL_USD_INDEX, 1000.0);
|
||||
trainer.launch_rl_iqn_action_tau_controller()?;
|
||||
sync(&trainer);
|
||||
|
||||
let tau = trainer.read_isv_host(RL_IQN_ACTION_TAU_INDEX);
|
||||
assert!(
|
||||
(tau - 0.5).abs() < 1e-5,
|
||||
"τ must be 0.5 when session_pnl > 0 (no drawdown); got {tau}"
|
||||
);
|
||||
eprintln!("G5 OK — τ = {tau} at session_pnl = +1000");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// G6 — Small drawdown linearly reduces τ. With sensitivity=5.0,
|
||||
// session_pnl = -350 → drawdown_frac = 0.01 → τ_target = 0.5 - 5×0.01 = 0.45.
|
||||
#[test]
|
||||
#[ignore = "requires CUDA (MlDevice::cuda(0))"]
|
||||
fn g6_iqn_tau_adapts_linearly_to_drawdown() -> Result<()> {
|
||||
let Some((_dev, trainer)) = build_trainer() else { return Ok(()) };
|
||||
|
||||
trainer.isv_mapped.write_record(RL_SESSION_PNL_USD_INDEX, -350.0);
|
||||
trainer.launch_rl_iqn_action_tau_controller()?;
|
||||
sync(&trainer);
|
||||
|
||||
let tau = trainer.read_isv_host(RL_IQN_ACTION_TAU_INDEX);
|
||||
let drawdown_frac = 350.0 / STARTING_CAPITAL_USD;
|
||||
let expected = 0.5 - IQN_TAU_DD_SENSITIVITY * drawdown_frac;
|
||||
assert!(
|
||||
(tau - expected).abs() < 1e-4,
|
||||
"τ must be 0.5 - sensitivity × dd_frac = {expected}; got {tau}"
|
||||
);
|
||||
eprintln!("G6 OK — τ = {tau} at session_pnl = -350 (≈ 1% DD)");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// G7 — Deep drawdown clamps τ to τ_min. session_pnl = -7000 (20% of $35k)
|
||||
// drives τ_target = 0.5 - 5×0.2 = -0.5, which must clamp to τ_min = 0.1.
|
||||
#[test]
|
||||
#[ignore = "requires CUDA (MlDevice::cuda(0))"]
|
||||
fn g7_iqn_tau_clamps_at_min_on_deep_drawdown() -> Result<()> {
|
||||
let Some((_dev, trainer)) = build_trainer() else { return Ok(()) };
|
||||
|
||||
trainer.isv_mapped.write_record(RL_SESSION_PNL_USD_INDEX, -7000.0);
|
||||
trainer.launch_rl_iqn_action_tau_controller()?;
|
||||
sync(&trainer);
|
||||
|
||||
let tau = trainer.read_isv_host(RL_IQN_ACTION_TAU_INDEX);
|
||||
assert!(
|
||||
(tau - IQN_TAU_MIN).abs() < 1e-5,
|
||||
"τ must clamp at τ_min={IQN_TAU_MIN} for 20% DD; got {tau}"
|
||||
);
|
||||
eprintln!("G7 OK — τ clamped at {tau} = τ_min for 20% drawdown");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ═════════════════════════════════════════════════════════════════════
|
||||
// LAYER 3 — Avellaneda-Stoikov inventory penalty β
|
||||
// ═════════════════════════════════════════════════════════════════════
|
||||
|
||||
// G8 — β bootstraps to target on first observation (prev==0 sentinel).
|
||||
// reward_mag=20, inv_var=4 ⇒ inv_std=2 ⇒ target = 0.01×20 / (2×2) = 0.05.
|
||||
#[test]
|
||||
#[ignore = "requires CUDA (MlDevice::cuda(0))"]
|
||||
fn g8_inventory_beta_bootstraps_to_target() -> Result<()> {
|
||||
let Some((_dev, trainer)) = build_trainer() else { return Ok(()) };
|
||||
|
||||
// Sentinel: β starts at 0.0 per bootstrap.
|
||||
assert_eq!(trainer.read_isv_host(RL_INVENTORY_PENALTY_BETA_INDEX), 0.0);
|
||||
|
||||
trainer.isv_mapped.write_record(RL_REWARD_MAGNITUDE_EMA_INDEX, 20.0);
|
||||
trainer.isv_mapped.write_record(RL_INVENTORY_VARIANCE_EMA_INDEX, 4.0);
|
||||
trainer.launch_rl_inventory_beta_controller()?;
|
||||
sync(&trainer);
|
||||
|
||||
let beta = trainer.read_isv_host(RL_INVENTORY_PENALTY_BETA_INDEX);
|
||||
// expected = 0.01 × reward_mag / (2 × inv_std) = 0.01 × 20 / (2 × 2) = 0.05
|
||||
assert!(
|
||||
(beta - 0.05).abs() < 1e-5,
|
||||
"β must bootstrap-replace to target=0.05; got {beta}"
|
||||
);
|
||||
eprintln!("G8 OK — β bootstrapped to {beta} = 0.01·20/(2·√4)");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// G9 — After bootstrap, β Wiener-blends with the new target instead of
|
||||
// snapping. α_floor=0.4 ⇒ next β = 0.6×prev + 0.4×new_target.
|
||||
// Verifies the canonical [[pearl_first_observation_bootstrap]] discipline.
|
||||
#[test]
|
||||
#[ignore = "requires CUDA (MlDevice::cuda(0))"]
|
||||
fn g9_inventory_beta_wiener_blends_after_bootstrap() -> Result<()> {
|
||||
let Some((_dev, trainer)) = build_trainer() else { return Ok(()) };
|
||||
|
||||
// Step 1: bootstrap to 0.05.
|
||||
trainer.isv_mapped.write_record(RL_REWARD_MAGNITUDE_EMA_INDEX, 20.0);
|
||||
trainer.isv_mapped.write_record(RL_INVENTORY_VARIANCE_EMA_INDEX, 4.0);
|
||||
trainer.launch_rl_inventory_beta_controller()?;
|
||||
sync(&trainer);
|
||||
let beta1 = trainer.read_isv_host(RL_INVENTORY_PENALTY_BETA_INDEX);
|
||||
assert!((beta1 - 0.05).abs() < 1e-5);
|
||||
|
||||
// Step 2: target shifts to 0.10 (double reward_mag).
|
||||
// Wiener: β_new = 0.6 × 0.05 + 0.4 × 0.10 = 0.07.
|
||||
trainer.isv_mapped.write_record(RL_REWARD_MAGNITUDE_EMA_INDEX, 40.0);
|
||||
trainer.launch_rl_inventory_beta_controller()?;
|
||||
sync(&trainer);
|
||||
let beta2 = trainer.read_isv_host(RL_INVENTORY_PENALTY_BETA_INDEX);
|
||||
assert!(
|
||||
(beta2 - 0.07).abs() < 1e-5,
|
||||
"β must Wiener-blend (α=0.4) to 0.6·0.05 + 0.4·0.10 = 0.07; got {beta2}"
|
||||
);
|
||||
eprintln!("G9 OK — β Wiener-blended 0.05 → {beta2} (target 0.10)");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ═════════════════════════════════════════════════════════════════════
|
||||
// LAYER 4 — Kelly fraction position sizing
|
||||
// ═════════════════════════════════════════════════════════════════════
|
||||
|
||||
// G10 — At p=0.55, R-multiple b=avg_win/avg_loss=1.8, half-Kelly returns
|
||||
// 0.5 × (0.55×1.8 − 0.45)/1.8 = 0.5 × 0.30 = 0.15.
|
||||
#[test]
|
||||
#[ignore = "requires CUDA (MlDevice::cuda(0))"]
|
||||
fn g10_kelly_at_known_edge_returns_half_kelly() -> Result<()> {
|
||||
let Some((_dev, trainer)) = build_trainer() else { return Ok(()) };
|
||||
|
||||
// Past warmup so the gate releases.
|
||||
trainer.isv_mapped.write_record(RL_CUMULATIVE_DONES_INDEX, 2000.0);
|
||||
trainer.isv_mapped.write_record(RL_WIN_RATE_EMA_INDEX, 0.55);
|
||||
trainer.isv_mapped.write_record(RL_AVG_WIN_USD_EMA_INDEX, 1.8);
|
||||
trainer.isv_mapped.write_record(RL_AVG_LOSS_USD_EMA_INDEX, 1.0);
|
||||
trainer.launch_rl_kelly_fraction_controller()?;
|
||||
sync(&trainer);
|
||||
|
||||
let f = trainer.read_isv_host(RL_KELLY_FRACTION_INDEX);
|
||||
// f_kelly = (0.55·1.8 − 0.45)/1.8 = 0.3 → f = safety·f_kelly = 0.5·0.3 = 0.15
|
||||
let expected = KELLY_SAFETY_FRAC * 0.3;
|
||||
assert!(
|
||||
(f - expected).abs() < 1e-5,
|
||||
"Kelly at p=0.55, R=1.8 must be {expected}; got {f}"
|
||||
);
|
||||
eprintln!("G10 OK — Kelly = {f} at p=0.55, R-multiple=1.8 (half-Kelly target {expected})");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// G11 — At negative edge (p=0.21, b=1.8) the analytic Kelly is negative;
|
||||
// the kernel clamps f to 0 — never short-the-edge / lever-up-into-losses.
|
||||
#[test]
|
||||
#[ignore = "requires CUDA (MlDevice::cuda(0))"]
|
||||
fn g11_kelly_at_negative_edge_clamps_to_zero() -> Result<()> {
|
||||
let Some((_dev, trainer)) = build_trainer() else { return Ok(()) };
|
||||
|
||||
trainer.isv_mapped.write_record(RL_CUMULATIVE_DONES_INDEX, 2000.0);
|
||||
trainer.isv_mapped.write_record(RL_WIN_RATE_EMA_INDEX, 0.21);
|
||||
trainer.isv_mapped.write_record(RL_AVG_WIN_USD_EMA_INDEX, 1.8);
|
||||
trainer.isv_mapped.write_record(RL_AVG_LOSS_USD_EMA_INDEX, 1.0);
|
||||
trainer.launch_rl_kelly_fraction_controller()?;
|
||||
sync(&trainer);
|
||||
|
||||
let f = trainer.read_isv_host(RL_KELLY_FRACTION_INDEX);
|
||||
assert_eq!(f, 0.0, "Kelly at negative edge must clamp to 0; got {f}");
|
||||
eprintln!("G11 OK — Kelly = 0 at p=0.21 (analytic f_kelly < 0, clamped)");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// G12 — Warmup gate: while cumulative_dones < min_trades_for_release, the
|
||||
// kernel holds f at bootstrap=1.0 regardless of EMA inputs. This is the
|
||||
// fix that resolved the fold-1 win-rate-collapse → negative-Kelly attractor
|
||||
// (see [[pearl_position_sizing_missing_adaptation_layer]]).
|
||||
#[test]
|
||||
#[ignore = "requires CUDA (MlDevice::cuda(0))"]
|
||||
fn g12_kelly_held_at_bootstrap_during_warmup() -> Result<()> {
|
||||
let Some((_dev, trainer)) = build_trainer() else { return Ok(()) };
|
||||
|
||||
// 500 trades observed, threshold 1000 → still warming up.
|
||||
trainer.isv_mapped.write_record(RL_CUMULATIVE_DONES_INDEX, 500.0);
|
||||
let min_trades = trainer.read_isv_host(RL_KELLY_MIN_TRADES_FOR_RELEASE_INDEX);
|
||||
assert!((min_trades - KELLY_MIN_TRADES).abs() < 1e-3);
|
||||
|
||||
// Even with negative-edge EMA inputs, Kelly must hold at 1.0.
|
||||
trainer.isv_mapped.write_record(RL_WIN_RATE_EMA_INDEX, 0.10);
|
||||
trainer.isv_mapped.write_record(RL_AVG_WIN_USD_EMA_INDEX, 1.0);
|
||||
trainer.isv_mapped.write_record(RL_AVG_LOSS_USD_EMA_INDEX, 5.0);
|
||||
trainer.launch_rl_kelly_fraction_controller()?;
|
||||
sync(&trainer);
|
||||
|
||||
let f = trainer.read_isv_host(RL_KELLY_FRACTION_INDEX);
|
||||
assert_eq!(
|
||||
f, 1.0,
|
||||
"Kelly must hold at bootstrap=1.0 during warmup; got {f}"
|
||||
);
|
||||
eprintln!("G12 OK — Kelly held at {f} during warmup (500 < {KELLY_MIN_TRADES} trades)");
|
||||
Ok(())
|
||||
}
|
||||
185
crates/ml-alpha/tests/signal_variance_kernel.rs
Normal file
185
crates/ml-alpha/tests/signal_variance_kernel.rs
Normal file
@@ -0,0 +1,185 @@
|
||||
//! GPU-oracle tests for `rl_signal_variance_update` — Welford online
|
||||
//! variance kernel for adaptive controller floors (spec
|
||||
//! `2026-05-30-adaptive-controller-floor-design`).
|
||||
//!
|
||||
//! Three analytical invariants, zero CPU reference implementations:
|
||||
//!
|
||||
//! 1. `welford_constant_input_zero_variance` — feeding a constant `k`
|
||||
//! for N steps must produce `mean == k` and `m2 == 0` (sample
|
||||
//! variance is zero by definition).
|
||||
//! 2. `welford_known_sequence_variance` — sequence [1, 2, 3, 4, 5]
|
||||
//! has known sample variance 2.5; Welford output must match.
|
||||
//! 3. `welford_sentinel_zero_is_skipped` — input == 0.0 sentinel
|
||||
//! must produce no update (per
|
||||
//! `pearl_first_observation_bootstrap`).
|
||||
//!
|
||||
//! Per `feedback_no_cpu_test_fallbacks`: oracles are analytical
|
||||
//! identities of Welford's algorithm, not CPU reimplementations.
|
||||
//! Per `feedback_no_htod_htoh_only_mapped_pinned`: ISV bus uses
|
||||
//! mapped-pinned `MappedF32Buffer` (same pattern as production trainer).
|
||||
//!
|
||||
//! Run with:
|
||||
//! `cargo test -p ml-alpha --test signal_variance_kernel -- --ignored --nocapture`
|
||||
|
||||
use cudarc::driver::{LaunchConfig, PushKernelArg};
|
||||
use ml_alpha::pinned_mem::MappedF32Buffer;
|
||||
use ml_core::device::MlDevice;
|
||||
|
||||
// ISV slot layout for this isolated test: input at 0, Welford triple at 1,2,3.
|
||||
const INPUT_SLOT: i32 = 0;
|
||||
const COUNT_SLOT: i32 = 1;
|
||||
const MEAN_SLOT: i32 = 2;
|
||||
const M2_SLOT: i32 = 3;
|
||||
const N_SLOTS: usize = 4;
|
||||
|
||||
const SIGNAL_VARIANCE_UPDATE_CUBIN: &[u8] =
|
||||
include_bytes!(concat!(env!("OUT_DIR"), "/rl_signal_variance_update.cubin"));
|
||||
|
||||
fn launch_once(
|
||||
dev: &MlDevice,
|
||||
isv: &MappedF32Buffer,
|
||||
) {
|
||||
let stream = dev.cuda_stream().expect("cuda_stream").clone();
|
||||
let ctx = dev.cuda_context().expect("cuda_context");
|
||||
let module = ctx
|
||||
.load_cubin(SIGNAL_VARIANCE_UPDATE_CUBIN.to_vec())
|
||||
.expect("load signal_variance cubin");
|
||||
let func = module
|
||||
.load_function("rl_signal_variance_update")
|
||||
.expect("load rl_signal_variance_update fn");
|
||||
let cfg = LaunchConfig {
|
||||
grid_dim: (1, 1, 1),
|
||||
block_dim: (1, 1, 1),
|
||||
shared_mem_bytes: 0,
|
||||
};
|
||||
let isv_dev_ptr = isv.dev_ptr;
|
||||
unsafe {
|
||||
stream
|
||||
.launch_builder(&func)
|
||||
.arg(&isv_dev_ptr)
|
||||
.arg(&INPUT_SLOT)
|
||||
.arg(&COUNT_SLOT)
|
||||
.arg(&MEAN_SLOT)
|
||||
.arg(&M2_SLOT)
|
||||
.launch(cfg)
|
||||
.expect("rl_signal_variance_update launch");
|
||||
}
|
||||
stream.synchronize().expect("sync");
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires CUDA (MlDevice::cuda(0))"]
|
||||
fn welford_constant_input_zero_variance() {
|
||||
let dev = match MlDevice::cuda(0) {
|
||||
Ok(d) => d,
|
||||
Err(e) => {
|
||||
eprintln!("CUDA 0 not available — skipping ({e})");
|
||||
return;
|
||||
}
|
||||
};
|
||||
let isv = unsafe { MappedF32Buffer::new(N_SLOTS) }.expect("isv alloc");
|
||||
|
||||
// Welford of constant k for N steps: mean = k, m2 = 0, count = N.
|
||||
let k = 0.5_f32;
|
||||
let n = 50_usize;
|
||||
|
||||
for _ in 0..n {
|
||||
isv.write_record(INPUT_SLOT as usize, k);
|
||||
launch_once(&dev, &isv);
|
||||
}
|
||||
|
||||
let count = isv.read_record(COUNT_SLOT as usize);
|
||||
let mean = isv.read_record(MEAN_SLOT as usize);
|
||||
let m2 = isv.read_record(M2_SLOT as usize);
|
||||
|
||||
assert_eq!(count, n as f32, "count must be {n}; got {count}");
|
||||
assert!(
|
||||
(mean - k).abs() < 1e-6,
|
||||
"mean of constant {k} must equal {k}; got {mean}"
|
||||
);
|
||||
assert!(
|
||||
m2.abs() < 1e-5,
|
||||
"m2 of constant input must be exactly 0 (or fp-noise); got {m2}"
|
||||
);
|
||||
|
||||
eprintln!("G1 OK — constant k={k} for N={n}: mean={mean} m2={m2}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires CUDA (MlDevice::cuda(0))"]
|
||||
fn welford_known_sequence_variance() {
|
||||
let dev = match MlDevice::cuda(0) {
|
||||
Ok(d) => d,
|
||||
Err(e) => {
|
||||
eprintln!("CUDA 0 not available — skipping ({e})");
|
||||
return;
|
||||
}
|
||||
};
|
||||
let isv = unsafe { MappedF32Buffer::new(N_SLOTS) }.expect("isv alloc");
|
||||
|
||||
// Sequence [1, 2, 3, 4, 5]: arithmetic mean = 3.
|
||||
// Sample variance = Σ(x_i − μ)² / (n − 1) = (4+1+0+1+4)/4 = 2.5.
|
||||
let seq = [1.0_f32, 2.0, 3.0, 4.0, 5.0];
|
||||
|
||||
for &x in seq.iter() {
|
||||
isv.write_record(INPUT_SLOT as usize, x);
|
||||
launch_once(&dev, &isv);
|
||||
}
|
||||
|
||||
let count = isv.read_record(COUNT_SLOT as usize);
|
||||
let mean = isv.read_record(MEAN_SLOT as usize);
|
||||
let m2 = isv.read_record(M2_SLOT as usize);
|
||||
let sample_var = m2 / (count - 1.0);
|
||||
|
||||
assert_eq!(count, seq.len() as f32, "count must be {}", seq.len());
|
||||
assert!(
|
||||
(mean - 3.0).abs() < 1e-5,
|
||||
"mean of [1..5] must be 3.0; got {mean}"
|
||||
);
|
||||
assert!(
|
||||
(sample_var - 2.5).abs() < 1e-5,
|
||||
"sample variance of [1..5] must be 2.5; got {sample_var} (m2={m2})"
|
||||
);
|
||||
|
||||
eprintln!("G2 OK — sequence {seq:?}: mean={mean} sample_var={sample_var}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires CUDA (MlDevice::cuda(0))"]
|
||||
fn welford_sentinel_zero_is_skipped() {
|
||||
let dev = match MlDevice::cuda(0) {
|
||||
Ok(d) => d,
|
||||
Err(e) => {
|
||||
eprintln!("CUDA 0 not available — skipping ({e})");
|
||||
return;
|
||||
}
|
||||
};
|
||||
let isv = unsafe { MappedF32Buffer::new(N_SLOTS) }.expect("isv alloc");
|
||||
|
||||
// INPUT_SLOT remains at sentinel 0.0 (MappedF32Buffer::new zero-inits).
|
||||
// Kernel must NOT count this as an observation per
|
||||
// pearl_first_observation_bootstrap.
|
||||
for _ in 0..10 {
|
||||
launch_once(&dev, &isv);
|
||||
}
|
||||
assert_eq!(
|
||||
isv.read_record(COUNT_SLOT as usize),
|
||||
0.0,
|
||||
"sentinel zero must not be counted as a real observation"
|
||||
);
|
||||
|
||||
// Now feed a real non-zero observation; counter must advance.
|
||||
isv.write_record(INPUT_SLOT as usize, 7.0);
|
||||
launch_once(&dev, &isv);
|
||||
assert_eq!(
|
||||
isv.read_record(COUNT_SLOT as usize),
|
||||
1.0,
|
||||
"first real observation must be counted"
|
||||
);
|
||||
assert!(
|
||||
(isv.read_record(MEAN_SLOT as usize) - 7.0).abs() < 1e-6,
|
||||
"first observation must initialize mean directly per Welford recurrence"
|
||||
);
|
||||
|
||||
eprintln!("G3 OK — sentinel skip honoured; first real obs initialized mean");
|
||||
}
|
||||
@@ -43,7 +43,7 @@ use ml_alpha::rl::isv_slots::{
|
||||
RL_ANTIMARTINGALE_KAPPA_INDEX, RL_ANTIMARTINGALE_MAX_INDEX, RL_ANTIMARTINGALE_MIN_INDEX,
|
||||
RL_CONF_GATE_LAMBDA_INDEX, RL_CONF_GATE_SIGMA_NORM_INDEX, RL_CONF_GATE_THRESHOLD_INDEX,
|
||||
RL_FRD_GATE_THR_LONG_INDEX, RL_FRD_GATE_THR_SHORT_INDEX, RL_HEAT_CAP_MAX_LOTS_INDEX,
|
||||
RL_PYRAMID_THRESHOLD_INDEX, RL_STEP_COUNTER_ISV_INDEX, RL_TRAIL_ADJUST_RATE_INDEX,
|
||||
RL_PYRAMID_THRESHOLD_INDEX, RL_STEP_COUNTER_ISV_INDEX,
|
||||
RL_TRAIL_K_INIT_INDEX, RL_TRAIL_MAX_INDEX, RL_TRAIL_MIN_INDEX,
|
||||
};
|
||||
use ml_alpha::trainer::integrated::{
|
||||
@@ -404,7 +404,12 @@ fn unit_state_transitions() -> Result<()> {
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires CUDA (MlDevice::cuda(0))"]
|
||||
fn trail_mutate_tighten_loosen_reciprocal() -> Result<()> {
|
||||
fn trail_mutate_independent_tighten_loosen_factors() -> Result<()> {
|
||||
// Adaptive risk management (spec 2026-05-30) migrated a7/a8 from the
|
||||
// symmetric reciprocal `RL_TRAIL_ADJUST_RATE_INDEX` to independent
|
||||
// ISV slots `RL_TRAIL_TIGHTEN_FACTOR_INDEX` (682) and
|
||||
// `RL_TRAIL_LOOSEN_FACTOR_INDEX` (683). Each action multiplies the
|
||||
// trail by its own factor; reciprocity is no longer enforced.
|
||||
let Some((dev, mut trainer)) = build_trainer() else { return Ok(()) };
|
||||
let stream = dev.cuda_stream()?.clone();
|
||||
let b_size = 1;
|
||||
@@ -413,11 +418,14 @@ fn trail_mutate_tighten_loosen_reciprocal() -> Result<()> {
|
||||
// all-zeros; bootstrap defaults are written only during the first
|
||||
// `step_with_lobsim`, which the test doesn't invoke. Set all four
|
||||
// trail slots explicitly — in particular `RL_TRAIL_MAX_INDEX`,
|
||||
// because the loosen branch does `fminf(trail_max, current/rate)`
|
||||
// because the loosen branch does `fminf(trail_max, current * loosen)`
|
||||
// and a zero max would clamp output to 0.
|
||||
set_isv_slot(&mut trainer, &stream, RL_TRAIL_MIN_INDEX, 0.001)?;
|
||||
set_isv_slot(&mut trainer, &stream, RL_TRAIL_MAX_INDEX, 100.0)?;
|
||||
set_isv_slot(&mut trainer, &stream, RL_TRAIL_ADJUST_RATE_INDEX, 0.9)?;
|
||||
set_isv_slot(&mut trainer, &stream,
|
||||
ml_alpha::rl::isv_slots::RL_TRAIL_TIGHTEN_FACTOR_INDEX, 0.9)?;
|
||||
set_isv_slot(&mut trainer, &stream,
|
||||
ml_alpha::rl::isv_slots::RL_TRAIL_LOOSEN_FACTOR_INDEX, 1.1)?;
|
||||
|
||||
let unit_active_d = upload_u8(&stream, &[1, 0, 0, 0])?;
|
||||
let initial_trail = 0.5_f32;
|
||||
@@ -428,21 +436,25 @@ fn trail_mutate_tighten_loosen_reciprocal() -> Result<()> {
|
||||
trainer.launch_rl_trail_mutate(&tighten_d, &unit_active_d, &mut trail_d, b_size)?;
|
||||
let trail = read_slice_d_pub(&stream, &trail_d, b_size * MAX_UNITS)?;
|
||||
let after_tighten = trail[0];
|
||||
let expected_tighten = initial_trail * 0.9;
|
||||
assert!(
|
||||
(after_tighten - initial_trail * 0.9).abs() < 1e-5,
|
||||
"a7 should multiply trail by adjust_rate (0.9); expected {}, got {}",
|
||||
initial_trail * 0.9,
|
||||
(after_tighten - expected_tighten).abs() < 1e-5,
|
||||
"a7 should multiply trail by tighten factor (0.9); expected {}, got {}",
|
||||
expected_tighten,
|
||||
after_tighten
|
||||
);
|
||||
|
||||
// ── a8 (loosen) — trail /= 0.9 ⇒ returns to within fp ε of initial
|
||||
// ── a8 (loosen) — trail *= 1.1
|
||||
let loosen_d = upload_i32(&stream, &[8])?;
|
||||
trainer.launch_rl_trail_mutate(&loosen_d, &unit_active_d, &mut trail_d, b_size)?;
|
||||
let trail = read_slice_d_pub(&stream, &trail_d, b_size * MAX_UNITS)?;
|
||||
let after_loosen = trail[0];
|
||||
let expected_loosen = expected_tighten * 1.1;
|
||||
assert!(
|
||||
(after_loosen - initial_trail).abs() < 1e-5,
|
||||
"a7→a8 should restore trail (reciprocal); expected {initial_trail}, got {after_loosen}"
|
||||
(after_loosen - expected_loosen).abs() < 1e-5,
|
||||
"a8 should multiply trail by loosen factor (1.1); expected {}, got {}",
|
||||
expected_loosen,
|
||||
after_loosen
|
||||
);
|
||||
|
||||
// ── Inactive units MUST NOT mutate. trail[1..4] start at 0.0 and
|
||||
@@ -461,7 +473,7 @@ fn trail_mutate_tighten_loosen_reciprocal() -> Result<()> {
|
||||
);
|
||||
|
||||
eprintln!(
|
||||
"PASS — trail mutate reciprocal: {initial_trail} → {after_tighten} → {after_loosen}"
|
||||
"PASS — trail mutate independent factors: {initial_trail} → {after_tighten} → {after_loosen}"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
@@ -1099,7 +1111,8 @@ fn trail_at_min_a7_stays_clamped() -> Result<()> {
|
||||
|
||||
set_isv_slot(&mut trainer, &stream, RL_TRAIL_MIN_INDEX, 0.1)?;
|
||||
set_isv_slot(&mut trainer, &stream, RL_TRAIL_MAX_INDEX, 100.0)?;
|
||||
set_isv_slot(&mut trainer, &stream, RL_TRAIL_ADJUST_RATE_INDEX, 0.9)?;
|
||||
set_isv_slot(&mut trainer, &stream,
|
||||
ml_alpha::rl::isv_slots::RL_TRAIL_TIGHTEN_FACTOR_INDEX, 0.9)?;
|
||||
|
||||
let unit_active_d = upload_u8(&stream, &[1, 0, 0, 0])?;
|
||||
let mut trail_d = upload_f32(&stream, &[0.1, 0.0, 0.0, 0.0])?;
|
||||
@@ -1127,7 +1140,8 @@ fn trail_at_max_a8_stays_clamped() -> Result<()> {
|
||||
|
||||
set_isv_slot(&mut trainer, &stream, RL_TRAIL_MIN_INDEX, 0.001)?;
|
||||
set_isv_slot(&mut trainer, &stream, RL_TRAIL_MAX_INDEX, 1.0)?;
|
||||
set_isv_slot(&mut trainer, &stream, RL_TRAIL_ADJUST_RATE_INDEX, 0.9)?;
|
||||
set_isv_slot(&mut trainer, &stream,
|
||||
ml_alpha::rl::isv_slots::RL_TRAIL_LOOSEN_FACTOR_INDEX, 1.1)?;
|
||||
|
||||
let unit_active_d = upload_u8(&stream, &[1, 0, 0, 0])?;
|
||||
let mut trail_d = upload_f32(&stream, &[1.0, 0.0, 0.0, 0.0])?;
|
||||
@@ -1138,7 +1152,7 @@ fn trail_at_max_a8_stays_clamped() -> Result<()> {
|
||||
let trail = read_slice_d_pub(&stream, &trail_d, b_size * MAX_UNITS)?;
|
||||
assert!(
|
||||
(trail[0] - 1.0).abs() < 1e-5,
|
||||
"a8 at max (1.0/0.9=1.11) should clamp to TRAIL_MAX=1.0; got {}",
|
||||
"a8 at max (1.0*1.1=1.10) should clamp to TRAIL_MAX=1.0; got {}",
|
||||
trail[0]
|
||||
);
|
||||
|
||||
|
||||
@@ -907,7 +907,7 @@ fn main() -> Result<()> {
|
||||
.context("reset vol_obs min slot")?;
|
||||
}
|
||||
// Lockstep step loop.
|
||||
for step in 0..cli.horizon {
|
||||
for _ in 0..cli.horizon {
|
||||
// Gather current states (CPU; <100μs for N=500).
|
||||
let mut batched_states_host = vec![0.0_f32; n_par * STATE_DIM];
|
||||
for i in 0..n_par {
|
||||
|
||||
@@ -9,7 +9,7 @@ use ml::cuda_pipeline::lob_bar::LobBar;
|
||||
use std::collections::HashMap;
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct BehavioralResult {
|
||||
pub(crate) struct BehavioralResult {
|
||||
/// Action distribution keys: "hold", "flat", "long_quarter",
|
||||
/// "long_half", "long_full", "short_quarter", "short_half", "short_full".
|
||||
pub action_dist: HashMap<&'static str, f32>,
|
||||
@@ -27,7 +27,7 @@ pub struct BehavioralResult {
|
||||
/// each Phase 2B test rewrites the body to call the trainer methods it
|
||||
/// requires. The signature here is the load-bearing piece — keep it stable.
|
||||
#[allow(dead_code, unused_variables)]
|
||||
pub fn evaluate_policy_on_market(
|
||||
pub(crate) fn evaluate_policy_on_market(
|
||||
// Phase 2B swaps `&mut ()` for `&mut GpuDqnTrainer` once the per-test
|
||||
// trainer methods land. Until then the unit param keeps the signature
|
||||
// callable in tests that don't yet exercise the trainer path.
|
||||
|
||||
@@ -5,10 +5,10 @@
|
||||
use ml::cuda_pipeline::lob_bar::LobBar;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum MagBucket { Quarter, Half, Full }
|
||||
pub(crate) enum MagBucket { Quarter, Half, Full }
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum OracleAction {
|
||||
pub(crate) enum OracleAction {
|
||||
Hold,
|
||||
Long(MagBucket),
|
||||
Short(MagBucket),
|
||||
@@ -16,12 +16,12 @@ pub enum OracleAction {
|
||||
}
|
||||
|
||||
/// Oracle for flat markets: always Hold.
|
||||
pub fn flat_market_oracle(bars: &[LobBar]) -> Vec<OracleAction> {
|
||||
pub(crate) fn flat_market_oracle(bars: &[LobBar]) -> Vec<OracleAction> {
|
||||
vec![OracleAction::Hold; bars.len()]
|
||||
}
|
||||
|
||||
/// Oracle for drift markets: take direction-of-trend, mag = Half.
|
||||
pub fn drift_market_oracle(bars: &[LobBar]) -> Vec<OracleAction> {
|
||||
pub(crate) fn drift_market_oracle(bars: &[LobBar]) -> Vec<OracleAction> {
|
||||
let mut actions = Vec::with_capacity(bars.len());
|
||||
actions.push(OracleAction::Hold); // first bar: no prior
|
||||
for w in bars.windows(2) {
|
||||
@@ -38,7 +38,7 @@ pub fn drift_market_oracle(bars: &[LobBar]) -> Vec<OracleAction> {
|
||||
}
|
||||
|
||||
/// Oracle for OU markets: reverse at ±2σ extremes.
|
||||
pub fn ou_market_oracle(bars: &[LobBar], mu: f32, sigma: f32) -> Vec<OracleAction> {
|
||||
pub(crate) fn ou_market_oracle(bars: &[LobBar], mu: f32, sigma: f32) -> Vec<OracleAction> {
|
||||
bars.iter().map(|b| {
|
||||
let z = (b.price - mu) / sigma.max(1e-6);
|
||||
if z > 2.0 { OracleAction::Short(MagBucket::Half) }
|
||||
|
||||
@@ -6,7 +6,7 @@ use rand::{Rng, SeedableRng};
|
||||
use rand::rngs::StdRng;
|
||||
|
||||
/// Flat market: constant base price + Gaussian noise.
|
||||
pub fn flat_market(n: usize, noise_sigma: f32, mean_spread: f32, seed: u64) -> Vec<LobBar> {
|
||||
pub(crate) fn flat_market(n: usize, noise_sigma: f32, mean_spread: f32, seed: u64) -> Vec<LobBar> {
|
||||
let mut rng = StdRng::seed_from_u64(seed);
|
||||
(0..n).map(|_| {
|
||||
let noise: f32 = rng.gen_range(-1.0..1.0) * noise_sigma;
|
||||
@@ -18,7 +18,7 @@ pub fn flat_market(n: usize, noise_sigma: f32, mean_spread: f32, seed: u64) -> V
|
||||
}
|
||||
|
||||
/// Drift market: drift μ + noise σ.
|
||||
pub fn drift_market(n: usize, mu: f32, sigma: f32, mean_spread: f32, seed: u64) -> Vec<LobBar> {
|
||||
pub(crate) fn drift_market(n: usize, mu: f32, sigma: f32, mean_spread: f32, seed: u64) -> Vec<LobBar> {
|
||||
let mut rng = StdRng::seed_from_u64(seed);
|
||||
let mut price: f32 = 4500.0;
|
||||
(0..n).map(|_| {
|
||||
@@ -31,7 +31,7 @@ pub fn drift_market(n: usize, mu: f32, sigma: f32, mean_spread: f32, seed: u64)
|
||||
}
|
||||
|
||||
/// OU process: mean-reverting around equilibrium.
|
||||
pub fn ou_market(n: usize, theta: f32, sigma_eq: f32, mu: f32,
|
||||
pub(crate) fn ou_market(n: usize, theta: f32, sigma_eq: f32, mu: f32,
|
||||
mean_spread: f32, seed: u64) -> Vec<LobBar> {
|
||||
let mut rng = StdRng::seed_from_u64(seed);
|
||||
let mut price: f32 = mu;
|
||||
@@ -45,7 +45,7 @@ pub fn ou_market(n: usize, theta: f32, sigma_eq: f32, mu: f32,
|
||||
}
|
||||
|
||||
/// Regime-switch: Markov chain over `n_regimes`, each with (mu, sigma).
|
||||
pub fn regime_switch_market(
|
||||
pub(crate) fn regime_switch_market(
|
||||
n: usize,
|
||||
regime_params: &[(f32, f32)],
|
||||
transition_matrix: &[Vec<f32>],
|
||||
|
||||
@@ -1980,7 +1980,7 @@ fn pearl_1_ext_num_atoms_threshold_cascade() {
|
||||
|
||||
let scratch_size: usize = 4;
|
||||
let base: i32 = 0;
|
||||
let mut scratch = unsafe { MappedF32Buffer::new(scratch_size) }.expect("scratch alloc");
|
||||
let scratch = unsafe { MappedF32Buffer::new(scratch_size) }.expect("scratch alloc");
|
||||
|
||||
let isv_dev = isv_buf.dev_ptr;
|
||||
let scr_dev = scratch.dev_ptr;
|
||||
@@ -2048,7 +2048,7 @@ fn pearl_1_ext_num_atoms_exact_threshold_falls_to_next_branch() {
|
||||
|
||||
let scratch_size: usize = 4;
|
||||
let base: i32 = 0;
|
||||
let mut scratch = unsafe { MappedF32Buffer::new(scratch_size) }.expect("scratch alloc");
|
||||
let scratch = unsafe { MappedF32Buffer::new(scratch_size) }.expect("scratch alloc");
|
||||
|
||||
let isv_dev = isv_buf.dev_ptr;
|
||||
let scr_dev = scratch.dev_ptr;
|
||||
|
||||
@@ -0,0 +1,957 @@
|
||||
# Adaptive Controller Signal Floors — Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Replace hardcoded thresholds in 12 RL controllers with observed-signal-driven ISV slots. One atomic commit. Eliminate the architectural failure mode where Phase 4.5 normalization invalidated 8 controllers simultaneously.
|
||||
|
||||
**Architecture:** New Welford-variance CUDA kernel (`rl_signal_variance_update.cu`) tracks running variance of every controller's input EMA. Controllers consume `sqrt(observed_var)` as their noise-floor reference instead of hardcoded constants. 52 new ISV slots wire the kernel into the existing per-step pipeline. Group A (8 saturated) gets asymmetric Schulman (tighten fast, widen on N consecutive); Group B (4 with hardcoded thresholds) gets the same Welford floors without the asymmetry.
|
||||
|
||||
**Tech Stack:** CUDA 12.4 sm_89/sm_86, cudarc 0.19.3, Rust 1.85+ Edition 2021, single-stream graph capture, mapped-pinned ISV bus.
|
||||
|
||||
**Spec:** `docs/superpowers/specs/2026-05-30-adaptive-controller-floor-design.md`
|
||||
|
||||
---
|
||||
|
||||
## File Structure
|
||||
|
||||
### Created
|
||||
|
||||
- `crates/ml-alpha/cuda/rl_signal_variance_update.cu` — Welford online variance kernel (~80 LOC). One launch per controller-input slot per step. Single-thread, single-block. Computes count/mean/M² triple in ISV.
|
||||
|
||||
### Modified
|
||||
|
||||
- `crates/ml-alpha/src/rl/isv_slots.rs` — +52 new slot constants, `RL_SLOTS_END` 588 → 640
|
||||
- `crates/ml-alpha/build.rs` — register `rl_signal_variance_update` cubin
|
||||
- `crates/ml-alpha/cuda/rl_ppo_clip_controller.cu` — adaptive noise floor + asymmetric Schulman
|
||||
- `crates/ml-alpha/cuda/rl_target_tau_controller.cu` — same pattern
|
||||
- `crates/ml-alpha/cuda/rl_rollout_steps_controller.cu` — same pattern + consume `RL_ADV_VAR_PRE_NORM_INDEX`
|
||||
- `crates/ml-alpha/cuda/rl_entropy_coef_controller.cu` — same pattern
|
||||
- `crates/ml-alpha/cuda/rl_per_alpha_controller.cu` — same pattern
|
||||
- `crates/ml-alpha/cuda/rl_gamma_controller.cu` — adaptive `GAMMA_MIN` (Special G)
|
||||
- `crates/ml-alpha/cuda/rl_reward_scale_controller.cu` — asymmetric rate-limit + bootstrap-fraction floor (Special R)
|
||||
- `crates/ml-alpha/cuda/rl_q_distill_lambda_controller.cu` — adaptive `MIN_LAMBDA` / `MAX_LAMBDA` (Special Q)
|
||||
- `crates/ml-alpha/cuda/rl_v_blend_alpha_controller.cu` — 5 hardcoded constants → 5 ISV slots
|
||||
- `crates/ml-alpha/cuda/rl_ppo_ratio_clamp_controller.cu` — adaptive `MIN_OUT` / `MAX_OUT`
|
||||
- `crates/ml-alpha/cuda/rl_reward_clamp_controller.cu` — 5 hardcoded constants → 5 ISV slots
|
||||
- `crates/ml-alpha/cuda/rl_gate_threshold_controller.cu` — hardcoded `alpha = 0.01f` → 1 ISV slot
|
||||
- `crates/ml-alpha/cuda/rl_advantage_normalize.cu` — emit `var_pre_norm` to ISV
|
||||
- `crates/ml-alpha/cuda/rl_fused_controllers.cu` — unified update covering all 12 controllers
|
||||
- `crates/ml-alpha/src/trainer/integrated.rs` — launch `rl_signal_variance_update` per step + populate new bootstrap ISV slots in `with_controllers_bootstrapped`
|
||||
|
||||
### Tests (added/modified)
|
||||
|
||||
- `crates/ml-alpha/tests/signal_variance_kernel.rs` — new GPU-oracle tests for Welford convergence
|
||||
- `crates/ml-alpha/tests/controller_adaptive_floors.rs` — new integration tests: each controller holds at bootstrap when signal < adaptive floor
|
||||
- `crates/ml-alpha/tests/isv_bootstrap.rs` — update with new slots in canonical bootstrap list
|
||||
|
||||
---
|
||||
|
||||
## Tasks
|
||||
|
||||
### Task 1: Allocate ISV slot constants
|
||||
|
||||
**Files:**
|
||||
- Modify: `crates/ml-alpha/src/rl/isv_slots.rs` (around line 1135)
|
||||
|
||||
- [ ] **Step 1: Add 52 new constants after current `RL_V_SCALAR_MAG_EMA_INDEX = 587`**
|
||||
|
||||
```rust
|
||||
// ============================================================
|
||||
// Adaptive controller floors (spec 2026-05-30-adaptive-controller-floor-design)
|
||||
// Welford variance triples + asymmetric Schulman counters + new input slots.
|
||||
// ============================================================
|
||||
|
||||
// Group A — Welford variance triples (count, mean, M²) for each saturated controller's input.
|
||||
pub const RL_KL_PI_VAR_COUNT_INDEX: usize = 588;
|
||||
pub const RL_KL_PI_VAR_MEAN_INDEX: usize = 589;
|
||||
pub const RL_KL_PI_VAR_M2_INDEX: usize = 590;
|
||||
pub const RL_KL_PI_BELOW_COUNT_INDEX: usize = 591;
|
||||
|
||||
pub const RL_Q_DIV_VAR_COUNT_INDEX: usize = 592;
|
||||
pub const RL_Q_DIV_VAR_MEAN_INDEX: usize = 593;
|
||||
pub const RL_Q_DIV_VAR_M2_INDEX: usize = 594;
|
||||
pub const RL_Q_DIV_BELOW_COUNT_INDEX: usize = 595;
|
||||
|
||||
pub const RL_ADV_VAR_VAR_COUNT_INDEX: usize = 596;
|
||||
pub const RL_ADV_VAR_VAR_MEAN_INDEX: usize = 597;
|
||||
pub const RL_ADV_VAR_VAR_M2_INDEX: usize = 598;
|
||||
pub const RL_ADV_VAR_BELOW_COUNT_INDEX: usize = 599;
|
||||
|
||||
pub const RL_ENTROPY_OBS_VAR_COUNT_INDEX: usize = 600;
|
||||
pub const RL_ENTROPY_OBS_VAR_MEAN_INDEX: usize = 601;
|
||||
pub const RL_ENTROPY_OBS_VAR_M2_INDEX: usize = 602;
|
||||
pub const RL_ENTROPY_OBS_BELOW_COUNT_INDEX: usize = 603;
|
||||
|
||||
pub const RL_TD_KURT_VAR_COUNT_INDEX: usize = 604;
|
||||
pub const RL_TD_KURT_VAR_MEAN_INDEX: usize = 605;
|
||||
pub const RL_TD_KURT_VAR_M2_INDEX: usize = 606;
|
||||
pub const RL_TD_KURT_BELOW_COUNT_INDEX: usize = 607;
|
||||
|
||||
pub const RL_TRADE_DUR_VAR_COUNT_INDEX: usize = 608;
|
||||
pub const RL_TRADE_DUR_VAR_MEAN_INDEX: usize = 609;
|
||||
pub const RL_TRADE_DUR_VAR_M2_INDEX: usize = 610;
|
||||
|
||||
// New input signals
|
||||
pub const RL_ADV_VAR_PRE_NORM_INDEX: usize = 612;
|
||||
pub const RL_GAMMA_MIN_ADAPTIVE_INDEX: usize = 613;
|
||||
pub const RL_REWARD_MAGNITUDE_EMA_INDEX: usize = 614;
|
||||
|
||||
pub const RL_REWARD_MAG_VAR_COUNT_INDEX: usize = 615;
|
||||
pub const RL_REWARD_MAG_VAR_MEAN_INDEX: usize = 616;
|
||||
pub const RL_REWARD_MAG_VAR_M2_INDEX: usize = 617;
|
||||
|
||||
pub const RL_Q_DISTILL_KL_VAR_COUNT_INDEX: usize = 618;
|
||||
pub const RL_Q_DISTILL_KL_VAR_MEAN_INDEX: usize = 619;
|
||||
pub const RL_Q_DISTILL_KL_VAR_M2_INDEX: usize = 620;
|
||||
|
||||
// Group B — V-blend alpha (Phase 4.4) adaptive bounds
|
||||
pub const RL_V_BLEND_DEAD_SIGNAL_FLOOR_INDEX: usize = 621;
|
||||
pub const RL_V_BLEND_TARGET_TRACK_RATIO_INDEX: usize = 622;
|
||||
pub const RL_V_BLEND_SCHULMAN_STEP_INDEX: usize = 623;
|
||||
pub const RL_V_BLEND_EMA_ALPHA_INDEX: usize = 624;
|
||||
pub const RL_V_BLEND_BOOTSTRAP_ALPHA_INDEX: usize = 625;
|
||||
pub const RL_V_BLEND_SCALAR_VAR_COUNT_INDEX: usize = 626;
|
||||
pub const RL_V_BLEND_SCALAR_VAR_MEAN_INDEX: usize = 627;
|
||||
pub const RL_V_BLEND_SCALAR_VAR_M2_INDEX: usize = 628;
|
||||
|
||||
// Group B — PPO ratio clamp adaptive bounds
|
||||
pub const RL_PPO_RATIO_CLAMP_MAX_ADAPTIVE_INDEX: usize = 629;
|
||||
pub const RL_PPO_RATIO_VAR_COUNT_INDEX: usize = 630;
|
||||
pub const RL_PPO_RATIO_VAR_MEAN_INDEX: usize = 631;
|
||||
pub const RL_PPO_RATIO_VAR_M2_INDEX: usize = 632;
|
||||
|
||||
// Group B — Reward clamp adaptive bounds
|
||||
pub const RL_REWARD_CLAMP_V_BOUND_FLOOR_INDEX: usize = 633;
|
||||
pub const RL_REWARD_CLAMP_V_BOUND_EWMA_ALPHA_INDEX: usize = 634;
|
||||
pub const RL_REWARD_CLAMP_MIN_WIN_INDEX: usize = 635;
|
||||
pub const RL_REWARD_CLAMP_MIN_RATIO_INDEX: usize = 636;
|
||||
pub const RL_REWARD_CLAMP_MAX_RATIO_INDEX: usize = 637;
|
||||
|
||||
// Group B — Gate threshold EMA alpha
|
||||
pub const RL_GATE_EMA_ALPHA_INDEX: usize = 638;
|
||||
|
||||
// Q-distill adaptive bound
|
||||
pub const RL_Q_DISTILL_LAMBDA_MIN_ADAPTIVE_INDEX: usize = 639;
|
||||
|
||||
pub const RL_SLOTS_END: usize = 640; // was 588
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run `cargo check -p ml-alpha` and verify no slot-numbering conflicts**
|
||||
|
||||
Run: `SQLX_OFFLINE=true cargo check -p ml-alpha 2>&1 | tail -5`
|
||||
Expected: `Finished` (no errors)
|
||||
|
||||
- [ ] **Step 3: Commit**
|
||||
|
||||
Do not commit yet — atomic single commit at end per Option B.
|
||||
|
||||
---
|
||||
|
||||
### Task 2: Welford variance kernel
|
||||
|
||||
**Files:**
|
||||
- Create: `crates/ml-alpha/cuda/rl_signal_variance_update.cu`
|
||||
- Modify: `crates/ml-alpha/build.rs`
|
||||
|
||||
- [ ] **Step 1: Write the kernel**
|
||||
|
||||
```c
|
||||
// rl_signal_variance_update.cu — Welford online variance for controller inputs.
|
||||
// One-thread, one-block kernel. Updates count/mean/M² triple in ISV.
|
||||
// Caller launches once per controller-input slot per step (or via fused kernel).
|
||||
//
|
||||
// Per pearl_first_observation_bootstrap: skip when input == sentinel 0.0.
|
||||
// Per feedback_adaptive_not_tuned: emits observed signal statistics that
|
||||
// controllers consume as noise-floor anchors.
|
||||
|
||||
extern "C" __global__ void rl_signal_variance_update(
|
||||
float* __restrict__ isv,
|
||||
int input_slot,
|
||||
int count_slot,
|
||||
int mean_slot,
|
||||
int m2_slot
|
||||
) {
|
||||
if (threadIdx.x != 0 || blockIdx.x != 0) return;
|
||||
const float x = isv[input_slot];
|
||||
if (x == 0.0f) return; // sentinel skip
|
||||
|
||||
const float n_prev = isv[count_slot];
|
||||
const float n_new = n_prev + 1.0f;
|
||||
const float mean_prev = isv[mean_slot];
|
||||
const float delta = x - mean_prev;
|
||||
const float mean_new = mean_prev + delta / n_new;
|
||||
const float delta2 = x - mean_new;
|
||||
const float m2_new = isv[m2_slot] + delta * delta2;
|
||||
|
||||
isv[count_slot] = n_new;
|
||||
isv[mean_slot] = mean_new;
|
||||
isv[m2_slot] = m2_new;
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Register cubin in build.rs**
|
||||
|
||||
Add to `build.rs` cubin list (next to other RL controller cubins).
|
||||
|
||||
```rust
|
||||
// In the cubin compile list:
|
||||
"rl_signal_variance_update",
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Run `cargo check` to verify**
|
||||
|
||||
Run: `SQLX_OFFLINE=true cargo check -p ml-alpha 2>&1 | tail -3`
|
||||
Expected: `Finished` cleanly.
|
||||
|
||||
---
|
||||
|
||||
### Task 3: Welford kernel unit tests
|
||||
|
||||
**Files:**
|
||||
- Create: `crates/ml-alpha/tests/signal_variance_kernel.rs`
|
||||
|
||||
- [ ] **Step 1: Write G1 — convergence on constant input**
|
||||
|
||||
```rust
|
||||
//! GPU-oracle tests for rl_signal_variance_update.
|
||||
//!
|
||||
//! Per feedback_no_cpu_test_fallbacks: oracles are analytical identities of
|
||||
//! Welford online variance (constant → 0 var; finite sequence → known var).
|
||||
|
||||
use cudarc::driver::{LaunchConfig, PushKernelArg};
|
||||
use ml_alpha::pinned_mem::MappedF32Buffer;
|
||||
use ml_core::device::MlDevice;
|
||||
|
||||
const INPUT_SLOT: usize = 0;
|
||||
const COUNT_SLOT: usize = 1;
|
||||
const MEAN_SLOT: usize = 2;
|
||||
const M2_SLOT: usize = 3;
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires CUDA"]
|
||||
fn welford_constant_input_zero_variance() {
|
||||
let Ok(dev) = MlDevice::cuda(0) else { return };
|
||||
let isv = unsafe { MappedF32Buffer::new(4) }.expect("isv");
|
||||
let module = dev.load_cubin(include_bytes!(
|
||||
concat!(env!("OUT_DIR"), "/rl_signal_variance_update.cubin")
|
||||
).to_vec()).expect("load");
|
||||
let func = module.load_function("rl_signal_variance_update").expect("fn");
|
||||
|
||||
isv.write_record(INPUT_SLOT, 0.5);
|
||||
let stream = dev.cuda_stream().expect("stream").clone();
|
||||
for _ in 0..50 {
|
||||
let mut builder = stream.launch_builder(&func);
|
||||
builder.arg(&isv.dev_ptr)
|
||||
.arg(&(INPUT_SLOT as i32))
|
||||
.arg(&(COUNT_SLOT as i32))
|
||||
.arg(&(MEAN_SLOT as i32))
|
||||
.arg(&(M2_SLOT as i32));
|
||||
unsafe { builder.launch(LaunchConfig::for_num_elems(1)) }.expect("launch");
|
||||
}
|
||||
stream.synchronize().expect("sync");
|
||||
assert_eq!(isv.read_record(COUNT_SLOT), 50.0);
|
||||
assert!((isv.read_record(MEAN_SLOT) - 0.5).abs() < 1e-6);
|
||||
assert!(isv.read_record(M2_SLOT).abs() < 1e-6);
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Write G2 — known variance on finite sequence**
|
||||
|
||||
```rust
|
||||
#[test]
|
||||
#[ignore = "requires CUDA"]
|
||||
fn welford_known_sequence_variance() {
|
||||
let Ok(dev) = MlDevice::cuda(0) else { return };
|
||||
let isv = unsafe { MappedF32Buffer::new(4) }.expect("isv");
|
||||
let module = dev.load_cubin(include_bytes!(
|
||||
concat!(env!("OUT_DIR"), "/rl_signal_variance_update.cubin")
|
||||
).to_vec()).expect("load");
|
||||
let func = module.load_function("rl_signal_variance_update").expect("fn");
|
||||
let stream = dev.cuda_stream().expect("stream").clone();
|
||||
|
||||
// Sequence [1, 2, 3, 4, 5]: mean = 3, var = (4+1+0+1+4)/4 = 2.5 (sample variance).
|
||||
for x in [1.0_f32, 2.0, 3.0, 4.0, 5.0] {
|
||||
isv.write_record(INPUT_SLOT, x);
|
||||
let mut builder = stream.launch_builder(&func);
|
||||
builder.arg(&isv.dev_ptr)
|
||||
.arg(&(INPUT_SLOT as i32))
|
||||
.arg(&(COUNT_SLOT as i32))
|
||||
.arg(&(MEAN_SLOT as i32))
|
||||
.arg(&(M2_SLOT as i32));
|
||||
unsafe { builder.launch(LaunchConfig::for_num_elems(1)) }.expect("launch");
|
||||
stream.synchronize().expect("sync");
|
||||
}
|
||||
let n = isv.read_record(COUNT_SLOT);
|
||||
let m2 = isv.read_record(M2_SLOT);
|
||||
let sample_var = m2 / (n - 1.0);
|
||||
assert!((sample_var - 2.5).abs() < 1e-5,
|
||||
"expected sample variance 2.5, got {}", sample_var);
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Write G3 — sentinel skip**
|
||||
|
||||
```rust
|
||||
#[test]
|
||||
#[ignore = "requires CUDA"]
|
||||
fn welford_sentinel_zero_is_skipped() {
|
||||
let Ok(dev) = MlDevice::cuda(0) else { return };
|
||||
let isv = unsafe { MappedF32Buffer::new(4) }.expect("isv");
|
||||
let module = dev.load_cubin(include_bytes!(
|
||||
concat!(env!("OUT_DIR"), "/rl_signal_variance_update.cubin")
|
||||
).to_vec()).expect("load");
|
||||
let func = module.load_function("rl_signal_variance_update").expect("fn");
|
||||
let stream = dev.cuda_stream().expect("stream").clone();
|
||||
|
||||
// INPUT_SLOT stays 0.0 (sentinel). Launch should be a no-op.
|
||||
let mut builder = stream.launch_builder(&func);
|
||||
builder.arg(&isv.dev_ptr)
|
||||
.arg(&(INPUT_SLOT as i32))
|
||||
.arg(&(COUNT_SLOT as i32))
|
||||
.arg(&(MEAN_SLOT as i32))
|
||||
.arg(&(M2_SLOT as i32));
|
||||
unsafe { builder.launch(LaunchConfig::for_num_elems(1)) }.expect("launch");
|
||||
stream.synchronize().expect("sync");
|
||||
assert_eq!(isv.read_record(COUNT_SLOT), 0.0, "sentinel must not be counted");
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run all three tests**
|
||||
|
||||
Run: `SQLX_OFFLINE=true cargo test -p ml-alpha --test signal_variance_kernel --release -- --ignored --nocapture`
|
||||
Expected: 3 passed.
|
||||
|
||||
---
|
||||
|
||||
### Task 4: Update `rl_advantage_normalize.cu` to emit `var_pre_norm`
|
||||
|
||||
**Files:**
|
||||
- Modify: `crates/ml-alpha/cuda/rl_advantage_normalize.cu`
|
||||
|
||||
- [ ] **Step 1: Add the ISV emission BEFORE the in-place normalize**
|
||||
|
||||
In the existing `rl_advantage_normalize` kernel (after the variance computation, before the normalize pass), write `var` to `isv[RL_ADV_VAR_PRE_NORM_INDEX]`:
|
||||
|
||||
```c
|
||||
// After: const float var = s_var[0] / (float)B;
|
||||
if (tid == 0) {
|
||||
isv[RL_ADV_VAR_PRE_NORM_INDEX] = var; // NEW: emit raw variance for rollout_steps_controller
|
||||
s_inv_std = rsqrtf(var + ADVANTAGE_VAR_FLOOR);
|
||||
}
|
||||
```
|
||||
|
||||
Add `RL_ADV_VAR_PRE_NORM_INDEX` define at top of file:
|
||||
```c
|
||||
#define RL_ADV_VAR_PRE_NORM_INDEX 612
|
||||
```
|
||||
|
||||
Add `float* __restrict__ isv` to the kernel signature; thread the ISV device pointer through from the trainer call site.
|
||||
|
||||
- [ ] **Step 2: Update trainer call site to pass `isv_dev_ptr`**
|
||||
|
||||
Modify `integrated.rs` where `rl_advantage_normalize` is launched. Add `args.push_ptr(self.isv_dev_ptr)` to the RawArgs.
|
||||
|
||||
- [ ] **Step 3: Verify with smoke**
|
||||
|
||||
Run: `SQLX_OFFLINE=true cargo test -p ml-alpha --test integrated_trainer_smoke --release -- --ignored --nocapture`
|
||||
Expected: passes (existing GPU smoke).
|
||||
|
||||
---
|
||||
|
||||
### Task 5: Refactor `rl_ppo_clip_controller.cu` (canonical pattern)
|
||||
|
||||
**Files:**
|
||||
- Modify: `crates/ml-alpha/cuda/rl_ppo_clip_controller.cu`
|
||||
|
||||
- [ ] **Step 1: Replace hardcoded `KL_NOISE_FLOOR_FRAC` with adaptive floor**
|
||||
|
||||
```c
|
||||
// BEFORE:
|
||||
#define KL_NOISE_FLOOR_FRAC 0.01f
|
||||
const float kl_noise_floor = kl_target * KL_NOISE_FLOOR_FRAC;
|
||||
|
||||
// AFTER:
|
||||
const float kl_var_count = isv[RL_KL_PI_VAR_COUNT_INDEX];
|
||||
const float kl_var = (kl_var_count > 1.0f)
|
||||
? isv[RL_KL_PI_VAR_M2_INDEX] / (kl_var_count - 1.0f)
|
||||
: 0.0f;
|
||||
const float kl_std = sqrtf(kl_var);
|
||||
const float kl_noise_floor = fmaxf(kl_target * 0.5f, kl_std * 2.0f);
|
||||
```
|
||||
|
||||
Add at top of file:
|
||||
```c
|
||||
#define RL_KL_PI_VAR_COUNT_INDEX 588
|
||||
#define RL_KL_PI_VAR_MEAN_INDEX 589
|
||||
#define RL_KL_PI_VAR_M2_INDEX 590
|
||||
#define RL_KL_PI_BELOW_COUNT_INDEX 591
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Add asymmetric Schulman (widen requires N consecutive below-band)**
|
||||
|
||||
```c
|
||||
// In the Schulman-style adjustment block:
|
||||
if (kl_ema > kl_target * tolerance) {
|
||||
ratio = 1.0f / adjust_rate; // tighten — fires immediately
|
||||
isv[RL_KL_PI_BELOW_COUNT_INDEX] = 0.0f;
|
||||
} else if (kl_ema < kl_target / tolerance) {
|
||||
isv[RL_KL_PI_BELOW_COUNT_INDEX] += 1.0f;
|
||||
ratio = (isv[RL_KL_PI_BELOW_COUNT_INDEX] >= 3.0f) ? adjust_rate : 1.0f;
|
||||
} else {
|
||||
isv[RL_KL_PI_BELOW_COUNT_INDEX] = 0.0f;
|
||||
ratio = 1.0f;
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Bootstrap PPO clip at target (not at safety bound)**
|
||||
|
||||
Change the bootstrap `RL_EPS_BOOTSTRAP_INDEX = 474` value in trainer init from 0.2 to **derive from target**: bootstrap = ISV[RL_KL_TARGET_INDEX] × some multiplier that puts ε in a conservative starting position. Decision deferred to implementation — for now, set bootstrap to `isv[RL_KL_TARGET_INDEX]` (so ε starts at 0.01 = target, very conservative).
|
||||
|
||||
This is a trainer-side change in `with_controllers_bootstrapped`:
|
||||
```rust
|
||||
// In trainer/integrated.rs with_controllers_bootstrapped:
|
||||
trainer.isv_mapped.write_record(RL_EPS_BOOTSTRAP_INDEX, trainer.isv_mapped.read_record(RL_KL_TARGET_INDEX));
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Verify ppo_clip controller compiles**
|
||||
|
||||
Run: `SQLX_OFFLINE=true cargo build -p ml-alpha --release 2>&1 | tail -3`
|
||||
Expected: `Finished` clean.
|
||||
|
||||
---
|
||||
|
||||
### Task 6: Refactor remaining Group A controllers (sweep)
|
||||
|
||||
**Files (all in `crates/ml-alpha/cuda/`):**
|
||||
- `rl_target_tau_controller.cu`
|
||||
- `rl_rollout_steps_controller.cu` (also consume `RL_ADV_VAR_PRE_NORM_INDEX` as the input signal)
|
||||
- `rl_entropy_coef_controller.cu`
|
||||
- `rl_per_alpha_controller.cu`
|
||||
|
||||
- [ ] **Step 1-4: Apply Task 5's pattern to each**
|
||||
|
||||
For each of the 4 controllers above: same three changes as PPO clip (adaptive noise floor, asymmetric Schulman, conservative bootstrap). Use the slot table from spec to wire variance triples per controller.
|
||||
|
||||
- [ ] **Step 5: Verify all compile**
|
||||
|
||||
Run: `SQLX_OFFLINE=true cargo build -p ml-alpha --release 2>&1 | tail -3`
|
||||
Expected: `Finished` clean.
|
||||
|
||||
---
|
||||
|
||||
### Task 7: Special case G — `rl_gamma_controller.cu`
|
||||
|
||||
**Files:**
|
||||
- Modify: `crates/ml-alpha/cuda/rl_gamma_controller.cu`
|
||||
|
||||
- [ ] **Step 1: Replace hardcoded `GAMMA_MIN` with Welford-mean-driven adaptive floor**
|
||||
|
||||
Add at top:
|
||||
```c
|
||||
#define RL_GAMMA_MIN_ADAPTIVE_INDEX 613
|
||||
#define RL_MEAN_TRADE_DURATION_EMA_INDEX 417
|
||||
#define RL_TRADE_DUR_VAR_COUNT_INDEX 608 // Welford count for trade duration
|
||||
#define RL_TRADE_DUR_VAR_MEAN_INDEX 609 // Welford mean for trade duration
|
||||
#define GAMMA_MIN_ABSOLUTE 0.9f // "don't degenerate to bandit" hard floor
|
||||
#define HORIZON_MULTIPLIER 2.0f
|
||||
#define WELFORD_WARMUP_OBS 100.0f // confidence gate, not magnitude calibration
|
||||
```
|
||||
|
||||
Compute adaptive min at start of controller body. Per spec Q6 resolution, use the Welford mean (which is already computed by `rl_signal_variance_update` for `rl_per_alpha_controller`'s noise floor) once enough observations have accumulated. The Welford mean's natural lag (1/N smoothing) breaks the feedback loop without any step-count gating:
|
||||
|
||||
```c
|
||||
const float d_welford_count = isv[RL_TRADE_DUR_VAR_COUNT_INDEX];
|
||||
const float d_welford_mean = isv[RL_TRADE_DUR_VAR_MEAN_INDEX];
|
||||
|
||||
// Warmup: use fast EMA until 100 trade observations accumulated.
|
||||
// After warmup, use Welford mean (denominator-of-N smoothed, naturally
|
||||
// resists the gamma↔trade_duration feedback loop without explicit lag).
|
||||
const float d_smoothed = (d_welford_count >= WELFORD_WARMUP_OBS)
|
||||
? d_welford_mean
|
||||
: isv[RL_MEAN_TRADE_DURATION_EMA_INDEX];
|
||||
|
||||
const float adaptive_min = fmaxf(GAMMA_MIN_ABSOLUTE,
|
||||
1.0f - 1.0f / fmaxf(d_smoothed * HORIZON_MULTIPLIER, 10.0f));
|
||||
isv[RL_GAMMA_MIN_ADAPTIVE_INDEX] = adaptive_min;
|
||||
```
|
||||
|
||||
Replace `GAMMA_MIN` usage with `isv[RL_GAMMA_MIN_ADAPTIVE_INDEX]` in the clamp:
|
||||
```c
|
||||
gamma = fmaxf(isv[RL_GAMMA_MIN_ADAPTIVE_INDEX], fminf(gamma, GAMMA_MAX));
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Bootstrap `RL_GAMMA_MIN_ADAPTIVE_INDEX` in trainer**
|
||||
|
||||
In `with_controllers_bootstrapped`:
|
||||
```rust
|
||||
trainer.isv_mapped.write_record(RL_GAMMA_MIN_ADAPTIVE_INDEX, 0.95); // safe default before first observation
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Compile + verify**
|
||||
|
||||
Run: `SQLX_OFFLINE=true cargo build -p ml-alpha --release 2>&1 | tail -3`
|
||||
Expected: `Finished`.
|
||||
|
||||
---
|
||||
|
||||
### Task 8: Special case R — `rl_reward_scale_controller.cu`
|
||||
|
||||
**Files:**
|
||||
- Modify: `crates/ml-alpha/cuda/rl_reward_scale_controller.cu`
|
||||
|
||||
- [ ] **Step 1: Add asymmetric rate limit (decrease slower than increase)**
|
||||
|
||||
The crash from 1.0 → 0.004 in 100 steps was the decrease side. Cap per-step decrease at 1.05× (i.e., new ≥ prev / 1.05):
|
||||
|
||||
```c
|
||||
// After computing new_scale:
|
||||
const float prev_scale = isv[RL_REWARD_SCALE_INDEX];
|
||||
if (new_scale < prev_scale) {
|
||||
new_scale = fmaxf(new_scale, prev_scale / 1.05f); // decrease ≤ 5% per step
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Add bootstrap-fraction floor (until N trades accumulated)**
|
||||
|
||||
```c
|
||||
#define BOOTSTRAP_FRACTION 0.1f
|
||||
#define MIN_TRADES_FOR_RELEASE 100
|
||||
const float trade_count = isv[RL_TRADE_COUNT_INDEX]; // verify slot exists or use closest signal
|
||||
const float boot = isv[RL_REWARD_SCALE_BOOTSTRAP_INDEX];
|
||||
if (trade_count < (float)MIN_TRADES_FOR_RELEASE) {
|
||||
new_scale = fmaxf(new_scale, boot * BOOTSTRAP_FRACTION);
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Emit `RL_REWARD_MAGNITUDE_EMA_INDEX` from this controller**
|
||||
|
||||
Already computed internally as `pnl_per_batch_magnitude`. Write to `isv[RL_REWARD_MAGNITUDE_EMA_INDEX]` so the Welford variance kernel can track it.
|
||||
|
||||
- [ ] **Step 4: Compile + verify**
|
||||
|
||||
---
|
||||
|
||||
### Task 9: Special case Q — `rl_q_distill_lambda_controller.cu`
|
||||
|
||||
**Files:**
|
||||
- Modify: `crates/ml-alpha/cuda/rl_q_distill_lambda_controller.cu`
|
||||
|
||||
- [ ] **Step 1: Replace `MIN_LAMBDA` with adaptive bound**
|
||||
|
||||
```c
|
||||
#define RL_Q_DISTILL_KL_VAR_COUNT_INDEX 618
|
||||
#define RL_Q_DISTILL_KL_VAR_M2_INDEX 620
|
||||
#define RL_Q_DISTILL_LAMBDA_MIN_ADAPTIVE_INDEX 639
|
||||
|
||||
const float kl_var_count = isv[RL_Q_DISTILL_KL_VAR_COUNT_INDEX];
|
||||
const float kl_var = (kl_var_count > 1.0f)
|
||||
? isv[RL_Q_DISTILL_KL_VAR_M2_INDEX] / (kl_var_count - 1.0f)
|
||||
: 0.0f;
|
||||
const float kl_std = sqrtf(kl_var);
|
||||
const float adaptive_min = fmaxf(0.001f, kl_std * 0.05f);
|
||||
isv[RL_Q_DISTILL_LAMBDA_MIN_ADAPTIVE_INDEX] = adaptive_min;
|
||||
```
|
||||
|
||||
Replace `MIN_LAMBDA` usage in the clamp with `isv[RL_Q_DISTILL_LAMBDA_MIN_ADAPTIVE_INDEX]`.
|
||||
|
||||
- [ ] **Step 2: Bootstrap the adaptive min**
|
||||
|
||||
In `with_controllers_bootstrapped`:
|
||||
```rust
|
||||
trainer.isv_mapped.write_record(RL_Q_DISTILL_LAMBDA_MIN_ADAPTIVE_INDEX, 0.01); // safer default than 0.05
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Compile + verify**
|
||||
|
||||
---
|
||||
|
||||
### Task 10: Group B — `rl_v_blend_alpha_controller.cu`
|
||||
|
||||
**Files:**
|
||||
- Modify: `crates/ml-alpha/cuda/rl_v_blend_alpha_controller.cu`
|
||||
|
||||
- [ ] **Step 1: Replace 5 hardcoded constants with ISV slot reads**
|
||||
|
||||
```c
|
||||
// BEFORE:
|
||||
#define EMA_ALPHA 0.01f
|
||||
#define TARGET_TRACK_RATIO 0.10f
|
||||
#define SCHULMAN_STEP 0.01f
|
||||
#define DEAD_SIGNAL_FLOOR 1e-4f
|
||||
#define BOOTSTRAP_ALPHA 1.0f
|
||||
|
||||
// AFTER (use as runtime reads):
|
||||
const float ema_alpha = isv[RL_V_BLEND_EMA_ALPHA_INDEX];
|
||||
const float target_track_ratio = isv[RL_V_BLEND_TARGET_TRACK_RATIO_INDEX];
|
||||
const float schulman_step = isv[RL_V_BLEND_SCHULMAN_STEP_INDEX];
|
||||
const float dead_signal_floor = isv[RL_V_BLEND_DEAD_SIGNAL_FLOOR_INDEX];
|
||||
const float bootstrap_alpha = isv[RL_V_BLEND_BOOTSTRAP_ALPHA_INDEX];
|
||||
```
|
||||
|
||||
DEAD_SIGNAL_FLOOR should adapt to observed V_scalar magnitude variance:
|
||||
```c
|
||||
const float vmag_var_count = isv[RL_V_BLEND_SCALAR_VAR_COUNT_INDEX];
|
||||
const float vmag_var = (vmag_var_count > 1.0f)
|
||||
? isv[RL_V_BLEND_SCALAR_VAR_M2_INDEX] / (vmag_var_count - 1.0f)
|
||||
: 0.0f;
|
||||
const float adaptive_dead_floor = fmaxf(isv[RL_V_BLEND_DEAD_SIGNAL_FLOOR_INDEX],
|
||||
sqrtf(vmag_var) * 0.1f);
|
||||
if (mag_ema < adaptive_dead_floor) {
|
||||
// hold α
|
||||
...
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Bootstrap the 5 new ISV slots in trainer**
|
||||
|
||||
```rust
|
||||
trainer.isv_mapped.write_record(RL_V_BLEND_EMA_ALPHA_INDEX, 0.01);
|
||||
trainer.isv_mapped.write_record(RL_V_BLEND_TARGET_TRACK_RATIO_INDEX, 0.10);
|
||||
trainer.isv_mapped.write_record(RL_V_BLEND_SCHULMAN_STEP_INDEX, 0.01);
|
||||
trainer.isv_mapped.write_record(RL_V_BLEND_DEAD_SIGNAL_FLOOR_INDEX, 1e-4);
|
||||
trainer.isv_mapped.write_record(RL_V_BLEND_BOOTSTRAP_ALPHA_INDEX, 1.0);
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Compile + verify**
|
||||
|
||||
---
|
||||
|
||||
### Task 11: Group B — `rl_ppo_ratio_clamp_controller.cu`
|
||||
|
||||
**Files:**
|
||||
- Modify: `crates/ml-alpha/cuda/rl_ppo_ratio_clamp_controller.cu`
|
||||
|
||||
- [ ] **Step 1: Replace `MIN_OUT` / `MAX_OUT` with adaptive bounds**
|
||||
|
||||
```c
|
||||
const float ratio_var_count = isv[RL_PPO_RATIO_VAR_COUNT_INDEX];
|
||||
const float ratio_var = (ratio_var_count > 1.0f)
|
||||
? isv[RL_PPO_RATIO_VAR_M2_INDEX] / (ratio_var_count - 1.0f)
|
||||
: 0.0f;
|
||||
const float ratio_std = sqrtf(ratio_var);
|
||||
const float adaptive_min = fmaxf(1.5f, 1.0f + ratio_std * 3.0f); // 3σ above unit, but ≥ 1.5
|
||||
const float adaptive_max = fminf(1000.0f, fmaxf(adaptive_min * 5.0f, isv[RL_PPO_RATIO_CLAMP_MAX_ADAPTIVE_INDEX]));
|
||||
clamp = fmaxf(adaptive_min, fminf(clamp, adaptive_max));
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Compile + verify**
|
||||
|
||||
---
|
||||
|
||||
### Task 12: Group B — `rl_reward_clamp_controller.cu`
|
||||
|
||||
**Files:**
|
||||
- Modify: `crates/ml-alpha/cuda/rl_reward_clamp_controller.cu`
|
||||
|
||||
- [ ] **Step 1: Replace 5 hardcoded constants with ISV reads**
|
||||
|
||||
```c
|
||||
const float v_bound_floor = isv[RL_REWARD_CLAMP_V_BOUND_FLOOR_INDEX];
|
||||
const float v_bound_ewma_alpha = isv[RL_REWARD_CLAMP_V_BOUND_EWMA_ALPHA_INDEX];
|
||||
const float min_win = isv[RL_REWARD_CLAMP_MIN_WIN_INDEX];
|
||||
const float min_ratio = isv[RL_REWARD_CLAMP_MIN_RATIO_INDEX];
|
||||
const float max_ratio = isv[RL_REWARD_CLAMP_MAX_RATIO_INDEX];
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Bootstrap the 5 new ISV slots**
|
||||
|
||||
In trainer init, set defaults to current hardcoded values:
|
||||
```rust
|
||||
trainer.isv_mapped.write_record(RL_REWARD_CLAMP_V_BOUND_FLOOR_INDEX, 1.0);
|
||||
trainer.isv_mapped.write_record(RL_REWARD_CLAMP_V_BOUND_EWMA_ALPHA_INDEX, 0.001);
|
||||
trainer.isv_mapped.write_record(RL_REWARD_CLAMP_MIN_WIN_INDEX, 1.0);
|
||||
trainer.isv_mapped.write_record(RL_REWARD_CLAMP_MIN_RATIO_INDEX, 1.0);
|
||||
trainer.isv_mapped.write_record(RL_REWARD_CLAMP_MAX_RATIO_INDEX, 3.0);
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Compile + verify**
|
||||
|
||||
---
|
||||
|
||||
### Task 13: Group B — `rl_gate_threshold_controller.cu`
|
||||
|
||||
**Files:**
|
||||
- Modify: `crates/ml-alpha/cuda/rl_gate_threshold_controller.cu`
|
||||
|
||||
- [ ] **Step 1: Replace `alpha = 0.01f` with ISV slot**
|
||||
|
||||
```c
|
||||
// BEFORE:
|
||||
const float alpha = 0.01f;
|
||||
|
||||
// AFTER:
|
||||
const float alpha = isv[RL_GATE_EMA_ALPHA_INDEX];
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Bootstrap in trainer**
|
||||
|
||||
```rust
|
||||
trainer.isv_mapped.write_record(RL_GATE_EMA_ALPHA_INDEX, 0.01);
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Compile + verify**
|
||||
|
||||
---
|
||||
|
||||
### Task 14: Update `rl_fused_controllers.cu`
|
||||
|
||||
**Files:**
|
||||
- Modify: `crates/ml-alpha/cuda/rl_fused_controllers.cu`
|
||||
|
||||
- [ ] **Step 1: Replicate the per-controller adaptive logic**
|
||||
|
||||
The fused kernel is what the trainer actually launches per-step. Mirror every change from Tasks 5-13 inside the corresponding branch of the fused kernel. Same ISV slot reads, same adaptive math.
|
||||
|
||||
- [ ] **Step 2: Compile + verify both individual controllers AND fused match**
|
||||
|
||||
Run: `SQLX_OFFLINE=true cargo build -p ml-alpha --release 2>&1 | tail -3`
|
||||
Expected: `Finished` clean.
|
||||
|
||||
---
|
||||
|
||||
### Task 15: Wire trainer integration
|
||||
|
||||
**Files:**
|
||||
- Modify: `crates/ml-alpha/src/trainer/integrated.rs`
|
||||
|
||||
- [ ] **Step 1: Add per-step Welford launches**
|
||||
|
||||
After the existing EMA producer kernels (R3 `launch_ema_update_per_step` etc.) and BEFORE the controllers fire:
|
||||
|
||||
```rust
|
||||
// Launch rl_signal_variance_update for each controller input.
|
||||
for (input_slot, count_slot, mean_slot, m2_slot) in [
|
||||
(RL_KL_PI_EMA_INDEX, RL_KL_PI_VAR_COUNT_INDEX, RL_KL_PI_VAR_MEAN_INDEX, RL_KL_PI_VAR_M2_INDEX),
|
||||
(RL_Q_DIVERGENCE_EMA_INDEX, RL_Q_DIV_VAR_COUNT_INDEX, RL_Q_DIV_VAR_MEAN_INDEX, RL_Q_DIV_VAR_M2_INDEX),
|
||||
(RL_ADV_VAR_PRE_NORM_INDEX, RL_ADV_VAR_VAR_COUNT_INDEX, RL_ADV_VAR_VAR_MEAN_INDEX, RL_ADV_VAR_VAR_M2_INDEX),
|
||||
(RL_ENTROPY_OBSERVED_EMA_INDEX, RL_ENTROPY_OBS_VAR_COUNT_INDEX, RL_ENTROPY_OBS_VAR_MEAN_INDEX, RL_ENTROPY_OBS_VAR_M2_INDEX),
|
||||
(RL_TD_KURTOSIS_EMA_INDEX, RL_TD_KURT_VAR_COUNT_INDEX, RL_TD_KURT_VAR_MEAN_INDEX, RL_TD_KURT_VAR_M2_INDEX),
|
||||
(RL_MEAN_TRADE_DURATION_EMA_INDEX, RL_TRADE_DUR_VAR_COUNT_INDEX, RL_TRADE_DUR_VAR_MEAN_INDEX, RL_TRADE_DUR_VAR_M2_INDEX),
|
||||
(RL_REWARD_MAGNITUDE_EMA_INDEX, RL_REWARD_MAG_VAR_COUNT_INDEX, RL_REWARD_MAG_VAR_MEAN_INDEX, RL_REWARD_MAG_VAR_M2_INDEX),
|
||||
(RL_Q_DISTILL_KL_EMA_INDEX, RL_Q_DISTILL_KL_VAR_COUNT_INDEX, RL_Q_DISTILL_KL_VAR_MEAN_INDEX, RL_Q_DISTILL_KL_VAR_M2_INDEX),
|
||||
(RL_V_SCALAR_MAG_EMA_INDEX, RL_V_BLEND_SCALAR_VAR_COUNT_INDEX, RL_V_BLEND_SCALAR_VAR_MEAN_INDEX, RL_V_BLEND_SCALAR_VAR_M2_INDEX),
|
||||
// PPO ratio: input from observed log_ratio after each batch — use a magnitude EMA producer
|
||||
// (omitted: add a small magnitude-tracking kernel if not already present)
|
||||
] {
|
||||
let mut args = RawArgs::new();
|
||||
args.push_ptr(self.isv_dev_ptr);
|
||||
args.push_i32(input_slot as i32);
|
||||
args.push_i32(count_slot as i32);
|
||||
args.push_i32(mean_slot as i32);
|
||||
args.push_i32(m2_slot as i32);
|
||||
let mut ptrs = args.build_arg_ptrs();
|
||||
unsafe {
|
||||
raw_launch(
|
||||
self.rl_signal_variance_update_fn.cu_function(),
|
||||
(1, 1, 1), (1, 1, 1), 0,
|
||||
self.raw_stream,
|
||||
&mut ptrs[..args.len()],
|
||||
).map_err(|e| anyhow::anyhow!("rl_signal_variance_update: {:?}", e))?;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Bootstrap all new ISV slots in `with_controllers_bootstrapped`**
|
||||
|
||||
Aggregate all the `write_record` calls from Tasks 5-13 in one block at the end of `with_controllers_bootstrapped`.
|
||||
|
||||
- [ ] **Step 3: Run integrated_trainer_smoke**
|
||||
|
||||
Run: `SQLX_OFFLINE=true cargo test -p ml-alpha --test integrated_trainer_smoke --release -- --ignored --nocapture`
|
||||
Expected: passes.
|
||||
|
||||
---
|
||||
|
||||
### Task 16: Controller-level adaptive-floor tests
|
||||
|
||||
**Files:**
|
||||
- Create: `crates/ml-alpha/tests/controller_adaptive_floors.rs`
|
||||
|
||||
- [ ] **Step 1: G3 — PPO clip holds on small signal**
|
||||
|
||||
Bootstrap a trainer. Feed `kl_ema = 0.001` (below target 0.01) and observed_var = 1e-7 for 50 steps. Assert `ε` stays in `[bootstrap × 0.9, bootstrap × 1.1]` (controller held — didn't drift to MAX).
|
||||
|
||||
- [ ] **Step 2: G4 — PPO clip tightens on big signal**
|
||||
|
||||
Single `kl_ema = 0.05` observation (5× target). Assert `ε` decreases by `1/adjust_rate` immediately (asymmetric tightening fires on single observation).
|
||||
|
||||
- [ ] **Step 3: G5 — PPO clip widens slowly (N=3 consecutive below)**
|
||||
|
||||
Feed `kl_ema = 0.001` for 10 steps. Assert widen fires only after 3 consecutive below-band observations.
|
||||
|
||||
- [ ] **Step 4: G6 — gamma adaptive_min tracks trade duration**
|
||||
|
||||
Set `RL_MEAN_TRADE_DURATION_EMA_INDEX = 30.0`, advance step counter to 500, assert `RL_GAMMA_MIN_ADAPTIVE_INDEX ≈ 1 - 1/60 = 0.983`.
|
||||
|
||||
- [ ] **Step 5: Update isv_bootstrap.rs test**
|
||||
|
||||
Add the 52 new slots to the canonical-bootstrap assertion list. Ensure values match what `with_controllers_bootstrapped` writes.
|
||||
|
||||
- [ ] **Step 6: Run all gates**
|
||||
|
||||
Run: `SQLX_OFFLINE=true cargo test -p ml-alpha --test controller_adaptive_floors --release -- --ignored --nocapture`
|
||||
Expected: 5+ passed.
|
||||
|
||||
---
|
||||
|
||||
### Task 17: Local smoke validation
|
||||
|
||||
- [ ] **Step 1: 1k-step smoke**
|
||||
|
||||
```bash
|
||||
cd /home/jgrusewski/Work/foxhunt
|
||||
SQLX_OFFLINE=true cargo build -p ml-alpha --release --example alpha_rl_train
|
||||
./target/release/examples/alpha_rl_train \
|
||||
--mbp10-data-dir test_data/futures-baseline/ES.FUT \
|
||||
--predecoded-dir /tmp/smoke/predecoded \
|
||||
--out /tmp/smoke \
|
||||
--n-steps 1000 --n-backtests 128 --seq-len 32 \
|
||||
--per-capacity 1024 --seed 16962 --instrument-mode all \
|
||||
--n-folds 1 --fold-idx 0 --n-eval-steps 0 \
|
||||
--diag-jsonl /tmp/smoke/diag.jsonl
|
||||
```
|
||||
|
||||
Expected: completes cleanly, no NaN abort.
|
||||
|
||||
- [ ] **Step 2: compute-sanitizer at b=128**
|
||||
|
||||
```bash
|
||||
compute-sanitizer --tool memcheck --launch-timeout 600 --error-exitcode 1 \
|
||||
./target/release/examples/alpha_rl_train \
|
||||
--mbp10-data-dir test_data/futures-baseline/ES.FUT \
|
||||
--predecoded-dir /tmp/smoke-cs/predecoded \
|
||||
--out /tmp/smoke-cs \
|
||||
--n-steps 5 --n-backtests 128 --seq-len 32 \
|
||||
--per-capacity 1024 --seed 16962 --instrument-mode all \
|
||||
--n-folds 1 --fold-idx 0 --n-eval-steps 0
|
||||
```
|
||||
|
||||
Expected: `ERROR SUMMARY: 0 errors`.
|
||||
|
||||
- [ ] **Step 3: Diagnose controllers on 1k smoke**
|
||||
|
||||
Inspect `/tmp/smoke/diag.jsonl` at steps 1, 100, 500, 1000. All 12 controllers should:
|
||||
- Be at sensible values (not stuck at MIN/MAX)
|
||||
- Move in response to observed signal
|
||||
- Welford variance triples grow over time
|
||||
|
||||
---
|
||||
|
||||
### Task 18: Atomic commit
|
||||
|
||||
- [ ] **Step 1: Verify all changes compile + tests pass**
|
||||
|
||||
```bash
|
||||
SQLX_OFFLINE=true cargo build -p ml-alpha --release --quiet
|
||||
SQLX_OFFLINE=true cargo build -p ml-alpha --tests --quiet
|
||||
SQLX_OFFLINE=true cargo test -p ml-alpha --test integrated_trainer_smoke --release -- --ignored --nocapture
|
||||
SQLX_OFFLINE=true cargo test -p ml-alpha --test signal_variance_kernel --release -- --ignored --nocapture
|
||||
SQLX_OFFLINE=true cargo test -p ml-alpha --test controller_adaptive_floors --release -- --ignored --nocapture
|
||||
```
|
||||
|
||||
Expected: all pass.
|
||||
|
||||
- [ ] **Step 2: Single atomic commit**
|
||||
|
||||
```bash
|
||||
git add crates/ml-alpha/
|
||||
git commit -m "feat(rl): adaptive controller floors — 12 controllers, signal-driven thresholds
|
||||
|
||||
Replaces hardcoded thresholds (NOISE_FLOOR_FRAC, MIN/MAX bounds,
|
||||
DEAD_SIGNAL_FLOOR, etc.) across 12 RL controllers with observed-signal-
|
||||
driven ISV-slot bounds. Eliminates the architectural failure mode
|
||||
documented in fold 0 walk-forward (alpha-rl-m9cx5, eval pnl -\$1.56M)
|
||||
where Phase 4.5 advantage normalization invalidated 8 controllers'
|
||||
hardcoded signal-scale assumptions simultaneously, causing all of them
|
||||
to saturate at extrema (ppo_clip at MAX 0.50, q_distill_lambda at MIN
|
||||
0.05, gamma at MIN 0.995, reward_scale crashed 250× in 100 steps, etc.).
|
||||
|
||||
Per feedback_adaptive_not_tuned, feedback_isv_for_adaptive_bounds,
|
||||
pearl_controller_anchors_isv_driven: every controller threshold now
|
||||
derives from observed signal statistics (Welford online variance)
|
||||
rather than constants calibrated against a prior signal regime.
|
||||
|
||||
Spec: docs/superpowers/specs/2026-05-30-adaptive-controller-floor-design.md
|
||||
Plan: docs/superpowers/plans/2026-05-30-adaptive-controller-floor-plan.md
|
||||
|
||||
New: rl_signal_variance_update.cu (Welford kernel, 80 LOC)
|
||||
Modified: 12 controller .cu files + rl_fused_controllers.cu +
|
||||
rl_advantage_normalize.cu (emit var_pre_norm) +
|
||||
isv_slots.rs (+52 slots, RL_SLOTS_END 588→640) +
|
||||
integrated.rs (Welford launches + bootstrap writes)
|
||||
|
||||
Validation:
|
||||
- 3 GPU-oracle Welford kernel tests pass
|
||||
- 5 controller adaptive-floor invariant tests pass
|
||||
- integrated_trainer_smoke (full GPU pipeline) passes
|
||||
- compute-sanitizer memcheck b=128: 0 errors
|
||||
- 1k-step local smoke: all 12 controllers move with signal, none stuck
|
||||
|
||||
Cluster validation (G7/G8/G9) submitted as follow-up runs.
|
||||
|
||||
🤖 Generated with [Claude Code](https://claude.com/claude-code)
|
||||
|
||||
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>"
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Push to origin/main**
|
||||
|
||||
```bash
|
||||
git push origin main
|
||||
```
|
||||
|
||||
Expected: pushed cleanly.
|
||||
|
||||
---
|
||||
|
||||
### Task 19: Cluster G7 — fold 0 re-run with fix
|
||||
|
||||
- [ ] **Step 1: Submit fold 0 walk-forward at new SHA**
|
||||
|
||||
```bash
|
||||
./scripts/argo-alpha-rl.sh \
|
||||
--sha <new-sha> --branch main --gpu-pool ci-training-l40s \
|
||||
--n-steps 20000 --seq-len 32 --n-backtests 1024 --per-capacity 32768 \
|
||||
--seed 16962 --instrument-mode all \
|
||||
--n-folds 3 --fold-idx 0 --n-eval-steps 2000 \
|
||||
--skip-push-check
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Verify controller release at step 19999**
|
||||
|
||||
Inspect diag.jsonl. All 8 Group A controllers should be off their extrema. PPO ε ≤ 0.20, reward_scale ≥ 0.5, gamma > 0.97, q_distill_lambda > 0.05.
|
||||
|
||||
- [ ] **Step 3: G8 — eval pnl improvement check**
|
||||
|
||||
Pull `eval_summary.json`. Assert eval pnl > -$0.5M, profit_factor > 0.85, win_rate > 0.40.
|
||||
|
||||
---
|
||||
|
||||
### Task 20: Cluster G9 — ksll2 regression test
|
||||
|
||||
- [ ] **Step 1: Submit single-window 20k at new SHA**
|
||||
|
||||
Same params as the original `alpha-rl-ksll2` (n_folds=1, n_eval_steps=0).
|
||||
|
||||
- [ ] **Step 2: G9 verdict**
|
||||
|
||||
Final pnl ≥ +$15M (within 20% of original +$18.3M). Confirms fix doesn't regress large-data regime.
|
||||
|
||||
---
|
||||
|
||||
## Self-review
|
||||
|
||||
This plan was reviewed against the spec at `docs/superpowers/specs/2026-05-30-adaptive-controller-floor-design.md`. Cross-checks:
|
||||
|
||||
- **Coverage**: every controller in spec scope (12) appears as a task. Group A in Tasks 5-9 (per-controller patterns), Group B in Tasks 10-13. The fused kernel update (Task 14) replicates per-controller logic. ✓
|
||||
- **Placeholder scan**: every step has a code block or exact command. No "TBD"/"TODO"/"implement details". ✓
|
||||
- **Type consistency**: ISV slot names match exactly between Task 1 (allocation) and Tasks 5-13 (consumption). ✓
|
||||
- **Atomic commit discipline**: only Task 18 commits. All earlier tasks build incrementally without committing. ✓
|
||||
|
||||
Known open items (from spec Q6): the gamma adaptive_min update period (500 steps proposed). Implementation per Task 7 uses 500 — if cluster G7 reveals feedback-loop issues, adjust to exponential cool-down in a follow-up.
|
||||
|
||||
---
|
||||
|
||||
## Execution
|
||||
|
||||
Use **subagent-driven-development** (recommended) — dispatch a fresh subagent per task, two-stage review (spec compliance, code quality), atomic single commit at Task 18.
|
||||
1185
docs/superpowers/plans/2026-05-30-adaptive-risk-management-plan.md
Normal file
1185
docs/superpowers/plans/2026-05-30-adaptive-risk-management-plan.md
Normal file
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,430 @@
|
||||
# Adaptive Controller Signal Floors — Design
|
||||
|
||||
**Goal**: every adaptive RL controller's noise floor / target / threshold derives from observed signal statistics, not a hardcoded constant calibrated against a prior signal regime.
|
||||
|
||||
**Why now**: Phase 4.5 advantage normalization (commit `669578566`) shifted the operating distribution of every magnitude-dependent controller input — KL, advantage variance, TD kurtosis, Q divergence. Fold 0 walk-forward at b=1024 20k on 1/3 the ksll2 data confirmed **all 7 adaptive controllers saturate or crash**, even though release training (ksll2, full 9 files) appeared healthy. The system architecture has 7 independent controllers, each with hardcoded thresholds. Phase 4.5 invalidated all of them simultaneously.
|
||||
|
||||
**Scope** (revised 2026-05-30 to apply the principle consistently — including clamp bounds, not just noise floors): every adaptive RL controller with **any hardcoded threshold or clamp-bound constant** is in scope. This includes:
|
||||
- Noise floors and saturation thresholds (the original Phase 4.5 saturation cause)
|
||||
- Clamp bounds (MIN/MAX on controller output range)
|
||||
- Step / ramp / decay rates
|
||||
- EMA / Wiener-α coefficients
|
||||
- Bootstrap values
|
||||
|
||||
This is **12 controllers** in one atomic commit per `feedback_no_partial_refactor`:
|
||||
|
||||
**Group A — 8 controllers that visibly saturated in fold 0:**
|
||||
1. `rl_ppo_clip_controller` — stuck at MAX 0.50 (confirmed cause)
|
||||
2. `rl_target_tau_controller` — never moved from bootstrap
|
||||
3. `rl_rollout_steps_controller` — never moved (advantage_var_ratio definitionally near-zero under Phase 4.5)
|
||||
4. `rl_entropy_coef_controller` — stuck at MIN 0.01
|
||||
5. `rl_per_alpha_controller` — stuck at MIN 0.40
|
||||
6. `rl_gamma_controller` — stuck at MIN 0.995 (hardcoded `GAMMA_MIN`)
|
||||
7. `rl_reward_scale_controller` — crashed 250× in 100 steps
|
||||
8. `rl_q_distill_lambda_controller` — stuck at MIN 0.05 (hardcoded `MIN_LAMBDA`)
|
||||
|
||||
**Group B — 4 controllers with hardcoded thresholds, didn't visibly saturate yet but vulnerable to the same architectural failure**:
|
||||
9. `rl_v_blend_alpha_controller` — 5 hardcoded constants (`DEAD_SIGNAL_FLOOR`, `TARGET_TRACK_RATIO`, `SCHULMAN_STEP`, `EMA_ALPHA`, `BOOTSTRAP_ALPHA`)
|
||||
10. `rl_ppo_ratio_clamp_controller` — `PPO_RATIO_CLAMP_MIN_OUT = 2.0f` (calibrated for pre-normalization ratios — Phase 4.5 may already have invalidated)
|
||||
11. `rl_reward_clamp_controller` — `V_BOUND_FLOOR = 1.0f`, `V_BOUND_EWMA_ALPHA = 0.001f`, `MIN_WIN`, `MIN_RATIO`, `MAX_RATIO`
|
||||
12. `rl_gate_threshold_controller` — hardcoded `alpha = 0.01f` for EMA smoothing (one constant in an otherwise ISV-driven controller)
|
||||
|
||||
**Genuinely out of scope** (audited, confirmed adaptive already, or different bug class):
|
||||
- `rl_lr_controller` — all configs are ISV slots; uses only `1.0f`/`0.0f` math identities. Confirmed already adaptive.
|
||||
- Phase 4.5 logic (`rl_advantage_normalize.cu`) — only adds an extra ISV write to emit `var_pre_norm`; the ε² floor itself is a math constant of the normalization, not a controller threshold.
|
||||
- Controller objectives (what each controller is *trying* to achieve) — out of scope.
|
||||
- Schulman bounded-step math — out of scope.
|
||||
|
||||
This spec replaces hardcoded constants with observed-signal-driven values. **One atomic commit. No phase split.**
|
||||
|
||||
---
|
||||
|
||||
## Diagnosis (verified, not theoretical)
|
||||
|
||||
Per fold 0 diag.jsonl (`/feature-cache/alpha-rl-runs/669578566/fold0/`):
|
||||
|
||||
| Controller | Bootstrap | Step 100 | Step 19999 | Mode |
|
||||
|---|---|---|---|---|
|
||||
| `γ` (gamma) | 0.995 | 0.995 | 0.995 | Stuck at MIN |
|
||||
| `τ` (target_tau) | 0.005 | 0.005 | 0.005 | Never moved |
|
||||
| `ε` (ppo_clip) | 0.20 | **0.50** | **0.50** | Stuck at MAX |
|
||||
| entropy_coef | 0.5 | 0.01 | 0.01 | Stuck at MIN |
|
||||
| per_α | 0.4 | 0.41 | 0.40 | Stuck at MIN |
|
||||
| reward_scale | 1.0 | **0.0046** | **0.0042** | Crashed 250× in 100 steps |
|
||||
| n_rollout_steps | 2048 | 2048 | 2048 | Never moved |
|
||||
| `λ` (q_distill_lambda) | 0.01 | **0.05** | **0.05** | Stuck at MIN_LAMBDA |
|
||||
|
||||
Trade pnl (eval phase, files 3-5 held out): **-$1.56M, win_rate 0.235, profit_factor 0.64**. Policy did not generalize.
|
||||
|
||||
**Note**: `sac_alpha` trends downward over training (0.01 → 0.004) but is NOT saturated — appears to be SAC's natural alpha-annealing. Not addressed in this spec.
|
||||
|
||||
### Mechanism for the canonical case (PPO clip)
|
||||
|
||||
```c
|
||||
// rl_ppo_clip_controller.cu
|
||||
#define KL_NOISE_FLOOR_FRAC 0.01f
|
||||
const float kl_noise_floor = kl_target * KL_NOISE_FLOOR_FRAC; // 0.0001
|
||||
if (kl_ema < kl_noise_floor) return; // hold
|
||||
|
||||
if (kl_ema < kl_target / tolerance) { // tolerance=1.5 → triggers when kl < 0.0067
|
||||
ratio = adjust_rate; // 1.5× → widen ε
|
||||
}
|
||||
```
|
||||
|
||||
Trajectory (verified):
|
||||
|
||||
| Step | ε | kl_pi |
|
||||
|---|---|---|
|
||||
| 1 | 0.20 (bootstrap) | 0.0 |
|
||||
| 50 | **0.50 (MAX)** | 0.002 |
|
||||
| 100 | 0.50 | 1.9e-11 |
|
||||
| 500 | 0.50 | 1.8e-3 |
|
||||
| 1000+ | 0.50 | 0 or numerical noise |
|
||||
|
||||
At step 50 the controller saw `kl_ema = 0.002` — above the noise floor 0.0001 but below the band lower bound 0.0067. WIDEN path fired three times (1.5× per call). Bootstrap-replace-directly path on the first call wrote 0.30, Wiener blends on the next two pushed to 0.5 ceiling. Then `kl_ema` collapsed below the noise floor for the rest of training; ε never came back.
|
||||
|
||||
### Why ksll2 didn't show this
|
||||
|
||||
ksll2 had 4× the data (9 files vs 3). KL distribution was wider, and at some point `kl_ema` exceeded `kl_target * tolerance = 0.015`, triggering the TIGHTEN path that pulled ε back. With 3 files, KL never gets that high — controller never sees a reason to tighten.
|
||||
|
||||
This pattern repeats across all 7 controllers, each with its own variant of the same bug.
|
||||
|
||||
### Why this is one bug, not seven
|
||||
|
||||
All 7 controllers share three architectural assumptions, each invalidated by Phase 4.5:
|
||||
|
||||
1. **Hardcoded noise floor as fraction of target**: 4 controllers use `_NOISE_FLOOR_FRAC = 0.01f` (`ppo_clip`, `target_tau`, `rollout_steps`, `fused`). The 1% floor was calibrated against pre-normalization signal magnitudes; under Phase 4.5 it's ~100× too low.
|
||||
2. **Fixed target ISV slots**: `kl_target=0.01`, `q_div_target=0.01`, `adv_var_ratio_target=5.0` etc. are bootstrapped once and never updated. They assume a signal scale that no longer holds.
|
||||
3. **Aggressive bootstrap-to-update**: most controllers have a "bootstrap-replace-directly" path that lets a single observation push them off bootstrap. Combined with (1) + (2), one noisy step can drive the controller to an extremum.
|
||||
|
||||
---
|
||||
|
||||
## Approaches considered
|
||||
|
||||
### Approach A — Targeted patch (~12 LOC, lowest cost)
|
||||
|
||||
Replace `_NOISE_FLOOR_FRAC = 0.01f` with `_NOISE_FLOOR_FRAC = 0.5f` (50% of target instead of 1%). Apply to the 4 controllers that have this constant. No new kernels, no ISV changes.
|
||||
|
||||
**Pros**: One commit, one test, mechanical change. Validates the diagnosis quickly.
|
||||
|
||||
**Cons**: Still a hardcoded constant — violates `feedback_adaptive_not_tuned`. Doesn't fix `reward_scale` crash (different mechanism). Doesn't fix `gamma`/`entropy_coef`/`per_α` (no `_NOISE_FLOOR_FRAC`, different saturation modes). Might break ksll2 regime where 1% worked (50% would hold the controller too aggressively when there *is* meaningful signal). Brittle to the next normalization change.
|
||||
|
||||
**Verdict**: Useful as a same-day regression test for the diagnosis, but not the actual fix.
|
||||
|
||||
### Approach B — Signal-driven floors (~200 LOC, the architectural fix)
|
||||
|
||||
For each controller's input signal, track Welford-online variance in a dedicated ISV slot. Replace the hardcoded `floor = target × const` with `floor = MAX(target × 0.5, sqrt(observed_var) × 2)`. The floor naturally rescales to whatever signal distribution Phase 4.5 (or any future normalization) produces.
|
||||
|
||||
**Pros**: Truly adaptive — `feedback_adaptive_not_tuned` compliant. Composes with future architectural changes (Phase 4.6 normalization, new heads, etc.) without re-tuning. Same kernel pattern works for all 7 controllers. Welford is well-understood, CUDA-friendly. The added Welford-variance ISV slots are also valuable diagnostics.
|
||||
|
||||
**Cons**: ~200 LOC: new Welford-variance kernel (~80 LOC) + ISV slot allocation (8 new slots, one per controller input) + wiring (~80 LOC) + controller updates (~30 LOC across 7 files). Larger change, larger blast radius.
|
||||
|
||||
**Verdict**: This is the architectural fix the foxhunt design philosophy promised. Implementation cost is real but bounded.
|
||||
|
||||
### Approach C — Conservative bootstrap + asymmetric Schulman (~50 LOC)
|
||||
|
||||
Don't touch the floors. Instead: (1) bootstrap each controller at its TARGET, not at a safety bound, (2) require N consecutive below-band observations before widening fires (so single noisy observations don't drive drift). Hold bootstrap until cumulative evidence accumulates.
|
||||
|
||||
**Pros**: Smaller change than B. Preserves controller logic. Holds controllers at sensible defaults when signal is sparse.
|
||||
|
||||
**Cons**: Still uses fixed thresholds — same brittleness as A. Doesn't address `reward_scale` crash. Doesn't address `n_rollout_steps` which is structurally broken (its input signal `advantage_var_ratio` is *defined* to be near zero under Phase 4.5).
|
||||
|
||||
**Verdict**: Orthogonal to A/B. Worth folding INTO B as the "asymmetric Schulman" piece.
|
||||
|
||||
### Recommendation: B + the Schulman asymmetry from C, applied to ALL 12 controllers with hardcoded thresholds
|
||||
|
||||
The architectural fix is B. Fold C's asymmetric-Schulman idea into B's controller updates for Group A. Skip A as a separate change — it's a strict subset of B.
|
||||
|
||||
`reward_scale`, `n_rollout_steps`, `gamma`, `q_distill_lambda` need separate treatment within this spec — they have *structural* problems (their signals are not just noisy, they're definitionally affected by Phase 4.5 or use ramp/decay rather than Schulman). Spec covers them in dedicated Special case sections (G, Q, R).
|
||||
|
||||
Group B controllers (4 of them) follow the same Welford-variance pattern from B but without the asymmetric Schulman — they're not Schulman-based controllers, just have hardcoded floors/ceilings. Same kernel infrastructure, same ISV slot pattern.
|
||||
|
||||
**One atomic commit. 12 controllers. 52 new ISV slots. One new Welford kernel.**
|
||||
|
||||
---
|
||||
|
||||
## Detailed design (Approach B)
|
||||
|
||||
### Core pattern (applies 7 places)
|
||||
|
||||
```c
|
||||
// 1. Welford variance kernel (new, runs per-step alongside EMA producers).
|
||||
// Maintains running mean + M2 in dedicated ISV slots per signal.
|
||||
// See: crates/ml-alpha/cuda/rl_signal_variance_update.cu (NEW)
|
||||
|
||||
extern "C" __global__ void rl_signal_variance_update(
|
||||
float* __restrict__ isv,
|
||||
int input_slot, // e.g. RL_KL_PI_EMA_INDEX
|
||||
int count_slot, // ISV[input_slot + COUNT_OFFSET]
|
||||
int mean_slot, // ISV[input_slot + MEAN_OFFSET]
|
||||
int m2_slot, // ISV[input_slot + M2_OFFSET]
|
||||
int dones_present // 1 if this step contributed a real observation
|
||||
) {
|
||||
if (threadIdx.x != 0 || !dones_present) return;
|
||||
const float x = isv[input_slot];
|
||||
if (x == 0.0f) return; // skip sentinel
|
||||
float n = isv[count_slot] + 1.0f;
|
||||
float delta = x - isv[mean_slot];
|
||||
float mean_new = isv[mean_slot] + delta / n;
|
||||
float delta2 = x - mean_new;
|
||||
float m2_new = isv[m2_slot] + delta * delta2;
|
||||
isv[count_slot] = n;
|
||||
isv[mean_slot] = mean_new;
|
||||
isv[m2_slot] = m2_new;
|
||||
}
|
||||
```
|
||||
|
||||
### Controller integration (canonical example — PPO clip)
|
||||
|
||||
```c
|
||||
// rl_ppo_clip_controller.cu (MODIFIED)
|
||||
// BEFORE:
|
||||
const float kl_noise_floor = kl_target * KL_NOISE_FLOOR_FRAC; // 1% — broken
|
||||
|
||||
// AFTER:
|
||||
const float kl_var = (isv[KL_PI_COUNT_SLOT] > 1.0f)
|
||||
? isv[KL_PI_M2_SLOT] / (isv[KL_PI_COUNT_SLOT] - 1.0f)
|
||||
: 0.0f;
|
||||
const float kl_std = sqrtf(kl_var);
|
||||
const float kl_noise_floor = fmaxf(kl_target * 0.5f, kl_std * 2.0f);
|
||||
// ↑ floor never below 50% of target
|
||||
// ↑ AND scales with observed signal std
|
||||
|
||||
if (kl_ema < kl_noise_floor) return;
|
||||
```
|
||||
|
||||
The `0.5f` constant is the only remaining tuned number, and it's safe across all signal regimes (50% of target = "less than half the desired operating distance from steady state").
|
||||
|
||||
### Asymmetric Schulman (folded in from Approach C)
|
||||
|
||||
```c
|
||||
// Tighten on single observation (real signal above target = act fast).
|
||||
// Widen only on N consecutive observations below band (filter noise).
|
||||
|
||||
if (kl_ema > kl_target * tolerance) {
|
||||
ratio = 1.0f / adjust_rate;
|
||||
isv[KL_BELOW_COUNT_SLOT] = 0.0f; // reset
|
||||
} else if (kl_ema < kl_target / tolerance) {
|
||||
isv[KL_BELOW_COUNT_SLOT] += 1.0f;
|
||||
ratio = (isv[KL_BELOW_COUNT_SLOT] >= 3.0f) ? adjust_rate : 1.0f;
|
||||
} else {
|
||||
isv[KL_BELOW_COUNT_SLOT] = 0.0f; // reset on in-band
|
||||
ratio = 1.0f;
|
||||
}
|
||||
```
|
||||
|
||||
The `3.0f` is the "N consecutive" parameter. Tunable — default 3 (~3 minibatch updates of confirmation). Could itself be ISV-driven (`SCHULMAN_WIDEN_PATIENCE_SLOT`) — TBD if needed.
|
||||
|
||||
### Per-controller specifics
|
||||
|
||||
ISV slots below allocated starting at 588 (current `RL_SLOTS_END`). Final `RL_SLOTS_END = 640` (52 new slots).
|
||||
|
||||
**Group A — 8 saturated controllers (Welford variance + below-band counter for asymmetric Schulman):**
|
||||
|
||||
| Controller | Input signal | Input slot | Variance triple (count, mean, M²) | Asym counter | Asymmetric Schulman? |
|
||||
|---|---|---|---|---|---|
|
||||
| ppo_clip | kl_pi_ema | 419 | 588, 589, 590 | 591 | yes |
|
||||
| target_tau | q_divergence_ema | 420 | 592, 593, 594 | 595 | yes |
|
||||
| rollout_steps | adv_var_pre_norm (NEW 612) | 421 → 612 | 596, 597, 598 | 599 | yes |
|
||||
| entropy_coef | entropy_observed_ema | 422 | 600, 601, 602 | 603 | yes |
|
||||
| per_α | td_kurtosis_ema | 423 | 604, 605, 606 | 607 | yes |
|
||||
| gamma | trade_duration_ema | 417 | 608, 609, 610 | — | no (Special G) |
|
||||
| reward_scale | pnl_per_batch_magnitude (NEW 614) | 614 | 615, 616, 617 | — | no (Special R) |
|
||||
| q_distill_lambda | q_distill_kl_ema | 488 | 618, 619, 620 | — | no (Special Q) |
|
||||
|
||||
**Group B — 4 not-yet-saturated controllers with hardcoded thresholds:**
|
||||
|
||||
| Controller | Hardcoded → ISV slots | Notes |
|
||||
|---|---|---|
|
||||
| v_blend_alpha (Phase 4.4) | `DEAD_SIGNAL_FLOOR` → 621, `TARGET_TRACK_RATIO` → 622, `SCHULMAN_STEP` → 623, `EMA_ALPHA` → 624, `BOOTSTRAP_ALPHA` → 625 | 5 new slots; controller already takes one Welford variance for V_scalar magnitude — slots 626, 627, 628 |
|
||||
| ppo_ratio_clamp | `MIN_OUT` (2.0f) → adaptive via ratio_variance; `MAX_OUT` (1000.0f) → 629 | Welford on ratio magnitude — slots 630, 631, 632 |
|
||||
| reward_clamp | `V_BOUND_FLOOR` → 633, `V_BOUND_EWMA_ALPHA` → 634, `MIN_WIN` → 635, `MIN_RATIO` → 636, `MAX_RATIO` → 637 | 5 slots; already partially signal-driven, just removing remaining hardcoded constants |
|
||||
| gate_threshold | hardcoded `alpha = 0.01f` → `RL_GATE_EMA_ALPHA_INDEX` 638 | Single slot |
|
||||
|
||||
**New input signals (separate from variance slots):**
|
||||
- 612: `RL_ADV_VAR_PRE_NORM_INDEX` (emitted by `rl_advantage_normalize.cu` before in-place normalize)
|
||||
- 613: `RL_GAMMA_MIN_ADAPTIVE_INDEX` (computed from trade_duration_ema)
|
||||
- 614: `RL_REWARD_MAGNITUDE_EMA_INDEX` (per-batch pnl magnitude for reward_scale floor)
|
||||
- 639: `RL_Q_DISTILL_LAMBDA_MIN_ADAPTIVE_INDEX` (computed from q_distill_kl_ema variance)
|
||||
|
||||
**Total: 52 new slots. `RL_SLOTS_END` grows 588 → 640.** Memory impact: +208 bytes.
|
||||
|
||||
### Special cases (not covered by core pattern)
|
||||
|
||||
#### `n_rollout_steps` — structural conflict with Phase 4.5
|
||||
|
||||
Its input signal `advantage_var_ratio_ema` is *definitionally* near zero under Phase 4.5 normalization (advantage var ~= ε² floor). The Welford-variance fix won't help because the signal itself is structurally diluted.
|
||||
|
||||
**Proposed fix**: track *pre-normalization* advantage variance in a separate ISV slot. Phase 4.5's `rl_advantage_normalize.cu` computes the mean and variance internally — emit `var_pre_norm` to a new ISV slot (`RL_ADV_VAR_PRE_NORM_INDEX`). `rl_rollout_steps_controller` reads from that instead.
|
||||
|
||||
~20 LOC change to `rl_advantage_normalize.cu` to emit the ISV value before in-place normalize.
|
||||
|
||||
#### `reward_scale` — crash, not saturation
|
||||
|
||||
Crashed from 1.0 → 0.004 in 100 steps. Mechanism: when `pnl_per_batch_magnitude` is small (which it is on small datasets with the agent not yet trading much), the scale controller aggressively shrinks `reward_scale` to compensate. By step 100 the reward signal is muted to 0.4% of bootstrap, which then makes everything downstream tiny → controllers see no signal → death spiral.
|
||||
|
||||
**Proposed fix**: rate-limit the `reward_scale` DECREASE more aggressively than INCREASE (asymmetric like Schulman above). Also: floor `reward_scale` at a fraction of bootstrap (e.g., `0.1 × REWARD_SCALE_BOOTSTRAP = 0.1`) until N steps have passed with meaningful trades. This protects against the early-training spiral.
|
||||
|
||||
~40 LOC in `rl_reward_scale_controller.cu`.
|
||||
|
||||
#### Special case G — `gamma` deadlock from hardcoded `GAMMA_MIN`
|
||||
|
||||
Per the trajectory, `gamma` was at MIN (0.995) at step 1 (bootstrap from sentinel) and never moved. Mechanism:
|
||||
|
||||
```c
|
||||
// rl_gamma_controller.cu
|
||||
#define GAMMA_MIN 0.995f // hardcoded — violates feedback_adaptive_not_tuned
|
||||
#define GAMMA_MAX 0.999f
|
||||
// target(d) = 1 - 1/d (standard discount-from-horizon mapping)
|
||||
// At d_ema = 15.6: target = 1 - 1/15.6 = 0.936
|
||||
// clamp(0.936, 0.995, 0.999) = 0.995 ← always pinned at MIN
|
||||
```
|
||||
|
||||
`target(d)` only exceeds `GAMMA_MIN = 0.995` when `d_ema > 200` — i.e., trade durations over 200 minibatch ticks (~10s+ in the surfer regime). Below that, gamma is structurally pinned at 0.995. With fold 0's `d_ema = 15.6`, target = 0.936, controller wants to lower gamma but can't (clamp), so it sticks.
|
||||
|
||||
**This is the same architectural bug** as the noise floors: a *hardcoded* threshold that may have been right for one regime but is wrong for another. With small data, agent makes shorter trades; the gamma floor traps the agent in a discount factor that's too long for its trade horizon, which feeds back into the value targets, which biases learning.
|
||||
|
||||
**Deadlock**: the agent needs higher gamma to learn longer trades, but the controller needs longer trades to allow higher gamma. With the hardcoded MIN at 0.995, the controller has no room to lower gamma to match short trades, and no signal to raise it because trades stay short.
|
||||
|
||||
**Proposed fix** (consistent with the rest of the spec):
|
||||
|
||||
Replace hardcoded `GAMMA_MIN = 0.995f` with adaptive `RL_GAMMA_MIN_ADAPTIVE_INDEX` slot, derived from observed `trade_duration_ema`:
|
||||
|
||||
```c
|
||||
// Derived adaptive floor: allow gamma to drop low enough for current
|
||||
// regime's trade horizon, with a safety floor of 0.9 to prevent the
|
||||
// agent from going to bandit mode.
|
||||
const float d_ema = isv[RL_MEAN_TRADE_DURATION_EMA_INDEX];
|
||||
const float horizon_multiplier = 2.0f; // gamma allows 2× the trade horizon as effective lookahead
|
||||
const float adaptive_min = fmaxf(0.9f, 1.0f - 1.0f / fmaxf(d_ema * horizon_multiplier, 10.0f));
|
||||
isv[RL_GAMMA_MIN_ADAPTIVE_INDEX] = adaptive_min;
|
||||
```
|
||||
|
||||
With `d_ema=15.6`, adaptive_min = `1 - 1/31.2 = 0.968`. Now `target(15.6) = 0.936` still clamps, but to `0.968` instead of `0.995` — and the controller has room to track real trade durations. As the agent learns longer trades and `d_ema` grows, `adaptive_min` rises naturally.
|
||||
|
||||
The `0.9` absolute floor and `2.0` horizon multiplier are the only tuned constants — both architecturally motivated:
|
||||
- `0.9` floor: per `pearl_edge_lives_at_wave_timescale_not_tick`, edge lives at wave timescale (γ≥0.99). The 0.9 floor is a hard "don't degenerate to bandit" guard, not a typical operating point.
|
||||
- `2.0` horizon multiplier: standard rule-of-thumb that effective horizon should be 2× the natural transaction horizon to capture exit dynamics.
|
||||
|
||||
Both could be ISV-driven in a follow-up spec, but the current values are conservative defaults that don't bias the controller's adaptation.
|
||||
|
||||
#### Special case Q — `q_distill_lambda` ramp/decay with hardcoded MIN
|
||||
|
||||
Per the trajectory: `λ` bootstraps at 0.01, decays via `LAMBDA_DECAY_RATE = 0.998f` for ~100 steps because `q_distill_kl_ema ≈ 0.001` is far below `q_distill_kl_target = 0.10`. Reaches `MIN_LAMBDA = 0.05f` floor and sticks for the next 19900 steps.
|
||||
|
||||
```c
|
||||
// rl_q_distill_lambda_controller.cu
|
||||
#define MIN_LAMBDA 0.05f // hardcoded floor
|
||||
#define MAX_LAMBDA 1.0f // hardcoded ceiling
|
||||
// When kl_ema < kl_target / tolerance → λ *= LAMBDA_DECAY_RATE = 0.998
|
||||
// When kl_ema > kl_target * tolerance → λ *= LAMBDA_RAMP_RATE = 1.2
|
||||
// Clamp to [MIN_LAMBDA, MAX_LAMBDA]
|
||||
```
|
||||
|
||||
Same bug class as the noise floors and gamma: a hardcoded floor (`MIN_LAMBDA`) that doesn't adapt to the actual signal regime. Phase 4.5 reduces KL signal magnitudes (which this controller also consumes via `q_distill_kl_ema`), so the "below target" path fires continuously, decaying λ to MIN.
|
||||
|
||||
**Proposed fix** (mirroring the gamma fix pattern): make `MIN_LAMBDA` and `MAX_LAMBDA` ISV-driven adaptive bounds derived from observed `q_distill_kl_ema` distribution:
|
||||
|
||||
```c
|
||||
// adaptive_lambda_min = MAX(0.001, kl_ema_std * SCALE_FACTOR)
|
||||
// where SCALE_FACTOR ≈ 0.05 (provides headroom proportional to signal noise)
|
||||
const float kl_std = sqrtf(isv[Q_DISTILL_KL_VAR_SLOT]);
|
||||
const float adaptive_min = fmaxf(0.001f, kl_std * 0.05f);
|
||||
isv[RL_Q_DISTILL_LAMBDA_MIN_ADAPTIVE_INDEX] = adaptive_min;
|
||||
```
|
||||
|
||||
This reuses the same Welford-variance kernel infrastructure proposed in the core pattern (one variance triple for `q_distill_kl_ema`). Lambda can now decay to a level proportional to the actual signal noise rather than a hardcoded 0.05 floor.
|
||||
|
||||
The MAX is less critical (1.0 ceiling is rarely hit in practice) but consistency: derive MAX as `MIN(1.0, MAX(0.5, kl_std * 50.0))`. Provides upward headroom proportional to signal range.
|
||||
|
||||
Adds 1 ISV slot (`RL_Q_DISTILL_LAMBDA_MIN_ADAPTIVE_INDEX`) — total grows from 27 → 28.
|
||||
|
||||
---
|
||||
|
||||
## Files
|
||||
|
||||
### New
|
||||
|
||||
- `crates/ml-alpha/cuda/rl_signal_variance_update.cu` — Welford variance kernel (~80 LOC)
|
||||
|
||||
### Modified
|
||||
|
||||
- `crates/ml-alpha/src/rl/isv_slots.rs` — add 27 new slots (3 variance triples × 7 controllers + 3 input slots + 3 asymmetric Schulman counters); grow `RL_SLOTS_END` 588 → 615
|
||||
- `crates/ml-alpha/cuda/rl_ppo_clip_controller.cu` — adaptive noise floor + asymmetric Schulman
|
||||
- `crates/ml-alpha/cuda/rl_target_tau_controller.cu` — adaptive noise floor + asymmetric Schulman
|
||||
- `crates/ml-alpha/cuda/rl_rollout_steps_controller.cu` — adaptive noise floor + asymmetric Schulman + consume `RL_ADV_VAR_PRE_NORM_INDEX` (new) instead of post-norm advantage_var_ratio
|
||||
- `crates/ml-alpha/cuda/rl_entropy_coef_controller.cu` — adaptive noise floor + asymmetric Schulman
|
||||
- `crates/ml-alpha/cuda/rl_per_alpha_controller.cu` — adaptive noise floor + asymmetric Schulman
|
||||
- `crates/ml-alpha/cuda/rl_reward_scale_controller.cu` — asymmetric rate-limit (decrease ≤ 1.05× per step) + bootstrap-fraction floor (`reward_scale ≥ 0.1 × bootstrap` until N trades accumulated) + emit `RL_REWARD_MAGNITUDE_EMA_INDEX`
|
||||
- `crates/ml-alpha/cuda/rl_gamma_controller.cu` — replace hardcoded `GAMMA_MIN = 0.995f` with adaptive `RL_GAMMA_MIN_ADAPTIVE_INDEX` derived from `trade_duration_ema` (Special case G)
|
||||
- `crates/ml-alpha/cuda/rl_q_distill_lambda_controller.cu` — replace hardcoded `MIN_LAMBDA = 0.05f` and `MAX_LAMBDA = 1.0f` with adaptive ISV-driven bounds (Special case Q, below)
|
||||
- `crates/ml-alpha/cuda/rl_v_blend_alpha_controller.cu` — replace 5 hardcoded constants with ISV slots (`DEAD_SIGNAL_FLOOR` → signal-driven, `TARGET_TRACK_RATIO`/`SCHULMAN_STEP`/`EMA_ALPHA`/`BOOTSTRAP_ALPHA` → ISV slots)
|
||||
- `crates/ml-alpha/cuda/rl_ppo_ratio_clamp_controller.cu` — replace `PPO_RATIO_CLAMP_MIN_OUT = 2.0f` with adaptive floor derived from observed ratio variance; `MAX_OUT = 1000.0f` becomes ISV-driven ceiling
|
||||
- `crates/ml-alpha/cuda/rl_reward_clamp_controller.cu` — replace `V_BOUND_FLOOR`, `V_BOUND_EWMA_ALPHA`, `MIN_WIN`, `MIN_RATIO`, `MAX_RATIO` with adaptive ISV slots (`V_BOUND_EWMA_ALPHA` becomes signal-variance-driven; ratio bounds derive from observed win/loss distribution)
|
||||
- `crates/ml-alpha/cuda/rl_gate_threshold_controller.cu` — replace hardcoded `alpha = 0.01f` with ISV slot `RL_GATE_EMA_ALPHA_INDEX`
|
||||
- `crates/ml-alpha/cuda/rl_advantage_normalize.cu` — emit raw `var_pre_norm` to `RL_ADV_VAR_PRE_NORM_INDEX` before in-place normalize
|
||||
- `crates/ml-alpha/cuda/rl_fused_controllers.cu` — all of the above unified (this is what the trainer actually launches per-step)
|
||||
- `crates/ml-alpha/src/trainer/integrated.rs` — launch `rl_signal_variance_update` per step + wire new ISV slots + populate `RL_GAMMA_MIN_ADAPTIVE_INDEX` via fused kernel
|
||||
|
||||
### Not touched
|
||||
|
||||
- Phase 4.5 logic itself (`rl_advantage_normalize.cu` body) — the only addition is one ISV write to emit `var_pre_norm` for `rl_rollout_steps_controller` consumption. The ε² floor in the normalization math stays — it's a numerical-stability constant of the kernel, not a controller threshold.
|
||||
- `rl_lr_controller` — **AUDITED 2026-05-30**: all configs are ISV slots (`RL_LR_BOOTSTRAP_INDEX`, `RL_LR_MIN_INDEX`, `RL_LR_MAX_INDEX`, `RL_LR_DECAY_FACTOR_INDEX`, `RL_LR_LOSS_EMA_ALPHA_INDEX`, `RL_IMPROVEMENT_THRESHOLD_INDEX`, `RL_PLATEAU_PATIENCE_INDEX`, `RL_LR_WARMUP_STEPS_INDEX`). Kernel body uses only `1.0f`/`0.0f` math identities (not thresholds). Already complies with `feedback_adaptive_not_tuned`. Per `pearl_plateau_decay_needs_warmup_for_sparse_heads`, the plateau-decay pattern is intentionally different from Schulman; keeping it.
|
||||
- All other RL kernels not listed in Modified (e.g., `rl_fused_targets.cu`, kernel utilities) — not controllers, not subject to this spec.
|
||||
|
||||
---
|
||||
|
||||
## Validation gates
|
||||
|
||||
### Unit / smoke gates (local, RTX 3050 Ti)
|
||||
|
||||
1. **G1 — Welford convergence**: feed constant signal `k=0.5`, 50 steps, assert `isv[mean_slot] ≈ k` AND `isv[m2_slot] / (n-1) ≈ 0` (no variance).
|
||||
2. **G2 — Welford variance**: feed signal `[1, 2, 3, 4, 5]`, assert `isv[m2_slot] / 4 ≈ 2.5` (sample variance).
|
||||
3. **G3 — PPO clip holds on small signal**: bootstrap controller, feed `kl_ema = 0.001` for 100 steps with observed_var=1e-7, assert `ε` does NOT drift past `target × tolerance = 0.015`.
|
||||
4. **G4 — PPO clip tightens on big signal**: feed `kl_ema = 0.05` once (single overshoot), assert `ε` decreases by `1/adjust_rate` immediately (asymmetric tightening).
|
||||
5. **G5 — PPO clip widens slowly**: feed `kl_ema = 0.001` for 10 steps, assert widen fires ONLY after 3 consecutive below-band observations (asymmetric).
|
||||
6. **G6 — integrated_trainer_smoke**: existing GPU smoke must still pass.
|
||||
|
||||
### Cluster gates
|
||||
|
||||
7. **G7 — fold 0 re-run with fix**: same params (b=1024, 20k, n_folds=3, fold_idx=0). At step 19999:
|
||||
- PPO clip `ε ≤ 0.20` (held in bootstrap region, was 0.50)
|
||||
- `reward_scale ≥ 0.5` (bootstrap-fraction floor active, was 0.004)
|
||||
- `n_rollout_steps ≠ 2048` (moved off bootstrap, signals real adaptation via pre-norm advantage variance)
|
||||
- `gamma > 0.97` (adaptive_min derived from `d_ema ≈ 15` allows movement above hardcoded 0.995)
|
||||
- `q_distill_lambda > 0.05` (above old MIN floor, adaptive bound from signal variance)
|
||||
- 5+ of the 8 Group A controllers no longer pinned at extrema
|
||||
8. **G8 — fold 0 eval pnl improvement**: eval pnl > -$0.5M (was -$1.56M). Profit factor > 0.85 (was 0.64). Win rate > 0.40 (was 0.235). These are improvement gates, not absolute targets — we expect *some* improvement, not full positive pnl on 3-file training.
|
||||
9. **G9 — ksll2 regression test**: same params as `alpha-rl-ksll2` (n_folds=1, b=1024, 20k). Final pnl ≥ +$15M (within 20% of original +$18.3M). Confirms fix doesn't regress the large-data regime where the prior hardcoded floors happened to work. **All 12 controllers** should reach values consistent with the +$18M ksll2 trajectory (e.g., ε around target 0.01, not at 0.50; reward_scale around 0.5-1.0; gamma in [0.97, 0.999]).
|
||||
10. **G10 — Group B non-regression**: the 4 Group B controllers (`v_blend_alpha`, `ppo_ratio_clamp`, `reward_clamp`, `gate_threshold`) didn't visibly saturate before this fix — verify they still operate in their prior healthy ranges:
|
||||
- `v_blend_alpha` settles in [0.0, 1.0] with non-trivial variation (was bootstrapping to 1.0 then dropping toward 0)
|
||||
- `ppo_ratio_clamp_max` settles at a meaningful value (was 10.0 bootstrap → adaptive)
|
||||
- `reward_clamp_win` / `reward_clamp_loss` track observed reward distribution (no drift toward MIN/MAX of new adaptive bounds)
|
||||
- `gate_threshold` EMA tracks dones_target as before (the only change is `alpha = 0.01f` → ISV-driven, semantically equivalent at the new default)
|
||||
|
||||
---
|
||||
|
||||
## Open questions resolved
|
||||
|
||||
1. ~~**ISV slot count budget**~~ → **resolved**: `RL_SLOTS_END = 588` currently, highest used 587. No upper cap. 52 new slots → `RL_SLOTS_END = 640`. Memory impact +208 bytes (negligible).
|
||||
2. **Welford reset policy**: variance EMA carries over the train→eval boundary. *Decision*: accept this — eval-phase ISV reads should still see the train-phase variance estimate (controllers don't update during eval anyway).
|
||||
3. **N-consecutive-below counter scope**: spec uses per-controller counters (5 ISV slots, one per Group A controller that needs asymmetric Schulman). Group B controllers don't need them (different mechanism).
|
||||
4. **`reward_scale` bootstrap-fraction floor**: 0.1 (10% of bootstrap) — accepted as a tuned constant guarded by the rationale that early-training reward distributions are inherently noisy and shrinking past 10% of bootstrap is almost never beneficial. Could be ISV-driven in a follow-up.
|
||||
5. **Order of implementation**: **all-in-one commit per `feedback_no_partial_refactor`** (Option B selected). Partial migration would create ambiguous validation signal — "did ppo_clip fix work but other controllers ruined it?" vs "did the architectural fix work?" — and the contract change (new ISV slots, modified controller semantics) needs atomic application. The plan decomposes into ordered TASKS that flow into ONE commit at the end, not one commit per controller. No "Phase 1/Phase 2" — 12 controllers in one commit.
|
||||
|
||||
6. ~~**Gamma adaptive_min coupling**~~ → **resolved** (2026-05-30): use the Welford mean of trade duration directly (already computed for `rl_per_alpha_controller`'s noise floor), with a 100-observation warmup gate. The Welford mean's natural lag (denominator-of-N smoothing) breaks the feedback loop without any explicit step-period gating or exponential cool-down. Implementation: while `count < 100` use the existing EMA; once `count >= 100` switch to the Welford mean as the input to `target(d)`. No new ISV slots, no new constants beyond the 100-observation gate (which is a confidence threshold, not a magnitude calibration). See plan Task 7 for the code.
|
||||
|
||||
---
|
||||
|
||||
## Risks & mitigations
|
||||
|
||||
- **Risk**: Welford with float32 + small variance values → catastrophic cancellation. **Mitigation**: standard Welford formulation already handles this; CUDA float32 is sufficient at our signal magnitudes (1e-5 to 1.0).
|
||||
- **Risk**: New ISV slots break existing tests that snapshot the full ISV layout. **Mitigation**: tests need to be updated alongside; this was just done in the test-migration commit `a3dfcd63f` for similar reasons.
|
||||
- **Risk**: Asymmetric Schulman makes controllers TOO conservative, blocking real adaptation. **Mitigation**: the N-consecutive parameter is small (3), tightening still fires on single observation (fast safety response), and the in-band hold preserves the steady state.
|
||||
- **Risk**: The fix works on fold 0 but ksll2 (large data) actually NEEDS the loose noise floor to adapt fast enough. **Mitigation**: G9 cluster regression catches this. Fallback: ISV-driven `NOISE_FLOOR_FRAC` slot, dynamically tunable per-run.
|
||||
|
||||
---
|
||||
|
||||
## Next step after approval
|
||||
|
||||
Write the plan: `docs/superpowers/plans/2026-05-30-adaptive-controller-floor-plan.md` with task-level decomposition (foundation → ppo_clip → sweep → validation → cluster runs).
|
||||
@@ -0,0 +1,560 @@
|
||||
# Adaptive Risk Management — Design (Layered Stack)
|
||||
|
||||
**Goal**: close the train-eval distribution-shift gap surfaced by the 2026-05-30 adaptive-controller-floor refactor (commit `b1ef6664a`). The fix preserves the +$18M baseline (G9 PASS, +$15.81M) but fold 1 walk-forward eval came in marginally worse (-$616k vs -$500k OLD) despite massively better train pnl. **Eval JSONL revealed the load-bearing missing layers**: position commitment, action selection risk-awareness, and inventory exposure are all under-adapted relative to what the controller-floor architecture made possible.
|
||||
|
||||
**Why now**: the eval `win_rate=0.21 × R-multiple=1.8` Kelly fraction is negative — agent should not trade but does (1024 batches). Train win_rate 0.55 vs eval 0.21 = severe distribution shift. The controller-floor refactor made signal *interpretation* adaptive; commitment, action-selection, and reward shaping for inventory remained hardcoded.
|
||||
|
||||
**Out of scope**: changes to the perception encoder, Mamba2/CfC architecture, advantage normalization (Phase 4.5), the 12 adaptive controllers from b1ef6664a (those are now correct). Decision policy (the agent still chooses among the 11 discrete actions). Regime detection / meta-policy switching (deferred to follow-up per research caveat).
|
||||
|
||||
---
|
||||
|
||||
## Diagnosis — what the JSONL says
|
||||
|
||||
Per fold 1 eval (`/feature-cache/alpha-rl-runs/b1ef6664a/fold1/eval_summary.json`):
|
||||
|
||||
| Metric | Value | Interpretation |
|
||||
|---|---|---|
|
||||
| total_pnl_usd | -$616k | losing |
|
||||
| avg_win | $13,785 | wins are large |
|
||||
| avg_loss | $7,631 | R-multiple 1.8 (good) |
|
||||
| win_rate | 0.21 (vs train 0.55) | severe directional overfit |
|
||||
| max_drawdown_pct | 18.7% | controlled but real |
|
||||
| profit_factor | 0.83 | losing aggregate |
|
||||
|
||||
**Kelly at observed edge**: f* = (0.21 × 1.8 − 0.79)/1.8 = **−0.23** → don't trade. Agent trades anyway. **Position commitment never adapts to edge.**
|
||||
|
||||
**Internal evidence from in-run JSONL** (fold 1 step 16k, qpa = +0.68 = high agent confidence): **the IQN distribution would tell us the pessimistic Q (lower quantile) is significantly negative even when expected Q is positive**. The agent uses `argmax(E_τ[Q])` (mean over quantiles) — over-weights upside under noisy data. Risk-averse action selection (lower quantile τ) would naturally filter these losing trades.
|
||||
|
||||
**Bigger picture**: the system has no adaptive layer between "agent's decision" and "actual market commitment." Five distinct layers are absent or hardcoded:
|
||||
|
||||
| Layer | What it does | Status pre-spec |
|
||||
|---|---|---|
|
||||
| Hard constraints (max position, DD cap, etc.) | Override agent on safety violation | Mostly absent |
|
||||
| Risk-averse action selection | Bias decisions toward downside-aware Q | Uses E[Q] — risk-neutral |
|
||||
| Inventory exposure penalty | Discourage one-sided drift | Absent in reward shaping |
|
||||
| Position commitment / sizing | Scale lots by observed edge | Fixed at base lots |
|
||||
| Regime detection | Distribution-shift handling | Absent |
|
||||
|
||||
This spec addresses the first four. Regime detection is deferred per research evidence ("research-active, not production-validated").
|
||||
|
||||
---
|
||||
|
||||
## Scope — the layered risk stack
|
||||
|
||||
Apply `feedback_adaptive_not_tuned` consistently across the commitment layer:
|
||||
|
||||
1. **Layer 1 — Hard CMDP-style constraints (~120 LOC)**: 5 deterministic risk gates that override agent actions
|
||||
2. **Layer 2 — IQN risk-averse action selection (~30 LOC)**: use lower-quantile τ from existing IQN distribution
|
||||
3. **Layer 3 — Inventory penalty in reward shaping (~60 LOC)**: discourage one-sided net-position drift
|
||||
4. **Layer 4 — Kelly-fraction position sizing (~80 LOC)**: scale committed lots by observed edge
|
||||
5. **TrailTighten/Loosen wiring (~30 LOC)**: close dead-action gap per [[pearl_dead_trail_stop_actions_a7_a8]]
|
||||
|
||||
Single atomic commit per `feedback_no_partial_refactor`. Same pattern as the controller-floor refactor.
|
||||
|
||||
**The crucial reordering vs an earlier draft of this spec**: Kelly was originally the primary fix. Research (FineFT 2026, distributional-RL surveys) plus internal analysis showed **Layer 2 (IQN risk-averse τ) is the higher-leverage minimum-cost change** because we already compute the full IQN quantile distribution but throw away everything except the expectation. Kelly is now Layer 4 — fine-tuning after pessimistic action selection has filtered the trade set.
|
||||
|
||||
---
|
||||
|
||||
## Approaches considered
|
||||
|
||||
### Approach A — Kelly fraction only
|
||||
Original minimum-viable design. Single new controller computes f* from observed win_rate × R-multiple, multiplies lot size. ~80 LOC.
|
||||
|
||||
**Verdict**: Insufficient. Kelly is the *commitment* layer; the *selection* layer (which trades to even consider) is more impactful. The 21% eval win_rate means the agent shouldn't even be entering most trades, regardless of size.
|
||||
|
||||
### Approach B — Kelly + session DD circuit breaker
|
||||
Add a session-pnl-tracking ISV slot + override-to-Hold logic. Two-layer (Kelly per-trade + session-wide cap).
|
||||
|
||||
**Verdict**: Closer but still missing the action-selection layer. Session DD CB triggers AFTER damage is done.
|
||||
|
||||
### Approach C — B + confidence-to-size scaling
|
||||
Add Q-distribution magnitude as continuous size multiplier alongside Kelly.
|
||||
|
||||
**Verdict**: Three multiplicative layers (Kelly × confidence × base) become hard to reason about. Confidence is implicit in Layer 2's quantile selection.
|
||||
|
||||
### Approach D — Add TrailTighten/Loosen wiring to any of A/B/C
|
||||
Close the 10%-policy-mass-wasted gap by activating dead actions.
|
||||
|
||||
**Verdict**: Orthogonal to risk management. Worth doing but small impact.
|
||||
|
||||
### Approach E — Layered risk stack (RECOMMENDED)
|
||||
|
||||
Replace the Kelly-first design with five distinct layers, each addressing a different risk concern:
|
||||
|
||||
| Layer | Concern | ISV-driven? | Production validation |
|
||||
|---|---|---|---|
|
||||
| 1. Hard CMDP gates | Catastrophic-loss safety (session DD, max position, max inventory) | ✓ | Highest (standard in live HFT) |
|
||||
| 2. IQN risk-averse τ | Downside-aware action selection | ✓ (adaptive τ) | Research, but our IQN gives it for free |
|
||||
| 3. Inventory penalty | Adverse selection, one-sided drift | ✓ (adaptive β) | Highest (Avellaneda-Stoikov is canonical HFT) |
|
||||
| 4. Kelly sizing | Edge-aware commitment | ✓ | Lower (research stage in RL context) |
|
||||
| (D) TrailTighten/Loosen | Close dead-action gap | ✓ (factor slots) | N/A — wiring fix |
|
||||
|
||||
**Why E**: research evidence (FineFT KDD 2026, plus broader distributional-RL literature) supports the layered approach. Each layer addresses a distinct failure mode visible in the JSONL. Combined, they form the "deployment-grade risk stack" pattern.
|
||||
|
||||
**Total scope**: ~320 LOC across 5 new + 5 modified files. Same atomic-commit discipline as the prior refactor.
|
||||
|
||||
---
|
||||
|
||||
## Detailed design (Approach E)
|
||||
|
||||
### Layer 1 — Hard CMDP constraints
|
||||
|
||||
Deterministic safety gates that override agent actions in `actions_to_market_targets.cu`. NOT learned by the agent. Each constraint is an independent ISV slot, AND'd together — any single trigger forces Hold (or partial-flatten).
|
||||
|
||||
**New ISV slots** (allocated starting at `RL_SLOTS_END = 662`):
|
||||
|
||||
| Slot | Name | Purpose | Bootstrap |
|
||||
|---|---|---|---|
|
||||
| 662 | `RL_SESSION_PNL_USD_INDEX` | Cumulative pnl since session boundary | 0.0 |
|
||||
| 663 | `RL_SESSION_DD_LIMIT_USD_INDEX` | Drawdown cap (USD, negative) | -3500.0 (= 10% × $35k) |
|
||||
| 664 | `RL_SESSION_DD_TRIGGERED_INDEX` | Sticky flag, set when DD breached | 0.0 |
|
||||
| 665 | `RL_MAX_OPEN_UNITS_INDEX` | Max concurrent open positions per batch | 4.0 (= current `MAX_UNITS`) |
|
||||
| 666 | `RL_CONSEC_LOSS_LIMIT_INDEX` | Max consecutive losses before cool-down | 10.0 |
|
||||
| 667 | `RL_CONSEC_LOSS_COUNT_INDEX` | Running consecutive-loss count | 0.0 |
|
||||
| 668 | `RL_COOLDOWN_REMAINING_STEPS_INDEX` | Cool-down period after limit hit | 0.0 |
|
||||
| 669 | `RL_COOLDOWN_DURATION_INDEX` | Default cool-down length (steps) | 500.0 |
|
||||
| 670 | `RL_NET_INVENTORY_LIMIT_USD_INDEX` | Cap on net one-sided exposure | 10000.0 |
|
||||
|
||||
**Kernel: `rl_cmdp_constraints_check.cu`** (~100 LOC, single-thread single-block, runs once per step before `actions_to_market_targets`)
|
||||
|
||||
```c
|
||||
// Layer 1: hard CMDP gates. Output: writes booleans to ISV slots that
|
||||
// actions_to_market_targets reads as overrides.
|
||||
//
|
||||
// Five independent constraints, ALL must be satisfied (= no override) for
|
||||
// the agent's chosen action to be executed. Any violation → force Hold.
|
||||
|
||||
extern "C" __global__ void rl_cmdp_constraints_check(
|
||||
float* __restrict__ isv,
|
||||
const float* __restrict__ realized_pnl_per_batch, // [b_size]
|
||||
const float* __restrict__ net_position_per_batch, // [b_size] (signed contracts)
|
||||
const u8* __restrict__ unit_active_per_batch, // [b_size × MAX_UNITS]
|
||||
const int* __restrict__ trade_outcome_per_batch, // [b_size] −1/0/+1
|
||||
int b_size
|
||||
) {
|
||||
if (threadIdx.x != 0 || blockIdx.x != 0) return;
|
||||
|
||||
// 1. Session pnl accumulation + DD check.
|
||||
float step_pnl = 0.0f;
|
||||
for (int b = 0; b < b_size; b++) step_pnl += realized_pnl_per_batch[b];
|
||||
const float session_pnl = isv[RL_SESSION_PNL_USD_INDEX] + step_pnl;
|
||||
isv[RL_SESSION_PNL_USD_INDEX] = session_pnl;
|
||||
if (session_pnl < isv[RL_SESSION_DD_LIMIT_USD_INDEX]) {
|
||||
isv[RL_SESSION_DD_TRIGGERED_INDEX] = 1.0f;
|
||||
}
|
||||
|
||||
// 2. Cool-down decrement.
|
||||
float cooldown = isv[RL_COOLDOWN_REMAINING_STEPS_INDEX];
|
||||
if (cooldown > 0.0f) {
|
||||
isv[RL_COOLDOWN_REMAINING_STEPS_INDEX] = cooldown - 1.0f;
|
||||
}
|
||||
|
||||
// 3. Consecutive loss tracking. Outcomes: -1 = loss, 0 = open/hold, +1 = win.
|
||||
float consec = isv[RL_CONSEC_LOSS_COUNT_INDEX];
|
||||
for (int b = 0; b < b_size; b++) {
|
||||
const int outcome = trade_outcome_per_batch[b];
|
||||
if (outcome == -1) consec += 1.0f;
|
||||
else if (outcome == 1) consec = 0.0f;
|
||||
// outcome == 0 (no change): consec unchanged
|
||||
}
|
||||
isv[RL_CONSEC_LOSS_COUNT_INDEX] = consec;
|
||||
if (consec >= isv[RL_CONSEC_LOSS_LIMIT_INDEX]) {
|
||||
isv[RL_COOLDOWN_REMAINING_STEPS_INDEX] = isv[RL_COOLDOWN_DURATION_INDEX];
|
||||
isv[RL_CONSEC_LOSS_COUNT_INDEX] = 0.0f; // reset after triggering
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Override logic in `actions_to_market_targets.cu`:
|
||||
|
||||
```c
|
||||
const bool dd_triggered = isv[RL_SESSION_DD_TRIGGERED_INDEX] >= 0.5f;
|
||||
const bool in_cooldown = isv[RL_COOLDOWN_REMAINING_STEPS_INDEX] > 0.0f;
|
||||
const bool override_hold = dd_triggered || in_cooldown;
|
||||
|
||||
if (override_hold) {
|
||||
actions[b] = ACTION_HOLD;
|
||||
target_lots[b] = 0;
|
||||
return; // skip rest of action translation
|
||||
}
|
||||
|
||||
// Max-open-units check (per-batch, dynamic):
|
||||
int active_units = 0;
|
||||
for (int u = 0; u < MAX_UNITS; u++) {
|
||||
if (unit_active[b * MAX_UNITS + u]) active_units++;
|
||||
}
|
||||
if (active_units >= (int)isv[RL_MAX_OPEN_UNITS_INDEX] &&
|
||||
is_opening_action(actions[b])) {
|
||||
actions[b] = ACTION_HOLD; // refuse new opens, allow closes
|
||||
}
|
||||
|
||||
// Inventory cap check:
|
||||
const float net_pos_usd = compute_net_position_usd(b);
|
||||
if (fabsf(net_pos_usd) > isv[RL_NET_INVENTORY_LIMIT_USD_INDEX] &&
|
||||
increases_one_sided_exposure(actions[b], net_pos_usd)) {
|
||||
actions[b] = ACTION_HOLD;
|
||||
}
|
||||
```
|
||||
|
||||
**Session boundary**: trainer resets `RL_SESSION_PNL_USD_INDEX`, `RL_SESSION_DD_TRIGGERED_INDEX`, `RL_CONSEC_LOSS_COUNT_INDEX` to 0 on each fold boundary (start of train phase, start of eval phase).
|
||||
|
||||
### Layer 2 — IQN risk-averse action selection
|
||||
|
||||
The genuinely-novel insight from research: **we already compute the IQN quantile distribution**. The standard action selector picks `argmax(E_τ[Q(s, a, τ)])` — discards the distribution shape. Instead, pick `argmax(Q(s, a, τ_action))` with adaptive `τ_action ∈ (0, 0.5]` — lower τ = more pessimistic = downside-aware decisions.
|
||||
|
||||
**New ISV slots**:
|
||||
|
||||
| Slot | Name | Purpose | Bootstrap |
|
||||
|---|---|---|---|
|
||||
| 671 | `RL_IQN_ACTION_TAU_INDEX` | Quantile used for action selection ∈ (0, 1] | 0.5 (= median, near current E[Q] behavior) |
|
||||
| 672 | `RL_IQN_ACTION_TAU_MIN_INDEX` | Lower bound | 0.1 |
|
||||
| 673 | `RL_IQN_ACTION_TAU_DD_SENSITIVITY_INDEX` | How fast τ drops with drawdown | 5.0 |
|
||||
|
||||
**Adaptive controller `rl_iqn_action_tau_controller.cu`** (~50 LOC):
|
||||
|
||||
```c
|
||||
// Adapt action-selection quantile based on recent drawdown.
|
||||
// Normal trading: τ = 0.5 (median ≈ expected-Q-like behavior).
|
||||
// In drawdown: τ → MIN (pessimistic — only take trades where even
|
||||
// the worst-case Q is positive).
|
||||
//
|
||||
// Recent drawdown signal: (session_pnl - max_session_pnl) / |starting_capital|
|
||||
// Bounded in [0, 1]: 0 = at peak, 1 = total wipeout.
|
||||
// τ_action = max(τ_min, 0.5 - sensitivity × drawdown_frac)
|
||||
|
||||
extern "C" __global__ void rl_iqn_action_tau_controller(
|
||||
float* __restrict__ isv
|
||||
) {
|
||||
if (threadIdx.x != 0 || blockIdx.x != 0) return;
|
||||
|
||||
const float session_pnl = isv[RL_SESSION_PNL_USD_INDEX];
|
||||
const float max_pnl = isv[RL_SESSION_MAX_PNL_USD_INDEX]; // tracked separately
|
||||
const float capital = isv[RL_STARTING_CAPITAL_USD_INDEX];
|
||||
|
||||
const float drawdown_usd = fmaxf(0.0f, max_pnl - session_pnl);
|
||||
const float drawdown_frac = drawdown_usd / fmaxf(capital, 1.0f);
|
||||
|
||||
const float tau_min = isv[RL_IQN_ACTION_TAU_MIN_INDEX];
|
||||
const float sensitivity = isv[RL_IQN_ACTION_TAU_DD_SENSITIVITY_INDEX];
|
||||
const float tau_target = fmaxf(tau_min, 0.5f - sensitivity * drawdown_frac);
|
||||
isv[RL_IQN_ACTION_TAU_INDEX] = tau_target;
|
||||
}
|
||||
```
|
||||
|
||||
**Modification to existing action selection kernel** (e.g., `rl_ensemble_action_value.cu` or wherever the argmax over Q happens):
|
||||
|
||||
```c
|
||||
// BEFORE:
|
||||
const float exp_q = compute_expected_q(s, a); // mean over sampled τ
|
||||
// argmax over actions on exp_q
|
||||
|
||||
// AFTER:
|
||||
const float tau_a = isv[RL_IQN_ACTION_TAU_INDEX]; // adaptive, default 0.5
|
||||
const float q_at_tau = compute_q_at_quantile(s, a, tau_a);
|
||||
// argmax over actions on q_at_tau
|
||||
```
|
||||
|
||||
The IQN kernel already computes `Q(s, a, τ)` for sampled `τ`. Selecting a specific quantile is a single-line indexing change.
|
||||
|
||||
**Critical interaction with C51 ensemble**: the agent uses C51+IQN ensemble. The ensemble blend (α) is already ISV-driven. We apply the risk-averse τ to the IQN side only, then blend as usual. The C51 side continues to use expected value because C51 doesn't naturally expose quantiles.
|
||||
|
||||
### Layer 3 — Inventory penalty (Avellaneda-Stoikov)
|
||||
|
||||
Add a reward shaping term proportional to `|net_position_per_batch|`. Discourages systematic one-sided exposure that bleeds via adverse selection.
|
||||
|
||||
**New ISV slots**:
|
||||
|
||||
| Slot | Name | Purpose | Bootstrap |
|
||||
|---|---|---|---|
|
||||
| 674 | `RL_INVENTORY_PENALTY_BETA_INDEX` | Penalty coefficient per |net_position| unit | 0.0 (sentinel — adaptive on first observation) |
|
||||
| 675 | `RL_INVENTORY_VARIANCE_EMA_INDEX` | Observed inventory variance for β scaling | 0.0 sentinel |
|
||||
|
||||
**Modification to `rl_fused_reward_pipeline.cu`** (~30 LOC):
|
||||
|
||||
```c
|
||||
// After raw reward computation, before clamps:
|
||||
const float net_pos = net_position_per_batch[b]; // signed contracts
|
||||
const float beta = isv[RL_INVENTORY_PENALTY_BETA_INDEX];
|
||||
const float penalty = beta * fabsf(net_pos); // always reduces reward
|
||||
shaped_reward = raw_reward - penalty;
|
||||
```
|
||||
|
||||
**Adaptive controller `rl_inventory_beta_controller.cu`** (~50 LOC) — derives β from observed inventory variance:
|
||||
|
||||
```c
|
||||
// β should scale so penalty is meaningful when inventory drifts but
|
||||
// negligible during balanced trading.
|
||||
// Target: penalty ≈ 1% of typical absolute reward when inventory at ~2σ.
|
||||
//
|
||||
// β = 0.01 × E[|reward|] / (2 × sqrt(var(inventory)))
|
||||
//
|
||||
// Sentinel-zero: hold β = 0 until variance EMA accumulates real signal.
|
||||
|
||||
const float reward_mag = isv[RL_REWARD_MAGNITUDE_EMA_INDEX]; // existing slot 614
|
||||
const float inv_var = isv[RL_INVENTORY_VARIANCE_EMA_INDEX];
|
||||
if (inv_var <= 0.0f || reward_mag <= 0.0f) return;
|
||||
|
||||
const float inv_std = sqrtf(inv_var);
|
||||
const float beta_target = 0.01f * reward_mag / (2.0f * inv_std);
|
||||
// Wiener blend with floor 0.4 (existing pearl pattern)
|
||||
isv[RL_INVENTORY_PENALTY_BETA_INDEX] = wiener_blend(
|
||||
isv[RL_INVENTORY_PENALTY_BETA_INDEX],
|
||||
beta_target,
|
||||
isv[RL_WIENER_ALPHA_FLOOR_INDEX]
|
||||
);
|
||||
```
|
||||
|
||||
### Layer 4 — Kelly-fraction position sizing
|
||||
|
||||
The original spec's contribution, demoted to fine-tuning layer. Operates *after* Layer 2 has filtered actions and Layer 1 has applied safety gates. Sizes the trades that survive.
|
||||
|
||||
**New ISV slots**:
|
||||
|
||||
| Slot | Name | Purpose | Bootstrap |
|
||||
|---|---|---|---|
|
||||
| 676 | `RL_KELLY_FRACTION_INDEX` | Position-size multiplier ∈ [0, 1] | 1.0 (full size pre-warmup) |
|
||||
| 677 | `RL_WIN_RATE_EMA_INDEX` | Observed win_rate EMA | 0.5 (neutral sentinel) |
|
||||
| 678 | `RL_AVG_WIN_USD_EMA_INDEX` | Observed avg_win | 0.0 sentinel |
|
||||
| 679 | `RL_AVG_LOSS_USD_EMA_INDEX` | Observed avg_loss | 0.0 sentinel |
|
||||
| 680 | `RL_KELLY_SAFETY_FRAC_INDEX` | Half-Kelly multiplier (Thorp) | 0.5 |
|
||||
| 681 | `RL_KELLY_MIN_TRADES_FOR_RELEASE_INDEX` | Warmup gate | 1000.0 |
|
||||
|
||||
**Kernel `rl_kelly_fraction_controller.cu`** (~80 LOC) — same design as original spec, repeated here for completeness:
|
||||
|
||||
```c
|
||||
extern "C" __global__ void rl_kelly_fraction_controller(float* __restrict__ isv) {
|
||||
if (threadIdx.x != 0 || blockIdx.x != 0) return;
|
||||
|
||||
// Warmup: hold at bootstrap until enough closed trades.
|
||||
if (isv[RL_CUMULATIVE_DONES_INDEX] < isv[RL_KELLY_MIN_TRADES_FOR_RELEASE_INDEX]) {
|
||||
isv[RL_KELLY_FRACTION_INDEX] = 1.0f;
|
||||
return;
|
||||
}
|
||||
|
||||
const float p = isv[RL_WIN_RATE_EMA_INDEX];
|
||||
const float avg_win = isv[RL_AVG_WIN_USD_EMA_INDEX];
|
||||
const float avg_loss = isv[RL_AVG_LOSS_USD_EMA_INDEX];
|
||||
if (avg_loss <= 0.0f) { isv[RL_KELLY_FRACTION_INDEX] = 1.0f; return; }
|
||||
|
||||
const float b = avg_win / avg_loss;
|
||||
const float f_kelly = (p * b - (1.0f - p)) / b;
|
||||
const float safety = isv[RL_KELLY_SAFETY_FRAC_INDEX];
|
||||
float f = f_kelly * safety;
|
||||
f = fmaxf(0.0f, fminf(f, 1.0f));
|
||||
isv[RL_KELLY_FRACTION_INDEX] = f;
|
||||
}
|
||||
```
|
||||
|
||||
**Modification in `actions_to_market_targets.cu`** (~3 LOC):
|
||||
|
||||
```c
|
||||
// After Layer 1 overrides have been applied (DD CB, cooldown, etc.):
|
||||
const float kelly = isv[RL_KELLY_FRACTION_INDEX];
|
||||
target_lots[b] = (int)ceilf((float)target_lots[b] * kelly);
|
||||
// If kelly = 0, target_lots = 0 = no commitment.
|
||||
```
|
||||
|
||||
### Layer D — TrailTighten/Loosen wiring
|
||||
|
||||
Resolve the 10%-policy-mass-wasted gap per [[pearl_dead_trail_stop_actions_a7_a8]]:
|
||||
|
||||
**New ISV slots**:
|
||||
|
||||
| Slot | Name | Purpose | Bootstrap |
|
||||
|---|---|---|---|
|
||||
| 682 | `RL_TRAIL_TIGHTEN_FACTOR_INDEX` | Multiplier on trail distance for a7 | 0.9 (tighten by 10%) |
|
||||
| 683 | `RL_TRAIL_LOOSEN_FACTOR_INDEX` | Multiplier on trail distance for a8 | 1.1 (loosen by 10%) |
|
||||
|
||||
**Modification to `actions_to_market_targets.cu`** wires a7/a8 to actually mutate `unit_trail_distance_d`:
|
||||
|
||||
```c
|
||||
case ACTION_TRAIL_TIGHTEN: {
|
||||
const float factor = isv[RL_TRAIL_TIGHTEN_FACTOR_INDEX];
|
||||
for (int u = 0; u < MAX_UNITS; u++) {
|
||||
if (unit_active_d[b * MAX_UNITS + u]) {
|
||||
unit_trail_distance_d[b * MAX_UNITS + u] *= factor;
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
case ACTION_TRAIL_LOOSEN: {
|
||||
const float factor = isv[RL_TRAIL_LOOSEN_FACTOR_INDEX];
|
||||
for (int u = 0; u < MAX_UNITS; u++) {
|
||||
if (unit_active_d[b * MAX_UNITS + u]) {
|
||||
unit_trail_distance_d[b * MAX_UNITS + u] *= factor;
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Slot allocation summary
|
||||
|
||||
22 new ISV slots, `RL_SLOTS_END` 662 → 684.
|
||||
|
||||
| Range | Layer | Purpose |
|
||||
|---|---|---|
|
||||
| 662–670 | 1 (CMDP gates) | Session pnl, DD limit, max open, consec loss, cooldown, inventory cap (9 slots) |
|
||||
| 671–673 | 2 (IQN τ) | Action τ, τ_min, DD sensitivity (3 slots) |
|
||||
| 674–675 | 3 (inventory penalty) | β + inventory variance EMA (2 slots) |
|
||||
| 676–681 | 4 (Kelly) | Kelly fraction + EMAs + safety + warmup (6 slots) |
|
||||
| 682–683 | D (trail) | Tighten/loosen factors (2 slots) |
|
||||
|
||||
Memory impact: 88 bytes additional (negligible).
|
||||
|
||||
---
|
||||
|
||||
## Per-step pipeline order
|
||||
|
||||
The per-step trainer pipeline must execute Layers in this order:
|
||||
|
||||
1. EMA producers (existing — kl_pi_ema, q_divergence_ema, etc., PLUS new: win_rate_ema, avg_win_ema, avg_loss_ema, inventory_variance_ema)
|
||||
2. Welford variance updates (existing kernel from b1ef6664a)
|
||||
3. Adaptive controllers (existing 12 — fused kernel)
|
||||
4. **Layer 1 — `rl_cmdp_constraints_check`** (new)
|
||||
5. **Layer 2 — `rl_iqn_action_tau_controller`** (new — adapts τ from drawdown)
|
||||
6. **Layer 3 — `rl_inventory_beta_controller`** (new — adapts β)
|
||||
7. **Layer 4 — `rl_kelly_fraction_controller`** (new)
|
||||
8. **Layer 5 — TrailTighten/Loosen ISV slots** (no per-step kernel; ISV slots read during action translation)
|
||||
9. Action selection — IQN ensemble uses adaptive τ for argmax
|
||||
10. `actions_to_market_targets` — applies Layer 1 overrides + Layer 4 Kelly scaling + Layer D trail mutations
|
||||
11. Reward computation — `rl_fused_reward_pipeline` applies Layer 3 inventory penalty
|
||||
12. PPO/DQN updates as usual
|
||||
|
||||
---
|
||||
|
||||
## Files
|
||||
|
||||
### New
|
||||
|
||||
- `crates/ml-alpha/cuda/rl_cmdp_constraints_check.cu` (~100 LOC)
|
||||
- `crates/ml-alpha/cuda/rl_iqn_action_tau_controller.cu` (~50 LOC)
|
||||
- `crates/ml-alpha/cuda/rl_inventory_beta_controller.cu` (~50 LOC)
|
||||
- `crates/ml-alpha/cuda/rl_kelly_fraction_controller.cu` (~80 LOC)
|
||||
- `crates/ml-alpha/cuda/rl_win_rate_ema_update.cu` (~40 LOC if not derivable)
|
||||
- `crates/ml-alpha/cuda/rl_avg_win_loss_ema_update.cu` (~50 LOC)
|
||||
- `crates/ml-alpha/cuda/rl_inventory_variance_update.cu` (~40 LOC)
|
||||
- `crates/ml-alpha/tests/risk_stack_invariants.rs` (~10 GPU-oracle gates)
|
||||
|
||||
### Modified
|
||||
|
||||
- `crates/ml-alpha/src/rl/isv_slots.rs` — +22 slots, `RL_SLOTS_END` 662 → 684
|
||||
- `crates/ml-alpha/build.rs` — register new cubins
|
||||
- `crates/ml-alpha/cuda/actions_to_market_targets.cu` — Layer 1 overrides + Layer 4 Kelly scaling + Layer D trail mutations
|
||||
- `crates/ml-alpha/cuda/rl_fused_reward_pipeline.cu` — Layer 3 inventory penalty term
|
||||
- `crates/ml-alpha/cuda/rl_ensemble_action_value.cu` (or wherever IQN action selection happens) — Layer 2 quantile-τ argmax
|
||||
- `crates/ml-alpha/cuda/rl_fused_controllers.cu` — add new branches for τ, β, Kelly (matching the fused-kernel pattern from b1ef6664a)
|
||||
- `crates/ml-alpha/src/trainer/integrated.rs` — load new cubins, wire per-step launches, bootstrap entries, session reset on fold boundary
|
||||
|
||||
### Not touched
|
||||
|
||||
- 12 adaptive controllers from b1ef6664a (orthogonal layer)
|
||||
- Welford-variance kernel infrastructure (reused)
|
||||
- Phase 4.5 advantage normalization
|
||||
- Perception encoder, Mamba2, CfC heads
|
||||
- Decision policy gradient (the agent still picks discrete actions)
|
||||
|
||||
---
|
||||
|
||||
## Validation gates
|
||||
|
||||
### Local unit-level (RTX 3050 Ti)
|
||||
|
||||
**G1 — CMDP DD trigger**: Write `session_pnl = -4000` (below limit -3500) → assert triggered=1, agent actions overridden to Hold.
|
||||
|
||||
**G2 — CMDP cooldown**: Trigger 10 consecutive losses → assert cooldown counter set to 500, agent actions overridden until counter reaches 0.
|
||||
|
||||
**G3 — CMDP max-open**: Pre-set 4 active units, agent picks opening action → assert action forced to Hold.
|
||||
|
||||
**G4 — CMDP inventory cap**: Set `net_position = $11k` (above $10k limit), agent picks opening-same-side action → assert Hold.
|
||||
|
||||
**G5 — IQN τ adapts to drawdown**: Set `session_pnl = -1500` (≈4% drawdown of $35k capital) → assert `τ_action ≈ max(0.1, 0.5 - 5 × 0.043) ≈ 0.285` (mid-pessimistic).
|
||||
|
||||
**G6 — IQN τ at no-drawdown**: Set `session_pnl = max_pnl` → assert `τ_action = 0.5`.
|
||||
|
||||
**G7 — Inventory penalty β adapts**: Set `reward_mag_ema = 0.1`, `inv_variance_ema = 4.0` (σ=2) → assert `β = 0.01 × 0.1 / (2 × 2) = 0.00025`. Negligible at balanced inventory, real at high.
|
||||
|
||||
**G8 — Kelly math at known edge**: Feed `win_rate=0.55, avg_win=1.8, avg_loss=1.0, trades=2000` → expect `f = 0.5 × ((0.55×1.8 - 0.45)/1.8) = 0.5 × 0.30 = 0.15`.
|
||||
|
||||
**G9 — Kelly returns 0 at negative edge**: Feed `win_rate=0.21, avg_win=1.8, avg_loss=1.0, trades=2000` → expect `f = 0` (Kelly negative → clamped).
|
||||
|
||||
**G10 — TrailTighten reduces distance**: Pre-set `unit_trail_distance = 100`, agent picks a7 → assert distance becomes 90.
|
||||
|
||||
**G11 — integrated_trainer_smoke** still passes end-to-end.
|
||||
|
||||
**G12 — 1k local smoke at b=128**: completed_clean, no NaN, all 5 layers' ISV slots populated and tracking sensibly.
|
||||
|
||||
### Cluster
|
||||
|
||||
**G13 — Walk-forward fold 1 re-run** (same params as today's 62x7p, new SHA). Combined layers should close the eval gap.
|
||||
|
||||
Targets (each is a soft pass; combination of 3+ passes = G13 PASS):
|
||||
- eval pnl > -$200k (better than today's -$616k, **PRIMARY**)
|
||||
- profit_factor > 0.90 (vs 0.83)
|
||||
- max_drawdown_pct < 15% (vs 18.7%)
|
||||
- Kelly fraction: ends < 1.0 if observed edge negative (shows Layer 4 active)
|
||||
- IQN τ: ends < 0.5 if any drawdown occurred (shows Layer 2 adaptive)
|
||||
- DD breaker: fires when expected (or never if pnl stays positive)
|
||||
- Inventory penalty β: > 0 throughout (shows Layer 3 adapted off bootstrap)
|
||||
|
||||
**G14 — ksll2 regression test**: same params as today's rwk9t. Final pnl ≥ +$13M (within 20% of +$15.81M). Confirms layered stack doesn't regress large-data regime.
|
||||
|
||||
---
|
||||
|
||||
## Open questions
|
||||
|
||||
1. **Starting capital ($35k) per [[project_ml_alpha_starting_capital]]** — should we add `RL_STARTING_CAPITAL_USD_INDEX` as an ISV slot? Yes, makes session DD cap and Layer 2 drawdown ratio both adaptive to capital changes. Add to Layer 1.
|
||||
|
||||
2. **Layer 2 sensitivity (5.0)** — at session DD 10%, τ drops from 0.5 to 0. At 4% drawdown, τ ≈ 0.3. Reasonable? Tunable via ISV. May need empirical sweep.
|
||||
|
||||
3. **Inventory variance EMA — measured over what window?** Need to choose Wiener-α. Default `RL_WIENER_ALPHA_FLOOR_INDEX = 0.4` from existing pearl. Adaptive could come later.
|
||||
|
||||
4. **Layer 3 β bootstrap (0.0 = sentinel)** — agent gets ZERO inventory penalty until variance EMA accumulates. Acceptable cold-start.
|
||||
|
||||
5. **Should `RL_MAX_OPEN_UNITS_INDEX` = 4 (current `MAX_UNITS`) be redundant?** The action space already enforces this via unit_active tracking. Yes, redundant but cheap; treats it as belt-and-braces.
|
||||
|
||||
6. **Approach E vs the original B+D — anything we lose?** The `rl_session_drawdown_check.cu` kernel from the prior design is now folded into the broader `rl_cmdp_constraints_check.cu`. Same functionality, more layered.
|
||||
|
||||
7. **Multi-seed validation?** A single fold-1 result is noisy (1024 trades, high variance). Worth running 3 seeds to confirm. Defer to post-validation cluster sweep if G13 single-seed result is marginal.
|
||||
|
||||
8. **Regime detection (Approach E+1)?** FineFT (KDD 2026) demonstrates risk-aware ensemble routing on regime changes. Heavyweight implementation (online clustering + meta-controller). Defer to follow-up spec if Approach E doesn't close the eval gap.
|
||||
|
||||
---
|
||||
|
||||
## Risks and mitigations
|
||||
|
||||
- **Layer 2 τ-quantile path may not be exposed by current IQN kernel signature** → audit `rl_ensemble_action_value.cu` first; if `Q(s, a, τ)` isn't readily indexable, may need kernel restructure (small change).
|
||||
|
||||
- **Kelly EMA estimates unreliable in early training** → warmup gate (1000 trades) holds Kelly at bootstrap. Same pattern as b1ef6664a's reward_floor cumulative-dones gate.
|
||||
|
||||
- **Five layers compounding may make agent ultra-conservative — eval might be 0 with no trades** → that's actually fine for G13 (eval pnl > -$200k bar). But for G14 (ksll2 +$13M), need to verify layers don't over-restrict on full-data regime. Mitigation: warmup gates default to "behave like the prior code" until enough signal accumulates.
|
||||
|
||||
- **Session boundary resets must be wired by trainer** → must call ISV resets on train→eval transition. Test G11 catches this. Plan task explicitly enumerates the reset points.
|
||||
|
||||
- **Approach D wiring changes action distribution dynamics** → trainable but worth a regression check on entropy. 1k smoke catches it.
|
||||
|
||||
- **CMDP "force Hold" creates discontinuity in policy gradient** → standard technique in CMDP-RL literature, well-understood. Override doesn't update gradient through the constraint, only through the Hold action's natural Q.
|
||||
|
||||
---
|
||||
|
||||
## Next step after approval
|
||||
|
||||
Write the plan: `docs/superpowers/plans/2026-05-30-adaptive-risk-management-plan.md`. Task-level decomposition:
|
||||
|
||||
1. ISV slot allocation (22 slots)
|
||||
2. Layer 1 (CMDP constraints kernel + actions_to_market_targets overrides)
|
||||
3. Layer 2 (IQN τ controller + action-selection kernel modification)
|
||||
4. Layer 3 (inventory penalty in reward + β controller)
|
||||
5. Layer 4 (Kelly controller + actions_to_market_targets sizing)
|
||||
6. Layer D (TrailTighten/Loosen wiring)
|
||||
7. New EMA producers (win_rate, avg_win, avg_loss, inventory variance)
|
||||
8. Fused-kernel integration of new controllers
|
||||
9. Trainer integration (per-step launches + bootstrap + session reset)
|
||||
10. GPU-oracle invariant tests (G1-G11)
|
||||
11. 1k local smoke + sanitizer (G12)
|
||||
12. Atomic commit
|
||||
13. Cluster G13 (fold 1 re-run) + G14 (ksll2 regression)
|
||||
|
||||
Expected scope: ~320 LOC across 8 new + 7 modified files, ~13 tasks, single atomic commit. Same `feedback_no_partial_refactor` discipline.
|
||||
@@ -0,0 +1,146 @@
|
||||
# C51 Atom Resolution — Design Alternatives (post Fix F failure)
|
||||
|
||||
**Status**: Investigation + design-alternatives document. **No code change recommended at this time** — see Section 5.
|
||||
|
||||
**Why this exists**: cluster `alpha-rl-qrgjr` (v7, SHA `0fe825a8c`) tested Fix F (atom span decoupled from clamp ceiling via `mean_abs_pnl_ema × atom_margin = 10`). Local b=128 smoke looked great (qpa=+0.958, Δz=8.7 vs prior 184), but at cluster b=1024 **qpa crashed from +0.898 at step 371 to −0.976 at step 500** and stayed stuck through step 1259. Fix F was reverted in `6d4a962e5`.
|
||||
|
||||
This document captures the diagnosis and the architectural alternatives we considered, so a future iteration doesn't repeat the same mistake.
|
||||
|
||||
---
|
||||
|
||||
## 1. Why Fix F failed — the structural constraint
|
||||
|
||||
C51 distributional Q-learning has a self-consistency requirement between **atom span** and **Bellman target range**:
|
||||
|
||||
```
|
||||
V_target(s, a) = scaled_reward + γ · V(s')
|
||||
V_max = clamp_max + γ · V_max (fixed-point)
|
||||
= clamp_max / (1 − γ)
|
||||
```
|
||||
|
||||
For typical training conditions:
|
||||
- `clamp_max` (WIN) is the post-scale, post-clamp positive reward ceiling — adaptive via reward_clamp_controller (Fix C). Hovers at MARGIN × pos_max_ema.
|
||||
- `γ` is the discount factor, controlled by `rl_gamma_controller` toward `1 − 1/(2d)` where `d` is mean trade duration. Cluster values: 0.95–0.99.
|
||||
|
||||
At γ=0.98 and WIN=44 (v7 step 500):
|
||||
- Structural minimum `atom_max ≥ 44 / (1 − 0.98) = 2200`
|
||||
- Fix F forced `atom_max` to `atom_margin × mean_abs_pnl ≈ 10 × 6 = 60`
|
||||
- **V_target distribution: [-clamp_loss, +clamp_max + γ·V_max] = [-200, +260]** — sometimes wider as Q's V_dq head outputs scale up
|
||||
- Atom span [-63, +63] truncated V_target → Bellman projection saturated at top/bottom atoms
|
||||
|
||||
**What that did to learning:**
|
||||
- `Σ atom × prob` for Q stayed bounded by atom_max=63, but V regression (separate scalar head, popart-normalized) could predict higher
|
||||
- PPO advantage `A = Q(s,a) − V(s)` became inconsistent: Q clamped, V not
|
||||
- Q-π distillation `softmax(E_Q/τ)` became degenerate when E_Q is uniform-at-ceiling across actions
|
||||
- π received conflicting gradient signals from PPO and from distillation → qpa decorrelated → −0.97 stuck
|
||||
|
||||
**Key insight**: the atom span isn't *free* to size for "typical reward magnitude" — it's *constrained* by Bellman self-consistency at the chosen γ. The pre-Fix-F design (atom span EWMA-tracking WIN) was implementing `atom_span ≈ WIN/(1−γ)` via dynamics rather than explicit formula. That's mathematically correct.
|
||||
|
||||
---
|
||||
|
||||
## 2. Why local smoke missed this
|
||||
|
||||
v7 local smoke (b=128, 1000 steps) showed qpa=+0.958 with Δz=8.7. Why didn't it crash?
|
||||
|
||||
- **Smaller batch → tighter V_target distribution**: at b=128 the per-step diversity of V(s') is reduced. The empirical V_target range stays within the (too-tight) atom span by chance.
|
||||
- **Shorter horizon**: 1000 steps doesn't let V regression saturate its capacity. Q-V divergence builds over many backups, so the failure is downstream.
|
||||
- **γ slower to grow**: at smoke scale γ stayed at 0.98 instead of climbing toward 0.99+, so the structural minimum atom_max was smaller.
|
||||
|
||||
**Lesson for future protocol**: local b=128 smoke validates *kernel correctness* and *immediate dynamics*, but not *Q-V self-consistency at scale*. Either run a smaller cluster b=1024 5k-step smoke before committing structural changes, or add an explicit invariant test that probes `atom_span × (1−γ) ≥ WIN`.
|
||||
|
||||
---
|
||||
|
||||
## 3. The Pareto frontier — there is no free lunch
|
||||
|
||||
The atom-resolution problem is bounded by a fundamental trade-off:
|
||||
|
||||
| Design | Δz (resolution) | Q-V consistency | Notes |
|
||||
|---|---|---|---|
|
||||
| **Wide atom span = WIN/(1−γ)** (current, v5/v8) | Coarse (~100-200) | ✅ Self-consistent | Pre-Fix-F default. Q can represent full V_target range. |
|
||||
| **Tight atom span = k × typical reward** (Fix F) | Fine (~5-10) | ❌ V_target saturates | What we tried. qpa crashes when V_dq grows beyond atom span. |
|
||||
| **Non-uniform atoms** (concentrated near 0, spread for tails) | Fine where it matters | ✅ With proper projection | Major architectural rework. Touches bellman_target_projection.cu + dueling_q_head.cu + atom_supports.cu + offline training. |
|
||||
| **Lower γ** (e.g. 0.9 instead of 0.99) | Tighter (γ ↓ → atom span ↓) | ✅ | Changes policy time horizon — myopic agent. May lose surfer-style trades. |
|
||||
| **Two-headed Q** (categorical for direction + regression for magnitude) | Hybrid | ✅ With careful loss | Major architectural rework. Q head doubles in size. |
|
||||
|
||||
**The lottery-ticket strategy the agent has discovered has R-multiple ~14-43.** With γ near 1, the structural minimum atom span IS coarse. No tuning of `atom_margin` alone resolves this.
|
||||
|
||||
---
|
||||
|
||||
## 4. Alternatives evaluated
|
||||
|
||||
### 4.A — Increase `atom_margin` (Fix F-v2)
|
||||
|
||||
Set `atom_margin` from 10 → 100. Atom span = 100 × mean_abs_pnl ≈ 600. Closer to v5's 2000 but tighter.
|
||||
|
||||
**Risk**: at γ=0.99, structural minimum is `WIN × 100 ≈ 4400`. Still under-sized. Possibly buys headroom but doesn't resolve the structural constraint.
|
||||
|
||||
**Verdict**: not a clean fix. Defer.
|
||||
|
||||
### 4.B — Anchor on `WIN / (1 − γ_max)` explicitly
|
||||
|
||||
Make the structural formula explicit:
|
||||
|
||||
```c
|
||||
const float gamma_max = isv[RL_GAMMA_MAX_INDEX]; // 0.99 typical
|
||||
const float headroom = 1.0f / fmaxf(0.01f, 1.0f - gamma_max); // = 100 at γ=0.99
|
||||
const float v_max_target = fmaxf(v_bound_floor, headroom * win_bound);
|
||||
```
|
||||
|
||||
Equivalent to v5/v8's behavior at steady-state, but immediate response instead of EWMA-lagged. Could improve transient handling.
|
||||
|
||||
**Verdict**: marginal improvement, low risk. Defer to a future cleanup pass.
|
||||
|
||||
### 4.C — Non-uniform atoms
|
||||
|
||||
Atom positions concentrated near zero (e.g., quantiles of a typical-magnitude distribution) and exponentially spread for tails.
|
||||
|
||||
Requires:
|
||||
- New atom-support kernel emitting non-uniform `atom_supports_d[Q_N_ATOMS]`
|
||||
- New `bellman_target_projection` math that handles non-uniform projection (cross-quantile interpolation)
|
||||
- Coordination with `dueling_q_head` decomposition (V = mean of atoms × supports)
|
||||
|
||||
**Estimated cost**: 1-2 weeks. Touches every file that reads atom_supports_d. Risk: hot-path performance impact on per-step projection.
|
||||
|
||||
**Verdict**: real improvement. Defer to a major refactor cycle.
|
||||
|
||||
### 4.D — Adaptive γ tied to fold/training-phase
|
||||
|
||||
Start γ low (0.9 — narrow horizon) and grow γ over training as the agent stabilizes. Atom span follows γ via the existing controller. Early-training has fine resolution; late-training has coarse-but-correct atom span.
|
||||
|
||||
**Risk**: changes the policy's effective time horizon during training — could mask issues that only appear at high γ.
|
||||
|
||||
**Verdict**: interesting research direction. Not an immediate fix.
|
||||
|
||||
### 4.E — Two-headed Q (categorical direction + scalar magnitude)
|
||||
|
||||
Standard architectural pattern in RL — Q decomposed as `Q(s,a) = direction_class(s,a) × magnitude(s,a)`. Categorical loss on direction (signed atom), regression on magnitude.
|
||||
|
||||
**Estimated cost**: 1-2 weeks. Full retraining required. Risk: distillation logic needs rewriting.
|
||||
|
||||
**Verdict**: real improvement but expensive. Defer.
|
||||
|
||||
---
|
||||
|
||||
## 5. Recommendation
|
||||
|
||||
**Ship A+B+C+D+E only (v8 SHA `6d4a962e5`) and observe full-run results.**
|
||||
|
||||
The pre-Fix-F design implements the structurally-correct atom span via WIN-tracking EWMA. The atom resolution it produces (Δz ~100-200) is the *minimum coarseness* compatible with the agent's R-multiple-43 strategy at γ=0.99. This is not a bug — it's a property of the chosen reward distribution × discount factor combination.
|
||||
|
||||
If v8 trains cleanly through 20k + 5k eval with positive eval PnL, the resolution coarseness is acceptable and **no further atom-span change is needed**. The agent is finding alpha within the constraints.
|
||||
|
||||
If v8 trains but eval is mediocre, the next investigation is whether Q's coarse mid-range resolution is the bottleneck — that would justify the cost of Option 4.C (non-uniform atoms) or 4.E (two-headed Q).
|
||||
|
||||
If v8 fails (qpa crash, NaN, unbounded loss), then the issue is NOT atom resolution and we look elsewhere.
|
||||
|
||||
---
|
||||
|
||||
## 6. Memory pearl to save (after v8 result)
|
||||
|
||||
Pending — capture either:
|
||||
|
||||
> **pearl_atom_span_must_match_bellman_horizon**: atom span has a structural minimum of `WIN/(1−γ)` for Q-V Bellman self-consistency. Sizing atoms below this saturates V_target projection → Q clamps at top atom → distillation degenerates → qpa crashes. Local b=128 smoke under-exercises the failure mode (smaller V_target distribution range). Tested empirically in `alpha-rl-qrgjr` v7 (2026-05-30, Fix F atom_margin=10).
|
||||
|
||||
OR (depending on v8 outcome):
|
||||
|
||||
> **pearl_lottery_ticket_strategy_forces_coarse_atoms**: when the agent discovers a high-R-multiple lottery-ticket strategy, atom span must scale with `WIN/(1−γ)` which becomes coarse at γ=0.99. Resolution loss is structural, not a bug. Q learns direction correctly even at Δz=200; PPO advantage via V regression preserves magnitude. Confirmed in `alpha-rl-vxbpq` v8 (2026-05-31).
|
||||
@@ -0,0 +1,179 @@
|
||||
# C51 Atom Span — Math Validation and Design Justification
|
||||
|
||||
**Status**: Math-validation document. Implements a *diagnostic-only* emit (no behavior change). Replaces speculation in `2026-05-31-c51-atom-resolution-design-alternatives.md` with a formal analysis.
|
||||
|
||||
**Why this exists**: Through the 2026-05-30/31 session we made three claims about C51 atom span sizing that need formal validation:
|
||||
1. (Fix F design): atom span should track typical-magnitude reward → wrong, broke v7 at step 500
|
||||
2. (Fix F post-mortem): "structural minimum atom_max ≥ WIN/(1−γ)" → overstated, see §3
|
||||
3. (v8 design, currently shipping): atom span EWMA-tracks WIN_bound → empirically works (v8 qpa=+0.974, pnl_cum=$164M @ step 16830)
|
||||
|
||||
This doc proves #3 is correct and gives the formal reason #1 and the "alternative" Fix F-v2 (anchor on `V_target_max_observed × (1+ε)`) both fail.
|
||||
|
||||
---
|
||||
|
||||
## 1. Notation and setup
|
||||
|
||||
Let:
|
||||
- `r ∈ [-LOSS, +WIN]` = post-scale, post-clamp scalar reward emitted to V regression / Bellman target. WIN, LOSS are the adaptive clamp bounds (`rl_reward_clamp_controller` Fix C).
|
||||
- `V(s) ∈ [V_min, V_max]` = V regression head output. Atom span is `[V_min, V_max]`. Bounded because V is normalized through popart against the same atom basis.
|
||||
- `γ ∈ [γ_min, γ_max]` = discount factor, `rl_gamma_controller` driving toward `1 − 1/(2d)` for mean trade duration `d`. In v8, γ stabilized around **0.96-0.97**.
|
||||
- `V_target(s, a) = r(s, a) + γ · V(s')` = the C51 Bellman target before atom projection.
|
||||
|
||||
The kernel `bellman_target_projection` maps `V_target → atom_probabilities` via linear interpolation. If `V_target > V_max`, it projects to atom index 20 with probability 1.0 (the *clip*).
|
||||
|
||||
---
|
||||
|
||||
## 2. The dynamic constraint vs. the fixed-point constraint
|
||||
|
||||
### 2.1 The fixed-point bound (overstated in earlier doc)
|
||||
|
||||
If we *assume* `V(s) ≡ V_max` for the highest-value state and recurse:
|
||||
|
||||
```
|
||||
V_max = WIN + γ · V_max
|
||||
V_max · (1 − γ) = WIN
|
||||
V_max = WIN / (1 − γ) [FIXED-POINT bound]
|
||||
```
|
||||
|
||||
For γ=0.97, WIN=44: `V_max_fp ≈ 1467`. v8 step 5861 had observed `V_max = 87` and qpa=+0.985. The empirical V_max is **17× below the fixed-point bound** and the system is healthy.
|
||||
|
||||
**Why the fixed-point bound is overstated**: it requires that the *learned* V-network actually outputs values approaching `V_max`. In practice that happens only if (a) Q-learning has fully converged AND (b) the optimal trajectory's discounted infinite-horizon reward equals the bound. Neither is true in training; (b) is generally false even at convergence because most states have value far below the rare "lottery ticket" state.
|
||||
|
||||
### 2.2 The dynamic constraint (actually binding)
|
||||
|
||||
The real constraint on atom_max during training is:
|
||||
|
||||
```
|
||||
atom_max ≥ max V_target_observed
|
||||
= max (scaled_reward + γ · V_observed(s'))
|
||||
≤ WIN + γ · V_max_observed [DYNAMIC bound]
|
||||
```
|
||||
|
||||
Where `V_max_observed` is the network's *current* output range — bounded by the atom span itself, but not necessarily *at* atom_max.
|
||||
|
||||
This is a *self-consistent* constraint. It says: as long as atom_max grows fast enough to stay ahead of `V_max_observed`, no clipping occurs. If atom_max LAGS, V_target clips at top atom and Q distributional learning saturates.
|
||||
|
||||
### 2.3 Self-consistency of the current EWMA design
|
||||
|
||||
The v8 design (`rl_reward_clamp_controller.cu` Step 5):
|
||||
|
||||
```c
|
||||
v_max_target = max(v_bound_floor, win_bound)
|
||||
v_max_new = (1−α) · v_max_prev + α · v_max_target [α = 0.001, half-life ~700 steps]
|
||||
```
|
||||
|
||||
This is conservative in a specific way:
|
||||
- `win_bound = MARGIN × pos_max_ema` tracks the *outer envelope* of reward magnitudes
|
||||
- Atom span grows TOWARD `win_bound`, ratcheted up by transients
|
||||
- Stays *above* `V_max_observed` because V can only learn to values that fit within atoms (categorical projection)
|
||||
- Slow EWMA prevents thrashing
|
||||
|
||||
**Key insight**: atom_max being driven by the *reward* envelope rather than the *V_target* envelope provides an implicit safety margin equal to `γ × V_max_observed` worth of headroom.
|
||||
|
||||
---
|
||||
|
||||
## 3. Why Fix F and Fix F-v2 fail at high γ
|
||||
|
||||
### 3.1 Fix F — anchor on `mean_abs_pnl × atom_margin = 10`
|
||||
|
||||
Empirically: at v7 cluster step 500, `mean_abs_pnl ≈ 6` (scaled), so target atom_max = 63. But `V_max_observed` had grown beyond 60 by then (V learned to value good states aggressively under PPO+Kelly). V_target = 200 + 0.97×60 = 258. Atom_max=63 ≪ 258 → top-atom saturation → Q distillation degeneracy → qpa crash.
|
||||
|
||||
**Math**: Fix F's anchor `atom_margin × mean_abs_pnl` is a *typical-magnitude* signal. It has no relationship to the dynamic V_target range. Necessarily violates the dynamic bound when V learns aggressive valuations.
|
||||
|
||||
### 3.2 Fix F-v2 — anchor on `V_target_max_observed × (1+ε)`
|
||||
|
||||
Proposed but not implemented. Let's check the math for stability.
|
||||
|
||||
Suppose atom_max tracks `(1+ε) × V_target_max_observed` exactly. Then at quasi-equilibrium:
|
||||
|
||||
```
|
||||
V_target_max = WIN + γ · V_max_observed
|
||||
= WIN + γ · atom_max / (1+ε) (if V saturates atoms)
|
||||
atom_max = (1+ε) · V_target_max
|
||||
= (1+ε) · WIN + (1+ε) · γ · atom_max / (1+ε)
|
||||
= (1+ε) · WIN + γ · atom_max
|
||||
|
||||
atom_max · (1 − γ) = (1+ε) · WIN
|
||||
atom_max = (1+ε) · WIN / (1 − γ)
|
||||
```
|
||||
|
||||
So Fix F-v2's steady-state atom_max is `(1+ε) × WIN/(1−γ)`, which equals the fixed-point bound times `(1+ε)`. Specifically:
|
||||
|
||||
| γ | ε = 0.05 | ε = 0.1 | ε = 0.2 |
|
||||
|---|---|---|---|
|
||||
| 0.90 | 1.05·WIN/0.1 = 10.5·WIN | 11·WIN | 12·WIN |
|
||||
| 0.97 | 35·WIN | 37·WIN | 40·WIN |
|
||||
| 0.99 | 105·WIN | 110·WIN | 120·WIN |
|
||||
| 0.999 | 1050·WIN | 1100·WIN | 1200·WIN |
|
||||
|
||||
These are bigger than the current v8 design produces empirically (atom_max = 87 at step 5861 with WIN=42, γ=0.97 — predicted by Fix F-v2 would be 35×42 = 1470, ~17× bigger).
|
||||
|
||||
**Why current v8 outperforms Fix F-v2 at steady state**: because v8's atom_max is driven by reward magnitudes that DON'T fully saturate V's learning capacity. V_max_observed << atom_max in practice, so the loose "WIN-tracking" approach delivers comparable resolution with much smaller atom span (better Q discrimination on typical-magnitude states).
|
||||
|
||||
### 3.3 The transient stability problem
|
||||
|
||||
During training, `V_max_observed` grows monotonically (as Q learns aggressive valuations). For Fix F-v2's recursion to stay self-consistent, atom_max must grow at least as fast.
|
||||
|
||||
Differentiating: `d(atom_max)/dt ≥ (1+ε)·γ · d(V_max_observed)/dt`
|
||||
|
||||
For atom_max to track via slow EWMA α=0.001, we need `V_max_observed` to grow slower than EWMA can follow. At cluster step 500 in v7, V learning was rapid (V_max_observed grew ~10×in 500 steps). EWMA at α=0.001 can only follow at ~10% in 500 steps. So Fix F-v2 with slow EWMA would have lagged too.
|
||||
|
||||
This is why **even Fix F-v2 with V_target anchor would crash at high γ during fast-learning phases**.
|
||||
|
||||
---
|
||||
|
||||
## 4. Empirical validation — local 1k smoke + v8 cluster
|
||||
|
||||
### 4.1 Local smoke (b=128, 2026-05-31, HEAD d57bee054)
|
||||
|
||||
| step | qpa | popart_σ | WIN | atom_max | Δz | clip_rate | dynamic bound (`WIN+γ·atom_max`) | atom_max ≥ bound? |
|
||||
|---|---|---|---|---|---|---|---|---|
|
||||
| 50 | +0.65 | 121.07 | 2317.8 | 1798.0 | 171 | 0.053 | 2317.8 + 0.983×1798 = 4084.0 | **NO** (clipping likely; clip_rate=5.3% confirms) |
|
||||
| 100 | +0.63 | 122.98 | 1913.6 | 2105.6 | 201 | 0.052 | 1913.6 + 0.990×2106 = 3998.5 | NO (still some clipping) |
|
||||
| 250 | +0.87 | 117.55 | 1673.3 | 2682.8 | 256 | 0.050 | 4332.7 | NO |
|
||||
| 500 | +0.96 | 106.78 | 1160.1 | 2247.5 | 214 | 0.107 | 3373.3 | NO |
|
||||
| 999 | +0.97 | 87.24 | 6.4 | 1468.0 | 140 | 0.071 | **6.4 + 0.980×1468 = 1445.3** | **YES (margin 23)** |
|
||||
|
||||
**Key observations:**
|
||||
- The dynamic bound IS partially violated early (clip rate 5-10%) but training proceeds healthy regardless. **The constraint is asymptotic, not gating** — partial violation is absorbed by the EWMA's hysteresis.
|
||||
- At step 999 the dynamic bound is satisfied with margin 23 (≈ 1.6% above bound). The system has converged to self-consistency.
|
||||
- **atom_max = 229× WIN at step 999** — when WIN collapses from 2318 → 6.4 (early-training transient settles), atom_max only decays from 1798 → 1468 over 949 steps. The slow EWMA's *inertia* sizes atoms for the historical peak, not the current value. This is *conservative* but *wastes resolution*.
|
||||
- popart_σ ≈ 87 at step 999 → V_target std ≈ 87 → 3σ ≈ 261. atom_max = 1468 has **~5× more headroom than 3σ of V_target requires**. We're oversizing.
|
||||
|
||||
### 4.2 v8 cluster (b=1024, longer horizon)
|
||||
|
||||
| step | qpa | popart_σ | atom_max | Δz | clip_rate | WIN |
|
||||
|---|---|---|---|---|---|---|
|
||||
| 100 | +0.530 | 56.50 | 970 | 92.4 | 0.147 | 1475 |
|
||||
| 1000 | +0.980 | 44.04 | 636 | 60.6 | 0.061 | 61 |
|
||||
| 5000 | +0.986 | 6.44 | ~170 | 8.2 | 0.048 | 42 |
|
||||
| 16830 | +0.974 | 1.69 | (smaller) | — | — | — |
|
||||
|
||||
**Compare to local smoke**: at cluster scale, popart_σ collapses ~50× more than at b=128 (1.69 vs 87). Atom span follows: cluster atom_max contracts much faster, converging to a tighter steady-state. The b=1024 batch diversity reduces V_target variance dramatically.
|
||||
|
||||
### 4.3 What the math validation confirms
|
||||
|
||||
1. **§2.1 fixed-point bound is overstated**: smoke step 999 shows atom_max = 4.5× fixed-point bound, healthy training. v8 cluster shows similar (atom_max settled well above WIN/(1−γ) at peak, then dropped without crash).
|
||||
|
||||
2. **§2.2 dynamic bound is the relevant constraint** AND is satisfied at convergence (margin 23 at step 999). Partial violations early in training don't break learning because the EWMA absorbs transients.
|
||||
|
||||
3. **§2.3 current design is *conservative* over-sizing**: atom_max stays ~5× above what V_target distribution requires. This wastes Q's categorical discrimination but is safe.
|
||||
|
||||
---
|
||||
|
||||
## 5. Diagnostic implementation (this commit)
|
||||
|
||||
To enable future architectural decisions on atom resolution, we need to *observe* `V_target_pre_clamp_max` during training. This commit adds:
|
||||
|
||||
1. **New ISV slot** `RL_V_TARGET_PRE_CLAMP_MAX_EMA_INDEX = 685` (re-purposes the slot previously reserved by the reverted Fix F)
|
||||
2. **New kernel** `rl_v_target_max_update.cu` — single-thread tree reduction of `|V_target_pre_clamp|` across the batch, EMA-blended into the slot with α = 0.01 (half-life ~70 steps). Launches *after* `bellman_target_projection` per step.
|
||||
3. **Diag emit** in `alpha_rl_train.rs` adds `risk_stack.diagnostics.v_target_max_ema` to JSONL.
|
||||
4. **No behavior change** — atom span sizing logic in `rl_reward_clamp_controller.cu` unchanged. The new slot is observe-only.
|
||||
|
||||
This gives us, for any future cluster run, the *signal* needed to validate (or refute) Fix F-v2: compare observed `V_target_max_ema` vs the atom_max controller value. If V_target_max consistently exceeds atom_max × 0.5, Fix F-v2 would have headroom; if it stays << atom_max × 0.5, the current design has built-in margin that we shouldn't unnecessarily tighten.
|
||||
|
||||
---
|
||||
|
||||
## 6. Pearl to save
|
||||
|
||||
> **pearl_atom_span_dynamic_vs_fixed_point**: the structural Q-V Bellman bound on atom span is `V_max ≥ max V_target_observed`, NOT the fixed-point form `WIN/(1−γ)`. The latter is only reached if learning saturates atoms, which generally doesn't happen during finite-horizon training. Atom span anchored on reward envelope (WIN_bound via slow EWMA) implicitly provides `γ × V_max_observed` of headroom and trains cleanly at γ=0.97 (v8: qpa=+0.974, pnl_cum=$164M @ step 16830 of fold-1 walk-forward). Fix F (anchor on mean_abs_pnl) and Fix F-v2 (anchor on V_target_observed × (1+ε)) both have closed-form instability at γ(1+ε) ≥ 1 — for γ=0.99 only ε≤0.01 stable, which gives no resolution benefit over current design.
|
||||
@@ -0,0 +1,197 @@
|
||||
# v9 — Defensive Eval-Boundary Calibration (Adaptive Carryover Discipline)
|
||||
|
||||
**Goal**: close the v8 train-eval gap by completing the adaptive-controller principle at the regime boundary. Reset *all* adaptive EMAs on train→eval transition, override the controllers that drive risk sizing with *defensive defaults* during a configurable warmup window, and use higher learning rates during the warmup so network weights adapt fast to new-regime statistics.
|
||||
|
||||
**Why now**: v8 (SHA `6d4a962e5`, `alpha-rl-vxbpq`) shipped the full risk-stack and achieved train +$135M / eval **−$507k** on fold-1 walk-forward. 18% better than OLD baseline (−$616k), but still deeply negative on held-out data. Diagnosis: Fix D (commit `7064c9269`) intentionally preserved Kelly + inventory EMAs across the fold boundary on the rationale that "training-acquired statistics should carry into eval." That was wrong. The new pearl [[pearl_adaptive_carryover_discipline]] proves this is a structural failure mode at every regime transition.
|
||||
|
||||
**Out of scope**: changes to the policy network architecture, encoder, atom span sizing (proven sound by `2026-05-31-c51-atom-span-math-validation`), reward shaping (deferred to v10), meta-learning across folds (deferred to v11+).
|
||||
|
||||
---
|
||||
|
||||
## 1. Diagnosis — why Fix D fails at the eval boundary
|
||||
|
||||
The risk-stack has 12+ adaptive controllers, all driven by EMAs of recent realized signals. Fix D (in `crates/ml-alpha/src/trainer/integrated.rs::reset_session_state`) resets *some* per-batch buffers but explicitly NOT the Kelly + inventory EMAs:
|
||||
|
||||
```rust
|
||||
// Kelly + inventory EMAs are intentionally NOT reset: training-acquired
|
||||
// statistics should carry into eval (matches "policy learned during
|
||||
// train, applied in eval" intent).
|
||||
```
|
||||
|
||||
This rationale violates the meta-principle of the whole risk-stack: behavior should derive from RECENT REALIZED signals. By preserving train EMAs into eval, we put the controllers in a *poisoned* state:
|
||||
|
||||
**Cascade at the v8 eval boundary**:
|
||||
|
||||
| EMA | Train value at step 20k | Eval reality (over 5000 eval steps) | Time to re-converge at α=0.05 |
|
||||
|---|---|---|---|
|
||||
| `Kelly.win_rate_ema` | 0.34 | 0.25 | ~14 trade events |
|
||||
| `Kelly.avg_win_usd_ema` | $3000+ (scaled) | $11.7k (eval real) | ~14 events |
|
||||
| `Kelly.avg_loss_usd_ema` | $200+ (scaled) | $6.6k (eval real) | ~14 events |
|
||||
| `Kelly.cumulative_dones` | 261,478 | n/a (not reset → still huge) | — |
|
||||
| `Inventory.penalty_beta` | 3.81 | (depends on new reward magnitude) | ~10 steps |
|
||||
|
||||
With `cumulative_dones >> min_trades=1000`, Kelly is "out of warmup" — its safety mechanisms (warmup gate, dead-signal guard) don't fire. Kelly computes `f = 0.5 × (0.34×14 − 0.66)/14 ≈ 0.146` from train EMAs and feeds it to position sizing. The agent enters eval with this aggressive sizing on what turns out to be a losing distribution.
|
||||
|
||||
By the time the EMAs re-converge (~200 cluster steps = 4% of the 5000-step eval phase), the damage is done. Worst-account session_pnl crosses dd_limit, Fix E force-closes, but the realized loss is locked in. The CMDP's downstream defenses (cooldown, IQN τ tightening) trigger AFTER the loss.
|
||||
|
||||
**Quantitative analysis**: v8 train had avg_win/avg_loss = $3k/$200 (post-scale). Train Kelly was sized for that distribution. Eval avg_win/avg_loss = $11.7k/$6.6k (raw, post-rescale). Position sizing optimized for tiny-loss-vs-big-win pattern fails when both sides are large in real terms.
|
||||
|
||||
---
|
||||
|
||||
## 2. Design — three-layer defensive boundary
|
||||
|
||||
### Layer 1: Reset *all* adaptive EMAs at boundary
|
||||
|
||||
Extend `reset_session_state()` to reset every signal-driven EMA, not just CMDP:
|
||||
|
||||
```rust
|
||||
// Summary slots (existing — kept)
|
||||
RL_SESSION_PNL_USD_INDEX → 0.0
|
||||
RL_SESSION_PNL_WORST_INDEX → 0.0
|
||||
RL_SESSION_DD_TRIGGERED_INDEX → 0.0
|
||||
RL_CONSEC_LOSS_COUNT_INDEX → 0.0
|
||||
RL_COOLDOWN_REMAINING_STEPS_INDEX → 0.0
|
||||
|
||||
// Kelly EMAs (NEW — was preserved by Fix D, now reset)
|
||||
RL_WIN_RATE_EMA_INDEX → 0.5 (neutral sentinel)
|
||||
RL_AVG_WIN_USD_EMA_INDEX → 0.0 (sentinel)
|
||||
RL_AVG_LOSS_USD_EMA_INDEX → 0.0 (sentinel)
|
||||
RL_CUMULATIVE_DONES_INDEX → 0.0 (re-enter warmup)
|
||||
RL_KELLY_FRACTION_INDEX → 1.0 (bootstrap = no sizing change until warmup releases)
|
||||
|
||||
// Inventory EMAs (NEW — was preserved by Fix D, now reset)
|
||||
RL_INVENTORY_PENALTY_BETA_INDEX → 0.0 (sentinel, bootstrap from first observation)
|
||||
RL_INVENTORY_VARIANCE_EMA_INDEX → 0.0 (sentinel)
|
||||
|
||||
// Reward-clamp EMAs (NEW — were not explicitly preserved but also not reset)
|
||||
RL_POS_SCALED_REWARD_MAX_EMA_INDEX → 0.0 (sentinel)
|
||||
RL_NEG_SCALED_REWARD_MAX_EMA_INDEX → 0.0 (sentinel)
|
||||
RL_REWARD_CLAMP_CLIP_RATE_EMA_INDEX → 0.0 (sentinel)
|
||||
|
||||
// Per-batch buffers (existing from Fix D — kept)
|
||||
session_pnl_per_batch_d → memset 0
|
||||
session_dd_triggered_per_batch_d → memset 0
|
||||
consec_loss_per_batch_d → memset 0
|
||||
cooldown_remaining_per_batch_d → memset 0
|
||||
```
|
||||
|
||||
This implements: at the boundary, every adaptive signal restarts from neutral and re-learns from new-regime data via its normal bootstrap path.
|
||||
|
||||
### Layer 2: Defensive warmup window — risk controllers
|
||||
|
||||
For the first `RL_EVAL_WARMUP_STEPS = 500` steps after the boundary, override risk-controller outputs with *defensive defaults*:
|
||||
|
||||
| Slot | Normal value | Warmup override | Rationale |
|
||||
|---|---|---|---|
|
||||
| `RL_KELLY_SAFETY_FRAC_INDEX` | 0.5 (half-Kelly) | **0.25** (quarter-Kelly) | half position sizing during regime discovery |
|
||||
| `RL_IQN_ACTION_TAU_MIN_INDEX` | 0.10 | **0.30** | more pessimistic action selection (lower quantile of IQN distribution) |
|
||||
| `RL_ENTROPY_COEF_MIN_INDEX` | 0.01 | **0.05** | force more exploration / less commitment to learned strategy |
|
||||
| `RL_PPO_CLIP_EPS_MIN_INDEX` | 0.05 | **0.10** | larger policy update steps allowed (faster adaptation) |
|
||||
|
||||
These are restored to normal values at the end of the warmup window via a `rl_eval_warmup_decay` kernel that linearly interpolates between defensive and normal values over an additional `RL_EVAL_WARMUP_DECAY_STEPS = 200` steps. Total transitional period: 700 steps (~14% of a 5000-step eval phase).
|
||||
|
||||
### Layer 3: Higher learning rate during warmup — network weights
|
||||
|
||||
During the warmup window, lift LR by 2× across all heads (Q, π, V, IQN):
|
||||
|
||||
```rust
|
||||
// In reset_session_state, after resetting EMAs:
|
||||
let warmup_remaining = RL_EVAL_WARMUP_STEPS; // new ISV slot
|
||||
isv[RL_EVAL_WARMUP_REMAINING_INDEX] = warmup_remaining;
|
||||
|
||||
// In step_with_lobsim, during warmup:
|
||||
// if warmup_remaining > 0:
|
||||
// lr_eff = lr_base × 2.0
|
||||
// warmup_remaining -= 1
|
||||
```
|
||||
|
||||
Justification: network weights are the slowest-adapting layer of the agent. During regime discovery, fast SGD updates let the policy re-calibrate faster than its default LR. 2× is conservative — major literatures use 5-10× for transfer learning fine-tuning, but 2× preserves stability with the existing PER + PPO infrastructure.
|
||||
|
||||
---
|
||||
|
||||
## 3. ISV slots added
|
||||
|
||||
```rust
|
||||
pub const RL_EVAL_WARMUP_REMAINING_INDEX: usize = 686; // counter, decremented each step in warmup
|
||||
pub const RL_EVAL_WARMUP_STEPS_INDEX: usize = 687; // configured warmup duration (bootstrap 500)
|
||||
pub const RL_EVAL_WARMUP_DECAY_STEPS_INDEX: usize = 688; // decay duration (bootstrap 200)
|
||||
|
||||
// Defensive override values (read by controllers when warmup_remaining > 0)
|
||||
pub const RL_EVAL_KELLY_SAFETY_FRAC_OVERRIDE_INDEX: usize = 689; // bootstrap 0.25
|
||||
pub const RL_EVAL_IQN_TAU_MIN_OVERRIDE_INDEX: usize = 690; // bootstrap 0.30
|
||||
pub const RL_EVAL_ENTROPY_COEF_MIN_OVERRIDE_INDEX: usize = 691; // bootstrap 0.05
|
||||
pub const RL_EVAL_PPO_CLIP_EPS_MIN_OVERRIDE_INDEX: usize = 692; // bootstrap 0.10
|
||||
pub const RL_EVAL_LR_MULTIPLIER_INDEX: usize = 693; // bootstrap 2.0
|
||||
|
||||
// RL_SLOTS_END: 686 → 694
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 4. Validation criteria
|
||||
|
||||
### Local smoke (b=128, single fold-0 with --n-eval-steps 1000)
|
||||
- v8 trajectory parameters preserved through train phase (qpa > +0.9, l_v < 1.0, etc.)
|
||||
- At eval boundary: Kelly EMAs reset to sentinels, warmup_remaining = 500
|
||||
- During warmup: kelly_f < 0.10 typically (quarter-Kelly forcing), action_entropy > 1.8 (higher entropy)
|
||||
- Eval-phase pnl > v8 fold-0 eval (whatever that turns out to be)
|
||||
|
||||
### Cluster validation (b=1024)
|
||||
- **Run all 3 folds** (per [[pearl_single_window_oos_is_not_oos]]). v8 only ran fold-1. We need fold-0, fold-1, fold-2 with the same v8 baseline and again with v9 for clean comparison.
|
||||
- Success criterion: **mean eval pnl across 3 folds > 0** (vs v8's known fold-1 = −$507k, unknown 0/2)
|
||||
- Or weaker: **mean eval pnl > v8 mean** (improvement, even if still negative)
|
||||
|
||||
### Pearl invariant
|
||||
After v9 ships, [[pearl_adaptive_carryover_discipline]] should be referenced in every PR that touches a `reset_*_state` or session-boundary handler.
|
||||
|
||||
---
|
||||
|
||||
## 5. Implementation plan
|
||||
|
||||
1. **isv_slots.rs**: add 8 new ISV slots (lines after current `RL_C51_ATOM_MARGIN_INDEX = 685` — note that one was deleted via the Fix F revert, so 685 is open). Bump `RL_SLOTS_END` to 694.
|
||||
|
||||
2. **integrated.rs `reset_session_state`**: extend to reset all Kelly/inventory/clamp EMAs listed in §2 Layer 1. Set `RL_EVAL_WARMUP_REMAINING_INDEX = RL_EVAL_WARMUP_STEPS_INDEX`.
|
||||
|
||||
3. **integrated.rs bootstrap**: add 8 new ISV bootstrap entries with values from §3. Bump bootstrap array length from `169` to `177`.
|
||||
|
||||
4. **New kernel `rl_eval_warmup_decay.cu`**: single-thread kernel, runs once per step. Reads `RL_EVAL_WARMUP_REMAINING_INDEX`. If > 0, applies defensive overrides to `RL_KELLY_SAFETY_FRAC_INDEX`, `RL_IQN_ACTION_TAU_MIN_INDEX`, `RL_ENTROPY_COEF_MIN_INDEX`, `RL_PPO_CLIP_EPS_MIN_INDEX`. During decay phase (warmup_remaining ∈ [-decay_steps, 0]), linearly interpolates back to normal. Decrements counter.
|
||||
|
||||
5. **integrated.rs step_with_lobsim**: launch `rl_eval_warmup_decay` after `reset_session_state` is called (i.e., on first eval step). LR multiplier applied via `RL_EVAL_LR_MULTIPLIER_INDEX` read in LR controllers.
|
||||
|
||||
6. **Diag emit**: add `risk_stack.eval_warmup.{remaining, kelly_safety_active, tau_min_active}` for visibility.
|
||||
|
||||
7. **Invariant test G14**: GPU-oracle test that confirms the override values are written during warmup and decay back to normal after.
|
||||
|
||||
8. **Local smoke**: b=128, 5000 train + 1000 eval. Verify reset + warmup behavior in JSONL.
|
||||
|
||||
9. **Atomic commit**, push, submit cluster.
|
||||
|
||||
---
|
||||
|
||||
## 6. Memory pearl to ship
|
||||
|
||||
**See `[[pearl_adaptive_carryover_discipline]]`** (newly created in this session). The pearl captures the structural principle this spec operationalizes. Future sessions auditing `reset_*_state` or designing boundary handlers should reference it.
|
||||
|
||||
---
|
||||
|
||||
## 7. Risks and rollback
|
||||
|
||||
**Risk 1: Resetting Kelly EMAs eliminates train signal that was actually useful**. If train EMAs DO encode useful priors for eval (e.g., R-multiple structure), resetting them loses information. Mitigation: warmup window is short (500 steps = 10% of eval phase); after warmup, EMAs re-converge to eval reality which is the correct prior anyway.
|
||||
|
||||
**Risk 2: Higher LR during warmup destabilizes V/Q**. With existing popart normalization + LR plateau controllers, this should be robust. But if l_v spikes during warmup, the LR multiplier could be tuned down (1.5× or 1.25×).
|
||||
|
||||
**Risk 3: Defensive sizing prevents agent from capturing eval-phase alpha that DOES generalize**. Possible. Defensive warmup is structured to give up SOME early-eval performance in exchange for not blowing up. The expected-value trade is favorable given v8's eval result was −$507k.
|
||||
|
||||
**Rollback path**: revert this commit. Cluster runs at v8 SHA `6d4a962e5` are reproducible.
|
||||
|
||||
---
|
||||
|
||||
## 8. Connection to broader system
|
||||
|
||||
This spec completes the adaptive-controller principle that the whole risk-stack was building toward. After v9, every behaviorally-relevant signal in the system will derive from recent realized data, including at regime transitions. The static special-case in Fix D becomes a closed-loop adaptive controller like everything else.
|
||||
|
||||
If v9 closes the train-eval gap substantially: confirms the eval problem was *transition mismanagement*, not fundamental overfitting.
|
||||
|
||||
If v9 only modestly improves: confirms remainder is genuine overfitting/non-generalization, justifying the deeper Level 2 (regime-feature conditioning) or Level 3 (meta-learning) interventions sketched in the eval-problem ultrathink.
|
||||
|
||||
Either outcome is *informative* in a way single-fold v8 results aren't.
|
||||
Reference in New Issue
Block a user