foxhunt/crates/ml-alpha/cuda/rl_gamma_controller.cu

// rl_gamma_controller.cu — emits γ to ISV[RL_GAMMA_INDEX=400].
//
// Phase C of the integrated RL trainer
// (docs/superpowers/plans/2026-05-22-integrated-rl-trainer.md).
//
// γ is the Bellman discount factor used by the categorical Bellman
// projection (Phase E) and by the PPO advantage estimator (Phase D).
// Per `pearl_controller_anchors_isv_driven` and
// `feedback_isv_for_adaptive_bounds`, γ is NOT a hardcoded constant; it
// adapts so that
//
//     γ^(mean_trade_duration_events) ≈ 0.5
//
// i.e. roughly half of the discounted value remains by the time the
// average trade closes. This anchors the credit-assignment horizon to
// the actual trade timescale rather than a tuned constant; if the
// strategy lengthens its holding period, γ rises automatically.
//
// Bootstrap discipline (per `pearl_first_observation_bootstrap`): the
// ISV slot starts at 0.0 (sentinel "uninitialised"). On the first
// emit the kernel DERIVES γ from the current input slot via the same
// target formula the per-step path uses, instead of writing a
// hardcoded `GAMMA_BOOTSTRAP = 0.99`. At sentinel input
// (trade_duration_ema = 0 → clamped d = 1) target = 0.5 → clamped to
// GAMMA_MIN = 0.90, so the cold-start γ is the floor. This eliminates
// the dead-zone where target(d ≈ 69) = 0.99 = hardcoded bootstrap froze
// the Wiener blend at canonical long-horizon γ for any realistic
// trade duration that happened to land near 69 events.
//
// Trade-off: cold-start γ = 0.90 (floor) is more myopic than the
// previous hardcoded 0.99. Once the trade_duration_ema stabilises
// (within ~10 episodes), the controller drifts γ up toward target
// (0.986 at d=50, 0.99 at d=69, etc.). The brief myopic warm-up is
// acceptable cost for guaranteed responsiveness — a frozen controller
// at canonical γ is worse than a controller that starts low and
// climbs.
//
// Subsequent emits use a Wiener-α blend with floor 0.4 per
// `pearl_wiener_alpha_floor_for_nonstationary` (the target γ_target
// drifts as the strategy's holding period co-adapts with the policy,
// which violates the stationarity precondition of Wiener-optimal α).
//
// Bounds: γ ∈ [0.90, 0.999] enforced by clamp to prevent runaway
// (γ → 1 makes credit assignment infinite-horizon; γ → 0.5 makes
// the policy myopic).

#define RL_GAMMA_INDEX   400
// γ MAX clamp ceiling is now ISV-driven per the 2026-05-30 clamp-bound
// extension. γ MIN already adaptive via RL_GAMMA_MIN_ADAPTIVE_INDEX (613).
#define RL_GAMMA_MAX_INDEX           649
// Wiener-α floor — shared across 9 controllers (slot 659).
#define RL_WIENER_ALPHA_FLOOR_INDEX  659

// Adaptive controller floors (spec 2026-05-30 Special case G):
// hardcoded `GAMMA_MIN = 0.995f` replaced with an adaptive bound derived
// from the Welford mean of trade duration. With short-horizon trade
// regimes (d_ema = 15.6 in fold 0) the static 0.995 floor pinned gamma
// for the entire run; adaptive bound `1 - 1/(d_smoothed × 2)` lets the
// controller drop gamma low enough for the current regime while keeping
// a hard `GAMMA_MIN_ABSOLUTE = 0.9` to prevent bandit-mode degenerate.
//
// Smoothed-duration choice: Welford mean (slot 609) is used after a
// 100-observation warmup so the gamma↔trade_duration feedback loop is
// damped by the natural N-smoothing of running mean. Before warmup, the
// EMA at slot 417 is used directly (less stable but available
// immediately at cold-start).
#define RL_GAMMA_MIN_ADAPTIVE_INDEX        613
#define RL_TRADE_DUR_VAR_COUNT_INDEX       608
#define RL_TRADE_DUR_VAR_MEAN_INDEX        609
#define RL_MEAN_TRADE_DURATION_EMA_INDEX   417
#define GAMMA_MIN_ABSOLUTE                 0.9f
#define HORIZON_MULTIPLIER                 2.0f
#define WELFORD_WARMUP_OBS                 100.0f
// Floor for `d × HORIZON_MULTIPLIER` to prevent the `1/d` term from
// blowing up when trade duration is very small (cold start, no closes).
#define HORIZON_FLOOR                      10.0f


// ─────────────────────────────────────────────────────────────────────
// rl_gamma_controller:
//   Single-thread controller — writes ONE float to isv[RL_GAMMA_INDEX].
//
// Inputs:
//   isv                          [≥ RL_GAMMA_INDEX+1] — ISV bus
//   alpha                        — Wiener-α from the controller's own
//                                  signal stats (caller computes this
//                                  upstream from γ-divergence variance).
//                                  Floored at WIENER_ALPHA_FLOOR before
//                                  the blend.
//   mean_trade_duration_events   — EMA of trade hold time in event count.
//                                  Caller is responsible for the EMA
//                                  (Phase E); we just read the scalar.
//
// Outputs:
//   isv[RL_GAMMA_INDEX]          — γ ∈ [GAMMA_MIN, GAMMA_MAX]
// ─────────────────────────────────────────────────────────────────────
// Phase R5: scalar input arg replaced with `input_slot` ISV index so the
// EMA producer (Phase R3 ema_update_per_step targeting
// ISV[RL_MEAN_TRADE_DURATION_EMA_INDEX=417]) feeds this controller
// without any host roundtrip per `feedback_cpu_is_read_only`. At
// bootstrap (R1) the kernel returns before the input read, so `input_slot`
// can be the same ISV[417] sentinel-zero slot — the read just doesn't
// happen on that path.
extern "C" __global__ void rl_gamma_controller(
    float* __restrict__ isv,
    float          alpha,
    int            input_slot
) {
    if (threadIdx.x != 0 || blockIdx.x != 0) return;

    const float gamma_prev = isv[RL_GAMMA_INDEX];

    // Adaptive GAMMA_MIN (spec 2026-05-30 Special case G).
    // Replaces hardcoded `GAMMA_MIN = 0.995f`. With d_ema = 15.6 (fold 0)
    // the static 0.995 floor pinned γ for the entire run — the target
    // formula `0.5^(1/d)` produces γ_target = 0.936 at d = 15.6, which
    // clamps to 0.995 = floor and never moves. Adaptive bound
    // `1 - 1/(d × 2)` gives γ_min ≈ 0.968 at d = 15.6 — leaving room
    // for the controller to track the actual trade horizon.
    //
    // After 100 Welford observations the running mean is used (more
    // stable than EMA against per-batch outliers); before that the EMA
    // at slot 417 is used directly so cold-start has a usable signal.
    const float d_welford_count = isv[RL_TRADE_DUR_VAR_COUNT_INDEX];
    const float d_welford_mean  = isv[RL_TRADE_DUR_VAR_MEAN_INDEX];
    const float d_smoothed = (d_welford_count >= WELFORD_WARMUP_OBS)
        ? d_welford_mean
        : isv[RL_MEAN_TRADE_DURATION_EMA_INDEX];
    const float adaptive_min = fmaxf(GAMMA_MIN_ABSOLUTE,
                                      1.0f - 1.0f / fmaxf(d_smoothed * HORIZON_MULTIPLIER,
                                                          HORIZON_FLOOR));
    isv[RL_GAMMA_MIN_ADAPTIVE_INDEX] = adaptive_min;

    // Compute target from the current input EMA. Shared between
    // bootstrap and per-step paths so the dead-zone coincidence with
    // a hardcoded bootstrap cannot recur.
    //   Target: γ^d ≈ 0.5  ⇒  γ = 0.5^(1/d). Clamp d ≥ 1 so a
    //   single-event trade doesn't push γ to 0.5.
    const float mean_trade_duration_events = isv[input_slot];
    const float d            = fmaxf(mean_trade_duration_events, 1.0f);
    const float gamma_max    = isv[RL_GAMMA_MAX_INDEX];
    float       gamma_target = powf(0.5f, 1.0f / d);
    gamma_target = fmaxf(adaptive_min, fminf(gamma_target, gamma_max));

    // Bootstrap on sentinel 0.0 per pearl_first_observation_bootstrap:
    // first emit replaces directly with the computed target. At cold
    // start (input EMA also sentinel-zero), clamped d=1, target=0.5,
    // clamped to adaptive_min (≥ GAMMA_MIN_ABSOLUTE = 0.90). Any
    // non-sentinel input produces target ≥ adaptive_min → ≤ 0.999.
    if (gamma_prev == 0.0f) {
        isv[RL_GAMMA_INDEX] = gamma_target;
        return;
    }

    // Wiener-α blend with floor per pearl_wiener_alpha_floor_for_nonstationary.
    const float a            = fmaxf(alpha, isv[RL_WIENER_ALPHA_FLOOR_INDEX]);
    float       gamma_new    = (1.0f - a) * gamma_prev + a * gamma_target;

    // Clamp into the bounded range; runaway protection.
    gamma_new = fmaxf(adaptive_min, fminf(gamma_new, gamma_max));
    isv[RL_GAMMA_INDEX] = gamma_new;
}