foxhunt/crates/ml-alpha/cuda/rl_gamma_controller.cu

// rl_gamma_controller.cu — emits γ to ISV[RL_GAMMA_INDEX=400].
//
// Phase C of the integrated RL trainer
// (docs/superpowers/plans/2026-05-22-integrated-rl-trainer.md).
//
// γ is the Bellman discount factor used by the categorical Bellman
// projection (Phase E) and by the PPO advantage estimator (Phase D).
// Per `pearl_controller_anchors_isv_driven` and
// `feedback_isv_for_adaptive_bounds`, γ is NOT a hardcoded constant; it
// adapts so that
//
//     γ^(mean_trade_duration_events) ≈ 0.5
//
// i.e. roughly half of the discounted value remains by the time the
// average trade closes. This anchors the credit-assignment horizon to
// the actual trade timescale rather than a tuned constant; if the
// strategy lengthens its holding period, γ rises automatically.
//
// Bootstrap discipline (per `pearl_first_observation_bootstrap`): the
// ISV slot starts at 0.0 (sentinel "uninitialised"). On the first
// emit the kernel DERIVES γ from the current input slot via the same
// target formula the per-step path uses, instead of writing a
// hardcoded `GAMMA_BOOTSTRAP = 0.99`. At sentinel input
// (trade_duration_ema = 0 → clamped d = 1) target = 0.5 → clamped to
// GAMMA_MIN = 0.90, so the cold-start γ is the floor. This eliminates
// the dead-zone where target(d ≈ 69) = 0.99 = hardcoded bootstrap froze
// the Wiener blend at canonical long-horizon γ for any realistic
// trade duration that happened to land near 69 events.
//
// Trade-off: cold-start γ = 0.90 (floor) is more myopic than the
// previous hardcoded 0.99. Once the trade_duration_ema stabilises
// (within ~10 episodes), the controller drifts γ up toward target
// (0.986 at d=50, 0.99 at d=69, etc.). The brief myopic warm-up is
// acceptable cost for guaranteed responsiveness — a frozen controller
// at canonical γ is worse than a controller that starts low and
// climbs.
//
// Subsequent emits use a Wiener-α blend with floor 0.4 per
// `pearl_wiener_alpha_floor_for_nonstationary` (the target γ_target
// drifts as the strategy's holding period co-adapts with the policy,
// which violates the stationarity precondition of Wiener-optimal α).
//
// Bounds: γ ∈ [0.90, 0.999] enforced by clamp to prevent runaway
// (γ → 1 makes credit assignment infinite-horizon; γ → 0.5 makes
// the policy myopic).

#define RL_GAMMA_INDEX   400
#define GAMMA_MIN        0.995f
#define GAMMA_MAX        0.999f
#define WIENER_ALPHA_FLOOR 0.4f


// ─────────────────────────────────────────────────────────────────────
// rl_gamma_controller:
//   Single-thread controller — writes ONE float to isv[RL_GAMMA_INDEX].
//
// Inputs:
//   isv                          [≥ RL_GAMMA_INDEX+1] — ISV bus
//   alpha                        — Wiener-α from the controller's own
//                                  signal stats (caller computes this
//                                  upstream from γ-divergence variance).
//                                  Floored at WIENER_ALPHA_FLOOR before
//                                  the blend.
//   mean_trade_duration_events   — EMA of trade hold time in event count.
//                                  Caller is responsible for the EMA
//                                  (Phase E); we just read the scalar.
//
// Outputs:
//   isv[RL_GAMMA_INDEX]          — γ ∈ [GAMMA_MIN, GAMMA_MAX]
// ─────────────────────────────────────────────────────────────────────
// Phase R5: scalar input arg replaced with `input_slot` ISV index so the
// EMA producer (Phase R3 ema_update_per_step targeting
// ISV[RL_MEAN_TRADE_DURATION_EMA_INDEX=417]) feeds this controller
// without any host roundtrip per `feedback_cpu_is_read_only`. At
// bootstrap (R1) the kernel returns before the input read, so `input_slot`
// can be the same ISV[417] sentinel-zero slot — the read just doesn't
// happen on that path.
extern "C" __global__ void rl_gamma_controller(
    float* __restrict__ isv,
    float          alpha,
    int            input_slot
) {
    if (threadIdx.x != 0 || blockIdx.x != 0) return;

    const float gamma_prev = isv[RL_GAMMA_INDEX];

    // Compute target from the current input EMA. Shared between
    // bootstrap and per-step paths so the dead-zone coincidence with
    // a hardcoded bootstrap cannot recur.
    //   Target: γ^d ≈ 0.5  ⇒  γ = 0.5^(1/d). Clamp d ≥ 1 so a
    //   single-event trade doesn't push γ to 0.5.
    const float mean_trade_duration_events = isv[input_slot];
    const float d            = fmaxf(mean_trade_duration_events, 1.0f);
    float       gamma_target = powf(0.5f, 1.0f / d);
    gamma_target = fmaxf(GAMMA_MIN, fminf(gamma_target, GAMMA_MAX));

    // Bootstrap on sentinel 0.0 per pearl_first_observation_bootstrap:
    // first emit replaces directly with the computed target. At cold
    // start (input EMA also sentinel-zero), clamped d=1, target=0.5,
    // clamped to GAMMA_MIN = 0.90 (the floor). Any non-sentinel input
    // produces target ≥ 0.90 → ≤ 0.999, distinct from the floor so
    // the per-step Wiener blend on subsequent calls always sees a
    // prev vs target delta.
    if (gamma_prev == 0.0f) {
        isv[RL_GAMMA_INDEX] = gamma_target;
        return;
    }

    // Wiener-α blend with floor per pearl_wiener_alpha_floor_for_nonstationary.
    const float a            = fmaxf(alpha, WIENER_ALPHA_FLOOR);
    float       gamma_new    = (1.0f - a) * gamma_prev + a * gamma_target;

    // Clamp into the bounded range; runaway protection.
    gamma_new = fmaxf(GAMMA_MIN, fminf(gamma_new, GAMMA_MAX));
    isv[RL_GAMMA_INDEX] = gamma_new;
}