foxhunt/crates/ml-alpha/cuda/rl_target_tau_controller.cu

// rl_target_tau_controller.cu — emits τ to ISV[RL_TARGET_TAU_INDEX=401].
//
// Phase C of the integrated RL trainer
// (docs/superpowers/plans/2026-05-22-integrated-rl-trainer.md).
//
// τ is the target-network soft-update fraction used by the Phase E
// `dqn_soft_update_target` step
//     W_target ← (1 - τ) * W_target + τ * W.
//
// Per `pearl_controller_anchors_isv_driven`, τ is NOT a hardcoded
// constant — it adapts to keep the target-net divergence
// ‖Q_online - Q_target‖₂ (caller-computed EMA) near a target anchor.
// High divergence → raise τ so the target tracks faster; low divergence
// → lower τ so the bootstrap stays stable.
//
// Bootstrap discipline (per `pearl_first_observation_bootstrap`): the
// ISV slot starts at 0.0 sentinel. First emit writes τ = 0.005 directly.
// Subsequent emits Wiener-α blend with floor 0.4 (per
// `pearl_wiener_alpha_floor_for_nonstationary` — the target divergence
// drifts as the online weights co-adapt with the policy, breaking
// stationarity).
//
// Bounds: τ ∈ [0.001, 0.05] (clamp). Below 0.001 the target is
// effectively frozen, eroding the Bellman bootstrap; above 0.05 the
// target tracks the online net too closely, destroying the stability
// the target-net design exists to provide.

#define RL_TARGET_TAU_INDEX 401
#define TAU_MIN             0.001f
#define TAU_MAX             0.05f
// ISV-driven bootstrap + target.
#define RL_TAU_BOOTSTRAP_INDEX            473
#define RL_DIV_TARGET_INDEX               457
// Schulman params from shared global slots (same as ppo_clip).
#define RL_SCHULMAN_TOLERANCE_INDEX       468
#define RL_SCHULMAN_ADJUST_RATE_INDEX     469
#define DIV_NOISE_FLOOR_FRAC              0.01f
#define WIENER_ALPHA_FLOOR                0.4f


// ─────────────────────────────────────────────────────────────────────
// rl_target_tau_controller:
//   Single-thread controller — writes ONE float to isv[RL_TARGET_TAU_INDEX].
//
// Inputs:
//   isv                    [≥ RL_TARGET_TAU_INDEX+1] — ISV bus
//   alpha                  — Wiener-α for the blend (floored at 0.4)
//   q_divergence_norm      — caller-computed EMA of
//                            ‖Q_online - Q_target‖₂. Phase E provides
//                            this via a small reduce-norm kernel against
//                            the target / online weight buffers.
//
// Outputs:
//   isv[RL_TARGET_TAU_INDEX] — τ ∈ [TAU_MIN, TAU_MAX]
// ─────────────────────────────────────────────────────────────────────
// Phase R5: scalar input arg replaced with `input_slot` ISV index so the
// EMA producer (Phase R3 ema_update_per_step targeting
// ISV[RL_Q_DIVERGENCE_EMA_INDEX=418]) feeds this controller without
// any host roundtrip per `feedback_cpu_is_read_only`.
extern "C" __global__ void rl_target_tau_controller(
    float* __restrict__ isv,
    float          alpha,
    int            input_slot
) {
    if (threadIdx.x != 0 || blockIdx.x != 0) return;

    const float tau_prev = isv[RL_TARGET_TAU_INDEX];
    // Bootstrap on sentinel 0.0 per pearl_first_observation_bootstrap.
    if (tau_prev == 0.0f) {
        isv[RL_TARGET_TAU_INDEX] = isv[RL_TAU_BOOTSTRAP_INDEX];
        return;
    }

    // Cold-start gate (sentinel-zero EMA — kernel hasn't fired yet).
    const float q_divergence_norm = isv[input_slot];
    if (q_divergence_norm == 0.0f) return;

    // ISV-driven divergence target (was hardcoded #define).
    const float div_target = isv[RL_DIV_TARGET_INDEX];
    const float div_noise_floor = div_target * DIV_NOISE_FLOOR_FRAC;

    // Noise-floor gate.
    if (q_divergence_norm < div_noise_floor) return;

    // Bounded multiplicative adjustment.
    float ratio;
    const float tolerance   = isv[RL_SCHULMAN_TOLERANCE_INDEX];
    const float adjust_rate = isv[RL_SCHULMAN_ADJUST_RATE_INDEX];
    if (q_divergence_norm > div_target * tolerance) {
        ratio = adjust_rate;
    } else if (q_divergence_norm < div_target / tolerance) {
        ratio = 1.0f / adjust_rate;
    } else {
        ratio = 1.0f;
    }
    float tau_target = tau_prev * ratio;
    tau_target = fmaxf(TAU_MIN, fminf(tau_target, TAU_MAX));

    // First-observation replace-directly: if
    // prev is still exactly the hardcoded bootstrap value, this is
    // the FIRST per-step fire that has real input signal. Per
    // `pearl_first_observation_bootstrap`, "first observation
    // replaces directly" — the Wiener blend `(1-α)·bootstrap +
    // α·target` would leave 60% bootstrap contamination, distorting
    // the controller's first emit. Write target directly instead.
    // Subsequent steps (where prev has drifted off bootstrap via
    // earlier blends) take the Wiener path below.
    if (tau_prev == isv[RL_TAU_BOOTSTRAP_INDEX]) {
        isv[RL_TARGET_TAU_INDEX] = tau_target;
        return;
    }

    // Wiener-α blend with floor per pearl_wiener_alpha_floor_for_nonstationary.
    const float a       = fmaxf(alpha, WIENER_ALPHA_FLOOR);
    float       tau_new = (1.0f - a) * tau_prev + a * tau_target;

    tau_new = fmaxf(TAU_MIN, fminf(tau_new, TAU_MAX));
    isv[RL_TARGET_TAU_INDEX] = tau_new;
}