// rl_target_tau_controller.cu — emits τ to ISV[RL_TARGET_TAU_INDEX=401].
//
// Phase C of the integrated RL trainer
// (docs/superpowers/plans/2026-05-22-integrated-rl-trainer.md).
//
// τ is the target-network soft-update fraction used by the Phase E
// `dqn_soft_update_target` step
//     W_target ← (1 - τ) * W_target + τ * W.
//
// Per `pearl_controller_anchors_isv_driven`, τ is NOT a hardcoded
// constant — it adapts to keep the target-net divergence
// ‖Q_online - Q_target‖₂ (caller-computed EMA) near a target anchor.
// High divergence → raise τ so the target tracks faster; low divergence
// → lower τ so the bootstrap stays stable.
//
// Bootstrap discipline (per `pearl_first_observation_bootstrap`): the
// ISV slot starts at 0.0 sentinel. First emit writes τ = 0.005 directly.
// Subsequent emits Wiener-α blend with floor 0.4 (per
// `pearl_wiener_alpha_floor_for_nonstationary` — the target divergence
// drifts as the online weights co-adapt with the policy, breaking
// stationarity).
//
// Bounds: τ ∈ [0.001, 0.05] (clamp). Below 0.001 the target is
// effectively frozen, eroding the Bellman bootstrap; above 0.05 the
// target tracks the online net too closely, destroying the stability
// the target-net design exists to provide.

#define RL_TARGET_TAU_INDEX     401
// τ MIN clamp bound is now ISV-driven per the 2026-05-30 clamp-bound
// extension. ISV-residence makes the floor runtime-tunable and aligns
// with the broader principle that every controller bound is observed-
// signal driven rather than calibrated against a prior regime.
#define RL_TARGET_TAU_MIN_INDEX           642
#define RL_TARGET_TAU_MAX_INDEX           573
// ISV-driven bootstrap + target.
#define RL_TAU_BOOTSTRAP_INDEX            473
#define RL_DIV_TARGET_INDEX               457
// Schulman params from shared global slots (same as ppo_clip).
#define RL_SCHULMAN_TOLERANCE_INDEX       468
#define RL_SCHULMAN_ADJUST_RATE_INDEX     469
// Wiener-α floor — shared across 9 controllers (slot 659).
#define RL_WIENER_ALPHA_FLOOR_INDEX       659

// Adaptive controller floors (spec 2026-05-30-adaptive-controller-floor-design):
// noise floor derives from observed q_divergence_ema variance via Welford
// triples, asymmetric Schulman widening requires N consecutive below-band
// observations. Replaces hardcoded DIV_NOISE_FLOOR_FRAC=0.01f which was
// calibrated against the pre-Phase-4.5 divergence regime — under Phase 4.5
// the 1%-of-target floor is ~100× too low and the controller never moved
// from bootstrap (alpha-rl-... fold 0 confirmed τ stuck at 0.005).
#define RL_Q_DIV_VAR_COUNT_INDEX           592
#define RL_Q_DIV_VAR_M2_INDEX              594
#define RL_Q_DIV_BELOW_COUNT_INDEX         595
#define NOISE_FLOOR_TARGET_FRAC            0.5f  // floor ≥ 50% of target
#define NOISE_FLOOR_STD_MULTIPLIER         2.0f  // floor ≥ 2σ of observed signal
#define WIDEN_PATIENCE_CONSECUTIVE         3.0f  // widen requires N below-band steps


// ─────────────────────────────────────────────────────────────────────
// rl_target_tau_controller:
//   Single-thread controller — writes ONE float to isv[RL_TARGET_TAU_INDEX].
//
// Inputs:
//   isv                    [≥ RL_TARGET_TAU_INDEX+1] — ISV bus
//   alpha                  — Wiener-α for the blend (floored at 0.4)
//   q_divergence_norm      — caller-computed EMA of
//                            ‖Q_online - Q_target‖₂. Phase E provides
//                            this via a small reduce-norm kernel against
//                            the target / online weight buffers.
//
// Outputs:
//   isv[RL_TARGET_TAU_INDEX] — τ ∈ [TAU_MIN, TAU_MAX]
// ─────────────────────────────────────────────────────────────────────
// Phase R5: scalar input arg replaced with `input_slot` ISV index so the
// EMA producer (Phase R3 ema_update_per_step targeting
// ISV[RL_Q_DIVERGENCE_EMA_INDEX=418]) feeds this controller without
// any host roundtrip per `feedback_cpu_is_read_only`.
extern "C" __global__ void rl_target_tau_controller(
    float* __restrict__ isv,
    float          alpha,
    int            input_slot
) {
    if (threadIdx.x != 0 || blockIdx.x != 0) return;

    const float tau_prev = isv[RL_TARGET_TAU_INDEX];
    // Bootstrap on sentinel 0.0 per pearl_first_observation_bootstrap.
    if (tau_prev == 0.0f) {
        isv[RL_TARGET_TAU_INDEX] = isv[RL_TAU_BOOTSTRAP_INDEX];
        return;
    }

    // Cold-start gate (sentinel-zero EMA — kernel hasn't fired yet).
    const float q_divergence_norm = isv[input_slot];
    if (q_divergence_norm == 0.0f) return;

    // ISV-driven divergence target (was hardcoded #define).
    const float div_target = isv[RL_DIV_TARGET_INDEX];

    // Adaptive noise floor — signal-driven per
    // `pearl_zscore_normalization_for_magnitude_asymmetric_signals` and
    // `feedback_adaptive_not_tuned`. Floor must scale with observed signal
    // magnitude AND have an absolute minimum tied to the target so a
    // momentarily-noisy signal can't trigger spurious adjustments.
    // Welford sample variance = M² / (count − 1) when count > 1.
    const float div_count = isv[RL_Q_DIV_VAR_COUNT_INDEX];
    const float div_var = (div_count > 1.0f)
        ? isv[RL_Q_DIV_VAR_M2_INDEX] / (div_count - 1.0f)
        : 0.0f;
    const float div_std = sqrtf(div_var);
    const float div_noise_floor = fmaxf(div_target * NOISE_FLOOR_TARGET_FRAC,
                                         div_std * NOISE_FLOOR_STD_MULTIPLIER);

    // Noise-floor gate: divergence below the adaptive floor is dominated by
    // numerical noise OR signal stays naturally below target under the
    // current architecture. Hold τ.
    if (q_divergence_norm < div_noise_floor) return;

    // Asymmetric Schulman (spec 2026-05-30 Section "Approach C, folded in"):
    // raising τ fires on a single above-band observation — real Q
    // divergence is a safety signal we act on fast (target net falling
    // behind = raise the bootstrap rate). Lowering τ requires N consecutive
    // below-band observations so a single noisy step can't push τ toward
    // the floor. Below-band counter is per-controller in ISV.
    const float tolerance   = isv[RL_SCHULMAN_TOLERANCE_INDEX];
    const float adjust_rate = isv[RL_SCHULMAN_ADJUST_RATE_INDEX];
    float ratio;
    if (q_divergence_norm > div_target * tolerance) {
        ratio = adjust_rate;                              // raise τ — single observation
        isv[RL_Q_DIV_BELOW_COUNT_INDEX] = 0.0f;           // reset patience
    } else if (q_divergence_norm < div_target / tolerance) {
        const float new_count = isv[RL_Q_DIV_BELOW_COUNT_INDEX] + 1.0f;
        isv[RL_Q_DIV_BELOW_COUNT_INDEX] = new_count;
        ratio = (new_count >= WIDEN_PATIENCE_CONSECUTIVE) ? (1.0f / adjust_rate) : 1.0f;
    } else {
        isv[RL_Q_DIV_BELOW_COUNT_INDEX] = 0.0f;           // in-band → reset
        ratio = 1.0f;
    }
    float tau_target = tau_prev * ratio;
    const float tau_min = isv[RL_TARGET_TAU_MIN_INDEX];
    const float tau_max = isv[RL_TARGET_TAU_MAX_INDEX];
    tau_target = fmaxf(tau_min, fminf(tau_target, tau_max));

    // First-observation replace-directly: if
    // prev is still exactly the hardcoded bootstrap value, this is
    // the FIRST per-step fire that has real input signal. Per
    // `pearl_first_observation_bootstrap`, "first observation
    // replaces directly" — the Wiener blend `(1-α)·bootstrap +
    // α·target` would leave 60% bootstrap contamination, distorting
    // the controller's first emit. Write target directly instead.
    // Subsequent steps (where prev has drifted off bootstrap via
    // earlier blends) take the Wiener path below.
    if (tau_prev == isv[RL_TAU_BOOTSTRAP_INDEX]) {
        isv[RL_TARGET_TAU_INDEX] = tau_target;
        return;
    }

    // Wiener-α blend with floor per pearl_wiener_alpha_floor_for_nonstationary.
    const float a       = fmaxf(alpha, isv[RL_WIENER_ALPHA_FLOOR_INDEX]);
    float       tau_new = (1.0f - a) * tau_prev + a * tau_target;

    tau_new = fmaxf(tau_min, fminf(tau_new, tau_max));
    isv[RL_TARGET_TAU_INDEX] = tau_new;
}
