foxhunt/crates/ml-alpha/cuda/rl_q_distill_lambda_controller.cu

// rl_q_distill_lambda_controller.cu — adaptive λ_distill controller.
//
// Adapts the Q→π distillation strength (λ, slot 486) from the KL
// divergence between Q's Boltzmann policy and π's softmax (slot 488).
//
// Controller logic (asymmetric bounded step):
//
//   if KL > target × TOLERANCE → λ *= RAMP_RATE    (Q signal not
//                                                     landing, pull harder)
//   if KL < target / TOLERANCE → λ *= DECAY_RATE   (aligned, ease off)
//   else                       → λ unchanged        (dead-zone)
//
// CRITICAL DESIGN: decay is 100× slower than ramp. This prevents the
// self-defeating oscillation where λ ramps up → Q and π align → λ
// collapses → they diverge again. The asymmetry means coupling
// establishes quickly but persists for thousands of steps, giving
// Q time to teach π meaningful action preferences.
//
// Per `feedback_no_atomicadd`: single-thread kernel, no atomics.
// Per `feedback_cpu_is_read_only`: all state in ISV.

#define RL_Q_DISTILL_LAMBDA_INDEX            486
#define RL_Q_DISTILL_KL_EMA_INDEX            488
#define RL_Q_DISTILL_KL_TARGET_INDEX         491

// Adaptive controller floors (spec 2026-05-30 Special case Q):
// hardcoded `MIN_LAMBDA = 0.05f` floor replaced with adaptive bound
// derived from Welford variance on q_distill_kl_ema. Phase 4.5 reduces
// KL signal magnitudes which this controller consumes; the "below target"
// path fires continuously, decaying λ to MIN. Adaptive bound
// `max(0.001, sqrt(var) × 0.05)` lets λ decay to a level proportional to
// the actual signal noise rather than the hardcoded 0.05 floor.
#define RL_Q_DISTILL_KL_VAR_COUNT_INDEX        618
#define RL_Q_DISTILL_KL_VAR_M2_INDEX           620
#define RL_Q_DISTILL_LAMBDA_MIN_ADAPTIVE_INDEX 639

// MAX_LAMBDA, KL_TOLERANCE, LAMBDA_RAMP_RATE, LAMBDA_DECAY_RATE are now
// ISV-driven per the 2026-05-30 clamp-bound extension. ADAPTIVE_MIN_*
// remain hardcoded — they're new constants from the noise-floor design
// itself per spec exemption ("not clamp bounds, new pattern constants").
#define RL_Q_DISTILL_LAMBDA_MAX_INDEX        650
#define RL_Q_DISTILL_KL_TOLERANCE_INDEX      651
#define RL_Q_DISTILL_LAMBDA_RAMP_RATE_INDEX  652
#define RL_Q_DISTILL_LAMBDA_DECAY_RATE_INDEX 653
#define ADAPTIVE_MIN_ABSOLUTE                0.001f
#define ADAPTIVE_MIN_STD_SCALE               0.05f

extern "C" __global__ void rl_q_distill_lambda_controller(
    float* __restrict__ isv
) {
    if (threadIdx.x != 0 || blockIdx.x != 0) return;

    const float kl_observed = isv[RL_Q_DISTILL_KL_EMA_INDEX];
    const float kl_target   = isv[RL_Q_DISTILL_KL_TARGET_INDEX];
    float       lambda      = isv[RL_Q_DISTILL_LAMBDA_INDEX];

    if (kl_observed <= 0.0f || kl_target <= 0.0f || lambda <= 0.0f) return;

    // Adaptive MIN bound (spec 2026-05-30 Special case Q): replaces
    // hardcoded `MIN_LAMBDA = 0.05f`. Welford sample variance =
    // M² / (count − 1) when count > 1. Adaptive min scales with signal
    // std so λ can decay proportional to actual KL noise rather than
    // pegging at an arbitrary 0.05 floor.
    const float kl_var_count = isv[RL_Q_DISTILL_KL_VAR_COUNT_INDEX];
    const float kl_var = (kl_var_count > 1.0f)
        ? isv[RL_Q_DISTILL_KL_VAR_M2_INDEX] / (kl_var_count - 1.0f)
        : 0.0f;
    const float kl_std = sqrtf(kl_var);
    const float adaptive_min = fmaxf(ADAPTIVE_MIN_ABSOLUTE,
                                      kl_std * ADAPTIVE_MIN_STD_SCALE);
    isv[RL_Q_DISTILL_LAMBDA_MIN_ADAPTIVE_INDEX] = adaptive_min;

    const float max_lambda        = isv[RL_Q_DISTILL_LAMBDA_MAX_INDEX];
    const float kl_tolerance      = isv[RL_Q_DISTILL_KL_TOLERANCE_INDEX];
    const float lambda_ramp_rate  = isv[RL_Q_DISTILL_LAMBDA_RAMP_RATE_INDEX];
    const float lambda_decay_rate = isv[RL_Q_DISTILL_LAMBDA_DECAY_RATE_INDEX];

    const float upper = kl_target * kl_tolerance;
    const float lower = kl_target / kl_tolerance;

    if (kl_observed > upper) {
        lambda = fminf(max_lambda, lambda * lambda_ramp_rate);
    } else if (kl_observed < lower) {
        lambda = fmaxf(adaptive_min, lambda * lambda_decay_rate);
    }

    isv[RL_Q_DISTILL_LAMBDA_INDEX] = lambda;
}