feat(B2/G3): health-coupled tau — floor at 0.01×(1-health) for collapse recovery

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-04-20 19:57:11 +02:00
parent 46666afef9
commit edc59ed6bd
3 changed files with 39 additions and 4 deletions

View File

@@ -1663,6 +1663,10 @@ pub struct GpuDqnTrainer {
/// Last effective cql_alpha applied (base × (1regime_stability) × health).
/// Populated by apply_cql_gradient. Initialized to 0.0.
pub(crate) last_cql_alpha_eff: f32,
/// Last effective tau value (cosine-annealed, potentially floored by 0.01×(1-health)).
/// Populated by `apply_health_coupled_tau_floor`. Initialized to 0.0.
pub(crate) last_tau_eff: f32,
}
impl GpuDqnTrainer {
@@ -7442,6 +7446,7 @@ impl GpuDqnTrainer {
tau_final,
tau_anneal_steps,
last_cql_alpha_eff: 0.0,
last_tau_eff: 0.0,
})
}
@@ -7623,6 +7628,22 @@ impl GpuDqnTrainer {
}
}
/// B2/G3: Apply a health-coupled minimum to a scheduled tau value.
/// Returns `max(tau_scheduled, 0.01 * (1.0 - health))`. During collapse
/// (health≈0), the floor rises to 0.01 to accelerate target-network adaptation.
/// Caches the effective value for HEALTH_DIAG logging.
pub fn apply_health_coupled_tau_floor(&mut self, tau_scheduled: f64) -> f64 {
let health = if self.isv_signals_pinned.is_null() {
0.5_f32
} else {
unsafe { (*self.isv_signals_pinned.add(LEARNING_HEALTH_INDEX)).clamp(0.0, 1.0) }
};
let tau_floor = (0.01 * (1.0 - health)) as f64;
let tau_eff = tau_scheduled.max(tau_floor);
self.last_tau_eff = tau_eff as f32;
tau_eff
}
// ── A3: LearningHealth signal writers / readers ───────────────────────
/// Write a scalar into the ISV pinned buffer at the given index.

View File

@@ -1168,12 +1168,13 @@ impl FusedTrainingCtx {
// EMA target update — adaptive tau based on Q-divergence.
{
let dqn = agent.primary_dqn_mut();
let tau = compute_cosine_annealed_tau(
let tau_scheduled = compute_cosine_annealed_tau(
dqn.get_training_steps(),
dqn.config.tau,
dqn.config.tau_final,
dqn.config.tau_anneal_steps,
);
let tau = self.trainer.apply_health_coupled_tau_floor(tau_scheduled);
// Use cosine-annealed tau directly — fully deterministic.
// Adaptive tau (from q_divergence) was removed because q_divergence
// uses atomicAdd in the C51 loss kernel, introducing non-determinism
@@ -1428,12 +1429,13 @@ impl FusedTrainingCtx {
).map_err(|e| anyhow::anyhow!("IQN trunk gradient: {e}"))?;
let dqn = agent.primary_dqn_mut();
let tau = compute_cosine_annealed_tau(
let tau_scheduled = compute_cosine_annealed_tau(
dqn.get_training_steps(),
dqn.config.tau,
dqn.config.tau_final,
dqn.config.tau_anneal_steps,
);
let tau = self.trainer.apply_health_coupled_tau_floor(tau_scheduled);
iqn.target_ema_update(tau as f32)
.map_err(|e| anyhow::anyhow!("IQN EMA update: {e}"))?;
@@ -1829,23 +1831,25 @@ impl FusedTrainingCtx {
iqn.increment_adam_step();
// Update IQN tau via stable host address
let dqn = agent.primary_dqn_mut();
let tau = compute_cosine_annealed_tau(
let tau_scheduled = compute_cosine_annealed_tau(
dqn.get_training_steps(),
dqn.config.tau,
dqn.config.tau_final,
dqn.config.tau_anneal_steps,
);
let tau = self.trainer.apply_health_coupled_tau_floor(tau_scheduled);
iqn.set_tau_host(tau as f32);
}
// Update DQN trainer tau via stable host address
{
let dqn = agent.primary_dqn_mut();
let tau = compute_cosine_annealed_tau(
let tau_scheduled = compute_cosine_annealed_tau(
dqn.get_training_steps(),
dqn.config.tau,
dqn.config.tau_final,
dqn.config.tau_anneal_steps,
);
let tau = self.trainer.apply_health_coupled_tau_floor(tau_scheduled);
self.trainer.set_tau_value(tau as f32);
}
}
@@ -2189,6 +2193,11 @@ impl FusedTrainingCtx {
self.trainer.last_cql_alpha_eff
}
/// Last effective tau (cosine-annealed, potentially health-floored).
pub(crate) fn last_tau_eff(&self) -> f32 {
self.trainer.last_tau_eff
}
/// Compute spectral gap on the last Q readback slice (coarse max/min ratio proxy).
pub(crate) fn compute_q_spectral_gap(&self) -> f32 {
self.trainer.compute_q_spectral_gap()

View File

@@ -1863,6 +1863,11 @@ impl DQNTrainer {
self.last_cql_alpha_eff = Some(fused.last_cql_alpha_eff());
}
// B2/G3: propagate last tau_eff for logging.
if let Some(ref fused) = self.fused_ctx {
self.last_tau_eff = Some(fused.last_tau_eff());
}
// HEALTH_DIAG: components are [0, 1] normalized. effective = hyperparams after health-adaptation. novels = mechanism states.
tracing::info!(
"HEALTH_DIAG[{}]: health={:.2} components [q_gap={:.2} q_var={:.2} atoms={:.2} grad_stable={:.2} ens_agree={:.2} grad_cos={:.2} spectral={:.2}] effective [cql_alpha={:.4} iqn_budget={:.2} cql_budget={:.2} c51_budget={:.2} tau={:.5} sarsa_tau={:.2} gamma={:.3} cf_ratio={:.2}] novels [distill={} barrier={:.3} plasticity={} ib={:.3} ensemble_collapse={:.2} contrarian={} meta_q_pred={:.2}]",