feat(B2/G3): health-coupled tau — floor at 0.01×(1-health) for collapse recovery
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -1663,6 +1663,10 @@ pub struct GpuDqnTrainer {
|
||||
/// Last effective cql_alpha applied (base × (1−regime_stability) × health).
|
||||
/// Populated by apply_cql_gradient. Initialized to 0.0.
|
||||
pub(crate) last_cql_alpha_eff: f32,
|
||||
|
||||
/// Last effective tau value (cosine-annealed, potentially floored by 0.01×(1-health)).
|
||||
/// Populated by `apply_health_coupled_tau_floor`. Initialized to 0.0.
|
||||
pub(crate) last_tau_eff: f32,
|
||||
}
|
||||
|
||||
impl GpuDqnTrainer {
|
||||
@@ -7442,6 +7446,7 @@ impl GpuDqnTrainer {
|
||||
tau_final,
|
||||
tau_anneal_steps,
|
||||
last_cql_alpha_eff: 0.0,
|
||||
last_tau_eff: 0.0,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -7623,6 +7628,22 @@ impl GpuDqnTrainer {
|
||||
}
|
||||
}
|
||||
|
||||
/// B2/G3: Apply a health-coupled minimum to a scheduled tau value.
|
||||
/// Returns `max(tau_scheduled, 0.01 * (1.0 - health))`. During collapse
|
||||
/// (health≈0), the floor rises to 0.01 to accelerate target-network adaptation.
|
||||
/// Caches the effective value for HEALTH_DIAG logging.
|
||||
pub fn apply_health_coupled_tau_floor(&mut self, tau_scheduled: f64) -> f64 {
|
||||
let health = if self.isv_signals_pinned.is_null() {
|
||||
0.5_f32
|
||||
} else {
|
||||
unsafe { (*self.isv_signals_pinned.add(LEARNING_HEALTH_INDEX)).clamp(0.0, 1.0) }
|
||||
};
|
||||
let tau_floor = (0.01 * (1.0 - health)) as f64;
|
||||
let tau_eff = tau_scheduled.max(tau_floor);
|
||||
self.last_tau_eff = tau_eff as f32;
|
||||
tau_eff
|
||||
}
|
||||
|
||||
// ── A3: LearningHealth signal writers / readers ───────────────────────
|
||||
|
||||
/// Write a scalar into the ISV pinned buffer at the given index.
|
||||
|
||||
@@ -1168,12 +1168,13 @@ impl FusedTrainingCtx {
|
||||
// EMA target update — adaptive tau based on Q-divergence.
|
||||
{
|
||||
let dqn = agent.primary_dqn_mut();
|
||||
let tau = compute_cosine_annealed_tau(
|
||||
let tau_scheduled = compute_cosine_annealed_tau(
|
||||
dqn.get_training_steps(),
|
||||
dqn.config.tau,
|
||||
dqn.config.tau_final,
|
||||
dqn.config.tau_anneal_steps,
|
||||
);
|
||||
let tau = self.trainer.apply_health_coupled_tau_floor(tau_scheduled);
|
||||
// Use cosine-annealed tau directly — fully deterministic.
|
||||
// Adaptive tau (from q_divergence) was removed because q_divergence
|
||||
// uses atomicAdd in the C51 loss kernel, introducing non-determinism
|
||||
@@ -1428,12 +1429,13 @@ impl FusedTrainingCtx {
|
||||
).map_err(|e| anyhow::anyhow!("IQN trunk gradient: {e}"))?;
|
||||
|
||||
let dqn = agent.primary_dqn_mut();
|
||||
let tau = compute_cosine_annealed_tau(
|
||||
let tau_scheduled = compute_cosine_annealed_tau(
|
||||
dqn.get_training_steps(),
|
||||
dqn.config.tau,
|
||||
dqn.config.tau_final,
|
||||
dqn.config.tau_anneal_steps,
|
||||
);
|
||||
let tau = self.trainer.apply_health_coupled_tau_floor(tau_scheduled);
|
||||
iqn.target_ema_update(tau as f32)
|
||||
.map_err(|e| anyhow::anyhow!("IQN EMA update: {e}"))?;
|
||||
|
||||
@@ -1829,23 +1831,25 @@ impl FusedTrainingCtx {
|
||||
iqn.increment_adam_step();
|
||||
// Update IQN tau via stable host address
|
||||
let dqn = agent.primary_dqn_mut();
|
||||
let tau = compute_cosine_annealed_tau(
|
||||
let tau_scheduled = compute_cosine_annealed_tau(
|
||||
dqn.get_training_steps(),
|
||||
dqn.config.tau,
|
||||
dqn.config.tau_final,
|
||||
dqn.config.tau_anneal_steps,
|
||||
);
|
||||
let tau = self.trainer.apply_health_coupled_tau_floor(tau_scheduled);
|
||||
iqn.set_tau_host(tau as f32);
|
||||
}
|
||||
// Update DQN trainer tau via stable host address
|
||||
{
|
||||
let dqn = agent.primary_dqn_mut();
|
||||
let tau = compute_cosine_annealed_tau(
|
||||
let tau_scheduled = compute_cosine_annealed_tau(
|
||||
dqn.get_training_steps(),
|
||||
dqn.config.tau,
|
||||
dqn.config.tau_final,
|
||||
dqn.config.tau_anneal_steps,
|
||||
);
|
||||
let tau = self.trainer.apply_health_coupled_tau_floor(tau_scheduled);
|
||||
self.trainer.set_tau_value(tau as f32);
|
||||
}
|
||||
}
|
||||
@@ -2189,6 +2193,11 @@ impl FusedTrainingCtx {
|
||||
self.trainer.last_cql_alpha_eff
|
||||
}
|
||||
|
||||
/// Last effective tau (cosine-annealed, potentially health-floored).
|
||||
pub(crate) fn last_tau_eff(&self) -> f32 {
|
||||
self.trainer.last_tau_eff
|
||||
}
|
||||
|
||||
/// Compute spectral gap on the last Q readback slice (coarse max/min ratio proxy).
|
||||
pub(crate) fn compute_q_spectral_gap(&self) -> f32 {
|
||||
self.trainer.compute_q_spectral_gap()
|
||||
|
||||
@@ -1863,6 +1863,11 @@ impl DQNTrainer {
|
||||
self.last_cql_alpha_eff = Some(fused.last_cql_alpha_eff());
|
||||
}
|
||||
|
||||
// B2/G3: propagate last tau_eff for logging.
|
||||
if let Some(ref fused) = self.fused_ctx {
|
||||
self.last_tau_eff = Some(fused.last_tau_eff());
|
||||
}
|
||||
|
||||
// HEALTH_DIAG: components are [0, 1] normalized. effective = hyperparams after health-adaptation. novels = mechanism states.
|
||||
tracing::info!(
|
||||
"HEALTH_DIAG[{}]: health={:.2} components [q_gap={:.2} q_var={:.2} atoms={:.2} grad_stable={:.2} ens_agree={:.2} grad_cos={:.2} spectral={:.2}] effective [cql_alpha={:.4} iqn_budget={:.2} cql_budget={:.2} c51_budget={:.2} tau={:.5} sarsa_tau={:.2} gamma={:.3} cf_ratio={:.2}] novels [distill={} barrier={:.3} plasticity={} ib={:.3} ensemble_collapse={:.2} contrarian={} meta_q_pred={:.2}]",
|
||||
|
||||
Reference in New Issue
Block a user