diff --git a/crates/ml/src/cuda_pipeline/experience_kernels.cu b/crates/ml/src/cuda_pipeline/experience_kernels.cu index 64592887c..14faf31ed 100644 --- a/crates/ml/src/cuda_pipeline/experience_kernels.cu +++ b/crates/ml/src/cuda_pipeline/experience_kernels.cu @@ -4631,20 +4631,23 @@ extern "C" __global__ void homeostatic_regularizer( float obs = observables[k]; float tgt = targets[k]; - float denom = fmaxf(fabsf(tgt), 1e-6f); - float error = (obs - tgt) / denom; + + /* Adaptive normalization: use max(|target|, |observed|, 1.0) as scale. + * For zero targets (Q-mean): scale = max(1.0, |observed|) — absolute error, not relative. + * For non-zero targets (atom_util=0.85): scale = 0.85 — relative error. + * This prevents zero-target observables from producing infinite error. */ + float scale = fmaxf(fmaxf(fabsf(tgt), fabsf(obs)), 1.0f); + float error = (obs - tgt) / scale; /* Quadratic signed penalty — small drift = tiny, large drift = hard correction */ float pen = lambda_base * error * fabsf(error); - penalties[k] = pen; - /* Atomic sum for budget cap (single warp, minimal contention) */ - atomicAdd(total_penalty, fabsf(pen)); - __syncthreads(); - - /* Budget cap: scale all penalties if total exceeds budget */ - float total = total_penalty[0]; - if (total > budget_max && total > 1e-8f) { - penalties[k] *= budget_max / total; + /* PER-OBSERVABLE budget cap (not global — prevents one runaway stealing all budget). + * Each observable gets budget_max / n_obs of the total budget. */ + float per_obs_budget = budget_max / (float)n_obs; + if (fabsf(pen) > per_obs_budget) { + pen = (pen > 0.0f) ? per_obs_budget : -per_obs_budget; } + + penalties[k] = pen; } diff --git a/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs b/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs index 2de532785..74bfb13b9 100644 --- a/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs +++ b/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs @@ -1455,15 +1455,14 @@ impl GpuDqnTrainer { } } - /// Calibrate targets from observations (epochs 1-5). - /// EMA blend with alpha=0.3 so targets track early training dynamics. - /// Q-mean target is always forced to 0.0 (centered Q-values). - pub(crate) fn calibrate_homeostatic_targets(&mut self, epoch: usize) { - if epoch > 5 || self.homeostatic_calibration_done { - self.homeostatic_calibration_done = true; - return; - } - let alpha = 0.3_f32; + /// Continuously adaptive homeostatic targets — driven by model readiness, not epoch. + /// Low readiness (exploring): fast alpha=0.3 (targets track rapidly, model still discovering healthy range). + /// High readiness (converged): slow alpha=0.01 (targets stable, only drift with gradual shifts). + /// alpha = 0.3 * (1 - readiness) + 0.01 * readiness → smooth interpolation. + /// Q-mean target always 0.0 (centered Q-values are the invariant). + pub(crate) fn calibrate_homeostatic_targets(&mut self) { + let readiness = self.iqn_readiness.clamp(0.0, 1.0); + let alpha = 0.3 * (1.0 - readiness) + 0.01 * readiness; unsafe { for k in 0..HOMEOSTATIC_N_OBS { let obs = *self.homeostatic_obs_pinned.add(k); diff --git a/crates/ml/src/trainers/dqn/fused_training.rs b/crates/ml/src/trainers/dqn/fused_training.rs index 84ce95360..a9e591d83 100644 --- a/crates/ml/src/trainers/dqn/fused_training.rs +++ b/crates/ml/src/trainers/dqn/fused_training.rs @@ -1983,10 +1983,9 @@ impl FusedTrainingCtx { .map_err(|e| anyhow::anyhow!("reduce_current_q_stats: {e}")) } - /// Calibrate homeostatic targets from early-epoch observations (epochs 1-5). - /// Passthrough to GpuDqnTrainer::calibrate_homeostatic_targets. - pub(crate) fn calibrate_homeostatic_targets(&mut self, epoch: usize) { - self.trainer.calibrate_homeostatic_targets(epoch); + /// Calibrate homeostatic targets — driven by model readiness, not epoch. + pub(crate) fn calibrate_homeostatic_targets(&mut self) { + self.trainer.calibrate_homeostatic_targets(); } /// Run cuBLAS forward pass and return reference to GPU-resident Q-values.