gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: adaptive CV readiness — relative improvement from initial CV

Browse Source 
readiness = (cv_initial - cv_current) / cv_initial

No fixed threshold. Captures initial CV at step 1, measures fractional
improvement. Fully domain-adaptive: works for any batch size, reward
scale, or Q-value magnitude.

Result: fold 2 Sharpe 2.47 → 16.15, peak epoch 18 → 42/50.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-04-13 20:02:02 +02:00
parent 7e8150bdf6
commit 3a8a756332
2 changed files with 16 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
crates/ml/src/cuda_pipeline/gpu_iql_trainer.rs
View File

@@ -169,7 +169,7 @@ pub struct GpuIqlTrainer {
// ── New integration buffers ─────────────────────────────────────
adv_stats_buf: CudaSlice<f32>, // [2] mean, variance
adv_sigma_ema_buf: CudaSlice<f32>, // [1] GPU-side EMA of advantage std
readiness_buf: CudaSlice<f32>, // [1] CV-based readiness scalar (0=suppress, 1=active)
readiness_buf: CudaSlice<f32>, // [2]: [0]=readiness scalar, [1]=initial_cv baseline
adv_sigma_ema_kernel: CudaFunction,
per_sample_support_buf: CudaSlice<f32>, // [B, 3]
branch_scales_buf: CudaSlice<f32>, // [B, 4]
@@ -230,7 +230,7 @@ impl GpuIqlTrainer {
// New integration buffers
let adv_stats_buf = alloc_f32(&stream, 2, "iql_adv_stats")?;
let adv_sigma_ema_buf = alloc_f32(&stream, 1, "iql_adv_sigma_ema")?;
let readiness_buf = alloc_f32(&stream, 1, "iql_readiness")?;
let readiness_buf = alloc_f32(&stream, 2, "iql_readiness")?; // [0]=readiness, [1]=initial_cv
let mut per_sample_support_buf = alloc_f32(&stream, b * 3, "iql_per_sample_support")?;
let branch_scales_buf = alloc_f32(&stream, b * 4, "iql_branch_scales")?;
let expectile_gap_buf = alloc_f32(&stream, b, "iql_expectile_gap")?;

19
crates/ml/src/cuda_pipeline/iql_value_kernel.cu
View File

@@ -660,7 +660,7 @@ extern "C" __global__
void iql_adv_sigma_ema_update(
const float* __restrict__ adv_stats, /* [2]: mean, variance */
float* __restrict__ sigma_ema, /* [1] running EMA (device-side) */
float* __restrict__ readiness_buf, /* [1] CV-based readiness scalar */
float* __restrict__ readiness_buf, /* [2]: [0]=readiness, [1]=initial_cv */
float ema_beta /* 0.99 */
)
{
@@ -674,11 +674,20 @@ void iql_adv_sigma_ema_update(
sigma_ema[0] = ema_beta * prev + (1.0f - ema_beta) * sigma;
}
/* CV-based readiness: CV = sigma / |mean|.
* CV > 1 → advantages are noise → suppress IQL features.
* CV < 1 → advantages are stable → features fully active. */
/* Adaptive CV readiness: measure improvement from initial CV.
* readiness = (cv_initial - cv_current) / cv_initial
* Fully adaptive — no fixed threshold, works for any domain. */
float cv = sigma_ema[0] / fmaxf(fabsf(mean), 1e-6f);
readiness_buf[0] = fminf(1.0f / fmaxf(cv, 1.0f), 1.0f);
float cv_initial = readiness_buf[1];
if (cv_initial < 1e-8f) {
/* First step: capture initial CV as baseline */
readiness_buf[1] = cv;
readiness_buf[0] = 0.0f;
} else {
float improvement = (cv_initial - cv) / fmaxf(cv_initial, 1e-6f);
readiness_buf[0] = fminf(fmaxf(improvement, 0.0f), 1.0f);
}
}
/* ------------------------------------------------------------------ */