gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: eval mode uses Boltzmann sampling — argmax froze val_Sharpe at 0.00

Browse Source 
All 4 branches (dir, mag, order, urgency) now use Boltzmann sampling
in eval mode instead of argmax. argmax(softmax(Q/tau)) = argmax(Q) —
temperature is meaningless with argmax. Every state picked the same
winning action → identical trades → Sharpe=0.00 from epoch 8 onward.
Boltzmann sampling with Philox seed gives deterministic but
action-diverse evaluation. When Q-values differentiate, Boltzmann
naturally sharpens toward the best action.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-04-14 16:58:26 +02:00
parent c55d9d0276
commit 0b59b30549
1 changed files with 16 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
crates/ml/src/cuda_pipeline/experience_kernels.cu
View File

@@ -849,22 +849,17 @@ extern "C" __global__ void experience_action_select(
exps_d[a] = expf((qv - q_max_d) / tau_d);
sum_e += exps_d[a];
}
if (eval_mode) {
/* Eval: argmax of softmax probs — same as argmax(Q) but with
temperature-aware tie-breaking when Q-values are equal */
dir_idx = 0;
for (int a = 1; a < b0_size; a++) {
if (exps_d[a] > exps_d[dir_idx]) dir_idx = a;
}
} else {
/* Training: sample from Boltzmann distribution */
float r = philox_uniform(i, timestep, rng_ctr++) * sum_e;
float cum = 0.0f;
dir_idx = b0_size - 1;
for (int a = 0; a < b0_size; a++) {
cum += exps_d[a];
if (r < cum) { dir_idx = a; break; }
}
/* Both eval and training: Boltzmann sampling from softmax probs.
* Eval determinism comes from Philox seed (same i+timestep → same
* sample), NOT from argmax. argmax(softmax) = argmax(Q) which
* picks the same action for every state → val_Sharpe=0.00 freeze.
* Boltzmann sampling differentiates even when Q-gap is small. */
float r = philox_uniform(i, timestep, rng_ctr++) * sum_e;
float cum = 0.0f;
dir_idx = b0_size - 1;
for (int a = 0; a < b0_size; a++) {
cum += exps_d[a];
if (r < cum) { dir_idx = a; break; }
}
}
}
@@ -888,15 +883,7 @@ extern "C" __global__ void experience_action_select(
*
* Temperature tau = Q-range / 3 (adaptive, prevents division by zero).
* At tau → 0: greedy argmax. At tau → ∞: uniform random. */
if (eval_mode) {
/* Eval mode: pure greedy argmax for magnitude */
mag_idx = 0;
float best_q = (q_b1[0]);
for (int a = 1; a < b1_size; a++) {
float qv = (q_b1[a]);
if (qv > best_q) { best_q = qv; mag_idx = a; }
}
} else if (philox_uniform(i, timestep, rng_ctr++) < eps_mag) {
if (!eval_mode && philox_uniform(i, timestep, rng_ctr++) < eps_mag) {
int r = (int)(philox_uniform(i, timestep, rng_ctr++) * (float)b1_size);
mag_idx = (r >= b1_size) ? b1_size - 1 : r;
} else {
@@ -934,14 +921,8 @@ extern "C" __global__ void experience_action_select(
}
} /* end else (not in_hold) */
/* Branch 2: order type */
if (eval_mode) {
a2 = 0;
float best_q = q_b2[0];
for (int a = 1; a < b2_size; a++) {
if (q_b2[a] > best_q) { best_q = q_b2[a]; a2 = a; }
}
} else if (philox_uniform(i, timestep, rng_ctr++) < eps_ord) {
/* Branch 2: order type — Boltzmann for both eval and training */
if (!eval_mode && philox_uniform(i, timestep, rng_ctr++) < eps_ord) {
int r = (int)(philox_uniform(i, timestep, rng_ctr++) * (float)b2_size);
a2 = (r >= b2_size) ? b2_size - 1 : r;
} else {
@@ -972,14 +953,8 @@ extern "C" __global__ void experience_action_select(
}
}
/* Branch 3: urgency */
if (eval_mode) {
a3 = 0;
float best_q = q_b3[0];
for (int a = 1; a < b3_size; a++) {
if (q_b3[a] > best_q) { best_q = q_b3[a]; a3 = a; }
}
} else if (philox_uniform(i, timestep, rng_ctr++) < eps_urg) {
/* Branch 3: urgency — Boltzmann for both eval and training */
if (!eval_mode && philox_uniform(i, timestep, rng_ctr++) < eps_urg) {
int r = (int)(philox_uniform(i, timestep, rng_ctr++) * (float)b3_size);
a3 = (r >= b3_size) ? b3_size - 1 : r;
} else {