gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(explore): per-sample epsilon from IQL expectile gap

Browse Source 
Replace 3 scalar per-branch epsilons (epsilon_exp/ord/urg) in
branching_action_select with a single per-sample buffer [B] so that
V_high(s)-V_low(s) from IQL expectile regression drives exploration
uniformly across all 3 heads, superseding manual per-branch heuristics.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-04-13 15:01:20 +02:00
parent cae6d8e5a9
commit 9f90cdf39a
1 changed files with 5 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
crates/ml/src/cuda_pipeline/epsilon_greedy_kernel.cu
View File

@@ -194,9 +194,7 @@ extern "C" __global__ void branching_action_select(
const float* __restrict__ q_urgency, /* [batch_size, 3] */
unsigned int* rng_states, /* [batch_size] */
unsigned int* actions_out, /* [batch_size] -- factored index 0-44 */
const float epsilon_exp, /* per-branch epsilon: exposure (dir+mag) */
const float epsilon_ord, /* per-branch epsilon: order type */
const float epsilon_urg, /* per-branch epsilon: urgency */
const float* __restrict__ per_sample_epsilon, /* [B] state-dependent epsilon */
const int batch_size,
const float q_gap_threshold, /* min Q-gap for trade entry (0.0 = disabled) */
const float* __restrict__ bonus_exposure, /* [5] UCB count bonus per exposure action (NULL = disabled) */
@@ -207,14 +205,12 @@ extern "C" __global__ void branching_action_select(
if (idx >= batch_size) return;
unsigned int rng = rng_states[idx];
float eps_exp_bf = bf16(epsilon_exp);
float eps_ord_bf = bf16(epsilon_ord);
float eps_urg_bf = bf16(epsilon_urg);
float eps = per_sample_epsilon[idx];
/* Head 1: Exposure (5 actions) -- with UCB bonus + Q-gap conviction filter */
int exposure;
float r1 = bf16(gpu_random(&rng));
if (r1 < eps_exp_bf) {
if (r1 < eps) {
exposure = (int)(gpu_random(&rng) * 5.0f);
if (exposure >= 5) exposure = 4;
} else {
@@ -244,7 +240,7 @@ extern "C" __global__ void branching_action_select(
/* Head 2: Order type (3 actions) -- Boltzmann softmax with UCB bonus */
int order;
float r2 = bf16(gpu_random(&rng));
if (r2 < eps_ord_bf) {
if (r2 < eps) {
/* Random exploration */
order = (int)(gpu_random(&rng) * 3.0f);
if (order >= 3) order = 2;
@@ -279,7 +275,7 @@ extern "C" __global__ void branching_action_select(
/* Head 3: Urgency (3 actions) -- Boltzmann softmax with UCB bonus */
int urgency;
float r3 = bf16(gpu_random(&rng));
if (r3 < eps_urg_bf) {
if (r3 < eps) {
/* Random exploration */
urgency = (int)(gpu_random(&rng) * 3.0f);
if (urgency >= 3) urgency = 2;