148454d37d
rl_curriculum_weights: per-segment Sharpe → z-score → softmax difficulty weights. Harder segments sampled more. Block tree-reduce, no atomics. rl_adversarial_boost: multiply PER priority by boost factor for negative-reward transitions. Self-regulating — fewer losses → fewer boosts. ISV-driven threshold + boost magnitude. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
59 lines
2.6 KiB
Plaintext
59 lines
2.6 KiB
Plaintext
// Adversarial regime injection: boost PER priority on negative-reward
|
|
// transitions to increase replay sampling of losing experiences.
|
|
//
|
|
// Self-regulating: as the agent improves, fewer negative-reward steps
|
|
// are generated, so fewer boosts are applied. The boost magnitude is
|
|
// ISV-driven (slot 607) so it adapts with the training regime.
|
|
//
|
|
// Grid=(1), Block=(min(B, 256)). Runs once per step after experience
|
|
// collection, before the next PER prefix scan.
|
|
//
|
|
// ISV slot layout (literal indices; canonical names in Rust-side
|
|
// adversarial_isv_slots module):
|
|
// 606 RL_ADVERSARIAL_DD_THRESHOLD (reserved, unused in V1)
|
|
// 607 RL_ADVERSARIAL_BOOST boost multiplier (default 1.5)
|
|
// 608 RL_MAX_DD_EMA (reserved, unused in V1)
|
|
|
|
extern "C" __global__ void rl_adversarial_boost(
|
|
const float* __restrict__ rewards, // [B] current step rewards
|
|
float* __restrict__ priorities_pa, // [capacity] flat PER sampling weights
|
|
const float* __restrict__ isv, // ISV bus (read boost factor)
|
|
int write_cursor, // host-side write position (post-insert)
|
|
int b_size, // batch size (number of transitions just inserted)
|
|
int capacity // replay buffer capacity
|
|
)
|
|
{
|
|
int b = threadIdx.x;
|
|
if (b >= b_size) return;
|
|
|
|
// Read boost factor from ISV[607]. Cold-start sentinel: if ISV is
|
|
// null or the slot is zero/negative, use 1.0 (no boost = no-op).
|
|
float boost = 1.0f;
|
|
if (isv != nullptr) {
|
|
float raw = isv[607];
|
|
// Sane range: boost in [1.0, 5.0]. Below 1.0 = no boost.
|
|
// Above 5.0 = cap to prevent priority explosion.
|
|
boost = fmaxf(1.0f, fminf(5.0f, raw));
|
|
}
|
|
|
|
// Only boost negative-reward transitions.
|
|
if (rewards[b] >= 0.0f) return;
|
|
|
|
// Map batch index b to the replay buffer entry that was just written.
|
|
// The caller passes write_cursor AFTER the insert, so the b_size
|
|
// entries occupy [write_cursor - b_size .. write_cursor) mod capacity.
|
|
// Entry for batch element b:
|
|
int entry = (write_cursor - b_size + b) % capacity;
|
|
// Handle negative modulo (C/CUDA % can return negative for negative dividend).
|
|
if (entry < 0) entry += capacity;
|
|
|
|
// Multiply the PER sampling weight by the boost factor.
|
|
// This makes negative-reward transitions more likely to be sampled
|
|
// in the next PER proportional-sampling pass.
|
|
//
|
|
// Each thread writes to a unique entry (the b→entry mapping is
|
|
// injective within the batch), so no synchronization needed.
|
|
float old_pa = priorities_pa[entry];
|
|
priorities_pa[entry] = old_pa * boost;
|
|
}
|