gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(4branch): config foundation — b0=3 direction, b1=3 magnitude, b2=3 order, b3=3 urgency

Browse Source 
Delete exposure_aux_weight, exposure_aux_warmup_epochs.
b0_size 9→3 (direction branch). b3_size=3 added (urgency).

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-04-08 12:18:50 +02:00
parent 977b47a440
commit 4889b6e968
16 changed files with 54 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
config/training/dqn-hyperopt.toml
View File

@@ -92,7 +92,8 @@ exit_timing_weight = [0.0, 0.2]
ofi_reward_weight = [0.0, 0.5]
kelly_sizing_weight = [0.0, 0.5]
reward_noise_scale = [0.01, 0.1]
exposure_aux_weight = [0.001, 0.1]
# exposure_aux_weight removed (4-branch refactor)
b3_size = 3
# v8 search ranges
micro_reward_scale = [0.0, 0.05]
@@ -139,8 +140,7 @@ exit_timing_weight = 0.05
ofi_reward_weight = 0.2
kelly_sizing_weight = 0.1
reward_noise_scale = 0.05
exposure_aux_weight = 0.1
exposure_aux_warmup_epochs = 5
b3_size = 3
[fixed]
@@ -182,5 +182,4 @@ exit_timing_weight = 0.05
ofi_reward_weight = 0.2
kelly_sizing_weight = 0.1
reward_noise_scale = 0.05
exposure_aux_weight = 0.1
exposure_aux_warmup_epochs = 5
b3_size = 3

3
config/training/dqn-localdev.toml
View File

@@ -131,5 +131,4 @@ exit_timing_weight = 0.05
ofi_reward_weight = 0.2
kelly_sizing_weight = 0.1
reward_noise_scale = 0.05
exposure_aux_weight = 0.1
exposure_aux_warmup_epochs = 5
b3_size = 3

3
config/training/dqn-production.toml
View File

@@ -142,5 +142,4 @@ exit_timing_weight = 0.05
ofi_reward_weight = 0.2
kelly_sizing_weight = 0.1
reward_noise_scale = 0.05
exposure_aux_weight = 0.1
exposure_aux_warmup_epochs = 5
b3_size = 3

3
config/training/dqn-smoketest.toml
View File

@@ -138,8 +138,7 @@ exit_timing_weight = 0.05
ofi_reward_weight = 0.2
kelly_sizing_weight = 0.1
reward_noise_scale = 0.05
exposure_aux_weight = 0.1
exposure_aux_warmup_epochs = 5
b3_size = 3
[walk_forward]
step_fraction = 0.25

16
crates/ml/src/cuda_pipeline/gpu_backtest_evaluator.rs
View File

@@ -103,12 +103,14 @@ pub struct DqnBacktestConfig {
pub adv_h: usize,
/// C51 distributional atom count (typically 51).
pub num_atoms: usize,
/// Exposure branch action count (branch 0, typically 5).
/// Direction branch action count (branch 0, default 3 — Short/Flat/Long).
pub branch_0_size: usize,
/// Order-type branch action count (branch 1, typically 3).
/// Magnitude branch action count (branch 1, default 3 — 25%/50%/100%).
pub branch_1_size: usize,
/// Urgency branch action count (branch 2, typically 3).
/// Order-type branch action count (branch 2, default 3).
pub branch_2_size: usize,
/// Urgency branch action count (branch 3, default 3 — Patient/Normal/Aggressive).
pub branch_3_size: usize,
/// C51 minimum support value.
pub v_min: f32,
/// C51 maximum support value.
@@ -118,7 +120,7 @@ pub struct DqnBacktestConfig {
impl DqnBacktestConfig {
/// Create from the legacy `network_dims` tuple with standard C51 defaults.
///
/// Uses branch sizes [9, 3, 3], num_atoms=51, v_range from reward_scale=10/gamma=0.95.
/// Uses branch sizes [3, 3, 3, 3], num_atoms=51, v_range from reward_scale=10/gamma=0.95.
pub fn from_network_dims(network_dims: (usize, usize, usize, usize)) -> Self {
// v_min/v_max computed from reward_scale=10, gamma=0.95:
// Q* ~ 10 / (1-0.95) * 1.2 = 240, clamped to [20, 300] → ±240
@@ -129,9 +131,10 @@ impl DqnBacktestConfig {
value_h: network_dims.2,
adv_h: network_dims.3,
num_atoms: 51,
branch_0_size: 9,
branch_0_size: 3,
branch_1_size: 3,
branch_2_size: 3,
branch_3_size: 3,
v_min: -v_range,
v_max: v_range,
}
@@ -2049,9 +2052,10 @@ mod tests {
assert_eq!(cfg.value_h, 128);
assert_eq!(cfg.adv_h, 128);
assert_eq!(cfg.num_atoms, 51);
assert_eq!(cfg.branch_0_size, 9);
assert_eq!(cfg.branch_0_size, 3);
assert_eq!(cfg.branch_1_size, 3);
assert_eq!(cfg.branch_2_size, 3);
assert_eq!(cfg.branch_3_size, 3);
// v_range = (10 / (1-0.95) * 1.2).clamp(20, 300) = 240
let expected_v_range = (10.0_f32 / (1.0 - 0.95) * 1.2).clamp(20.0, 300.0);
assert!((cfg.v_min - (-expected_v_range)).abs() < f32::EPSILON);

11
crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs
View File

@@ -132,12 +132,14 @@ pub struct GpuDqnTrainConfig {
pub v_min: f32,
/// C51 maximum support value (default: 50.0).
pub v_max: f32,
/// Branch 0 (exposure) action count (default: 5).
/// Branch 0 (direction) action count (default: 3 — Short/Flat/Long).
pub branch_0_size: usize,
/// Branch 1 (order) action count (default: 3).
/// Branch 1 (magnitude) action count (default: 3 — 25%/50%/100%).
pub branch_1_size: usize,
/// Branch 2 (urgency) action count (default: 3).
/// Branch 2 (order) action count (default: 3).
pub branch_2_size: usize,
/// Branch 3 (urgency) action count (default: 3 — Patient/Normal/Aggressive).
pub branch_3_size: usize,
/// Training batch size (fixed for CUDA Graph capture).
pub batch_size: usize,
/// Discount factor for Bellman projection.
@@ -235,9 +237,10 @@ impl Default for GpuDqnTrainConfig {
// Q* ~ 10 / (1-0.95) * 1.2 = 240, clamped to [20, 300] → ±240
v_min: -(10.0_f32 / (1.0 - 0.95) * 1.2).clamp(20.0, 300.0),
v_max: (10.0_f32 / (1.0 - 0.95) * 1.2).clamp(20.0, 300.0),
branch_0_size: 9,
branch_0_size: 3,
branch_1_size: 3,
branch_2_size: 3,
branch_3_size: 3,
batch_size: 64, // Smoke test default (production overrides via from_hyperparams)
gamma: 0.99,
n_steps: 1,

1
crates/ml/src/cuda_pipeline/gpu_experience_collector.rs
View File

@@ -753,6 +753,7 @@ impl GpuExperienceCollector {
branch_0_size: branch_sizes[0],
branch_1_size: branch_sizes[1],
branch_2_size: branch_sizes[2],
branch_3_size: 3, // urgency — fixed at 3 (4-branch refactor)
bottleneck_dim: 0,
market_dim: 0,
..GpuDqnTrainConfig::default()

11
crates/ml/src/cuda_pipeline/gpu_iqn_head.rs
View File

@@ -57,12 +57,14 @@ pub struct GpuIqnConfig {
pub state_dim: usize,
/// First shared hidden layer width from DQN trunk (default 256).
pub shared_h1: usize,
/// Branch 0 (exposure) action count.
/// Branch 0 (direction) action count.
pub branch_0_size: usize,
/// Branch 1 (order) action count.
/// Branch 1 (magnitude) action count.
pub branch_1_size: usize,
/// Branch 2 (urgency) action count.
/// Branch 2 (order) action count.
pub branch_2_size: usize,
/// Branch 3 (urgency) action count.
pub branch_3_size: usize,
/// Training batch size (must match DQN trainer).
pub batch_size: usize,
/// Learning rate for IQN Adam optimizer.
@@ -90,9 +92,10 @@ impl Default for GpuIqnConfig {
kappa: 1.0,
state_dim: 72,
shared_h1: 256,
branch_0_size: 9,
branch_0_size: 3,
branch_1_size: 3,
branch_2_size: 3,
branch_3_size: 3,
batch_size: 256,
lr: 3e-4,
beta1: 0.9,

1
crates/ml/src/hyperopt/adapters/dqn.rs
View File

@@ -1478,6 +1478,7 @@ impl DQNTrainer {
branch_0_size: b0,
branch_1_size: if is_branching { b1 } else { b0 },
branch_2_size: if is_branching { b2 } else { b0 },
branch_3_size: 3, // urgency — fixed at 3 (4-branch refactor)
v_min: hp.v_min as f32,
v_max: hp.v_max as f32,
};

12
crates/ml/src/trainers/dqn/config.rs
View File

@@ -795,10 +795,7 @@ pub struct DQNHyperparameters {
pub kelly_sizing_weight: f64,
/// Reward v7 gem: Reward label smoothing noise scale.
pub reward_noise_scale: f64,
/// v7.1: Exposure auxiliary loss base weight (scheduled decay).
pub exposure_aux_weight: f64,
/// v7.1: Epochs over which exposure aux weight decays to 10%.
pub exposure_aux_warmup_epochs: usize,
// exposure_aux_weight and exposure_aux_warmup_epochs removed (4-branch refactor)
/// Position staleness rent per step held
pub time_decay_rate: f64,
/// Q-value gap threshold for trade conviction filter.
@@ -1538,7 +1535,7 @@ impl DQNHyperparameters {
self.ofi_reward_weight = (self.ofi_reward_weight * li).clamp(0.0, 1.0);
self.kelly_sizing_weight = (self.kelly_sizing_weight * li).clamp(0.0, 1.0);
self.reward_noise_scale = (self.reward_noise_scale * li).clamp(0.0, 0.2);
self.exposure_aux_weight = (self.exposure_aux_weight * li).clamp(0.0, 2.0);
// exposure_aux_weight removed (4-branch refactor)
self.micro_reward_scale = (self.micro_reward_scale * li).clamp(0.0, 0.01);
self.position_entropy_weight *= li;
// v6 legacy (zeroed defaults, no-op)
@@ -1653,8 +1650,7 @@ impl DQNHyperparameters {
ofi_reward_weight: 0.2,
kelly_sizing_weight: 0.1,
reward_noise_scale: 0.05,
exposure_aux_weight: 0.1,
exposure_aux_warmup_epochs: 5,
// exposure_aux_weight / exposure_aux_warmup_epochs removed (4-branch refactor)
// v8: Dense micro-reward, TD(λ), PopArt, curriculum, hindsight
micro_reward_scale: 0.01,
td_lambda: 0.9,
@@ -1927,7 +1923,7 @@ pub(crate) fn dqn_default_config() -> DQNConfig {
DQNConfig {
// Network Architecture
state_dim: 48, // Default placeholder; overridden by constructor to (42+8+16+7)&!7=72
num_actions: 9, // 9 exposure levels (25% steps)
num_actions: 3, // 3 direction levels (Short/Flat/Long) — 4-branch refactor
hidden_dims: vec![512, 256, 128], // Sufficient capacity without overfitting (Trial 19)
leaky_relu_alpha: 0.01,

6
crates/ml/src/trainers/dqn/fused_training.rs
View File

@@ -209,7 +209,7 @@ pub(crate) struct FusedTrainingCtx {
graph_mega: Option<RawCudaGraph>,
/// Host-side tau for graph replay. Graph captures HtoD from this address.
tau_host: f32,
/// v7.1: Exposure auxiliary gradient weight (decayed over warmup epochs).
// exposure_aux_weight removed (4-branch refactor) — kept as zero sentinel
pub(crate) exposure_aux_weight: f64,
/// v7.1: Exposure targets buffer [batch_size] -- from experience collector.
pub(crate) exposure_targets: cudarc::driver::CudaSlice<i32>,
@@ -278,6 +278,7 @@ impl FusedTrainingCtx {
branch_0_size: dqn.config.num_actions,
branch_1_size: dqn.config.num_order_types,
branch_2_size: dqn.config.num_urgency_levels,
branch_3_size: 3, // urgency — fixed at 3 (4-branch refactor)
batch_size,
gamma: hyperparams.gamma as f32,
n_steps: hyperparams.n_steps,
@@ -429,6 +430,7 @@ impl FusedTrainingCtx {
branch_0_size: dqn.config.num_actions,
branch_1_size: dqn.config.num_order_types,
branch_2_size: dqn.config.num_urgency_levels,
branch_3_size: 3, // urgency — fixed at 3 (4-branch refactor)
batch_size,
lr: hyperparams.learning_rate as f32,
max_grad_norm: resolved_grad_norm as f32,
@@ -611,7 +613,7 @@ impl FusedTrainingCtx {
graph_spectral: None,
graph_mega: None,
tau_host: 0.0,
exposure_aux_weight: hyperparams.exposure_aux_weight,
exposure_aux_weight: 0.0, // removed in 4-branch refactor
exposure_targets,
popart_enabled: hyperparams.popart_enabled,
hindsight_fraction: hyperparams.hindsight_fraction,

2
crates/ml/src/trainers/dqn/smoke_tests/gradient_budget.rs
View File

@@ -22,7 +22,7 @@ fn test_config() -> GpuDqnTrainConfig {
num_atoms: 11,
v_min: -10.0,
v_max: 10.0,
branch_0_size: 9,
branch_0_size: 3,
branch_1_size: 3,
branch_2_size: 3,
batch_size: 8,

4
crates/ml/src/trainers/dqn/smoke_tests/reward_v8.rs
View File

@@ -75,8 +75,8 @@ mod tests {
#[test]
fn test_exposure_aux_gradient_unique() {
let b0 = 9;
let target = 3;
let b0 = 3; // 4-branch: direction branch now 3 (Short/Flat/Long)
let target = 1; // Flat
let prob = 1.0 / b0 as f64;
let grads: Vec<f64> = (0..b0).map(|a| prob - if a == target { 1.0 } else { 0.0 }).collect();
assert!(grads[target] < 0.0, "Target gets negative gradient");

1
crates/ml/src/trainers/dqn/trainer/metrics.rs
View File

@@ -508,6 +508,7 @@ impl DQNTrainer {
branch_0_size: b0,
branch_1_size: if is_branching { b1 } else { b0 },
branch_2_size: if is_branching { b2 } else { b0 },
branch_3_size: 3, // urgency — fixed at 3 (4-branch refactor)
v_min: hp.v_min as f32,
v_max: hp.v_max as f32,
};

10
crates/ml/src/trainers/dqn/trainer/training_loop.rs
View File

@@ -191,7 +191,7 @@ impl DQNTrainer {
}
// v8: Ensure fused context exists for pre-training (normally lazy-init in Phase 3)
if self.hyperparams.exposure_aux_weight > 0.0 && self.fused_ctx.is_none() && self.device.is_cuda() {
if false && self.fused_ctx.is_none() && self.device.is_cuda() { // exposure_aux removed (4-branch refactor)
if let Some(ref stream) = self.cuda_stream {
let agent = self.agent.read().await;
match super::super::fused_training::FusedTrainingCtx::new(
@@ -204,7 +204,7 @@ impl DQNTrainer {
}
// v8: Supervised pre-training (epoch 0) — initialize exposure branch with directional knowledge
if self.hyperparams.exposure_aux_weight > 0.0 {
if false { // exposure_aux removed (4-branch refactor)
let pretrain_total_bars = self.gpu_data.as_ref().map(|d| d.num_bars);
let pretrain_batch_size = pretrain_total_bars.map(|t| self.current_batch_size.min(t));
let max_pos = self.max_position as f32;
@@ -1177,10 +1177,8 @@ impl DQNTrainer {
// v7.1: Exposure aux weight decay over warmup epochs
if let Some(ref mut fused) = self.fused_ctx {
let base = self.hyperparams.exposure_aux_weight;
let warmup = self.hyperparams.exposure_aux_warmup_epochs.max(1) as f64;
let progress = (self.current_epoch as f64 / warmup).min(1.0);
fused.exposure_aux_weight = base * (1.0 - 0.9 * progress);
// exposure_aux_weight decay removed (4-branch refactor)
fused.exposure_aux_weight = 0.0;
}
let gpu_batch = collector.collect_experiences_gpu(

8
crates/ml/src/training_profile.rs
View File

@@ -135,12 +135,14 @@ pub struct DistributionalSection {
/// Branching DQN per-head action-space sizes.
#[derive(Debug, Clone, Deserialize, Default)]
pub struct BranchingSection {
/// Branch 0: exposure head size (default 5).
/// Branch 0: direction head size (default 3 — Short/Flat/Long).
pub branch_0_size: Option<usize>,
/// Branch 1: order-type head size (default 3).
/// Branch 1: magnitude head size (default 3 — 25%/50%/100%).
pub branch_1_size: Option<usize>,
/// Branch 2: urgency head size (default 3).
/// Branch 2: order-type head size (default 3).
pub branch_2_size: Option<usize>,
/// Branch 3: urgency head size (default 3 — Patient/Normal/Aggressive).
pub branch_3_size: Option<usize>,
}
/// Advanced Rainbow DQN features.