From 4889b6e968fecc00ef3e28f436e712688da4a068 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Wed, 8 Apr 2026 12:18:50 +0200 Subject: [PATCH] =?UTF-8?q?feat(4branch):=20config=20foundation=20?= =?UTF-8?q?=E2=80=94=20b0=3D3=20direction,=20b1=3D3=20magnitude,=20b2=3D3?= =?UTF-8?q?=20order,=20b3=3D3=20urgency?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Delete exposure_aux_weight, exposure_aux_warmup_epochs. b0_size 9→3 (direction branch). b3_size=3 added (urgency). Co-Authored-By: Claude Opus 4.6 (1M context) --- config/training/dqn-hyperopt.toml | 9 ++++----- config/training/dqn-localdev.toml | 3 +-- config/training/dqn-production.toml | 3 +-- config/training/dqn-smoketest.toml | 3 +-- .../src/cuda_pipeline/gpu_backtest_evaluator.rs | 16 ++++++++++------ crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs | 11 +++++++---- .../cuda_pipeline/gpu_experience_collector.rs | 1 + crates/ml/src/cuda_pipeline/gpu_iqn_head.rs | 11 +++++++---- crates/ml/src/hyperopt/adapters/dqn.rs | 1 + crates/ml/src/trainers/dqn/config.rs | 12 ++++-------- crates/ml/src/trainers/dqn/fused_training.rs | 6 ++++-- .../trainers/dqn/smoke_tests/gradient_budget.rs | 2 +- .../ml/src/trainers/dqn/smoke_tests/reward_v8.rs | 4 ++-- crates/ml/src/trainers/dqn/trainer/metrics.rs | 1 + .../ml/src/trainers/dqn/trainer/training_loop.rs | 10 ++++------ crates/ml/src/training_profile.rs | 8 +++++--- 16 files changed, 54 insertions(+), 47 deletions(-) diff --git a/config/training/dqn-hyperopt.toml b/config/training/dqn-hyperopt.toml index 478f6ad86..8cb38f028 100644 --- a/config/training/dqn-hyperopt.toml +++ b/config/training/dqn-hyperopt.toml @@ -92,7 +92,8 @@ exit_timing_weight = [0.0, 0.2] ofi_reward_weight = [0.0, 0.5] kelly_sizing_weight = [0.0, 0.5] reward_noise_scale = [0.01, 0.1] -exposure_aux_weight = [0.001, 0.1] +# exposure_aux_weight removed (4-branch refactor) +b3_size = 3 # v8 search ranges micro_reward_scale = [0.0, 0.05] @@ -139,8 +140,7 @@ exit_timing_weight = 0.05 ofi_reward_weight = 0.2 kelly_sizing_weight = 0.1 reward_noise_scale = 0.05 -exposure_aux_weight = 0.1 -exposure_aux_warmup_epochs = 5 +b3_size = 3 [fixed] @@ -182,5 +182,4 @@ exit_timing_weight = 0.05 ofi_reward_weight = 0.2 kelly_sizing_weight = 0.1 reward_noise_scale = 0.05 -exposure_aux_weight = 0.1 -exposure_aux_warmup_epochs = 5 +b3_size = 3 diff --git a/config/training/dqn-localdev.toml b/config/training/dqn-localdev.toml index 7ebd67609..8b813d221 100644 --- a/config/training/dqn-localdev.toml +++ b/config/training/dqn-localdev.toml @@ -131,5 +131,4 @@ exit_timing_weight = 0.05 ofi_reward_weight = 0.2 kelly_sizing_weight = 0.1 reward_noise_scale = 0.05 -exposure_aux_weight = 0.1 -exposure_aux_warmup_epochs = 5 +b3_size = 3 diff --git a/config/training/dqn-production.toml b/config/training/dqn-production.toml index de179d569..7607b120c 100644 --- a/config/training/dqn-production.toml +++ b/config/training/dqn-production.toml @@ -142,5 +142,4 @@ exit_timing_weight = 0.05 ofi_reward_weight = 0.2 kelly_sizing_weight = 0.1 reward_noise_scale = 0.05 -exposure_aux_weight = 0.1 -exposure_aux_warmup_epochs = 5 +b3_size = 3 diff --git a/config/training/dqn-smoketest.toml b/config/training/dqn-smoketest.toml index a36ea24b9..c7112c822 100644 --- a/config/training/dqn-smoketest.toml +++ b/config/training/dqn-smoketest.toml @@ -138,8 +138,7 @@ exit_timing_weight = 0.05 ofi_reward_weight = 0.2 kelly_sizing_weight = 0.1 reward_noise_scale = 0.05 -exposure_aux_weight = 0.1 -exposure_aux_warmup_epochs = 5 +b3_size = 3 [walk_forward] step_fraction = 0.25 diff --git a/crates/ml/src/cuda_pipeline/gpu_backtest_evaluator.rs b/crates/ml/src/cuda_pipeline/gpu_backtest_evaluator.rs index b5a1966b2..78ba44f8d 100644 --- a/crates/ml/src/cuda_pipeline/gpu_backtest_evaluator.rs +++ b/crates/ml/src/cuda_pipeline/gpu_backtest_evaluator.rs @@ -103,12 +103,14 @@ pub struct DqnBacktestConfig { pub adv_h: usize, /// C51 distributional atom count (typically 51). pub num_atoms: usize, - /// Exposure branch action count (branch 0, typically 5). + /// Direction branch action count (branch 0, default 3 — Short/Flat/Long). pub branch_0_size: usize, - /// Order-type branch action count (branch 1, typically 3). + /// Magnitude branch action count (branch 1, default 3 — 25%/50%/100%). pub branch_1_size: usize, - /// Urgency branch action count (branch 2, typically 3). + /// Order-type branch action count (branch 2, default 3). pub branch_2_size: usize, + /// Urgency branch action count (branch 3, default 3 — Patient/Normal/Aggressive). + pub branch_3_size: usize, /// C51 minimum support value. pub v_min: f32, /// C51 maximum support value. @@ -118,7 +120,7 @@ pub struct DqnBacktestConfig { impl DqnBacktestConfig { /// Create from the legacy `network_dims` tuple with standard C51 defaults. /// - /// Uses branch sizes [9, 3, 3], num_atoms=51, v_range from reward_scale=10/gamma=0.95. + /// Uses branch sizes [3, 3, 3, 3], num_atoms=51, v_range from reward_scale=10/gamma=0.95. pub fn from_network_dims(network_dims: (usize, usize, usize, usize)) -> Self { // v_min/v_max computed from reward_scale=10, gamma=0.95: // Q* ~ 10 / (1-0.95) * 1.2 = 240, clamped to [20, 300] → ±240 @@ -129,9 +131,10 @@ impl DqnBacktestConfig { value_h: network_dims.2, adv_h: network_dims.3, num_atoms: 51, - branch_0_size: 9, + branch_0_size: 3, branch_1_size: 3, branch_2_size: 3, + branch_3_size: 3, v_min: -v_range, v_max: v_range, } @@ -2049,9 +2052,10 @@ mod tests { assert_eq!(cfg.value_h, 128); assert_eq!(cfg.adv_h, 128); assert_eq!(cfg.num_atoms, 51); - assert_eq!(cfg.branch_0_size, 9); + assert_eq!(cfg.branch_0_size, 3); assert_eq!(cfg.branch_1_size, 3); assert_eq!(cfg.branch_2_size, 3); + assert_eq!(cfg.branch_3_size, 3); // v_range = (10 / (1-0.95) * 1.2).clamp(20, 300) = 240 let expected_v_range = (10.0_f32 / (1.0 - 0.95) * 1.2).clamp(20.0, 300.0); assert!((cfg.v_min - (-expected_v_range)).abs() < f32::EPSILON); diff --git a/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs b/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs index e0c0ed457..f1fe1d0ee 100644 --- a/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs +++ b/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs @@ -132,12 +132,14 @@ pub struct GpuDqnTrainConfig { pub v_min: f32, /// C51 maximum support value (default: 50.0). pub v_max: f32, - /// Branch 0 (exposure) action count (default: 5). + /// Branch 0 (direction) action count (default: 3 — Short/Flat/Long). pub branch_0_size: usize, - /// Branch 1 (order) action count (default: 3). + /// Branch 1 (magnitude) action count (default: 3 — 25%/50%/100%). pub branch_1_size: usize, - /// Branch 2 (urgency) action count (default: 3). + /// Branch 2 (order) action count (default: 3). pub branch_2_size: usize, + /// Branch 3 (urgency) action count (default: 3 — Patient/Normal/Aggressive). + pub branch_3_size: usize, /// Training batch size (fixed for CUDA Graph capture). pub batch_size: usize, /// Discount factor for Bellman projection. @@ -235,9 +237,10 @@ impl Default for GpuDqnTrainConfig { // Q* ~ 10 / (1-0.95) * 1.2 = 240, clamped to [20, 300] → ±240 v_min: -(10.0_f32 / (1.0 - 0.95) * 1.2).clamp(20.0, 300.0), v_max: (10.0_f32 / (1.0 - 0.95) * 1.2).clamp(20.0, 300.0), - branch_0_size: 9, + branch_0_size: 3, branch_1_size: 3, branch_2_size: 3, + branch_3_size: 3, batch_size: 64, // Smoke test default (production overrides via from_hyperparams) gamma: 0.99, n_steps: 1, diff --git a/crates/ml/src/cuda_pipeline/gpu_experience_collector.rs b/crates/ml/src/cuda_pipeline/gpu_experience_collector.rs index 084c61060..44d1b5e05 100644 --- a/crates/ml/src/cuda_pipeline/gpu_experience_collector.rs +++ b/crates/ml/src/cuda_pipeline/gpu_experience_collector.rs @@ -753,6 +753,7 @@ impl GpuExperienceCollector { branch_0_size: branch_sizes[0], branch_1_size: branch_sizes[1], branch_2_size: branch_sizes[2], + branch_3_size: 3, // urgency — fixed at 3 (4-branch refactor) bottleneck_dim: 0, market_dim: 0, ..GpuDqnTrainConfig::default() diff --git a/crates/ml/src/cuda_pipeline/gpu_iqn_head.rs b/crates/ml/src/cuda_pipeline/gpu_iqn_head.rs index a7a22db93..b14700885 100644 --- a/crates/ml/src/cuda_pipeline/gpu_iqn_head.rs +++ b/crates/ml/src/cuda_pipeline/gpu_iqn_head.rs @@ -57,12 +57,14 @@ pub struct GpuIqnConfig { pub state_dim: usize, /// First shared hidden layer width from DQN trunk (default 256). pub shared_h1: usize, - /// Branch 0 (exposure) action count. + /// Branch 0 (direction) action count. pub branch_0_size: usize, - /// Branch 1 (order) action count. + /// Branch 1 (magnitude) action count. pub branch_1_size: usize, - /// Branch 2 (urgency) action count. + /// Branch 2 (order) action count. pub branch_2_size: usize, + /// Branch 3 (urgency) action count. + pub branch_3_size: usize, /// Training batch size (must match DQN trainer). pub batch_size: usize, /// Learning rate for IQN Adam optimizer. @@ -90,9 +92,10 @@ impl Default for GpuIqnConfig { kappa: 1.0, state_dim: 72, shared_h1: 256, - branch_0_size: 9, + branch_0_size: 3, branch_1_size: 3, branch_2_size: 3, + branch_3_size: 3, batch_size: 256, lr: 3e-4, beta1: 0.9, diff --git a/crates/ml/src/hyperopt/adapters/dqn.rs b/crates/ml/src/hyperopt/adapters/dqn.rs index e3d47c289..f206319bb 100644 --- a/crates/ml/src/hyperopt/adapters/dqn.rs +++ b/crates/ml/src/hyperopt/adapters/dqn.rs @@ -1478,6 +1478,7 @@ impl DQNTrainer { branch_0_size: b0, branch_1_size: if is_branching { b1 } else { b0 }, branch_2_size: if is_branching { b2 } else { b0 }, + branch_3_size: 3, // urgency — fixed at 3 (4-branch refactor) v_min: hp.v_min as f32, v_max: hp.v_max as f32, }; diff --git a/crates/ml/src/trainers/dqn/config.rs b/crates/ml/src/trainers/dqn/config.rs index eed0f87db..51fb11fff 100644 --- a/crates/ml/src/trainers/dqn/config.rs +++ b/crates/ml/src/trainers/dqn/config.rs @@ -795,10 +795,7 @@ pub struct DQNHyperparameters { pub kelly_sizing_weight: f64, /// Reward v7 gem: Reward label smoothing noise scale. pub reward_noise_scale: f64, - /// v7.1: Exposure auxiliary loss base weight (scheduled decay). - pub exposure_aux_weight: f64, - /// v7.1: Epochs over which exposure aux weight decays to 10%. - pub exposure_aux_warmup_epochs: usize, + // exposure_aux_weight and exposure_aux_warmup_epochs removed (4-branch refactor) /// Position staleness rent per step held pub time_decay_rate: f64, /// Q-value gap threshold for trade conviction filter. @@ -1538,7 +1535,7 @@ impl DQNHyperparameters { self.ofi_reward_weight = (self.ofi_reward_weight * li).clamp(0.0, 1.0); self.kelly_sizing_weight = (self.kelly_sizing_weight * li).clamp(0.0, 1.0); self.reward_noise_scale = (self.reward_noise_scale * li).clamp(0.0, 0.2); - self.exposure_aux_weight = (self.exposure_aux_weight * li).clamp(0.0, 2.0); + // exposure_aux_weight removed (4-branch refactor) self.micro_reward_scale = (self.micro_reward_scale * li).clamp(0.0, 0.01); self.position_entropy_weight *= li; // v6 legacy (zeroed defaults, no-op) @@ -1653,8 +1650,7 @@ impl DQNHyperparameters { ofi_reward_weight: 0.2, kelly_sizing_weight: 0.1, reward_noise_scale: 0.05, - exposure_aux_weight: 0.1, - exposure_aux_warmup_epochs: 5, + // exposure_aux_weight / exposure_aux_warmup_epochs removed (4-branch refactor) // v8: Dense micro-reward, TD(λ), PopArt, curriculum, hindsight micro_reward_scale: 0.01, td_lambda: 0.9, @@ -1927,7 +1923,7 @@ pub(crate) fn dqn_default_config() -> DQNConfig { DQNConfig { // Network Architecture state_dim: 48, // Default placeholder; overridden by constructor to (42+8+16+7)&!7=72 - num_actions: 9, // 9 exposure levels (25% steps) + num_actions: 3, // 3 direction levels (Short/Flat/Long) — 4-branch refactor hidden_dims: vec![512, 256, 128], // Sufficient capacity without overfitting (Trial 19) leaky_relu_alpha: 0.01, diff --git a/crates/ml/src/trainers/dqn/fused_training.rs b/crates/ml/src/trainers/dqn/fused_training.rs index fff326901..ba283ef5b 100644 --- a/crates/ml/src/trainers/dqn/fused_training.rs +++ b/crates/ml/src/trainers/dqn/fused_training.rs @@ -209,7 +209,7 @@ pub(crate) struct FusedTrainingCtx { graph_mega: Option, /// Host-side tau for graph replay. Graph captures HtoD from this address. tau_host: f32, - /// v7.1: Exposure auxiliary gradient weight (decayed over warmup epochs). + // exposure_aux_weight removed (4-branch refactor) — kept as zero sentinel pub(crate) exposure_aux_weight: f64, /// v7.1: Exposure targets buffer [batch_size] -- from experience collector. pub(crate) exposure_targets: cudarc::driver::CudaSlice, @@ -278,6 +278,7 @@ impl FusedTrainingCtx { branch_0_size: dqn.config.num_actions, branch_1_size: dqn.config.num_order_types, branch_2_size: dqn.config.num_urgency_levels, + branch_3_size: 3, // urgency — fixed at 3 (4-branch refactor) batch_size, gamma: hyperparams.gamma as f32, n_steps: hyperparams.n_steps, @@ -429,6 +430,7 @@ impl FusedTrainingCtx { branch_0_size: dqn.config.num_actions, branch_1_size: dqn.config.num_order_types, branch_2_size: dqn.config.num_urgency_levels, + branch_3_size: 3, // urgency — fixed at 3 (4-branch refactor) batch_size, lr: hyperparams.learning_rate as f32, max_grad_norm: resolved_grad_norm as f32, @@ -611,7 +613,7 @@ impl FusedTrainingCtx { graph_spectral: None, graph_mega: None, tau_host: 0.0, - exposure_aux_weight: hyperparams.exposure_aux_weight, + exposure_aux_weight: 0.0, // removed in 4-branch refactor exposure_targets, popart_enabled: hyperparams.popart_enabled, hindsight_fraction: hyperparams.hindsight_fraction, diff --git a/crates/ml/src/trainers/dqn/smoke_tests/gradient_budget.rs b/crates/ml/src/trainers/dqn/smoke_tests/gradient_budget.rs index 8f2fe9070..30414725a 100644 --- a/crates/ml/src/trainers/dqn/smoke_tests/gradient_budget.rs +++ b/crates/ml/src/trainers/dqn/smoke_tests/gradient_budget.rs @@ -22,7 +22,7 @@ fn test_config() -> GpuDqnTrainConfig { num_atoms: 11, v_min: -10.0, v_max: 10.0, - branch_0_size: 9, + branch_0_size: 3, branch_1_size: 3, branch_2_size: 3, batch_size: 8, diff --git a/crates/ml/src/trainers/dqn/smoke_tests/reward_v8.rs b/crates/ml/src/trainers/dqn/smoke_tests/reward_v8.rs index 4c276617b..42c5893f6 100644 --- a/crates/ml/src/trainers/dqn/smoke_tests/reward_v8.rs +++ b/crates/ml/src/trainers/dqn/smoke_tests/reward_v8.rs @@ -75,8 +75,8 @@ mod tests { #[test] fn test_exposure_aux_gradient_unique() { - let b0 = 9; - let target = 3; + let b0 = 3; // 4-branch: direction branch now 3 (Short/Flat/Long) + let target = 1; // Flat let prob = 1.0 / b0 as f64; let grads: Vec = (0..b0).map(|a| prob - if a == target { 1.0 } else { 0.0 }).collect(); assert!(grads[target] < 0.0, "Target gets negative gradient"); diff --git a/crates/ml/src/trainers/dqn/trainer/metrics.rs b/crates/ml/src/trainers/dqn/trainer/metrics.rs index ebed741c6..27f87cedf 100644 --- a/crates/ml/src/trainers/dqn/trainer/metrics.rs +++ b/crates/ml/src/trainers/dqn/trainer/metrics.rs @@ -508,6 +508,7 @@ impl DQNTrainer { branch_0_size: b0, branch_1_size: if is_branching { b1 } else { b0 }, branch_2_size: if is_branching { b2 } else { b0 }, + branch_3_size: 3, // urgency — fixed at 3 (4-branch refactor) v_min: hp.v_min as f32, v_max: hp.v_max as f32, }; diff --git a/crates/ml/src/trainers/dqn/trainer/training_loop.rs b/crates/ml/src/trainers/dqn/trainer/training_loop.rs index 4f2d46027..9327fbb48 100644 --- a/crates/ml/src/trainers/dqn/trainer/training_loop.rs +++ b/crates/ml/src/trainers/dqn/trainer/training_loop.rs @@ -191,7 +191,7 @@ impl DQNTrainer { } // v8: Ensure fused context exists for pre-training (normally lazy-init in Phase 3) - if self.hyperparams.exposure_aux_weight > 0.0 && self.fused_ctx.is_none() && self.device.is_cuda() { + if false && self.fused_ctx.is_none() && self.device.is_cuda() { // exposure_aux removed (4-branch refactor) if let Some(ref stream) = self.cuda_stream { let agent = self.agent.read().await; match super::super::fused_training::FusedTrainingCtx::new( @@ -204,7 +204,7 @@ impl DQNTrainer { } // v8: Supervised pre-training (epoch 0) — initialize exposure branch with directional knowledge - if self.hyperparams.exposure_aux_weight > 0.0 { + if false { // exposure_aux removed (4-branch refactor) let pretrain_total_bars = self.gpu_data.as_ref().map(|d| d.num_bars); let pretrain_batch_size = pretrain_total_bars.map(|t| self.current_batch_size.min(t)); let max_pos = self.max_position as f32; @@ -1177,10 +1177,8 @@ impl DQNTrainer { // v7.1: Exposure aux weight decay over warmup epochs if let Some(ref mut fused) = self.fused_ctx { - let base = self.hyperparams.exposure_aux_weight; - let warmup = self.hyperparams.exposure_aux_warmup_epochs.max(1) as f64; - let progress = (self.current_epoch as f64 / warmup).min(1.0); - fused.exposure_aux_weight = base * (1.0 - 0.9 * progress); + // exposure_aux_weight decay removed (4-branch refactor) + fused.exposure_aux_weight = 0.0; } let gpu_batch = collector.collect_experiences_gpu( diff --git a/crates/ml/src/training_profile.rs b/crates/ml/src/training_profile.rs index 223bec6e1..ce5564ce4 100644 --- a/crates/ml/src/training_profile.rs +++ b/crates/ml/src/training_profile.rs @@ -135,12 +135,14 @@ pub struct DistributionalSection { /// Branching DQN per-head action-space sizes. #[derive(Debug, Clone, Deserialize, Default)] pub struct BranchingSection { - /// Branch 0: exposure head size (default 5). + /// Branch 0: direction head size (default 3 — Short/Flat/Long). pub branch_0_size: Option, - /// Branch 1: order-type head size (default 3). + /// Branch 1: magnitude head size (default 3 — 25%/50%/100%). pub branch_1_size: Option, - /// Branch 2: urgency head size (default 3). + /// Branch 2: order-type head size (default 3). pub branch_2_size: Option, + /// Branch 3: urgency head size (default 3 — Patient/Normal/Aggressive). + pub branch_3_size: Option, } /// Advanced Rainbow DQN features.