From 614d4dbf8463fcdef2adff4856866a6034fca0e9 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Fri, 17 Apr 2026 09:28:49 +0200 Subject: [PATCH] =?UTF-8?q?cleanup:=20remove=20ofi=5Fenabled/ofi=5Fpre=20c?= =?UTF-8?q?onditionals=20=E2=80=94=20OFI=20always=20on?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit OFI (20 microstructure features) is unconditionally enabled. mbp10_data_dir always set. state_dim unconditionally 96. Removed dead else branches (ofi_dim=0, state_dim=72). Simplifies 7 files across the workspace. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../cuda_pipeline/gpu_experience_collector.rs | 4 +- crates/ml/src/cuda_pipeline/mod.rs | 6 +-- crates/ml/src/hyperopt/adapters/dqn.rs | 51 ++++++++----------- .../src/trainers/dqn/trainer/constructor.rs | 16 +++--- crates/ml/src/trainers/dqn/trainer/metrics.rs | 21 ++++---- crates/ml/src/trainers/dqn/trainer/mod.rs | 6 +-- crates/ml/src/trainers/dqn/trainer/state.rs | 12 ++--- 7 files changed, 49 insertions(+), 67 deletions(-) diff --git a/crates/ml/src/cuda_pipeline/gpu_experience_collector.rs b/crates/ml/src/cuda_pipeline/gpu_experience_collector.rs index b9e8a9e0a..67a29b1c6 100644 --- a/crates/ml/src/cuda_pipeline/gpu_experience_collector.rs +++ b/crates/ml/src/cuda_pipeline/gpu_experience_collector.rs @@ -755,8 +755,8 @@ impl GpuExperienceCollector { let alloc_timesteps = timesteps_per_episode.min(MAX_TIMESTEPS_LIMIT); let (shared_h1, shared_h2, value_h, adv_h) = network_dims; let (state_dim, market_dim, num_atoms_max) = kernel_dims; - let portfolio_dim: usize = 12; // 8 base + 4 plan progress features - let ofi_dim: usize = if state_dim >= market_dim + portfolio_dim + 20 { 20 } else { 0 }; + let _portfolio_dim: usize = 12; // 8 base + 4 plan progress features + let ofi_dim: usize = 20; // Branch sizes for 4-branch hierarchical DQN (always enabled). // [direction(3), magnitude(3), order(3), urgency(3)] diff --git a/crates/ml/src/cuda_pipeline/mod.rs b/crates/ml/src/cuda_pipeline/mod.rs index b8fe755f4..978447dcd 100644 --- a/crates/ml/src/cuda_pipeline/mod.rs +++ b/crates/ml/src/cuda_pipeline/mod.rs @@ -399,7 +399,7 @@ impl DqnGpuData { /// Estimated VRAM usage in bytes. pub fn vram_bytes(&self) -> usize { - let ofi_dim = if self.ofi_features.is_some() { 20 } else { 0 }; + let ofi_dim: usize = 20; estimate_vram_bytes(self.num_bars * (self.feature_dim + 4 + ofi_dim)) } @@ -474,7 +474,7 @@ impl DqnGpuData { portfolio_features: &[f32; 3], stream: &Arc, ) -> Result, MLError> { - let ofi_dim = if self.ofi_features.is_some() { 20 } else { 0 }; + let ofi_dim: usize = 20; let raw_dim = self.feature_dim + 3 + ofi_dim; let final_dim = self.aligned_state_dim.unwrap_or(raw_dim); @@ -513,7 +513,7 @@ impl DqnGpuData { if count == 0 { return Err(MLError::ModelError("Empty batch for state construction".to_owned())); } - let ofi_dim = if self.ofi_features.is_some() { 20 } else { 0 }; + let ofi_dim: usize = 20; let raw_dim = self.feature_dim + 3 + ofi_dim; let final_dim = self.aligned_state_dim.unwrap_or(raw_dim); diff --git a/crates/ml/src/hyperopt/adapters/dqn.rs b/crates/ml/src/hyperopt/adapters/dqn.rs index 588b9fea9..fdf6ca802 100644 --- a/crates/ml/src/hyperopt/adapters/dqn.rs +++ b/crates/ml/src/hyperopt/adapters/dqn.rs @@ -1346,31 +1346,26 @@ impl DQNTrainer { // The gather kernel places live portfolio at [feat_dim..feat_dim+8] and // multi-timeframe features at [feat_dim+8..feat_dim+24]. // GpuBacktestEvaluator adds PORTFOLIO_AND_MTF_DIM=24 on top of feature_dim - // to reach the full state_dim (72 without OFI, 88 with OFI, both 8-aligned). - // - // For OFI-enabled models, feature_dim = 62 (42 market + 20 OFI) - // so state_dim = (62+24+7)&!7 = 88. Without OFI: (42+24+7)&!7 = 72. - let ofi_enabled = self.mbp10_data_dir.is_some(); + // to reach the full state_dim: (62+24+7)&!7 = 88. + // feature_dim = 62 (42 market + 20 OFI). OFI always enabled. let market_dim: usize = 42; - let feature_dim: usize = if ofi_enabled { 62 } else { market_dim }; + let feature_dim: usize = 62; // 42 market + 20 OFI // OFI feature overlay: preloaded OFI covers ALL bars (train+val), left-aligned. // Validation bars start at index `train_end` in the global OFI array. let ofi_offset = self.preloaded_train_end; let ofi_ref = self.preloaded_ofi_features.as_deref(); - if ofi_enabled { - let ofi_total = ofi_ref.map_or(0, |o| o.len()); - let ofi_avail = ofi_total.saturating_sub(ofi_offset); - debug!( - ofi_offset, - ofi_total, - ofi_avail, - total_bars, - "walk-forward OFI overlay: {} of {} val bars have real OFI", - ofi_avail.min(total_bars), - total_bars, - ); - } + let ofi_total = ofi_ref.map_or(0, |o| o.len()); + let ofi_avail = ofi_total.saturating_sub(ofi_offset); + debug!( + ofi_offset, + ofi_total, + ofi_avail, + total_bars, + "walk-forward OFI overlay: {} of {} val bars have real OFI", + ofi_avail.min(total_bars), + total_bars, + ); let mut window_prices = Vec::with_capacity(window_count); let mut window_features = Vec::with_capacity(window_count); @@ -1405,15 +1400,13 @@ impl DQNTrainer { let mut fv_f32: Vec = fv_slice.iter().map(|&v| v as f32).collect(); // Overlay real OFI features from preloaded MBP-10 data. // Falls back to zero-padding only when OFI data is unavailable. - if ofi_enabled { - let ofi_idx = ofi_offset + i; - if let Some(ofi) = ofi_ref.filter(|o| ofi_idx < o.len()) { - for &v in ofi.get(ofi_idx).iter().flat_map(|f| f.iter()) { - fv_f32.push(v as f32); - } - } else { - fv_f32.resize(feature_dim, 0.0); + let ofi_idx = ofi_offset + i; + if let Some(ofi) = ofi_ref.filter(|o| ofi_idx < o.len()) { + for &v in ofi.get(ofi_idx).iter().flat_map(|f| f.iter()) { + fv_f32.push(v as f32); } + } else { + fv_f32.resize(feature_dim, 0.0); } features.push(fv_f32); } @@ -1441,7 +1434,7 @@ impl DQNTrainer { max_leverage: internal_trainer.hyperparams().max_leverage as f32, // OFI reorder in gather kernel: produces [market, portfolio, OFI, pad] // directly, eliminating the Candle narrow+cat closure. - ofi_dim: if ofi_enabled { 20 } else { 0 }, + ofi_dim: 20, min_hold_bars: internal_trainer.hyperparams().min_hold_bars as i32, bars_per_day: internal_trainer.hyperparams().bars_per_day as f32, trading_days_per_year: internal_trainer.hyperparams().trading_days_per_year as f32, @@ -2028,7 +2021,7 @@ impl HyperparameterOptimizable for DQNTrainer { let scaled_atoms = ((base_hp.num_atoms as f64 * params.architecture_intensity) as usize).max(11); let scaled_buffer = (base_hp.buffer_size as f64 * params.replay_intensity).max(1000.0) as usize; let vram_buffer = if budget.gpu_memory_mb <= 8192 { 0 } else { scaled_buffer.min(self.buffer_size_max) }; - let aligned_state_dim: usize = if self.mbp10_data_dir.is_some() { 80 } else { 72 }; + let aligned_state_dim: usize = 96; // (92+7)&!7 = 96, OFI always on let vram_check = budget.trial_fits_vram( scaled_hidden, base_hp.batch_size, diff --git a/crates/ml/src/trainers/dqn/trainer/constructor.rs b/crates/ml/src/trainers/dqn/trainer/constructor.rs index 0db7a4eec..c4e4abe87 100644 --- a/crates/ml/src/trainers/dqn/trainer/constructor.rs +++ b/crates/ml/src/trainers/dqn/trainer/constructor.rs @@ -48,9 +48,8 @@ impl DQNTrainer { // Pre-compute hidden dims to get accurate model size for batch sizing. // Align input_dim to 8 so the log matches the actual model dimensions. // (device not yet created, so use the formula directly -- CUDA always aligns) - let ofi_pre = !hyperparams.mbp10_data_dir.is_empty(); - // 42 market + 14 portfolio (8 base + 6 plan) + 16 multi-timeframe = 72 base, +20 OFI = 92 - let input_dim: usize = if ofi_pre { 96 } else { 72 }; // (92+7)&!7=96, (72+7)&!7=72 + // 42 market + 14 portfolio (8 base + 6 plan) + 16 multi-timeframe + 20 OFI = 92, aligned (92+7)&!7=96 + let input_dim: usize = 96; let output_dim: usize = 5; let hidden_dims: Vec = match hyperparams.hidden_dim_base { Some(base) => { @@ -242,18 +241,17 @@ impl DQNTrainer { // [0..42) 42 market features (OHLCV, technical, patterns, volume, time, statistical) // [42..50) 8 portfolio features (position, P&L, drawdown, etc.) // [50..66) 16 multi-timeframe features (4 windows x 4 features) - // [66..86) 20 OFI features (from MBP-10 order book data, when mbp10_data_dir set) - // Raw state_dim: 66 without OFI, 86 with OFI. + // [66..86) 20 OFI features (from MBP-10 order book data, always loaded) + // Raw state_dim: 92 (42+14+16+20). OFI always enabled. // market_dim: always 42 (bottleneck compresses only base market features). // OFI features bypass bottleneck via portfolio_dim (fed directly to shared trunk). // // GpuTensor core alignment: state_dim is rounded up to the next multiple of 8 - // (66->72, 86->88) so that cuBLAS dispatches BF16 HMMA instructions instead + // (92->96) so that cuBLAS dispatches TF32 HMMA instructions instead // of falling back to scalar FMA. The extra columns are zero-padded at the // data pipeline boundaries (GpuPreloadedData and train_batch CPU path). - let ofi_enabled = !hyperparams.mbp10_data_dir.is_empty(); - let raw_state_dim: usize = if ofi_enabled { 92 } else { 72 }; // 42 market + 14 portfolio + 16 MTF + (20 OFI) - let full_state_dim = (raw_state_dim + 7) & !7; // aligned: 88 with OFI, 72 without + let raw_state_dim: usize = 92; // 42 market + 14 portfolio + 16 MTF + 20 OFI + let full_state_dim = (raw_state_dim + 7) & !7; // aligned: 96 // state_dim is always the FULL padded dimension. The GpuDqnTrainer // handles bottleneck reduction internally via compute_param_sizes(). diff --git a/crates/ml/src/trainers/dqn/trainer/metrics.rs b/crates/ml/src/trainers/dqn/trainer/metrics.rs index 15e499b3b..f475e0102 100644 --- a/crates/ml/src/trainers/dqn/trainer/metrics.rs +++ b/crates/ml/src/trainers/dqn/trainer/metrics.rs @@ -442,9 +442,8 @@ impl DQNTrainer { // ── Lazy-init the GPU evaluator (once per fold, reused across epochs) ── if self.gpu_evaluator.is_none() { - let ofi_enabled = self.ofi_features.is_some(); let market_dim: usize = 42; - let feature_dim: usize = if ofi_enabled { 62 } else { market_dim }; // 42 market + 20 OFI = 62 (portfolio+MTF in state_dim, not feature_dim) + let feature_dim: usize = 62; // 42 market + 20 OFI (portfolio+MTF in state_dim, not feature_dim) // Build a single window from all val_data let mut prices: Vec<[f32; 4]> = Vec::with_capacity(self.val_data.len()); @@ -458,18 +457,16 @@ impl DQNTrainer { let close = if target.len() >= 2 { target[0] as f32 } else { fv[3] as f32 }; prices.push([close, close, close, close]); - // Features: 42 market features, plus 20 OFI if enabled + // Features: 42 market features + 20 OFI let fv_slice = &fv[..market_dim.min(fv.len())]; let mut fv_f32: Vec = fv_slice.iter().map(|&v| v as f32).collect(); - if ofi_enabled { - let ofi_idx = self.ofi_val_offset + i; - if let Some(ofi_row) = self.ofi_features.as_ref().and_then(|o| o.get(ofi_idx)) { - for &v in ofi_row.iter() { - fv_f32.push(v as f32); - } - } else { - fv_f32.resize(feature_dim, 0.0); + let ofi_idx = self.ofi_val_offset + i; + if let Some(ofi_row) = self.ofi_features.as_ref().and_then(|o| o.get(ofi_idx)) { + for &v in ofi_row.iter() { + fv_f32.push(v as f32); } + } else { + fv_f32.resize(feature_dim, 0.0); } features.push(fv_f32); } @@ -487,7 +484,7 @@ impl DQNTrainer { contract_multiplier: hp.contract_multiplier as f32, margin_pct: hp.margin_pct as f32, max_leverage: hp.max_leverage as f32, - ofi_dim: if ofi_enabled { 20 } else { 0 }, + ofi_dim: 20, min_hold_bars: hp.min_hold_bars as i32, bars_per_day: hp.bars_per_day as f32, trading_days_per_year: hp.trading_days_per_year as f32, diff --git a/crates/ml/src/trainers/dqn/trainer/mod.rs b/crates/ml/src/trainers/dqn/trainer/mod.rs index 91ae0fcf8..980c0e9e6 100644 --- a/crates/ml/src/trainers/dqn/trainer/mod.rs +++ b/crates/ml/src/trainers/dqn/trainer/mod.rs @@ -1222,16 +1222,14 @@ impl DQNTrainer { features, targets, ofi, stream, ).map_err(|e| anyhow::anyhow!("DqnGpuData::upload_slices failed: {e}"))?; - let ofi_enabled = gpu_data.ofi_features.is_some(); - let raw_dim = if ofi_enabled { 65 } else { 45 }; + let raw_dim: usize = 65; // 42 market + 3 portfolio + 20 OFI let aligned_dim = (raw_dim + 7) & !7; gpu_data.set_aligned_state_dim(aligned_dim); tracing::info!( - "init_from_fxcache: {} bars uploaded to GPU ({:.1} MB, OFI={})", + "init_from_fxcache: {} bars uploaded to GPU ({:.1} MB, OFI=true)", features.len(), (features.len() * (42 + 4 + 20) * 4) as f64 / 1_048_576.0, - ofi_enabled, ); self.gpu_data = Some(gpu_data); diff --git a/crates/ml/src/trainers/dqn/trainer/state.rs b/crates/ml/src/trainers/dqn/trainer/state.rs index 7fb05cda3..86c8542fc 100644 --- a/crates/ml/src/trainers/dqn/trainer/state.rs +++ b/crates/ml/src/trainers/dqn/trainer/state.rs @@ -78,18 +78,14 @@ impl DQNTrainer { vec![0.0, 0.0, 0.0] // Fallback if no price provided }; - // OFI regime features: 8 features from MBP-10 order book data. - // When OFI is enabled (mbp10_data_dir set), always return 8 features - // (zeros if data didn't load). GPU training uses state_dim=80 (74 raw, 8-aligned). - let ofi_enabled = !self.hyperparams.mbp10_data_dir.is_empty(); + // OFI regime features: 20 features from MBP-10 order book data (always enabled). + // Returns real OFI when available, zeros otherwise. state_dim=96. let regime_features: Vec = if let (Some(ofi), Some(idx)) = (&self.ofi_features, ofi_index) { ofi.get(idx) .map(|f| f.iter().map(|&v| v as f32).collect()) - .unwrap_or_else(|| vec![0.0; 8]) - } else if ofi_enabled { - vec![0.0; 8] + .unwrap_or_else(|| vec![0.0; 20]) } else { - vec![] + vec![0.0; 20] }; // Use from_normalized() to preserve sign information