From 6e8cb318d71030f84523a1136acae49c3b3490d5 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Tue, 3 Mar 2026 15:42:35 +0100 Subject: [PATCH 01/11] feat(common): add 11 Prometheus gauges for epoch financial metrics Co-Authored-By: Claude Opus 4.6 --- crates/common/src/metrics/training_metrics.rs | 108 +++++++++++++++++- 1 file changed, 106 insertions(+), 2 deletions(-) diff --git a/crates/common/src/metrics/training_metrics.rs b/crates/common/src/metrics/training_metrics.rs index eae70ea29..606dbab5d 100644 --- a/crates/common/src/metrics/training_metrics.rs +++ b/crates/common/src/metrics/training_metrics.rs @@ -1,6 +1,6 @@ //! Prometheus metrics for ML training binaries. //! -//! Registers the 41 metrics expected by the Training Cockpit Grafana dashboard +//! Registers the 52 metrics expected by the Training Cockpit Grafana dashboard //! and provides typed helper functions so callers never mis-spell metric names. //! //! # Usage @@ -35,7 +35,7 @@ pub fn verbose_enabled() -> bool { // Registration // --------------------------------------------------------------------------- -/// Register all 41 training metrics with the global Prometheus registry. +/// Register all 52 training metrics with the global Prometheus registry. /// /// Safe to call multiple times — the registry silently ignores duplicates. pub fn init() { @@ -183,6 +183,63 @@ pub fn init() { mf, ); + // Epoch-level financial metrics (model + fold) + _ = register_gauge_vec( + "foxhunt_training_epoch_sharpe", + "Epoch Sharpe ratio from validation backtest", + mf, + ); + _ = register_gauge_vec( + "foxhunt_training_epoch_sortino", + "Epoch Sortino ratio from validation backtest", + mf, + ); + _ = register_gauge_vec( + "foxhunt_training_epoch_win_rate", + "Epoch win rate 0-1", + mf, + ); + _ = register_gauge_vec( + "foxhunt_training_epoch_max_drawdown", + "Epoch max drawdown 0-1", + mf, + ); + _ = register_gauge_vec( + "foxhunt_training_epoch_profit_factor", + "Epoch profit factor (gross profit / gross loss)", + mf, + ); + _ = register_gauge_vec( + "foxhunt_training_epoch_total_return", + "Epoch total return (fractional)", + mf, + ); + _ = register_gauge_vec( + "foxhunt_training_epoch_avg_return", + "Epoch average return per trade", + mf, + ); + _ = register_gauge_vec( + "foxhunt_training_epoch_total_trades", + "Epoch total trade count", + mf, + ); + _ = register_gauge_vec( + "foxhunt_training_epoch_action_buy_pct", + "BUY action percentage 0-1", + mf, + ); + _ = register_gauge_vec( + "foxhunt_training_epoch_action_sell_pct", + "SELL action percentage 0-1", + mf, + ); + _ = register_gauge_vec( + "foxhunt_training_epoch_action_hold_pct", + "HOLD action percentage 0-1", + mf, + ); + // Hyperopt gauges (model only) _ = register_gauge_vec( "foxhunt_hyperopt_trial_current", @@ -448,6 +505,46 @@ pub fn set_action_diversity(model: &str, fold: &str, diversity: f64) { set_gauge_vec("foxhunt_training_action_diversity", &[model, fold], diversity); } +// --------------------------------------------------------------------------- +// Tier 1: Epoch financial metrics +// --------------------------------------------------------------------------- + +/// Push a full set of epoch-level financial metrics from a backtest evaluation. +pub fn set_epoch_financial_metrics( + model: &str, + fold: &str, + sharpe: f64, + sortino: f64, + win_rate: f64, + max_drawdown: f64, + profit_factor: f64, + total_return: f64, + avg_return: f64, + total_trades: f64, +) { + set_gauge_vec("foxhunt_training_epoch_sharpe", &[model, fold], sharpe); + set_gauge_vec("foxhunt_training_epoch_sortino", &[model, fold], sortino); + set_gauge_vec("foxhunt_training_epoch_win_rate", &[model, fold], win_rate); + set_gauge_vec("foxhunt_training_epoch_max_drawdown", &[model, fold], max_drawdown); + set_gauge_vec("foxhunt_training_epoch_profit_factor", &[model, fold], profit_factor); + set_gauge_vec("foxhunt_training_epoch_total_return", &[model, fold], total_return); + set_gauge_vec("foxhunt_training_epoch_avg_return", &[model, fold], avg_return); + set_gauge_vec("foxhunt_training_epoch_total_trades", &[model, fold], total_trades); +} + +/// Push epoch action distribution percentages (all 0-1). +pub fn set_epoch_action_distribution( + model: &str, + fold: &str, + buy_pct: f64, + sell_pct: f64, + hold_pct: f64, +) { + set_gauge_vec("foxhunt_training_epoch_action_buy_pct", &[model, fold], buy_pct); + set_gauge_vec("foxhunt_training_epoch_action_sell_pct", &[model, fold], sell_pct); + set_gauge_vec("foxhunt_training_epoch_action_hold_pct", &[model, fold], hold_pct); +} + // --------------------------------------------------------------------------- // Tier 1: Hyperopt intra-trial // --------------------------------------------------------------------------- @@ -573,4 +670,11 @@ mod tests { set_hyperopt_trial_best_loss("dqn", 0.042); set_hyperopt_elapsed("dqn", 123.4); } + + #[test] + fn test_epoch_financial_metrics_no_panic() { + init(); + set_epoch_financial_metrics("dqn", "0", 2.31, 3.12, 0.55, 0.08, 1.84, 0.124, 0.003, 142.0); + set_epoch_action_distribution("dqn", "0", 0.35, 0.25, 0.40); + } } From 8a77a2d700490ad818698abe2076120c662b2709 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Tue, 3 Mar 2026 15:47:22 +0100 Subject: [PATCH 02/11] style(common): add clippy allow for too_many_arguments on epoch metrics Co-Authored-By: Claude Opus 4.6 --- crates/common/src/metrics/training_metrics.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/common/src/metrics/training_metrics.rs b/crates/common/src/metrics/training_metrics.rs index 606dbab5d..9618f0f60 100644 --- a/crates/common/src/metrics/training_metrics.rs +++ b/crates/common/src/metrics/training_metrics.rs @@ -510,6 +510,7 @@ pub fn set_action_diversity(model: &str, fold: &str, diversity: f64) { // --------------------------------------------------------------------------- /// Push a full set of epoch-level financial metrics from a backtest evaluation. +#[allow(clippy::too_many_arguments)] pub fn set_epoch_financial_metrics( model: &str, fold: &str, From eca76e86d3791f2087aa0d153bd2155ef289a0cd Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Tue, 3 Mar 2026 15:50:02 +0100 Subject: [PATCH 03/11] feat(ml): add compute_epoch_financials helper for DQN/PPO Co-Authored-By: Claude Opus 4.6 --- crates/ml/src/trainers/dqn/financials.rs | 189 +++++++++++++++++++++++ crates/ml/src/trainers/dqn/mod.rs | 1 + 2 files changed, 190 insertions(+) create mode 100644 crates/ml/src/trainers/dqn/financials.rs diff --git a/crates/ml/src/trainers/dqn/financials.rs b/crates/ml/src/trainers/dqn/financials.rs new file mode 100644 index 000000000..4e293d338 --- /dev/null +++ b/crates/ml/src/trainers/dqn/financials.rs @@ -0,0 +1,189 @@ +//! Epoch-level financial metrics computed from the DQN trainer's PnL history +//! and action counts. These are pushed to Prometheus at the end of each epoch. + +use std::collections::VecDeque; + +/// Financial metrics summary for a single training epoch. +#[derive(Debug, Clone, Default)] +pub(crate) struct EpochFinancials { + pub sharpe: f64, + pub sortino: f64, + pub win_rate: f64, + pub max_drawdown: f64, + pub profit_factor: f64, + pub total_return: f64, + pub avg_return: f64, + pub total_trades: usize, + pub buy_pct: f64, + pub sell_pct: f64, + pub hold_pct: f64, +} + +/// Compute financial metrics from the trainer's PnL history and action counts. +/// +/// - `pnl_history`: per-step PnL values accumulated this epoch +/// - `action_counts`: 45-element array (FactoredAction), grouped into BUY/SELL/HOLD +/// - `initial_capital`: starting equity for return calculation (default 100_000) +pub(crate) fn compute_epoch_financials( + pnl_history: &VecDeque, + action_counts: &[usize; 45], + initial_capital: f64, +) -> EpochFinancials { + if pnl_history.is_empty() { + return EpochFinancials::default(); + } + + let returns: Vec = pnl_history.iter().copied().collect(); + let n = returns.len(); + + // Total return + let total_pnl: f64 = returns.iter().sum(); + let total_return = total_pnl / initial_capital; + + // Win rate + let winning = returns.iter().filter(|&&r| r > 0.0).count(); + let win_rate = winning as f64 / n as f64; + + // Average return per trade + let avg_return = total_pnl / n as f64; + + // Sharpe ratio (annualized, 252 trading days) + let mean = total_pnl / n as f64; + let variance: f64 = returns.iter().map(|r| (r - mean).powi(2)).sum::() / n as f64; + let std = variance.sqrt(); + let sharpe = if std > 1e-10 { + (mean / std) * (252.0_f64).sqrt() + } else { + 0.0 + }; + + // Sortino ratio (only downside deviation) + let downside_returns: Vec = returns.iter().filter(|&&r| r < 0.0).copied().collect(); + let sortino = if downside_returns.len() > 1 { + let down_var: f64 = downside_returns.iter().map(|r| r.powi(2)).sum::() + / downside_returns.len() as f64; + let down_std = down_var.sqrt(); + if down_std > 1e-10 { + (mean / down_std) * (252.0_f64).sqrt() + } else { + 0.0 + } + } else { + 0.0 + }; + + // Max drawdown + let mut equity = initial_capital; + let mut peak = equity; + let mut max_dd = 0.0_f64; + for &pnl in &returns { + equity += pnl; + if equity > peak { + peak = equity; + } + let dd = (peak - equity) / peak; + if dd > max_dd { + max_dd = dd; + } + } + + // Profit factor + let gross_profit: f64 = returns.iter().filter(|&&r| r > 0.0).sum(); + let gross_loss: f64 = returns.iter().filter(|&&r| r < 0.0).map(|r| r.abs()).sum(); + let profit_factor = if gross_loss > 1e-10 { + gross_profit / gross_loss + } else if gross_profit > 0.0 { + f64::INFINITY + } else { + 0.0 + }; + + // Action distribution: group 45 actions into BUY/SELL/HOLD + // FactoredAction: exposure(5) x order(3) x urgency(3) + // Exposure 0,1 = reduce/close (SELL-like), 2 = hold, 3,4 = add/aggressive (BUY-like) + let total_actions: usize = action_counts.iter().sum(); + let (buy_pct, sell_pct, hold_pct) = if total_actions > 0 { + let mut buy = 0usize; + let mut sell = 0usize; + let mut hold = 0usize; + for (i, &count) in action_counts.iter().enumerate() { + let exposure = i / 9; // 0-4 + match exposure { + 0 | 1 => sell += count, // reduce/close + 2 => hold += count, // neutral + 3 | 4 => buy += count, // add/aggressive + _ => {} + } + } + let t = total_actions as f64; + (buy as f64 / t, sell as f64 / t, hold as f64 / t) + } else { + (0.0, 0.0, 0.0) + }; + + EpochFinancials { + sharpe, + sortino, + win_rate, + max_drawdown: max_dd, + profit_factor, + total_return, + avg_return, + total_trades: n, + buy_pct, + sell_pct, + hold_pct, + } +} + +#[cfg(test)] +#[allow(clippy::unwrap_used)] +mod tests { + use super::*; + + #[test] + fn test_empty_history() { + let f = compute_epoch_financials(&VecDeque::new(), &[0; 45], 100_000.0); + assert_eq!(f.total_trades, 0); + assert_eq!(f.sharpe, 0.0); + } + + #[test] + fn test_all_winning() { + let pnl: VecDeque = vec![10.0, 20.0, 30.0, 15.0, 25.0].into(); + let f = compute_epoch_financials(&pnl, &[0; 45], 100_000.0); + assert_eq!(f.win_rate, 1.0); + assert_eq!(f.total_trades, 5); + assert!(f.sharpe > 0.0); + assert!(f.max_drawdown < 1e-10); // No drawdown with all wins + assert!(f.profit_factor.is_infinite()); // No losses + } + + #[test] + fn test_mixed_pnl() { + let pnl: VecDeque = vec![100.0, -50.0, 75.0, -25.0, 50.0].into(); + let f = compute_epoch_financials(&pnl, &[0; 45], 100_000.0); + assert_eq!(f.total_trades, 5); + assert!((f.win_rate - 0.6).abs() < 1e-10); + assert!(f.total_return > 0.0); + assert!(f.profit_factor > 1.0); + assert!(f.max_drawdown > 0.0); + assert!(f.sortino > 0.0); + } + + #[test] + fn test_action_distribution() { + let mut actions = [0usize; 45]; + // Exposure 3 (add), order 0, urgency 0 → index 27 + actions[27] = 100; // BUY + // Exposure 0 (reduce), order 0, urgency 0 → index 0 + actions[0] = 50; // SELL + // Exposure 2 (neutral), order 0, urgency 0 → index 18 + actions[18] = 50; // HOLD + let pnl: VecDeque = vec![1.0].into(); + let f = compute_epoch_financials(&pnl, &actions, 100_000.0); + assert!((f.buy_pct - 0.5).abs() < 1e-10); + assert!((f.sell_pct - 0.25).abs() < 1e-10); + assert!((f.hold_pct - 0.25).abs() < 1e-10); + } +} diff --git a/crates/ml/src/trainers/dqn/mod.rs b/crates/ml/src/trainers/dqn/mod.rs index 0b7828343..d42722b81 100644 --- a/crates/ml/src/trainers/dqn/mod.rs +++ b/crates/ml/src/trainers/dqn/mod.rs @@ -21,6 +21,7 @@ mod config; mod data_loading; mod early_stopping; +pub(crate) mod financials; mod features; pub mod lr_scheduler; mod monitoring; From 5c7228375bebb006cd61ae8c89a194945ce9e09e Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Tue, 3 Mar 2026 15:52:26 +0100 Subject: [PATCH 04/11] feat(ml): push epoch financial metrics from DQN trainer Co-Authored-By: Claude Opus 4.6 --- crates/ml/src/trainers/dqn/trainer.rs | 34 +++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/crates/ml/src/trainers/dqn/trainer.rs b/crates/ml/src/trainers/dqn/trainer.rs index 2f2676cbf..5086263cb 100644 --- a/crates/ml/src/trainers/dqn/trainer.rs +++ b/crates/ml/src/trainers/dqn/trainer.rs @@ -41,6 +41,7 @@ use crate::memory_optimization::auto_batch_size::{AutoBatchSizer, BatchSizeConfi // Import from sibling modules use super::config::{DQNAgentType, DQNHyperparameters}; +use super::financials::compute_epoch_financials; use super::monitoring::TrainingMonitor; use super::statistics::{FeatureStatistics, QValueStats}; use super::{EPISODE_LENGTH, FeatureVector51}; @@ -2788,6 +2789,39 @@ impl DQNTrainer { ); } + // Epoch financial metrics for monitoring service + { + let financials = compute_epoch_financials( + &self.pnl_history, + &monitor.action_counts, + 100_000.0, + ); + training_metrics::set_epoch_financial_metrics( + "dqn", "current", + financials.sharpe, + financials.sortino, + financials.win_rate, + financials.max_drawdown, + financials.profit_factor, + financials.total_return, + financials.avg_return, + financials.total_trades as f64, + ); + training_metrics::set_epoch_action_distribution( + "dqn", "current", + financials.buy_pct, + financials.sell_pct, + financials.hold_pct, + ); + info!( + "Epoch {}/{}: Sharpe={:.2} WinRate={:.1}% MaxDD={:.1}% PF={:.2} Return={:+.2}% Trades={}", + epoch + 1, self.hyperparams.epochs, + financials.sharpe, financials.win_rate * 100.0, + financials.max_drawdown * 100.0, financials.profit_factor, + financials.total_return * 100.0, financials.total_trades, + ); + } + // Track metrics for early stopping self.loss_history.push(avg_loss); self.q_value_history.push(avg_q_value); From 26c7c3b5b8fc45542d4233101ec54759bdaed050 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Tue, 3 Mar 2026 15:54:19 +0100 Subject: [PATCH 05/11] feat(ml): push epoch financial metrics from PPO trainer Co-Authored-By: Claude Opus 4.6 --- crates/ml/src/trainers/ppo.rs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/crates/ml/src/trainers/ppo.rs b/crates/ml/src/trainers/ppo.rs index b260af645..a618e2c05 100644 --- a/crates/ml/src/trainers/ppo.rs +++ b/crates/ml/src/trainers/ppo.rs @@ -678,6 +678,28 @@ impl PpoTrainer { training_metrics::set_advantage_std("ppo", "current", std_reward as f64); training_metrics::set_value_explained_variance("ppo", "current", explained_variance as f64); + // Epoch financial metrics (simplified for PPO — derived from reward stats) + // PPO doesn't run a backtest per epoch; use reward mean/std as proxy + { + let epoch_sharpe = if std_reward > 1e-10 { + (mean_reward / std_reward) as f64 * (252.0_f64).sqrt() + } else { + 0.0 + }; + training_metrics::set_epoch_financial_metrics( + "ppo", + "current", + epoch_sharpe, + 0.0, // sortino: not available without per-step returns + 0.0, // win_rate: not tracked per epoch in PPO + 0.0, // max_drawdown: not tracked per epoch in PPO + 0.0, // profit_factor: not tracked + mean_reward as f64, // total_return proxy + mean_reward as f64, // avg_return proxy + 0.0, // total_trades: not applicable for PPO + ); + } + // Track metrics for early stopping (bounded ring buffer) { let mut loss_history = self.value_loss_history.lock().await; From a471c913d70e35f793d1610bdc5478c8fa37ce6a Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Tue, 3 Mar 2026 15:57:27 +0100 Subject: [PATCH 06/11] feat(proto): add epoch financial metrics + GetEpochHistory to monitoring.proto Add 11 financial fields (sharpe, sortino, win_rate, max_drawdown, profit_factor, total_return, avg_return, total_trades, action distribution) to TrainingSession (fields 36-46), a new GetEpochHistory RPC with request/response messages, and wire the Prometheus metric mapping in the monitoring service with a stub RPC handler for task 7. Co-Authored-By: Claude Opus 4.6 --- bin/fxt/proto/monitoring.proto | 48 +++++++++++++++++++ .../monitoring_service/proto/monitoring.proto | 48 +++++++++++++++++++ services/monitoring_service/src/service.rs | 38 ++++++++++++++- 3 files changed, 132 insertions(+), 2 deletions(-) diff --git a/bin/fxt/proto/monitoring.proto b/bin/fxt/proto/monitoring.proto index d7233faf6..157a89453 100644 --- a/bin/fxt/proto/monitoring.proto +++ b/bin/fxt/proto/monitoring.proto @@ -9,6 +9,10 @@ service MonitoringService { // Server-streaming: pushes updates every N seconds rpc StreamTrainingMetrics(StreamTrainingMetricsRequest) returns (stream GetLiveTrainingMetricsResponse); + + // Epoch history for a specific session (ring buffer, max 50 epochs) + rpc GetEpochHistory(GetEpochHistoryRequest) + returns (GetEpochHistoryResponse); } message GetLiveTrainingMetricsRequest { @@ -81,6 +85,20 @@ message TrainingSession { uint32 hyperopt_trial_epoch = 33; float hyperopt_trial_best_loss = 34; float hyperopt_elapsed_seconds = 35; + + // Epoch-level financial metrics + float epoch_sharpe = 36; + float epoch_sortino = 37; + float epoch_win_rate = 38; + float epoch_max_drawdown = 39; + float epoch_profit_factor = 40; + float epoch_total_return = 41; + float epoch_avg_return = 42; + uint32 epoch_total_trades = 43; + // Action distribution + float action_buy_pct = 44; + float action_sell_pct = 45; + float action_hold_pct = 46; } message GpuSnapshot { @@ -90,3 +108,33 @@ message GpuSnapshot { float temperature_celsius = 4; float power_watts = 5; } + +message GetEpochHistoryRequest { + string model = 1; + string fold = 2; + uint32 max_epochs = 3; // 0 = all (up to 50) +} + +message EpochFinancialSnapshot { + uint32 epoch = 1; + float sharpe = 2; + float sortino = 3; + float win_rate = 4; + float max_drawdown = 5; + float profit_factor = 6; + float total_return = 7; + float avg_return = 8; + uint32 total_trades = 9; + float loss = 10; + float val_loss = 11; + float learning_rate = 12; + float action_buy_pct = 13; + float action_sell_pct = 14; + float action_hold_pct = 15; +} + +message GetEpochHistoryResponse { + string model = 1; + string fold = 2; + repeated EpochFinancialSnapshot epochs = 3; +} diff --git a/services/monitoring_service/proto/monitoring.proto b/services/monitoring_service/proto/monitoring.proto index d7233faf6..157a89453 100644 --- a/services/monitoring_service/proto/monitoring.proto +++ b/services/monitoring_service/proto/monitoring.proto @@ -9,6 +9,10 @@ service MonitoringService { // Server-streaming: pushes updates every N seconds rpc StreamTrainingMetrics(StreamTrainingMetricsRequest) returns (stream GetLiveTrainingMetricsResponse); + + // Epoch history for a specific session (ring buffer, max 50 epochs) + rpc GetEpochHistory(GetEpochHistoryRequest) + returns (GetEpochHistoryResponse); } message GetLiveTrainingMetricsRequest { @@ -81,6 +85,20 @@ message TrainingSession { uint32 hyperopt_trial_epoch = 33; float hyperopt_trial_best_loss = 34; float hyperopt_elapsed_seconds = 35; + + // Epoch-level financial metrics + float epoch_sharpe = 36; + float epoch_sortino = 37; + float epoch_win_rate = 38; + float epoch_max_drawdown = 39; + float epoch_profit_factor = 40; + float epoch_total_return = 41; + float epoch_avg_return = 42; + uint32 epoch_total_trades = 43; + // Action distribution + float action_buy_pct = 44; + float action_sell_pct = 45; + float action_hold_pct = 46; } message GpuSnapshot { @@ -90,3 +108,33 @@ message GpuSnapshot { float temperature_celsius = 4; float power_watts = 5; } + +message GetEpochHistoryRequest { + string model = 1; + string fold = 2; + uint32 max_epochs = 3; // 0 = all (up to 50) +} + +message EpochFinancialSnapshot { + uint32 epoch = 1; + float sharpe = 2; + float sortino = 3; + float win_rate = 4; + float max_drawdown = 5; + float profit_factor = 6; + float total_return = 7; + float avg_return = 8; + uint32 total_trades = 9; + float loss = 10; + float val_loss = 11; + float learning_rate = 12; + float action_buy_pct = 13; + float action_sell_pct = 14; + float action_hold_pct = 15; +} + +message GetEpochHistoryResponse { + string model = 1; + string fold = 2; + repeated EpochFinancialSnapshot epochs = 3; +} diff --git a/services/monitoring_service/src/service.rs b/services/monitoring_service/src/service.rs index 85404dd49..656d22e43 100644 --- a/services/monitoring_service/src/service.rs +++ b/services/monitoring_service/src/service.rs @@ -8,8 +8,9 @@ use tonic::{Request, Response, Status}; use tracing::error; use crate::monitoring::{ - monitoring_service_server::MonitoringService, GetLiveTrainingMetricsRequest, - GetLiveTrainingMetricsResponse, GpuSnapshot, StreamTrainingMetricsRequest, TrainingSession, + monitoring_service_server::MonitoringService, GetEpochHistoryRequest, + GetEpochHistoryResponse, GetLiveTrainingMetricsRequest, GetLiveTrainingMetricsResponse, + GpuSnapshot, StreamTrainingMetricsRequest, TrainingSession, }; use crate::prometheus_client::{MetricSample, PrometheusClient}; @@ -92,6 +93,16 @@ impl MonitoringService for MonitoringServiceImpl { Ok(Response::new(Box::pin(stream))) } + + async fn get_epoch_history( + &self, + _request: Request, + ) -> Result, Status> { + // TODO(task-7): wire to epoch ring buffer storage + Err(Status::unimplemented( + "GetEpochHistory not yet wired — see task 7", + )) + } } /// Group flat metric samples into TrainingSession structs keyed by (model, fold) @@ -178,6 +189,29 @@ fn group_into_sessions(samples: &[MetricSample], model_filter: &str) -> Vec { session.hyperopt_elapsed_seconds = s.value as f32; } + // Epoch-level financial metrics + "foxhunt_training_epoch_sharpe" => session.epoch_sharpe = s.value as f32, + "foxhunt_training_epoch_sortino" => session.epoch_sortino = s.value as f32, + "foxhunt_training_epoch_win_rate" => session.epoch_win_rate = s.value as f32, + "foxhunt_training_epoch_max_drawdown" => { + session.epoch_max_drawdown = s.value as f32; + } + "foxhunt_training_epoch_profit_factor" => { + session.epoch_profit_factor = s.value as f32; + } + "foxhunt_training_epoch_total_return" => { + session.epoch_total_return = s.value as f32; + } + "foxhunt_training_epoch_avg_return" => { + session.epoch_avg_return = s.value as f32; + } + "foxhunt_training_epoch_total_trades" => { + session.epoch_total_trades = s.value as u32; + } + // Action distribution + "foxhunt_training_action_buy_pct" => session.action_buy_pct = s.value as f32, + "foxhunt_training_action_sell_pct" => session.action_sell_pct = s.value as f32, + "foxhunt_training_action_hold_pct" => session.action_hold_pct = s.value as f32, _ => {} } } From 5842859d78aea15bd8cd95bf9174ec4a53970ecd Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Tue, 3 Mar 2026 16:01:04 +0100 Subject: [PATCH 07/11] feat(monitoring): wire epoch financial metrics mapper + epoch history store - Fix action distribution metric names (add epoch_ prefix) - Implement epoch history ring buffer (50 epochs per session) - Wire GetEpochHistory RPC with real data - Add test for financial metric mapping Co-Authored-By: Claude Opus 4.6 --- services/monitoring_service/src/service.rs | 130 ++++++++++++++++++--- 1 file changed, 116 insertions(+), 14 deletions(-) diff --git a/services/monitoring_service/src/service.rs b/services/monitoring_service/src/service.rs index 656d22e43..510f342c6 100644 --- a/services/monitoring_service/src/service.rs +++ b/services/monitoring_service/src/service.rs @@ -1,22 +1,27 @@ -use std::collections::HashMap; +use std::collections::{HashMap, VecDeque}; use std::pin::Pin; use std::sync::Arc; use std::time::Duration; +use tokio::sync::RwLock; use tokio_stream::Stream; use tonic::{Request, Response, Status}; use tracing::error; use crate::monitoring::{ - monitoring_service_server::MonitoringService, GetEpochHistoryRequest, - GetEpochHistoryResponse, GetLiveTrainingMetricsRequest, GetLiveTrainingMetricsResponse, - GpuSnapshot, StreamTrainingMetricsRequest, TrainingSession, + monitoring_service_server::MonitoringService, EpochFinancialSnapshot, + GetEpochHistoryRequest, GetEpochHistoryResponse, GetLiveTrainingMetricsRequest, + GetLiveTrainingMetricsResponse, GpuSnapshot, StreamTrainingMetricsRequest, TrainingSession, }; use crate::prometheus_client::{MetricSample, PrometheusClient}; +const MAX_EPOCH_HISTORY: usize = 50; + pub struct MonitoringServiceImpl { prom: Arc, default_interval: u32, + epoch_histories: Arc>>>, + last_epochs: Arc>>, } impl MonitoringServiceImpl { @@ -24,12 +29,16 @@ impl MonitoringServiceImpl { Self { prom: Arc::new(prom), default_interval, + epoch_histories: Arc::new(RwLock::new(HashMap::new())), + last_epochs: Arc::new(RwLock::new(HashMap::new())), } } async fn build_response( prom: &PrometheusClient, model_filter: &str, + epoch_histories: &RwLock>>, + last_epochs: &RwLock>, ) -> Result { let (training, gpu, jobs) = tokio::try_join!( prom.fetch_training_metrics(), @@ -41,6 +50,43 @@ impl MonitoringServiceImpl { let sessions = group_into_sessions(&training, model_filter); let gpu_snapshot = build_gpu_snapshot(&gpu); + // Record epoch history snapshots for sessions with new epoch data + { + let mut last = last_epochs.write().await; + let mut histories = epoch_histories.write().await; + for session in &sessions { + let key = format!("{}/{}", session.model, session.fold); + let prev_epoch = last.get(&key).copied().unwrap_or_default(); + if session.current_epoch > prev_epoch && session.epoch_sharpe != 0.0 { + last.insert(key.clone(), session.current_epoch); + let snapshot = EpochFinancialSnapshot { + epoch: session.current_epoch as u32, + sharpe: session.epoch_sharpe, + sortino: session.epoch_sortino, + win_rate: session.epoch_win_rate, + max_drawdown: session.epoch_max_drawdown, + profit_factor: session.epoch_profit_factor, + total_return: session.epoch_total_return, + avg_return: session.epoch_avg_return, + total_trades: session.epoch_total_trades, + loss: session.epoch_loss, + val_loss: session.validation_loss, + learning_rate: session.learning_rate, + action_buy_pct: session.action_buy_pct, + action_sell_pct: session.action_sell_pct, + action_hold_pct: session.action_hold_pct, + }; + let history = histories + .entry(key) + .or_insert_with(|| VecDeque::with_capacity(MAX_EPOCH_HISTORY)); + if history.len() >= MAX_EPOCH_HISTORY { + history.pop_front(); + } + history.push_back(snapshot); + } + } + } + Ok(GetLiveTrainingMetricsResponse { sessions, gpu: Some(gpu_snapshot), @@ -57,7 +103,9 @@ impl MonitoringService for MonitoringServiceImpl { request: Request, ) -> Result, Status> { let filter = &request.into_inner().model_filter; - let resp = Self::build_response(&self.prom, filter).await?; + let resp = + Self::build_response(&self.prom, filter, &self.epoch_histories, &self.last_epochs) + .await?; Ok(Response::new(resp)) } @@ -76,12 +124,14 @@ impl MonitoringService for MonitoringServiceImpl { }; let filter = req.model_filter; let prom = self.prom.clone(); + let epoch_histories = self.epoch_histories.clone(); + let last_epochs = self.last_epochs.clone(); let stream = async_stream::stream! { let mut interval = tokio::time::interval(Duration::from_secs(u64::from(interval_secs))); loop { interval.tick().await; - match Self::build_response(&prom, &filter).await { + match Self::build_response(&prom, &filter, &epoch_histories, &last_epochs).await { Ok(resp) => yield Ok(resp), Err(e) => { error!("Stream tick failed: {}", e); @@ -96,12 +146,27 @@ impl MonitoringService for MonitoringServiceImpl { async fn get_epoch_history( &self, - _request: Request, + request: Request, ) -> Result, Status> { - // TODO(task-7): wire to epoch ring buffer storage - Err(Status::unimplemented( - "GetEpochHistory not yet wired — see task 7", - )) + let req = request.into_inner(); + let key = format!("{}/{}", req.model, req.fold); + let histories = self.epoch_histories.read().await; + let epochs = match histories.get(&key) { + Some(deque) => { + let max = if req.max_epochs == 0 { + MAX_EPOCH_HISTORY + } else { + req.max_epochs as usize + }; + deque.iter().rev().take(max).rev().cloned().collect() + } + None => vec![], + }; + Ok(Response::new(GetEpochHistoryResponse { + model: req.model, + fold: req.fold, + epochs, + })) } } @@ -209,9 +274,9 @@ fn group_into_sessions(samples: &[MetricSample], model_filter: &str) -> Vec session.action_buy_pct = s.value as f32, - "foxhunt_training_action_sell_pct" => session.action_sell_pct = s.value as f32, - "foxhunt_training_action_hold_pct" => session.action_hold_pct = s.value as f32, + "foxhunt_training_epoch_action_buy_pct" => session.action_buy_pct = s.value as f32, + "foxhunt_training_epoch_action_sell_pct" => session.action_sell_pct = s.value as f32, + "foxhunt_training_epoch_action_hold_pct" => session.action_hold_pct = s.value as f32, _ => {} } } @@ -409,6 +474,43 @@ mod tests { assert!((s.hyperopt_elapsed_seconds - 123.4).abs() < 0.1); } + #[test] + fn test_group_financial_metrics() { + let samples = vec![ + MetricSample { + name: "foxhunt_training_epoch_sharpe".to_owned(), + model: "dqn".to_owned(), + fold: "0".to_owned(), + value: 2.31, + }, + MetricSample { + name: "foxhunt_training_epoch_win_rate".to_owned(), + model: "dqn".to_owned(), + fold: "0".to_owned(), + value: 0.552, + }, + MetricSample { + name: "foxhunt_training_epoch_max_drawdown".to_owned(), + model: "dqn".to_owned(), + fold: "0".to_owned(), + value: 0.081, + }, + MetricSample { + name: "foxhunt_training_epoch_action_buy_pct".to_owned(), + model: "dqn".to_owned(), + fold: "0".to_owned(), + value: 0.35, + }, + ]; + let sessions = group_into_sessions(&samples, ""); + assert_eq!(sessions.len(), 1); + let s = &sessions[0]; + assert!((s.epoch_sharpe - 2.31).abs() < 0.01); + assert!((s.epoch_win_rate - 0.552).abs() < 0.001); + assert!((s.epoch_max_drawdown - 0.081).abs() < 0.001); + assert!((s.action_buy_pct - 0.35).abs() < 0.01); + } + #[test] fn test_build_gpu_snapshot() { let samples = vec![ From 778adb7ce05ab8471c0e00deda6a666f7b22c582 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Tue, 3 Mar 2026 16:06:00 +0100 Subject: [PATCH 08/11] feat(fxt): render epoch financial metrics in watch TUI list + detail views Add Sharpe/Win% columns to training list table, financial summary lines (Sharpe, Sortino, Win Rate, Max DD, PF, Return, Avg, Trades) to the detail overview, action distribution (BUY/SELL/HOLD %) to current metrics, and four new sparklines (Sharpe, Win Rate, Max DD, Total Return) to the Metrics sub-tab. Co-Authored-By: Claude Opus 4.6 --- bin/fxt/src/commands/watch/render.rs | 45 +++++++++++++++++++++++----- 1 file changed, 38 insertions(+), 7 deletions(-) diff --git a/bin/fxt/src/commands/watch/render.rs b/bin/fxt/src/commands/watch/render.rs index a96b7347c..6653716b6 100644 --- a/bin/fxt/src/commands/watch/render.rs +++ b/bin/fxt/src/commands/watch/render.rs @@ -149,6 +149,8 @@ fn render_training_list(frame: &mut Frame, area: Rect, tab: &TrainingTabState) { Cell::from("LR"), Cell::from("Batch/s"), Cell::from("Grad"), + Cell::from("Sharpe"), + Cell::from("Win%"), ]) .style( Style::default() @@ -171,6 +173,8 @@ fn render_training_list(frame: &mut Frame, area: Rect, tab: &TrainingTabState) { Cell::from(format!("{:.2e}", s.learning_rate)), Cell::from(format!("{:.1}", s.batches_per_second)), Cell::from(format!("{:.3}", s.gradient_norm)), + Cell::from(if s.epoch_sharpe != 0.0 { format!("{:.2}", s.epoch_sharpe) } else { "-".to_owned() }), + Cell::from(if s.epoch_win_rate > 0.0 { format!("{:.0}%", s.epoch_win_rate * 100.0) } else { "-".to_owned() }), ]) }) .collect(); @@ -188,6 +192,8 @@ fn render_training_list(frame: &mut Frame, area: Rect, tab: &TrainingTabState) { Constraint::Length(10), Constraint::Length(8), Constraint::Length(8), + Constraint::Length(8), + Constraint::Length(8), ], ) .header(header) @@ -351,8 +357,8 @@ fn render_detail_overview( let chunks = Layout::default() .direction(Direction::Vertical) .constraints([ - Constraint::Length(8), // key stats - Constraint::Min(0), // loss sparkline + Constraint::Length(12), // key stats (incl. financial metrics) + Constraint::Min(0), // loss sparkline ]) .split(area); @@ -377,6 +383,17 @@ fn render_detail_overview( " NaN: {} Grad Explosions: {} Feature Errors: {}", session.nan_detected, session.gradient_explosions, session.feature_errors, )), + Line::from(""), + Line::from(format!( + " Sharpe: {:.2} Sortino: {:.2} Win Rate: {:.1}% Max DD: {:.1}%", + session.epoch_sharpe, session.epoch_sortino, + session.epoch_win_rate * 100.0, session.epoch_max_drawdown * 100.0, + )), + Line::from(format!( + " PF: {:.2} Return: {:+.2}% Avg: {:+.4} Trades: {}", + session.epoch_profit_factor, session.epoch_total_return * 100.0, + session.epoch_avg_return, session.epoch_total_trades, + )), ]) .block( Block::default() @@ -454,7 +471,7 @@ fn render_detail_metrics( let chunks = Layout::default() .direction(Direction::Vertical) .constraints([ - Constraint::Length(5), // current values + Constraint::Length(7), // current values (incl. action distribution) Constraint::Min(0), // sparklines ]) .split(area); @@ -468,6 +485,12 @@ fn render_detail_metrics( " Precision: {:.4} Recall: {:.4}", session.eval_precision, session.eval_recall, )), + Line::from(format!( + " Action: BUY {:.0}% SELL {:.0}% HOLD {:.0}%", + session.action_buy_pct * 100.0, + session.action_sell_pct * 100.0, + session.action_hold_pct * 100.0, + )), ]) .block( Block::default() @@ -479,10 +502,14 @@ fn render_detail_metrics( let spark_area = Layout::default() .direction(Direction::Vertical) .constraints([ - Constraint::Percentage(25), - Constraint::Percentage(25), - Constraint::Percentage(25), - Constraint::Percentage(25), + Constraint::Ratio(1, 8), + Constraint::Ratio(1, 8), + Constraint::Ratio(1, 8), + Constraint::Ratio(1, 8), + Constraint::Ratio(1, 8), + Constraint::Ratio(1, 8), + Constraint::Ratio(1, 8), + Constraint::Ratio(1, 8), ]) .split(chunks[1]); @@ -490,6 +517,10 @@ fn render_detail_metrics( render_sparkline_row(frame, spark_area[1], "Precision", &history.precision, 1.0, Color::Cyan); render_sparkline_row(frame, spark_area[2], "Recall", &history.recall, 1.0, Color::Magenta); render_sparkline_row(frame, spark_area[3], "F1", &history.f1, 1.0, Color::Yellow); + render_sparkline_row(frame, spark_area[4], "Sharpe", &history.sharpe, 20.0, Color::Cyan); + render_sparkline_row(frame, spark_area[5], "Win Rate", &history.win_rate, 1.0, Color::Green); + render_sparkline_row(frame, spark_area[6], "Max DD", &history.max_drawdown, 1.0, Color::Red); + render_sparkline_row(frame, spark_area[7], "Total Return", &history.total_return, 1.0, Color::Magenta); } fn render_detail_hyperopt( From 9501d581adbd12e9f3ea7e2edd89d466850272b4 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Tue, 3 Mar 2026 16:08:19 +0100 Subject: [PATCH 09/11] feat(fxt): add epoch financial metrics to train monitor output Add print_financial_metrics() to the monitor command, displaying per-model Sharpe ratio (color-coded green/yellow/red), win rate, max drawdown, profit factor, total return, trade count, and action distribution (BUY/SELL/HOLD percentages). Only shown when sessions report non-zero financial data. Co-Authored-By: Claude Opus 4.6 --- bin/fxt/src/commands/train/monitor.rs | 40 +++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/bin/fxt/src/commands/train/monitor.rs b/bin/fxt/src/commands/train/monitor.rs index b8a13e00f..56c32d899 100644 --- a/bin/fxt/src/commands/train/monitor.rs +++ b/bin/fxt/src/commands/train/monitor.rs @@ -101,6 +101,7 @@ fn render_snapshot(resp: &GetLiveTrainingMetricsResponse) { print_rl_diagnostics(&resp.sessions); print_hyperopt_summary(&resp.sessions); print_health_summary(&resp.sessions); + print_financial_metrics(&resp.sessions); } fn render_tui(resp: &GetLiveTrainingMetricsResponse) { @@ -232,3 +233,42 @@ fn print_health_summary(sessions: &[TrainingSession]) { total_nan, total_grad, total_ckpt ); } + +fn print_financial_metrics(sessions: &[TrainingSession]) { + let financial: Vec<_> = sessions + .iter() + .filter(|s| s.epoch_sharpe != 0.0 || s.epoch_win_rate > 0.0) + .collect(); + if financial.is_empty() { + return; + } + println!(); + println!("{}", "Epoch Financial Metrics:".bright_cyan()); + for s in &financial { + let sharpe_colored = if s.epoch_sharpe >= 2.0 { + format!("{:.2}", s.epoch_sharpe).green() + } else if s.epoch_sharpe >= 1.0 { + format!("{:.2}", s.epoch_sharpe).yellow() + } else { + format!("{:.2}", s.epoch_sharpe).red() + }; + println!( + " {}: Sharpe={} WinRate={:.1}% MaxDD={:.1}% PF={:.2} Return={:+.2}% Trades={}", + s.model.bright_white(), + sharpe_colored, + s.epoch_win_rate * 100.0, + s.epoch_max_drawdown * 100.0, + s.epoch_profit_factor, + s.epoch_total_return * 100.0, + s.epoch_total_trades, + ); + if s.action_buy_pct > 0.0 || s.action_sell_pct > 0.0 { + println!( + " Actions: BUY {:.0}% | SELL {:.0}% | HOLD {:.0}%", + s.action_buy_pct * 100.0, + s.action_sell_pct * 100.0, + s.action_hold_pct * 100.0, + ); + } + } +} From 392655d5fbe62a2bc640f970f0a44fc382e575ad Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Tue, 3 Mar 2026 16:20:11 +0100 Subject: [PATCH 10/11] fix: add GetEpochHistory to api_gateway proxy + clippy fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Implement GetEpochHistory forwarding in MonitoringServiceProxy - Fix clippy integer suffix style (0usize → 0_usize) Co-Authored-By: Claude Opus 4.6 --- crates/ml/src/trainers/dqn/financials.rs | 6 +++--- .../api_gateway/src/grpc/monitoring_proxy.rs | 20 ++++++++++++++++++- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/crates/ml/src/trainers/dqn/financials.rs b/crates/ml/src/trainers/dqn/financials.rs index 4e293d338..ea17f9b12 100644 --- a/crates/ml/src/trainers/dqn/financials.rs +++ b/crates/ml/src/trainers/dqn/financials.rs @@ -103,9 +103,9 @@ pub(crate) fn compute_epoch_financials( // Exposure 0,1 = reduce/close (SELL-like), 2 = hold, 3,4 = add/aggressive (BUY-like) let total_actions: usize = action_counts.iter().sum(); let (buy_pct, sell_pct, hold_pct) = if total_actions > 0 { - let mut buy = 0usize; - let mut sell = 0usize; - let mut hold = 0usize; + let mut buy = 0_usize; + let mut sell = 0_usize; + let mut hold = 0_usize; for (i, &count) in action_counts.iter().enumerate() { let exposure = i / 9; // 0-4 match exposure { diff --git a/services/api_gateway/src/grpc/monitoring_proxy.rs b/services/api_gateway/src/grpc/monitoring_proxy.rs index b4182df28..5af4b321f 100644 --- a/services/api_gateway/src/grpc/monitoring_proxy.rs +++ b/services/api_gateway/src/grpc/monitoring_proxy.rs @@ -11,7 +11,8 @@ use tracing::{error, info, instrument}; use crate::monitoring::monitoring_service_client::MonitoringServiceClient; use crate::monitoring::monitoring_service_server::{MonitoringService, MonitoringServiceServer}; use crate::monitoring::{ - GetLiveTrainingMetricsRequest, GetLiveTrainingMetricsResponse, StreamTrainingMetricsRequest, + GetEpochHistoryRequest, GetEpochHistoryResponse, GetLiveTrainingMetricsRequest, + GetLiveTrainingMetricsResponse, StreamTrainingMetricsRequest, }; /// Monitoring Service Proxy @@ -79,6 +80,23 @@ impl MonitoringService for MonitoringServiceProxy { info!("StreamTrainingMetrics forwarded successfully"); Ok(Response::new(boxed_stream)) } + + #[instrument(skip(self, request), fields(request_id = %uuid::Uuid::new_v4()), err)] + async fn get_epoch_history( + &self, + request: Request, + ) -> Result, Status> { + info!("Proxying GetEpochHistory request"); + + let mut client = self.client.clone(); + let response = client.get_epoch_history(request).await.map_err(|e| { + error!("Backend GetEpochHistory failed: {}", e); + e + })?; + + info!("GetEpochHistory forwarded successfully"); + Ok(response) + } } #[cfg(test)] From 7cbcfce781392f1149ccb90bf0c2ef21f82c19df Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Tue, 3 Mar 2026 16:21:07 +0100 Subject: [PATCH 11/11] fix: restore accidentally deleted bf16 plan files Co-Authored-By: Claude Opus 4.6 --- docs/plans/2026-03-03-bf16-training-design.md | 146 ++++ docs/plans/2026-03-03-bf16-training.md | 644 ++++++++++++++++++ 2 files changed, 790 insertions(+) create mode 100644 docs/plans/2026-03-03-bf16-training-design.md create mode 100644 docs/plans/2026-03-03-bf16-training.md diff --git a/docs/plans/2026-03-03-bf16-training-design.md b/docs/plans/2026-03-03-bf16-training-design.md new file mode 100644 index 000000000..b7db49be4 --- /dev/null +++ b/docs/plans/2026-03-03-bf16-training-design.md @@ -0,0 +1,146 @@ +# BF16 Training — Design Document + +**Date:** 2026-03-03 +**Scope:** All 10 models (DQN, PPO, TFT, Mamba2, TGGN, TLOB, Liquid, KAN, xLSTM, Diffusion) +**Target hardware:** L40S, H100 (Ampere+ GPUs with native BF16) +**Approach:** Pure BF16 — weights, gradients, optimizer state, activations all BF16 + +## Motivation + +Current training runs entirely in FP32. On Ampere+ GPUs (L40S, H100), BF16 unlocks: +- Tensor core utilization for all matmuls (up to 10x theoretical FLOPS vs FP32) +- 50% VRAM reduction on weights, activations, and replay buffer states +- More parallel hyperopt trials (more VRAM headroom per GPU) +- Smaller checkpoint files (50%) + +## Core Principle: Cast Once at Boundaries + +Zero casts in the training hot path. BF16 flows end-to-end inside the training loop. + +Cast points (exhaustive list): +1. **Data ingestion** — f32 from Rust/OHLCV → BF16 at replay buffer `store()` or dataset creation +2. **Loss scalar** — BF16 → F32 before `backward()` (single number) +3. **CUDA kernel boundary** — BF16 weights → f32 at extraction for experience collector (once per epoch) + +## Design + +### 1. Dynamic DType Detection + +A single function determines dtype from the device at runtime: + +```rust +// mixed_precision.rs +pub fn training_dtype(device: &Device) -> DType { + match device { + Device::Cuda(_) if is_ampere_or_newer(device) => DType::BF16, + _ => DType::F32, + } +} +``` + +Reuses existing `detect_from_gpu_name()` logic for GPU identification. No config fields, no env vars. CPU always returns F32. + +### 2. Model Construction (~30 VarBuilder sites) + +Every model changes from hardcoded F32 to dynamic: + +```rust +// Before: +VarBuilder::from_varmap(&vars, DType::F32, &device) +// After: +VarBuilder::from_varmap(&vars, training_dtype(&device), &device) +``` + +Affected models and key sites: +- **DQN**: `dqn.rs` Sequential, DuelingQNetwork; `dueling.rs`; `distributional_dueling.rs`; `quantile_regression.rs` +- **PPO**: `ppo.rs` PolicyNetwork, ValueNetwork +- **TFT**: `mod.rs`, `quantized_grn.rs`, `quantile_outputs.rs` +- **Mamba2**: `mod.rs` +- **Liquid/CfC**: `adapter.rs`, `candle_cfc.rs` +- **xLSTM**: `trainable.rs`, `slstm.rs`, `mlstm.rs` +- **Diffusion**: `trainable.rs` +- **TLOB**: `tlob.rs` +- **TGGN**: model construction site +- **KAN**: model construction site + +### 3. Training Data Path + +**RL models (DQN, PPO):** +- Replay buffer `store()`: cast f32 states/next_states to BF16 at insertion +- Replay buffer tensors pre-allocated as BF16 (states, next_states columns) +- `sample()` returns BF16 tensors directly — zero cast in training loop +- Actions stay U32, rewards/priorities/dones stay F32 + +**Supervised models (TFT, Mamba2, etc.):** +- Feature extraction produces f32 from OHLCV data +- Single cast to BF16 when building epoch dataset tensor +- All training iterations read BF16 directly + +### 4. Loss Computation + +Loss scalar cast to F32 before `backward()`: + +```rust +let loss = model.compute_loss(batch_bf16)?; +let loss_f32 = loss.to_dtype(DType::F32)?; +loss_f32.backward()?; +``` + +Distributional/categorical DQN loss already has F32 enforcement — keep as-is. Gradients flow back in BF16 automatically via Candle's autograd. + +### 5. CUDA Pipeline + +| Component | Current | After | +|-----------|---------|-------| +| GPU replay buffer: states/next_states | F32 | **BF16** | +| GPU replay buffer: actions | U32 | U32 | +| GPU replay buffer: rewards/priorities/dones | F32 | F32 | +| GPU weights extraction | f32 | **BF16→f32 cast at extraction** | +| GPU experience collector | CudaSlice | CudaSlice (unchanged) | +| GPU portfolio simulator | CudaSlice | CudaSlice (unchanged) | + +Experience collector and portfolio simulator CUDA kernels stay f32 — rewriting PTX is out of scope. + +### 6. Checkpoint Compatibility + +- Save: weights saved as BF16 in safetensors (50% smaller files) +- Load: `training_dtype(&device)` at load time — BF16 checkpoint on CPU auto-casts to F32 + +```rust +// Before: +VarBuilder::from_mmaped_safetensors(&[path], DType::F32, &device) +// After: +VarBuilder::from_mmaped_safetensors(&[path], training_dtype(&device), &device) +``` + +### 7. Mamba2 Fix + +Remove BF16/F16 rejection in `mamba/mod.rs::scalar_tensor()`: + +```rust +// Before: return Err(...) for BF16/F16 +// After: +Tensor::new(value, device)?.to_dtype(dtype) +``` + +### 8. Testing + +- Existing 2506 ml tests run on CPU (F32) — no behavior change +- One integration test: DQN 1-epoch training on synthetic data with BF16, verify loss decreases +- Real validation: hyperopt run on L40S comparing F32 vs BF16 Sharpe distributions + +## Expected Impact + +| Metric | F32 (current) | BF16 (expected) | +|--------|--------------|-----------------| +| VRAM per DQN trial | ~8-12 GB | ~5-7 GB | +| Parallel hyperopt trials (L40S 48GB) | 3 | 5-6 | +| Parallel hyperopt trials (H100 80GB) | 4-5 | 8-10 | +| Per-trial training time | baseline | ~1.5-2x faster (tensor cores) | +| Checkpoint size | baseline | ~50% smaller | + +## Risks + +- **Training divergence**: BF16 optimizer state has 8-bit mantissa vs F32's 24-bit. Some models may fail to converge. Mitigation: if a model diverges, fall back to F32 for that model. +- **Numerical edge cases**: Small gradients may underflow to zero in BF16. BF16's dynamic range (same exponent bits as F32) makes this unlikely, unlike FP16. +- **CUDA kernel mismatch**: Experience collector outputs f32 into a BF16 replay buffer. The cast at insertion handles this. diff --git a/docs/plans/2026-03-03-bf16-training.md b/docs/plans/2026-03-03-bf16-training.md new file mode 100644 index 000000000..38d30defd --- /dev/null +++ b/docs/plans/2026-03-03-bf16-training.md @@ -0,0 +1,644 @@ +# BF16 Training Implementation Plan + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Switch all 10 ML models from FP32 to BF16 training on Ampere+ GPUs with dynamic detection and zero casts in the training hot path. + +**Architecture:** Add a single `training_dtype(device) -> DType` function that returns BF16 on Ampere+ CUDA, F32 elsewhere. Thread it through all ~150 VarBuilder sites, training tensor creation, GPU replay buffer, and checkpoint loading. Loss stays F32 (single scalar cast). CUDA experience collector kernels untouched (stay f32). + +**Tech Stack:** Candle (v0.9.1 git pin), half crate (2.6.0), cudarc, safetensors + +--- + +## Phase 1: Core Infrastructure + +### Task 1: Add `training_dtype()` function + +**Files:** +- Modify: `crates/ml/src/dqn/mixed_precision.rs` + +**Step 1: Add the public function after `detect_from_gpu_name()` (~line 180)** + +```rust +/// Returns the optimal training DType for the given device. +/// Ampere+ CUDA GPUs → BF16 (tensor core acceleration). +/// Everything else (CPU, older GPUs) → F32. +pub fn training_dtype(device: &candle_core::Device) -> candle_core::DType { + match device { + candle_core::Device::Cuda(_) => { + // Check if GPU supports BF16 natively + if let Some(config) = detect_from_gpu_name_auto() { + match config.dtype { + DTypeSelection::BF16 => candle_core::DType::BF16, + DTypeSelection::F16 => candle_core::DType::F32, // F16 needs loss scaling, stay F32 + } + } else { + candle_core::DType::F32 + } + } + _ => candle_core::DType::F32, + } +} +``` + +Note: `detect_from_gpu_name_auto()` already exists at line 182 — it reads the GPU name from the CUDA device and calls `detect_from_gpu_name()`. Reuse it. + +**Step 2: Re-export from the module's public API** + +Ensure `training_dtype` is accessible as `crate::dqn::mixed_precision::training_dtype`. Check the module's `pub use` or `mod` visibility. + +**Step 3: Build check** + +```bash +SQLX_OFFLINE=true cargo check -p ml 2>&1 | head -20 +``` + +**Step 4: Commit** + +```bash +git add crates/ml/src/dqn/mixed_precision.rs +git commit -m "feat(ml): add training_dtype() for dynamic BF16 detection" +``` + +--- + +### Task 2: Fix Mamba2 scalar_tensor BF16 rejection + +**Files:** +- Modify: `crates/ml/src/mamba/mod.rs:605` + +**Step 1: Change the match arm at line 605** + +Replace the BF16/F16 rejection: + +```rust +// Before (line 605): +DType::F8E4M3 | DType::U8 | DType::U32 | DType::I64 | DType::BF16 | DType::F16 => { + Err(MLError::ModelError(format!( + "Unsupported dtype: {:?}", + dtype + ))) +}, + +// After: +DType::BF16 | DType::F16 => { + // Create in F32 then cast — half types can't be created directly from f64 + Tensor::new(&[value as f32], device)? + .to_dtype(dtype)? + .reshape(())? + .ok_or_else(|| MLError::ModelError("scalar reshape failed".into())) +}, +DType::F8E4M3 | DType::U8 | DType::U32 | DType::I64 => { + Err(MLError::ModelError(format!( + "Unsupported dtype: {:?}", + dtype + ))) +}, +``` + +Note: Check the exact return type — `scalar_tensor` may return `Result`. Adjust the reshape/return accordingly. The key is: create as f32, cast to target dtype, return scalar. + +**Step 2: Build check** + +```bash +SQLX_OFFLINE=true cargo check -p ml 2>&1 | head -20 +``` + +**Step 3: Commit** + +```bash +git add crates/ml/src/mamba/mod.rs +git commit -m "fix(ml): allow BF16/F16 in Mamba2 scalar_tensor helper" +``` + +--- + +## Phase 2: DQN Module (largest surface area) + +### Task 3: VarBuilder sites — DQN core networks + +Change `DType::F32` → `training_dtype(&device)` (or `training_dtype(device)` if device is a reference) in all VarBuilder::from_varmap calls across the DQN module. + +**Files (all need the same mechanical change):** +- `crates/ml/src/dqn/dqn.rs:669` +- `crates/ml/src/dqn/network.rs:256,261,301,363` +- `crates/ml/src/dqn/agent.rs:366,371,597` +- `crates/ml/src/dqn/dueling.rs:139` +- `crates/ml/src/dqn/distributional_dueling.rs:155` +- `crates/ml/src/dqn/quantile_regression.rs:90` +- `crates/ml/src/dqn/curiosity.rs:38` +- `crates/ml/src/dqn/factored_q_network.rs:72` +- `crates/ml/src/dqn/rainbow_agent.rs:41,45` +- `crates/ml/src/dqn/rainbow_agent_impl.rs:69,72` +- `crates/ml/src/dqn/rainbow_network.rs:429,450` + +**Pattern for each site:** + +```rust +// Before: +VarBuilder::from_varmap(&vars, DType::F32, &device) +// After: +VarBuilder::from_varmap(&vars, training_dtype(&device), &device) +``` + +Add `use crate::dqn::mixed_precision::training_dtype;` at the top of each file that doesn't already import it. + +**Step 1:** Apply the change to all files listed above. Use `replace_all` where `DType::F32` appears only in VarBuilder contexts. Where `DType::F32` also appears in non-VarBuilder contexts (tensor creation, loss), change only the VarBuilder lines. + +**Step 2: Build check** + +```bash +SQLX_OFFLINE=true cargo check -p ml 2>&1 | head -30 +``` + +**Step 3: Commit** + +```bash +git add crates/ml/src/dqn/ +git commit -m "feat(ml): BF16 VarBuilder for DQN core networks" +``` + +--- + +### Task 4: VarBuilder sites — DQN layer modules + +Same pattern for the layer-level modules that have many VarBuilder sites: + +**Files:** +- `crates/ml/src/dqn/noisy_layers.rs:314,324,346,391,431,447,487` +- `crates/ml/src/dqn/residual.rs:178,196,222,249,276,301,330,353` +- `crates/ml/src/dqn/attention.rs:465,479,509,549,583,614` +- `crates/ml/src/dqn/spectral_norm.rs:240,254,275,307,330,355,376` +- `crates/ml/src/dqn/rmsnorm.rs:239,254,269,309,353,358,406,410,458,484` + +Same mechanical change. These files likely have `DType::F32` ONLY in VarBuilder contexts, so `replace_all` may be safe. Verify by reading each file first. + +**Step 1:** Apply changes. +**Step 2:** Build check. +**Step 3: Commit** + +```bash +git add crates/ml/src/dqn/ +git commit -m "feat(ml): BF16 VarBuilder for DQN layers (noisy, residual, attention, spectral, rmsnorm)" +``` + +--- + +### Task 5: GPU replay buffer — BF16 states + +**Files:** +- Modify: `crates/ml/src/cuda_pipeline/gpu_replay_buffer.rs:74-75` + +**Step 1: Change states/next_states allocation to use dynamic dtype** + +```rust +// Line 74-75, change: +let states = Tensor::zeros(&[cap, sdim], DType::F32, device)?; +let next_states = Tensor::zeros(&[cap, sdim], DType::F32, device)?; +// To: +let dtype = training_dtype(device); +let states = Tensor::zeros(&[cap, sdim], dtype, device)?; +let next_states = Tensor::zeros(&[cap, sdim], dtype, device)?; +``` + +Keep rewards, dones, priorities as `DType::F32`. Keep actions as `DType::U32`. + +**Step 2: Verify `insert_batch()` callers cast correctly** + +The `insert_batch()` at line 167 uses `slice_scatter` which requires matching dtypes. The caller (DQN trainer) builds state tensors from f32 experience data. Add a `.to_dtype(self.states.dtype())?` cast on the incoming states/next_states args inside `insert_batch()`: + +```rust +// Inside insert_batch(), before slice_scatter: +let states = states.to_dtype(self.states.dtype())?; +let next_states = next_states.to_dtype(self.next_states.dtype())?; +``` + +This is the ONE cast at the data ingestion boundary. After this, all `sample()` returns match the buffer dtype (BF16 on Ampere+). + +**Step 3:** Build check. +**Step 4: Commit** + +```bash +git add crates/ml/src/cuda_pipeline/gpu_replay_buffer.rs +git commit -m "feat(ml): BF16 states in GPU replay buffer (50% VRAM savings)" +``` + +--- + +### Task 6: GPU weights extraction — handle BF16 weights + +**Files:** +- Modify: `crates/ml/src/cuda_pipeline/gpu_weights.rs:179,204` + +**Step 1: Cast to F32 before extraction** + +The CUDA experience collector kernel expects f32 weights. When model weights are BF16, cast before extracting: + +```rust +// In extract_one() at line 179, change: +.to_vec1::() +// To: +.to_dtype(candle_core::DType::F32)? +.to_vec1::() +``` + +Same for `sync_one()` at line 204. This is the boundary cast from BF16 model weights → f32 CUDA kernel. Happens once per epoch during experience collection, not in the training hot path. + +**Step 2:** Build check. +**Step 3: Commit** + +```bash +git add crates/ml/src/cuda_pipeline/gpu_weights.rs +git commit -m "feat(ml): handle BF16 weights in GPU weight extraction" +``` + +--- + +### Task 7: GPU data pre-upload — BF16 feature tensors + +**Files:** +- Modify: `crates/ml/src/cuda_pipeline/mod.rs:132,135,346,353` + +**Step 1: Cast feature/target uploads to training dtype** + +In `DqnGpuData::upload()` (line 132): + +```rust +// After creating the tensor from f32 data, cast: +let features = Tensor::from_vec(flat_features, (num_bars, feature_dim), device)? + .to_dtype(training_dtype(device))?; +let targets = Tensor::from_vec(flat_targets, (num_bars, target_dim), device)? + .to_dtype(training_dtype(device))?; +``` + +Same pattern for `GpuBufferPool::upload_dqn` (lines 346, 353) — cast after `from_slice`. + +For PPO `PpoGpuData::upload()` (line 418) — same cast. + +This is the data ingestion boundary cast. All downstream `build_batch_states()` and `bar_features()` calls return BF16 directly. + +**Step 2:** Build check. +**Step 3: Commit** + +```bash +git add crates/ml/src/cuda_pipeline/mod.rs +git commit -m "feat(ml): BF16 GPU data pre-upload for DQN and PPO" +``` + +--- + +### Task 8: DQN training tensors — CPU replay buffer path + +**Files:** +- Modify: `crates/ml/src/dqn/dqn.rs` — `compute_loss_internal()` + +The CPU replay buffer path creates training batch tensors from `Vec`. These need to match the model's weight dtype. + +**Step 1: Cast batch tensors at creation** + +At lines 1540-1569, after each `Tensor::from_vec`: + +```rust +// States/next_states — cast to model dtype for matmul compatibility +let states_tensor = Tensor::from_vec(states, (batch_size, self.config.state_dim), device)? + .to_dtype(training_dtype(device))?; +let next_states_tensor = Tensor::from_vec(next_states, (batch_size, self.config.state_dim), device)? + .to_dtype(training_dtype(device))?; +``` + +Actions stay U32. Rewards/dones/importance-weights stay F32 — they're used in loss math, not matmuls. + +For the GPU replay buffer path, states already come back as BF16 from `sample()` (Task 5), so no change needed there. + +**Step 2: Verify loss stays F32** + +The distributional loss at line 1650/1658 already has `to_dtype(DType::F32)` enforcement. Keep as-is. + +**Step 3:** Build check. +**Step 4: Commit** + +```bash +git add crates/ml/src/dqn/dqn.rs +git commit -m "feat(ml): BF16 training tensors in DQN compute_loss" +``` + +--- + +### Task 9: DQN trainer auxiliary tensors + +**Files:** +- Modify: `crates/ml/src/trainers/dqn/trainer.rs:1015,1130,2134,2141,3123,3647` + +Same pattern — cast state tensors used in `select_actions_batch`, curiosity, and Q-value logging to `training_dtype(&self.device)`: + +```rust +let tensor = Tensor::from_vec(states, shape, &self.device)? + .to_dtype(training_dtype(&self.device))?; +``` + +These are not in the training hot path (they're action selection and logging), so the single cast is fine. + +**Step 1:** Apply casts at listed lines. +**Step 2:** Build check. +**Step 3: Commit** + +```bash +git add crates/ml/src/trainers/dqn/trainer.rs +git commit -m "feat(ml): BF16 auxiliary tensors in DQN trainer" +``` + +--- + +## Phase 3: PPO Module + +### Task 10: VarBuilder sites — PPO networks + +**Files:** +- `crates/ml/src/ppo/ppo.rs:301,549` +- `crates/ml/src/ppo/lstm_networks.rs:52,267` +- `crates/ml/src/ppo/continuous_policy.rs:83` +- `crates/ml/src/ppo/flow_policy/mod.rs:124` +- `crates/ml/src/ppo/flow_policy/coupling_layer.rs:260` + +Same pattern: `DType::F32` → `training_dtype(&device)`. + +Also change checkpoint loading at lines 1795 and 1852: +```rust +// Before: +VarBuilder::from_mmaped_safetensors(&[path], DType::F32, &device) +// After: +VarBuilder::from_mmaped_safetensors(&[path], training_dtype(&device), &device) +``` + +**Step 1:** Apply all changes. +**Step 2:** Build check. +**Step 3: Commit** + +```bash +git add crates/ml/src/ppo/ +git commit -m "feat(ml): BF16 VarBuilder and checkpoints for PPO networks" +``` + +--- + +### Task 11: PPO training tensors + +**Files:** +- Modify: `crates/ml/src/trainers/ppo.rs:834,895,960,1014` + +Cast state tensors to training dtype for forward pass compatibility: + +```rust +let states = Tensor::from_vec(state_floats, shape, &self.device)? + .to_dtype(training_dtype(&self.device))?; +``` + +Lines 1156, 1346, 1348, 1372 (rewards, returns, values) — keep F32, these are loss/metric tensors not fed to the network. + +**Step 1:** Apply casts to state tensors only. +**Step 2:** Build check. +**Step 3: Commit** + +```bash +git add crates/ml/src/trainers/ppo.rs +git commit -m "feat(ml): BF16 training tensors in PPO trainer" +``` + +--- + +## Phase 4: Supervised Models (8 models) + +### Task 12: TFT VarBuilder sites + +**Files:** +- `crates/ml/src/tft/mod.rs:339` +- `crates/ml/src/tft/quantized_grn.rs:295,315` +- `crates/ml/src/tft/quantized_attention.rs:417` +- `crates/ml/src/tft/quantized_lstm.rs:417,442` +- `crates/ml/src/tft/quantized_vsn.rs:61,249` +- `crates/ml/src/tft/varmap_quantization.rs:676,722` + +Same `DType::F32` → `training_dtype(&device)` pattern. + +**Step 1:** Apply, build, commit. + +```bash +git commit -m "feat(ml): BF16 VarBuilder for TFT" +``` + +--- + +### Task 13: Mamba2 VarBuilder site + +**Files:** +- `crates/ml/src/mamba/mod.rs:631` +- `crates/ml/src/mamba/ssd_layer.rs:556` + +Already fixed scalar_tensor in Task 2. Now change VarBuilder dtype. + +**Step 1:** Apply, build, commit. + +```bash +git commit -m "feat(ml): BF16 VarBuilder for Mamba2" +``` + +--- + +### Task 14: Liquid/CfC VarBuilder sites + +**Files:** +- `crates/ml/src/liquid/candle_cfc.rs:450,460,475,489,499,524,548,558,578,601,639` (11 sites) +- `crates/ml/src/liquid/adapter.rs:58` +- `crates/ml/src/liquid/training.rs:505` + +**Step 1:** Apply, build, commit. + +```bash +git commit -m "feat(ml): BF16 VarBuilder for Liquid/CfC" +``` + +--- + +### Task 15: KAN VarBuilder sites + +**Files:** +- `crates/ml/src/kan/layer.rs:138,150` +- `crates/ml/src/kan/network.rs:94,107` +- `crates/ml/src/kan/trainable.rs:47` + +**Step 1:** Apply, build, commit. + +```bash +git commit -m "feat(ml): BF16 VarBuilder for KAN" +``` + +--- + +### Task 16: xLSTM VarBuilder sites + +**Files:** +- `crates/ml/src/xlstm/slstm.rs:136,148,164,172` +- `crates/ml/src/xlstm/mlstm.rs:229,243,259,267,282` +- `crates/ml/src/xlstm/block.rs:127,139,152,161` +- `crates/ml/src/xlstm/network.rs:172,185,198,214,236,248` +- `crates/ml/src/xlstm/trainable.rs:45` + +**Step 1:** Apply, build, commit. + +```bash +git commit -m "feat(ml): BF16 VarBuilder for xLSTM" +``` + +--- + +### Task 17: Diffusion VarBuilder sites + +**Files:** +- `crates/ml/src/diffusion/sampler.rs:163,230` +- `crates/ml/src/diffusion/denoiser.rs:233,244,258,270,285` +- `crates/ml/src/diffusion/trainable.rs:45` + +**Step 1:** Apply, build, commit. + +```bash +git commit -m "feat(ml): BF16 VarBuilder for Diffusion" +``` + +--- + +### Task 18: TGGN + TLOB VarBuilder sites + +**Files:** +- `crates/ml/src/tgnn/trainable_adapter.rs:87` +- `crates/ml/src/tlob/trainable_adapter.rs:116` +- `crates/ml/src/trainers/tlob.rs:210` + +**Step 1:** Apply, build, commit. + +```bash +git commit -m "feat(ml): BF16 VarBuilder for TGGN and TLOB" +``` + +--- + +## Phase 5: Remaining Sites + +### Task 19: Ensemble adapters + misc + +**Files:** +- `crates/ml/src/ensemble/adapters/liquid.rs:46,62` +- `crates/ml/src/ensemble/adapters/diffusion.rs:49,72` +- `crates/ml/src/ensemble/adapters/kan.rs:44,60` +- `crates/ml/src/ensemble/adapters/tlob.rs:100,123` +- `crates/ml/src/ensemble/adapters/tggn.rs:77,96` +- `crates/ml/src/ensemble/adapters/xlstm.rs:57,79` +- `crates/ml/src/portfolio_transformer.rs:190` +- `crates/ml/src/features/multi_timeframe.rs:269,560,576` +- `crates/ml/src/trainers/online_learning.rs:581` +- `crates/ml/src/explainability/integrated_gradients.rs:200,247,301` + +**Step 1:** Apply, build, commit. + +```bash +git commit -m "feat(ml): BF16 VarBuilder for ensemble adapters and misc modules" +``` + +--- + +## Phase 6: Validation + +### Task 20: Full workspace build and test + +**Step 1: Workspace build** + +```bash +SQLX_OFFLINE=true cargo check --workspace 2>&1 | tail -5 +``` + +Expected: 0 errors. Fix any dtype mismatches — common issues: +- `expected F32 but got BF16` — a tensor created without the dtype cast feeding into a module that expects matched dtypes +- `cannot add BF16 and F32` — missing cast at a boundary + +**Step 2: Clippy** + +```bash +SQLX_OFFLINE=true cargo clippy --workspace -- -D warnings 2>&1 | tail -10 +``` + +**Step 3: ML crate tests** + +```bash +SQLX_OFFLINE=true cargo test -p ml --lib 2>&1 | tail -20 +``` + +Expected: 2506+ tests pass. Tests run on CPU → F32 path, no behavior change. + +**Step 4: Commit if any fixes were needed** + +```bash +git commit -m "fix(ml): resolve BF16 dtype mismatches" +``` + +--- + +### Task 21: BF16 integration test (optional — requires GPU) + +Create a minimal integration test verifying BF16 training works end-to-end on CUDA: + +**Files:** +- Create: `crates/ml/tests/bf16_training_integration.rs` + +```rust +//! Integration test: verify BF16 training on Ampere+ GPU +//! Run with: SQLX_OFFLINE=true cargo test -p ml --test bf16_training_integration + +#[cfg(feature = "cuda")] +mod bf16_tests { + use ml::dqn::mixed_precision::training_dtype; + use candle_core::{Device, DType}; + + #[test] + fn test_training_dtype_returns_bf16_on_cuda() { + if let Ok(device) = Device::new_cuda(0) { + let dtype = training_dtype(&device); + // On Ampere+ (L40S, H100), should be BF16 + // On older GPUs, F32 is fine too + assert!(dtype == DType::BF16 || dtype == DType::F32); + } + } + + #[test] + fn test_training_dtype_returns_f32_on_cpu() { + let device = Device::Cpu; + assert_eq!(training_dtype(&device), DType::F32); + } +} +``` + +Real validation is the hyperopt run on L40S — compare trial Sharpe distributions. + +**Step 1:** Create test, build, commit. + +```bash +git commit -m "test(ml): add BF16 training dtype integration test" +``` + +--- + +## Summary + +| Phase | Tasks | Sites Changed | Commit Count | +|-------|-------|---------------|-------------| +| 1: Infrastructure | 1-2 | training_dtype fn + mamba fix | 2 | +| 2: DQN | 3-9 | ~80 VarBuilder + CUDA pipeline + training tensors | 7 | +| 3: PPO | 10-11 | ~9 VarBuilder + training tensors + checkpoints | 2 | +| 4: Supervised | 12-18 | ~60 VarBuilder across 8 models | 7 | +| 5: Remaining | 19 | ~20 ensemble/misc sites | 1 | +| 6: Validation | 20-21 | Build + test + integration test | 2 | +| **Total** | **21 tasks** | **~150 sites** | **~21 commits** | + +## Risk Checkpoints + +After Phase 2 (DQN complete): full workspace build must pass. DQN is the most complex module — if it compiles, the rest is mechanical. + +After Phase 6: all 2506+ tests must pass on CPU. GPU validation via hyperopt run on L40S.