From 4ad9d45d7e6ce6ed457919fc56e76e0280222ce3 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Fri, 20 Feb 2026 17:33:05 +0100 Subject: [PATCH] test(ml): add real-data validation integration test for 6E.FUT Runs full ValidationHarness on actual Databento 6E.FUT 1-minute OHLCV bars (~30k bars). Extracts 15-dim features (5 OHLCV + 10 technical indicators), configures DQN with walk-forward validation, and prints a detailed report including DSR, PBO, permutation test, and per-regime breakdown. Co-Authored-By: Claude Opus 4.6 --- ml/tests/validation_real_data_test.rs | 252 ++++++++++++++++++++++++++ 1 file changed, 252 insertions(+) create mode 100644 ml/tests/validation_real_data_test.rs diff --git a/ml/tests/validation_real_data_test.rs b/ml/tests/validation_real_data_test.rs new file mode 100644 index 000000000..33e754138 --- /dev/null +++ b/ml/tests/validation_real_data_test.rs @@ -0,0 +1,252 @@ +//! Real-data validation test: runs the full ValidationHarness on actual Databento 6E.FUT data. +//! +//! This test loads 1-minute OHLCV bars from a DBN file, extracts 15-dimensional +//! features (5 OHLCV + 10 technical indicators), builds a TimeSeriesData, +//! wraps a DQN agent via DqnStrategy, and runs walk-forward validation with +//! DSR, PBO, permutation tests, and per-regime breakdown. +//! +//! Requires: `test_data/real/databento/6E.FUT_ohlcv-1m_*.dbn` to exist. +//! Run with: `SQLX_OFFLINE=true cargo test --manifest-path ml/Cargo.toml --test validation_real_data_test -- --nocapture` + +use chrono::{DateTime, Utc}; +use ml::dqn::DQNConfig; +use ml::real_data_loader::RealDataLoader; +use ml::validation::{ + DqnStrategy, TimeSeriesData, ValidationHarness, ValidationHarnessConfig, WalkForwardConfig, +}; + +/// Build a 15-dimensional feature vector per bar from RealDataLoader output. +/// +/// Features: +/// 0-4: Normalized OHLCV (open, high, low, close, volume) +/// 5: RSI(14) +/// 6-7: EMA fast(12), EMA slow(26) +/// 8-10: MACD (line, signal, histogram = line - signal) +/// 11-13: Bollinger Bands (upper, middle, lower) +/// 14: ATR(14) +fn build_features_from_loader(loader: &RealDataLoader, bars: &[ml::real_data_loader::OHLCVBar]) -> Vec> { + let feat_matrix = loader + .extract_features(bars) + .expect("extract_features failed"); + + let indicators = loader + .calculate_indicators(bars) + .expect("calculate_indicators failed"); + + let n = bars.len(); + let mut features = Vec::with_capacity(n); + + for i in 0..n { + let mut row = Vec::with_capacity(15); + + // 0-4: normalized OHLCV + if let Some(price_row) = feat_matrix.prices.get(i) { + row.extend_from_slice(price_row); + } else { + row.extend_from_slice(&[0.0_f32; 5]); + } + + // 5: RSI + row.push(indicators.rsi.get(i).copied().unwrap_or(50.0) / 100.0); // normalize to 0-1 + + // 6-7: EMA fast, slow (normalize relative to close) + let close = bars.get(i).map(|b| b.close as f32).unwrap_or(1.0); + let denom = if close.abs() > 1e-10 { close } else { 1.0 }; + row.push(indicators.ema_fast.get(i).copied().unwrap_or(0.0) / denom); + row.push(indicators.ema_slow.get(i).copied().unwrap_or(0.0) / denom); + + // 8-10: MACD line, signal, histogram (already small values) + let macd_line = indicators.macd.get(i).copied().unwrap_or(0.0); + let macd_signal = indicators.macd_signal.get(i).copied().unwrap_or(0.0); + row.push(macd_line); + row.push(macd_signal); + row.push(macd_line - macd_signal); // histogram + + // 11-13: Bollinger Bands (normalized relative to close) + row.push(indicators.bb_upper.get(i).copied().unwrap_or(0.0) / denom); + row.push(indicators.bb_middle.get(i).copied().unwrap_or(0.0) / denom); + row.push(indicators.bb_lower.get(i).copied().unwrap_or(0.0) / denom); + + // 14: ATR (as fraction of close) + row.push(indicators.atr.get(i).copied().unwrap_or(0.0) / denom); + + features.push(row); + } + + features +} + +fn make_dqn_config() -> DQNConfig { + let mut config = DQNConfig::default(); + config.state_dim = 15; // 5 OHLCV + 10 indicators + config.num_actions = 3; // short / flat / long + config.hidden_dims = vec![64, 32]; + config.batch_size = 16; + config.min_replay_size = 16; + config.warmup_steps = 0; + config.use_noisy_nets = false; + config.use_iqn = false; + config.use_distributional = false; + config.use_dueling = true; + config.use_per = false; + config.epsilon_start = 0.3; + config.epsilon_end = 0.01; + config +} + +/// Full walk-forward validation on real 6E.FUT minute-bar data. +/// +/// Loads ~30 days of 1-minute OHLCV, extracts 15-dim features, +/// runs walk-forward with embargo, and prints the complete +/// ValidationReport including DSR, PBO, permutation test, and per-regime metrics. +#[tokio::test] +async fn test_validation_on_real_6e_data() { + // 1. Load real data (auto-detect workspace root) + let mut loader = RealDataLoader::new_from_workspace() + .expect("Failed to find workspace root — run from foxhunt repo"); + let bars = loader + .load_symbol_data("6E.FUT") + .await + .expect("Failed to load 6E.FUT data — check test_data/real/databento/ exists"); + + println!("Loaded {} bars for 6E.FUT", bars.len()); + assert!( + bars.len() > 500, + "Expected at least 500 bars from 6E.FUT DBN, got {}", + bars.len() + ); + + // Print data range + if let (Some(first), Some(last)) = (bars.first(), bars.last()) { + println!( + "Data range: {} to {}", + first.timestamp, last.timestamp + ); + println!( + "First bar: O={:.5} H={:.5} L={:.5} C={:.5} V={:.0}", + first.open, first.high, first.low, first.close, first.volume + ); + } + + // 2. Build features + let features = build_features_from_loader(&loader, &bars); + assert_eq!(features.len(), bars.len()); + + // Verify features are finite + for (i, row) in features.iter().enumerate() { + assert_eq!(row.len(), 15, "Bar {} has {} features, expected 15", i, row.len()); + for (j, v) in row.iter().enumerate() { + assert!( + v.is_finite(), + "Feature [{},{}] is not finite: {}", + i, + j, + v + ); + } + } + println!("Features: {} bars × {} dims, all finite", features.len(), 15); + + // 3. Build TimeSeriesData + let timestamps: Vec> = bars.iter().map(|b| b.timestamp).collect(); + let prices: Vec = bars.iter().map(|b| b.close).collect(); + + let data = TimeSeriesData::new(timestamps, features, prices) + .expect("Failed to create TimeSeriesData"); + + println!( + "TimeSeriesData: {} bars, {} returns", + data.len(), + data.returns.len() + ); + + // 4. Create DQN strategy + let config = make_dqn_config(); + let mut strategy = + DqnStrategy::new(config).expect("Failed to create DqnStrategy"); + + // 5. Configure walk-forward harness + // For 1-min bars: 500 bars ≈ 8 hours of training data + // test = 100 bars ≈ 1.5 hours + // embargo = 20 bars ≈ 20 minutes + let num_bars = data.len(); + let train_bars = (num_bars / 5).max(200); + let test_bars = (num_bars / 20).max(50); + + let harness_config = ValidationHarnessConfig { + wf_config: WalkForwardConfig { + train_bars, + test_bars, + embargo_bars: 20, + step_bars: test_bars, + min_train_samples: 100, + }, + num_permutations: 500, // Reduced for speed; 10k for final + num_trials: 1, + seed: 42, + }; + + println!("\n=== Walk-Forward Config ==="); + println!(" train_bars: {}", train_bars); + println!(" test_bars: {}", test_bars); + println!(" embargo: 20"); + println!(" step: {}", test_bars); + println!(" total bars: {}", num_bars); + + let harness = ValidationHarness::new(harness_config); + + // 6. Run validation + println!("\n=== Running validation... ==="); + let report = harness + .validate(&mut strategy, &data) + .expect("Validation harness failed"); + + // 7. Print full report + println!("\n╔══════════════════════════════════════════╗"); + println!("║ VALIDATION REPORT: {} ║", report.strategy_name); + println!("╠══════════════════════════════════════════╣"); + println!("║ Folds: {:>20} ║", report.num_folds); + println!("║ Aggregate Sharpe: {:>20.4} ║", report.aggregate_sharpe); + println!("╠══════════════════════════════════════════╣"); + println!("║ DEFLATED SHARPE RATIO ║"); + println!("║ Observed SR: {:>20.4} ║", report.dsr.observed_sharpe); + println!("║ Expected max: {:>20.4} ║", report.dsr.expected_max_sharpe); + println!("║ SE: {:>20.4} ║", report.dsr.sharpe_std_error); + println!("║ DSR statistic: {:>20.4} ║", report.dsr.deflated_sharpe); + println!("║ p-value: {:>20.4} ║", report.dsr.pvalue); + println!("╠══════════════════════════════════════════╣"); + println!("║ PBO (CSCV) ║"); + println!("║ PBO: {:>20.4} ║", report.pbo.pbo); + println!("║ Combinations: {:>20} ║", report.pbo.num_combinations); + println!("╠══════════════════════════════════════════╣"); + println!("║ PERMUTATION TEST ║"); + println!("║ Observed SR: {:>20.4} ║", report.permutation.observed_sharpe); + println!("║ Null mean: {:>20.4} ║", report.permutation.null_mean); + println!("║ Null std: {:>20.4} ║", report.permutation.null_std); + println!("║ p-value: {:>20.4} ║", report.permutation.pvalue); + println!("║ Permutations: {:>20} ║", report.permutation.num_permutations); + println!("╠══════════════════════════════════════════╣"); + println!("║ PER-FOLD SHARPES ║"); + for (i, sr) in report.per_fold_sharpes.iter().enumerate() { + println!("║ Fold {:>2}: {:>20.4} ║", i, sr); + } + println!("╠══════════════════════════════════════════╣"); + println!("║ PER-REGIME BREAKDOWN ║"); + for (regime, m) in &report.per_regime_metrics { + println!( + "║ {:?}: SR={:.4}, bars={}, WR={:.2}%, avg_ret={:.6}", + regime, m.sharpe, m.num_bars, m.win_rate * 100.0, m.avg_return + ); + } + println!("╠══════════════════════════════════════════╣"); + println!("║ VERDICT: {:>32} ║", format!("{}", report.verdict)); + println!("╚══════════════════════════════════════════╝"); + + // 8. Structural assertions (not outcome-dependent) + assert!(report.num_folds >= 2, "Need at least 2 folds"); + assert!(report.aggregate_sharpe.is_finite()); + assert!((0.0..=1.0).contains(&report.dsr.pvalue)); + assert!((0.0..=1.0).contains(&report.pbo.pbo)); + assert!((0.0..=1.0).contains(&report.permutation.pvalue)); + assert!(!report.per_regime_metrics.is_empty()); +}