diff --git a/ml/tests/validation_harness_integration_test.rs b/ml/tests/validation_harness_integration_test.rs new file mode 100644 index 000000000..6e913caf9 --- /dev/null +++ b/ml/tests/validation_harness_integration_test.rs @@ -0,0 +1,284 @@ +//! End-to-end integration tests for the validation harness pipeline. +//! +//! Tests the full flow: DQN strategy creation -> walk-forward splitting -> +//! train/evaluate per fold -> DSR/PBO/permutation -> report with verdict. + +use chrono::{Duration, TimeZone, Utc}; +use ml::dqn::DQNConfig; +use ml::validation::{ + deflated_sharpe_ratio, probability_of_backtest_overfitting, walk_forward_split, + DqnStrategy, TimeSeriesData, ValidationHarness, + ValidationHarnessConfig, WalkForwardConfig, +}; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +/// Create `n` UTC timestamps starting from 2024-01-01, one day apart. +fn make_timestamps(n: usize) -> Vec> { + (0..n) + .map(|i| { + Utc.with_ymd_and_hms(2024, 1, 1, 0, 0, 0) + .single() + .unwrap_or_else(Utc::now) + + Duration::days(i as i64) + }) + .collect() +} + +/// Create `n` feature rows of dimension `dim` with deterministic non-zero values. +fn make_features(n: usize, dim: usize) -> Vec> { + (0..n) + .map(|i| { + (0..dim) + .map(|j| ((i * 7 + j * 3) % 100) as f32 * 0.01) + .collect() + }) + .collect() +} + +/// Build a minimal DQNConfig suitable for fast integration testing. +fn make_small_dqn_config() -> DQNConfig { + let mut config = DQNConfig::default(); + config.state_dim = 8; + config.num_actions = 3; + config.hidden_dims = vec![16, 8]; + config.batch_size = 4; + config.min_replay_size = 4; + config.warmup_steps = 0; + config.use_noisy_nets = false; + config.use_iqn = false; + config.use_distributional = false; + config.use_dueling = false; + config.use_per = false; + config.epsilon_start = 0.3; + config +} + +// --------------------------------------------------------------------------- +// Test 1: Full validation pipeline with DQN +// --------------------------------------------------------------------------- + +#[test] +fn test_full_validation_pipeline_with_dqn() { + // 1. Create DQN strategy + let config = make_small_dqn_config(); + let mut strategy = + DqnStrategy::new(config).unwrap_or_else(|e| panic!("DqnStrategy::new failed: {}", e)); + + // 2. Create synthetic TimeSeriesData (300 bars, feature_dim=8) + // Prices follow sine + trend: 100.0 + sin(i*0.1)*5.0 + i*0.01 + let n = 300_usize; + let prices: Vec = (0..n) + .map(|i| 100.0 + (i as f64 * 0.1).sin() * 5.0 + i as f64 * 0.01) + .collect(); + let timestamps = make_timestamps(n); + let features = make_features(n, 8); + + let data = TimeSeriesData::new(timestamps, features, prices) + .unwrap_or_else(|e| panic!("TimeSeriesData::new failed: {}", e)); + + // 3. Configure harness + let harness_config = ValidationHarnessConfig { + wf_config: WalkForwardConfig { + train_bars: 50, + test_bars: 30, + embargo_bars: 5, + step_bars: 30, + min_train_samples: 20, + }, + num_permutations: 100, + num_trials: 1, + seed: 42, + }; + + let harness = ValidationHarness::new(harness_config); + + // 4. Run validation + let report = harness + .validate(&mut strategy, &data) + .unwrap_or_else(|e| panic!("validate failed: {}", e)); + + // 5. Assertions + assert_eq!( + report.strategy_name, "DQN", + "Strategy name should be 'DQN', got '{}'", + report.strategy_name + ); + + assert!( + report.num_folds >= 2, + "Expected at least 2 folds, got {}", + report.num_folds + ); + + assert!( + report.aggregate_sharpe.is_finite(), + "Aggregate Sharpe should be finite, got {}", + report.aggregate_sharpe + ); + + // DSR p-value in [0, 1] + assert!( + (0.0..=1.0).contains(&report.dsr.pvalue), + "DSR p-value out of range [0,1]: {}", + report.dsr.pvalue + ); + + // PBO in [0, 1] + assert!( + (0.0..=1.0).contains(&report.pbo.pbo), + "PBO out of range [0,1]: {}", + report.pbo.pbo + ); + + // Permutation p-value in [0, 1] + assert!( + (0.0..=1.0).contains(&report.permutation.pvalue), + "Permutation p-value out of range [0,1]: {}", + report.permutation.pvalue + ); + + // Per-regime metrics should not be empty + assert!( + !report.per_regime_metrics.is_empty(), + "per_regime_metrics should not be empty" + ); + + // Print summary + println!("=== Validation Report Summary ==="); + println!("Strategy: {}", report.strategy_name); + println!("Folds: {}", report.num_folds); + println!("Aggregate SR: {:.4}", report.aggregate_sharpe); + println!("DSR p-value: {:.4}", report.dsr.pvalue); + println!("PBO: {:.4}", report.pbo.pbo); + println!("MC perm p: {:.4}", report.permutation.pvalue); + println!("Verdict: {}", report.verdict); + println!("Regimes: {}", report.per_regime_metrics.len()); + for (regime, metrics) in &report.per_regime_metrics { + println!( + " {:?}: sharpe={:.4}, bars={}, win_rate={:.2}, avg_ret={:.6}", + regime, metrics.sharpe, metrics.num_bars, metrics.win_rate, metrics.avg_return + ); + } +} + +// --------------------------------------------------------------------------- +// Test 2: Walk-forward split standalone +// --------------------------------------------------------------------------- + +#[test] +fn test_walk_forward_split_standalone() { + let config = WalkForwardConfig { + train_bars: 50, + test_bars: 20, + embargo_bars: 5, + step_bars: 20, + min_train_samples: 20, + }; + + let folds = walk_forward_split(200, &config); + + assert!( + !folds.is_empty(), + "Expected at least one fold from 200 bars" + ); + + for fold in &folds { + // test_range.end must not exceed total bars + assert!( + fold.test_range.end <= 200, + "Fold {} test_range.end ({}) exceeds 200", + fold.fold_index, + fold.test_range.end + ); + + // train.end <= embargo.start (they should be equal by construction) + assert!( + fold.train_range.end <= fold.embargo_range.start, + "Fold {} train.end ({}) > embargo.start ({})", + fold.fold_index, + fold.train_range.end, + fold.embargo_range.start + ); + + // embargo.end <= test.start (they should be equal by construction) + assert!( + fold.embargo_range.end <= fold.test_range.start, + "Fold {} embargo.end ({}) > test.start ({})", + fold.fold_index, + fold.embargo_range.end, + fold.test_range.start + ); + } + + println!( + "Walk-forward split: {} folds from 200 bars", + folds.len() + ); + for fold in &folds { + println!( + " Fold {}: train={:?}, embargo={:?}, test={:?}", + fold.fold_index, fold.train_range, fold.embargo_range, fold.test_range + ); + } +} + +// --------------------------------------------------------------------------- +// Test 3: DSR and PBO standalone +// --------------------------------------------------------------------------- + +#[test] +fn test_dsr_and_pbo_standalone() { + // --- DSR --- + // High Sharpe (2.5) with few trials (5) should be significant -> p < 0.1 + let dsr = deflated_sharpe_ratio(2.5, 5, 0.5, 0.1, 3.5, 500); + assert!( + dsr.pvalue < 0.1, + "DSR: SR=2.5 with 5 trials should have p < 0.1, got {:.4}", + dsr.pvalue + ); + assert!( + dsr.observed_sharpe.is_finite(), + "DSR observed_sharpe should be finite" + ); + assert!( + dsr.deflated_sharpe.is_finite(), + "DSR deflated_sharpe should be finite" + ); + assert!( + dsr.sharpe_std_error.is_finite() && dsr.sharpe_std_error >= 0.0, + "DSR sharpe_std_error should be non-negative and finite, got {}", + dsr.sharpe_std_error + ); + + println!("DSR result: observed={:.4}, deflated={:.4}, p={:.4}", + dsr.observed_sharpe, dsr.deflated_sharpe, dsr.pvalue); + + // --- PBO --- + // Consistent positive Sharpes across 8 folds -> should have num_combinations > 0 + let sharpes = vec![1.0, 1.2, 0.8, 1.1, 0.9, 1.3, 1.0, 0.95]; + let pbo = probability_of_backtest_overfitting(&sharpes); + assert!( + pbo.num_combinations > 0, + "PBO should have evaluated combinations, got {}", + pbo.num_combinations + ); + assert!( + (0.0..=1.0).contains(&pbo.pbo), + "PBO value should be in [0,1], got {}", + pbo.pbo + ); + assert!( + !pbo.logit_distribution.is_empty(), + "PBO logit_distribution should not be empty" + ); + + println!( + "PBO result: pbo={:.4}, num_combinations={}, logit_entries={}", + pbo.pbo, + pbo.num_combinations, + pbo.logit_distribution.len() + ); +}