test(validation): add end-to-end integration test for full validation pipeline

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

ml/tests/validation_harness_integration_test.rs

@@ -0,0 +1,284 @@
+//! End-to-end integration tests for the validation harness pipeline.
+//!
+//! Tests the full flow: DQN strategy creation -> walk-forward splitting ->
+//! train/evaluate per fold -> DSR/PBO/permutation -> report with verdict.
+
+use chrono::{Duration, TimeZone, Utc};
+use ml::dqn::DQNConfig;
+use ml::validation::{
+    deflated_sharpe_ratio, probability_of_backtest_overfitting, walk_forward_split,
+    DqnStrategy, TimeSeriesData, ValidationHarness,
+    ValidationHarnessConfig, WalkForwardConfig,
+};
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+/// Create `n` UTC timestamps starting from 2024-01-01, one day apart.
+fn make_timestamps(n: usize) -> Vec<chrono::DateTime<Utc>> {
+    (0..n)
+        .map(|i| {
+            Utc.with_ymd_and_hms(2024, 1, 1, 0, 0, 0)
+                .single()
+                .unwrap_or_else(Utc::now)
+                + Duration::days(i as i64)
+        })
+        .collect()
+}
+
+/// Create `n` feature rows of dimension `dim` with deterministic non-zero values.
+fn make_features(n: usize, dim: usize) -> Vec<Vec<f32>> {
+    (0..n)
+        .map(|i| {
+            (0..dim)
+                .map(|j| ((i * 7 + j * 3) % 100) as f32 * 0.01)
+                .collect()
+        })
+        .collect()
+}
+
+/// Build a minimal DQNConfig suitable for fast integration testing.
+fn make_small_dqn_config() -> DQNConfig {
+    let mut config = DQNConfig::default();
+    config.state_dim = 8;
+    config.num_actions = 3;
+    config.hidden_dims = vec![16, 8];
+    config.batch_size = 4;
+    config.min_replay_size = 4;
+    config.warmup_steps = 0;
+    config.use_noisy_nets = false;
+    config.use_iqn = false;
+    config.use_distributional = false;
+    config.use_dueling = false;
+    config.use_per = false;
+    config.epsilon_start = 0.3;
+    config
+}
+
+// ---------------------------------------------------------------------------
+// Test 1: Full validation pipeline with DQN
+// ---------------------------------------------------------------------------
+
+#[test]
+fn test_full_validation_pipeline_with_dqn() {
+    // 1. Create DQN strategy
+    let config = make_small_dqn_config();
+    let mut strategy =
+        DqnStrategy::new(config).unwrap_or_else(|e| panic!("DqnStrategy::new failed: {}", e));
+
+    // 2. Create synthetic TimeSeriesData (300 bars, feature_dim=8)
+    //    Prices follow sine + trend: 100.0 + sin(i*0.1)*5.0 + i*0.01
+    let n = 300_usize;
+    let prices: Vec<f64> = (0..n)
+        .map(|i| 100.0 + (i as f64 * 0.1).sin() * 5.0 + i as f64 * 0.01)
+        .collect();
+    let timestamps = make_timestamps(n);
+    let features = make_features(n, 8);
+
+    let data = TimeSeriesData::new(timestamps, features, prices)
+        .unwrap_or_else(|e| panic!("TimeSeriesData::new failed: {}", e));
+
+    // 3. Configure harness
+    let harness_config = ValidationHarnessConfig {
+        wf_config: WalkForwardConfig {
+            train_bars: 50,
+            test_bars: 30,
+            embargo_bars: 5,
+            step_bars: 30,
+            min_train_samples: 20,
+        },
+        num_permutations: 100,
+        num_trials: 1,
+        seed: 42,
+    };
+
+    let harness = ValidationHarness::new(harness_config);
+
+    // 4. Run validation
+    let report = harness
+        .validate(&mut strategy, &data)
+        .unwrap_or_else(|e| panic!("validate failed: {}", e));
+
+    // 5. Assertions
+    assert_eq!(
+        report.strategy_name, "DQN",
+        "Strategy name should be 'DQN', got '{}'",
+        report.strategy_name
+    );
+
+    assert!(
+        report.num_folds >= 2,
+        "Expected at least 2 folds, got {}",
+        report.num_folds
+    );
+
+    assert!(
+        report.aggregate_sharpe.is_finite(),
+        "Aggregate Sharpe should be finite, got {}",
+        report.aggregate_sharpe
+    );
+
+    // DSR p-value in [0, 1]
+    assert!(
+        (0.0..=1.0).contains(&report.dsr.pvalue),
+        "DSR p-value out of range [0,1]: {}",
+        report.dsr.pvalue
+    );
+
+    // PBO in [0, 1]
+    assert!(
+        (0.0..=1.0).contains(&report.pbo.pbo),
+        "PBO out of range [0,1]: {}",
+        report.pbo.pbo
+    );
+
+    // Permutation p-value in [0, 1]
+    assert!(
+        (0.0..=1.0).contains(&report.permutation.pvalue),
+        "Permutation p-value out of range [0,1]: {}",
+        report.permutation.pvalue
+    );
+
+    // Per-regime metrics should not be empty
+    assert!(
+        !report.per_regime_metrics.is_empty(),
+        "per_regime_metrics should not be empty"
+    );
+
+    // Print summary
+    println!("=== Validation Report Summary ===");
+    println!("Strategy:       {}", report.strategy_name);
+    println!("Folds:          {}", report.num_folds);
+    println!("Aggregate SR:   {:.4}", report.aggregate_sharpe);
+    println!("DSR p-value:    {:.4}", report.dsr.pvalue);
+    println!("PBO:            {:.4}", report.pbo.pbo);
+    println!("MC perm p:      {:.4}", report.permutation.pvalue);
+    println!("Verdict:        {}", report.verdict);
+    println!("Regimes:        {}", report.per_regime_metrics.len());
+    for (regime, metrics) in &report.per_regime_metrics {
+        println!(
+            "  {:?}: sharpe={:.4}, bars={}, win_rate={:.2}, avg_ret={:.6}",
+            regime, metrics.sharpe, metrics.num_bars, metrics.win_rate, metrics.avg_return
+        );
+    }
+}
+
+// ---------------------------------------------------------------------------
+// Test 2: Walk-forward split standalone
+// ---------------------------------------------------------------------------
+
+#[test]
+fn test_walk_forward_split_standalone() {
+    let config = WalkForwardConfig {
+        train_bars: 50,
+        test_bars: 20,
+        embargo_bars: 5,
+        step_bars: 20,
+        min_train_samples: 20,
+    };
+
+    let folds = walk_forward_split(200, &config);
+
+    assert!(
+        !folds.is_empty(),
+        "Expected at least one fold from 200 bars"
+    );
+
+    for fold in &folds {
+        // test_range.end must not exceed total bars
+        assert!(
+            fold.test_range.end <= 200,
+            "Fold {} test_range.end ({}) exceeds 200",
+            fold.fold_index,
+            fold.test_range.end
+        );
+
+        // train.end <= embargo.start (they should be equal by construction)
+        assert!(
+            fold.train_range.end <= fold.embargo_range.start,
+            "Fold {} train.end ({}) > embargo.start ({})",
+            fold.fold_index,
+            fold.train_range.end,
+            fold.embargo_range.start
+        );
+
+        // embargo.end <= test.start (they should be equal by construction)
+        assert!(
+            fold.embargo_range.end <= fold.test_range.start,
+            "Fold {} embargo.end ({}) > test.start ({})",
+            fold.fold_index,
+            fold.embargo_range.end,
+            fold.test_range.start
+        );
+    }
+
+    println!(
+        "Walk-forward split: {} folds from 200 bars",
+        folds.len()
+    );
+    for fold in &folds {
+        println!(
+            "  Fold {}: train={:?}, embargo={:?}, test={:?}",
+            fold.fold_index, fold.train_range, fold.embargo_range, fold.test_range
+        );
+    }
+}
+
+// ---------------------------------------------------------------------------
+// Test 3: DSR and PBO standalone
+// ---------------------------------------------------------------------------
+
+#[test]
+fn test_dsr_and_pbo_standalone() {
+    // --- DSR ---
+    // High Sharpe (2.5) with few trials (5) should be significant -> p < 0.1
+    let dsr = deflated_sharpe_ratio(2.5, 5, 0.5, 0.1, 3.5, 500);
+    assert!(
+        dsr.pvalue < 0.1,
+        "DSR: SR=2.5 with 5 trials should have p < 0.1, got {:.4}",
+        dsr.pvalue
+    );
+    assert!(
+        dsr.observed_sharpe.is_finite(),
+        "DSR observed_sharpe should be finite"
+    );
+    assert!(
+        dsr.deflated_sharpe.is_finite(),
+        "DSR deflated_sharpe should be finite"
+    );
+    assert!(
+        dsr.sharpe_std_error.is_finite() && dsr.sharpe_std_error >= 0.0,
+        "DSR sharpe_std_error should be non-negative and finite, got {}",
+        dsr.sharpe_std_error
+    );
+
+    println!("DSR result: observed={:.4}, deflated={:.4}, p={:.4}",
+        dsr.observed_sharpe, dsr.deflated_sharpe, dsr.pvalue);
+
+    // --- PBO ---
+    // Consistent positive Sharpes across 8 folds -> should have num_combinations > 0
+    let sharpes = vec![1.0, 1.2, 0.8, 1.1, 0.9, 1.3, 1.0, 0.95];
+    let pbo = probability_of_backtest_overfitting(&sharpes);
+    assert!(
+        pbo.num_combinations > 0,
+        "PBO should have evaluated combinations, got {}",
+        pbo.num_combinations
+    );
+    assert!(
+        (0.0..=1.0).contains(&pbo.pbo),
+        "PBO value should be in [0,1], got {}",
+        pbo.pbo
+    );
+    assert!(
+        !pbo.logit_distribution.is_empty(),
+        "PBO logit_distribution should not be empty"
+    );
+
+    println!(
+        "PBO result: pbo={:.4}, num_combinations={}, logit_entries={}",
+        pbo.pbo,
+        pbo.num_combinations,
+        pbo.logit_distribution.len()
+    );
+}