#![allow( clippy::assertions_on_constants, clippy::assertions_on_result_states, clippy::clone_on_copy, clippy::decimal_literal_representation, clippy::doc_markdown, clippy::empty_line_after_doc_comments, clippy::field_reassign_with_default, clippy::get_unwrap, clippy::identity_op, clippy::inconsistent_digit_grouping, clippy::indexing_slicing, clippy::integer_division, clippy::len_zero, clippy::let_underscore_must_use, clippy::manual_div_ceil, clippy::manual_let_else, clippy::manual_range_contains, clippy::modulo_arithmetic, clippy::needless_range_loop, clippy::non_ascii_literal, clippy::redundant_clone, clippy::shadow_reuse, clippy::shadow_same, clippy::shadow_unrelated, clippy::single_match_else, clippy::str_to_string, clippy::string_slice, clippy::tests_outside_test_module, clippy::too_many_lines, clippy::unnecessary_wraps, clippy::unseparated_literal_suffix, clippy::use_debug, clippy::useless_vec, clippy::wildcard_enum_match_arm, clippy::else_if_without_else, clippy::expect_used, clippy::missing_const_for_fn, clippy::similar_names, clippy::type_complexity, clippy::collapsible_else_if, clippy::doc_lazy_continuation, clippy::items_after_test_module, clippy::map_clone, clippy::multiple_unsafe_ops_per_block, clippy::unwrap_or_default, clippy::assign_op_pattern, clippy::needless_borrow, clippy::println_empty_string, clippy::unnecessary_cast, clippy::used_underscore_binding, clippy::create_dir, clippy::implicit_saturating_sub, clippy::exit, clippy::expect_fun_call, clippy::too_many_arguments, clippy::unnecessary_map_or, clippy::unwrap_used, dead_code, unused_imports, unused_variables, clippy::cloned_ref_to_slice_refs, clippy::neg_multiply, clippy::while_let_loop, clippy::bool_assert_comparison, clippy::excessive_precision, clippy::trivially_copy_pass_by_ref, clippy::op_ref, clippy::redundant_closure, clippy::unnecessary_lazy_evaluations, clippy::if_then_some_else_none, clippy::unnecessary_to_owned, clippy::single_component_path_imports, )] //! End-to-end integration tests for the validation harness pipeline. //! //! Tests the full flow: DQN strategy creation -> walk-forward splitting -> //! train/evaluate per fold -> DSR/PBO/permutation -> report with verdict. use chrono::{Duration, TimeZone, Utc}; use ml::dqn::DQNConfig; use ml::validation::{ deflated_sharpe_ratio, probability_of_backtest_overfitting, walk_forward_split, DqnStrategy, TimeSeriesData, ValidationHarness, ValidationHarnessConfig, WalkForwardConfig, }; use tracing::info; // --------------------------------------------------------------------------- // Helpers // --------------------------------------------------------------------------- /// Create `n` UTC timestamps starting from 2024-01-01, one day apart. fn make_timestamps(n: usize) -> Vec> { (0..n) .map(|i| { Utc.with_ymd_and_hms(2024, 1, 1, 0, 0, 0) .single() .unwrap_or_else(Utc::now) + Duration::days(i as i64) }) .collect() } /// Create `n` feature rows of dimension `dim` with deterministic non-zero values. fn make_features(n: usize, dim: usize) -> Vec> { (0..n) .map(|i| { (0..dim) .map(|j| ((i * 7 + j * 3) % 100) as f32 * 0.01) .collect() }) .collect() } /// Build a minimal DQNConfig suitable for fast integration testing. fn make_small_dqn_config() -> DQNConfig { let mut config = DQNConfig::default(); config.state_dim = 8; config.num_actions = 3; config.hidden_dims = vec![16, 8]; config.batch_size = 4; config.min_replay_size = 4; config.warmup_steps = 0; config.use_iqn = false; config.epsilon_start = 0.3; config } // --------------------------------------------------------------------------- // Test 1: Full validation pipeline with DQN // --------------------------------------------------------------------------- #[test] fn test_full_validation_pipeline_with_dqn() { // 1. Create DQN strategy let config = make_small_dqn_config(); let mut strategy = DqnStrategy::new(config).unwrap_or_else(|e| panic!("DqnStrategy::new failed: {}", e)); // 2. Create synthetic TimeSeriesData (300 bars, feature_dim=8) // Prices follow sine + trend: 100.0 + sin(i*0.1)*5.0 + i*0.01 let n = 300_usize; let prices: Vec = (0..n) .map(|i| 100.0 + (i as f64 * 0.1).sin() * 5.0 + i as f64 * 0.01) .collect(); let timestamps = make_timestamps(n); let features = make_features(n, 8); let data = TimeSeriesData::new(timestamps, features, prices) .unwrap_or_else(|e| panic!("TimeSeriesData::new failed: {}", e)); // 3. Configure harness let harness_config = ValidationHarnessConfig { wf_config: WalkForwardConfig { train_bars: 50, test_bars: 30, embargo_bars: 5, step_bars: 30, min_train_samples: 20, }, num_permutations: 100, num_trials: 1, seed: 42, }; let harness = ValidationHarness::new(harness_config); // 4. Run validation let report = harness .validate(&mut strategy, &data) .unwrap_or_else(|e| panic!("validate failed: {}", e)); // 5. Assertions assert_eq!( report.strategy_name, "DQN", "Strategy name should be 'DQN', got '{}'", report.strategy_name ); assert!( report.num_folds >= 2, "Expected at least 2 folds, got {}", report.num_folds ); assert!( report.aggregate_sharpe.is_finite(), "Aggregate Sharpe should be finite, got {}", report.aggregate_sharpe ); // DSR p-value in [0, 1] assert!( (0.0..=1.0).contains(&report.dsr.pvalue), "DSR p-value out of range [0,1]: {}", report.dsr.pvalue ); // PBO in [0, 1] assert!( (0.0..=1.0).contains(&report.pbo.pbo), "PBO out of range [0,1]: {}", report.pbo.pbo ); // Permutation p-value in [0, 1] assert!( (0.0..=1.0).contains(&report.permutation.pvalue), "Permutation p-value out of range [0,1]: {}", report.permutation.pvalue ); // Per-regime metrics should not be empty assert!( !report.per_regime_metrics.is_empty(), "per_regime_metrics should not be empty" ); // Print summary info!( strategy = %report.strategy_name, folds = report.num_folds, aggregate_sharpe = report.aggregate_sharpe, dsr_pvalue = report.dsr.pvalue, pbo = report.pbo.pbo, permutation_pvalue = report.permutation.pvalue, verdict = %report.verdict, regimes = report.per_regime_metrics.len(), "Validation Report Summary" ); for (regime, metrics) in &report.per_regime_metrics { info!( regime = ?regime, sharpe = metrics.sharpe, bars = metrics.num_bars, win_rate = metrics.win_rate, avg_return = metrics.avg_return, "Regime metrics" ); } } // --------------------------------------------------------------------------- // Test 2: Walk-forward split standalone // --------------------------------------------------------------------------- #[test] fn test_walk_forward_split_standalone() { let config = WalkForwardConfig { train_bars: 50, test_bars: 20, embargo_bars: 5, step_bars: 20, min_train_samples: 20, }; let folds = walk_forward_split(200, &config); assert!( !folds.is_empty(), "Expected at least one fold from 200 bars" ); for fold in &folds { // test_range.end must not exceed total bars assert!( fold.test_range.end <= 200, "Fold {} test_range.end ({}) exceeds 200", fold.fold_index, fold.test_range.end ); // train.end <= embargo.start (they should be equal by construction) assert!( fold.train_range.end <= fold.embargo_range.start, "Fold {} train.end ({}) > embargo.start ({})", fold.fold_index, fold.train_range.end, fold.embargo_range.start ); // embargo.end <= test.start (they should be equal by construction) assert!( fold.embargo_range.end <= fold.test_range.start, "Fold {} embargo.end ({}) > test.start ({})", fold.fold_index, fold.embargo_range.end, fold.test_range.start ); } info!(num_folds = folds.len(), total_bars = 200, "Walk-forward split complete"); for fold in &folds { info!( fold = fold.fold_index, train = ?fold.train_range, embargo = ?fold.embargo_range, test = ?fold.test_range, "Fold ranges" ); } } // --------------------------------------------------------------------------- // Test 3: DSR and PBO standalone // --------------------------------------------------------------------------- #[test] fn test_dsr_and_pbo_standalone() { // --- DSR --- // High Sharpe (2.5) with few trials (5) should be significant -> p < 0.1 let dsr = deflated_sharpe_ratio(2.5, 5, 0.5, 0.1, 3.5, 500); assert!( dsr.pvalue < 0.1, "DSR: SR=2.5 with 5 trials should have p < 0.1, got {:.4}", dsr.pvalue ); assert!( dsr.observed_sharpe.is_finite(), "DSR observed_sharpe should be finite" ); assert!( dsr.deflated_sharpe.is_finite(), "DSR deflated_sharpe should be finite" ); assert!( dsr.sharpe_std_error.is_finite() && dsr.sharpe_std_error >= 0.0, "DSR sharpe_std_error should be non-negative and finite, got {}", dsr.sharpe_std_error ); info!(observed_sharpe = dsr.observed_sharpe, deflated_sharpe = dsr.deflated_sharpe, pvalue = dsr.pvalue, "DSR result"); // --- PBO --- // Consistent positive Sharpes across 8 folds -> should have num_combinations > 0 let sharpes = vec![1.0, 1.2, 0.8, 1.1, 0.9, 1.3, 1.0, 0.95]; let pbo = probability_of_backtest_overfitting(&sharpes); assert!( pbo.num_combinations > 0, "PBO should have evaluated combinations, got {}", pbo.num_combinations ); assert!( (0.0..=1.0).contains(&pbo.pbo), "PBO value should be in [0,1], got {}", pbo.pbo ); assert!( !pbo.logit_distribution.is_empty(), "PBO logit_distribution should not be empty" ); info!(pbo = pbo.pbo, num_combinations = pbo.num_combinations, logit_entries = pbo.logit_distribution.len(), "PBO result"); }