foxhunt/crates/ml/tests/validation_harness_integration_test.rs

#![allow(
    clippy::assertions_on_constants,
    clippy::assertions_on_result_states,
    clippy::clone_on_copy,
    clippy::decimal_literal_representation,
    clippy::doc_markdown,
    clippy::empty_line_after_doc_comments,
    clippy::field_reassign_with_default,
    clippy::get_unwrap,
    clippy::identity_op,
    clippy::inconsistent_digit_grouping,
    clippy::indexing_slicing,
    clippy::integer_division,
    clippy::len_zero,
    clippy::let_underscore_must_use,
    clippy::manual_div_ceil,
    clippy::manual_let_else,
    clippy::manual_range_contains,
    clippy::modulo_arithmetic,
    clippy::needless_range_loop,
    clippy::non_ascii_literal,
    clippy::redundant_clone,
    clippy::shadow_reuse,
    clippy::shadow_same,
    clippy::shadow_unrelated,
    clippy::single_match_else,
    clippy::str_to_string,
    clippy::string_slice,
    clippy::tests_outside_test_module,
    clippy::too_many_lines,
    clippy::unnecessary_wraps,
    clippy::unseparated_literal_suffix,
    clippy::use_debug,
    clippy::useless_vec,
    clippy::wildcard_enum_match_arm,
    clippy::else_if_without_else,
    clippy::expect_used,
    clippy::missing_const_for_fn,
    clippy::similar_names,
    clippy::type_complexity,
    clippy::collapsible_else_if,
    clippy::doc_lazy_continuation,
    clippy::items_after_test_module,
    clippy::map_clone,
    clippy::multiple_unsafe_ops_per_block,
    clippy::unwrap_or_default,
    clippy::assign_op_pattern,
    clippy::needless_borrow,
    clippy::println_empty_string,
    clippy::unnecessary_cast,
    clippy::used_underscore_binding,
    clippy::create_dir,
    clippy::implicit_saturating_sub,
    clippy::exit,
    clippy::expect_fun_call,
    clippy::too_many_arguments,
    clippy::unnecessary_map_or,
    clippy::unwrap_used,
    dead_code,
    unused_imports,
    unused_variables,
    clippy::cloned_ref_to_slice_refs,
    clippy::neg_multiply,
    clippy::while_let_loop,
    clippy::bool_assert_comparison,
    clippy::excessive_precision,
    clippy::trivially_copy_pass_by_ref,
    clippy::op_ref,
    clippy::redundant_closure,
    clippy::unnecessary_lazy_evaluations,
    clippy::if_then_some_else_none,
    clippy::unnecessary_to_owned,
    clippy::single_component_path_imports,
)]
//! End-to-end integration tests for the validation harness pipeline.
//!
//! Tests the full flow: DQN strategy creation -> walk-forward splitting ->
//! train/evaluate per fold -> DSR/PBO/permutation -> report with verdict.

use chrono::{Duration, TimeZone, Utc};
use ml::dqn::DQNConfig;
use ml::validation::{
    deflated_sharpe_ratio, probability_of_backtest_overfitting, walk_forward_split,
    DqnStrategy, TimeSeriesData, ValidationHarness,
    ValidationHarnessConfig, WalkForwardConfig,
};
use tracing::info;

// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------

/// Create `n` UTC timestamps starting from 2024-01-01, one day apart.
fn make_timestamps(n: usize) -> Vec<chrono::DateTime<Utc>> {
    (0..n)
        .map(|i| {
            Utc.with_ymd_and_hms(2024, 1, 1, 0, 0, 0)
                .single()
                .unwrap_or_else(Utc::now)
                + Duration::days(i as i64)
        })
        .collect()
}

/// Create `n` feature rows of dimension `dim` with deterministic non-zero values.
fn make_features(n: usize, dim: usize) -> Vec<Vec<f32>> {
    (0..n)
        .map(|i| {
            (0..dim)
                .map(|j| ((i * 7 + j * 3) % 100) as f32 * 0.01)
                .collect()
        })
        .collect()
}

/// Build a minimal DQNConfig suitable for fast integration testing.
fn make_small_dqn_config() -> DQNConfig {
    let mut config = DQNConfig::default();
    config.state_dim = 8;
    config.num_actions = 3;
    config.hidden_dims = vec![16, 8];
    config.batch_size = 4;
    config.min_replay_size = 4;
    config.warmup_steps = 0;
    config.use_iqn = false;
    config.epsilon_start = 0.3;
    config
}

// ---------------------------------------------------------------------------
// Test 1: Full validation pipeline with DQN
// ---------------------------------------------------------------------------

#[test]
fn test_full_validation_pipeline_with_dqn() {
    // 1. Create DQN strategy
    let config = make_small_dqn_config();
    let mut strategy =
        DqnStrategy::new(config).unwrap_or_else(|e| panic!("DqnStrategy::new failed: {}", e));

    // 2. Create synthetic TimeSeriesData (300 bars, feature_dim=8)
    //    Prices follow sine + trend: 100.0 + sin(i*0.1)*5.0 + i*0.01
    let n = 300_usize;
    let prices: Vec<f64> = (0..n)
        .map(|i| 100.0 + (i as f64 * 0.1).sin() * 5.0 + i as f64 * 0.01)
        .collect();
    let timestamps = make_timestamps(n);
    let features = make_features(n, 8);

    let data = TimeSeriesData::new(timestamps, features, prices)
        .unwrap_or_else(|e| panic!("TimeSeriesData::new failed: {}", e));

    // 3. Configure harness
    let harness_config = ValidationHarnessConfig {
        wf_config: WalkForwardConfig {
            train_bars: 50,
            test_bars: 30,
            embargo_bars: 5,
            step_bars: 30,
            min_train_samples: 20,
        },
        num_permutations: 100,
        num_trials: 1,
        seed: 42,
    };

    let harness = ValidationHarness::new(harness_config);

    // 4. Run validation
    let report = harness
        .validate(&mut strategy, &data)
        .unwrap_or_else(|e| panic!("validate failed: {}", e));

    // 5. Assertions
    assert_eq!(
        report.strategy_name, "DQN",
        "Strategy name should be 'DQN', got '{}'",
        report.strategy_name
    );

    assert!(
        report.num_folds >= 2,
        "Expected at least 2 folds, got {}",
        report.num_folds
    );

    assert!(
        report.aggregate_sharpe.is_finite(),
        "Aggregate Sharpe should be finite, got {}",
        report.aggregate_sharpe
    );

    // DSR p-value in [0, 1]
    assert!(
        (0.0..=1.0).contains(&report.dsr.pvalue),
        "DSR p-value out of range [0,1]: {}",
        report.dsr.pvalue
    );

    // PBO in [0, 1]
    assert!(
        (0.0..=1.0).contains(&report.pbo.pbo),
        "PBO out of range [0,1]: {}",
        report.pbo.pbo
    );

    // Permutation p-value in [0, 1]
    assert!(
        (0.0..=1.0).contains(&report.permutation.pvalue),
        "Permutation p-value out of range [0,1]: {}",
        report.permutation.pvalue
    );

    // Per-regime metrics should not be empty
    assert!(
        !report.per_regime_metrics.is_empty(),
        "per_regime_metrics should not be empty"
    );

    // Print summary
    info!(
        strategy = %report.strategy_name,
        folds = report.num_folds,
        aggregate_sharpe = report.aggregate_sharpe,
        dsr_pvalue = report.dsr.pvalue,
        pbo = report.pbo.pbo,
        permutation_pvalue = report.permutation.pvalue,
        verdict = %report.verdict,
        regimes = report.per_regime_metrics.len(),
        "Validation Report Summary"
    );
    for (regime, metrics) in &report.per_regime_metrics {
        info!(
            regime = ?regime,
            sharpe = metrics.sharpe,
            bars = metrics.num_bars,
            win_rate = metrics.win_rate,
            avg_return = metrics.avg_return,
            "Regime metrics"
        );
    }
}

// ---------------------------------------------------------------------------
// Test 2: Walk-forward split standalone
// ---------------------------------------------------------------------------

#[test]
fn test_walk_forward_split_standalone() {
    let config = WalkForwardConfig {
        train_bars: 50,
        test_bars: 20,
        embargo_bars: 5,
        step_bars: 20,
        min_train_samples: 20,
    };

    let folds = walk_forward_split(200, &config);

    assert!(
        !folds.is_empty(),
        "Expected at least one fold from 200 bars"
    );

    for fold in &folds {
        // test_range.end must not exceed total bars
        assert!(
            fold.test_range.end <= 200,
            "Fold {} test_range.end ({}) exceeds 200",
            fold.fold_index,
            fold.test_range.end
        );

        // train.end <= embargo.start (they should be equal by construction)
        assert!(
            fold.train_range.end <= fold.embargo_range.start,
            "Fold {} train.end ({}) > embargo.start ({})",
            fold.fold_index,
            fold.train_range.end,
            fold.embargo_range.start
        );

        // embargo.end <= test.start (they should be equal by construction)
        assert!(
            fold.embargo_range.end <= fold.test_range.start,
            "Fold {} embargo.end ({}) > test.start ({})",
            fold.fold_index,
            fold.embargo_range.end,
            fold.test_range.start
        );
    }

    info!(num_folds = folds.len(), total_bars = 200, "Walk-forward split complete");
    for fold in &folds {
        info!(
            fold = fold.fold_index,
            train = ?fold.train_range,
            embargo = ?fold.embargo_range,
            test = ?fold.test_range,
            "Fold ranges"
        );
    }
}

// ---------------------------------------------------------------------------
// Test 3: DSR and PBO standalone
// ---------------------------------------------------------------------------

#[test]
fn test_dsr_and_pbo_standalone() {
    // --- DSR ---
    // High Sharpe (2.5) with few trials (5) should be significant -> p < 0.1
    let dsr = deflated_sharpe_ratio(2.5, 5, 0.5, 0.1, 3.5, 500);
    assert!(
        dsr.pvalue < 0.1,
        "DSR: SR=2.5 with 5 trials should have p < 0.1, got {:.4}",
        dsr.pvalue
    );
    assert!(
        dsr.observed_sharpe.is_finite(),
        "DSR observed_sharpe should be finite"
    );
    assert!(
        dsr.deflated_sharpe.is_finite(),
        "DSR deflated_sharpe should be finite"
    );
    assert!(
        dsr.sharpe_std_error.is_finite() && dsr.sharpe_std_error >= 0.0,
        "DSR sharpe_std_error should be non-negative and finite, got {}",
        dsr.sharpe_std_error
    );

    info!(observed_sharpe = dsr.observed_sharpe, deflated_sharpe = dsr.deflated_sharpe, pvalue = dsr.pvalue, "DSR result");

    // --- PBO ---
    // Consistent positive Sharpes across 8 folds -> should have num_combinations > 0
    let sharpes = vec![1.0, 1.2, 0.8, 1.1, 0.9, 1.3, 1.0, 0.95];
    let pbo = probability_of_backtest_overfitting(&sharpes);
    assert!(
        pbo.num_combinations > 0,
        "PBO should have evaluated combinations, got {}",
        pbo.num_combinations
    );
    assert!(
        (0.0..=1.0).contains(&pbo.pbo),
        "PBO value should be in [0,1], got {}",
        pbo.pbo
    );
    assert!(
        !pbo.logit_distribution.is_empty(),
        "PBO logit_distribution should not be empty"
    );

    info!(pbo = pbo.pbo, num_combinations = pbo.num_combinations, logit_entries = pbo.logit_distribution.len(), "PBO result");
}