foxhunt/crates/ml/tests/validation_real_data_test.rs

#![allow(
    clippy::assertions_on_constants,
    clippy::assertions_on_result_states,
    clippy::clone_on_copy,
    clippy::decimal_literal_representation,
    clippy::doc_markdown,
    clippy::empty_line_after_doc_comments,
    clippy::field_reassign_with_default,
    clippy::get_unwrap,
    clippy::identity_op,
    clippy::inconsistent_digit_grouping,
    clippy::indexing_slicing,
    clippy::integer_division,
    clippy::len_zero,
    clippy::let_underscore_must_use,
    clippy::manual_div_ceil,
    clippy::manual_let_else,
    clippy::manual_range_contains,
    clippy::modulo_arithmetic,
    clippy::needless_range_loop,
    clippy::non_ascii_literal,
    clippy::redundant_clone,
    clippy::shadow_reuse,
    clippy::shadow_same,
    clippy::shadow_unrelated,
    clippy::single_match_else,
    clippy::str_to_string,
    clippy::string_slice,
    clippy::tests_outside_test_module,
    clippy::too_many_lines,
    clippy::unnecessary_wraps,
    clippy::unseparated_literal_suffix,
    clippy::use_debug,
    clippy::useless_vec,
    clippy::wildcard_enum_match_arm,
    clippy::else_if_without_else,
    clippy::expect_used,
    clippy::missing_const_for_fn,
    clippy::similar_names,
    clippy::type_complexity,
    clippy::collapsible_else_if,
    clippy::doc_lazy_continuation,
    clippy::items_after_test_module,
    clippy::map_clone,
    clippy::multiple_unsafe_ops_per_block,
    clippy::unwrap_or_default,
    clippy::assign_op_pattern,
    clippy::needless_borrow,
    clippy::println_empty_string,
    clippy::unnecessary_cast,
    clippy::used_underscore_binding,
    clippy::create_dir,
    clippy::implicit_saturating_sub,
    clippy::exit,
    clippy::expect_fun_call,
    clippy::too_many_arguments,
    clippy::unnecessary_map_or,
    clippy::unwrap_used,
    dead_code,
    unused_imports,
    unused_variables,
    clippy::cloned_ref_to_slice_refs,
    clippy::neg_multiply,
    clippy::while_let_loop,
    clippy::bool_assert_comparison,
    clippy::excessive_precision,
    clippy::trivially_copy_pass_by_ref,
    clippy::op_ref,
    clippy::redundant_closure,
    clippy::unnecessary_lazy_evaluations,
    clippy::if_then_some_else_none,
    clippy::unnecessary_to_owned,
    clippy::single_component_path_imports,
)]
//! Real-data validation test: runs the full ValidationHarness on actual Databento 6E.FUT data.
//!
//! This test loads 1-minute OHLCV bars from a DBN file, extracts 15-dimensional
//! features (5 OHLCV + 10 technical indicators), builds a TimeSeriesData,
//! wraps a DQN agent via DqnStrategy, and runs walk-forward validation with
//! DSR, PBO, permutation tests, and per-regime breakdown.
//!
//! Requires: `test_data/real/databento/6E.FUT_ohlcv-1m_*.dbn` to exist.
//! Run with: `SQLX_OFFLINE=true cargo test --manifest-path ml/Cargo.toml --test validation_real_data_test -- --nocapture`

use chrono::{DateTime, Utc};
use ml::dqn::DQNConfig;
use ml::data_loader::RealDataLoader;
use ml::validation::{
    DqnStrategy, TimeSeriesData, ValidationHarness, ValidationHarnessConfig, WalkForwardConfig,
};
use tracing::info;

/// Build a 15-dimensional feature vector per bar from RealDataLoader output.
///
/// Features:
///   0-4: Normalized OHLCV (open, high, low, close, volume)
///   5:   RSI(14)
///   6-7: EMA fast(12), EMA slow(26)
///   8-10: MACD (line, signal, histogram = line - signal)
///   11-13: Bollinger Bands (upper, middle, lower)
///   14:  ATR(14)
fn build_features_from_loader(loader: &RealDataLoader, bars: &[ml::types::OHLCVBar]) -> Vec<Vec<f32>> {
    let feat_matrix = loader
        .extract_features(bars)
        .expect("extract_features failed");

    let indicators = loader
        .calculate_indicators(bars)
        .expect("calculate_indicators failed");

    let n = bars.len();
    let mut features = Vec::with_capacity(n);

    for i in 0..n {
        let mut row = Vec::with_capacity(16);

        // 0-4: normalized OHLCV
        if let Some(price_row) = feat_matrix.prices.get(i) {
            row.extend_from_slice(price_row);
        } else {
            row.extend_from_slice(&[0.0_f32; 5]);
        }

        // 5: RSI
        row.push(indicators.rsi.get(i).copied().unwrap_or(50.0) / 100.0); // normalize to 0-1

        // 6-7: EMA fast, slow (normalize relative to close)
        let close = bars.get(i).map(|b| b.close as f32).unwrap_or(1.0);
        let denom = if close.abs() > 1e-10 { close } else { 1.0 };
        row.push(indicators.ema_fast.get(i).copied().unwrap_or(0.0) / denom);
        row.push(indicators.ema_slow.get(i).copied().unwrap_or(0.0) / denom);

        // 8-10: MACD line, signal, histogram (already small values)
        let macd_line = indicators.macd.get(i).copied().unwrap_or(0.0);
        let macd_signal = indicators.macd_signal.get(i).copied().unwrap_or(0.0);
        row.push(macd_line);
        row.push(macd_signal);
        row.push(macd_line - macd_signal); // histogram

        // 11-13: Bollinger Bands (normalized relative to close)
        row.push(indicators.bb_upper.get(i).copied().unwrap_or(0.0) / denom);
        row.push(indicators.bb_middle.get(i).copied().unwrap_or(0.0) / denom);
        row.push(indicators.bb_lower.get(i).copied().unwrap_or(0.0) / denom);

        // 14: ATR (as fraction of close)
        row.push(indicators.atr.get(i).copied().unwrap_or(0.0) / denom);

        // 15: zero-pad to 16-dim (tensor core alignment, multiple of 8)
        row.push(0.0_f32);

        features.push(row);
    }

    features
}

fn make_dqn_config() -> DQNConfig {
    let mut config = DQNConfig::default();
    config.state_dim = 16; // 5 OHLCV + 10 indicators + 1 zero-pad (aligned to 8)
    config.num_actions = 3; // short / flat / long
    config.hidden_dims = vec![64, 32];
    config.batch_size = 16;
    config.min_replay_size = 16;
    config.warmup_steps = 0;
    config.use_iqn = false;
    config.epsilon_start = 0.3;
    config.epsilon_end = 0.01;
    config
}

/// Full walk-forward validation on real 6E.FUT minute-bar data.
///
/// Loads ~30 days of 1-minute OHLCV, extracts 15-dim features,
/// runs walk-forward with embargo, and prints the complete
/// ValidationReport including DSR, PBO, permutation test, and per-regime metrics.
#[tokio::test]
async fn test_validation_on_real_6e_data() {
    // 1. Load real data (auto-detect workspace root)
    let mut loader = RealDataLoader::new_from_workspace()
        .expect("Failed to find workspace root — run from foxhunt repo");
    let bars = loader
        .load_symbol_data("6E.FUT")
        .await
        .expect("Failed to load 6E.FUT data — check test_data/real/databento/ exists");

    info!(bars = bars.len(), "Loaded bars for 6E.FUT");
    assert!(
        bars.len() > 500,
        "Expected at least 500 bars from 6E.FUT DBN, got {}",
        bars.len()
    );

    // Print data range
    if let (Some(first), Some(last)) = (bars.first(), bars.last()) {
        info!(
            start = %first.timestamp,
            end = %last.timestamp,
            open = first.open,
            high = first.high,
            low = first.low,
            close = first.close,
            volume = first.volume,
            "Data range and first bar"
        );
    }

    // 2. Build features
    let features = build_features_from_loader(&loader, &bars);
    assert_eq!(features.len(), bars.len());

    // Verify features are finite
    for (i, row) in features.iter().enumerate() {
        assert_eq!(row.len(), 15, "Bar {} has {} features, expected 15", i, row.len());
        for (j, v) in row.iter().enumerate() {
            assert!(
                v.is_finite(),
                "Feature [{},{}] is not finite: {}",
                i,
                j,
                v
            );
        }
    }
    info!(bars = features.len(), dims = 15, "Features validated (all finite)");

    // 3. Build TimeSeriesData
    let timestamps: Vec<DateTime<Utc>> = bars.iter().map(|b| b.timestamp).collect();
    let prices: Vec<f64> = bars.iter().map(|b| b.close).collect();

    let data = TimeSeriesData::new(timestamps, features, prices)
        .expect("Failed to create TimeSeriesData");

    info!(bars = data.len(), returns = data.returns.len(), "TimeSeriesData created");

    // 4. Create DQN strategy
    let config = make_dqn_config();
    let mut strategy =
        DqnStrategy::new(config).expect("Failed to create DqnStrategy");

    // 5. Configure walk-forward harness
    //    For 1-min bars: 500 bars ≈ 8 hours of training data
    //    test = 100 bars ≈ 1.5 hours
    //    embargo = 20 bars ≈ 20 minutes
    let num_bars = data.len();
    let train_bars = (num_bars / 5).max(200);
    let test_bars = (num_bars / 20).max(50);

    let harness_config = ValidationHarnessConfig {
        wf_config: WalkForwardConfig {
            train_bars,
            test_bars,
            embargo_bars: 20,
            step_bars: test_bars,
            min_train_samples: 100,
        },
        num_permutations: 500, // Reduced for speed; 10k for final
        num_trials: 1,
        seed: 42,
    };

    info!(train_bars, test_bars, embargo_bars = 20, step_bars = test_bars, total_bars = num_bars, "Walk-Forward Config");

    let harness = ValidationHarness::new(harness_config);

    // 6. Run validation
    info!("Running validation...");
    let report = harness
        .validate(&mut strategy, &data)
        .expect("Validation harness failed");

    // 7. Log full report
    info!(
        strategy = %report.strategy_name,
        folds = report.num_folds,
        aggregate_sharpe = report.aggregate_sharpe,
        dsr_observed = report.dsr.observed_sharpe,
        dsr_expected_max = report.dsr.expected_max_sharpe,
        dsr_se = report.dsr.sharpe_std_error,
        dsr_statistic = report.dsr.deflated_sharpe,
        dsr_pvalue = report.dsr.pvalue,
        pbo = report.pbo.pbo,
        pbo_combinations = report.pbo.num_combinations,
        perm_observed = report.permutation.observed_sharpe,
        perm_null_mean = report.permutation.null_mean,
        perm_null_std = report.permutation.null_std,
        perm_pvalue = report.permutation.pvalue,
        perm_count = report.permutation.num_permutations,
        verdict = %report.verdict,
        "Validation Report"
    );
    for (i, sr) in report.per_fold_sharpes.iter().enumerate() {
        info!(fold = i, sharpe = sr, "Per-fold Sharpe");
    }
    for (regime, m) in &report.per_regime_metrics {
        info!(
            regime = ?regime,
            sharpe = m.sharpe,
            bars = m.num_bars,
            win_rate_pct = m.win_rate * 100.0,
            avg_return = m.avg_return,
            "Per-regime metrics"
        );
    }

    // 8. Structural assertions (not outcome-dependent)
    assert!(report.num_folds >= 2, "Need at least 2 folds");
    assert!(report.aggregate_sharpe.is_finite());
    assert!((0.0..=1.0).contains(&report.dsr.pvalue));
    assert!((0.0..=1.0).contains(&report.pbo.pbo));
    assert!((0.0..=1.0).contains(&report.permutation.pvalue));
    assert!(!report.per_regime_metrics.is_empty());
}