foxhunt/crates/ml/tests/ppo_validation_real_data_test.rs

#![allow(
    clippy::assertions_on_constants,
    clippy::assertions_on_result_states,
    clippy::clone_on_copy,
    clippy::decimal_literal_representation,
    clippy::doc_markdown,
    clippy::empty_line_after_doc_comments,
    clippy::field_reassign_with_default,
    clippy::get_unwrap,
    clippy::identity_op,
    clippy::inconsistent_digit_grouping,
    clippy::indexing_slicing,
    clippy::integer_division,
    clippy::len_zero,
    clippy::let_underscore_must_use,
    clippy::manual_div_ceil,
    clippy::manual_let_else,
    clippy::manual_range_contains,
    clippy::modulo_arithmetic,
    clippy::needless_range_loop,
    clippy::non_ascii_literal,
    clippy::redundant_clone,
    clippy::shadow_reuse,
    clippy::shadow_same,
    clippy::shadow_unrelated,
    clippy::single_match_else,
    clippy::str_to_string,
    clippy::string_slice,
    clippy::tests_outside_test_module,
    clippy::too_many_lines,
    clippy::unnecessary_wraps,
    clippy::unseparated_literal_suffix,
    clippy::use_debug,
    clippy::useless_vec,
    clippy::wildcard_enum_match_arm,
    clippy::else_if_without_else,
    clippy::expect_used,
    clippy::missing_const_for_fn,
    clippy::similar_names,
    clippy::type_complexity,
    clippy::collapsible_else_if,
    clippy::doc_lazy_continuation,
    clippy::items_after_test_module,
    clippy::map_clone,
    clippy::multiple_unsafe_ops_per_block,
    clippy::unwrap_or_default,
    clippy::assign_op_pattern,
    clippy::needless_borrow,
    clippy::println_empty_string,
    clippy::unnecessary_cast,
    clippy::used_underscore_binding,
    clippy::create_dir,
    clippy::implicit_saturating_sub,
    clippy::exit,
    clippy::expect_fun_call,
    clippy::too_many_arguments,
    clippy::unnecessary_map_or,
    clippy::unwrap_used,
    dead_code,
    unused_imports,
    unused_variables,
    clippy::cloned_ref_to_slice_refs,
    clippy::neg_multiply,
    clippy::while_let_loop,
    clippy::bool_assert_comparison,
    clippy::excessive_precision,
    clippy::trivially_copy_pass_by_ref,
    clippy::op_ref,
    clippy::redundant_closure,
    clippy::unnecessary_lazy_evaluations,
    clippy::if_then_some_else_none,
    clippy::unnecessary_to_owned,
    clippy::single_component_path_imports,
    unused_crate_dependencies,
)]
//! Real-data validation test: runs the full ValidationHarness on actual Databento 6E.FUT data
//! using PPO (both MLP and LSTM variants).
//!
//! This test loads 1-minute OHLCV bars from a DBN file, extracts 15-dimensional
//! features (5 OHLCV + 10 technical indicators), builds a TimeSeriesData,
//! wraps a PPO agent via PpoStrategy / PpoLstmStrategy, and runs walk-forward
//! validation with DSR, PBO, permutation tests, and per-regime breakdown.
//!
//! Requires: `test_data/real/databento/6E.FUT_ohlcv-1m_*.dbn` to exist.
//! Run with: `SQLX_OFFLINE=true cargo test --manifest-path ml/Cargo.toml --test ppo_validation_real_data_test -- --nocapture`

use chrono::{DateTime, Utc};
use ml::ppo::gae::GAEConfig;
use ml::ppo::ppo::PPOConfig;
use ml::data_loader::RealDataLoader;
use tracing::info;
use ml::validation::{
    PpoLstmStrategy, PpoStrategy, TimeSeriesData, ValidationHarness, ValidationHarnessConfig,
    WalkForwardConfig,
};

/// Build a 15-dimensional feature vector per bar from RealDataLoader output.
///
/// Features:
///   0-4: Normalized OHLCV (open, high, low, close, volume)
///   5:   RSI(14)
///   6-7: EMA fast(12), EMA slow(26)
///   8-10: MACD (line, signal, histogram = line - signal)
///   11-13: Bollinger Bands (upper, middle, lower)
///   14:  ATR(14)
fn build_features_from_loader(loader: &RealDataLoader, bars: &[ml::types::OHLCVBar]) -> Vec<Vec<f32>> {
    let feat_matrix = loader
        .extract_features(bars)
        .expect("extract_features failed");

    let indicators = loader
        .calculate_indicators(bars)
        .expect("calculate_indicators failed");

    let n = bars.len();
    let mut features = Vec::with_capacity(n);

    for i in 0..n {
        let mut row = Vec::with_capacity(15);

        // 0-4: normalized OHLCV
        if let Some(price_row) = feat_matrix.prices.get(i) {
            row.extend_from_slice(price_row);
        } else {
            row.extend_from_slice(&[0.0_f32; 5]);
        }

        // 5: RSI
        row.push(indicators.rsi.get(i).copied().unwrap_or(50.0) / 100.0); // normalize to 0-1

        // 6-7: EMA fast, slow (normalize relative to close)
        let close = bars.get(i).map(|b| b.close as f32).unwrap_or(1.0);
        let denom = if close.abs() > 1e-10 { close } else { 1.0 };
        row.push(indicators.ema_fast.get(i).copied().unwrap_or(0.0) / denom);
        row.push(indicators.ema_slow.get(i).copied().unwrap_or(0.0) / denom);

        // 8-10: MACD line, signal, histogram (already small values)
        let macd_line = indicators.macd.get(i).copied().unwrap_or(0.0);
        let macd_signal = indicators.macd_signal.get(i).copied().unwrap_or(0.0);
        row.push(macd_line);
        row.push(macd_signal);
        row.push(macd_line - macd_signal); // histogram

        // 11-13: Bollinger Bands (normalized relative to close)
        row.push(indicators.bb_upper.get(i).copied().unwrap_or(0.0) / denom);
        row.push(indicators.bb_middle.get(i).copied().unwrap_or(0.0) / denom);
        row.push(indicators.bb_lower.get(i).copied().unwrap_or(0.0) / denom);

        // 14: ATR (as fraction of close)
        row.push(indicators.atr.get(i).copied().unwrap_or(0.0) / denom);

        features.push(row);
    }

    features
}

fn make_ppo_mlp_config() -> PPOConfig {
    PPOConfig {
        state_dim: 15,
        num_actions: 3,
        policy_hidden_dims: vec![64, 32],
        value_hidden_dims: vec![64, 32],
        batch_size: 64,
        mini_batch_size: 32,
        num_epochs: 3,
        policy_learning_rate: 3e-4,
        value_learning_rate: 1e-3,
        clip_epsilon: 0.2,
        value_loss_coeff: 0.5,
        entropy_coeff: 0.01,
        max_grad_norm: 0.5,
        gae_config: GAEConfig {
            gamma: 0.99,
            lambda: 0.95,
            normalize_advantages: true,
        },
        early_stopping_enabled: false,
        early_stopping_patience: 10,
        early_stopping_min_delta: 1e-4,
        early_stopping_min_epochs: 10,
        use_lstm: false,
        ..PPOConfig::default()
    }
}

fn make_ppo_lstm_config() -> PPOConfig {
    let mut config = make_ppo_mlp_config();
    config.use_lstm = true;
    config.lstm_hidden_dim = 64;
    config.lstm_num_layers = 1;
    config.lstm_sequence_length = 16;
    config
}

/// Full walk-forward validation on real 6E.FUT minute-bar data using PPO (MLP).
///
/// Loads ~30 days of 1-minute OHLCV, extracts 15-dim features,
/// runs walk-forward with embargo, and prints the complete
/// ValidationReport including DSR, PBO, permutation test, and per-regime metrics.
#[tokio::test]
async fn test_ppo_mlp_validation_on_real_6e_data() {
    // 1. Load real data (auto-detect workspace root)
    let mut loader = RealDataLoader::new_from_workspace()
        .expect("Failed to find workspace root — run from foxhunt repo");
    let bars = loader
        .load_symbol_data("6E.FUT")
        .await
        .expect("Failed to load 6E.FUT data — check test_data/real/databento/ exists");

    info!(bar_count = bars.len(), "Loaded bars for 6E.FUT");
    assert!(
        bars.len() > 500,
        "Expected at least 500 bars from 6E.FUT DBN, got {}",
        bars.len()
    );

    // Log data range
    if let (Some(first), Some(last)) = (bars.first(), bars.last()) {
        info!(from = %first.timestamp, to = %last.timestamp, "Data range");
        info!(open = first.open, high = first.high, low = first.low, close = first.close, volume = first.volume, "First bar");
    }

    // 2. Build features
    let features = build_features_from_loader(&loader, &bars);
    assert_eq!(features.len(), bars.len());

    // Verify features are finite
    for (i, row) in features.iter().enumerate() {
        assert_eq!(row.len(), 15, "Bar {} has {} features, expected 15", i, row.len());
        for (j, v) in row.iter().enumerate() {
            assert!(
                v.is_finite(),
                "Feature [{},{}] is not finite: {}",
                i,
                j,
                v
            );
        }
    }
    info!(bar_count = features.len(), dims = 15, "Features loaded, all finite");

    // 3. Build TimeSeriesData
    let timestamps: Vec<DateTime<Utc>> = bars.iter().map(|b| b.timestamp).collect();
    let prices: Vec<f64> = bars.iter().map(|b| b.close).collect();

    let data = TimeSeriesData::new(timestamps, features, prices)
        .expect("Failed to create TimeSeriesData");

    info!(bars = data.len(), returns = data.returns.len(), "TimeSeriesData created");

    // 4. Create PPO MLP strategy
    let config = make_ppo_mlp_config();
    let mut strategy =
        PpoStrategy::new(config).expect("Failed to create PpoStrategy");

    // 5. Configure walk-forward harness
    //    For 1-min bars: 500 bars ~ 8 hours of training data
    //    test = 100 bars ~ 1.5 hours
    //    embargo = 20 bars ~ 20 minutes
    let num_bars = data.len();
    let train_bars = (num_bars / 5).max(200);
    let test_bars = (num_bars / 20).max(50);

    let harness_config = ValidationHarnessConfig {
        wf_config: WalkForwardConfig {
            train_bars,
            test_bars,
            embargo_bars: 20,
            step_bars: test_bars,
            min_train_samples: 100,
        },
        num_permutations: 500, // Reduced for speed; 10k for final
        num_trials: 1,
        seed: 42,
    };

    info!(train_bars, test_bars, embargo = 20, total_bars = num_bars, "Walk-Forward Config (PPO MLP)");

    let harness = ValidationHarness::new(harness_config);

    // 6. Run validation
    info!("Running PPO MLP validation");
    let report = harness
        .validate(&mut strategy, &data)
        .expect("Validation harness failed");

    // 7. Log full report
    info!(
        strategy = %report.strategy_name,
        folds = report.num_folds,
        aggregate_sharpe = report.aggregate_sharpe,
        dsr_observed = report.dsr.observed_sharpe,
        dsr_expected_max = report.dsr.expected_max_sharpe,
        dsr_se = report.dsr.sharpe_std_error,
        dsr_statistic = report.dsr.deflated_sharpe,
        dsr_pvalue = report.dsr.pvalue,
        pbo = report.pbo.pbo,
        pbo_combinations = report.pbo.num_combinations,
        perm_observed = report.permutation.observed_sharpe,
        perm_null_mean = report.permutation.null_mean,
        perm_null_std = report.permutation.null_std,
        perm_pvalue = report.permutation.pvalue,
        perm_count = report.permutation.num_permutations,
        verdict = %report.verdict,
        "Validation report"
    );
    for (i, sr) in report.per_fold_sharpes.iter().enumerate() {
        info!(fold = i, sharpe = sr, "Per-fold Sharpe");
    }
    for (regime, m) in &report.per_regime_metrics {
        info!(regime = ?regime, sharpe = m.sharpe, bars = m.num_bars, win_rate = m.win_rate, avg_return = m.avg_return, "Per-regime metrics");
    }

    // 8. Structural assertions (not outcome-dependent)
    assert!(report.num_folds >= 2, "Need at least 2 folds");
    assert!(report.aggregate_sharpe.is_finite());
    assert!((0.0..=1.0).contains(&report.dsr.pvalue));
    assert!((0.0..=1.0).contains(&report.pbo.pbo));
    assert!((0.0..=1.0).contains(&report.permutation.pvalue));
    assert!(!report.per_regime_metrics.is_empty());
}

/// Full walk-forward validation on real 6E.FUT minute-bar data using PPO (LSTM).
///
/// Same pipeline as the MLP variant but with LSTM temporal modeling enabled.
#[tokio::test]
async fn test_ppo_lstm_validation_on_real_6e_data() {
    // 1. Load real data (auto-detect workspace root)
    let mut loader = RealDataLoader::new_from_workspace()
        .expect("Failed to find workspace root — run from foxhunt repo");
    let bars = loader
        .load_symbol_data("6E.FUT")
        .await
        .expect("Failed to load 6E.FUT data — check test_data/real/databento/ exists");

    info!(bar_count = bars.len(), "Loaded bars for 6E.FUT");
    assert!(
        bars.len() > 500,
        "Expected at least 500 bars from 6E.FUT DBN, got {}",
        bars.len()
    );

    // Log data range
    if let (Some(first), Some(last)) = (bars.first(), bars.last()) {
        info!(from = %first.timestamp, to = %last.timestamp, "Data range");
        info!(open = first.open, high = first.high, low = first.low, close = first.close, volume = first.volume, "First bar");
    }

    // 2. Build features
    let features = build_features_from_loader(&loader, &bars);
    assert_eq!(features.len(), bars.len());

    // Verify features are finite
    for (i, row) in features.iter().enumerate() {
        assert_eq!(row.len(), 15, "Bar {} has {} features, expected 15", i, row.len());
        for (j, v) in row.iter().enumerate() {
            assert!(
                v.is_finite(),
                "Feature [{},{}] is not finite: {}",
                i,
                j,
                v
            );
        }
    }
    info!(bar_count = features.len(), dims = 15, "Features loaded, all finite");

    // 3. Build TimeSeriesData
    let timestamps: Vec<DateTime<Utc>> = bars.iter().map(|b| b.timestamp).collect();
    let prices: Vec<f64> = bars.iter().map(|b| b.close).collect();

    let data = TimeSeriesData::new(timestamps, features, prices)
        .expect("Failed to create TimeSeriesData");

    info!(bars = data.len(), returns = data.returns.len(), "TimeSeriesData created");

    // 4. Create PPO LSTM strategy
    let config = make_ppo_lstm_config();
    let mut strategy =
        PpoLstmStrategy::new(config).expect("Failed to create PpoLstmStrategy");

    // 5. Configure walk-forward harness
    //    For 1-min bars: 500 bars ~ 8 hours of training data
    //    test = 100 bars ~ 1.5 hours
    //    embargo = 20 bars ~ 20 minutes
    let num_bars = data.len();
    let train_bars = (num_bars / 5).max(200);
    let test_bars = (num_bars / 20).max(50);

    let harness_config = ValidationHarnessConfig {
        wf_config: WalkForwardConfig {
            train_bars,
            test_bars,
            embargo_bars: 20,
            step_bars: test_bars,
            min_train_samples: 100,
        },
        num_permutations: 500, // Reduced for speed; 10k for final
        num_trials: 1,
        seed: 42,
    };

    info!(train_bars, test_bars, embargo = 20, total_bars = num_bars, "Walk-Forward Config (PPO LSTM)");

    let harness = ValidationHarness::new(harness_config);

    // 6. Run validation
    info!("Running PPO LSTM validation");
    let report = harness
        .validate(&mut strategy, &data)
        .expect("Validation harness failed");

    // 7. Log full report
    info!(
        strategy = %report.strategy_name,
        folds = report.num_folds,
        aggregate_sharpe = report.aggregate_sharpe,
        dsr_observed = report.dsr.observed_sharpe,
        dsr_expected_max = report.dsr.expected_max_sharpe,
        dsr_se = report.dsr.sharpe_std_error,
        dsr_statistic = report.dsr.deflated_sharpe,
        dsr_pvalue = report.dsr.pvalue,
        pbo = report.pbo.pbo,
        pbo_combinations = report.pbo.num_combinations,
        perm_observed = report.permutation.observed_sharpe,
        perm_null_mean = report.permutation.null_mean,
        perm_null_std = report.permutation.null_std,
        perm_pvalue = report.permutation.pvalue,
        perm_count = report.permutation.num_permutations,
        verdict = %report.verdict,
        "Validation report"
    );
    for (i, sr) in report.per_fold_sharpes.iter().enumerate() {
        info!(fold = i, sharpe = sr, "Per-fold Sharpe");
    }
    for (regime, m) in &report.per_regime_metrics {
        info!(regime = ?regime, sharpe = m.sharpe, bars = m.num_bars, win_rate = m.win_rate, avg_return = m.avg_return, "Per-regime metrics");
    }

    // 8. Structural assertions (not outcome-dependent)
    assert!(report.num_folds >= 2, "Need at least 2 folds");
    assert!(report.aggregate_sharpe.is_finite());
    assert!((0.0..=1.0).contains(&report.dsr.pvalue));
    assert!((0.0..=1.0).contains(&report.pbo.pbo));
    assert!((0.0..=1.0).contains(&report.permutation.pvalue));
    assert!(!report.per_regime_metrics.is_empty());
}