foxhunt/ml/tests/ppo_validation_real_data_test.rs

#![allow(unused_crate_dependencies)]
//! Real-data validation test: runs the full ValidationHarness on actual Databento 6E.FUT data
//! using PPO (both MLP and LSTM variants).
//!
//! This test loads 1-minute OHLCV bars from a DBN file, extracts 15-dimensional
//! features (5 OHLCV + 10 technical indicators), builds a TimeSeriesData,
//! wraps a PPO agent via PpoStrategy / PpoLstmStrategy, and runs walk-forward
//! validation with DSR, PBO, permutation tests, and per-regime breakdown.
//!
//! Requires: `test_data/real/databento/6E.FUT_ohlcv-1m_*.dbn` to exist.
//! Run with: `SQLX_OFFLINE=true cargo test --manifest-path ml/Cargo.toml --test ppo_validation_real_data_test -- --nocapture`

use chrono::{DateTime, Utc};
use ml::ppo::gae::GAEConfig;
use ml::ppo::ppo::PPOConfig;
use ml::real_data_loader::RealDataLoader;
use ml::validation::{
    PpoLstmStrategy, PpoStrategy, TimeSeriesData, ValidationHarness, ValidationHarnessConfig,
    WalkForwardConfig,
};

/// Build a 15-dimensional feature vector per bar from RealDataLoader output.
///
/// Features:
///   0-4: Normalized OHLCV (open, high, low, close, volume)
///   5:   RSI(14)
///   6-7: EMA fast(12), EMA slow(26)
///   8-10: MACD (line, signal, histogram = line - signal)
///   11-13: Bollinger Bands (upper, middle, lower)
///   14:  ATR(14)
fn build_features_from_loader(loader: &RealDataLoader, bars: &[ml::types::OHLCVBar]) -> Vec<Vec<f32>> {
    let feat_matrix = loader
        .extract_features(bars)
        .expect("extract_features failed");

    let indicators = loader
        .calculate_indicators(bars)
        .expect("calculate_indicators failed");

    let n = bars.len();
    let mut features = Vec::with_capacity(n);

    for i in 0..n {
        let mut row = Vec::with_capacity(15);

        // 0-4: normalized OHLCV
        if let Some(price_row) = feat_matrix.prices.get(i) {
            row.extend_from_slice(price_row);
        } else {
            row.extend_from_slice(&[0.0_f32; 5]);
        }

        // 5: RSI
        row.push(indicators.rsi.get(i).copied().unwrap_or(50.0) / 100.0); // normalize to 0-1

        // 6-7: EMA fast, slow (normalize relative to close)
        let close = bars.get(i).map(|b| b.close as f32).unwrap_or(1.0);
        let denom = if close.abs() > 1e-10 { close } else { 1.0 };
        row.push(indicators.ema_fast.get(i).copied().unwrap_or(0.0) / denom);
        row.push(indicators.ema_slow.get(i).copied().unwrap_or(0.0) / denom);

        // 8-10: MACD line, signal, histogram (already small values)
        let macd_line = indicators.macd.get(i).copied().unwrap_or(0.0);
        let macd_signal = indicators.macd_signal.get(i).copied().unwrap_or(0.0);
        row.push(macd_line);
        row.push(macd_signal);
        row.push(macd_line - macd_signal); // histogram

        // 11-13: Bollinger Bands (normalized relative to close)
        row.push(indicators.bb_upper.get(i).copied().unwrap_or(0.0) / denom);
        row.push(indicators.bb_middle.get(i).copied().unwrap_or(0.0) / denom);
        row.push(indicators.bb_lower.get(i).copied().unwrap_or(0.0) / denom);

        // 14: ATR (as fraction of close)
        row.push(indicators.atr.get(i).copied().unwrap_or(0.0) / denom);

        features.push(row);
    }

    features
}

fn make_ppo_mlp_config() -> PPOConfig {
    PPOConfig {
        state_dim: 15,
        num_actions: 3,
        policy_hidden_dims: vec![64, 32],
        value_hidden_dims: vec![64, 32],
        batch_size: 64,
        mini_batch_size: 32,
        num_epochs: 3,
        policy_learning_rate: 3e-4,
        value_learning_rate: 1e-3,
        clip_epsilon: 0.2,
        value_loss_coeff: 0.5,
        entropy_coeff: 0.01,
        max_grad_norm: 0.5,
        gae_config: GAEConfig {
            gamma: 0.99,
            lambda: 0.95,
            normalize_advantages: true,
        },
        early_stopping_enabled: false,
        early_stopping_patience: 10,
        early_stopping_min_delta: 1e-4,
        early_stopping_min_epochs: 10,
        use_lstm: false,
        ..PPOConfig::default()
    }
}

fn make_ppo_lstm_config() -> PPOConfig {
    let mut config = make_ppo_mlp_config();
    config.use_lstm = true;
    config.lstm_hidden_dim = 64;
    config.lstm_num_layers = 1;
    config.lstm_sequence_length = 16;
    config
}

/// Full walk-forward validation on real 6E.FUT minute-bar data using PPO (MLP).
///
/// Loads ~30 days of 1-minute OHLCV, extracts 15-dim features,
/// runs walk-forward with embargo, and prints the complete
/// ValidationReport including DSR, PBO, permutation test, and per-regime metrics.
#[tokio::test]
async fn test_ppo_mlp_validation_on_real_6e_data() {
    // 1. Load real data (auto-detect workspace root)
    let mut loader = RealDataLoader::new_from_workspace()
        .expect("Failed to find workspace root — run from foxhunt repo");
    let bars = loader
        .load_symbol_data("6E.FUT")
        .await
        .expect("Failed to load 6E.FUT data — check test_data/real/databento/ exists");

    println!("Loaded {} bars for 6E.FUT", bars.len());
    assert!(
        bars.len() > 500,
        "Expected at least 500 bars from 6E.FUT DBN, got {}",
        bars.len()
    );

    // Print data range
    if let (Some(first), Some(last)) = (bars.first(), bars.last()) {
        println!(
            "Data range: {} to {}",
            first.timestamp, last.timestamp
        );
        println!(
            "First bar: O={:.5} H={:.5} L={:.5} C={:.5} V={:.0}",
            first.open, first.high, first.low, first.close, first.volume
        );
    }

    // 2. Build features
    let features = build_features_from_loader(&loader, &bars);
    assert_eq!(features.len(), bars.len());

    // Verify features are finite
    for (i, row) in features.iter().enumerate() {
        assert_eq!(row.len(), 15, "Bar {} has {} features, expected 15", i, row.len());
        for (j, v) in row.iter().enumerate() {
            assert!(
                v.is_finite(),
                "Feature [{},{}] is not finite: {}",
                i,
                j,
                v
            );
        }
    }
    println!("Features: {} bars × {} dims, all finite", features.len(), 15);

    // 3. Build TimeSeriesData
    let timestamps: Vec<DateTime<Utc>> = bars.iter().map(|b| b.timestamp).collect();
    let prices: Vec<f64> = bars.iter().map(|b| b.close).collect();

    let data = TimeSeriesData::new(timestamps, features, prices)
        .expect("Failed to create TimeSeriesData");

    println!(
        "TimeSeriesData: {} bars, {} returns",
        data.len(),
        data.returns.len()
    );

    // 4. Create PPO MLP strategy
    let config = make_ppo_mlp_config();
    let mut strategy =
        PpoStrategy::new(config).expect("Failed to create PpoStrategy");

    // 5. Configure walk-forward harness
    //    For 1-min bars: 500 bars ~ 8 hours of training data
    //    test = 100 bars ~ 1.5 hours
    //    embargo = 20 bars ~ 20 minutes
    let num_bars = data.len();
    let train_bars = (num_bars / 5).max(200);
    let test_bars = (num_bars / 20).max(50);

    let harness_config = ValidationHarnessConfig {
        wf_config: WalkForwardConfig {
            train_bars,
            test_bars,
            embargo_bars: 20,
            step_bars: test_bars,
            min_train_samples: 100,
        },
        num_permutations: 500, // Reduced for speed; 10k for final
        num_trials: 1,
        seed: 42,
    };

    println!("\n=== Walk-Forward Config (PPO MLP) ===");
    println!("  train_bars:  {}", train_bars);
    println!("  test_bars:   {}", test_bars);
    println!("  embargo:     20");
    println!("  step:        {}", test_bars);
    println!("  total bars:  {}", num_bars);

    let harness = ValidationHarness::new(harness_config);

    // 6. Run validation
    println!("\n=== Running PPO MLP validation... ===");
    let report = harness
        .validate(&mut strategy, &data)
        .expect("Validation harness failed");

    // 7. Print full report
    println!("\n╔══════════════════════════════════════════╗");
    println!("║   VALIDATION REPORT: {}              ║", report.strategy_name);
    println!("╠══════════════════════════════════════════╣");
    println!("║ Folds:            {:>20}   ║", report.num_folds);
    println!("║ Aggregate Sharpe: {:>20.4}   ║", report.aggregate_sharpe);
    println!("╠══════════════════════════════════════════╣");
    println!("║ DEFLATED SHARPE RATIO                    ║");
    println!("║   Observed SR:    {:>20.4}   ║", report.dsr.observed_sharpe);
    println!("║   Expected max:   {:>20.4}   ║", report.dsr.expected_max_sharpe);
    println!("║   SE:             {:>20.4}   ║", report.dsr.sharpe_std_error);
    println!("║   DSR statistic:  {:>20.4}   ║", report.dsr.deflated_sharpe);
    println!("║   p-value:        {:>20.4}   ║", report.dsr.pvalue);
    println!("╠══════════════════════════════════════════╣");
    println!("║ PBO (CSCV)                               ║");
    println!("║   PBO:            {:>20.4}   ║", report.pbo.pbo);
    println!("║   Combinations:   {:>20}   ║", report.pbo.num_combinations);
    println!("╠══════════════════════════════════════════╣");
    println!("║ PERMUTATION TEST                         ║");
    println!("║   Observed SR:    {:>20.4}   ║", report.permutation.observed_sharpe);
    println!("║   Null mean:      {:>20.4}   ║", report.permutation.null_mean);
    println!("║   Null std:       {:>20.4}   ║", report.permutation.null_std);
    println!("║   p-value:        {:>20.4}   ║", report.permutation.pvalue);
    println!("║   Permutations:   {:>20}   ║", report.permutation.num_permutations);
    println!("╠══════════════════════════════════════════╣");
    println!("║ PER-FOLD SHARPES                         ║");
    for (i, sr) in report.per_fold_sharpes.iter().enumerate() {
        println!("║   Fold {:>2}:        {:>20.4}   ║", i, sr);
    }
    println!("╠══════════════════════════════════════════╣");
    println!("║ PER-REGIME BREAKDOWN                     ║");
    for (regime, m) in &report.per_regime_metrics {
        println!(
            "║   {:?}: SR={:.4}, bars={}, WR={:.2}%, avg_ret={:.6}",
            regime, m.sharpe, m.num_bars, m.win_rate * 100.0, m.avg_return
        );
    }
    println!("╠══════════════════════════════════════════╣");
    println!("║ VERDICT: {:>32}   ║", format!("{}", report.verdict));
    println!("╚══════════════════════════════════════════╝");

    // 8. Structural assertions (not outcome-dependent)
    assert!(report.num_folds >= 2, "Need at least 2 folds");
    assert!(report.aggregate_sharpe.is_finite());
    assert!((0.0..=1.0).contains(&report.dsr.pvalue));
    assert!((0.0..=1.0).contains(&report.pbo.pbo));
    assert!((0.0..=1.0).contains(&report.permutation.pvalue));
    assert!(!report.per_regime_metrics.is_empty());
}

/// Full walk-forward validation on real 6E.FUT minute-bar data using PPO (LSTM).
///
/// Same pipeline as the MLP variant but with LSTM temporal modeling enabled.
#[tokio::test]
async fn test_ppo_lstm_validation_on_real_6e_data() {
    // 1. Load real data (auto-detect workspace root)
    let mut loader = RealDataLoader::new_from_workspace()
        .expect("Failed to find workspace root — run from foxhunt repo");
    let bars = loader
        .load_symbol_data("6E.FUT")
        .await
        .expect("Failed to load 6E.FUT data — check test_data/real/databento/ exists");

    println!("Loaded {} bars for 6E.FUT", bars.len());
    assert!(
        bars.len() > 500,
        "Expected at least 500 bars from 6E.FUT DBN, got {}",
        bars.len()
    );

    // Print data range
    if let (Some(first), Some(last)) = (bars.first(), bars.last()) {
        println!(
            "Data range: {} to {}",
            first.timestamp, last.timestamp
        );
        println!(
            "First bar: O={:.5} H={:.5} L={:.5} C={:.5} V={:.0}",
            first.open, first.high, first.low, first.close, first.volume
        );
    }

    // 2. Build features
    let features = build_features_from_loader(&loader, &bars);
    assert_eq!(features.len(), bars.len());

    // Verify features are finite
    for (i, row) in features.iter().enumerate() {
        assert_eq!(row.len(), 15, "Bar {} has {} features, expected 15", i, row.len());
        for (j, v) in row.iter().enumerate() {
            assert!(
                v.is_finite(),
                "Feature [{},{}] is not finite: {}",
                i,
                j,
                v
            );
        }
    }
    println!("Features: {} bars × {} dims, all finite", features.len(), 15);

    // 3. Build TimeSeriesData
    let timestamps: Vec<DateTime<Utc>> = bars.iter().map(|b| b.timestamp).collect();
    let prices: Vec<f64> = bars.iter().map(|b| b.close).collect();

    let data = TimeSeriesData::new(timestamps, features, prices)
        .expect("Failed to create TimeSeriesData");

    println!(
        "TimeSeriesData: {} bars, {} returns",
        data.len(),
        data.returns.len()
    );

    // 4. Create PPO LSTM strategy
    let config = make_ppo_lstm_config();
    let mut strategy =
        PpoLstmStrategy::new(config).expect("Failed to create PpoLstmStrategy");

    // 5. Configure walk-forward harness
    //    For 1-min bars: 500 bars ~ 8 hours of training data
    //    test = 100 bars ~ 1.5 hours
    //    embargo = 20 bars ~ 20 minutes
    let num_bars = data.len();
    let train_bars = (num_bars / 5).max(200);
    let test_bars = (num_bars / 20).max(50);

    let harness_config = ValidationHarnessConfig {
        wf_config: WalkForwardConfig {
            train_bars,
            test_bars,
            embargo_bars: 20,
            step_bars: test_bars,
            min_train_samples: 100,
        },
        num_permutations: 500, // Reduced for speed; 10k for final
        num_trials: 1,
        seed: 42,
    };

    println!("\n=== Walk-Forward Config (PPO LSTM) ===");
    println!("  train_bars:  {}", train_bars);
    println!("  test_bars:   {}", test_bars);
    println!("  embargo:     20");
    println!("  step:        {}", test_bars);
    println!("  total bars:  {}", num_bars);

    let harness = ValidationHarness::new(harness_config);

    // 6. Run validation
    println!("\n=== Running PPO LSTM validation... ===");
    let report = harness
        .validate(&mut strategy, &data)
        .expect("Validation harness failed");

    // 7. Print full report
    println!("\n╔══════════════════════════════════════════╗");
    println!("║   VALIDATION REPORT: {}              ║", report.strategy_name);
    println!("╠══════════════════════════════════════════╣");
    println!("║ Folds:            {:>20}   ║", report.num_folds);
    println!("║ Aggregate Sharpe: {:>20.4}   ║", report.aggregate_sharpe);
    println!("╠══════════════════════════════════════════╣");
    println!("║ DEFLATED SHARPE RATIO                    ║");
    println!("║   Observed SR:    {:>20.4}   ║", report.dsr.observed_sharpe);
    println!("║   Expected max:   {:>20.4}   ║", report.dsr.expected_max_sharpe);
    println!("║   SE:             {:>20.4}   ║", report.dsr.sharpe_std_error);
    println!("║   DSR statistic:  {:>20.4}   ║", report.dsr.deflated_sharpe);
    println!("║   p-value:        {:>20.4}   ║", report.dsr.pvalue);
    println!("╠══════════════════════════════════════════╣");
    println!("║ PBO (CSCV)                               ║");
    println!("║   PBO:            {:>20.4}   ║", report.pbo.pbo);
    println!("║   Combinations:   {:>20}   ║", report.pbo.num_combinations);
    println!("╠══════════════════════════════════════════╣");
    println!("║ PERMUTATION TEST                         ║");
    println!("║   Observed SR:    {:>20.4}   ║", report.permutation.observed_sharpe);
    println!("║   Null mean:      {:>20.4}   ║", report.permutation.null_mean);
    println!("║   Null std:       {:>20.4}   ║", report.permutation.null_std);
    println!("║   p-value:        {:>20.4}   ║", report.permutation.pvalue);
    println!("║   Permutations:   {:>20}   ║", report.permutation.num_permutations);
    println!("╠══════════════════════════════════════════╣");
    println!("║ PER-FOLD SHARPES                         ║");
    for (i, sr) in report.per_fold_sharpes.iter().enumerate() {
        println!("║   Fold {:>2}:        {:>20.4}   ║", i, sr);
    }
    println!("╠══════════════════════════════════════════╣");
    println!("║ PER-REGIME BREAKDOWN                     ║");
    for (regime, m) in &report.per_regime_metrics {
        println!(
            "║   {:?}: SR={:.4}, bars={}, WR={:.2}%, avg_ret={:.6}",
            regime, m.sharpe, m.num_bars, m.win_rate * 100.0, m.avg_return
        );
    }
    println!("╠══════════════════════════════════════════╣");
    println!("║ VERDICT: {:>32}   ║", format!("{}", report.verdict));
    println!("╚══════════════════════════════════════════╝");

    // 8. Structural assertions (not outcome-dependent)
    assert!(report.num_folds >= 2, "Need at least 2 folds");
    assert!(report.aggregate_sharpe.is_finite());
    assert!((0.0..=1.0).contains(&report.dsr.pvalue));
    assert!((0.0..=1.0).contains(&report.pbo.pbo));
    assert!((0.0..=1.0).contains(&report.permutation.pvalue));
    assert!(!report.per_regime_metrics.is_empty());
}