foxhunt/crates/ml/tests/wave16q_ohlcv_mutation_test.rs

// WAVE 16Q: OHLCV Mutation Bug Regression Test
//
// **Bug Description**: DQN training corrupts OHLCV bar close prices from realistic values
// ($4000-$5500 for ES futures) to preprocessed z-scores (0.001-2.0 range). This causes:
// 1. max_position explosion to 100M (should be 10-50)
// 2. P&L calculation completely broken
// 3. Reward signals meaningless
//
// **Root Cause**: target[2] (raw_current) receives preprocessed z-scores instead of raw dollars,
// suggesting either data aliasing, index mismatch, or hidden mutation in preprocessing/feature extraction.
//
// **Fix Strategy**: Extract raw close prices into separate vector before any preprocessing or
// feature extraction to guarantee data preservation.

use anyhow::Result;
use ml::trainers::DQNHyperparameters;
use ml::trainers::dqn::DQNTrainer;

#[tokio::test]
async fn test_ohlcv_raw_prices_preserved_after_preprocessing() -> Result<()> {
    // Initialize trainer with preprocessing enabled
    let mut hyperparams = DQNHyperparameters::conservative();
    hyperparams.enable_preprocessing = true;
    hyperparams.preprocessing_window = 50;
    hyperparams.preprocessing_clip_sigma = 3.0;
    hyperparams.epochs = 1; // Single epoch test

    let mut trainer = DQNTrainer::new(hyperparams)?;

    // Use real DBN data (same as production)
    let data_dir = "test_data/real/databento/ml_training";

    // Run 1 epoch training to trigger preprocessing and feature extraction
    // Use no-op checkpoint callback (just return empty string)
    let noop_callback = |_epoch: usize, _data: Vec<u8>, _is_best: bool| -> Result<String> {
        Ok(String::new())
    };

    let result = trainer.train(data_dir, noop_callback).await;

    // Training may fail for other reasons, but we need to check OHLCV corruption
    // even if training fails (the bug manifests during data loading)
    if let Err(e) = result {
        eprintln!("⚠️ Training failed (may be expected): {}", e);
    }

    // Get validation data to inspect target values
    let val_data = trainer.get_val_data();

    // Validation: target[2] should contain raw prices ($4000-$5500), NOT z-scores (0.001-2.0)
    let mut raw_prices_valid = 0;
    let mut preprocessed_zscores = 0;
    let mut min_raw = f64::MAX;
    let mut max_raw = f64::MIN;

    for (_features, target) in val_data.iter().take(100) {
        let raw_current = target[2];

        min_raw = min_raw.min(raw_current);
        max_raw = max_raw.max(raw_current);

        // ES futures trade around $4000-$5500
        // Z-scores are in range -3 to +3 (clipped), typically 0.001 to 2.0
        if raw_current.abs() > 100.0 {
            // Realistic dollar price
            raw_prices_valid += 1;
        } else {
            // Suspiciously small (likely z-score)
            preprocessed_zscores += 1;
        }
    }

    println!("📊 OHLCV Validation Results:");
    println!("  • Raw prices (>$100): {}", raw_prices_valid);
    println!("  • Preprocessed z-scores (<$100): {}", preprocessed_zscores);
    println!("  • Min raw_current: {:.2}", min_raw);
    println!("  • Max raw_current: {:.2}", max_raw);

    // ASSERTION: At least 90% of samples should have realistic raw prices
    assert!(
        raw_prices_valid >= 90,
        "❌ BUG CONFIRMED: target[2] contains z-scores instead of raw prices! \
         Valid: {}/100, Z-scores: {}/100, Range: [{:.2}, {:.2}]",
        raw_prices_valid,
        preprocessed_zscores,
        min_raw,
        max_raw
    );

    // ASSERTION: Min/max should be in realistic ES futures range ($4000-$5500)
    assert!(
        min_raw > 3000.0 && max_raw < 6000.0,
        "❌ BUG CONFIRMED: Raw prices outside realistic range! \
         Expected: $4000-$5500, Got: [{:.2}, {:.2}]",
        min_raw,
        max_raw
    );

    println!("✅ PASS: OHLCV raw prices preserved correctly");
    Ok(())
}

#[tokio::test]
async fn test_target_2_is_raw_price_not_zscore() -> Result<()> {
    // Regression test: Verify target[2] contains raw dollars, not preprocessed z-scores
    let mut hyperparams = DQNHyperparameters::conservative();
    hyperparams.enable_preprocessing = true; // Enable preprocessing to trigger bug
    hyperparams.epochs = 1;

    let mut trainer = DQNTrainer::new(hyperparams)?;
    let data_dir = "test_data/real/databento/ml_training";

    let noop_callback = |_: usize, _: Vec<u8>, _: bool| -> Result<String> { Ok(String::new()) };
    let _ = trainer.train(data_dir, noop_callback).await;

    let val_data = trainer.get_val_data();
    assert!(!val_data.is_empty(), "Validation data should not be empty");

    // Check first 10 samples
    for (i, (_features, target)) in val_data.iter().take(10).enumerate() {
        let raw_current = target[2];

        println!("Sample {}: target[2] = {:.2}", i, raw_current);

        // Assertion: raw_current should be in realistic price range, NOT z-score range
        assert!(
            raw_current.abs() > 100.0,
            "Sample {}: target[2] = {:.6} looks like a z-score, not raw price",
            i,
            raw_current
        );
    }

    println!("✅ PASS: All samples have realistic raw prices in target[2]");
    Ok(())
}

#[tokio::test]
async fn test_max_position_realistic_not_100m() -> Result<()> {
    // Regression test: max_position should be 10-50, NOT 100M (indicates price corruption)
    let mut hyperparams = DQNHyperparameters::conservative();
    hyperparams.epochs = 1;

    let mut trainer = DQNTrainer::new(hyperparams)?;
    let data_dir = "test_data/real/databento/ml_training";

    let noop_callback = |_: usize, _: Vec<u8>, _: bool| -> Result<String> { Ok(String::new()) };
    let _ = trainer.train(data_dir, noop_callback).await;

    let val_data = trainer.get_val_data();

    // Simulate position calculation (simplified)
    let mut max_position = 0.0f64;

    for (_features, target) in val_data.iter() {
        let price_change = (target[1] - target[0]).abs();

        // Typical position sizing based on price change
        let position = 10000.0 / price_change.max(0.01); // Avoid div by zero

        max_position = max_position.max(position);
    }

    println!("📊 Max position calculated: {:.2}", max_position);

    // ASSERTION: max_position should be reasonable (10-50), NOT astronomical (100M)
    assert!(
        max_position < 1000.0,
        "❌ BUG CONFIRMED: max_position = {:.2e} suggests price corruption (expected <1000)",
        max_position
    );

    println!("✅ PASS: max_position = {:.2} is realistic", max_position);
    Ok(())
}