foxhunt/ml/tests/parquet_feature_extraction_test.rs

//! Parquet Feature Extraction TDD Test Suite
//!
//! This test suite validates the integration of the production 54-feature pipeline
//! with Parquet data loading. Tests are designed following strict TDD methodology:
//! - Test 1: Module existence (validates module structure)
//! - Test 2: Parquet loading (validates file I/O)
//! - Test 3: Feature dimensionality (validates 54-dim output)
//! - Test 4: NaN/Inf validation (validates data quality)
//! - Test 5: Warmup period (validates 50-bar removal)
//! - Test 6: Production consistency (validates Wave C + Wave D features)
//! - Test 7: End-to-end pipeline (validates full integration)

use ml::data_loaders::parquet_utils::load_parquet_data;
use std::path::PathBuf;

/// Helper function to find test data file across different working directories
fn find_test_data_file() -> Option<PathBuf> {
    let possible_paths = [
        "test_data/ES_FUT_unseen.parquet",
        "../test_data/ES_FUT_unseen.parquet",
        "/home/jgrusewski/Work/foxhunt/test_data/ES_FUT_unseen.parquet",
    ];

    possible_paths
        .iter()
        .map(|p| PathBuf::from(p))
        .find(|p| p.exists())
}

#[test]
fn test_1_parquet_loader_module_exists() {
    // TDD STEP 1: Verify module is accessible
    // This test will FAIL until we create ml/src/data_loaders/parquet_utils.rs

    // If we can import the function, the module exists
    let _ = load_parquet_data; // Type check only

    println!("✅ Test 1 PASSED: Module ml::data_loaders::parquet_utils exists");
}

#[test]
fn test_2_load_parquet_successfully() {
    // TDD STEP 4: Verify Parquet file loading
    // This test validates that we can load a real Parquet file with OHLCV data

    let Some(parquet_path) = find_test_data_file() else {
        println!("⚠️  Test 2 SKIPPED: Test data file not found");
        return;
    };

    let result = load_parquet_data(&parquet_path, 50);

    assert!(
        result.is_ok(),
        "❌ Test 2 FAILED: Parquet loading failed with error: {:?}",
        result.err()
    );

    let features = result.unwrap();

    assert!(
        !features.is_empty(),
        "❌ Test 2 FAILED: Feature vectors should not be empty"
    );

    println!(
        "✅ Test 2 PASSED: Loaded {} feature vectors from Parquet file",
        features.len()
    );
}

#[test]
fn test_3_feature_vectors_have_225_dimensions() {
    // TDD STEP 5: Verify all feature vectors have exactly 54 dimensions
    // Critical for model compatibility (DQN, TFT, PPO, MAMBA-2 all expect 54 inputs)

    let Some(parquet_path) = find_test_data_file() else {
        println!("⚠️  Test 3 SKIPPED: Test data file not found");
        return;
    };

    let features = load_parquet_data(&parquet_path, 50).expect("Failed to load Parquet data");

    // Validate ALL feature vectors have 54 dimensions
    for (idx, feature_vec) in features.iter().enumerate() {
        assert_eq!(
            feature_vec.len(),
            54,
            "❌ Test 3 FAILED: Feature vector {} has {} dimensions, expected 54",
            idx,
            feature_vec.len()
        );
    }

    println!(
        "✅ Test 3 PASSED: All {} feature vectors have exactly 54 dimensions",
        features.len()
    );
}

#[test]
fn test_4_no_nan_inf_in_features() {
    // TDD STEP 6: Verify no NaN/Inf values in feature vectors
    // NaN/Inf causes model training failures and inference crashes

    let Some(parquet_path) = find_test_data_file() else {
        println!("⚠️  Test 4 SKIPPED: Test data file not found");
        return;
    };

    let features = load_parquet_data(&parquet_path, 50).expect("Failed to load Parquet data");

    // Check ALL values in ALL feature vectors
    let mut nan_count = 0;
    let mut inf_count = 0;

    for (vec_idx, feature_vec) in features.iter().enumerate() {
        for (feat_idx, &value) in feature_vec.iter().enumerate() {
            if value.is_nan() {
                nan_count += 1;
                eprintln!(
                    "⚠️  NaN detected at vector {} feature {}: value={}",
                    vec_idx, feat_idx, value
                );
            }
            if value.is_infinite() {
                inf_count += 1;
                eprintln!(
                    "⚠️  Inf detected at vector {} feature {}: value={}",
                    vec_idx, feat_idx, value
                );
            }
        }
    }

    assert_eq!(
        nan_count, 0,
        "❌ Test 4 FAILED: Found {} NaN values in feature vectors",
        nan_count
    );

    assert_eq!(
        inf_count, 0,
        "❌ Test 4 FAILED: Found {} Inf values in feature vectors",
        inf_count
    );

    println!(
        "✅ Test 4 PASSED: No NaN/Inf values in {} feature vectors ({} total values checked)",
        features.len(),
        features.len() * 54
    );
}

#[test]
fn test_5_warmup_period_removes_exactly_50_bars() {
    // TDD STEP 7: Verify warmup period logic
    // Warmup is critical for rolling window features (SMA, EMA, ATR, etc.)

    let Some(parquet_path) = find_test_data_file() else {
        println!("⚠️  Test 5 SKIPPED: Test data file not found");
        return;
    };

    // Load with warmup=0 (all bars)
    let features_no_warmup =
        load_parquet_data(&parquet_path, 0).expect("Failed to load with warmup=0");

    // Load with warmup=50 (skip first 50)
    let features_with_warmup =
        load_parquet_data(&parquet_path, 50).expect("Failed to load with warmup=50");

    // Warmup should remove exactly 50 feature vectors
    let expected_diff = 50;
    let actual_diff = features_no_warmup.len() - features_with_warmup.len();

    assert_eq!(
        actual_diff, expected_diff,
        "❌ Test 5 FAILED: Warmup removed {} bars, expected {} bars",
        actual_diff, expected_diff
    );

    println!(
        "✅ Test 5 PASSED: Warmup correctly removed {} bars ({} → {} feature vectors)",
        expected_diff,
        features_no_warmup.len(),
        features_with_warmup.len()
    );
}

#[test]
fn test_6_production_consistency_wave_d_features() {
    // TDD STEP 8: Verify advanced features are non-zero
    // Advanced features (indices 40-53) should contain microstructure and regime data
    // If these are zero/constant, it indicates mock features instead of production pipeline

    let Some(parquet_path) = find_test_data_file() else {
        println!("⚠️  Test 6 SKIPPED: Test data file not found");
        return;
    };

    let features = load_parquet_data(&parquet_path, 50).expect("Failed to load Parquet data");

    // Check advanced features (indices 40-53, 14 features)
    // These should be non-zero for production pipeline
    let wave_d_start = 40;
    let wave_d_end = 53;

    let mut non_zero_count = 0;
    let total_wave_d_features = (wave_d_end - wave_d_start + 1) * features.len();

    for feature_vec in &features {
        for idx in wave_d_start..=wave_d_end {
            if feature_vec[idx].abs() > 1e-10 {
                non_zero_count += 1;
            }
        }
    }

    // At least 10% of Wave D features should be non-zero
    let non_zero_ratio = non_zero_count as f64 / total_wave_d_features as f64;

    assert!(
        non_zero_ratio > 0.1,
        "❌ Test 6 FAILED: Advanced features are mostly zero ({:.2}% non-zero). This indicates mock features instead of production pipeline.",
        non_zero_ratio * 100.0
    );

    println!(
        "✅ Test 6 PASSED: Advanced features are non-zero ({:.2}% non-zero, {} / {} values)",
        non_zero_ratio * 100.0,
        non_zero_count,
        total_wave_d_features
    );
}

#[test]
fn test_7_end_to_end_parquet_to_inference_ready() {
    // TDD STEP 9: End-to-end validation
    // Simulate the full pipeline: Parquet → Features → Model Input

    let Some(parquet_path) = find_test_data_file() else {
        println!("⚠️  Test 7 SKIPPED: Test data file not found");
        return;
    };

    // Load features
    let features = load_parquet_data(&parquet_path, 50).expect("Failed to load Parquet data");

    // Validate minimum data requirement (100 bars for meaningful evaluation)
    assert!(
        features.len() >= 100,
        "❌ Test 7 FAILED: Insufficient data for inference ({} bars, expected >= 100)",
        features.len()
    );

    // Validate feature statistics (sanity checks)
    // 1. OHLCV features (0-4) should be in reasonable ranges
    for (idx, feature_vec) in features.iter().take(10).enumerate() {
        // Normalized OHLCV should be roughly 0.0 - 2.0 range
        for i in 0..5 {
            assert!(
                feature_vec[i].abs() < 10.0,
                "❌ Test 7 FAILED: OHLCV feature {} out of reasonable range at vector {}: value={}",
                i,
                idx,
                feature_vec[i]
            );
        }
    }

    // 2. Features should have non-zero variance (not all constant)
    let first_vec = &features[0];
    let mut all_same = true;
    for feature_vec in &features[1..] {
        for i in 0..54 {
            if (feature_vec[i] - first_vec[i]).abs() > 1e-8 {
                all_same = false;
                break;
            }
        }
        if !all_same {
            break;
        }
    }

    assert!(
        !all_same,
        "❌ Test 7 FAILED: All feature vectors are identical (no variance)"
    );

    println!(
        "✅ Test 7 PASSED: End-to-end pipeline produces {} inference-ready feature vectors",
        features.len()
    );
    println!("   - Feature dimensionality: 54 ✓");
    println!("   - No NaN/Inf values ✓");
    println!("   - Non-zero variance ✓");
    println!("   - Production Wave D features ✓");
}