foxhunt/services/ml_training_service/tests/training_pipeline_comprehensive.rs

#![allow(
    unused_variables,
    clippy::unwrap_used,
    clippy::expect_used,
    clippy::indexing_slicing,
    clippy::unnecessary_cast,
    clippy::field_reassign_with_default
)]
//! Comprehensive ML Training Pipeline Tests
//!
//! This test suite validates the complete training data pipeline from database
//! to feature extraction, addressing critical gaps identified in Wave 81 coverage analysis.
//!
//! ## Test Coverage Areas
//!
//! 1. **Normalization Methods**: Z-score, min-max, robust scaling
//! 2. **Risk Metrics**: VaR, Expected Shortfall, Max Drawdown, Sharpe Ratio
//! 3. **Technical Indicators**: RSI, MACD, EMA calculations
//! 4. **Edge Cases**: Empty data, malformed data, connection failures
//! 5. **Data Quality**: Missing data, outliers, timestamp inconsistencies
//! 6. **Data Leakage Prevention**: Validation set normalization
//!
//! ## Wave 81 Discovery
//!
//! The "mock data in production" concern is OUTDATED. The production data pipeline
//! is fully implemented with 1,082 lines of code. Mock data is only active when
//! explicitly compiled with `--features mock-data`. These tests validate the
//! production implementation.

use chrono::Utc;
use ml_training_service::data_config::{
    CacheConfig, DataSourceType, DataValidationConfig, DatabaseConfig, DatabaseTables,
    FeatureExtractionConfig, TimeRangeConfig, TrainingDataSourceConfig,
};
use ml_training_service::data_loader::HistoricalDataLoader;
use sqlx::PgPool;
use std::env;

// ============================================================================
// TEST INFRASTRUCTURE
// ============================================================================

fn get_test_database_url() -> String {
    env::var("TEST_DATABASE_URL").unwrap_or_else(|_| {
        "postgresql://postgres:password@localhost:5432/foxhunt_test".to_string()
    })
}

async fn create_test_pool() -> Result<PgPool, sqlx::Error> {
    let database_url = get_test_database_url();
    sqlx::postgres::PgPoolOptions::new()
        .max_connections(5)
        .connect(&database_url)
        .await
}

/// Setup test database with sample data for normalization and risk calculation tests
async fn setup_comprehensive_test_data(pool: &PgPool) -> Result<(), sqlx::Error> {
    // Clean existing test data
    sqlx::query("DELETE FROM market_events WHERE symbol LIKE 'NORM_%'")
        .execute(pool)
        .await?;
    sqlx::query("DELETE FROM trade_executions WHERE symbol LIKE 'NORM_%'")
        .execute(pool)
        .await?;
    sqlx::query("DELETE FROM order_book_snapshots WHERE symbol LIKE 'NORM_%'")
        .execute(pool)
        .await?;

    // Insert order book snapshots with varying price patterns for risk calculation
    // Pattern 1: Upward trend with volatility (NORM_SYMBOL_1)
    for i in 0..200 {
        let timestamp = Utc::now() - chrono::Duration::minutes(200 - i);
        let base_price = 100.0;
        let trend = i as f64 * 0.05; // Upward trend
        let volatility = (i as f64 * 0.1).sin() * 5.0; // Sine wave volatility
        let price = base_price + trend + volatility;

        sqlx::query(
            r#"
            INSERT INTO order_book_snapshots
            (timestamp, symbol, best_bid, best_ask, bid_volume, ask_volume, spread_bps, mid_price, imbalance, data_quality)
            VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
            "#,
        )
        .bind(timestamp)
        .bind("NORM_SYMBOL_1")
        .bind(rust_decimal::Decimal::from_f64_retain(price - 0.02).unwrap())
        .bind(rust_decimal::Decimal::from_f64_retain(price + 0.02).unwrap())
        .bind(rust_decimal::Decimal::new(1000 + (i * 10) as i64, 0))
        .bind(rust_decimal::Decimal::new(900 + (i * 8) as i64, 0))
        .bind(4 + (i % 10) as i32)
        .bind(rust_decimal::Decimal::from_f64_retain(price).unwrap())
        .bind(0.05 * (i as f64 * 0.1).cos())
        .bind(95 + (i % 6) as i32) // Varying data quality
        .execute(pool)
        .await?;
    }

    // Pattern 2: Downward trend with high volatility (NORM_SYMBOL_2)
    for i in 0..200 {
        let timestamp = Utc::now() - chrono::Duration::minutes(200 - i);
        let base_price = 200.0;
        let trend = -(i as f64 * 0.08); // Downward trend
        let volatility = (i as f64 * 0.15).sin() * 8.0; // Higher volatility
        let price = base_price + trend + volatility;

        sqlx::query(
            r#"
            INSERT INTO order_book_snapshots
            (timestamp, symbol, best_bid, best_ask, bid_volume, ask_volume, spread_bps, mid_price, imbalance, data_quality)
            VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
            "#,
        )
        .bind(timestamp)
        .bind("NORM_SYMBOL_2")
        .bind(rust_decimal::Decimal::from_f64_retain(price - 0.03).unwrap())
        .bind(rust_decimal::Decimal::from_f64_retain(price + 0.03).unwrap())
        .bind(rust_decimal::Decimal::new(2000 + (i * 15) as i64, 0))
        .bind(rust_decimal::Decimal::new(1800 + (i * 12) as i64, 0))
        .bind(6 + (i % 8) as i32)
        .bind(rust_decimal::Decimal::from_f64_retain(price).unwrap())
        .bind(-0.08 * (i as f64 * 0.12).cos())
        .bind(90 + (i % 11) as i32)
        .execute(pool)
        .await?;
    }

    // Insert trade executions with volume patterns
    for i in 0..100 {
        let timestamp = Utc::now() - chrono::Duration::minutes(100 - i);
        let price_1 = 100.0 + (i as f64 * 0.05) + (i as f64 * 0.1).sin() * 5.0;
        let price_2 = 200.0 - (i as f64 * 0.08) + (i as f64 * 0.15).sin() * 8.0;

        // Trades for NORM_SYMBOL_1
        sqlx::query(
            r#"
            INSERT INTO trade_executions
            (timestamp, symbol, price, quantity, side, data_quality)
            VALUES ($1, $2, $3, $4, $5, $6)
            "#,
        )
        .bind(timestamp)
        .bind("NORM_SYMBOL_1")
        .bind(rust_decimal::Decimal::from_f64_retain(price_1).unwrap())
        .bind(rust_decimal::Decimal::new(100 + (i * 5) as i64, 0))
        .bind(if i % 2 == 0 { "buy" } else { "sell" })
        .bind(95 + (i % 6) as i32)
        .execute(pool)
        .await?;

        // Trades for NORM_SYMBOL_2
        sqlx::query(
            r#"
            INSERT INTO trade_executions
            (timestamp, symbol, price, quantity, side, data_quality)
            VALUES ($1, $2, $3, $4, $5, $6)
            "#,
        )
        .bind(timestamp)
        .bind("NORM_SYMBOL_2")
        .bind(rust_decimal::Decimal::from_f64_retain(price_2).unwrap())
        .bind(rust_decimal::Decimal::new(150 + (i * 7) as i64, 0))
        .bind(if i % 3 == 0 { "buy" } else { "sell" })
        .bind(88 + (i % 8) as i32)
        .execute(pool)
        .await?;
    }

    Ok(())
}

fn create_test_config(normalization: &str) -> TrainingDataSourceConfig {
    let database_url = get_test_database_url();

    let mut features = FeatureExtractionConfig::default();
    features.normalization = normalization.to_string();

    TrainingDataSourceConfig {
        source_type: DataSourceType::Historical,
        database: Some(DatabaseConfig {
            connection_url: database_url,
            max_connections: 5,
            query_timeout_secs: 30,
            tables: DatabaseTables::default(),
        }),
        s3: None,
        time_range: TimeRangeConfig {
            start: Some(Utc::now() - chrono::Duration::hours(4)),
            end: Some(Utc::now()),
            duration_days: None,
            train_split: 0.8,
        },
        symbols: vec!["NORM_SYMBOL_1".to_string(), "NORM_SYMBOL_2".to_string()],
        features,
        validation: DataValidationConfig {
            min_samples: 50,
            max_missing_ratio: 0.2,
            enable_outlier_detection: true,
            outlier_threshold: 3.0,
        },
        cache: CacheConfig::default(),
    }
}

// ============================================================================
// NORMALIZATION TESTS
// ============================================================================

#[tokio::test]
#[ignore = "Requires test database setup"]
async fn test_normalization_zscore() {
    let pool = create_test_pool()
        .await
        .expect("Failed to create test pool");
    setup_comprehensive_test_data(&pool)
        .await
        .expect("Failed to setup test data");

    let config = create_test_config("zscore");
    let mut loader = HistoricalDataLoader::new(config)
        .await
        .expect("Failed to create data loader");

    let (training_data, _) = loader
        .load_training_data()
        .await
        .expect("Failed to load training data");

    assert!(
        !training_data.is_empty(),
        "Training data should not be empty"
    );

    // Verify Z-score normalization: values should have ~mean=0, ~std=1
    let (features, _) = &training_data[0];

    // Check that spread_bps_normalized exists (created by normalization)
    assert!(
        features
            .technical_indicators
            .contains_key("spread_bps_normalized"),
        "Z-score normalization should create normalized spread_bps"
    );

    // Collect all imbalance values to check distribution
    let imbalances: Vec<f64> = training_data
        .iter()
        .map(|(f, _)| f.microstructure.imbalance)
        .collect();

    let mean = imbalances.iter().sum::<f64>() / imbalances.len() as f64;
    let variance =
        imbalances.iter().map(|v| (v - mean).powi(2)).sum::<f64>() / imbalances.len() as f64;
    let std_dev = variance.sqrt();

    // After Z-score normalization, mean should be ~0, std should be ~1
    assert!(
        mean.abs() < 0.2,
        "Z-score normalized mean should be near 0, got {}",
        mean
    );
    assert!(
        (std_dev - 1.0).abs() < 0.3,
        "Z-score normalized std dev should be near 1, got {}",
        std_dev
    );

    println!(
        "✅ Z-score normalization: mean={:.4}, std={:.4}",
        mean, std_dev
    );
}

#[tokio::test]
#[ignore = "Requires test database setup"]
async fn test_normalization_minmax() {
    let pool = create_test_pool()
        .await
        .expect("Failed to create test pool");
    setup_comprehensive_test_data(&pool)
        .await
        .expect("Failed to setup test data");

    let config = create_test_config("minmax");
    let mut loader = HistoricalDataLoader::new(config)
        .await
        .expect("Failed to create data loader");

    let (training_data, _) = loader
        .load_training_data()
        .await
        .expect("Failed to load training data");

    assert!(
        !training_data.is_empty(),
        "Training data should not be empty"
    );

    // Verify min-max normalization: values should be in [0, 1]
    let imbalances: Vec<f64> = training_data
        .iter()
        .map(|(f, _)| f.microstructure.imbalance)
        .collect();

    let min_val = imbalances.iter().fold(f64::INFINITY, |a, &b| a.min(b));
    let max_val = imbalances.iter().fold(f64::NEG_INFINITY, |a, &b| a.max(b));

    assert!(
        min_val >= -0.1,
        "Min-max normalized values should be >= 0, got min={}",
        min_val
    );
    assert!(
        max_val <= 1.1,
        "Min-max normalized values should be <= 1, got max={}",
        max_val
    );

    println!(
        "✅ Min-max normalization: range=[{:.4}, {:.4}]",
        min_val, max_val
    );
}

#[tokio::test]
#[ignore = "Requires test database setup"]
async fn test_normalization_robust() {
    let pool = create_test_pool()
        .await
        .expect("Failed to create test pool");
    setup_comprehensive_test_data(&pool)
        .await
        .expect("Failed to setup test data");

    let config = create_test_config("robust");
    let mut loader = HistoricalDataLoader::new(config)
        .await
        .expect("Failed to create data loader");

    let (training_data, _) = loader
        .load_training_data()
        .await
        .expect("Failed to load training data");

    assert!(
        !training_data.is_empty(),
        "Training data should not be empty"
    );

    // Verify robust normalization: uses IQR instead of std dev
    // Should be less sensitive to outliers than Z-score
    let imbalances: Vec<f64> = training_data
        .iter()
        .map(|(f, _)| f.microstructure.imbalance)
        .collect();

    // Calculate IQR for verification
    let mut sorted = imbalances.clone();
    sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));

    let q1_idx = (sorted.len() as f64 * 0.25) as usize;
    let q3_idx = (sorted.len() as f64 * 0.75) as usize;
    let q1 = sorted[q1_idx];
    let q3 = sorted[q3_idx];
    let iqr = q3 - q1;

    assert!(iqr > 0.0, "IQR should be positive for robust normalization");

    println!(
        "✅ Robust normalization: IQR={:.4}, Q1={:.4}, Q3={:.4}",
        iqr, q1, q3
    );
}

// ============================================================================
// DATA LEAKAGE PREVENTION TESTS (Critical Issue from Expert Analysis)
// ============================================================================

#[tokio::test]
#[ignore = "Requires test database setup"]
async fn test_validation_set_normalization_leakage_prevention() {
    // This test validates that normalization parameters are fitted on training data
    // and applied to validation data, preventing data leakage.
    //
    // CRITICAL: Expert analysis identified this as a high-impact issue.
    // Current implementation normalizes validation set independently, causing data leakage.

    let pool = create_test_pool()
        .await
        .expect("Failed to create test pool");
    setup_comprehensive_test_data(&pool)
        .await
        .expect("Failed to setup test data");

    let config = create_test_config("zscore");
    let mut loader = HistoricalDataLoader::new(config)
        .await
        .expect("Failed to create data loader");

    let (training_data, validation_data) = loader
        .load_training_data()
        .await
        .expect("Failed to load training data");

    // VERIFICATION: Check that validation set has different statistical properties
    // This test documents the CURRENT BEHAVIOR (which has data leakage)
    // After fixing the data leakage issue, this test should verify that:
    // 1. Training set mean ≈ 0, std ≈ 1 (fitted on training)
    // 2. Validation set mean != 0, std != 1 (transformed with training params)

    let train_imbalances: Vec<f64> = training_data
        .iter()
        .map(|(f, _)| f.microstructure.imbalance)
        .collect();
    let val_imbalances: Vec<f64> = validation_data
        .iter()
        .map(|(f, _)| f.microstructure.imbalance)
        .collect();

    let train_mean = train_imbalances.iter().sum::<f64>() / train_imbalances.len() as f64;
    let val_mean = val_imbalances.iter().sum::<f64>() / val_imbalances.len() as f64;

    // CURRENT BEHAVIOR: Both are normalized independently (data leakage)
    // EXPECTED AFTER FIX: val_mean should NOT be ~0 (should use training params)
    println!(
        "⚠️  Current behavior (data leakage): train_mean={:.4}, val_mean={:.4}",
        train_mean, val_mean
    );
    println!("⚠️  After fix: val_mean should != 0 (transformed with training params)");

    // Document current behavior for regression testing
    assert!(
        train_mean.abs() < 0.3,
        "Training set should have mean ~0 after normalization"
    );
}

// ============================================================================
// RISK METRICS CALCULATION TESTS
// ============================================================================

#[tokio::test]
#[ignore = "Requires test database setup"]
async fn test_risk_metrics_var_calculation() {
    let pool = create_test_pool()
        .await
        .expect("Failed to create test pool");
    setup_comprehensive_test_data(&pool)
        .await
        .expect("Failed to setup test data");

    let config = create_test_config("none"); // No normalization for raw risk metrics
    let mut loader = HistoricalDataLoader::new(config)
        .await
        .expect("Failed to create data loader");

    let (training_data, _) = loader
        .load_training_data()
        .await
        .expect("Failed to load training data");

    assert!(
        !training_data.is_empty(),
        "Training data should not be empty"
    );

    // Verify VaR calculation
    let (features, _) = &training_data[training_data.len() / 2]; // Middle sample with history

    // VaR should be negative (loss metric)
    assert!(
        features.risk_metrics.var_5pct < 0.0,
        "VaR at 5% should be negative, got {}",
        features.risk_metrics.var_5pct
    );

    // VaR should be reasonable (not too extreme)
    assert!(
        features.risk_metrics.var_5pct > -1.0,
        "VaR should not be extremely negative, got {}",
        features.risk_metrics.var_5pct
    );

    println!(
        "✅ VaR calculation: 5% VaR={:.6}",
        features.risk_metrics.var_5pct
    );
}

#[tokio::test]
#[ignore = "Requires test database setup"]
async fn test_risk_metrics_expected_shortfall() {
    let pool = create_test_pool()
        .await
        .expect("Failed to create test pool");
    setup_comprehensive_test_data(&pool)
        .await
        .expect("Failed to setup test data");

    let config = create_test_config("none");
    let mut loader = HistoricalDataLoader::new(config)
        .await
        .expect("Failed to create data loader");

    let (training_data, _) = loader
        .load_training_data()
        .await
        .expect("Failed to load training data");

    let (features, _) = &training_data[training_data.len() / 2];

    // Expected Shortfall (CVaR) should be <= VaR (more extreme loss)
    assert!(
        features.risk_metrics.expected_shortfall <= features.risk_metrics.var_5pct,
        "Expected Shortfall ({}) should be <= VaR ({})",
        features.risk_metrics.expected_shortfall,
        features.risk_metrics.var_5pct
    );

    println!(
        "✅ Expected Shortfall: ES={:.6}, VaR={:.6}",
        features.risk_metrics.expected_shortfall, features.risk_metrics.var_5pct
    );
}

#[tokio::test]
#[ignore = "Requires test database setup"]
async fn test_risk_metrics_max_drawdown() {
    let pool = create_test_pool()
        .await
        .expect("Failed to create test pool");
    setup_comprehensive_test_data(&pool)
        .await
        .expect("Failed to setup test data");

    let config = create_test_config("none");
    let mut loader = HistoricalDataLoader::new(config)
        .await
        .expect("Failed to create data loader");

    let (training_data, _) = loader
        .load_training_data()
        .await
        .expect("Failed to load training data");

    let (features, _) = &training_data[training_data.len() / 2];

    // Max drawdown should be negative or zero
    assert!(
        features.risk_metrics.max_drawdown <= 0.0,
        "Max drawdown should be negative or zero, got {}",
        features.risk_metrics.max_drawdown
    );

    println!(
        "✅ Max Drawdown: DD={:.6}",
        features.risk_metrics.max_drawdown
    );
}

#[tokio::test]
#[ignore = "Requires test database setup"]
async fn test_risk_metrics_sharpe_ratio() {
    let pool = create_test_pool()
        .await
        .expect("Failed to create test pool");
    setup_comprehensive_test_data(&pool)
        .await
        .expect("Failed to setup test data");

    let config = create_test_config("none");
    let mut loader = HistoricalDataLoader::new(config)
        .await
        .expect("Failed to create data loader");

    let (training_data, _) = loader
        .load_training_data()
        .await
        .expect("Failed to load training data");

    let (features, _) = &training_data[training_data.len() / 2];

    // Sharpe ratio can be any real number, but should be finite
    assert!(
        features.risk_metrics.sharpe_ratio.is_finite(),
        "Sharpe ratio should be finite, got {}",
        features.risk_metrics.sharpe_ratio
    );

    println!(
        "✅ Sharpe Ratio: SR={:.4}",
        features.risk_metrics.sharpe_ratio
    );
}

// ============================================================================
// TECHNICAL INDICATOR TESTS
// ============================================================================

#[tokio::test]
#[ignore = "Requires test database setup"]
async fn test_technical_indicators_presence() {
    let pool = create_test_pool()
        .await
        .expect("Failed to create test pool");
    setup_comprehensive_test_data(&pool)
        .await
        .expect("Failed to setup test data");

    let config = create_test_config("none");
    let mut loader = HistoricalDataLoader::new(config)
        .await
        .expect("Failed to create data loader");

    let (training_data, _) = loader
        .load_training_data()
        .await
        .expect("Failed to load training data");

    let (features, _) = &training_data[50]; // Sample with enough history

    // Verify technical indicators are calculated
    assert!(
        features.technical_indicators.contains_key("spread_bps"),
        "Should have spread_bps indicator"
    );
    assert!(
        features.technical_indicators.contains_key("imbalance"),
        "Should have imbalance indicator"
    );

    // Check for stateful indicators (from TechnicalIndicatorCalculator)
    // These should be present after enough data points
    println!(
        "✅ Technical indicators: {} indicators calculated",
        features.technical_indicators.len()
    );

    for (key, value) in &features.technical_indicators {
        println!("   - {}: {:.6}", key, value);
    }
}

// ============================================================================
// EDGE CASE TESTS
// ============================================================================

#[tokio::test]
#[ignore = "Requires test database setup"]
async fn test_empty_dataset_handling() {
    let pool = create_test_pool()
        .await
        .expect("Failed to create test pool");

    // Clean all test data to create empty dataset
    sqlx::query("DELETE FROM order_book_snapshots WHERE symbol LIKE 'EMPTY_%'")
        .execute(&pool)
        .await
        .expect("Failed to clean test data");

    let mut config = create_test_config("none");
    config.symbols = vec!["EMPTY_SYMBOL".to_string()];
    config.validation.min_samples = 1; // Reduce minimum to test empty case

    let mut loader = HistoricalDataLoader::new(config)
        .await
        .expect("Failed to create data loader");

    let result = loader.load_training_data().await;

    // Should fail with insufficient data error
    assert!(result.is_err(), "Empty dataset should produce error");

    let error_msg = result.unwrap_err().to_string();
    assert!(
        error_msg.contains("Insufficient data") || error_msg.contains("no rows"),
        "Error should mention insufficient data or no rows, got: {}",
        error_msg
    );

    println!("✅ Empty dataset handling: Correctly rejected");
}

#[tokio::test]
#[ignore = "Requires test database setup"]
async fn test_insufficient_samples_validation() {
    let pool = create_test_pool()
        .await
        .expect("Failed to create test pool");
    setup_comprehensive_test_data(&pool)
        .await
        .expect("Failed to setup test data");

    let mut config = create_test_config("none");
    config.validation.min_samples = 100000; // Unrealistically high

    let mut loader = HistoricalDataLoader::new(config)
        .await
        .expect("Failed to create data loader");

    let result = loader.load_training_data().await;

    assert!(result.is_err(), "Should fail with insufficient samples");

    let error_msg = result.unwrap_err().to_string();
    assert!(
        error_msg.contains("Insufficient data"),
        "Error should mention insufficient data, got: {}",
        error_msg
    );

    println!("✅ Insufficient samples validation: Correctly enforced minimum");
}

// ============================================================================
// DATA QUALITY TESTS
// ============================================================================

#[tokio::test]
#[ignore = "Requires test database setup"]
async fn test_data_quality_filtering() {
    let pool = create_test_pool()
        .await
        .expect("Failed to create test pool");
    setup_comprehensive_test_data(&pool)
        .await
        .expect("Failed to setup test data");

    let config = create_test_config("none");
    let mut loader = HistoricalDataLoader::new(config)
        .await
        .expect("Failed to create data loader");

    let (training_data, _) = loader
        .load_training_data()
        .await
        .expect("Failed to load training data");

    // All loaded data should have quality >= 80 (per SQL query filter)
    // We can't directly check this from FinancialFeatures, but the data
    // should be present and valid
    assert!(
        !training_data.is_empty(),
        "Should have loaded high-quality data"
    );

    println!(
        "✅ Data quality filtering: {} samples loaded",
        training_data.len()
    );
}

#[tokio::test]
#[ignore = "Requires test database setup"]
async fn test_train_validation_split_ratio() {
    let pool = create_test_pool()
        .await
        .expect("Failed to create test pool");
    setup_comprehensive_test_data(&pool)
        .await
        .expect("Failed to setup test data");

    let mut config = create_test_config("none");
    config.time_range.train_split = 0.75; // 75/25 split

    let mut loader = HistoricalDataLoader::new(config)
        .await
        .expect("Failed to create data loader");

    let (training_data, validation_data) = loader
        .load_training_data()
        .await
        .expect("Failed to load training data");

    let total = training_data.len() + validation_data.len();
    let actual_split = training_data.len() as f64 / total as f64;

    assert!(
        (actual_split - 0.75).abs() < 0.1,
        "Train split should be approximately 0.75, got {:.4}",
        actual_split
    );

    println!(
        "✅ Train/validation split: {}/{} (target: 75/25)",
        training_data.len(),
        validation_data.len()
    );
}

// ============================================================================
// MICROSTRUCTURE FEATURE TESTS
// ============================================================================

#[tokio::test]
#[ignore = "Requires test database setup"]
async fn test_microstructure_spread_calculation() {
    let pool = create_test_pool()
        .await
        .expect("Failed to create test pool");
    setup_comprehensive_test_data(&pool)
        .await
        .expect("Failed to setup test data");

    let config = create_test_config("none");
    let mut loader = HistoricalDataLoader::new(config)
        .await
        .expect("Failed to create data loader");

    let (training_data, _) = loader
        .load_training_data()
        .await
        .expect("Failed to load training data");

    let (features, _) = &training_data[0];

    // Spread should be positive
    assert!(
        features.microstructure.spread_bps > 0,
        "Spread should be positive, got {}",
        features.microstructure.spread_bps
    );

    // Spread should be reasonable (< 100 bps for normal markets)
    assert!(
        features.microstructure.spread_bps < 100,
        "Spread should be reasonable, got {}",
        features.microstructure.spread_bps
    );

    println!(
        "✅ Spread calculation: {} bps",
        features.microstructure.spread_bps
    );
}

#[tokio::test]
#[ignore = "Requires test database setup"]
async fn test_microstructure_imbalance_bounds() {
    let pool = create_test_pool()
        .await
        .expect("Failed to create test pool");
    setup_comprehensive_test_data(&pool)
        .await
        .expect("Failed to setup test data");

    let config = create_test_config("none");
    let mut loader = HistoricalDataLoader::new(config)
        .await
        .expect("Failed to create data loader");

    let (training_data, _) = loader
        .load_training_data()
        .await
        .expect("Failed to load training data");

    // Check all imbalance values are within bounds
    for (features, _) in &training_data {
        assert!(
            features.microstructure.imbalance.abs() <= 1.1, // Allow small tolerance
            "Imbalance should be in [-1, 1], got {}",
            features.microstructure.imbalance
        );
    }

    println!("✅ Imbalance bounds: All values within [-1, 1]");
}

// ============================================================================
// CONFIGURATION VALIDATION TESTS
// ============================================================================

#[tokio::test]
async fn test_config_validation_invalid_split() {
    let mut config = create_test_config("none");
    config.time_range.train_split = 1.5; // Invalid: > 1.0

    let result = config.validate();

    assert!(result.is_err(), "Should fail validation with invalid split");
    assert!(
        result.unwrap_err().to_string().contains("train_split"),
        "Error should mention train_split"
    );

    println!("✅ Config validation: Invalid split correctly rejected");
}

#[tokio::test]
async fn test_config_validation_missing_database() {
    let mut config = create_test_config("none");
    config.database = None; // Missing required database config

    let result = config.validate();

    assert!(
        result.is_err(),
        "Should fail validation without database config"
    );
    assert!(
        result.unwrap_err().to_string().contains("Database"),
        "Error should mention database requirement"
    );

    println!("✅ Config validation: Missing database correctly rejected");
}