foxhunt/services/ml_training_service/tests/validation_pipeline_tests.rs

//! TDD Tests for Model Validation Pipeline
//!
//! Test-Driven Development approach:
//! 1. Write tests FIRST (these tests will FAIL initially)
//! 2. Implement validation_pipeline.rs to make ALL tests GREEN
//! 3. Validation triggers after training completion
//! 4. Uses backtesting service for out-of-sample validation
//! 5. Promotes models to production only if validation passes

use std::collections::HashMap;

use chrono::Utc;
use ml::training_pipeline::{
    FinancialValidationConfig, ModelArchitectureConfig, PerformanceConfig,
    ProductionTrainingConfig, TrainingHyperparameters,
};
use ml_training_service::{
    orchestrator::{JobStatus, TrainingJob},
    validation_pipeline::{
        PromotionDecision, ValidationConfig, ValidationMetrics, ValidationPipeline,
    },
};
use uuid::Uuid;

// ============================================================================
// Test 1: Validation Pipeline Creation
// ============================================================================

#[tokio::test]
async fn test_validation_pipeline_creation() {
    let config = ValidationConfig {
        holdout_data_path: "test_data/real/databento/ml_training".to_string(),
        backtest_duration_days: 30,
        min_sharpe_ratio: 1.5,
        min_win_rate: 0.52,
        max_drawdown: 0.15,
        enable_promotion: true,
    };

    let pipeline = ValidationPipeline::new(config);
    assert!(pipeline.is_ok(), "Pipeline creation should succeed");

    let pipeline = pipeline.unwrap();
    assert_eq!(pipeline.get_config().backtest_duration_days, 30);
    assert_eq!(pipeline.get_config().min_sharpe_ratio, 1.5);
}

// ============================================================================
// Test 2: Training Completion Trigger
// ============================================================================

#[tokio::test]
async fn test_validation_triggered_on_training_complete() {
    let config = ValidationConfig::default();
    let pipeline = ValidationPipeline::new(config).expect("Pipeline creation failed");

    // Simulate training job completion
    let job_id = Uuid::new_v4();
    let training_job = create_completed_training_job(job_id);

    // Validation should trigger automatically
    let result = pipeline.validate_on_completion(&training_job).await;

    assert!(result.is_ok(), "Validation trigger should succeed");
    let validation_result = result.unwrap();
    assert_eq!(validation_result.job_id, job_id);
    assert!(!validation_result.validation_id.is_empty());
}

// ============================================================================
// Test 3: Holdout Dataset Loading
// ============================================================================

#[tokio::test]
async fn test_holdout_dataset_loading() {
    let config = ValidationConfig {
        holdout_data_path:
            "test_data/real/databento/ZN.FUT_ohlcv-1m_2024-01-02_to_2024-01-31.uncompressed.dbn"
                .to_string(),
        ..Default::default()
    };

    let pipeline = ValidationPipeline::new(config).expect("Pipeline creation failed");

    // Load holdout dataset (out-of-sample data)
    let holdout_data = pipeline.load_holdout_dataset().await;

    if let Err(ref e) = holdout_data {
        eprintln!("Holdout data loading error: {:?}", e);
        eprintln!("Error details: {}", e);
        eprintln!("Error source: {:?}", e.source());
    }

    assert!(
        holdout_data.is_ok(),
        "Holdout data loading should succeed: {:?}",
        holdout_data.as_ref().err()
    );
    let data = holdout_data.unwrap();
    assert!(!data.is_empty(), "Holdout dataset should not be empty");
    assert!(
        data.len() >= 100,
        "Holdout dataset should have at least 100 bars for 30-day backtest"
    );
}

// ============================================================================
// Test 4: Backtesting Integration
// ============================================================================

#[tokio::test]
async fn test_backtesting_integration() {
    let config = ValidationConfig {
        holdout_data_path:
            "test_data/real/databento/ZN.FUT_ohlcv-1m_2024-01-02_to_2024-01-31.uncompressed.dbn"
                .to_string(),
        backtest_duration_days: 30,
        ..Default::default()
    };

    let pipeline = ValidationPipeline::new(config).expect("Pipeline creation failed");
    let job_id = Uuid::new_v4();
    let training_job = create_completed_training_job(job_id);

    // Run backtest on holdout data
    let backtest_result = pipeline
        .run_backtest(&training_job, "validation_data_path")
        .await;

    assert!(
        backtest_result.is_ok(),
        "Backtesting integration should succeed"
    );
    let result = backtest_result.unwrap();

    // Verify backtest executed and returned metrics
    assert!(
        result.sharpe_ratio.is_finite(),
        "Sharpe ratio should be finite"
    );
    assert!(result.win_rate >= 0.0 && result.win_rate <= 1.0);
    assert!(result.max_drawdown >= 0.0 && result.max_drawdown <= 1.0);
}

// ============================================================================
// Test 5: Metrics Calculation
// ============================================================================

#[tokio::test]
async fn test_metrics_calculation() {
    let config = ValidationConfig {
        min_sharpe_ratio: 1.5,
        min_win_rate: 0.52,
        max_drawdown: 0.15,
        ..Default::default()
    };

    let pipeline = ValidationPipeline::new(config).expect("Pipeline creation failed");

    // Mock backtest results
    let mock_trades = vec![
        // Winning trades
        (100.0, 101.5), // +1.5% profit
        (101.5, 103.0), // +1.5% profit
        (103.0, 105.0), // +2.0% profit
        // Losing trades
        (105.0, 104.0), // -1.0% loss
        (104.0, 105.5), // +1.5% profit
    ];

    let metrics = pipeline.calculate_metrics(&mock_trades).await;

    assert!(metrics.is_ok(), "Metrics calculation should succeed");
    let result = metrics.unwrap();

    assert_eq!(result.total_trades, 5);
    assert!(result.win_rate > 0.5, "Win rate should be > 50%");
    assert!(result.sharpe_ratio > 0.0, "Sharpe ratio should be positive");
    assert!(
        result.max_drawdown >= 0.0 && result.max_drawdown <= 1.0,
        "Max drawdown should be between 0 and 1"
    );
}

// ============================================================================
// Test 6: Promotion Decision Logic (PASS)
// ============================================================================

#[tokio::test]
async fn test_promotion_decision_pass() {
    let config = ValidationConfig {
        min_sharpe_ratio: 1.5,
        min_win_rate: 0.52,
        max_drawdown: 0.15,
        enable_promotion: true,
        ..Default::default()
    };

    let pipeline = ValidationPipeline::new(config).expect("Pipeline creation failed");

    // Excellent metrics (should PASS)
    let metrics = ValidationMetrics {
        sharpe_ratio: 2.0,  // Above threshold (1.5)
        win_rate: 0.58,     // Above threshold (0.52)
        max_drawdown: 0.10, // Below threshold (0.15)
        total_trades: 150,
        avg_profit_per_trade: 0.015,
        profit_factor: 2.5,
        total_return: 0.45,
    };

    let decision = pipeline.make_promotion_decision(&metrics).await;

    assert!(decision.is_ok(), "Promotion decision should succeed");
    let result = decision.unwrap();

    assert_eq!(result.decision, PromotionDecision::Promote);
    assert!(
        result.reason.contains("PASS"),
        "Reason should indicate validation passed"
    );
}

// ============================================================================
// Test 7: Promotion Decision Logic (FAIL - Low Sharpe)
// ============================================================================

#[tokio::test]
async fn test_promotion_decision_fail_low_sharpe() {
    let config = ValidationConfig {
        min_sharpe_ratio: 1.5,
        min_win_rate: 0.52,
        max_drawdown: 0.15,
        enable_promotion: true,
        ..Default::default()
    };

    let pipeline = ValidationPipeline::new(config).expect("Pipeline creation failed");

    // Poor metrics (LOW SHARPE - should FAIL)
    let metrics = ValidationMetrics {
        sharpe_ratio: 0.8,  // BELOW threshold (1.5) ❌
        win_rate: 0.58,     // Above threshold
        max_drawdown: 0.10, // Below threshold
        total_trades: 150,
        avg_profit_per_trade: 0.005,
        profit_factor: 1.2,
        total_return: 0.15,
    };

    let decision = pipeline.make_promotion_decision(&metrics).await;

    assert!(decision.is_ok(), "Promotion decision should succeed");
    let result = decision.unwrap();

    assert_eq!(result.decision, PromotionDecision::Reject);
    assert!(
        result.reason.contains("Sharpe") || result.reason.contains("sharpe"),
        "Reason should mention Sharpe ratio failure"
    );
}

// ============================================================================
// Test 8: Promotion Decision Logic (FAIL - Low Win Rate)
// ============================================================================

#[tokio::test]
async fn test_promotion_decision_fail_low_win_rate() {
    let config = ValidationConfig {
        min_sharpe_ratio: 1.5,
        min_win_rate: 0.52,
        max_drawdown: 0.15,
        enable_promotion: true,
        ..Default::default()
    };

    let pipeline = ValidationPipeline::new(config).expect("Pipeline creation failed");

    // Poor metrics (LOW WIN RATE - should FAIL)
    let metrics = ValidationMetrics {
        sharpe_ratio: 2.0,  // Above threshold
        win_rate: 0.48,     // BELOW threshold (0.52) ❌
        max_drawdown: 0.10, // Below threshold
        total_trades: 150,
        avg_profit_per_trade: 0.015,
        profit_factor: 1.8,
        total_return: 0.35,
    };

    let decision = pipeline.make_promotion_decision(&metrics).await;

    assert!(decision.is_ok(), "Promotion decision should succeed");
    let result = decision.unwrap();

    assert_eq!(result.decision, PromotionDecision::Reject);
    assert!(
        result.reason.contains("win rate") || result.reason.contains("Win rate"),
        "Reason should mention win rate failure"
    );
}

// ============================================================================
// Test 9: Promotion Decision Logic (FAIL - High Drawdown)
// ============================================================================

#[tokio::test]
async fn test_promotion_decision_fail_high_drawdown() {
    let config = ValidationConfig {
        min_sharpe_ratio: 1.5,
        min_win_rate: 0.52,
        max_drawdown: 0.15,
        enable_promotion: true,
        ..Default::default()
    };

    let pipeline = ValidationPipeline::new(config).expect("Pipeline creation failed");

    // Poor metrics (HIGH DRAWDOWN - should FAIL)
    let metrics = ValidationMetrics {
        sharpe_ratio: 2.0,  // Above threshold
        win_rate: 0.58,     // Above threshold
        max_drawdown: 0.25, // ABOVE threshold (0.15) ❌
        total_trades: 150,
        avg_profit_per_trade: 0.015,
        profit_factor: 2.0,
        total_return: 0.40,
    };

    let decision = pipeline.make_promotion_decision(&metrics).await;

    assert!(decision.is_ok(), "Promotion decision should succeed");
    let result = decision.unwrap();

    assert_eq!(result.decision, PromotionDecision::Reject);
    assert!(
        result.reason.contains("drawdown") || result.reason.contains("Drawdown"),
        "Reason should mention drawdown failure"
    );
}

// ============================================================================
// Test 10: End-to-End Validation Flow
// ============================================================================

#[tokio::test]
async fn test_e2e_validation_flow() {
    let config = ValidationConfig {
        holdout_data_path:
            "test_data/real/databento/ZN.FUT_ohlcv-1m_2024-01-02_to_2024-01-31.uncompressed.dbn"
                .to_string(),
        backtest_duration_days: 30,
        min_sharpe_ratio: 1.0, // Relaxed for testing
        min_win_rate: 0.50,    // Relaxed for testing
        max_drawdown: 0.20,    // Relaxed for testing
        enable_promotion: true,
    };

    let pipeline = ValidationPipeline::new(config).expect("Pipeline creation failed");
    let job_id = Uuid::new_v4();
    let training_job = create_completed_training_job(job_id);

    // Complete validation flow:
    // 1. Trigger validation
    let validation_result = pipeline.validate_on_completion(&training_job).await;

    if let Err(ref e) = validation_result {
        eprintln!("Validation trigger error: {:?}", e);
    }

    assert!(
        validation_result.is_ok(),
        "Validation should trigger: {:?}",
        validation_result.as_ref().err()
    );

    let result = validation_result.unwrap();

    // 2. Verify validation executed
    assert_eq!(result.job_id, job_id);
    assert!(!result.validation_id.is_empty());

    // 3. Check metrics were calculated
    assert!(result.metrics.is_some(), "Metrics should be calculated");
    let metrics = result.metrics.unwrap();
    assert!(metrics.sharpe_ratio.is_finite());
    assert!(metrics.total_trades > 0);

    // 4. Verify promotion decision was made
    assert!(result.promotion_decision.is_some());
    let decision = result.promotion_decision.unwrap();
    assert!(
        matches!(
            decision.decision,
            PromotionDecision::Promote | PromotionDecision::Reject
        ),
        "Decision should be Promote or Reject"
    );
}

// ============================================================================
// Helper Functions
// ============================================================================

/// Create a completed training job for testing
fn create_completed_training_job(job_id: Uuid) -> TrainingJob {
    let config = ProductionTrainingConfig {
        model_config: ModelArchitectureConfig {
            input_dim: 50,
            output_dim: 1,
            hidden_dims: vec![128, 64],
            dropout_rate: 0.1,
            activation: "relu".to_string(),
            batch_norm: false,
            residual_connections: false,
        },
        training_params: TrainingHyperparameters {
            learning_rate: 0.001,
            batch_size: 32,
            max_epochs: 10,
            patience: 3,
            validation_split: 0.2,
            l2_regularization: 0.0001,
            lr_decay_factor: 0.1,
            lr_decay_patience: 5,
        },
        safety_config: ml::safety::MLSafetyConfig::default(),
        gradient_config: ml::safety::GradientSafetyConfig::default(),
        financial_config: FinancialValidationConfig {
            max_prediction_multiple: 2.0,
            min_prediction_confidence: 0.6,
            validate_position_sizing: true,
            max_position_fraction: 0.25,
            min_sharpe_threshold: 0.5,
        },
        performance_config: PerformanceConfig {
            device_preference: "cpu".to_string(),
            max_memory_bytes: 8 * 1024 * 1024 * 1024,
            num_workers: 4,
            gradient_accumulation_steps: 1,
        },
    };

    let mut job = TrainingJob::new(
        "DQN".to_string(),
        config,
        "Test training job for validation".to_string(),
        HashMap::new(),
    );

    // Set job as completed with mock training results
    job.id = job_id;
    job.status = JobStatus::Completed;
    job.started_at = Some(Utc::now() - chrono::Duration::hours(2));
    job.completed_at = Some(Utc::now());
    job.progress_percentage = 100.0;
    job.current_epoch = 10;
    job.total_epochs = 10;
    job.model_artifact_path = Some(format!("models/{}.bin", job_id));

    // Mock training metrics
    job.metrics.insert("final_train_loss".to_string(), 0.015);
    job.metrics.insert("final_val_loss".to_string(), 0.018);
    job.metrics.insert("accuracy".to_string(), 0.85);

    job
}