foxhunt/ml/tests/tft_hyperopt_real_metrics_test.rs

//! TFT Hyperopt Real Metrics Validation Test
//!
//! This test explicitly proves that TFT hyperopt returns REAL metrics,
//! not hardcoded mock values.
//!
//! ## Test Strategy
//!
//! 1. Run 3 training trials with different hyperparameters
//! 2. Verify metrics vary between trials (not constant 0.5, 0.4, 0.3)
//! 3. Verify loss decreases during training (learning occurs)
//! 4. Verify metrics are in reasonable ranges

use anyhow::Result;
use ml::hyperopt::adapters::tft::{TFTParams, TFTTrainer};
use ml::hyperopt::traits::HyperparameterOptimizable;

#[test]
fn test_tft_metrics_are_not_mock() -> Result<()> {
    println!("╔═══════════════════════════════════════════════════════════╗");
    println!("║  TFT Hyperopt Real Metrics Validation                    ║");
    println!("╚═══════════════════════════════════════════════════════════╝");
    println!();

    // Use absolute path from workspace root
    let workspace_root = std::env::current_dir()?.to_string_lossy().to_string();
    let parquet_file = if workspace_root.ends_with("foxhunt") {
        "test_data/ES_FUT_small.parquet"
    } else {
        "../test_data/ES_FUT_small.parquet"
    };

    println!("Workspace: {}", workspace_root);
    println!("Dataset: {}", parquet_file);
    println!();

    let mut trainer = TFTTrainer::new(parquet_file, 3).expect("Failed to create TFT trainer");

    // Test 3 different hyperparameter configurations
    let test_configs = vec![
        (
            "Config 1: Small model (LR=1e-3)",
            TFTParams {
                learning_rate: 1e-3,
                batch_size: 16,
                hidden_size: 128,
                num_heads: 4,
                dropout: 0.1,
            },
        ),
        (
            "Config 2: Medium model (LR=5e-4)",
            TFTParams {
                learning_rate: 5e-4,
                batch_size: 32,
                hidden_size: 256,
                num_heads: 8,
                dropout: 0.15,
            },
        ),
        (
            "Config 3: Large model (LR=1e-4)",
            TFTParams {
                learning_rate: 1e-4,
                batch_size: 32,
                hidden_size: 256,
                num_heads: 8,
                dropout: 0.2,
            },
        ),
    ];

    let mut val_losses = Vec::new();
    let mut train_losses = Vec::new();
    let mut rmse_values = Vec::new();

    println!("Running 3 training trials with different hyperparameters...");
    println!();

    for (i, (name, params)) in test_configs.iter().enumerate() {
        println!("Trial {}/3: {}", i + 1, name);
        println!("  • Learning rate: {:.6}", params.learning_rate);
        println!("  • Batch size: {}", params.batch_size);
        println!("  • Hidden size: {}", params.hidden_size);
        println!("  • Num heads: {}", params.num_heads);
        println!("  • Dropout: {:.3}", params.dropout);

        let metrics = trainer
            .train_with_params(params.clone())
            .expect("Training failed");

        println!("  ✓ Training completed:");
        println!("    - Validation loss: {:.6}", metrics.val_loss);
        println!("    - Training loss: {:.6}", metrics.train_loss);
        println!("    - Validation RMSE: {:.4}", metrics.val_rmse);
        println!("    - Epochs: {}", metrics.epochs_completed);
        println!();

        val_losses.push(metrics.val_loss);
        train_losses.push(metrics.train_loss);
        rmse_values.push(metrics.val_rmse);
    }

    println!("╔═══════════════════════════════════════════════════════════╗");
    println!("║  Validation Results                                       ║");
    println!("╚═══════════════════════════════════════════════════════════╝");
    println!();

    // Test 1: Verify metrics are not hardcoded mock values
    println!("Test 1: Checking for hardcoded mock values...");

    let mock_val_loss = 0.5;
    let mock_train_loss = 0.4;
    let mock_rmse = 0.3;

    for (i, loss) in val_losses.iter().enumerate() {
        assert_ne!(
            *loss,
            mock_val_loss,
            "Trial {} val_loss is hardcoded to 0.5 (MOCK!)",
            i + 1
        );
    }

    for (i, loss) in train_losses.iter().enumerate() {
        assert_ne!(
            *loss,
            mock_train_loss,
            "Trial {} train_loss is hardcoded to 0.4 (MOCK!)",
            i + 1
        );
    }

    for (i, rmse) in rmse_values.iter().enumerate() {
        assert_ne!(
            *rmse,
            mock_rmse,
            "Trial {} RMSE is hardcoded to 0.3 (MOCK!)",
            i + 1
        );
    }

    println!("  ✓ No hardcoded mock values detected");
    println!();

    // Test 2: Verify metrics vary between trials
    println!("Test 2: Checking metric variation between trials...");

    let all_val_losses_same = val_losses.windows(2).all(|w| (w[0] - w[1]).abs() < 1e-10);
    assert!(
        !all_val_losses_same,
        "Validation losses are constant across trials: {:?} (MOCK!)",
        val_losses
    );

    println!("  ✓ Validation losses vary between trials:");
    for (i, loss) in val_losses.iter().enumerate() {
        println!("    Trial {}: {:.6}", i + 1, loss);
    }
    println!();

    // Test 3: Verify metrics are in reasonable ranges
    println!("Test 3: Checking metric ranges...");

    for (i, loss) in val_losses.iter().enumerate() {
        assert!(
            loss.is_finite(),
            "Trial {} val_loss is not finite: {}",
            i + 1,
            loss
        );
        assert!(
            *loss > 0.0,
            "Trial {} val_loss is negative or zero: {}",
            i + 1,
            loss
        );
        assert!(
            *loss < 100.0,
            "Trial {} val_loss is unreasonably high: {} (model not learning?)",
            i + 1,
            loss
        );
    }

    println!("  ✓ All metrics are finite and in reasonable ranges");
    println!();

    // Test 4: Verify training occurred (not skipped)
    println!("Test 4: Checking training completion...");

    let expected_epochs = 3;
    for metrics in [
        val_losses.clone(),
        train_losses.clone(),
        rmse_values.clone(),
    ] {
        assert_eq!(
            metrics.len(),
            expected_epochs,
            "Not all trials completed (expected {}, got {})",
            expected_epochs,
            metrics.len()
        );
    }

    println!(
        "  ✓ All {} trials completed {} epochs each",
        test_configs.len(),
        expected_epochs
    );
    println!();

    // Test 5: Verify train loss < 1000 (not penalty value)
    println!("Test 5: Checking for penalty values...");

    let penalty_value = 1000.0;
    for (i, loss) in val_losses.iter().enumerate() {
        assert_ne!(
            *loss,
            penalty_value,
            "Trial {} val_loss is penalty value (invalid config?)",
            i + 1
        );
    }

    println!("  ✓ No penalty values detected (all configs valid)");
    println!();

    println!("╔═══════════════════════════════════════════════════════════╗");
    println!("║  ✅ ALL TESTS PASSED - METRICS ARE REAL                  ║");
    println!("╚═══════════════════════════════════════════════════════════╝");
    println!();
    println!("Summary:");
    println!("  • Validation losses: {:?}", val_losses);
    println!("  • Training losses: {:?}", train_losses);
    println!("  • RMSE values: {:?}", rmse_values);
    println!();
    println!("Conclusion:");
    println!("  TFT hyperopt adapter returns REAL metrics from actual training,");
    println!("  not hardcoded mock values (0.5, 0.4, 0.3).");

    Ok(())
}

#[test]
#[ignore] // Run with: cargo test test_tft_learning_occurs -- --ignored --nocapture
fn test_tft_learning_occurs() -> Result<()> {
    println!("╔═══════════════════════════════════════════════════════════╗");
    println!("║  TFT Learning Validation Test                             ║");
    println!("╚═══════════════════════════════════════════════════════════╝");
    println!();

    // Use absolute path from workspace root
    let workspace_root = std::env::current_dir()?.to_string_lossy().to_string();
    let parquet_file = if workspace_root.ends_with("foxhunt") {
        "test_data/ES_FUT_small.parquet"
    } else {
        "../test_data/ES_FUT_small.parquet"
    };

    println!("Dataset: {}", parquet_file);
    println!("Configuration:");
    println!("  • Epochs: 10 (sufficient for learning)");
    println!("  • Batch size: 16");
    println!("  • Hidden size: 256");
    println!();

    let mut trainer = TFTTrainer::new(parquet_file, 10).expect("Failed to create TFT trainer");

    let params = TFTParams {
        learning_rate: 1e-3,
        batch_size: 16,
        hidden_size: 256,
        num_heads: 8,
        dropout: 0.1,
    };

    println!("Training for 10 epochs...");
    let metrics = trainer.train_with_params(params)?;

    println!("Training completed:");
    println!("  • Validation loss: {:.6}", metrics.val_loss);
    println!("  • Training loss: {:.6}", metrics.train_loss);
    println!("  • Validation RMSE: {:.4}", metrics.val_rmse);
    println!();

    // Verify training loss < validation loss (typical for good training)
    if metrics.train_loss < metrics.val_loss {
        println!("✓ Train loss < Val loss (model learning, no overfitting)");
    } else {
        println!("⚠ Train loss >= Val loss (may indicate underfitting or small dataset)");
    }

    // Verify loss is reasonable for financial data
    assert!(
        metrics.val_loss < 10.0,
        "Validation loss too high: {} (model not learning)",
        metrics.val_loss
    );

    println!();
    println!("✅ Learning validation PASSED");

    Ok(())
}

#[test]
fn test_tft_invalid_config_penalty() -> Result<()> {
    println!("╔═══════════════════════════════════════════════════════════╗");
    println!("║  TFT Invalid Config Penalty Test                          ║");
    println!("╚═══════════════════════════════════════════════════════════╝");
    println!();

    // Use absolute path from workspace root
    let workspace_root = std::env::current_dir()?.to_string_lossy().to_string();
    let parquet_file = if workspace_root.ends_with("foxhunt") {
        "test_data/ES_FUT_small.parquet"
    } else {
        "../test_data/ES_FUT_small.parquet"
    };

    let mut trainer = TFTTrainer::new(parquet_file, 3)?;

    // Test invalid config: hidden_size not divisible by num_heads
    let invalid_params = TFTParams {
        learning_rate: 1e-3,
        batch_size: 16,
        hidden_size: 127, // Not divisible by 8
        num_heads: 8,
        dropout: 0.1,
    };

    println!("Testing invalid config:");
    println!("  • Hidden size: 127 (not divisible by num_heads=8)");
    println!();

    let metrics = trainer.train_with_params(invalid_params)?;

    println!("Result:");
    println!("  • Validation loss: {:.1}", metrics.val_loss);
    println!("  • Training loss: {:.1}", metrics.train_loss);
    println!("  • RMSE: {:.1}", metrics.val_rmse);
    println!();

    // Verify penalty value is applied (1000.0)
    assert_eq!(
        metrics.val_loss, 1000.0,
        "Should return penalty value for invalid config"
    );
    assert_eq!(
        metrics.train_loss, 1000.0,
        "Should return penalty value for invalid config"
    );
    assert_eq!(
        metrics.val_rmse, 1000.0,
        "Should return penalty value for invalid config"
    );
    assert_eq!(
        metrics.epochs_completed, 0,
        "Should not complete any epochs for invalid config"
    );

    println!("✅ Invalid config penalty PASSED");
    println!("   (Penalty value 1000.0 is NOT a mock metric - it's for invalid configs)");

    Ok(())
}