foxhunt/crates/ml/tests/tft_hyperopt_real_metrics_test.rs

#![allow(unexpected_cfgs)]
#![cfg(feature = "__ml_integration_tests")]
#![allow(
    clippy::assertions_on_constants,
    clippy::assertions_on_result_states,
    clippy::clone_on_copy,
    clippy::decimal_literal_representation,
    clippy::doc_markdown,
    clippy::empty_line_after_doc_comments,
    clippy::field_reassign_with_default,
    clippy::get_unwrap,
    clippy::identity_op,
    clippy::inconsistent_digit_grouping,
    clippy::indexing_slicing,
    clippy::integer_division,
    clippy::len_zero,
    clippy::let_underscore_must_use,
    clippy::manual_div_ceil,
    clippy::manual_let_else,
    clippy::manual_range_contains,
    clippy::modulo_arithmetic,
    clippy::needless_range_loop,
    clippy::non_ascii_literal,
    clippy::redundant_clone,
    clippy::shadow_reuse,
    clippy::shadow_same,
    clippy::shadow_unrelated,
    clippy::single_match_else,
    clippy::str_to_string,
    clippy::string_slice,
    clippy::tests_outside_test_module,
    clippy::too_many_lines,
    clippy::unnecessary_wraps,
    clippy::unseparated_literal_suffix,
    clippy::use_debug,
    clippy::useless_vec,
    clippy::wildcard_enum_match_arm,
    clippy::else_if_without_else,
    clippy::expect_used,
    clippy::missing_const_for_fn,
    clippy::similar_names,
    clippy::type_complexity,
    clippy::collapsible_else_if,
    clippy::doc_lazy_continuation,
    clippy::items_after_test_module,
    clippy::map_clone,
    clippy::multiple_unsafe_ops_per_block,
    clippy::unwrap_or_default,
    clippy::assign_op_pattern,
    clippy::needless_borrow,
    clippy::println_empty_string,
    clippy::unnecessary_cast,
    clippy::used_underscore_binding,
    clippy::create_dir,
    clippy::implicit_saturating_sub,
    clippy::exit,
    clippy::expect_fun_call,
    clippy::too_many_arguments,
    clippy::unnecessary_map_or,
    clippy::unwrap_used,
    dead_code,
    unused_imports,
    unused_variables,
    clippy::cloned_ref_to_slice_refs,
    clippy::neg_multiply,
    clippy::while_let_loop,
    clippy::bool_assert_comparison,
    clippy::excessive_precision,
    clippy::trivially_copy_pass_by_ref,
    clippy::op_ref,
    clippy::redundant_closure,
    clippy::unnecessary_lazy_evaluations,
    clippy::if_then_some_else_none,
    clippy::unnecessary_to_owned,
    clippy::single_component_path_imports,
)]
//! TFT Hyperopt Real Metrics Validation Test
//!
//! This test explicitly proves that TFT hyperopt returns REAL metrics,
//! not hardcoded mock values.
//!
//! ## Test Strategy
//!
//! 1. Run 3 training trials with different hyperparameters
//! 2. Verify metrics vary between trials (not constant 0.5, 0.4, 0.3)
//! 3. Verify loss decreases during training (learning occurs)
//! 4. Verify metrics are in reasonable ranges

use anyhow::Result;
use ml::hyperopt::adapters::tft::{TFTParams, TFTTrainer};
use ml::hyperopt::traits::HyperparameterOptimizable;
use tracing::info;

#[test]
fn test_tft_metrics_are_not_mock() -> Result<()> {
    info!("TFT Hyperopt Real Metrics Validation");

    // Use absolute path from workspace root
    let workspace_root = std::env::current_dir()?.to_string_lossy().to_string();
    let parquet_file = if workspace_root.ends_with("foxhunt") {
        "test_data/ES_FUT_small.parquet"
    } else {
        "../test_data/ES_FUT_small.parquet"
    };

    info!(workspace = %workspace_root, dataset = parquet_file, "Environment");

    let mut trainer = TFTTrainer::new(parquet_file, 3).expect("Failed to create TFT trainer");

    // Test 3 different hyperparameter configurations
    let test_configs = vec![
        (
            "Config 1: Small model (LR=1e-3)",
            TFTParams {
                learning_rate: 1e-3,
                batch_size: 16,
                hidden_size: 128,
                num_heads: 4,
                dropout: 0.1,
                signal_high_bps: 10.0,
                signal_low_bps: 5.0,
            },
        ),
        (
            "Config 2: Medium model (LR=5e-4)",
            TFTParams {
                learning_rate: 5e-4,
                batch_size: 32,
                hidden_size: 256,
                num_heads: 8,
                dropout: 0.15,
                signal_high_bps: 10.0,
                signal_low_bps: 5.0,
            },
        ),
        (
            "Config 3: Large model (LR=1e-4)",
            TFTParams {
                learning_rate: 1e-4,
                batch_size: 32,
                hidden_size: 256,
                num_heads: 8,
                dropout: 0.2,
                signal_high_bps: 10.0,
                signal_low_bps: 5.0,
            },
        ),
    ];

    let mut val_losses = Vec::new();
    let mut train_losses = Vec::new();
    let mut rmse_values = Vec::new();

    info!("Running 3 training trials with different hyperparameters");

    for (i, (name, params)) in test_configs.iter().enumerate() {
        info!(
            trial = i + 1,
            name,
            learning_rate = params.learning_rate,
            batch_size = params.batch_size,
            hidden_size = params.hidden_size,
            num_heads = params.num_heads,
            dropout = params.dropout,
            "Trial config"
        );

        let metrics = trainer
            .train_with_params(params.clone())
            .expect("Training failed");

        info!(
            val_loss = metrics.val_loss,
            train_loss = metrics.train_loss,
            val_rmse = metrics.val_rmse,
            epochs = metrics.epochs_completed,
            "Trial training completed"
        );

        val_losses.push(metrics.val_loss);
        train_losses.push(metrics.train_loss);
        rmse_values.push(metrics.val_rmse);
    }

    info!("Validation Results");

    // Test 1: Verify metrics are not hardcoded mock values
    info!("Test 1: Checking for hardcoded mock values");

    let mock_val_loss = 0.5;
    let mock_train_loss = 0.4;
    let mock_rmse = 0.3;

    for (i, loss) in val_losses.iter().enumerate() {
        assert_ne!(
            *loss,
            mock_val_loss,
            "Trial {} val_loss is hardcoded to 0.5 (MOCK!)",
            i + 1
        );
    }

    for (i, loss) in train_losses.iter().enumerate() {
        assert_ne!(
            *loss,
            mock_train_loss,
            "Trial {} train_loss is hardcoded to 0.4 (MOCK!)",
            i + 1
        );
    }

    for (i, rmse) in rmse_values.iter().enumerate() {
        assert_ne!(
            *rmse,
            mock_rmse,
            "Trial {} RMSE is hardcoded to 0.3 (MOCK!)",
            i + 1
        );
    }

    info!("No hardcoded mock values detected");

    // Test 2: Verify metrics vary between trials
    info!("Test 2: Checking metric variation between trials");

    let all_val_losses_same = val_losses.windows(2).all(|w| (w[0] - w[1]).abs() < 1e-10);
    assert!(
        !all_val_losses_same,
        "Validation losses are constant across trials: {:?} (MOCK!)",
        val_losses
    );

    for (i, loss) in val_losses.iter().enumerate() {
        info!(trial = i + 1, val_loss = loss, "Validation loss per trial");
    }

    // Test 3: Verify metrics are in reasonable ranges
    info!("Test 3: Checking metric ranges");

    for (i, loss) in val_losses.iter().enumerate() {
        assert!(
            loss.is_finite(),
            "Trial {} val_loss is not finite: {}",
            i + 1,
            loss
        );
        assert!(
            *loss > 0.0,
            "Trial {} val_loss is negative or zero: {}",
            i + 1,
            loss
        );
        assert!(
            *loss < 100.0,
            "Trial {} val_loss is unreasonably high: {} (model not learning?)",
            i + 1,
            loss
        );
    }

    info!("All metrics are finite and in reasonable ranges");

    // Test 4: Verify training occurred (not skipped)
    info!("Test 4: Checking training completion");

    let expected_epochs = 3;
    for metrics in [
        val_losses.clone(),
        train_losses.clone(),
        rmse_values.clone(),
    ] {
        assert_eq!(
            metrics.len(),
            expected_epochs,
            "Not all trials completed (expected {}, got {})",
            expected_epochs,
            metrics.len()
        );
    }

    info!(trials = test_configs.len(), epochs_each = expected_epochs, "All trials completed");

    // Test 5: Verify train loss < 1000 (not penalty value)
    info!("Test 5: Checking for penalty values");

    let penalty_value = 1000.0;
    for (i, loss) in val_losses.iter().enumerate() {
        assert_ne!(
            *loss,
            penalty_value,
            "Trial {} val_loss is penalty value (invalid config?)",
            i + 1
        );
    }

    info!("No penalty values detected (all configs valid)");
    info!(?val_losses, ?train_losses, ?rmse_values, "All tests passed - metrics are real");

    Ok(())
}

#[test]
#[ignore] // Run with: cargo test test_tft_learning_occurs -- --ignored --nocapture
fn test_tft_learning_occurs() -> Result<()> {
    info!("TFT Learning Validation Test");

    // Use absolute path from workspace root
    let workspace_root = std::env::current_dir()?.to_string_lossy().to_string();
    let parquet_file = if workspace_root.ends_with("foxhunt") {
        "test_data/ES_FUT_small.parquet"
    } else {
        "../test_data/ES_FUT_small.parquet"
    };

    info!(dataset = parquet_file, epochs = 10, batch_size = 16, hidden_size = 256, "Learning validation config");

    let mut trainer = TFTTrainer::new(parquet_file, 10).expect("Failed to create TFT trainer");

    let params = TFTParams {
        learning_rate: 1e-3,
        batch_size: 16,
        hidden_size: 256,
        num_heads: 8,
        dropout: 0.1,
        signal_high_bps: 10.0,
        signal_low_bps: 5.0,
    };

    info!("Training for 10 epochs");
    let metrics = trainer.train_with_params(params)?;

    info!(val_loss = metrics.val_loss, train_loss = metrics.train_loss, val_rmse = metrics.val_rmse, "Training completed");

    // Verify training loss < validation loss (typical for good training)
    if metrics.train_loss < metrics.val_loss {
        info!("Train loss < Val loss (model learning, no overfitting)");
    } else {
        info!("Train loss >= Val loss (may indicate underfitting or small dataset)");
    }

    // Verify loss is reasonable for financial data
    assert!(
        metrics.val_loss < 10.0,
        "Validation loss too high: {} (model not learning)",
        metrics.val_loss
    );

    info!("Learning validation PASSED");

    Ok(())
}

#[test]
fn test_tft_invalid_config_penalty() -> Result<()> {
    info!("TFT Invalid Config Penalty Test");

    // Use absolute path from workspace root
    let workspace_root = std::env::current_dir()?.to_string_lossy().to_string();
    let parquet_file = if workspace_root.ends_with("foxhunt") {
        "test_data/ES_FUT_small.parquet"
    } else {
        "../test_data/ES_FUT_small.parquet"
    };

    let mut trainer = TFTTrainer::new(parquet_file, 3)?;

    // Test invalid config: hidden_size not divisible by num_heads
    let invalid_params = TFTParams {
        learning_rate: 1e-3,
        batch_size: 16,
        hidden_size: 127, // Not divisible by 8
        num_heads: 8,
        dropout: 0.1,
        signal_high_bps: 10.0,
        signal_low_bps: 5.0,
    };

    info!(hidden_size = 127, num_heads = 8, "Testing invalid config (hidden_size not divisible by num_heads)");

    let metrics = trainer.train_with_params(invalid_params)?;

    info!(val_loss = metrics.val_loss, train_loss = metrics.train_loss, rmse = metrics.val_rmse, "Invalid config result");

    // Verify penalty value is applied (1000.0)
    assert_eq!(
        metrics.val_loss, 1000.0,
        "Should return penalty value for invalid config"
    );
    assert_eq!(
        metrics.train_loss, 1000.0,
        "Should return penalty value for invalid config"
    );
    assert_eq!(
        metrics.val_rmse, 1000.0,
        "Should return penalty value for invalid config"
    );
    assert_eq!(
        metrics.epochs_completed, 0,
        "Should not complete any epochs for invalid config"
    );

    info!("Invalid config penalty PASSED (penalty value 1000.0 is for invalid configs, not a mock metric)");

    Ok(())
}