foxhunt/crates/ml/tests/real_data_pipeline_test.rs

#![allow(
    clippy::assertions_on_constants,
    clippy::assertions_on_result_states,
    clippy::clone_on_copy,
    clippy::decimal_literal_representation,
    clippy::doc_markdown,
    clippy::empty_line_after_doc_comments,
    clippy::field_reassign_with_default,
    clippy::get_unwrap,
    clippy::identity_op,
    clippy::inconsistent_digit_grouping,
    clippy::indexing_slicing,
    clippy::integer_division,
    clippy::len_zero,
    clippy::let_underscore_must_use,
    clippy::manual_div_ceil,
    clippy::manual_let_else,
    clippy::manual_range_contains,
    clippy::modulo_arithmetic,
    clippy::needless_range_loop,
    clippy::non_ascii_literal,
    clippy::redundant_clone,
    clippy::shadow_reuse,
    clippy::shadow_same,
    clippy::shadow_unrelated,
    clippy::single_match_else,
    clippy::str_to_string,
    clippy::string_slice,
    clippy::tests_outside_test_module,
    clippy::too_many_lines,
    clippy::unnecessary_wraps,
    clippy::unseparated_literal_suffix,
    clippy::use_debug,
    clippy::useless_vec,
    clippy::wildcard_enum_match_arm,
    clippy::else_if_without_else,
    clippy::expect_used,
    clippy::missing_const_for_fn,
    clippy::similar_names,
    clippy::type_complexity,
    clippy::collapsible_else_if,
    clippy::doc_lazy_continuation,
    clippy::items_after_test_module,
    clippy::map_clone,
    clippy::multiple_unsafe_ops_per_block,
    clippy::unwrap_or_default,
    clippy::assign_op_pattern,
    clippy::needless_borrow,
    clippy::println_empty_string,
    clippy::unnecessary_cast,
    clippy::used_underscore_binding,
    clippy::create_dir,
    clippy::implicit_saturating_sub,
    clippy::exit,
    clippy::expect_fun_call,
    clippy::too_many_arguments,
    clippy::unnecessary_map_or,
    clippy::unwrap_used,
    dead_code,
    unused_imports,
    unused_variables,
    clippy::cloned_ref_to_slice_refs,
    clippy::neg_multiply,
    clippy::while_let_loop,
    clippy::bool_assert_comparison,
    clippy::excessive_precision,
    clippy::trivially_copy_pass_by_ref,
    clippy::op_ref,
    clippy::redundant_closure,
    clippy::unnecessary_lazy_evaluations,
    clippy::if_then_some_else_none,
    clippy::unnecessary_to_owned,
    clippy::single_component_path_imports,
)]
//! Integration tests for the real-data training pipeline.
//!
//! Validates the full pipeline with synthetic data (no Databento API needed):
//! generate bars -> extract features -> walk-forward split -> normalization -> verify dimensions.

#![allow(unused_crate_dependencies)]

use chrono::{Datelike, NaiveDate, NaiveTime, TimeZone, Utc, Weekday};
use ml::features::extraction::extract_ml_features;
use ml::types::OHLCVBar;
use ml::walk_forward::{generate_walk_forward_windows, NormStats, WalkForwardConfig};

/// Expected feature dimension from `extract_ml_features` (40 base + 2 regime: ADX, CUSUM).
const EXPECTED_FEATURE_DIM: usize = 42;

// ---------------------------------------------------------------------------
// Synthetic bar generator
// ---------------------------------------------------------------------------

/// Generate realistic-ish synthetic OHLCV bars starting from `start_date`.
///
/// - Skips weekends (Sat/Sun).
/// - Produces `bars_per_day` intraday bars per trading day (390 = 6.5h * 60min).
/// - Base price ~4500 with small drift and intraday noise.
/// - Volume varies with approximate U-shaped intraday pattern.
fn generate_synthetic_bars(start_date: NaiveDate, num_calendar_days: u32, bars_per_day: u32) -> Vec<OHLCVBar> {
    let mut bars = Vec::new();
    let mut price = 4500.0_f64;
    let mut current = start_date;

    for _day_offset in 0..num_calendar_days {
        let weekday = current.weekday();
        if weekday == Weekday::Sat || weekday == Weekday::Sun {
            current = current.succ_opt().unwrap_or(current);
            continue;
        }

        // Small daily drift (-0.05% to +0.05%)
        let daily_drift = ((_day_offset as f64 * 0.7123).sin()) * 0.0005;
        price *= 1.0 + daily_drift;

        for minute in 0..bars_per_day {
            // Intraday time: market opens at 09:30, each bar is 1 minute
            let total_minutes = 9 * 60 + 30 + minute;
            let hour = total_minutes / 60;
            let min = total_minutes % 60;

            // Clamp hour/minute to valid ranges
            let hour_clamped = hour.min(23);
            let min_clamped = min.min(59);

            let time = NaiveTime::from_hms_opt(hour_clamped, min_clamped, 0)
                .unwrap_or_default();
            let dt = current.and_time(time);
            let timestamp = Utc.from_utc_datetime(&dt);

            // Small intrabar noise for realistic OHLCV
            let noise_factor = ((bars.len() as f64 * 1.3217).sin()) * 0.001;
            let open = price * (1.0 + noise_factor);
            let close = price * (1.0 + noise_factor * 0.8 + daily_drift * 0.001);

            // high is always >= max(open, close), low <= min(open, close)
            let bar_max = open.max(close);
            let bar_min = open.min(close);
            let high = bar_max + bar_max.abs() * 0.0005;
            let low = bar_min - bar_min.abs() * 0.0005;

            // U-shaped volume: higher at open/close, lower midday
            let session_pct = minute as f64 / bars_per_day.max(1) as f64;
            let u_shape = (session_pct - 0.5).powi(2) * 4.0 + 0.5;
            let volume = 50_000.0 * u_shape + 10_000.0;

            bars.push(OHLCVBar {
                timestamp,
                open,
                high,
                low,
                close,
                volume,
            });

            // Evolve price slightly per bar
            let bar_drift = ((bars.len() as f64 * 0.4567).sin()) * 0.0001;
            price *= 1.0 + bar_drift;
        }

        current = current.succ_opt().unwrap_or(current);
    }

    bars
}

// ---------------------------------------------------------------------------
// Test 1: Feature extraction from synthetic bars
// ---------------------------------------------------------------------------

#[test]
fn test_pipeline_features_extract_from_synthetic() {
    // Generate 90 calendar days of synthetic bars, 390 per trading day
    let start = NaiveDate::from_ymd_opt(2024, 3, 1).unwrap_or_default();
    let bars = generate_synthetic_bars(start, 90, 390);

    // Verify we generated a reasonable number of bars (~63 trading days * 390)
    assert!(
        bars.len() > 20_000,
        "Expected >20k bars for 90 days, got {}",
        bars.len()
    );

    // Extract features
    let features = match extract_ml_features(&bars) {
        Ok(f) => f,
        Err(e) => {
            assert!(false, "Feature extraction failed: {e}");
            return; // unreachable, satisfies type checker
        }
    };

    // Features should be non-empty (bars - warmup period of 50)
    assert!(
        !features.is_empty(),
        "Feature extraction returned empty vector"
    );
    assert!(
        features.len() > 19_000,
        "Expected >19k feature vectors, got {}",
        features.len()
    );

    // Each feature vector must be 51-dimensional
    for (i, fv) in features.iter().enumerate() {
        assert_eq!(
            fv.len(),
            EXPECTED_FEATURE_DIM,
            "Feature vector at index {} has {} dims, expected {}",
            i,
            fv.len(),
            EXPECTED_FEATURE_DIM
        );

        // No NaN or Inf in any feature
        for (j, &val) in fv.iter().enumerate() {
            assert!(
                val.is_finite(),
                "NaN/Inf at feature[{}][{}] = {}",
                i,
                j,
                val
            );
        }
    }
}

// ---------------------------------------------------------------------------
// Test 2: Walk-forward windows with feature normalization
// ---------------------------------------------------------------------------

#[test]
fn test_pipeline_walk_forward_with_features() {
    // Generate 730 calendar days (~24 months) of synthetic bars
    // Use fewer bars per day (20) to keep test runtime reasonable
    let start = NaiveDate::from_ymd_opt(2022, 3, 1).unwrap_or_default();
    let bars = generate_synthetic_bars(start, 730, 20);

    assert!(
        !bars.is_empty(),
        "Bar generation produced no bars"
    );

    // Create walk-forward windows with default config (12/3/3/3 months)
    let config = WalkForwardConfig::default();
    let windows = generate_walk_forward_windows(&bars, &config);

    assert!(
        windows.len() >= 2,
        "Expected at least 2 walk-forward windows, got {}",
        windows.len()
    );

    let mut validated_folds = 0_usize;

    for window in &windows {
        // Extract features from training data
        let train_features = if window.train.len() >= EXPECTED_FEATURE_DIM {
            extract_ml_features(&window.train).ok()
        } else {
            None
        };

        // Extract features from validation data
        let val_features = if window.val.len() >= EXPECTED_FEATURE_DIM {
            extract_ml_features(&window.val).ok()
        } else {
            None
        };

        // Training features should exist and be non-empty
        let train_feats = match train_features {
            Some(ref f) if !f.is_empty() => f,
            _ => continue, // Skip folds with insufficient data
        };

        validated_folds += 1;

        // Compute NormStats from training data ONLY
        let stats = NormStats::from_features(train_feats);

        // Normalize training data
        let normalized_train = stats.normalize_batch(train_feats);

        // Verify normalized training mean is approximately 0
        if !normalized_train.is_empty() {
            let n = normalized_train.len() as f64;
            // Compute per-feature mean of normalized training data
            let mut mean_per_feature = vec![0.0_f64; EXPECTED_FEATURE_DIM];
            for fv in &normalized_train {
                for (m, &v) in mean_per_feature.iter_mut().zip(fv.iter()) {
                    *m += v;
                }
            }
            for m in &mut mean_per_feature {
                *m /= n;
            }

            // Each feature's mean should be close to 0
            for (feat_idx, &m) in mean_per_feature.iter().enumerate() {
                assert!(
                    m.abs() < 0.1,
                    "Fold {}: normalized training mean for feature {} = {}, expected ~0",
                    window.fold,
                    feat_idx,
                    m
                );
            }
        }

        // Normalize validation data using training stats
        if let Some(ref val_feats) = val_features {
            if !val_feats.is_empty() {
                let normalized_val = stats.normalize_batch(val_feats);
                assert!(
                    !normalized_val.is_empty(),
                    "Fold {}: normalized val features should be non-empty",
                    window.fold
                );

                // Verify all normalized values are finite
                for (i, fv) in normalized_val.iter().enumerate() {
                    for (j, &val) in fv.iter().enumerate() {
                        assert!(
                            val.is_finite(),
                            "Fold {}: NaN/Inf in normalized val[{}][{}] = {}",
                            window.fold,
                            i,
                            j,
                            val
                        );
                    }
                }
            }
        }
    }

    assert!(
        validated_folds >= 1,
        "No walk-forward folds were actually validated (all skipped due to insufficient data)"
    );
}

// ---------------------------------------------------------------------------
// Test 3: No look-ahead bias
// ---------------------------------------------------------------------------

#[test]
fn test_pipeline_no_lookahead_bias() {
    // Generate 730 calendar days (~24 months) of synthetic bars
    let start = NaiveDate::from_ymd_opt(2022, 3, 1).unwrap_or_default();
    let bars = generate_synthetic_bars(start, 730, 20);

    assert!(!bars.is_empty(), "Bar generation produced no bars");

    let config = WalkForwardConfig::default();
    let windows = generate_walk_forward_windows(&bars, &config);

    assert!(
        !windows.is_empty(),
        "Expected at least 1 walk-forward window"
    );

    for window in &windows {
        // --- Train timestamps must all be < val start ---
        // Get the earliest val timestamp
        let val_start_ts = window
            .val
            .first()
            .map(|b| b.timestamp);

        if let Some(val_start) = val_start_ts {
            // Every training bar must have timestamp < val_start
            for (i, bar) in window.train.iter().enumerate() {
                assert!(
                    bar.timestamp < val_start,
                    "Fold {}: look-ahead leak! train bar {} timestamp ({}) >= val start ({})",
                    window.fold,
                    i,
                    bar.timestamp,
                    val_start
                );
            }
        }

        // --- Val timestamps must all be < test start ---
        let test_start_ts = window
            .test
            .first()
            .map(|b| b.timestamp);

        if let Some(test_start) = test_start_ts {
            for (i, bar) in window.val.iter().enumerate() {
                assert!(
                    bar.timestamp < test_start,
                    "Fold {}: look-ahead leak! val bar {} timestamp ({}) >= test start ({})",
                    window.fold,
                    i,
                    bar.timestamp,
                    test_start
                );
            }
        }

        // --- Also verify via date boundaries on the window struct ---
        assert!(
            window.train_end <= window.val_end,
            "Fold {}: train_end ({}) > val_end ({})",
            window.fold,
            window.train_end,
            window.val_end
        );
        assert!(
            window.val_end <= window.test_end,
            "Fold {}: val_end ({}) > test_end ({})",
            window.fold,
            window.val_end,
            window.test_end
        );
    }
}