foxhunt/services/ml_training_service/tests/integration_regime_persistence.rs

//! Integration Test: Database Regime Persistence
//!
//! **Agent IMPL-24**: Verifies that regime_states, regime_transitions, and
//! adaptive_strategy_metrics are properly populated during ML training operations.
//!
//! ## Test Coverage
//! - Regime state persistence during feature extraction
//! - Regime transition tracking across multiple bars
//! - Adaptive strategy metrics population
//! - Database schema validation (constraints, indices)
//! - Grafana dashboard compatibility (query validation)
//! - Multi-symbol regime tracking
//!
//! ## Dependencies
//! - Requires PostgreSQL running with migration 045 applied
//! - Uses real DatabasePool (no mocks)
//! - Tests actual SQL queries used by Grafana dashboards

use anyhow::Result;
use chrono::{DateTime, Utc};
use common::database::DatabasePool;
use common::regime_persistence::RegimePersistenceManager;
use sqlx::PgPool;

/// Helper to create test database pool
async fn setup_test_db() -> Result<DatabasePool> {
    let database_url = std::env::var("DATABASE_URL").unwrap_or_else(|_| {
        "postgresql://foxhunt:foxhunt_dev_password@localhost:5432/foxhunt".to_string()
    });

    use common::database::{LocalDatabaseConfig, PoolConfig, PerformanceConfig};
    let config = LocalDatabaseConfig {
        url: database_url,
        pool: PoolConfig {
            max_connections: 5,
            min_connections: 1,
            connect_timeout_ms: 10000,
            acquire_timeout_ms: 10000,
            max_lifetime_seconds: 3600,
            idle_timeout_seconds: 600,
        },
        performance: PerformanceConfig {
            query_timeout_micros: 100_000,
            enable_prewarming: false,
            enable_prepared_statements: true,
            enable_slow_query_logging: false,
            slow_query_threshold_micros: 50_000,
        },
    };

    DatabasePool::new(config).await.map_err(|e| anyhow::anyhow!("Failed to create pool: {}", e))
}

/// Helper to get the underlying PgPool for raw SQL queries
async fn get_pg_pool() -> Result<PgPool> {
    let database_url = std::env::var("DATABASE_URL").unwrap_or_else(|_| {
        "postgresql://foxhunt:foxhunt_dev_password@localhost:5432/foxhunt".to_string()
    });
    Ok(PgPool::connect(&database_url).await?)
}

/// Helper to clear regime tables for clean testing
async fn clear_regime_tables(pg_pool: &PgPool) -> Result<()> {
    // Clear in reverse dependency order  
    sqlx::query!("DELETE FROM adaptive_strategy_metrics").execute(pg_pool).await?;
    sqlx::query!("DELETE FROM regime_transitions").execute(pg_pool).await?;
    sqlx::query!("DELETE FROM regime_states").execute(pg_pool).await?;
    Ok(())
}

/// Generate mock regime features for testing
///
/// # Arguments
/// * `cusum_mean` - CUSUM mean (feature 201)
/// * `cusum_std` - CUSUM std (feature 202)
/// * `adx` - ADX value (feature 211)
/// * `position_mult` - Position multiplier (feature 221)
/// * `stop_mult` - Stop-loss multiplier (feature 222)
fn generate_regime_features(
    cusum_mean: f64,
    cusum_std: f64,
    adx: f64,
    position_mult: f64,
    stop_mult: f64,
) -> [f64; 24] {
    [
        // CUSUM Statistics (features 201-210)
        cusum_mean,
        cusum_std,
        0.5,   // cusum_s_plus
        -0.3,  // cusum_s_minus
        0.0,   // cusum_range
        0.0,   // cusum_crossings
        0.0,   // cusum_max
        0.0,   // cusum_min
        0.0,   // cusum_mean_abs
        0.0,   // cusum_trend
        // ADX & Directional (features 211-215)
        adx,
        0.0,   // plus_di
        0.0,   // minus_di
        0.0,   // adx_slope
        0.0,   // di_diff
        // Transition Probabilities (features 216-220)
        0.7,   // prob_stay
        0.2,   // prob_volatile
        0.1,   // prob_trending
        0.0,   // prob_ranging
        0.0,   // prob_normal
        // Adaptive Metrics (features 221-224)
        position_mult,
        stop_mult,
        0.0,   // regime_sharpe (calculated during backtest)
        0.0,   // risk_utilization (calculated during backtest)
    ]
}

#[tokio::test]
#[ignore = "Requires PostgreSQL with migration 045 applied"]
async fn test_regime_states_persisted_during_training() -> Result<()> {
    let pool = setup_test_db().await?;
    let pg_pool = get_pg_pool().await?;

    // 1. Clear regime tables
    clear_regime_tables(&pg_pool).await?;

    // 2. Create regime persistence manager
    let pool_clone = pool.clone();
    let mut manager = RegimePersistenceManager::new(pool_clone);

    // 3. Simulate ML training with Wave D features for ES.FUT
    let symbols = vec!["ES.FUT", "NQ.FUT"];
    let base_timestamp = Utc::now();

    for (idx, symbol) in symbols.iter().enumerate() {
        let timestamp = base_timestamp + chrono::Duration::seconds(idx as i64 * 60);

        // Generate features for volatile regime (cusum_std > 2.0)
        let features = generate_regime_features(
            0.5,  // cusum_mean
            3.0,  // cusum_std (VOLATILE)
            35.0, // adx
            0.8,  // position_mult (reduced for volatile)
            3.5,  // stop_mult (wider stops)
        );

        manager
            .process_regime_features(symbol, &features, timestamp)
            .await?;
    }

    // 4. Verify regime_states populated
    let state_count = sqlx::query_scalar!(
        r#"SELECT COUNT(*) as "count!" FROM regime_states"#
    )
    .fetch_one(&pg_pool)
    .await?;

    assert!(
        state_count > 0,
        "No regime states persisted! Expected at least 2, got {}",
        state_count
    );
    assert_eq!(
        state_count, 2,
        "Expected 2 regime states (ES.FUT + NQ.FUT), got {}",
        state_count
    );

    // 5. Verify ES.FUT regime state details
    let es_state = pool.get_latest_regime("ES.FUT").await?;
    assert_eq!(es_state.symbol, "ES.FUT");
    assert_eq!(es_state.regime, "Volatile"); // cusum_std=3.0 > 2.0
    assert!(es_state.confidence > 0.0 && es_state.confidence <= 1.0);
    assert!(es_state.cusum_s_plus.is_some());
    assert!(es_state.cusum_s_minus.is_some());
    assert!(es_state.adx.is_some());
    assert_eq!(es_state.adx.unwrap(), 35.0);

    // 6. Verify adaptive_strategy_metrics populated
    let metrics_count = sqlx::query_scalar!(
        r#"SELECT COUNT(*) as "count!" FROM adaptive_strategy_metrics WHERE symbol = 'ES.FUT'"#
    )
    .fetch_one(&pg_pool)
    .await?;

    assert!(
        metrics_count > 0,
        "No adaptive metrics persisted for ES.FUT!"
    );

    // 7. Verify metrics values
    let metrics = sqlx::query!(
        r#"
        SELECT
            position_multiplier,
            stop_loss_multiplier,
            regime
        FROM adaptive_strategy_metrics
        WHERE symbol = 'ES.FUT'
        ORDER BY event_timestamp DESC
        LIMIT 1
        "#
    )
    .fetch_one(&pg_pool)
    .await?;

    assert_eq!(metrics.position_multiplier, 0.8);
    assert_eq!(metrics.stop_loss_multiplier, 3.5);
    assert!(
        metrics.position_multiplier > 0.0 && metrics.position_multiplier <= 2.0,
        "Position multiplier out of range: {}",
        metrics.position_multiplier
    );
    assert!(
        metrics.stop_loss_multiplier >= 1.0 && metrics.stop_loss_multiplier <= 5.0,
        "Stop-loss multiplier out of range: {}",
        metrics.stop_loss_multiplier
    );

    Ok(())
}

#[tokio::test]
#[ignore = "Requires PostgreSQL with migration 045 applied"]
async fn test_regime_transitions_tracked() -> Result<()> {
    let pool = setup_test_db().await?;
    let pg_pool = get_pg_pool().await?;
    clear_regime_tables(&pg_pool).await?;

    let pool_clone = pool.clone();
    let mut manager = RegimePersistenceManager::new(pool_clone);
    let symbol = "TRANSITION.TEST";
    let base_timestamp = Utc::now();

    // First regime: Volatile (3 bars)
    for i in 0..3 {
        let timestamp = base_timestamp + chrono::Duration::seconds(i * 60);
        let features = generate_regime_features(0.5, 3.0, 35.0, 0.8, 3.5);
        manager
            .process_regime_features(symbol, &features, timestamp)
            .await?;
    }

    // Second regime: Trending (2 bars)
    for i in 3..5 {
        let timestamp = base_timestamp + chrono::Duration::seconds(i * 60);
        // Trending: cusum_mean > 1.5 && adx > 25
        let features = generate_regime_features(2.0, 1.0, 30.0, 1.2, 2.0);
        manager
            .process_regime_features(symbol, &features, timestamp)
            .await?;
    }

    // Third regime: Ranging (1 bar)
    let timestamp = base_timestamp + chrono::Duration::seconds(5 * 60);
    // Ranging: adx < 20 && cusum_std < 1.0
    let features = generate_regime_features(0.2, 0.5, 15.0, 1.0, 1.5);
    manager
        .process_regime_features(symbol, &features, timestamp)
        .await?;

    // Verify transitions were recorded
    let transition_count = sqlx::query_scalar!(
        r#"SELECT COUNT(*) as "count!" FROM regime_transitions WHERE symbol = $1"#,
        symbol
    )
    .fetch_one(&pg_pool)
    .await?;

    assert!(
        transition_count >= 2,
        "Expected at least 2 transitions (Volatile->Trending, Trending->Ranging), got {}",
        transition_count
    );

    // Get transition details
    let transitions = pool.get_regime_transitions(symbol, 10).await?;
    assert!(!transitions.is_empty(), "No transitions recorded!");

    // Verify first transition: Volatile -> Trending
    let first_transition = &transitions
        .iter()
        .find(|t| t.from_regime == "Volatile" && t.to_regime == "Trending");
    assert!(
        first_transition.is_some(),
        "Expected Volatile->Trending transition"
    );

    let transition = first_transition.unwrap();
    assert_eq!(transition.duration_bars, Some(3)); // 3 bars in Volatile regime

    // Verify transition matrix query works (used by Grafana)
    let matrix = sqlx::query!(
        r#"
        SELECT
            from_regime,
            to_regime,
            COUNT(*) as "count!"
        FROM regime_transitions
        WHERE symbol = $1
        GROUP BY from_regime, to_regime
        ORDER BY from_regime, to_regime
        "#,
        symbol
    )
    .fetch_all(&pg_pool)
    .await?;

    assert!(!matrix.is_empty(), "No transition matrix data!");
    assert!(matrix.len() >= 2, "Expected at least 2 transition pairs");

    Ok(())
}

#[tokio::test]
#[ignore = "Requires PostgreSQL with migration 045 applied"]
async fn test_grafana_can_query_regime_states() -> Result<()> {
    let pool = setup_test_db().await?;
    let pg_pool = get_pg_pool().await?;
    clear_regime_tables(&pg_pool).await?;

    let mut manager = RegimePersistenceManager::new(pool);
    let symbol = "ES.FUT";
    let base_timestamp = Utc::now();

    // Populate some regime states
    for i in 0..10 {
        let timestamp = base_timestamp + chrono::Duration::seconds(i * 60);
        let features = generate_regime_features(0.5, 3.0, 35.0, 0.8, 3.5);
        manager
            .process_regime_features(symbol, &features, timestamp)
            .await?;
    }

    // Test Grafana-style query: regime distribution by symbol
    let regime_distribution = sqlx::query!(
        r#"
        SELECT
            symbol,
            regime,
            COUNT(*) as "count!",
            AVG(confidence) as "avg_confidence!"
        FROM regime_states
        WHERE event_timestamp >= NOW() - INTERVAL '1 hour'
        GROUP BY symbol, regime
        ORDER BY symbol, regime
        "#
    )
    .fetch_all(&pg_pool)
    .await?;

    assert!(!regime_distribution.is_empty(), "Expected regime data");

    // Verify data structure is correct for Grafana
    for row in regime_distribution {
        assert!(!row.symbol.is_empty());
        assert!(!row.regime.is_empty());
        assert!(row.count > 0);
        assert!(row.avg_confidence >= 0.0 && row.avg_confidence <= 1.0);
    }

    // Test time-series query (used by Grafana timeseries panel)
    let timeseries = sqlx::query!(
        r#"
        SELECT
            event_timestamp,
            regime,
            confidence,
            adx
        FROM regime_states
        WHERE symbol = $1
        ORDER BY event_timestamp DESC
        LIMIT 100
        "#,
        symbol
    )
    .fetch_all(&pg_pool)
    .await?;

    assert!(!timeseries.is_empty(), "Expected time-series data");
    assert!(timeseries.len() <= 100, "Query limit not enforced");

    Ok(())
}

#[tokio::test]
#[ignore = "Requires PostgreSQL with migration 045 applied"]
async fn test_regime_state_has_valid_timestamp() -> Result<()> {
    let pool = setup_test_db().await?;
    let pg_pool = get_pg_pool().await?;
    clear_regime_tables(&pg_pool).await?;

    let pool_clone = pool.clone();
    let mut manager = RegimePersistenceManager::new(pool_clone);
    let symbol = "TIMESTAMP.TEST";
    let timestamp = Utc::now();

    let features = generate_regime_features(0.5, 3.0, 35.0, 0.8, 3.5);
    manager
        .process_regime_features(symbol, &features, timestamp)
        .await?;

    // Verify timestamp is stored correctly
    let state = pool.get_latest_regime(symbol).await?;
    let time_diff = (Utc::now() - state.event_timestamp)
        .num_seconds()
        .abs();

    assert!(
        time_diff < 60,
        "Timestamp should be recent (within 60 seconds), got diff: {} seconds",
        time_diff
    );

    Ok(())
}

#[tokio::test]
#[ignore = "Requires PostgreSQL with migration 045 applied"]
async fn test_confidence_scores_in_valid_range() -> Result<()> {
    let pool = setup_test_db().await?;
    let pg_pool = get_pg_pool().await?;
    clear_regime_tables(&pg_pool).await?;

    let pool_clone = pool.clone();
    let mut manager = RegimePersistenceManager::new(pool_clone);
    let symbol = "CONFIDENCE.TEST";

    // Test various ADX values (confidence = adx / 50.0)
    let test_cases = vec![
        (10.0, 0.2),  // Low ADX
        (25.0, 0.5),  // Medium ADX
        (50.0, 1.0),  // High ADX
        (100.0, 1.0), // Out of range (clamped to 1.0)
    ];

    for (idx, (adx, expected_confidence)) in test_cases.iter().enumerate() {
        let timestamp = Utc::now() + chrono::Duration::seconds(idx as i64 * 60);
        let features = generate_regime_features(0.5, 3.0, *adx, 0.8, 3.5);
        manager
            .process_regime_features(symbol, &features, timestamp)
            .await?;

        let state = pool.get_latest_regime(symbol).await?;
        assert!(
            state.confidence >= 0.0 && state.confidence <= 1.0,
            "Confidence out of range [0.0, 1.0]: {}",
            state.confidence
        );
        assert!(
            (state.confidence - expected_confidence).abs() < 0.01,
            "Expected confidence {}, got {}",
            expected_confidence,
            state.confidence
        );
    }

    Ok(())
}

#[tokio::test]
#[ignore = "Requires PostgreSQL with migration 045 applied"]
async fn test_adaptive_metrics_update_on_backtest() -> Result<()> {
    let pool = setup_test_db().await?;
    let pg_pool = get_pg_pool().await?;
    clear_regime_tables(&pg_pool).await?;

    let pool_clone = pool.clone();
    let mut manager = RegimePersistenceManager::new(pool_clone);
    let symbol = "BACKTEST.TEST";
    let regime = "Trending";
    let timestamp = Utc::now();

    // Initial metrics population
    let features = generate_regime_features(2.0, 1.0, 30.0, 1.2, 2.0);
    manager
        .process_regime_features(symbol, &features, timestamp)
        .await?;

    // Simulate trade execution updates
    manager
        .update_trade_metrics(symbol, regime, timestamp, 1000, true)
        .await?; // +$1000, winner

    manager
        .update_trade_metrics(symbol, regime, timestamp, -500, false)
        .await?; // -$500, loser

    manager
        .update_trade_metrics(symbol, regime, timestamp, 750, true)
        .await?; // +$750, winner

    // Verify performance metrics
    let performance = pool.get_regime_performance(Some(symbol), 24).await?;

    let trending_perf = performance
        .iter()
        .find(|p| p.regime == Some("Trending".to_string()))
        .expect("Expected Trending regime performance");

    assert_eq!(trending_perf.total_trades, Some(3));
    assert_eq!(trending_perf.win_rate, Some(2.0 / 3.0)); // 2 winners out of 3
    assert_eq!(trending_perf.total_pnl, Some(rust_decimal::Decimal::from(1250))); // 1000 - 500 + 750

    Ok(())
}

#[tokio::test]
#[ignore = "Requires PostgreSQL with migration 045 applied"]
async fn test_database_coverage_by_symbol() -> Result<()> {
    let pool = setup_test_db().await?;
    let pg_pool = get_pg_pool().await?;
    clear_regime_tables(&pg_pool).await?;

    let mut manager = RegimePersistenceManager::new(pool);
    let symbols = vec!["ES.FUT", "NQ.FUT", "6E.FUT", "ZN.FUT"];

    // Populate regime states for all symbols
    for (idx, symbol) in symbols.iter().enumerate() {
        let timestamp = Utc::now() + chrono::Duration::seconds(idx as i64 * 60);
        let features = generate_regime_features(0.5, 3.0, 35.0, 0.8, 3.5);
        manager
            .process_regime_features(symbol, &features, timestamp)
            .await?;
    }

    // Verify regime coverage by symbol
    let coverage = sqlx::query!(
        r#"
        SELECT
            symbol,
            COUNT(*) as "count!"
        FROM regime_states
        GROUP BY symbol
        ORDER BY symbol
        "#
    )
    .fetch_all(&pg_pool)
    .await?;

    assert_eq!(coverage.len(), 4, "Expected 4 symbols");

    for (symbol, row) in symbols.iter().zip(coverage.iter()) {
        assert_eq!(row.symbol, *symbol);
        assert!(row.count > 0);
    }

    Ok(())
}

#[tokio::test]
#[ignore = "Requires PostgreSQL with migration 045 applied"]
async fn test_latest_adaptive_metrics_query() -> Result<()> {
    let pool = setup_test_db().await?;
    let pg_pool = get_pg_pool().await?;
    clear_regime_tables(&pg_pool).await?;

    let mut manager = RegimePersistenceManager::new(pool);
    let symbol = "METRICS.LATEST";

    // Insert multiple metrics over time
    for i in 0..5 {
        let timestamp = Utc::now() + chrono::Duration::seconds(i * 60);
        let features = generate_regime_features(2.0, 1.0, 30.0, 1.2 + (i as f64 * 0.1), 2.0);
        manager
            .process_regime_features(symbol, &features, timestamp)
            .await?;
    }

    // Query latest adaptive metrics (Grafana dashboard query)
    let latest_metrics = sqlx::query!(
        r#"
        SELECT
            symbol,
            regime,
            position_multiplier,
            stop_loss_multiplier,
            event_timestamp
        FROM adaptive_strategy_metrics
        WHERE symbol = $1
        ORDER BY event_timestamp DESC
        LIMIT 10
        "#,
        symbol
    )
    .fetch_all(&pg_pool)
    .await?;

    assert!(!latest_metrics.is_empty(), "Expected metrics");
    assert!(latest_metrics.len() <= 10, "Query limit not enforced");

    // Verify ordering (newest first)
    let mut prev_timestamp: Option<DateTime<Utc>> = None;
    for metric in &latest_metrics {
        if let Some(prev) = prev_timestamp {
            assert!(
                metric.event_timestamp <= prev,
                "Metrics not ordered by timestamp DESC"
            );
        }
        prev_timestamp = Some(metric.event_timestamp);
    }

    // Verify latest has highest position multiplier
    let latest = &latest_metrics[0];
    assert_eq!(latest.position_multiplier, 1.6); // 1.2 + 0.4

    Ok(())
}

#[tokio::test]
#[ignore = "Requires PostgreSQL with migration 045 applied"]
async fn test_transition_probability_calculation() -> Result<()> {
    let pool = setup_test_db().await?;
    let pg_pool = get_pg_pool().await?;
    clear_regime_tables(&pg_pool).await?;

    let mut manager = RegimePersistenceManager::new(pool);
    let symbol = "PROB.TEST";
    let base_timestamp = Utc::now();

    // Create transition pattern: Volatile -> Trending -> Volatile -> Trending
    let regime_sequence = vec![
        (0.5, 3.0, 35.0), // Volatile
        (2.0, 1.0, 30.0), // Trending
        (0.5, 3.0, 35.0), // Volatile
        (2.0, 1.0, 30.0), // Trending
        (0.5, 3.0, 35.0), // Volatile
    ];

    for (i, (cusum_mean, cusum_std, adx)) in regime_sequence.iter().enumerate() {
        let timestamp = base_timestamp + chrono::Duration::seconds(i as i64 * 60);
        let features = generate_regime_features(*cusum_mean, *cusum_std, *adx, 1.0, 2.0);
        manager
            .process_regime_features(symbol, &features, timestamp)
            .await?;
    }

    // Verify transition probabilities using SQL function
    let transition_matrix = sqlx::query!(
        r#"
        SELECT
            from_regime,
            to_regime,
            transition_count,
            transition_probability
        FROM get_regime_transition_matrix($1, 24)
        "#,
        symbol
    )
    .fetch_all(&pg_pool)
    .await?;

    assert!(!transition_matrix.is_empty(), "No transition matrix data!");

    // Verify probabilities sum to ~1.0 for each from_regime
    use std::collections::HashMap;
    let mut prob_sums: HashMap<String, f64> = HashMap::new();
    for row in &transition_matrix {
        *prob_sums.entry(row.from_regime.clone().unwrap_or_default()).or_insert(0.0) +=
            row.transition_probability.unwrap_or(0.0);
    }

    for (from_regime, sum) in prob_sums {
        assert!(
            (sum - 1.0).abs() < 0.01,
            "Transition probabilities from {} should sum to 1.0, got {}",
            from_regime,
            sum
        );
    }

    Ok(())
}