foxhunt/data/examples/training_pipeline_demo.rs.disabled

//! Training Data Pipeline Comprehensive Demo
//!
//! This example demonstrates the complete training data pipeline for ML models including:
//! - Multi-source data ingestion (Databento, Benzinga, IB TWS, ICMarkets)
//! - Real-time and historical data collection
//! - Feature engineering with technical indicators and microstructure features
//! - Data validation and quality control
//! - Efficient storage and dataset management
//! - TLOB processing for order book analytics
//! - Portfolio performance tracking

use chrono::{DateTime, Duration, Utc};
use rust_decimal::Decimal;
use data::features::{MicrostructureAnalyzer, TechnicalIndicators, TemporalFeatures};
use data::training_pipeline::{
    BenzingaConfig, CompressionAlgorithm, CompressionConfig, DataSourcesConfig,
    DataValidationConfig, DatabentConfig, FeatureEngineeringConfig, HistoricalDataConfig,
    MACDConfig, MicrostructureConfig, MissingDataHandling, OutlierDetectionMethod,
    ProcessingConfig, RegimeDetectionConfig, StorageFormat, TLOBConfig, TechnicalIndicatorsConfig,
    TemporalConfig, TrainingDataPipeline, TrainingPipelineConfig, TrainingStorageConfig,
};
use common::MarketDataEvent;
use common::QuoteEvent;
use common::TradeEvent;
use data::validation::{DataValidator, ValidationResult};
use std::collections::HashMap;
use std::path::PathBuf;
use tokio::time::{sleep, timeout};
use tracing::{debug, error, info, warn};
use tracing_subscriber;

#[tokio::main]
async fn main() -> anyhow::Result<()> {
    // Initialize logging
    tracing_subscriber::fmt()
        .with_env_filter("info,data=debug")
        .with_target(false)
        .init();

    info!("🚀 Starting Training Data Pipeline Demo");

    // Demo configuration
    let config = create_demo_config();

    // Demo 1: Data ingestion and validation
    demo_data_ingestion_and_validation(&config).await?;

    // Demo 2: Feature engineering
    demo_feature_engineering().await?;

    // Demo 3: Complete pipeline workflow
    demo_complete_pipeline(config).await?;

    info!("✅ Training Data Pipeline Demo completed successfully");
    Ok(())
}

/// Create demonstration configuration
fn create_demo_config() -> TrainingPipelineConfig {
    info!("📋 Creating training pipeline configuration");

    TrainingPipelineConfig {
        sources: DataSourcesConfig {
            databento: Some(DatabentConfig {
                api_key: std::env::var("DATABENTO_API_KEY").unwrap_or_else(|_| {
                    warn!("DATABENTO_API_KEY not set, using demo key");
                    "demo_key".to_string()
                }),
                symbols: vec![
                    "AAPL".to_string(),
                    "MSFT".to_string(),
                    "TSLA".to_string(),
                    "SPY".to_string(),
                    "QQQ".to_string(),
                ],
                data_types: vec![
                    "trades".to_string(),
                    "quotes".to_string(),
                    "ohlcv".to_string(),
                ],
                rate_limit: 100,
                timeout: 30,
            }),
            benzinga: Some(BenzingaConfig {
                api_key: std::env::var("BENZINGA_API_KEY").unwrap_or_else(|_| {
                    warn!("BENZINGA_API_KEY not set, using demo key");
                    "demo_key".to_string()
                }),
                symbols: vec![
                    "AAPL".to_string(),
                    "MSFT".to_string(),
                    "TSLA".to_string(),
                    "SPY".to_string(),
                    "QQQ".to_string(),
                ],
                data_types: vec![
                    "news".to_string(),
                    "earnings".to_string(),
                    "guidance".to_string(),
                ],
                rate_limit: 60,
                timeout: 30,
            }),
            interactive_brokers: Some(data::training_pipeline::IBDataConfig {
                host: "127.0.0.1".to_string(),
                port: 7497,
                client_id: 1001,
                symbols: vec!["AAPL".to_string(), "MSFT".to_string()],
                enable_level2: true,
            }),
            icmarkets: Some(data::training_pipeline::ICMarketsDataConfig {
                host: "fix-demo.icmarkets.com".to_string(),
                port: 9880,
                username: std::env::var("ICMARKETS_USERNAME").unwrap_or_default(),
                password: std::env::var("ICMARKETS_PASSWORD").unwrap_or_default(),
                symbols: vec!["EURUSD".to_string(), "GBPUSD".to_string()],
            }),
            enable_realtime: true,
            historical: HistoricalDataConfig {
                start_date: Utc::now() - Duration::days(7),
                end_date: Utc::now(),
                timeframe: "1min".to_string(),
                max_concurrent_requests: 5,
                batch_size: 1000,
            },
        },
        features: FeatureEngineeringConfig {
            technical_indicators: TechnicalIndicatorsConfig {
                ma_periods: vec![5, 10, 20, 50, 100, 200],
                rsi_periods: vec![14, 21, 30],
                bollinger_periods: vec![20, 50],
                macd: MACDConfig {
                    fast_period: 12,
                    slow_period: 26,
                    signal_period: 9,
                },
                volume_indicators: true,
            },
            microstructure: MicrostructureConfig {
                bid_ask_spread: true,
                volume_imbalance: true,
                price_impact: true,
                kyle_lambda: true,
                amihud_ratio: true,
                roll_spread: true,
            },
            tlob: TLOBConfig {
                book_depth: 10,
                time_window: 300, // 5 minutes
                volume_buckets: vec![100.0, 500.0, 1000.0, 5000.0, 10000.0],
                order_flow_analytics: true,
                imbalance_calculations: true,
            },
            temporal: TemporalConfig {
                time_of_day: true,
                day_of_week: true,
                market_session: true,
                holiday_effects: true,
                expiration_effects: true,
            },
            regime_detection: RegimeDetectionConfig {
                volatility_regime: true,
                trend_regime: true,
                volume_regime: true,
                correlation_regime: true,
                lookback_period: 100,
            },
        },
        validation: DataValidationConfig {
            enable_price_validation: true,
            enable_volume_validation: true,
            price_threshold: 0.01,
            volume_threshold: 100.0,
            price_validation: true,
            max_price_change: 15.0, // 15% max price change
            volume_validation: true,
            max_volume_change: 2000.0, // 2000% max volume change
            timestamp_validation: true,
            max_timestamp_drift: 5000, // 5 seconds
            outlier_detection: true,
            outlier_method: OutlierDetectionMethod::ZScore,
            missing_data_handling: MissingDataHandling::Skip,
        },
        storage: TrainingStorageConfig {
            base_directory: PathBuf::from("./demo_training_data"),
            format: StorageFormat::Parquet,
            compression: CompressionConfig {
                algorithm: CompressionAlgorithm::ZSTD,
                level: 3,
                enabled: true,
            },
            versioning: data::training_pipeline::VersioningConfig {
                enabled: true,
                version_format: "v%Y%m%d_%H%M%S".to_string(),
                keep_versions: 5,
            },
            retention: data::training_pipeline::RetentionConfig {
                retention_days: 90,
                auto_cleanup: true,
                cleanup_schedule: "0 2 * * *".to_string(),
            },
        },
        processing: ProcessingConfig {
            worker_threads: num_cpus::get(),
            batch_size: 1000,
            buffer_size: 10000,
            timeout: 300,
            parallel_processing: true,
        },
    }
}

/// Demonstrate data ingestion and validation
async fn demo_data_ingestion_and_validation(config: &TrainingPipelineConfig) -> anyhow::Result<()> {
    info!("📊 === Data Ingestion and Validation Demo ===");

    // Create data validator
    let mut validator = DataValidator::new(config.validation.clone())?;
    info!("✅ Data validator initialized");

    // Create sample market data events
    let sample_events = create_sample_market_data();
    info!(
        "📈 Created {} sample market data events",
        sample_events.len()
    );

    // Validate each event
    let mut validation_results = Vec::new();
    for (i, event) in sample_events.iter().enumerate() {
        let result = validator.validate_event(event).await;

        info!(
            "Event {}: {} - Valid: {}, Errors: {}, Warnings: {}, Quality: {:.2}",
            i + 1,
            event.symbol(),
            result.is_valid,
            result.errors.len(),
            result.warnings.len(),
            result.quality_score
        );

        if !result.errors.is_empty() {
            for error in &result.errors {
                warn!(
                    "  ❌ Error: {} - {}",
                    error.field.as_deref().unwrap_or("unknown"),
                    error.message
                );
            }
        }

        if !result.warnings.is_empty() {
            for warning in &result.warnings {
                debug!(
                    "  ⚠️ Warning: {} - {}",
                    warning.field.as_deref().unwrap_or("unknown"),
                    warning.message
                );
            }
        }

        validation_results.push(result);
    }

    // Batch validation demo
    info!("🔄 Demonstrating batch validation");
    let batch_results = validator.validate_batch(&sample_events).await;
    let valid_count = batch_results.iter().filter(|r| r.is_valid).count();
    let avg_quality =
        batch_results.iter().map(|r| r.quality_score).sum::<f64>() / batch_results.len() as f64;

    info!(
        "📊 Batch validation results: {}/{} valid events, average quality: {:.2}",
        valid_count,
        batch_results.len(),
        avg_quality
    );

    Ok(())
}

/// Demonstrate feature engineering
async fn demo_feature_engineering() -> anyhow::Result<()> {
    info!("🔧 === Feature Engineering Demo ===");

    // Technical indicators demo
    demo_technical_indicators().await?;

    // Microstructure features demo
    demo_microstructure_features().await?;

    // Temporal features demo
    demo_temporal_features().await?;

    Ok(())
}

/// Demo technical indicators
async fn demo_technical_indicators() -> anyhow::Result<()> {
    info!("📈 Technical Indicators Demo");

    let config = TechnicalIndicatorsConfig {
        ma_periods: vec![10, 20, 50],
        rsi_periods: vec![14],
        bollinger_periods: vec![20],
        macd: MACDConfig {
            fast_period: 12,
            slow_period: 26,
            signal_period: 9,
        },
        volume_indicators: true,
    };

    let mut indicators = TechnicalIndicators::new(config);

    // Create sample price data
    let symbol = "AAPL";
    let mut base_price = 150.0;

    for i in 0..100 {
        // Simulate price movement
        base_price += (i as f64 * 0.1).sin() * 2.0 + (rand::random::<f64>() - 0.5) * 1.0;

        let price_point = data::features::PricePoint {
            timestamp: Utc::now() - Duration::minutes(100 - i),
            open: base_price - 0.5,
            high: base_price + 1.0,
            low: base_price - 1.0,
            close: base_price,
        };

        indicators.update_price(symbol, price_point);
    }

    // Calculate features
    let features = indicators.calculate_features(symbol);
    info!(
        "📊 Calculated {} technical indicator features",
        features.len()
    );

    for (name, value) in features.iter().take(10) {
        info!("  {} = {:.4}", name, value);
    }

    Ok(())
}

/// Demo microstructure features
async fn demo_microstructure_features() -> anyhow::Result<()> {
    info!("🏗️ Microstructure Features Demo");

    let config = MicrostructureConfig {
        bid_ask_spread: true,
        volume_imbalance: true,
        price_impact: true,
        kyle_lambda: false, // Requires more data
        amihud_ratio: true,
        roll_spread: true,
    };

    let mut analyzer = MicrostructureAnalyzer::new(config);
    let symbol = "AAPL";

    // Add sample quote data
    for i in 0..50 {
        let base_price = 150.0 + (i as f64 * 0.05);
        let quote = data::features::QuoteData {
            timestamp: Utc::now() - Duration::seconds(50 - i),
            bid: base_price - 0.01,
            ask: base_price + 0.01,
            bid_size: 1000.0 + (i as f64 * 10.0),
            ask_size: 800.0 + (i as f64 * 8.0),
        };
        analyzer.update_quote(symbol, quote);
    }

    // Add sample trade data
    for i in 0..30 {
        let trade = data::features::TradeData {
            timestamp: Utc::now() - Duration::seconds(30 - i),
            price: 150.0 + (i as f64 * 0.02),
            size: 100.0 + (i as f64 * 5.0),
            direction: if i % 2 == 0 {
                data::features::TradeDirection::Buy
            } else {
                data::features::TradeDirection::Sell
            },
        };
        analyzer.update_trade(symbol, trade);
    }

    // Calculate microstructure features
    let features = analyzer.calculate_features(symbol);
    info!("📊 Calculated {} microstructure features", features.len());

    for (name, value) in features.iter() {
        info!("  {} = {:.6}", name, value);
    }

    Ok(())
}

/// Demo temporal features
async fn demo_temporal_features() -> anyhow::Result<()> {
    info!("⏰ Temporal Features Demo");

    let timestamps = vec![
        Utc::now(),
        Utc::now() - Duration::hours(1),
        Utc::now() - Duration::days(1),
        Utc::now() - Duration::days(7),
    ];

    for (i, timestamp) in timestamps.iter().enumerate() {
        let features = TemporalFeatures::extract_features(*timestamp);
        info!("Timestamp {}: {} features", i + 1, features.len());

        for (name, value) in features.iter().take(8) {
            info!("  {} = {:.2}", name, value);
        }
    }

    Ok(())
}

/// Demonstrate complete pipeline workflow
async fn demo_complete_pipeline(config: TrainingPipelineConfig) -> anyhow::Result<()> {
    info!("🔄 === Complete Pipeline Workflow Demo ===");

    // Initialize training pipeline
    info!("🚀 Initializing training data pipeline");
    let mut pipeline = TrainingDataPipeline::new(config).await?;
    info!("✅ Pipeline initialized successfully");

    // Start real-time data collection (simulated)
    info!("📡 Starting real-time data collection (simulated)");
    // Note: In production, this would start actual data connections
    // pipeline.start_realtime_collection().await?;

    // Collect historical data
    info!("📚 Collecting historical data");
    let dataset_id = pipeline.collect_historical_data().await?;
    info!("✅ Historical data collected: {}", dataset_id);

    // Process features
    info!("🔧 Processing features");
    let processed_dataset_id = pipeline.process_features(&dataset_id).await?;
    info!("✅ Features processed: {}", processed_dataset_id);

    // Get processing statistics
    let stats = pipeline.get_stats().await;
    info!("📊 Processing Statistics:");
    info!("  Total records: {}", stats.total_records);
    info!("  Errors: {}", stats.errors);
    info!("  Validation failures: {}", stats.validation_failures);
    info!("  Start time: {}", stats.start_time);
    info!("  Last update: {}", stats.last_update);

    // Simulate processing some real-time data
    info!("⚡ Simulating real-time data processing");
    simulate_realtime_processing().await?;

    Ok(())
}

/// Create sample market data events for testing
fn create_sample_market_data() -> Vec<MarketDataEvent> {
    let mut events = Vec::new();
    let symbols = vec!["AAPL", "MSFT", "TSLA"];

    for (i, symbol) in symbols.iter().enumerate() {
        // Create trade events
        for j in 0..5 {
            let price = 100.0 + (i as f64 * 50.0) + (j as f64 * 2.0);
            let trade = TradeEvent {
                symbol: symbol.to_string(),
                timestamp: Utc::now() - Duration::seconds((j * 10) as i64),
                price: Decimal::try_from(price).unwrap(),
                size: Decimal::try_from(100.0 + (j as f64 * 50.0)).unwrap(),
                trade_id: Some(format!("{}_{}", symbol, j)),
                exchange: Some("NASDAQ".to_string()),
                conditions: vec!["regular".to_string()],
            };
            events.push(MarketDataEvent::Trade(trade));
        }

        // Create quote events
        for j in 0..3 {
            let price = 100.0 + (i as f64 * 50.0) + (j as f64 * 2.0);
            let quote = QuoteEvent {
                symbol: symbol.to_string(),
                timestamp: Utc::now() - Duration::seconds((j * 15) as i64),
                bid: Some(Decimal::try_from(price - 0.01).unwrap()),
                ask: Some(Decimal::try_from(price + 0.01).unwrap()),
                bid_size: Some(Decimal::try_from(1000.0).unwrap()),
                ask_size: Some(Decimal::try_from(800.0).unwrap()),
                exchange: Some("NASDAQ".to_string()),
            };
            events.push(MarketDataEvent::Quote(quote));
        }
    }

    // Add some problematic data for validation testing
    events.push(MarketDataEvent::Trade(TradeEvent {
        symbol: "TEST".to_string(),
        timestamp: Utc::now(),
        price: Decimal::try_from(-10.0).unwrap(), // Invalid negative price
        size: Decimal::try_from(100.0).unwrap(),
        trade_id: Some("invalid_price".to_string()),
        exchange: Some("TEST".to_string()),
        conditions: vec!["invalid".to_string()],
    }));

    events.push(MarketDataEvent::Quote(QuoteEvent {
        symbol: "TEST2".to_string(),
        timestamp: Utc::now(),
        bid: Some(Decimal::try_from(100.0).unwrap()),
        ask: Some(Decimal::try_from(99.0).unwrap()), // Invalid: bid > ask
        bid_size: Some(Decimal::try_from(1000.0).unwrap()),
        ask_size: Some(Decimal::try_from(800.0).unwrap()),
        exchange: Some("TEST".to_string()),
    }));

    events
}

/// Simulate real-time data processing
async fn simulate_realtime_processing() -> anyhow::Result<()> {
    info!("⚡ Simulating 10 seconds of real-time data processing");

    for i in 0..10 {
        // Simulate receiving market data
        let price = 150.0 + (i as f64 * 0.1);
        info!("📈 Received market data: AAPL @ ${:.2}", price);

        // Simulate feature calculation
        sleep(std::time::Duration::from_millis(100)).await;
        debug!("🔧 Calculated features for tick {}", i + 1);

        // Simulate validation
        debug!("✅ Validated data for tick {}", i + 1);

        sleep(std::time::Duration::from_millis(900)).await;
    }

    info!("✅ Real-time simulation completed");
    Ok(())
}

/// Configuration examples for different ML models
#[allow(dead_code)]
fn create_model_specific_configs() -> HashMap<String, TrainingPipelineConfig> {
    let mut configs = HashMap::new();

    // TLOB Transformer configuration - optimized for Databento market data
    let mut tlob_config = create_demo_config();
    tlob_config.features.tlob.book_depth = 20; // Deeper order book
    tlob_config.features.tlob.time_window = 60; // 1-minute windows
    tlob_config.features.microstructure.kyle_lambda = true;
    // Enhanced for Databento high-frequency data
    if let Some(ref mut databento) = tlob_config.sources.databento {
        databento.rate_limit = 200; // Higher rate for order book data
        databento.data_types = vec![
            "trades".to_string(),
            "quotes".to_string(),
            "depth".to_string(),
        ];
    }
    configs.insert("tlob_transformer".to_string(), tlob_config);

    // MAMBA configuration (for sequential modeling) - combines market + news data
    let mut mamba_config = create_demo_config();
    mamba_config.features.regime_detection.lookback_period = 500; // Longer lookback
    mamba_config.features.temporal.market_session = true;
    // Optimize Benzinga for news sentiment features
    if let Some(ref mut benzinga) = mamba_config.sources.benzinga {
        benzinga.data_types.push("analyst_ratings".to_string());
        benzinga.data_types.push("sec_filings".to_string());
    }
    configs.insert("mamba".to_string(), mamba_config);

    // DQN configuration (for reinforcement learning)
    let mut dqn_config = create_demo_config();
    dqn_config.features.technical_indicators.ma_periods = vec![5, 10, 20]; // Shorter periods
    dqn_config.processing.batch_size = 128; // RL batch size
    configs.insert("dqn".to_string(), dqn_config);

    // TFT configuration (for time series forecasting) - enhanced with news events
    let mut tft_config = create_demo_config();
    tft_config.features.temporal.holiday_effects = true;
    tft_config.features.temporal.expiration_effects = true;
    // Include corporate events from Benzinga
    if let Some(ref mut benzinga) = tft_config.sources.benzinga {
        benzinga.data_types.push("corporate_actions".to_string());
        benzinga.data_types.push("dividends".to_string());
    }
    configs.insert("tft".to_string(), tft_config);

    configs
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_demo_config_creation() {
        let config = create_demo_config();
        assert!(config.sources.databento.is_some());
        assert!(config.sources.benzinga.is_some());
        assert!(config.features.technical_indicators.ma_periods.len() > 0);
        assert!(config.validation.price_validation);
    }

    #[test]
    fn test_sample_data_creation() {
        let events = create_sample_market_data();
        assert!(!events.is_empty());
        assert!(events.len() >= 20); // 3 symbols * 8 events each + 2 invalid
    }

    #[test]
    fn test_model_specific_configs() {
        let configs = create_model_specific_configs();
        assert!(configs.contains_key("tlob_transformer"));
        assert!(configs.contains_key("mamba"));
        assert!(configs.contains_key("dqn"));
        assert!(configs.contains_key("tft"));
    }
}