foxhunt/services/ml_training_service/src/data_config.rs

//! Training Data Source Configuration
//!
//! Defines configuration for ML training data sources to replace mock data generator.
//! This is Phase 1 of the 6-phase ML Training Data Pipeline implementation.
//!
//! ## Data Source Types
//!
//! - **Historical**: Load data from PostgreSQL database (backtests, historical trades)
//! - **RealTime**: Stream data from live trading (requires active trading session)
//! - **Hybrid**: Combine historical baseline with recent real-time data
//! - **Parquet**: Load from S3 parquet files (feature engineered datasets)
//!
//! ## Configuration Sources
//!
//! 1. Environment variables (runtime override)
//! 2. PostgreSQL config table (persistent settings)
//! 3. Default values (fallback)

use anyhow::{Context, Result};
use chrono::{DateTime, Duration, Utc};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::env;
use tracing::{debug, info, warn};

/// Training data source type
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum DataSourceType {
    /// Load from Postgre`SQL` historical data tables
    Historical,
    /// Stream from live trading system (requires active session)
    RealTime,
    /// Combine historical baseline with recent real-time data
    Hybrid,
    /// Load from S3 parquet files (pre-processed features)
    Parquet,
}

impl Default for DataSourceType {
    fn default() -> Self {
        Self::Historical
    }
}

impl std::str::FromStr for DataSourceType {
    type Err = anyhow::Error;

    fn from_str(s: &str) -> Result<Self> {
        match s.to_lowercase().as_str() {
            "historical" => Ok(Self::Historical),
            "realtime" | "real-time" | "real_time" => Ok(Self::RealTime),
            "hybrid" => Ok(Self::Hybrid),
            "parquet" => Ok(Self::Parquet),
            _ => Err(anyhow::anyhow!(
                "Invalid data source type '{}'. Expected: historical, realtime, hybrid, or parquet",
                s
            )),
        }
    }
}

/// Complete training data source configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TrainingDataSourceConfig {
    /// Data source type
    pub source_type: DataSourceType,

    /// Database connection settings (for Historical/Hybrid sources)
    pub database: Option<DatabaseConfig>,

    /// S3 configuration (for Parquet source)
    pub s3: Option<S3Config>,

    /// Time range for data loading
    pub time_range: TimeRangeConfig,

    /// Symbol filters (empty = all symbols)
    pub symbols: Vec<String>,

    /// Feature extraction settings
    pub features: FeatureExtractionConfig,

    /// Data validation settings
    pub validation: DataValidationConfig,

    /// Caching settings
    pub cache: CacheConfig,
}

/// Database connection configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DatabaseConfig {
    /// Postgre`SQL` connection URL (from env or config)
    pub connection_url: String,

    /// Maximum connection pool size
    pub max_connections: u32,

    /// Query timeout (seconds)
    pub query_timeout_secs: u64,

    /// Tables to query
    pub tables: DatabaseTables,
}

/// Database table names for training data
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DatabaseTables {
    /// `Order` book snapshots table
    pub order_books: String,

    /// `Trade` executions table
    pub trades: String,

    /// Market data events table
    pub market_data: String,

    /// Feature cache table (if available)
    pub feature_cache: Option<String>,
}

impl Default for DatabaseTables {
    fn default() -> Self {
        Self {
            order_books: "order_book_snapshots".to_string(),
            trades: "trade_executions".to_string(),
            market_data: "market_events".to_string(),
            feature_cache: Some("ml_feature_cache".to_string()),
        }
    }
}

/// S3 storage configuration for parquet files
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct S3Config {
    /// S3 bucket name
    pub bucket: String,

    /// AWS region
    pub region: String,

    /// S3 path prefix (e.g., "training-data/features/")
    pub path_prefix: String,

    /// File pattern (e.g., "features-{date}.parquet")
    pub file_pattern: String,

    /// AWS credentials source (iam_role, env_vars, or profile)
    pub credentials_source: String,
}

/// Time range configuration for data loading
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TimeRangeConfig {
    /// Start time (None = earliest available)
    pub start: Option<DateTime<Utc>>,

    /// End time (None = most recent)
    pub end: Option<DateTime<Utc>>,

    /// Duration (alternative to start/end)
    pub duration_days: Option<i64>,

    /// Training/validation split ratio (0.0-1.0)
    pub train_split: f64,
}

impl Default for TimeRangeConfig {
    fn default() -> Self {
        Self {
            start: None,
            end: None,
            duration_days: Some(30), // Default: last 30 days
            train_split: 0.8,        // 80% training, 20% validation
        }
    }
}

/// Feature extraction configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct FeatureExtractionConfig {
    /// Technical indicators to compute
    pub technical_indicators: Vec<String>,

    /// Microstructure features to extract
    pub microstructure_features: Vec<String>,

    /// Time-based aggregation windows (seconds)
    pub aggregation_windows: Vec<u64>,

    /// Enable TLOB (Temporal Limit `Order` Book) features
    pub enable_tlob: bool,

    /// Enable regime detection features
    pub enable_regime_detection: bool,

    /// Normalization method (zscore, minmax, none)
    pub normalization: String,
}

impl Default for FeatureExtractionConfig {
    fn default() -> Self {
        Self {
            technical_indicators: vec![
                "rsi".to_string(),
                "macd".to_string(),
                "ema_fast".to_string(),
                "ema_slow".to_string(),
            ],
            microstructure_features: vec![
                "spread_bps".to_string(),
                "imbalance".to_string(),
                "vwap".to_string(),
            ],
            aggregation_windows: vec![60, 300, 900], // 1min, 5min, 15min
            enable_tlob: true,
            enable_regime_detection: true,
            normalization: "zscore".to_string(),
        }
    }
}

/// Data validation configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DataValidationConfig {
    /// Minimum data points required
    pub min_samples: usize,

    /// Maximum missing data ratio (0.0-1.0)
    pub max_missing_ratio: f64,

    /// Enable outlier detection
    pub enable_outlier_detection: bool,

    /// Outlier detection threshold (z-score)
    pub outlier_threshold: f64,
}

impl Default for DataValidationConfig {
    fn default() -> Self {
        Self {
            min_samples: 1000,
            max_missing_ratio: 0.1, // Max 10% missing data
            enable_outlier_detection: true,
            outlier_threshold: 3.0, // 3 standard deviations
        }
    }
}

/// Caching configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CacheConfig {
    /// Enable local disk caching
    pub enable_disk_cache: bool,

    /// Cache directory path
    pub cache_dir: String,

    /// Cache TTL in hours
    pub ttl_hours: u64,

    /// Maximum cache size in MB
    pub max_size_mb: u64,
}

impl Default for CacheConfig {
    fn default() -> Self {
        Self {
            enable_disk_cache: true,
            cache_dir: "/tmp/foxhunt-training-cache".to_string(),
            ttl_hours: 24,
            max_size_mb: 10240, // 10GB
        }
    }
}

impl TrainingDataSourceConfig {
    /// Load configuration from environment variables and defaults
    pub fn from_env() -> Result<Self> {
        let source_type = env::var("DATA_SOURCE_TYPE")
            .unwrap_or_else(|_| "historical".to_string())
            .parse()
            .context("Invalid DATA_SOURCE_TYPE")?;

        info!("Configuring training data source: {:?}", source_type);

        let database = if matches!(
            source_type,
            DataSourceType::Historical | DataSourceType::Hybrid
        ) {
            Some(Self::load_database_config()?)
        } else {
            None
        };

        let s3 = if source_type == DataSourceType::Parquet {
            Some(Self::load_s3_config()?)
        } else {
            None
        };

        let time_range = Self::load_time_range_config()?;
        let symbols = Self::load_symbols_filter();
        let features = Self::load_feature_config();
        let validation = DataValidationConfig::default();
        let cache = CacheConfig::default();

        Ok(Self {
            source_type,
            database,
            s3,
            time_range,
            symbols,
            features,
            validation,
            cache,
        })
    }

    /// Load database configuration from environment
    fn load_database_config() -> Result<DatabaseConfig> {
        let connection_url = env::var("DATABASE_URL")
            .context("DATABASE_URL must be set for Historical/Hybrid data sources")?;

        let max_connections = env::var("DB_MAX_CONNECTIONS")
            .ok()
            .and_then(|s| s.parse().ok())
            .unwrap_or(10);

        let query_timeout_secs = env::var("DB_QUERY_TIMEOUT_SECS")
            .ok()
            .and_then(|s| s.parse().ok())
            .unwrap_or(300); // 5 minutes default

        debug!(
            "Database config: {} connections, {}s timeout",
            max_connections, query_timeout_secs
        );

        Ok(DatabaseConfig {
            connection_url,
            max_connections,
            query_timeout_secs,
            tables: DatabaseTables::default(),
        })
    }

    /// Load S3 configuration from environment
    fn load_s3_config() -> Result<S3Config> {
        let bucket =
            env::var("S3_BUCKET").context("S3_BUCKET must be set for Parquet data source")?;

        let region = env::var("S3_REGION").unwrap_or_else(|_| "us-east-1".to_string());

        let path_prefix =
            env::var("S3_PATH_PREFIX").unwrap_or_else(|_| "training-data/features/".to_string());

        let file_pattern =
            env::var("S3_FILE_PATTERN").unwrap_or_else(|_| "features-*.parquet".to_string());

        let credentials_source =
            env::var("AWS_CREDENTIALS_SOURCE").unwrap_or_else(|_| "iam_role".to_string());

        debug!("S3 config: s3://{}/{}", bucket, path_prefix);

        Ok(S3Config {
            bucket,
            region,
            path_prefix,
            file_pattern,
            credentials_source,
        })
    }

    /// Load time range configuration from environment
    fn load_time_range_config() -> Result<TimeRangeConfig> {
        let duration_days = env::var("DATA_DURATION_DAYS")
            .ok()
            .and_then(|s| s.parse().ok());

        let train_split = env::var("TRAIN_SPLIT")
            .ok()
            .and_then(|s| s.parse().ok())
            .unwrap_or(0.8);

        let mut config = TimeRangeConfig {
            start: None,
            end: None,
            duration_days,
            train_split,
        };

        // If duration specified, calculate start/end
        if let Some(days) = duration_days {
            let end = Utc::now();
            let start = Utc::now() - Duration::days(days);
            config.end = Some(end);
            config.start = Some(start);
            debug!(
                "Time range: {} days ({} to {})",
                days,
                start,
                end
            );
        }

        Ok(config)
    }

    /// Load symbols filter from environment
    fn load_symbols_filter() -> Vec<String> {
        env::var("TRAINING_SYMBOLS")
            .map(|s| {
                s.split(',')
                    .map(|sym| sym.trim().to_uppercase())
                    .filter(|sym| !sym.is_empty())
                    .collect()
            })
            .unwrap_or_default()
    }

    /// Load feature extraction configuration
    fn load_feature_config() -> FeatureExtractionConfig {
        let mut config = FeatureExtractionConfig::default();

        if let Ok(indicators) = env::var("FEATURE_TECHNICAL_INDICATORS") {
            config.technical_indicators = indicators
                .split(',')
                .map(|s| s.trim().to_string())
                .collect();
        }

        if let Ok(enable_tlob) = env::var("FEATURE_ENABLE_TLOB") {
            config.enable_tlob = enable_tlob.parse().unwrap_or(true);
        }

        if let Ok(normalization) = env::var("FEATURE_NORMALIZATION") {
            config.normalization = normalization;
        }

        debug!(
            "Feature config: {:?} indicators, TLOB={}, normalization={}",
            config.technical_indicators.len(),
            config.enable_tlob,
            config.normalization
        );

        config
    }

    /// Validate configuration
    pub fn validate(&self) -> Result<()> {
        match self.source_type {
            DataSourceType::Historical | DataSourceType::Hybrid => {
                if self.database.is_none() {
                    anyhow::bail!(
                        "Database configuration required for {:?} source",
                        self.source_type
                    );
                }
            },
            DataSourceType::Parquet => {
                if self.s3.is_none() {
                    anyhow::bail!("S3 configuration required for Parquet source");
                }
            },
            DataSourceType::RealTime => {
                warn!("RealTime data source requires active trading session");
            },
        }

        if self.time_range.train_split < 0.0 || self.time_range.train_split > 1.0 {
            anyhow::bail!(
                "Invalid train_split: {} (must be 0.0-1.0)",
                self.time_range.train_split
            );
        }

        if self.validation.max_missing_ratio < 0.0 || self.validation.max_missing_ratio > 1.0 {
            anyhow::bail!(
                "Invalid max_missing_ratio: {} (must be 0.0-1.0)",
                self.validation.max_missing_ratio
            );
        }

        Ok(())
    }

    /// Get summary for logging
    pub fn summary(&self) -> HashMap<String, String> {
        let mut summary = HashMap::new();
        summary.insert("source_type".to_string(), format!("{:?}", self.source_type));
        summary.insert("symbols_count".to_string(), self.symbols.len().to_string());
        summary.insert(
            "train_split".to_string(),
            self.time_range.train_split.to_string(),
        );

        if let Some(days) = self.time_range.duration_days {
            summary.insert("duration_days".to_string(), days.to_string());
        }

        summary.insert(
            "features".to_string(),
            format!(
                "{}+{} (TLOB={})",
                self.features.technical_indicators.len(),
                self.features.microstructure_features.len(),
                self.features.enable_tlob
            ),
        );

        summary
    }
}

#[cfg(test)]
#[allow(clippy::unwrap_used, clippy::expect_used)]
mod tests {
    use super::*;

    #[test]
    fn test_data_source_type_parsing() {
        assert_eq!(
            "historical".parse::<DataSourceType>().unwrap(),
            DataSourceType::Historical
        );
        assert_eq!(
            "REALTIME".parse::<DataSourceType>().unwrap(),
            DataSourceType::RealTime
        );
        assert_eq!(
            "hybrid".parse::<DataSourceType>().unwrap(),
            DataSourceType::Hybrid
        );
        assert_eq!(
            "parquet".parse::<DataSourceType>().unwrap(),
            DataSourceType::Parquet
        );

        assert!("invalid".parse::<DataSourceType>().is_err());
    }

    #[test]
    fn test_time_range_defaults() {
        let config = TimeRangeConfig::default();
        assert_eq!(config.duration_days, Some(30));
        assert_eq!(config.train_split, 0.8);
    }

    #[test]
    fn test_config_validation() {
        let mut config = TrainingDataSourceConfig {
            source_type: DataSourceType::Historical,
            database: None, // Missing required database
            s3: None,
            time_range: TimeRangeConfig::default(),
            symbols: vec![],
            features: FeatureExtractionConfig::default(),
            validation: DataValidationConfig::default(),
            cache: CacheConfig::default(),
        };

        // Should fail - missing database
        assert!(config.validate().is_err());

        // Add database config
        config.database = Some(DatabaseConfig {
            connection_url: "postgresql://localhost/test".to_string(),
            max_connections: 10,
            query_timeout_secs: 300,
            tables: DatabaseTables::default(),
        });

        // Should pass
        assert!(config.validate().is_ok());
    }
}