//! Data configuration use num_cpus; use serde::{Deserialize, Serialize}; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct DataConfig { pub provider: String, pub symbols: Vec, pub batch_size: usize, pub buffer_size: usize, } #[derive(Debug, Clone, Serialize, Deserialize)] pub struct DataMicrostructureConfig { pub enable_bid_ask_spread: bool, pub enable_order_flow: bool, pub tick_size: f64, pub lot_size: f64, pub bid_ask_spread: bool, pub volume_imbalance: bool, pub price_impact: bool, pub kyle_lambda: bool, pub amihud_ratio: bool, } impl Default for DataMicrostructureConfig { fn default() -> Self { Self { enable_bid_ask_spread: true, enable_order_flow: true, tick_size: 0.01, lot_size: 100.0, bid_ask_spread: true, volume_imbalance: true, price_impact: false, kyle_lambda: false, amihud_ratio: false, } } } #[derive(Debug, Clone, Serialize, Deserialize)] pub struct DataTLOBConfig { pub depth_levels: usize, pub enable_imbalance: bool, pub enable_pressure: bool, pub window_size: usize, } #[derive(Debug, Clone, Serialize, Deserialize)] pub struct DataTechnicalIndicatorsConfig { pub enable_moving_averages: bool, pub enable_momentum: bool, pub enable_volatility: bool, pub window_sizes: Vec, pub ma_periods: Vec, pub rsi_periods: Vec, pub bollinger_periods: Vec, pub macd: DataMACDConfig, } impl Default for DataTechnicalIndicatorsConfig { fn default() -> Self { Self { enable_moving_averages: true, enable_momentum: true, enable_volatility: true, window_sizes: vec![10, 20, 50], ma_periods: vec![10, 20, 50, 200], rsi_periods: vec![14], bollinger_periods: vec![20], macd: DataMACDConfig::default(), } } } #[derive(Debug, Clone, Serialize, Deserialize)] pub struct TrainingBenzingaConfig { pub api_key: String, pub api_key_env: String, pub symbols: Vec, pub data_types: Vec, pub timeout: u64, pub rate_limit: usize, pub batch_size: usize, pub enable_caching: bool, } impl Default for TrainingBenzingaConfig { fn default() -> Self { Self { api_key: String::new(), api_key_env: "BENZINGA_API_KEY".to_owned(), symbols: vec!["SPY".to_owned(), "AAPL".to_owned()], data_types: vec![ "news".to_owned(), "sentiment".to_owned(), "ratings".to_owned(), "options".to_owned(), ], timeout: 30, rate_limit: 60, batch_size: 1000, enable_caching: true, } } } #[derive(Debug, Clone, Serialize, Deserialize)] pub enum DataCompressionAlgorithm { GZIP, ZSTD, LZ4, Snappy, None, } #[derive(Debug, Clone, Serialize, Deserialize)] pub struct DataCompressionConfig { pub algorithm: DataCompressionAlgorithm, pub enabled: bool, pub level: Option, } impl Default for DataCompressionConfig { fn default() -> Self { Self { algorithm: DataCompressionAlgorithm::ZSTD, enabled: true, level: Some(3), } } } #[derive(Debug, Clone, Serialize, Deserialize)] pub struct DataVersioningConfig { pub enabled: bool, pub version_format: String, pub keep_versions: usize, } impl Default for DataVersioningConfig { fn default() -> Self { Self { enabled: false, version_format: "v%Y%m%d_%H%M%S".to_owned(), keep_versions: 5, } } } #[derive(Debug, Clone, Serialize, Deserialize)] pub struct DataRetentionConfig { pub auto_cleanup: bool, pub retention_days: u32, } impl Default for DataRetentionConfig { fn default() -> Self { Self { auto_cleanup: false, retention_days: 30, } } } #[derive(Debug, Clone, Serialize, Deserialize)] pub enum DataStorageFormat { Parquet, Arrow, Json, Csv, CSV, HDF5, } #[derive(Debug, Clone, Serialize, Deserialize)] pub struct DataStorageConfig { pub format: DataStorageFormat, pub compression: DataCompressionConfig, pub path: String, pub base_directory: std::path::PathBuf, pub partition_by: Vec, pub versioning: DataVersioningConfig, pub retention: DataRetentionConfig, } impl Default for DataStorageConfig { fn default() -> Self { Self { format: DataStorageFormat::Parquet, compression: DataCompressionConfig::default(), path: "./data".to_owned(), base_directory: std::path::PathBuf::from("./data"), partition_by: vec!["symbol".to_owned(), "date".to_owned()], versioning: DataVersioningConfig::default(), retention: DataRetentionConfig::default(), } } } #[derive(Debug, Clone, Serialize, Deserialize)] pub struct DataRegimeDetectionConfig { pub enable_hmm: bool, pub enable_clustering: bool, pub window_size: usize, pub n_states: usize, pub volatility_regime: bool, pub trend_regime: bool, pub volume_regime: bool, pub correlation_regime: bool, pub lookback_period: usize, } impl Default for DataRegimeDetectionConfig { fn default() -> Self { Self { enable_hmm: false, enable_clustering: false, window_size: 100, n_states: 3, volatility_regime: true, trend_regime: true, volume_regime: false, correlation_regime: false, lookback_period: 252, } } } #[derive(Debug, Clone, Serialize, Deserialize)] pub struct DataProcessingConfig { pub worker_threads: usize, pub batch_size: usize, pub buffer_size: usize, pub timeout: u64, pub parallel_processing: bool, } impl Default for DataProcessingConfig { fn default() -> Self { Self { worker_threads: num_cpus::get(), batch_size: 1000, buffer_size: 10000, timeout: 300, parallel_processing: true, } } } #[derive(Debug, Clone, Serialize, Deserialize)] pub struct DataTrainingConfig { pub batch_size: usize, pub sequence_length: usize, pub validation_split: f64, pub test_split: f64, pub sources: DataSourcesConfig, pub features: TrainingFeatureEngineeringConfig, pub validation: DataValidationConfig, pub storage: DataStorageConfig, pub processing: DataProcessingConfig, pub rate_limit: usize, } impl Default for DataTrainingConfig { fn default() -> Self { Self { batch_size: 32, sequence_length: 100, validation_split: 0.2, test_split: 0.1, sources: DataSourcesConfig::default(), features: TrainingFeatureEngineeringConfig::default(), validation: DataValidationConfig::default(), storage: DataStorageConfig::default(), processing: DataProcessingConfig::default(), rate_limit: 100, } } } #[derive(Debug, Clone, Serialize, Deserialize, Default)] pub struct DataSourcesConfig { pub databento: Option, pub benzinga: Option, #[serde(default)] pub enable_realtime: bool, pub interactive_brokers: Option, pub icmarkets: Option, pub historical: Option, } #[derive(Debug, Clone, Serialize, Deserialize)] pub struct InteractiveBrokersConfig { pub host: String, pub port: u16, pub client_id: i32, pub timeout_seconds: u64, } #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ICMarketsConfig { pub api_key: String, pub environment: String, } #[derive(Debug, Clone, Serialize, Deserialize)] pub struct HistoricalDataConfig { pub enabled: bool, pub batch_size: usize, pub parallel_downloads: usize, } #[derive(Debug, Clone, Serialize, Deserialize)] pub struct DatabentoConfig { pub api_key: String, pub dataset: String, pub symbols: Vec, pub schema: String, pub stype_in: String, } #[derive(Debug, Clone, Serialize, Deserialize)] pub struct DataValidationConfig { #[serde(default)] pub enable_price_validation: bool, #[serde(default)] pub enable_volume_validation: bool, #[serde(default)] pub price_threshold: f64, #[serde(default)] pub volume_threshold: f64, #[serde(default)] pub outlier_method: OutlierDetectionMethod, #[serde(default)] pub max_price_change: f64, #[serde(default)] pub max_volume_change: f64, #[serde(default)] pub max_timestamp_drift: i64, #[serde(default)] pub price_validation: bool, #[serde(default)] pub volume_validation: bool, #[serde(default)] pub timestamp_validation: bool, #[serde(default)] pub outlier_detection: bool, #[serde(default)] pub missing_data_handling: MissingDataHandling, } impl Default for DataValidationConfig { fn default() -> Self { Self { enable_price_validation: true, enable_volume_validation: true, price_threshold: 0.1, volume_threshold: 0.2, outlier_method: OutlierDetectionMethod::ZScore, max_price_change: 0.05, max_volume_change: 2.0, max_timestamp_drift: 1000, price_validation: true, volume_validation: true, timestamp_validation: true, outlier_detection: true, missing_data_handling: MissingDataHandling::Skip, } } } #[derive(Debug, Clone, Serialize, Deserialize, Default)] pub enum MissingDataHandling { #[default] Skip, Drop, Interpolate, ForwardFill, BackwardFill, FillForward, FillBackward, Mean, Median, Error, } #[derive(Debug, Clone, Serialize, Deserialize)] pub struct TrainingFeatureEngineeringConfig { pub enable_normalization: bool, pub enable_scaling: bool, pub enable_log_returns: bool, pub lookback_window: usize, pub regime_detection: DataRegimeDetectionConfig, pub technical_indicators: DataTechnicalIndicatorsConfig, pub microstructure: DataMicrostructureConfig, } impl Default for TrainingFeatureEngineeringConfig { fn default() -> Self { Self { enable_normalization: true, enable_scaling: true, enable_log_returns: true, lookback_window: 100, regime_detection: DataRegimeDetectionConfig::default(), technical_indicators: DataTechnicalIndicatorsConfig::default(), microstructure: DataMicrostructureConfig::default(), } } } #[derive(Debug, Clone, Serialize, Deserialize)] pub struct DataTemporalConfig { pub enable_time_features: bool, pub enable_seasonal: bool, pub timezone: String, pub business_hours_only: bool, pub market_session: bool, pub holiday_effects: bool, pub expiration_effects: bool, } #[derive(Debug, Clone, Serialize, Deserialize, Default)] pub enum OutlierDetectionMethod { #[default] ZScore, IQR, Isolation, IsolationForest, LocalOutlierFactor, None, } #[derive(Debug, Clone, Serialize, Deserialize)] pub struct DataModuleConfig { pub data_path: String, pub batch_size: usize, pub num_workers: usize, pub cache_size: usize, pub settings: DataModuleSettings, pub interactive_brokers: Option, } #[derive(Debug, Clone, Serialize, Deserialize)] pub struct DataModuleSettings { pub enable_preprocessing: bool, pub enable_validation: bool, pub max_memory_usage: usize, pub market_data_buffer_size: usize, pub order_event_buffer_size: usize, } #[derive(Debug, Clone, Serialize, Deserialize)] pub struct DataMACDConfig { pub fast_period: usize, pub slow_period: usize, pub signal_period: usize, pub enabled: bool, } impl Default for DataMACDConfig { fn default() -> Self { Self { fast_period: 12, slow_period: 26, signal_period: 9, enabled: true, } } }