030a15ee05
- Fixed systematic array indexing corruption: [0_i32] → [0] - Fixed numeric literal suffixes across 835 files - Fixed iterator patterns on RwLockReadGuard (.iter() required) - Fixed float type annotations (365.25_f64 for sqrt) - Fixed missing semicolons in position manager - Fixed reference dereferencing in data loader Root cause: Mass refactoring incorrectly added _i32 suffixes to array indices Impact: Complete compilation failure (463 errors) Resolution: Automated regex + targeted fixes Result: 100% compilation success (0 errors) Validated: cargo check --workspace passes Ready for: Production deployment
473 lines
12 KiB
Rust
473 lines
12 KiB
Rust
//! Data configuration
|
|
|
|
use num_cpus;
|
|
use serde::{Deserialize, Serialize};
|
|
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct DataConfig {
|
|
pub provider: String,
|
|
pub symbols: Vec<String>,
|
|
pub batch_size: usize,
|
|
pub buffer_size: usize,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct DataMicrostructureConfig {
|
|
pub enable_bid_ask_spread: bool,
|
|
pub enable_order_flow: bool,
|
|
pub tick_size: f64,
|
|
pub lot_size: f64,
|
|
pub bid_ask_spread: bool,
|
|
pub volume_imbalance: bool,
|
|
pub price_impact: bool,
|
|
pub kyle_lambda: bool,
|
|
pub amihud_ratio: bool,
|
|
}
|
|
|
|
impl Default for DataMicrostructureConfig {
|
|
fn default() -> Self {
|
|
Self {
|
|
enable_bid_ask_spread: true,
|
|
enable_order_flow: true,
|
|
tick_size: 0.01,
|
|
lot_size: 100.0,
|
|
bid_ask_spread: true,
|
|
volume_imbalance: true,
|
|
price_impact: false,
|
|
kyle_lambda: false,
|
|
amihud_ratio: false,
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct DataTLOBConfig {
|
|
pub depth_levels: usize,
|
|
pub enable_imbalance: bool,
|
|
pub enable_pressure: bool,
|
|
pub window_size: usize,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct DataTechnicalIndicatorsConfig {
|
|
pub enable_moving_averages: bool,
|
|
pub enable_momentum: bool,
|
|
pub enable_volatility: bool,
|
|
pub window_sizes: Vec<usize>,
|
|
pub ma_periods: Vec<usize>,
|
|
pub rsi_periods: Vec<usize>,
|
|
pub bollinger_periods: Vec<usize>,
|
|
pub macd: DataMACDConfig,
|
|
}
|
|
|
|
impl Default for DataTechnicalIndicatorsConfig {
|
|
fn default() -> Self {
|
|
Self {
|
|
enable_moving_averages: true,
|
|
enable_momentum: true,
|
|
enable_volatility: true,
|
|
window_sizes: vec![10, 20, 50],
|
|
ma_periods: vec![10, 20, 50, 200],
|
|
rsi_periods: vec![14],
|
|
bollinger_periods: vec![20],
|
|
macd: DataMACDConfig::default(),
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct TrainingBenzingaConfig {
|
|
pub api_key: String,
|
|
pub api_key_env: String,
|
|
pub symbols: Vec<String>,
|
|
pub data_types: Vec<String>,
|
|
pub timeout: u64,
|
|
pub rate_limit: usize,
|
|
pub batch_size: usize,
|
|
pub enable_caching: bool,
|
|
}
|
|
|
|
impl Default for TrainingBenzingaConfig {
|
|
fn default() -> Self {
|
|
Self {
|
|
api_key: String::new(),
|
|
api_key_env: "BENZINGA_API_KEY".to_owned(),
|
|
symbols: vec!["SPY".to_owned(), "AAPL".to_owned()],
|
|
data_types: vec![
|
|
"news".to_owned(),
|
|
"sentiment".to_owned(),
|
|
"ratings".to_owned(),
|
|
"options".to_owned(),
|
|
],
|
|
timeout: 30,
|
|
rate_limit: 60,
|
|
batch_size: 1000,
|
|
enable_caching: true,
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub enum DataCompressionAlgorithm {
|
|
GZIP,
|
|
ZSTD,
|
|
LZ4,
|
|
Snappy,
|
|
None,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct DataCompressionConfig {
|
|
pub algorithm: DataCompressionAlgorithm,
|
|
pub enabled: bool,
|
|
pub level: Option<i32>,
|
|
}
|
|
|
|
impl Default for DataCompressionConfig {
|
|
fn default() -> Self {
|
|
Self {
|
|
algorithm: DataCompressionAlgorithm::ZSTD,
|
|
enabled: true,
|
|
level: Some(3),
|
|
}
|
|
}
|
|
}
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct DataVersioningConfig {
|
|
pub enabled: bool,
|
|
pub version_format: String,
|
|
pub keep_versions: usize,
|
|
}
|
|
|
|
impl Default for DataVersioningConfig {
|
|
fn default() -> Self {
|
|
Self {
|
|
enabled: false,
|
|
version_format: "v%Y%m%d_%H%M%S".to_owned(),
|
|
keep_versions: 5,
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct DataRetentionConfig {
|
|
pub auto_cleanup: bool,
|
|
pub retention_days: u32,
|
|
}
|
|
|
|
impl Default for DataRetentionConfig {
|
|
fn default() -> Self {
|
|
Self {
|
|
auto_cleanup: false,
|
|
retention_days: 30,
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub enum DataStorageFormat {
|
|
Parquet,
|
|
Arrow,
|
|
Json,
|
|
Csv,
|
|
CSV,
|
|
HDF5,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct DataStorageConfig {
|
|
pub format: DataStorageFormat,
|
|
pub compression: DataCompressionConfig,
|
|
pub path: String,
|
|
pub base_directory: std::path::PathBuf,
|
|
pub partition_by: Vec<String>,
|
|
pub versioning: DataVersioningConfig,
|
|
pub retention: DataRetentionConfig,
|
|
}
|
|
|
|
impl Default for DataStorageConfig {
|
|
fn default() -> Self {
|
|
Self {
|
|
format: DataStorageFormat::Parquet,
|
|
compression: DataCompressionConfig::default(),
|
|
path: "./data".to_owned(),
|
|
base_directory: std::path::PathBuf::from("./data"),
|
|
partition_by: vec!["symbol".to_owned(), "date".to_owned()],
|
|
versioning: DataVersioningConfig::default(),
|
|
retention: DataRetentionConfig::default(),
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct DataRegimeDetectionConfig {
|
|
pub enable_hmm: bool,
|
|
pub enable_clustering: bool,
|
|
pub window_size: usize,
|
|
pub n_states: usize,
|
|
pub volatility_regime: bool,
|
|
pub trend_regime: bool,
|
|
pub volume_regime: bool,
|
|
pub correlation_regime: bool,
|
|
pub lookback_period: usize,
|
|
}
|
|
|
|
impl Default for DataRegimeDetectionConfig {
|
|
fn default() -> Self {
|
|
Self {
|
|
enable_hmm: false,
|
|
enable_clustering: false,
|
|
window_size: 100,
|
|
n_states: 3,
|
|
volatility_regime: true,
|
|
trend_regime: true,
|
|
volume_regime: false,
|
|
correlation_regime: false,
|
|
lookback_period: 252,
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct DataProcessingConfig {
|
|
pub worker_threads: usize,
|
|
pub batch_size: usize,
|
|
pub buffer_size: usize,
|
|
pub timeout: u64,
|
|
pub parallel_processing: bool,
|
|
}
|
|
|
|
impl Default for DataProcessingConfig {
|
|
fn default() -> Self {
|
|
Self {
|
|
worker_threads: num_cpus::get(),
|
|
batch_size: 1000,
|
|
buffer_size: 10000,
|
|
timeout: 300,
|
|
parallel_processing: true,
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct DataTrainingConfig {
|
|
pub batch_size: usize,
|
|
pub sequence_length: usize,
|
|
pub validation_split: f64,
|
|
pub test_split: f64,
|
|
pub sources: DataSourcesConfig,
|
|
pub features: TrainingFeatureEngineeringConfig,
|
|
pub validation: DataValidationConfig,
|
|
pub storage: DataStorageConfig,
|
|
pub processing: DataProcessingConfig,
|
|
pub rate_limit: usize,
|
|
}
|
|
|
|
impl Default for DataTrainingConfig {
|
|
fn default() -> Self {
|
|
Self {
|
|
batch_size: 32,
|
|
sequence_length: 100,
|
|
validation_split: 0.2,
|
|
test_split: 0.1,
|
|
sources: DataSourcesConfig::default(),
|
|
features: TrainingFeatureEngineeringConfig::default(),
|
|
validation: DataValidationConfig::default(),
|
|
storage: DataStorageConfig::default(),
|
|
processing: DataProcessingConfig::default(),
|
|
rate_limit: 100,
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
|
pub struct DataSourcesConfig {
|
|
pub databento: Option<DatabentoConfig>,
|
|
pub benzinga: Option<TrainingBenzingaConfig>,
|
|
#[serde(default)]
|
|
pub enable_realtime: bool,
|
|
pub interactive_brokers: Option<InteractiveBrokersConfig>,
|
|
pub icmarkets: Option<ICMarketsConfig>,
|
|
pub historical: Option<HistoricalDataConfig>,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct InteractiveBrokersConfig {
|
|
pub host: String,
|
|
pub port: u16,
|
|
pub client_id: i32,
|
|
pub timeout_seconds: u64,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct ICMarketsConfig {
|
|
pub api_key: String,
|
|
pub environment: String,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct HistoricalDataConfig {
|
|
pub enabled: bool,
|
|
pub batch_size: usize,
|
|
pub parallel_downloads: usize,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct DatabentoConfig {
|
|
pub api_key: String,
|
|
pub dataset: String,
|
|
pub symbols: Vec<String>,
|
|
pub schema: String,
|
|
pub stype_in: String,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct DataValidationConfig {
|
|
#[serde(default)]
|
|
pub enable_price_validation: bool,
|
|
#[serde(default)]
|
|
pub enable_volume_validation: bool,
|
|
#[serde(default)]
|
|
pub price_threshold: f64,
|
|
#[serde(default)]
|
|
pub volume_threshold: f64,
|
|
#[serde(default)]
|
|
pub outlier_method: OutlierDetectionMethod,
|
|
#[serde(default)]
|
|
pub max_price_change: f64,
|
|
#[serde(default)]
|
|
pub max_volume_change: f64,
|
|
#[serde(default)]
|
|
pub max_timestamp_drift: i64,
|
|
#[serde(default)]
|
|
pub price_validation: bool,
|
|
#[serde(default)]
|
|
pub volume_validation: bool,
|
|
#[serde(default)]
|
|
pub timestamp_validation: bool,
|
|
#[serde(default)]
|
|
pub outlier_detection: bool,
|
|
#[serde(default)]
|
|
pub missing_data_handling: MissingDataHandling,
|
|
}
|
|
|
|
impl Default for DataValidationConfig {
|
|
fn default() -> Self {
|
|
Self {
|
|
enable_price_validation: true,
|
|
enable_volume_validation: true,
|
|
price_threshold: 0.1,
|
|
volume_threshold: 0.2,
|
|
outlier_method: OutlierDetectionMethod::ZScore,
|
|
max_price_change: 0.05,
|
|
max_volume_change: 2.0,
|
|
max_timestamp_drift: 1000,
|
|
price_validation: true,
|
|
volume_validation: true,
|
|
timestamp_validation: true,
|
|
outlier_detection: true,
|
|
missing_data_handling: MissingDataHandling::Skip,
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
|
pub enum MissingDataHandling {
|
|
#[default]
|
|
Skip,
|
|
Drop,
|
|
Interpolate,
|
|
ForwardFill,
|
|
BackwardFill,
|
|
FillForward,
|
|
FillBackward,
|
|
Mean,
|
|
Median,
|
|
Error,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct TrainingFeatureEngineeringConfig {
|
|
pub enable_normalization: bool,
|
|
pub enable_scaling: bool,
|
|
pub enable_log_returns: bool,
|
|
pub lookback_window: usize,
|
|
pub regime_detection: DataRegimeDetectionConfig,
|
|
pub technical_indicators: DataTechnicalIndicatorsConfig,
|
|
pub microstructure: DataMicrostructureConfig,
|
|
}
|
|
|
|
impl Default for TrainingFeatureEngineeringConfig {
|
|
fn default() -> Self {
|
|
Self {
|
|
enable_normalization: true,
|
|
enable_scaling: true,
|
|
enable_log_returns: true,
|
|
lookback_window: 100,
|
|
regime_detection: DataRegimeDetectionConfig::default(),
|
|
technical_indicators: DataTechnicalIndicatorsConfig::default(),
|
|
microstructure: DataMicrostructureConfig::default(),
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct DataTemporalConfig {
|
|
pub enable_time_features: bool,
|
|
pub enable_seasonal: bool,
|
|
pub timezone: String,
|
|
pub business_hours_only: bool,
|
|
pub market_session: bool,
|
|
pub holiday_effects: bool,
|
|
pub expiration_effects: bool,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
|
pub enum OutlierDetectionMethod {
|
|
#[default]
|
|
ZScore,
|
|
IQR,
|
|
Isolation,
|
|
IsolationForest,
|
|
LocalOutlierFactor,
|
|
None,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct DataModuleConfig {
|
|
pub data_path: String,
|
|
pub batch_size: usize,
|
|
pub num_workers: usize,
|
|
pub cache_size: usize,
|
|
pub settings: DataModuleSettings,
|
|
pub interactive_brokers: Option<InteractiveBrokersConfig>,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct DataModuleSettings {
|
|
pub enable_preprocessing: bool,
|
|
pub enable_validation: bool,
|
|
pub max_memory_usage: usize,
|
|
pub market_data_buffer_size: usize,
|
|
pub order_event_buffer_size: usize,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct DataMACDConfig {
|
|
pub fast_period: usize,
|
|
pub slow_period: usize,
|
|
pub signal_period: usize,
|
|
pub enabled: bool,
|
|
}
|
|
|
|
impl Default for DataMACDConfig {
|
|
fn default() -> Self {
|
|
Self {
|
|
fast_period: 12,
|
|
slow_period: 26,
|
|
signal_period: 9,
|
|
enabled: true,
|
|
}
|
|
}
|
|
}
|