#![deny(clippy::unwrap_used, clippy::expect_used)] //! Trading Service - Main Entry Point //! //! This is the main entry point for the Foxhunt HFT Trading Service. //! It initializes all components including the PostgreSQL ConfigLoader //! for direct configuration management with hot-reload support. use anyhow::{Context, Result}; use std::sync::Arc; use std::time::Duration; use tokio::signal; use tonic::transport::Server; use tracing::{debug, error, info, warn}; use trading_service::auth_interceptor::{AuthConfig, TonicAuthInterceptor}; // Use central configuration and shared libraries with canonical imports use common::DatabasePool; use config::manager::ConfigManager; use config::DatabaseConfig; // Import repository dependencies with explicit imports use trading_service::repositories::{ ConfigRepository, MarketDataRepository, RiskRepository, TradingRepository, }; use trading_service::repository_impls::{ PostgresConfigRepository, PostgresMarketDataRepository, PostgresRiskRepository, PostgresTradingRepository, }; use trading_service::compliance_service::{ComplianceConfig, ComplianceService}; use trading_service::event_persistence::EventPersistence; use trading_service::kill_switch_integration::TradingServiceKillSwitch; use trading_service::rate_limiter::{RateLimitConfig, RateLimiter}; use trading_service::services::ml::MLService; use trading_service::services::ml_fallback_manager::MLFallbackManager; use trading_service::services::ml_performance_monitor::MLPerformanceMonitor; use trading_service::services::monitoring::MonitoringServiceImpl; use trading_service::services::risk::RiskServiceImpl; use trading_service::services::trading::TradingServiceImpl; use trading_service::feedback_loop::{FeedbackLoop, FeedbackLoopConfig}; use trading_service::questdb_metrics::QuestDBMetricsProvider; use trading_service::state::TradingServiceState; use common::metrics::GrpcMetricsLayer; /// Default configuration values const DEFAULT_GRPC_PORT: u16 = 50052; // Trading Service port (API Gateway uses 50051) const DEFAULT_HEALTH_PORT: u16 = 8081; // Health check port (separate from API Gateway 8080) #[tokio::main] async fn main() -> Result<()> { // Initialize observability (JSON logging + OpenTelemetry tracing via OTLP) let otlp_endpoint = std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT") .unwrap_or_else(|_| "http://localhost:4317".to_string()); if let Err(e) = common::observability::init_observability( "trading_service", Some(&otlp_endpoint), ) { eprintln!("Failed to initialize observability: {}", e); } info!("Starting Foxhunt Trading Service..."); // Create service config manually let service_config = config::ServiceConfig { name: "trading_service".to_string(), environment: std::env::var("ENVIRONMENT").unwrap_or_else(|_| "production".to_string()), version: env!("CARGO_PKG_VERSION").to_string(), settings: serde_json::json!({}), }; let _config_manager = Arc::new(ConfigManager::new(service_config)); info!("Central ConfigManager initialized successfully"); // Get database URL from environment (DatabaseConfig::new() already handles this) let mut database_config = DatabaseConfig::new(); // DatabaseConfig::new() already sets url from DATABASE_URL env var database_config.max_connections = 20; database_config.min_connections = 5; // Initialize HFT-optimized database pool directly from config let db_pool_wrapper = DatabasePool::new(database_config.into()) .await .context("Failed to create HFT-optimized database pool")?; let db_pool = db_pool_wrapper.pool().clone(); info!("Database connection pool initialized"); // Initialize repositories with dependency injection let trading_repository: Arc = Arc::new(PostgresTradingRepository::new(db_pool.clone())); let market_data_repository: Arc = Arc::new(PostgresMarketDataRepository::new(db_pool.clone())); let risk_repository: Arc = Arc::new(PostgresRiskRepository::new(db_pool.clone())); let config_repository_impl = Arc::new(PostgresConfigRepository::new(db_pool.clone())); info!("Repository layer initialized with dependency injection"); // Initialize event persistence for compliance and audit trail let process_id = std::process::id(); let event_persistence = Arc::new(EventPersistence::new( db_pool.clone(), "trading_service".to_string(), process_id, )); info!("Event persistence initialized for compliance audit trail"); // Initialize default configurations if they don't exist initialize_default_configs(&config_repository_impl).await?; // Initialize kill switch system for regulatory compliance let redis_url = std::env::var("REDIS_URL").unwrap_or_else(|_| "redis://localhost:6379".to_string()); let kill_switch_system = Arc::new( TradingServiceKillSwitch::new(redis_url) .await .context("Failed to initialize kill switch system")?, ); info!("Kill switch system initialized for regulatory compliance"); // Start kill switch monitoring (Unix socket, signal handlers, etc.) kill_switch_system .start_monitoring() .await .context("Failed to start kill switch monitoring")?; info!("Kill switch monitoring started - emergency shutdown ready"); // Start configuration hot-reload monitoring start_config_monitoring(Arc::clone(&config_repository_impl)).await?; // The centralized config manager now handles all configuration persistence // and provenance tracking internally // Initialize authentication configuration using Tonic interceptor let auth_config = initialize_auth_config().await; let auth_interceptor = TonicAuthInterceptor::new(auth_config); info!("✅ Authentication interceptor initialized with Tonic 0.14 compatibility"); // Initialize compliance service for SOX and MiFID II regulatory requirements let compliance_config = ComplianceConfig { enable_sox_audit: std::env::var("ENABLE_SOX_AUDIT") .ok() .and_then(|s| s.parse().ok()) .unwrap_or(true), // Production default: enabled enable_mifid_reporting: std::env::var("ENABLE_MIFID_REPORTING") .ok() .and_then(|s| s.parse().ok()) .unwrap_or(true), // Production default: enabled enable_position_monitoring: std::env::var("ENABLE_POSITION_MONITORING") .ok() .and_then(|s| s.parse().ok()) .unwrap_or(true), // Production default: enabled enable_best_execution_analysis: std::env::var("ENABLE_BEST_EXECUTION_ANALYSIS") .ok() .and_then(|s| s.parse().ok()) .unwrap_or(true), // Production default: enabled kill_switch_enabled: std::env::var("COMPLIANCE_KILL_SWITCH_ENABLED") .ok() .and_then(|s| s.parse().ok()) .unwrap_or(true), // Production default: enabled max_position_utilization: std::env::var("MAX_POSITION_UTILIZATION") .ok() .and_then(|s| s.parse().ok()) .unwrap_or(0.95), // 95% default critical_risk_threshold: std::env::var("CRITICAL_RISK_THRESHOLD") .ok() .and_then(|s| s.parse().ok()) .unwrap_or(0.8), // 80% default }; let compliance_service = Arc::new(ComplianceService::new(db_pool.clone(), compliance_config)); // Verify compliance service health if let Err(e) = compliance_service.health_check().await { error!("Compliance service health check failed: {}", e); return Err(anyhow::anyhow!("Failed to initialize compliance service")); } info!("Compliance service initialized with SOX and MiFID II audit trails"); // Initialize advanced rate limiter with production defaults let rate_limit_config = RateLimitConfig { user_requests_per_minute: std::env::var("USER_REQUESTS_PER_MINUTE") .ok() .and_then(|s| s.parse().ok()) .unwrap_or(1000), // 1000 requests per minute per user user_burst_capacity: std::env::var("USER_BURST_CAPACITY") .ok() .and_then(|s| s.parse().ok()) .unwrap_or(100), // Allow bursts up to 100 requests ip_requests_per_minute: std::env::var("IP_REQUESTS_PER_MINUTE") .ok() .and_then(|s| s.parse().ok()) .unwrap_or(2000), // 2000 requests per minute per IP ip_burst_capacity: std::env::var("IP_BURST_CAPACITY") .ok() .and_then(|s| s.parse().ok()) .unwrap_or(200), // Allow bursts up to 200 requests global_requests_per_minute: std::env::var("GLOBAL_REQUESTS_PER_MINUTE") .ok() .and_then(|s| s.parse().ok()) .unwrap_or(50000), // 50k requests per minute globally global_burst_capacity: std::env::var("GLOBAL_BURST_CAPACITY") .ok() .and_then(|s| s.parse().ok()) .unwrap_or(5000), // Allow bursts up to 5k requests auth_failures_per_minute: std::env::var("AUTH_FAILURES_PER_MINUTE") .ok() .and_then(|s| s.parse().ok()) .unwrap_or(5), // Max 5 auth failures per minute auth_failure_penalty_minutes: std::env::var("AUTH_FAILURE_PENALTY_MINUTES") .ok() .and_then(|s| s.parse().ok()) .unwrap_or(15), // 15 minute penalty orders_per_minute: std::env::var("ORDERS_PER_MINUTE") .ok() .and_then(|s| s.parse().ok()) .unwrap_or(600), // 600 orders per minute (10/second) order_burst_capacity: std::env::var("ORDER_BURST_CAPACITY") .ok() .and_then(|s| s.parse().ok()) .unwrap_or(60), // Allow bursts up to 60 orders cleanup_interval_minutes: std::env::var("RATE_LIMIT_CLEANUP_INTERVAL") .ok() .and_then(|s| s.parse().ok()) .unwrap_or(60), // Cleanup every hour }; let rate_limiter = Arc::new(RateLimiter::new(rate_limit_config)); // Start background cleanup task for rate limiter Arc::clone(&rate_limiter).start_cleanup_task().await; info!("Advanced rate limiter initialized with per-user, per-IP, and global limits"); info!("Authentication system initialized with mTLS and JWT support"); // Initialize ensemble coordinator with real ML inference adapters let ensemble_coordinator = { use trading_service::adapter_bridge::InferenceAdapterBridge; use trading_service::ensemble_coordinator::EnsembleCoordinator; let coordinator = EnsembleCoordinator::new(); // Create real candle-backed inference adapters and register them // via the InferenceAdapterBridge (ModelInferenceAdapter -> MLModel). // Models are initialised with random weights; production checkpoints // are hot-loaded later via the model registry's shadow-buffer swap. // DQN adapter (51-dim input, 45 factored actions) match ml::ensemble::adapters::DqnInferenceAdapter::new(ml::dqn::dqn::DQNConfig { state_dim: 51, num_actions: 45, hidden_dims: vec![64, 64], ..Default::default() }) { Ok(adapter) => { let bridge = Arc::new(InferenceAdapterBridge::new( Box::new(adapter), "DQN".to_string(), ml::ModelType::DQN, )); if let Err(e) = coordinator .register_loaded_model("DQN".to_string(), bridge, 0.10) .await { error!("Failed to register DQN adapter: {}", e); } else { info!("Registered DQN inference adapter (51-dim, candle)"); } } Err(e) => error!("Failed to create DQN inference adapter: {}", e), } // PPO adapter (51-dim input, 3 actions: buy/sell/hold) match ml::ensemble::adapters::PpoInferenceAdapter::new(ml::ppo::ppo::PPOConfig { state_dim: 51, num_actions: 3, policy_hidden_dims: vec![64, 64], value_hidden_dims: vec![64, 64], ..Default::default() }) { Ok(adapter) => { let bridge = Arc::new(InferenceAdapterBridge::new( Box::new(adapter), "PPO".to_string(), ml::ModelType::PPO, )); if let Err(e) = coordinator .register_loaded_model("PPO".to_string(), bridge, 0.10) .await { error!("Failed to register PPO adapter: {}", e); } else { info!("Registered PPO inference adapter (51-dim, candle)"); } } Err(e) => error!("Failed to create PPO inference adapter: {}", e), } // TFT adapter (51-dim split: 3 static + 8 known + 40 unknown) match ml::ensemble::adapters::TftInferenceAdapter::new( ml::tft::TFTConfig { input_dim: 51, hidden_dim: 32, num_heads: 4, num_layers: 1, prediction_horizon: 1, sequence_length: 10, num_quantiles: 3, num_static_features: 3, num_known_features: 8, num_unknown_features: 40, ..Default::default() }, 10, // sequence_length for buffer ) { Ok(adapter) => { let bridge = Arc::new(InferenceAdapterBridge::new( Box::new(adapter), "TFT".to_string(), ml::ModelType::TFT, )); if let Err(e) = coordinator .register_loaded_model("TFT".to_string(), bridge, 0.10) .await { error!("Failed to register TFT adapter: {}", e); } else { info!("Registered TFT inference adapter (51-dim, candle)"); } } Err(e) => error!("Failed to create TFT inference adapter: {}", e), } // Mamba2 adapter (d_model=64, zero-pads from 51-dim features) match ml::ensemble::adapters::Mamba2InferenceAdapter::new( ml::mamba::Mamba2Config { d_model: 64, d_state: 16, d_head: 16, num_heads: 2, expand: 1, num_layers: 1, ..Default::default() }, 10, // sequence_length for buffer ) { Ok(adapter) => { let bridge = Arc::new(InferenceAdapterBridge::new( Box::new(adapter), "MAMBA2".to_string(), ml::ModelType::MAMBA, )); if let Err(e) = coordinator .register_loaded_model("MAMBA2".to_string(), bridge, 0.10) .await { error!("Failed to register MAMBA2 adapter: {}", e); } else { info!("Registered MAMBA2 inference adapter (64-dim SSM, candle)"); } } Err(e) => error!("Failed to create MAMBA2 inference adapter: {}", e), } // Liquid CfC adapter (51-dim input, 3 output: buy/hold/sell) match ml::ensemble::adapters::LiquidInferenceAdapter::new( ml::liquid::candle_cfc::CfCTrainConfig { input_size: 51, hidden_size: 32, output_size: 3, backbone_hidden_sizes: vec![32], seq_len: 1, device: ml::liquid::candle_cfc::DeviceConfig::Cpu, ..ml::liquid::candle_cfc::CfCTrainConfig::default() }, ) { Ok(adapter) => { let bridge = Arc::new(InferenceAdapterBridge::new( Box::new(adapter), "Liquid-CfC".to_string(), ml::ModelType::LNN, )); if let Err(e) = coordinator .register_loaded_model("Liquid-CfC".to_string(), bridge, 0.10) .await { error!("Failed to register Liquid-CfC adapter: {}", e); } else { info!("Registered Liquid-CfC inference adapter (51-dim, candle ODE)"); } } Err(e) => error!("Failed to create Liquid-CfC inference adapter: {}", e), } // TGGN adapter (51-dim input, 2-layer candle projection) match ml::ensemble::adapters::TggnInferenceAdapter::new(51, 64) { Ok(adapter) => { let bridge = Arc::new(InferenceAdapterBridge::new( Box::new(adapter), "TGGN".to_string(), ml::ModelType::TGGN, )); if let Err(e) = coordinator .register_loaded_model("TGGN".to_string(), bridge, 0.10) .await { error!("Failed to register TGGN adapter: {}", e); } else { info!("Registered TGGN inference adapter (51-dim, candle projection)"); } } Err(e) => error!("Failed to create TGGN inference adapter: {}", e), } // TLOB adapter (51-dim features, 3-layer MLP, seq_len=10) match ml::ensemble::adapters::TlobInferenceAdapter::new(51, 64, 10) { Ok(adapter) => { let bridge = Arc::new(InferenceAdapterBridge::new( Box::new(adapter), "TLOB".to_string(), ml::ModelType::TLOB, )); if let Err(e) = coordinator .register_loaded_model("TLOB".to_string(), bridge, 0.10) .await { error!("Failed to register TLOB adapter: {}", e); } else { info!("Registered TLOB inference adapter (51-dim, seq=10, candle MLP)"); } } Err(e) => error!("Failed to create TLOB inference adapter: {}", e), } // KAN adapter (51-dim input, B-spline activations) match ml::ensemble::adapters::KanInferenceAdapter::new(ml::kan::KANConfig { layer_widths: vec![51, 32, 16, 1], ..Default::default() }) { Ok(adapter) => { let bridge = Arc::new(InferenceAdapterBridge::new( Box::new(adapter), "KAN".to_string(), ml::ModelType::KAN, )); if let Err(e) = coordinator .register_loaded_model("KAN".to_string(), bridge, 0.10) .await { error!("Failed to register KAN adapter: {}", e); } else { info!("Registered KAN inference adapter (51-dim, B-spline, candle)"); } } Err(e) => error!("Failed to create KAN inference adapter: {}", e), } // xLSTM adapter (51-dim input, sLSTM+mLSTM, seq_len=10) match ml::ensemble::adapters::XlstmInferenceAdapter::new( ml::xlstm::XLSTMConfig { input_dim: 51, hidden_dim: 64, num_blocks: 2, num_heads: 2, ..Default::default() }, 10, // sequence_length for buffer ) { Ok(adapter) => { let bridge = Arc::new(InferenceAdapterBridge::new( Box::new(adapter), "xLSTM".to_string(), ml::ModelType::XLSTM, )); if let Err(e) = coordinator .register_loaded_model("xLSTM".to_string(), bridge, 0.10) .await { error!("Failed to register xLSTM adapter: {}", e); } else { info!("Registered xLSTM inference adapter (51-dim, seq=10, candle)"); } } Err(e) => error!("Failed to create xLSTM inference adapter: {}", e), } // Diffusion adapter (51-dim features, DDPM denoiser, t=1 inference) match ml::ensemble::adapters::DiffusionInferenceAdapter::new(ml::diffusion::DiffusionConfig { seq_len: 1, feature_dim: 51, hidden_dim: 64, num_layers: 2, time_embed_dim: 32, num_timesteps: 100, sampling_steps: 5, ..Default::default() }) { Ok(adapter) => { let bridge = Arc::new(InferenceAdapterBridge::new( Box::new(adapter), "Diffusion".to_string(), ml::ModelType::Diffusion, )); if let Err(e) = coordinator .register_loaded_model("Diffusion".to_string(), bridge, 0.10) .await { error!("Failed to register Diffusion adapter: {}", e); } else { info!("Registered Diffusion inference adapter (51-dim, DDPM, candle)"); } } Err(e) => error!("Failed to create Diffusion inference adapter: {}", e), } let model_count = coordinator.model_count().await; info!( "Ensemble coordinator initialized with {} real inference adapters \ (all 10 models at 0.10 weight each)", model_count ); Arc::new(coordinator) }; // Initialize service state with repository dependency injection let service_state = TradingServiceState::new_with_repositories( trading_repository, market_data_repository, risk_repository, Arc::clone(&config_repository_impl), db_pool.clone(), Arc::clone(&event_persistence), Some(Arc::clone(&kill_switch_system)), Some(ensemble_coordinator), ) .await?; info!("Trading service state initialized with repository dependency injection and ensemble coordinator"); // Initialize ML performance monitoring and fallback management let ml_performance_monitor = Arc::new(MLPerformanceMonitor::new()); let ml_fallback_manager = Arc::new(MLFallbackManager::new()); info!("ML performance monitoring and fallback management initialized"); // Initialize paper trading executor for prediction consumption use trading_service::paper_trading_executor::{PaperTradingConfig, PaperTradingExecutor}; let paper_trading_config = PaperTradingConfig { enabled: std::env::var("PAPER_TRADING_ENABLED") .ok() .and_then(|s| s.parse().ok()) .unwrap_or(true), // Default: enabled min_confidence: std::env::var("PAPER_TRADING_MIN_CONFIDENCE") .ok() .and_then(|s| s.parse().ok()) .unwrap_or(0.60), // 60% minimum confidence poll_interval_ms: std::env::var("PAPER_TRADING_POLL_INTERVAL_MS") .ok() .and_then(|s| s.parse().ok()) .unwrap_or(100), // 100ms polling max_position_size: std::env::var("PAPER_TRADING_MAX_POSITION_SIZE") .ok() .and_then(|s| s.parse().ok()) .unwrap_or(10_000.0), // $10,000 max position allowed_symbols: std::env::var("PAPER_TRADING_ALLOWED_SYMBOLS") .ok() .map(|s| s.split(',').map(|sym| sym.trim().to_string()).collect()) .unwrap_or_else(|| { vec![ "ES.FUT".to_string(), "NQ.FUT".to_string(), "ZN.FUT".to_string(), "6E.FUT".to_string(), ] }), account_id: std::env::var("PAPER_TRADING_ACCOUNT_ID") .unwrap_or_else(|_| "paper_trading_001".to_string()), initial_capital: std::env::var("PAPER_TRADING_INITIAL_CAPITAL") .ok() .and_then(|s| s.parse().ok()) .unwrap_or(100_000.0), // $100,000 initial capital batch_size: std::env::var("PAPER_TRADING_BATCH_SIZE") .ok() .and_then(|s| s.parse().ok()) .unwrap_or(100), // Process 100 predictions per batch }; let paper_trading_executor = Arc::new(PaperTradingExecutor::new( db_pool.clone(), paper_trading_config.clone(), )?); info!( "Paper trading executor initialized: enabled={}, min_confidence={:.1}%, poll_interval={}ms", paper_trading_config.enabled, paper_trading_config.min_confidence * 100.0, paper_trading_config.poll_interval_ms ); // Spawn paper trading executor background task let executor_clone = Arc::clone(&paper_trading_executor); tokio::spawn(async move { info!("Paper trading executor background task starting..."); if let Err(e) = executor_clone.start().await { error!("Paper trading executor failed: {}", e); } }); // Spawn background ML prediction generation loop (Wave 14.2 Agent 7) // This generates predictions every 60 seconds and saves to ensemble_predictions table use trading_service::prediction_generation_loop::{ PredictionGenerationLoop, PredictionLoopConfig, }; // Shutdown senders for background loops -- dropped at end of main to signal shutdown. let mut prediction_shutdown_handles: Vec> = Vec::new(); if let Some(ensemble_coordinator) = service_state.ensemble_coordinator() { let prediction_config = PredictionLoopConfig::from_env(); let prediction_loop = PredictionGenerationLoop::new( Arc::clone(ensemble_coordinator), db_pool.clone(), prediction_config.clone(), ); // Create shutdown channel for prediction loop let (prediction_shutdown_tx, prediction_shutdown_rx) = tokio::sync::broadcast::channel(1); // Spawn prediction generation loop tokio::spawn(async move { info!( "ML prediction generation loop starting: symbols={:?}, interval={:?}", prediction_config.symbols, prediction_config.prediction_interval ); if let Err(e) = prediction_loop.run(prediction_shutdown_rx).await { error!("Prediction generation loop failed: {}", e); } }); info!("✅ Background ML prediction generation loop initialized"); // Store shutdown sender so it is dropped during graceful shutdown, // which closes the channel and signals the prediction loop to stop. prediction_shutdown_handles.push(prediction_shutdown_tx); } else { info!("Ensemble coordinator not available - prediction generation loop disabled"); } // Spawn background market data feed for ensemble feature extractor warmup. // The ensemble coordinator's `update_market_data()` feeds per-symbol // `ProductionFeatureExtractorAdapter`s that require ~51 bars before they can // produce real 51-dim feature vectors. Without this feed the extractors stay // cold and `fetch_features_for_symbol` returns zeros. // // In production this task would be replaced by a Databento or exchange feed; // for now it generates synthetic OHLCV bars at 1-second intervals. if let Some(ensemble_coordinator) = service_state.ensemble_coordinator() { let coord = Arc::clone(ensemble_coordinator); let warmup_symbols: Vec = std::env::var("MARKET_FEED_SYMBOLS") .ok() .map(|s| s.split(',').map(|sym| sym.trim().to_string()).collect()) .unwrap_or_else(|| { vec![ "ES.FUT".to_string(), "NQ.FUT".to_string(), "ZN.FUT".to_string(), "6E.FUT".to_string(), ] }); let feed_interval_ms: u64 = std::env::var("MARKET_FEED_INTERVAL_MS") .ok() .and_then(|s| s.parse().ok()) .unwrap_or(1_000); // 1 second default // Base prices per symbol (representative of real futures contracts) let base_prices: Vec = warmup_symbols .iter() .map(|sym| match sym.as_str() { "ES.FUT" => 4_500.0, "NQ.FUT" => 15_800.0, "ZN.FUT" => 110.5, "6E.FUT" => 1.085, _ => 100.0, }) .collect(); tokio::spawn(async move { use rand::Rng; use rand::SeedableRng; info!( "Market data feed starting: symbols={:?}, interval={}ms, warmup_target={} bars", warmup_symbols, feed_interval_ms, trading_service::ensemble_coordinator::FEATURE_WARMUP_BARS, ); let mut interval = tokio::time::interval(Duration::from_millis(feed_interval_ms)); let mut tick: u64 = 0; let mut rng = rand::rngs::StdRng::from_entropy(); let mut prices: Vec = base_prices; let warmup_target = trading_service::ensemble_coordinator::FEATURE_WARMUP_BARS as u64; loop { interval.tick().await; let now = chrono::Utc::now(); for (idx, symbol) in warmup_symbols.iter().enumerate() { let price = match prices.get(idx) { Some(p) => *p, None => continue, }; // Random walk: +/- 0.02% per tick (realistic micro-volatility) let pct_change: f64 = rng.gen_range(-0.0002..0.0002); let new_price = price * (1.0 + pct_change); let volume = rng.gen_range(500.0..5_000.0); if let Some(slot) = prices.get_mut(idx) { *slot = new_price; } if let Err(e) = coord .update_market_data(symbol, new_price, volume, now) .await { debug!( "Market data feed: failed to update {} (tick {}): {}", symbol, tick, e ); } } tick += 1; // Log warmup progress at key milestones if tick == warmup_target { info!( "Market data feed: warmup complete ({} bars) -- \ feature extraction now producing real vectors for {:?}", warmup_target, warmup_symbols, ); } else if tick < warmup_target && tick % 10 == 0 { info!( "Market data feed: warmup progress {}/{} bars", tick, warmup_target, ); } } }); info!( "Background market data feed spawned for ensemble feature extractor warmup" ); } else { info!("Ensemble coordinator not available - market data feed disabled"); } // Initialize autonomous feedback loop (weight + gate optimization with kill switch) { let feedback_config = FeedbackLoopConfig { cycle_interval: Duration::from_secs( std::env::var("FEEDBACK_LOOP_INTERVAL_SECS") .ok() .and_then(|s| s.parse().ok()) .unwrap_or(24 * 3600), ), kill_switch_threshold: std::env::var("FEEDBACK_KILL_SWITCH_SHARPE") .ok() .and_then(|s| s.parse().ok()) .unwrap_or(-1.0), ..Default::default() }; let feedback_loop = Arc::new(FeedbackLoop::new(feedback_config)); // Connect to QuestDB for real metrics (pg wire protocol on port 8812) let questdb_url = std::env::var("QUESTDB_PG_URL") .unwrap_or_else(|_| "postgresql://admin:quest@localhost:8812/qdb".to_string()); match QuestDBMetricsProvider::try_new(&questdb_url).await { Some(provider) => { if let Err(e) = provider.ensure_tables().await { warn!("Failed to create QuestDB tables: {} — feedback loop will retry", e); } let metrics_provider: Arc = Arc::new(provider); let _feedback_handle = Arc::clone(&feedback_loop).spawn(metrics_provider); info!("Autonomous feedback loop spawned with QuestDB metrics provider"); } None => { info!("QuestDB unavailable — feedback loop disabled until QUESTDB_PG_URL is reachable"); } } } // Subscribe to ML performance alerts let monitor_clone = Arc::clone(&ml_performance_monitor); tokio::spawn(async move { use trading_service::ml_metrics; let mut alert_receiver = monitor_clone.subscribe_alerts(); info!("ML performance alert handler started"); while let Ok(alert) = alert_receiver.recv().await { // Log alert based on severity match alert.severity { trading_service::services::ml_performance_monitor::AlertSeverity::Emergency | trading_service::services::ml_performance_monitor::AlertSeverity::Critical => { error!("ML ALERT [{}]: {}", alert.model_id, alert.message); }, trading_service::services::ml_performance_monitor::AlertSeverity::Warning => { warn!("ML ALERT [{}]: {}", alert.model_id, alert.message); }, trading_service::services::ml_performance_monitor::AlertSeverity::Info => { info!("ML ALERT [{}]: {}", alert.model_id, alert.message); }, } // Update Prometheus alert counter ml_metrics::ML_ALERTS_TOTAL .with_label_values(&[ &alert.model_id, &ml_metrics::alert_type_str(&alert.alert_type).to_string(), &ml_metrics::alert_severity_str(&alert.severity).to_string(), ]) .inc(); } info!("ML performance alert handler stopped"); }); // Create gRPC services with enhanced ML capabilities // Services expect TradingServiceState directly, not Arc let trading_service = TradingServiceImpl::new(Arc::new(service_state.clone())); let risk_service = RiskServiceImpl::new(service_state.clone()); let ml_service = MLService::new( service_state.clone(), Arc::clone(&ml_performance_monitor), Arc::clone(&ml_fallback_manager), ); let config_service = trading_service::services::config::ConfigServiceImpl::new(db_pool.clone()); let monitoring_service = MonitoringServiceImpl::new(service_state); // Create health service let (health_reporter, health_service) = tonic_health::server::health_reporter(); health_reporter .set_serving::>() .await; // Build gRPC server with TLS, authentication, and rate limiting let grpc_port = std::env::var("GRPC_PORT") .ok() .and_then(|s| s.parse().ok()) .unwrap_or(DEFAULT_GRPC_PORT); let addr = format!("0.0.0.0:{}", grpc_port).parse()?; // Load TLS configuration if enabled let tls_config = if std::env::var("TLS_ENABLED") .unwrap_or_else(|_| "false".to_string()) .parse::() .unwrap_or(false) { info!("Loading TLS configuration..."); use trading_service::tls_config::TradingServiceTlsConfig; let tls = TradingServiceTlsConfig::from_files( &std::env::var("TLS_CERT_PATH") .unwrap_or_else(|_| "/app/certs/trading_service/server.crt".to_string()), &std::env::var("TLS_KEY_PATH") .unwrap_or_else(|_| "/app/certs/trading_service/server.key".to_string()), &std::env::var("TLS_CA_PATH") .unwrap_or_else(|_| "/app/certs/trading_service/ca.crt".to_string()), std::env::var("TLS_REQUIRE_CLIENT_CERT") .unwrap_or_else(|_| "false".to_string()) .parse::() .unwrap_or(false), ) .await?; info!("✓ TLS 1.3 enabled with mTLS client certificate validation"); Some(tls.to_server_tls_config()) } else { warn!("⚠ TLS DISABLED - Running in insecure mode (development only)"); None }; info!("🔒 Starting gRPC server with authentication enabled"); // Wave 67 Agent 3: HTTP/2 streaming performance optimizations // Load streaming configuration with feature flag support let streaming_config = trading_service::streaming::StreamingConfig::from_env(); if streaming_config.is_enabled() { info!("✅ HTTP/2 optimizations enabled:"); info!( " - tcp_nodelay: {} (-40ms Nagle delay)", streaming_config.tcp_nodelay ); info!( " - Stream window: {}KB", streaming_config.initial_stream_window_size / 1024 ); info!( " - Connection window: {}MB", streaming_config.initial_connection_window_size / (1024 * 1024) ); info!( " - Adaptive window: {}", streaming_config.http2_adaptive_window ); info!(" - Max streams: 10,000 (production scale)"); } else { info!("⚠️ HTTP/2 optimizations disabled via feature flag"); } // Start Prometheus metrics HTTP endpoint on port 9092 use trading_service::metrics_server::{MetricsServerConfig, TradingMetricsServer}; let metrics_config = MetricsServerConfig { bind_address: "0.0.0.0".to_string(), bind_port: 9092, scrape_timeout_ms: 500, max_metrics_per_scrape: 10000, }; let metrics_server = TradingMetricsServer::new(metrics_config); tokio::spawn(async move { if let Err(e) = metrics_server.start().await { tracing::error!("Metrics server failed: {}", e); } }); // Apply authentication interceptor to all gRPC services let mut server_builder = match tls_config { Some(tls) => Server::builder() .layer(GrpcMetricsLayer::new("trading-service")) .tls_config(tls) .context("Failed to configure TLS")?, None => Server::builder() .layer(GrpcMetricsLayer::new("trading-service")), }; // Apply HTTP/2 optimizations if enabled if streaming_config.is_enabled() { server_builder = server_builder .tcp_nodelay(streaming_config.tcp_nodelay) // Critical: eliminates 40ms Nagle delay .http2_keepalive_interval(Some(streaming_config.http2_keepalive_interval)) .http2_keepalive_timeout(Some(streaming_config.http2_keepalive_timeout)) .initial_stream_window_size(Some(streaming_config.initial_stream_window_size)) .initial_connection_window_size(Some(streaming_config.initial_connection_window_size)) .http2_adaptive_window(Some(streaming_config.http2_adaptive_window)) .max_concurrent_streams(Some(10_000)); // Increased from 1,024 to 10,000 for production scale } let server = server_builder .add_service(health_service) .add_service( trading_service::proto::trading::trading_service_server::TradingServiceServer::with_interceptor( trading_service, auth_interceptor.clone() ) ) .add_service( trading_service::proto::risk::risk_service_server::RiskServiceServer::with_interceptor( risk_service, auth_interceptor.clone() ) ) .add_service( trading_service::proto::ml::ml_service_server::MlServiceServer::with_interceptor( ml_service, auth_interceptor.clone() ) ) .add_service( trading_service::proto::monitoring::monitoring_service_server::MonitoringServiceServer::with_interceptor( monitoring_service, auth_interceptor.clone() ) ) .add_service( trading_service::proto::config::config_service_server::ConfigServiceServer::with_interceptor( config_service, auth_interceptor.clone() ) ) .serve_with_shutdown(addr, shutdown_signal()); info!("Trading Service listening on {}", addr); // Start background tasks with kill switch monitoring tokio::select! { result = server => { if let Err(e) = result { error!("gRPC server error: {}", e); } } _ = start_health_endpoint(std::env::var("HEALTH_PORT") .ok() .and_then(|s| s.parse().ok()) .unwrap_or(DEFAULT_HEALTH_PORT)) => { info!("Health endpoint stopped"); } _ = monitor_kill_switch_status(Arc::clone(&kill_switch_system)) => { info!("Kill switch monitoring stopped"); } } // Signal prediction loop(s) to stop, then drop senders for tx in &prediction_shutdown_handles { let _ = tx.send(()); } drop(prediction_shutdown_handles); info!("Background prediction loops signalled for shutdown"); // Cleanup kill switch monitoring on shutdown info!("Stopping kill switch monitoring..."); if let Err(e) = kill_switch_system.stop_monitoring().await { warn!("Error stopping kill switch monitoring: {}", e); } info!("Trading Service shutdown complete"); Ok(()) } /// Initialize authentication configuration /// /// NOTE: Authentication is handled by API Gateway (Wave 70) /// This function returns a minimal stub configuration for compatibility async fn initialize_auth_config() -> AuthConfig { // API Gateway handles all authentication - this is just a stub // All auth configuration is managed by API Gateway, not this service AuthConfig::default() } /// Initialize default configuration values if they don't exist async fn initialize_default_configs(_config_repository: &PostgresConfigRepository) -> Result<()> { info!("Initializing default configurations..."); // Note: Configuration initialization is now handled by ConfigManager // This function is kept for backward compatibility but is a no-op // The ConfigManager handles all configuration persistence and defaults info!("Default configurations initialized via ConfigManager"); Ok(()) } /// Start configuration monitoring for hot-reload async fn start_config_monitoring(config_repository: Arc) -> Result<()> { let mut change_receiver = config_repository .subscribe_to_changes() .await .context("Failed to subscribe to configuration changes")?; tokio::spawn(async move { info!("Configuration hot-reload monitoring started"); while let Ok((category, key)) = change_receiver.recv().await { info!("Configuration changed: {}.{}", category, key); // Handle specific configuration changes using typed methods match (category.as_str(), key.as_str()) { ("Trading", "max_order_size") => { if let Ok(Some(value)) = config_repository .get_config_f64("Trading", "max_order_size") .await { info!("Updated max order size to: ${}", value); } }, ("MachineLearning", "inference_timeout_ms") => { if let Ok(Some(value)) = config_repository .get_config_u64("MachineLearning", "inference_timeout_ms") .await { info!("Updated ML inference timeout to: {}ms", value); } }, ("Risk", "var_confidence") => { if let Ok(Some(value)) = config_repository .get_config_f64("Risk", "var_confidence") .await { info!("Updated VaR confidence to: {}", value); } }, _ => { info!("Configuration updated: {}.{}", category, key); }, } } info!("Configuration monitoring stopped"); }); Ok(()) } /// Start health check endpoint async fn start_health_endpoint(port: u16) -> Result<()> { use hyper::server::conn::http1; use hyper::service::service_fn; use hyper_util::rt::TokioIo; use tokio::net::TcpListener; let addr: std::net::SocketAddr = ([0, 0, 0, 0], port).into(); let listener = TcpListener::bind(addr) .await .context("Failed to bind health endpoint")?; info!("Health endpoint listening on http://{}", addr); loop { let (stream, _) = match listener.accept().await { Ok(conn) => conn, Err(e) => { error!("Failed to accept connection: {}", e); continue; }, }; tokio::spawn(async move { let io = TokioIo::new(stream); if let Err(e) = http1::Builder::new() .serve_connection(io, service_fn(health_handler)) .await { error!("Health server error: {}", e); } }); } } /// Health check handler with database pool statistics async fn health_handler( _: hyper::Request, ) -> Result>, std::convert::Infallible> { use bytes::Bytes; use http_body_util::Full; // Note: In a production system, you'd pass the db_pool_wrapper here // For now, we'll indicate that database connection pooling is active let health_response = serde_json::json!({ "status": "healthy", "service": "trading_service", "timestamp": chrono::Utc::now().to_rfc3339(), "version": env!("CARGO_PKG_VERSION"), "database": { "connection_pool": "HFT-optimized", "query_timeout_micros": 800, "connection_prewarming": "enabled", "prepared_statements": "enabled" } }); // Build HTTP response with error handling to prevent panics in production let response = hyper::Response::builder() .status(200) .header("content-type", "application/json") .body(Full::new(Bytes::from(health_response.to_string()))) .unwrap_or_else(|e| { // Return a minimal error response if primary response building fails error!("Failed to build primary health response: {}", e); // Try to build error response match hyper::Response::builder() .status(500) .header("content-type", "application/json") .body(Full::new(Bytes::from( r#"{"status":"error","message":"Health check failed"}"#, ))) { Ok(err_response) => err_response, Err(fatal_err) => { // Last resort: return minimal response without builder error!("FATAL: Cannot build any HTTP response: {}", fatal_err); hyper::Response::new(Full::new(Bytes::from(r#"{"status":"error"}"#))) }, } }); Ok(response) } /// Handle shutdown signals async fn shutdown_signal() { let ctrl_c = async { if let Err(e) = signal::ctrl_c().await { error!("Failed to install Ctrl+C handler: {}", e); } }; #[cfg(unix)] let terminate = async { match signal::unix::signal(signal::unix::SignalKind::terminate()) { Ok(mut signal_stream) => { signal_stream.recv().await; }, Err(e) => { error!("Failed to install SIGTERM handler: {}", e); }, } }; #[cfg(not(unix))] let terminate = std::future::pending::<()>(); tokio::select! { _ = ctrl_c => {}, _ = terminate => {}, } info!("Shutdown signal received"); } /// Monitor kill switch status and log important events async fn monitor_kill_switch_status(kill_switch_system: Arc) { let mut interval = tokio::time::interval(Duration::from_secs(30)); loop { interval.tick().await; match kill_switch_system.get_status().await { Ok(status) => { if status.is_active { error!( "KILL SWITCH ACTIVE - Trading blocked | Checks: {} | Commands: {} | Error Rate: {:.2}%", status.total_checks, status.total_commands, status.error_rate * 100.0 ); } if status.is_emergency_active { error!("🚨🚨🚨 EMERGENCY SHUTDOWN ACTIVE - System halted"); } if !status.is_healthy { warn!( "⚠️ Kill switch system unhealthy - Error rate: {:.2}% | Consecutive failures: {}", status.error_rate * 100.0, status.consecutive_failures ); } // Log periodic status (every 5 minutes) if status.total_checks % 100 == 0 { info!( "Kill switch status: Active={} | Healthy={} | Checks={} | Commands={}", status.is_active, status.is_healthy, status.total_checks, status.total_commands ); } }, Err(e) => { error!("Failed to get kill switch status: {}", e); }, } } }