foxhunt/services/trading_service/src/main.rs

#![deny(clippy::unwrap_used, clippy::expect_used)]

//! Trading Service - Main Entry Point
//!
//! This is the main entry point for the Foxhunt HFT Trading Service.
//! It initializes all components including the PostgreSQL ConfigLoader
//! for direct configuration management with hot-reload support.

use anyhow::{Context, Result};
use std::sync::Arc;
use std::time::Duration;
use tokio::signal;
use tonic::transport::Server;
use tracing::{debug, error, info, warn};

use trading_service::auth_interceptor::{AuthConfig, TonicAuthInterceptor};

// Use central configuration and shared libraries with canonical imports
use common::DatabasePool;
use config::manager::ConfigManager;
use config::DatabaseConfig;

// Import repository dependencies with explicit imports
use trading_service::repositories::{
    ConfigRepository, MarketDataRepository, RiskRepository, TradingRepository,
};
use trading_service::repository_impls::{
    PostgresConfigRepository, PostgresMarketDataRepository, PostgresRiskRepository,
    PostgresTradingRepository,
};

use trading_service::compliance_service::{ComplianceConfig, ComplianceService};
use trading_service::event_persistence::EventPersistence;
use trading_service::kill_switch_integration::TradingServiceKillSwitch;
use trading_service::rate_limiter::{RateLimitConfig, RateLimiter};
use trading_service::services::ml::MLService;
use trading_service::services::ml_fallback_manager::MLFallbackManager;
use trading_service::services::ml_performance_monitor::MLPerformanceMonitor;
use trading_service::services::monitoring::MonitoringServiceImpl;
use trading_service::services::risk::RiskServiceImpl;
use trading_service::services::trading::TradingServiceImpl;
use trading_service::feedback_loop::{FeedbackLoop, FeedbackLoopConfig};
use trading_service::questdb_metrics::QuestDBMetricsProvider;
use trading_service::state::TradingServiceState;

use common::metrics::GrpcMetricsLayer;

/// Default configuration values
const DEFAULT_GRPC_PORT: u16 = 50052; // Trading Service port (API Gateway uses 50051)
const DEFAULT_HEALTH_PORT: u16 = 8081; // Health check port (separate from API Gateway 8080)

#[tokio::main]
async fn main() -> Result<()> {
    // Initialize observability (JSON logging + OpenTelemetry tracing via OTLP)
    let otlp_endpoint = std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT")
        .unwrap_or_else(|_| "http://localhost:4317".to_string());
    if let Err(e) = common::observability::init_observability(
        "trading_service",
        Some(&otlp_endpoint),
    ) {
        eprintln!("Failed to initialize observability: {}", e);
    }

    info!("Starting Foxhunt Trading Service...");

    // Create service config manually
    let service_config = config::ServiceConfig {
        name: "trading_service".to_string(),
        environment: std::env::var("ENVIRONMENT").unwrap_or_else(|_| "production".to_string()),
        version: env!("CARGO_PKG_VERSION").to_string(),
        settings: serde_json::json!({}),
    };
    let _config_manager = Arc::new(ConfigManager::new(service_config));

    info!("Central ConfigManager initialized successfully");

    // Get database URL from environment (DatabaseConfig::new() already handles this)
    let mut database_config = DatabaseConfig::new();
    // DatabaseConfig::new() already sets url from DATABASE_URL env var
    database_config.max_connections = 20;
    database_config.min_connections = 5;

    // Initialize HFT-optimized database pool directly from config
    let db_pool_wrapper = DatabasePool::new(database_config.into())
        .await
        .context("Failed to create HFT-optimized database pool")?;

    let db_pool = db_pool_wrapper.pool().clone();

    info!("Database connection pool initialized");

    // Initialize repositories with dependency injection
    let trading_repository: Arc<dyn TradingRepository> =
        Arc::new(PostgresTradingRepository::new(db_pool.clone()));
    let market_data_repository: Arc<dyn MarketDataRepository> =
        Arc::new(PostgresMarketDataRepository::new(db_pool.clone()));
    let risk_repository: Arc<dyn RiskRepository> =
        Arc::new(PostgresRiskRepository::new(db_pool.clone()));
    let config_repository_impl = Arc::new(PostgresConfigRepository::new(db_pool.clone()));

    info!("Repository layer initialized with dependency injection");

    // Initialize event persistence for compliance and audit trail
    let process_id = std::process::id();
    let event_persistence = Arc::new(EventPersistence::new(
        db_pool.clone(),
        "trading_service".to_string(),
        process_id,
    ));
    info!("Event persistence initialized for compliance audit trail");

    // Initialize default configurations if they don't exist
    initialize_default_configs(&config_repository_impl).await?;

    // Initialize kill switch system for regulatory compliance
    let redis_url =
        std::env::var("REDIS_URL").unwrap_or_else(|_| "redis://localhost:6379".to_string());
    let kill_switch_system = Arc::new(
        TradingServiceKillSwitch::new(redis_url)
            .await
            .context("Failed to initialize kill switch system")?,
    );
    info!("Kill switch system initialized for regulatory compliance");

    // Start kill switch monitoring (Unix socket, signal handlers, etc.)
    kill_switch_system
        .start_monitoring()
        .await
        .context("Failed to start kill switch monitoring")?;
    info!("Kill switch monitoring started - emergency shutdown ready");

    // Start configuration hot-reload monitoring
    start_config_monitoring(Arc::clone(&config_repository_impl)).await?;

    // The centralized config manager now handles all configuration persistence
    // and provenance tracking internally

    // Initialize authentication configuration using Tonic interceptor
    let auth_config = initialize_auth_config().await;
    let auth_interceptor = TonicAuthInterceptor::new(auth_config);

    info!("✅ Authentication interceptor initialized with Tonic 0.14 compatibility");

    // Initialize compliance service for SOX and MiFID II regulatory requirements
    let compliance_config = ComplianceConfig {
        enable_sox_audit: std::env::var("ENABLE_SOX_AUDIT")
            .ok()
            .and_then(|s| s.parse().ok())
            .unwrap_or(true), // Production default: enabled
        enable_mifid_reporting: std::env::var("ENABLE_MIFID_REPORTING")
            .ok()
            .and_then(|s| s.parse().ok())
            .unwrap_or(true), // Production default: enabled
        enable_position_monitoring: std::env::var("ENABLE_POSITION_MONITORING")
            .ok()
            .and_then(|s| s.parse().ok())
            .unwrap_or(true), // Production default: enabled
        enable_best_execution_analysis: std::env::var("ENABLE_BEST_EXECUTION_ANALYSIS")
            .ok()
            .and_then(|s| s.parse().ok())
            .unwrap_or(true), // Production default: enabled
        kill_switch_enabled: std::env::var("COMPLIANCE_KILL_SWITCH_ENABLED")
            .ok()
            .and_then(|s| s.parse().ok())
            .unwrap_or(true), // Production default: enabled
        max_position_utilization: std::env::var("MAX_POSITION_UTILIZATION")
            .ok()
            .and_then(|s| s.parse().ok())
            .unwrap_or(0.95), // 95% default
        critical_risk_threshold: std::env::var("CRITICAL_RISK_THRESHOLD")
            .ok()
            .and_then(|s| s.parse().ok())
            .unwrap_or(0.8), // 80% default
    };

    let compliance_service = Arc::new(ComplianceService::new(db_pool.clone(), compliance_config));

    // Verify compliance service health
    if let Err(e) = compliance_service.health_check().await {
        error!("Compliance service health check failed: {}", e);
        return Err(anyhow::anyhow!("Failed to initialize compliance service"));
    }

    info!("Compliance service initialized with SOX and MiFID II audit trails");

    // Initialize advanced rate limiter with production defaults
    let rate_limit_config = RateLimitConfig {
        user_requests_per_minute: std::env::var("USER_REQUESTS_PER_MINUTE")
            .ok()
            .and_then(|s| s.parse().ok())
            .unwrap_or(1000), // 1000 requests per minute per user
        user_burst_capacity: std::env::var("USER_BURST_CAPACITY")
            .ok()
            .and_then(|s| s.parse().ok())
            .unwrap_or(100), // Allow bursts up to 100 requests
        ip_requests_per_minute: std::env::var("IP_REQUESTS_PER_MINUTE")
            .ok()
            .and_then(|s| s.parse().ok())
            .unwrap_or(2000), // 2000 requests per minute per IP
        ip_burst_capacity: std::env::var("IP_BURST_CAPACITY")
            .ok()
            .and_then(|s| s.parse().ok())
            .unwrap_or(200), // Allow bursts up to 200 requests
        global_requests_per_minute: std::env::var("GLOBAL_REQUESTS_PER_MINUTE")
            .ok()
            .and_then(|s| s.parse().ok())
            .unwrap_or(50000), // 50k requests per minute globally
        global_burst_capacity: std::env::var("GLOBAL_BURST_CAPACITY")
            .ok()
            .and_then(|s| s.parse().ok())
            .unwrap_or(5000), // Allow bursts up to 5k requests
        auth_failures_per_minute: std::env::var("AUTH_FAILURES_PER_MINUTE")
            .ok()
            .and_then(|s| s.parse().ok())
            .unwrap_or(5), // Max 5 auth failures per minute
        auth_failure_penalty_minutes: std::env::var("AUTH_FAILURE_PENALTY_MINUTES")
            .ok()
            .and_then(|s| s.parse().ok())
            .unwrap_or(15), // 15 minute penalty
        orders_per_minute: std::env::var("ORDERS_PER_MINUTE")
            .ok()
            .and_then(|s| s.parse().ok())
            .unwrap_or(600), // 600 orders per minute (10/second)
        order_burst_capacity: std::env::var("ORDER_BURST_CAPACITY")
            .ok()
            .and_then(|s| s.parse().ok())
            .unwrap_or(60), // Allow bursts up to 60 orders
        cleanup_interval_minutes: std::env::var("RATE_LIMIT_CLEANUP_INTERVAL")
            .ok()
            .and_then(|s| s.parse().ok())
            .unwrap_or(60), // Cleanup every hour
    };

    let rate_limiter = Arc::new(RateLimiter::new(rate_limit_config));

    // Start background cleanup task for rate limiter
    Arc::clone(&rate_limiter).start_cleanup_task().await;

    info!("Advanced rate limiter initialized with per-user, per-IP, and global limits");
    info!("Authentication system initialized with mTLS and JWT support");

    // Initialize ensemble coordinator with real ML inference adapters
    let ensemble_coordinator = {
        use trading_service::adapter_bridge::InferenceAdapterBridge;
        use trading_service::ensemble_coordinator::EnsembleCoordinator;

        let coordinator = EnsembleCoordinator::new();

        // Create real candle-backed inference adapters and register them
        // via the InferenceAdapterBridge (ModelInferenceAdapter -> MLModel).
        // Models are initialised with random weights; production checkpoints
        // are hot-loaded later via the model registry's shadow-buffer swap.

        // DQN adapter (51-dim input, 45 factored actions)
        match ml::ensemble::adapters::DqnInferenceAdapter::new(ml::dqn::dqn::DQNConfig {
            state_dim: 51,
            num_actions: 45,
            hidden_dims: vec![64, 64],
            ..Default::default()
        }) {
            Ok(adapter) => {
                let bridge = Arc::new(InferenceAdapterBridge::new(
                    Box::new(adapter),
                    "DQN".to_string(),
                    ml::ModelType::DQN,
                ));
                if let Err(e) = coordinator
                    .register_loaded_model("DQN".to_string(), bridge, 0.10)
                    .await
                {
                    error!("Failed to register DQN adapter: {}", e);
                } else {
                    info!("Registered DQN inference adapter (51-dim, candle)");
                }
            }
            Err(e) => error!("Failed to create DQN inference adapter: {}", e),
        }

        // PPO adapter (51-dim input, 3 actions: buy/sell/hold)
        match ml::ensemble::adapters::PpoInferenceAdapter::new(ml::ppo::ppo::PPOConfig {
            state_dim: 51,
            num_actions: 3,
            policy_hidden_dims: vec![64, 64],
            value_hidden_dims: vec![64, 64],
            ..Default::default()
        }) {
            Ok(adapter) => {
                let bridge = Arc::new(InferenceAdapterBridge::new(
                    Box::new(adapter),
                    "PPO".to_string(),
                    ml::ModelType::PPO,
                ));
                if let Err(e) = coordinator
                    .register_loaded_model("PPO".to_string(), bridge, 0.10)
                    .await
                {
                    error!("Failed to register PPO adapter: {}", e);
                } else {
                    info!("Registered PPO inference adapter (51-dim, candle)");
                }
            }
            Err(e) => error!("Failed to create PPO inference adapter: {}", e),
        }

        // TFT adapter (51-dim split: 3 static + 8 known + 40 unknown)
        match ml::ensemble::adapters::TftInferenceAdapter::new(
            ml::tft::TFTConfig {
                input_dim: 51,
                hidden_dim: 32,
                num_heads: 4,
                num_layers: 1,
                prediction_horizon: 1,
                sequence_length: 10,
                num_quantiles: 3,
                num_static_features: 3,
                num_known_features: 8,
                num_unknown_features: 40,
                ..Default::default()
            },
            10, // sequence_length for buffer
        ) {
            Ok(adapter) => {
                let bridge = Arc::new(InferenceAdapterBridge::new(
                    Box::new(adapter),
                    "TFT".to_string(),
                    ml::ModelType::TFT,
                ));
                if let Err(e) = coordinator
                    .register_loaded_model("TFT".to_string(), bridge, 0.10)
                    .await
                {
                    error!("Failed to register TFT adapter: {}", e);
                } else {
                    info!("Registered TFT inference adapter (51-dim, candle)");
                }
            }
            Err(e) => error!("Failed to create TFT inference adapter: {}", e),
        }

        // Mamba2 adapter (d_model=64, zero-pads from 51-dim features)
        match ml::ensemble::adapters::Mamba2InferenceAdapter::new(
            ml::mamba::Mamba2Config {
                d_model: 64,
                d_state: 16,
                d_head: 16,
                num_heads: 2,
                expand: 1,
                num_layers: 1,
                ..Default::default()
            },
            10, // sequence_length for buffer
        ) {
            Ok(adapter) => {
                let bridge = Arc::new(InferenceAdapterBridge::new(
                    Box::new(adapter),
                    "MAMBA2".to_string(),
                    ml::ModelType::MAMBA,
                ));
                if let Err(e) = coordinator
                    .register_loaded_model("MAMBA2".to_string(), bridge, 0.10)
                    .await
                {
                    error!("Failed to register MAMBA2 adapter: {}", e);
                } else {
                    info!("Registered MAMBA2 inference adapter (64-dim SSM, candle)");
                }
            }
            Err(e) => error!("Failed to create MAMBA2 inference adapter: {}", e),
        }

        // Liquid CfC adapter (51-dim input, 3 output: buy/hold/sell)
        match ml::ensemble::adapters::LiquidInferenceAdapter::new(
            ml::liquid::candle_cfc::CfCTrainConfig {
                input_size: 51,
                hidden_size: 32,
                output_size: 3,
                backbone_hidden_sizes: vec![32],
                seq_len: 1,
                device: ml::liquid::candle_cfc::DeviceConfig::Cpu,
                ..ml::liquid::candle_cfc::CfCTrainConfig::default()
            },
        ) {
            Ok(adapter) => {
                let bridge = Arc::new(InferenceAdapterBridge::new(
                    Box::new(adapter),
                    "Liquid-CfC".to_string(),
                    ml::ModelType::LNN,
                ));
                if let Err(e) = coordinator
                    .register_loaded_model("Liquid-CfC".to_string(), bridge, 0.10)
                    .await
                {
                    error!("Failed to register Liquid-CfC adapter: {}", e);
                } else {
                    info!("Registered Liquid-CfC inference adapter (51-dim, candle ODE)");
                }
            }
            Err(e) => error!("Failed to create Liquid-CfC inference adapter: {}", e),
        }

        // TGGN adapter (51-dim input, 2-layer candle projection)
        match ml::ensemble::adapters::TggnInferenceAdapter::new(51, 64) {
            Ok(adapter) => {
                let bridge = Arc::new(InferenceAdapterBridge::new(
                    Box::new(adapter),
                    "TGGN".to_string(),
                    ml::ModelType::TGGN,
                ));
                if let Err(e) = coordinator
                    .register_loaded_model("TGGN".to_string(), bridge, 0.10)
                    .await
                {
                    error!("Failed to register TGGN adapter: {}", e);
                } else {
                    info!("Registered TGGN inference adapter (51-dim, candle projection)");
                }
            }
            Err(e) => error!("Failed to create TGGN inference adapter: {}", e),
        }

        // TLOB adapter (51-dim features, 3-layer MLP, seq_len=10)
        match ml::ensemble::adapters::TlobInferenceAdapter::new(51, 64, 10) {
            Ok(adapter) => {
                let bridge = Arc::new(InferenceAdapterBridge::new(
                    Box::new(adapter),
                    "TLOB".to_string(),
                    ml::ModelType::TLOB,
                ));
                if let Err(e) = coordinator
                    .register_loaded_model("TLOB".to_string(), bridge, 0.10)
                    .await
                {
                    error!("Failed to register TLOB adapter: {}", e);
                } else {
                    info!("Registered TLOB inference adapter (51-dim, seq=10, candle MLP)");
                }
            }
            Err(e) => error!("Failed to create TLOB inference adapter: {}", e),
        }

        // KAN adapter (51-dim input, B-spline activations)
        match ml::ensemble::adapters::KanInferenceAdapter::new(ml::kan::KANConfig {
            layer_widths: vec![51, 32, 16, 1],
            ..Default::default()
        }) {
            Ok(adapter) => {
                let bridge = Arc::new(InferenceAdapterBridge::new(
                    Box::new(adapter),
                    "KAN".to_string(),
                    ml::ModelType::KAN,
                ));
                if let Err(e) = coordinator
                    .register_loaded_model("KAN".to_string(), bridge, 0.10)
                    .await
                {
                    error!("Failed to register KAN adapter: {}", e);
                } else {
                    info!("Registered KAN inference adapter (51-dim, B-spline, candle)");
                }
            }
            Err(e) => error!("Failed to create KAN inference adapter: {}", e),
        }

        // xLSTM adapter (51-dim input, sLSTM+mLSTM, seq_len=10)
        match ml::ensemble::adapters::XlstmInferenceAdapter::new(
            ml::xlstm::XLSTMConfig {
                input_dim: 51,
                hidden_dim: 64,
                num_blocks: 2,
                num_heads: 2,
                ..Default::default()
            },
            10, // sequence_length for buffer
        ) {
            Ok(adapter) => {
                let bridge = Arc::new(InferenceAdapterBridge::new(
                    Box::new(adapter),
                    "xLSTM".to_string(),
                    ml::ModelType::XLSTM,
                ));
                if let Err(e) = coordinator
                    .register_loaded_model("xLSTM".to_string(), bridge, 0.10)
                    .await
                {
                    error!("Failed to register xLSTM adapter: {}", e);
                } else {
                    info!("Registered xLSTM inference adapter (51-dim, seq=10, candle)");
                }
            }
            Err(e) => error!("Failed to create xLSTM inference adapter: {}", e),
        }

        // Diffusion adapter (51-dim features, DDPM denoiser, t=1 inference)
        match ml::ensemble::adapters::DiffusionInferenceAdapter::new(ml::diffusion::DiffusionConfig {
            seq_len: 1,
            feature_dim: 51,
            hidden_dim: 64,
            num_layers: 2,
            time_embed_dim: 32,
            num_timesteps: 100,
            sampling_steps: 5,
            ..Default::default()
        }) {
            Ok(adapter) => {
                let bridge = Arc::new(InferenceAdapterBridge::new(
                    Box::new(adapter),
                    "Diffusion".to_string(),
                    ml::ModelType::Diffusion,
                ));
                if let Err(e) = coordinator
                    .register_loaded_model("Diffusion".to_string(), bridge, 0.10)
                    .await
                {
                    error!("Failed to register Diffusion adapter: {}", e);
                } else {
                    info!("Registered Diffusion inference adapter (51-dim, DDPM, candle)");
                }
            }
            Err(e) => error!("Failed to create Diffusion inference adapter: {}", e),
        }

        let model_count = coordinator.model_count().await;
        info!(
            "Ensemble coordinator initialized with {} real inference adapters \
             (all 10 models at 0.10 weight each)",
            model_count
        );

        Arc::new(coordinator)
    };

    // Initialize service state with repository dependency injection
    let service_state = TradingServiceState::new_with_repositories(
        trading_repository,
        market_data_repository,
        risk_repository,
        Arc::clone(&config_repository_impl),
        db_pool.clone(),
        Arc::clone(&event_persistence),
        Some(Arc::clone(&kill_switch_system)),
        Some(ensemble_coordinator),
    )
    .await?;
    info!("Trading service state initialized with repository dependency injection and ensemble coordinator");

    // Initialize ML performance monitoring and fallback management
    let ml_performance_monitor = Arc::new(MLPerformanceMonitor::new());
    let ml_fallback_manager = Arc::new(MLFallbackManager::new());

    info!("ML performance monitoring and fallback management initialized");

    // Initialize paper trading executor for prediction consumption
    use trading_service::paper_trading_executor::{PaperTradingConfig, PaperTradingExecutor};

    let paper_trading_config = PaperTradingConfig {
        enabled: std::env::var("PAPER_TRADING_ENABLED")
            .ok()
            .and_then(|s| s.parse().ok())
            .unwrap_or(true), // Default: enabled
        min_confidence: std::env::var("PAPER_TRADING_MIN_CONFIDENCE")
            .ok()
            .and_then(|s| s.parse().ok())
            .unwrap_or(0.60), // 60% minimum confidence
        poll_interval_ms: std::env::var("PAPER_TRADING_POLL_INTERVAL_MS")
            .ok()
            .and_then(|s| s.parse().ok())
            .unwrap_or(100), // 100ms polling
        max_position_size: std::env::var("PAPER_TRADING_MAX_POSITION_SIZE")
            .ok()
            .and_then(|s| s.parse().ok())
            .unwrap_or(10_000.0), // $10,000 max position
        allowed_symbols: std::env::var("PAPER_TRADING_ALLOWED_SYMBOLS")
            .ok()
            .map(|s| s.split(',').map(|sym| sym.trim().to_string()).collect())
            .unwrap_or_else(|| {
                vec![
                    "ES.FUT".to_string(),
                    "NQ.FUT".to_string(),
                    "ZN.FUT".to_string(),
                    "6E.FUT".to_string(),
                ]
            }),
        account_id: std::env::var("PAPER_TRADING_ACCOUNT_ID")
            .unwrap_or_else(|_| "paper_trading_001".to_string()),
        initial_capital: std::env::var("PAPER_TRADING_INITIAL_CAPITAL")
            .ok()
            .and_then(|s| s.parse().ok())
            .unwrap_or(100_000.0), // $100,000 initial capital
        batch_size: std::env::var("PAPER_TRADING_BATCH_SIZE")
            .ok()
            .and_then(|s| s.parse().ok())
            .unwrap_or(100), // Process 100 predictions per batch
    };

    let paper_trading_executor = Arc::new(PaperTradingExecutor::new(
        db_pool.clone(),
        paper_trading_config.clone(),
    )?);

    info!(
        "Paper trading executor initialized: enabled={}, min_confidence={:.1}%, poll_interval={}ms",
        paper_trading_config.enabled,
        paper_trading_config.min_confidence * 100.0,
        paper_trading_config.poll_interval_ms
    );

    // Spawn paper trading executor background task
    let executor_clone = Arc::clone(&paper_trading_executor);
    tokio::spawn(async move {
        info!("Paper trading executor background task starting...");
        if let Err(e) = executor_clone.start().await {
            error!("Paper trading executor failed: {}", e);
        }
    });

    // Spawn background ML prediction generation loop (Wave 14.2 Agent 7)
    // This generates predictions every 60 seconds and saves to ensemble_predictions table
    use trading_service::prediction_generation_loop::{
        PredictionGenerationLoop, PredictionLoopConfig,
    };

    // Shutdown senders for background loops -- dropped at end of main to signal shutdown.
    let mut prediction_shutdown_handles: Vec<tokio::sync::broadcast::Sender<()>> = Vec::new();

    if let Some(ensemble_coordinator) = service_state.ensemble_coordinator() {
        let prediction_config = PredictionLoopConfig::from_env();
        let prediction_loop = PredictionGenerationLoop::new(
            Arc::clone(ensemble_coordinator),
            db_pool.clone(),
            prediction_config.clone(),
        );

        // Create shutdown channel for prediction loop
        let (prediction_shutdown_tx, prediction_shutdown_rx) = tokio::sync::broadcast::channel(1);

        // Spawn prediction generation loop
        tokio::spawn(async move {
            info!(
                "ML prediction generation loop starting: symbols={:?}, interval={:?}",
                prediction_config.symbols, prediction_config.prediction_interval
            );
            if let Err(e) = prediction_loop.run(prediction_shutdown_rx).await {
                error!("Prediction generation loop failed: {}", e);
            }
        });

        info!("✅ Background ML prediction generation loop initialized");

        // Store shutdown sender so it is dropped during graceful shutdown,
        // which closes the channel and signals the prediction loop to stop.
        prediction_shutdown_handles.push(prediction_shutdown_tx);
    } else {
        info!("Ensemble coordinator not available - prediction generation loop disabled");
    }

    // Spawn background market data feed for ensemble feature extractor warmup.
    // The ensemble coordinator's `update_market_data()` feeds per-symbol
    // `ProductionFeatureExtractorAdapter`s that require ~51 bars before they can
    // produce real 51-dim feature vectors.  Without this feed the extractors stay
    // cold and `fetch_features_for_symbol` returns zeros.
    //
    // In production this task would be replaced by a Databento or exchange feed;
    // for now it generates synthetic OHLCV bars at 1-second intervals.
    if let Some(ensemble_coordinator) = service_state.ensemble_coordinator() {
        let coord = Arc::clone(ensemble_coordinator);

        let warmup_symbols: Vec<String> = std::env::var("MARKET_FEED_SYMBOLS")
            .ok()
            .map(|s| s.split(',').map(|sym| sym.trim().to_string()).collect())
            .unwrap_or_else(|| {
                vec![
                    "ES.FUT".to_string(),
                    "NQ.FUT".to_string(),
                    "ZN.FUT".to_string(),
                    "6E.FUT".to_string(),
                ]
            });

        let feed_interval_ms: u64 = std::env::var("MARKET_FEED_INTERVAL_MS")
            .ok()
            .and_then(|s| s.parse().ok())
            .unwrap_or(1_000); // 1 second default

        // Base prices per symbol (representative of real futures contracts)
        let base_prices: Vec<f64> = warmup_symbols
            .iter()
            .map(|sym| match sym.as_str() {
                "ES.FUT" => 4_500.0,
                "NQ.FUT" => 15_800.0,
                "ZN.FUT" => 110.5,
                "6E.FUT" => 1.085,
                _ => 100.0,
            })
            .collect();

        tokio::spawn(async move {
            use rand::Rng;
            use rand::SeedableRng;

            info!(
                "Market data feed starting: symbols={:?}, interval={}ms, warmup_target={} bars",
                warmup_symbols,
                feed_interval_ms,
                trading_service::ensemble_coordinator::FEATURE_WARMUP_BARS,
            );

            let mut interval =
                tokio::time::interval(Duration::from_millis(feed_interval_ms));
            let mut tick: u64 = 0;
            let mut rng = rand::rngs::StdRng::from_entropy();
            let mut prices: Vec<f64> = base_prices;
            let warmup_target = trading_service::ensemble_coordinator::FEATURE_WARMUP_BARS as u64;

            loop {
                interval.tick().await;
                let now = chrono::Utc::now();

                for (idx, symbol) in warmup_symbols.iter().enumerate() {
                    let price = match prices.get(idx) {
                        Some(p) => *p,
                        None => continue,
                    };

                    // Random walk: +/- 0.02% per tick (realistic micro-volatility)
                    let pct_change: f64 = rng.gen_range(-0.0002..0.0002);
                    let new_price = price * (1.0 + pct_change);
                    let volume = rng.gen_range(500.0..5_000.0);

                    if let Some(slot) = prices.get_mut(idx) {
                        *slot = new_price;
                    }

                    if let Err(e) = coord
                        .update_market_data(symbol, new_price, volume, now)
                        .await
                    {
                        debug!(
                            "Market data feed: failed to update {} (tick {}): {}",
                            symbol, tick, e
                        );
                    }
                }

                tick += 1;

                // Log warmup progress at key milestones
                if tick == warmup_target {
                    info!(
                        "Market data feed: warmup complete ({} bars) -- \
                         feature extraction now producing real vectors for {:?}",
                        warmup_target, warmup_symbols,
                    );
                } else if tick < warmup_target && tick % 10 == 0 {
                    info!(
                        "Market data feed: warmup progress {}/{} bars",
                        tick, warmup_target,
                    );
                }
            }
        });

        info!(
            "Background market data feed spawned for ensemble feature extractor warmup"
        );
    } else {
        info!("Ensemble coordinator not available - market data feed disabled");
    }

    // Initialize autonomous feedback loop (weight + gate optimization with kill switch)
    {
        let feedback_config = FeedbackLoopConfig {
            cycle_interval: Duration::from_secs(
                std::env::var("FEEDBACK_LOOP_INTERVAL_SECS")
                    .ok()
                    .and_then(|s| s.parse().ok())
                    .unwrap_or(24 * 3600),
            ),
            kill_switch_threshold: std::env::var("FEEDBACK_KILL_SWITCH_SHARPE")
                .ok()
                .and_then(|s| s.parse().ok())
                .unwrap_or(-1.0),
            ..Default::default()
        };

        let feedback_loop = Arc::new(FeedbackLoop::new(feedback_config));

        // Connect to QuestDB for real metrics (pg wire protocol on port 8812)
        let questdb_url = std::env::var("QUESTDB_PG_URL")
            .unwrap_or_else(|_| "postgresql://admin:quest@localhost:8812/qdb".to_string());

        match QuestDBMetricsProvider::try_new(&questdb_url).await {
            Some(provider) => {
                if let Err(e) = provider.ensure_tables().await {
                    warn!("Failed to create QuestDB tables: {} — feedback loop will retry", e);
                }
                let metrics_provider: Arc<dyn trading_service::feedback_loop::MetricsProvider> =
                    Arc::new(provider);
                let _feedback_handle = Arc::clone(&feedback_loop).spawn(metrics_provider);
                info!("Autonomous feedback loop spawned with QuestDB metrics provider");
            }
            None => {
                info!("QuestDB unavailable — feedback loop disabled until QUESTDB_PG_URL is reachable");
            }
        }
    }

    // Subscribe to ML performance alerts
    let monitor_clone = Arc::clone(&ml_performance_monitor);
    tokio::spawn(async move {
        use trading_service::ml_metrics;
        let mut alert_receiver = monitor_clone.subscribe_alerts();

        info!("ML performance alert handler started");

        while let Ok(alert) = alert_receiver.recv().await {
            // Log alert based on severity
            match alert.severity {
                trading_service::services::ml_performance_monitor::AlertSeverity::Emergency
                | trading_service::services::ml_performance_monitor::AlertSeverity::Critical => {
                    error!("ML ALERT [{}]: {}", alert.model_id, alert.message);
                },
                trading_service::services::ml_performance_monitor::AlertSeverity::Warning => {
                    warn!("ML ALERT [{}]: {}", alert.model_id, alert.message);
                },
                trading_service::services::ml_performance_monitor::AlertSeverity::Info => {
                    info!("ML ALERT [{}]: {}", alert.model_id, alert.message);
                },
            }

            // Update Prometheus alert counter
            ml_metrics::ML_ALERTS_TOTAL
                .with_label_values(&[
                    &alert.model_id,
                    &ml_metrics::alert_type_str(&alert.alert_type).to_string(),
                    &ml_metrics::alert_severity_str(&alert.severity).to_string(),
                ])
                .inc();
        }

        info!("ML performance alert handler stopped");
    });

    // Create gRPC services with enhanced ML capabilities
    // Services expect TradingServiceState directly, not Arc
    let trading_service = TradingServiceImpl::new(Arc::new(service_state.clone()));
    let risk_service = RiskServiceImpl::new(service_state.clone());
    let ml_service = MLService::new(
        service_state.clone(),
        Arc::clone(&ml_performance_monitor),
        Arc::clone(&ml_fallback_manager),
    );
    let config_service = trading_service::services::config::ConfigServiceImpl::new(db_pool.clone());
    let monitoring_service = MonitoringServiceImpl::new(service_state);

    // Create health service
    let (health_reporter, health_service) = tonic_health::server::health_reporter();
    health_reporter
        .set_serving::<trading_service::proto::trading::trading_service_server::TradingServiceServer<TradingServiceImpl>>()
        .await;

    // Build gRPC server with TLS, authentication, and rate limiting
    let grpc_port = std::env::var("GRPC_PORT")
        .ok()
        .and_then(|s| s.parse().ok())
        .unwrap_or(DEFAULT_GRPC_PORT);
    let addr = format!("0.0.0.0:{}", grpc_port).parse()?;

    // Load TLS configuration if enabled
    let tls_config = if std::env::var("TLS_ENABLED")
        .unwrap_or_else(|_| "false".to_string())
        .parse::<bool>()
        .unwrap_or(false)
    {
        info!("Loading TLS configuration...");
        use trading_service::tls_config::TradingServiceTlsConfig;

        let tls = TradingServiceTlsConfig::from_files(
            &std::env::var("TLS_CERT_PATH")
                .unwrap_or_else(|_| "/app/certs/trading_service/server.crt".to_string()),
            &std::env::var("TLS_KEY_PATH")
                .unwrap_or_else(|_| "/app/certs/trading_service/server.key".to_string()),
            &std::env::var("TLS_CA_PATH")
                .unwrap_or_else(|_| "/app/certs/trading_service/ca.crt".to_string()),
            std::env::var("TLS_REQUIRE_CLIENT_CERT")
                .unwrap_or_else(|_| "false".to_string())
                .parse::<bool>()
                .unwrap_or(false),
        )
        .await?;

        info!("✓ TLS 1.3 enabled with mTLS client certificate validation");
        Some(tls.to_server_tls_config())
    } else {
        warn!("⚠ TLS DISABLED - Running in insecure mode (development only)");
        None
    };

    info!("🔒 Starting gRPC server with authentication enabled");

    // Wave 67 Agent 3: HTTP/2 streaming performance optimizations
    // Load streaming configuration with feature flag support
    let streaming_config = trading_service::streaming::StreamingConfig::from_env();

    if streaming_config.is_enabled() {
        info!("✅ HTTP/2 optimizations enabled:");
        info!(
            "   - tcp_nodelay: {} (-40ms Nagle delay)",
            streaming_config.tcp_nodelay
        );
        info!(
            "   - Stream window: {}KB",
            streaming_config.initial_stream_window_size / 1024
        );
        info!(
            "   - Connection window: {}MB",
            streaming_config.initial_connection_window_size / (1024 * 1024)
        );
        info!(
            "   - Adaptive window: {}",
            streaming_config.http2_adaptive_window
        );
        info!("   - Max streams: 10,000 (production scale)");
    } else {
        info!("⚠️  HTTP/2 optimizations disabled via feature flag");
    }

    // Start Prometheus metrics HTTP endpoint on port 9092
    use trading_service::metrics_server::{MetricsServerConfig, TradingMetricsServer};

    let metrics_config = MetricsServerConfig {
        bind_address: "0.0.0.0".to_string(),
        bind_port: 9092,
        scrape_timeout_ms: 500,
        max_metrics_per_scrape: 10000,
    };

    let metrics_server = TradingMetricsServer::new(metrics_config);
    tokio::spawn(async move {
        if let Err(e) = metrics_server.start().await {
            tracing::error!("Metrics server failed: {}", e);
        }
    });

    // Apply authentication interceptor to all gRPC services
    let mut server_builder = match tls_config {
        Some(tls) => Server::builder()
            .layer(GrpcMetricsLayer::new("trading-service"))
            .tls_config(tls)
            .context("Failed to configure TLS")?,
        None => Server::builder()
            .layer(GrpcMetricsLayer::new("trading-service")),
    };

    // Apply HTTP/2 optimizations if enabled
    if streaming_config.is_enabled() {
        server_builder = server_builder
            .tcp_nodelay(streaming_config.tcp_nodelay)  // Critical: eliminates 40ms Nagle delay
            .http2_keepalive_interval(Some(streaming_config.http2_keepalive_interval))
            .http2_keepalive_timeout(Some(streaming_config.http2_keepalive_timeout))
            .initial_stream_window_size(Some(streaming_config.initial_stream_window_size))
            .initial_connection_window_size(Some(streaming_config.initial_connection_window_size))
            .http2_adaptive_window(Some(streaming_config.http2_adaptive_window))
            .max_concurrent_streams(Some(10_000)); // Increased from 1,024 to 10,000 for production scale
    }

    let server = server_builder
        .add_service(health_service)
        .add_service(
            trading_service::proto::trading::trading_service_server::TradingServiceServer::with_interceptor(
                trading_service,
                auth_interceptor.clone()
            )
        )
        .add_service(
            trading_service::proto::risk::risk_service_server::RiskServiceServer::with_interceptor(
                risk_service,
                auth_interceptor.clone()
            )
        )
        .add_service(
            trading_service::proto::ml::ml_service_server::MlServiceServer::with_interceptor(
                ml_service,
                auth_interceptor.clone()
            )
        )
        .add_service(
            trading_service::proto::monitoring::monitoring_service_server::MonitoringServiceServer::with_interceptor(
                monitoring_service,
                auth_interceptor.clone()
            )
        )
        .add_service(
            trading_service::proto::config::config_service_server::ConfigServiceServer::with_interceptor(
                config_service,
                auth_interceptor.clone()
            )
        )
        .serve_with_shutdown(addr, shutdown_signal());

    info!("Trading Service listening on {}", addr);

    // Start background tasks with kill switch monitoring
    tokio::select! {
        result = server => {
            if let Err(e) = result {
                error!("gRPC server error: {}", e);
            }
        }
        _ = start_health_endpoint(std::env::var("HEALTH_PORT")
            .ok()
            .and_then(|s| s.parse().ok())
            .unwrap_or(DEFAULT_HEALTH_PORT)) => {
            info!("Health endpoint stopped");
        }
        _ = monitor_kill_switch_status(Arc::clone(&kill_switch_system)) => {
            info!("Kill switch monitoring stopped");
        }
    }

    // Signal prediction loop(s) to stop, then drop senders
    for tx in &prediction_shutdown_handles {
        let _ = tx.send(());
    }
    drop(prediction_shutdown_handles);
    info!("Background prediction loops signalled for shutdown");

    // Cleanup kill switch monitoring on shutdown
    info!("Stopping kill switch monitoring...");
    if let Err(e) = kill_switch_system.stop_monitoring().await {
        warn!("Error stopping kill switch monitoring: {}", e);
    }

    info!("Trading Service shutdown complete");
    Ok(())
}

/// Initialize authentication configuration
///
/// NOTE: Authentication is handled by API Gateway (Wave 70)
/// This function returns a minimal stub configuration for compatibility
async fn initialize_auth_config() -> AuthConfig {
    // API Gateway handles all authentication - this is just a stub
    // All auth configuration is managed by API Gateway, not this service
    AuthConfig::default()
}

/// Initialize default configuration values if they don't exist
async fn initialize_default_configs(_config_repository: &PostgresConfigRepository) -> Result<()> {
    info!("Initializing default configurations...");

    // Note: Configuration initialization is now handled by ConfigManager
    // This function is kept for backward compatibility but is a no-op
    // The ConfigManager handles all configuration persistence and defaults

    info!("Default configurations initialized via ConfigManager");
    Ok(())
}

/// Start configuration monitoring for hot-reload
async fn start_config_monitoring(config_repository: Arc<PostgresConfigRepository>) -> Result<()> {
    let mut change_receiver = config_repository
        .subscribe_to_changes()
        .await
        .context("Failed to subscribe to configuration changes")?;

    tokio::spawn(async move {
        info!("Configuration hot-reload monitoring started");

        while let Ok((category, key)) = change_receiver.recv().await {
            info!("Configuration changed: {}.{}", category, key);

            // Handle specific configuration changes using typed methods
            match (category.as_str(), key.as_str()) {
                ("Trading", "max_order_size") => {
                    if let Ok(Some(value)) = config_repository
                        .get_config_f64("Trading", "max_order_size")
                        .await
                    {
                        info!("Updated max order size to: ${}", value);
                    }
                },
                ("MachineLearning", "inference_timeout_ms") => {
                    if let Ok(Some(value)) = config_repository
                        .get_config_u64("MachineLearning", "inference_timeout_ms")
                        .await
                    {
                        info!("Updated ML inference timeout to: {}ms", value);
                    }
                },
                ("Risk", "var_confidence") => {
                    if let Ok(Some(value)) = config_repository
                        .get_config_f64("Risk", "var_confidence")
                        .await
                    {
                        info!("Updated VaR confidence to: {}", value);
                    }
                },
                _ => {
                    info!("Configuration updated: {}.{}", category, key);
                },
            }
        }

        info!("Configuration monitoring stopped");
    });

    Ok(())
}

/// Start health check endpoint
async fn start_health_endpoint(port: u16) -> Result<()> {
    use hyper::server::conn::http1;
    use hyper::service::service_fn;
    use hyper_util::rt::TokioIo;

    use tokio::net::TcpListener;

    let addr: std::net::SocketAddr = ([0, 0, 0, 0], port).into();
    let listener = TcpListener::bind(addr)
        .await
        .context("Failed to bind health endpoint")?;

    info!("Health endpoint listening on http://{}", addr);

    loop {
        let (stream, _) = match listener.accept().await {
            Ok(conn) => conn,
            Err(e) => {
                error!("Failed to accept connection: {}", e);
                continue;
            },
        };

        tokio::spawn(async move {
            let io = TokioIo::new(stream);
            if let Err(e) = http1::Builder::new()
                .serve_connection(io, service_fn(health_handler))
                .await
            {
                error!("Health server error: {}", e);
            }
        });
    }
}

/// Health check handler with database pool statistics
async fn health_handler(
    _: hyper::Request<hyper::body::Incoming>,
) -> Result<hyper::Response<http_body_util::Full<bytes::Bytes>>, std::convert::Infallible> {
    use bytes::Bytes;
    use http_body_util::Full;

    // Note: In a production system, you'd pass the db_pool_wrapper here
    // For now, we'll indicate that database connection pooling is active
    let health_response = serde_json::json!({
        "status": "healthy",
        "service": "trading_service",
        "timestamp": chrono::Utc::now().to_rfc3339(),
        "version": env!("CARGO_PKG_VERSION"),
        "database": {
            "connection_pool": "HFT-optimized",
            "query_timeout_micros": 800,
            "connection_prewarming": "enabled",
            "prepared_statements": "enabled"
        }
    });

    // Build HTTP response with error handling to prevent panics in production
    let response = hyper::Response::builder()
        .status(200)
        .header("content-type", "application/json")
        .body(Full::new(Bytes::from(health_response.to_string())))
        .unwrap_or_else(|e| {
            // Return a minimal error response if primary response building fails
            error!("Failed to build primary health response: {}", e);

            // Try to build error response
            match hyper::Response::builder()
                .status(500)
                .header("content-type", "application/json")
                .body(Full::new(Bytes::from(
                    r#"{"status":"error","message":"Health check failed"}"#,
                ))) {
                Ok(err_response) => err_response,
                Err(fatal_err) => {
                    // Last resort: return minimal response without builder
                    error!("FATAL: Cannot build any HTTP response: {}", fatal_err);
                    hyper::Response::new(Full::new(Bytes::from(r#"{"status":"error"}"#)))
                },
            }
        });

    Ok(response)
}

/// Handle shutdown signals
async fn shutdown_signal() {
    let ctrl_c = async {
        if let Err(e) = signal::ctrl_c().await {
            error!("Failed to install Ctrl+C handler: {}", e);
        }
    };

    #[cfg(unix)]
    let terminate = async {
        match signal::unix::signal(signal::unix::SignalKind::terminate()) {
            Ok(mut signal_stream) => {
                signal_stream.recv().await;
            },
            Err(e) => {
                error!("Failed to install SIGTERM handler: {}", e);
            },
        }
    };

    #[cfg(not(unix))]
    let terminate = std::future::pending::<()>();

    tokio::select! {
        _ = ctrl_c => {},
        _ = terminate => {},
    }

    info!("Shutdown signal received");
}

/// Monitor kill switch status and log important events
async fn monitor_kill_switch_status(kill_switch_system: Arc<TradingServiceKillSwitch>) {
    let mut interval = tokio::time::interval(Duration::from_secs(30));

    loop {
        interval.tick().await;

        match kill_switch_system.get_status().await {
            Ok(status) => {
                if status.is_active {
                    error!(
                        "KILL SWITCH ACTIVE - Trading blocked | Checks: {} | Commands: {} | Error Rate: {:.2}%",
                        status.total_checks,
                        status.total_commands,
                        status.error_rate * 100.0
                    );
                }

                if status.is_emergency_active {
                    error!("🚨🚨🚨 EMERGENCY SHUTDOWN ACTIVE - System halted");
                }

                if !status.is_healthy {
                    warn!(
                        "⚠️ Kill switch system unhealthy - Error rate: {:.2}% | Consecutive failures: {}",
                        status.error_rate * 100.0,
                        status.consecutive_failures
                    );
                }

                // Log periodic status (every 5 minutes)
                if status.total_checks % 100 == 0 {
                    info!(
                        "Kill switch status: Active={} | Healthy={} | Checks={} | Commands={}",
                        status.is_active,
                        status.is_healthy,
                        status.total_checks,
                        status.total_commands
                    );
                }
            },
            Err(e) => {
                error!("Failed to get kill switch status: {}", e);
            },
        }
    }
}