# ML Inference Configuration # Centralized configuration for all inference parameters [inference] # Basic inference parameters max_latency_us = 100 # Maximum inference latency target (microseconds) inference_threads = 8 # Number of inference threads batch_size = 32 # Batch size for batch inference enable_gpu = true # Enable GPU acceleration warmup_iterations = 100 # Model warmup iterations on startup max_concurrent_requests = 1000 # Maximum concurrent inference requests [gpu] # GPU configuration device_id = 0 # CUDA device ID memory_pool_mb = 1024 # GPU memory pool size in MB cuda_streams = 4 # Number of CUDA streams tensor_rt_optimization = false # Enable TensorRT optimization [cpu] # CPU configuration num_threads = 8 # Number of CPU threads for inference enable_mkldnn = true # Enable MKL-DNN optimization inter_op_threads = 4 # Inter-operation threads intra_op_threads = 8 # Intra-operation threads [caching] # Inference caching enable_caching = true # Enable prediction caching default_ttl_ms = 1000 # Default TTL for predictions (milliseconds) max_cache_entries = 10000 # Maximum cache entries cache_compression = true # Enable cache compression redis_url = "redis://localhost:6379" # Redis connection for distributed caching [batching] # Dynamic batching enable_dynamic_batching = true # Enable dynamic batching max_batch_size = 64 # Maximum batch size batch_timeout_ms = 5 # Batch timeout (milliseconds) preferred_batch_size = 32 # Preferred batch size [optimization] # Model optimization enable_onnx_optimization = true # Enable ONNX optimization graph_optimization_level = 2 # Graph optimization level (0-2) enable_quantization = false # Enable model quantization quantization_bits = 8 # Quantization bits enable_pruning = false # Enable model pruning sparsity_threshold = 0.1 # Sparsity threshold for pruning [model_serving] # Model serving configuration model_format = "onnx" # Model format (onnx, pytorch, tensorflow) enable_model_versioning = true # Enable model versioning max_model_versions = 3 # Maximum model versions to keep loaded model_reload_check_interval_s = 30 # Model reload check interval enable_a_b_testing = false # Enable A/B testing between model versions [tier1] # Tier 1 inference (MAMBA/LIQUID) - Ultra-low latency max_concurrent_requests = 1000 # Maximum concurrent requests timeout_ms = 1 # Timeout in milliseconds memory_pool_size = 1000000 # Memory pool size max_model_memory_mb = 8192 # Maximum model memory in MB enable_onnx_optimization = true # Enable ONNX optimization device_preference = "auto" # Device preference: gpu, cpu, auto [tier2] # Tier 2 inference (TFT/TGGN) - Medium latency max_concurrent_requests = 100 # Maximum concurrent requests timeout_ms = 100 # Timeout in milliseconds historical_data_window = 1000 # Historical data window size prediction_horizon = 10 # Prediction horizon prediction_horizons = [1, 5, 10, 30] # Multiple prediction horizons max_nodes = 1000 # Maximum nodes for TGGN device_preference = "auto" # Device preference: gpu, cpu, auto [tier3] # Tier 3 inference (NLP) - Higher latency acceptable max_concurrent_requests = 10 # Maximum concurrent requests timeout_ms = 1000 # Timeout in milliseconds sentiment_models = ["finbert"] # Sentiment models to load news_sources = ["reuters", "bloomberg"] # News sources to process device_preference = "auto" # Device preference: gpu, cpu, auto [features] # Feature processing enable_feature_store = true # Enable feature store feature_store_url = "redis://localhost:6379" # Feature store URL lookback_periods = [5, 10, 30, 60, 300] # Lookback periods in seconds technical_indicators = ["sma", "ema", "rsi", "bollinger", "vwap"] # Technical indicators max_feature_age_ms = 1000 # Maximum feature age in milliseconds [monitoring] # Inference monitoring enable_metrics = true # Enable metrics collection metrics_port = 9090 # Metrics export port latency_buckets = [0.001, 0.01, 0.05, 0.1, 0.5, 1.0, 5.0, 10.0, 50.0, 100.0] # Latency histogram buckets enable_health_checks = true # Enable health checks health_check_interval_s = 30 # Health check interval log_level = "info" # Log level [safety] # Inference safety enable_input_validation = true # Enable input validation max_input_size = 10000 # Maximum input size enable_output_validation = true # Enable output validation nan_inf_check = true # Check for NaN/Inf in outputs output_range_check = true # Check output ranges min_confidence_threshold = 0.1 # Minimum confidence threshold [load_balancing] # Load balancing enable_load_balancing = true # Enable load balancing load_balancing_strategy = "round_robin" # Strategy: round_robin, least_connections, weighted health_check_enabled = true # Enable health checks for load balancing circuit_breaker_enabled = true # Enable circuit breaker circuit_breaker_failure_threshold = 5 # Circuit breaker failure threshold circuit_breaker_timeout_s = 60 # Circuit breaker timeout [model_configs] # Individual model configurations [model_configs.venue_selection] model_path = "./models/venue_selection.onnx" model_type = "onnx" input_size = 50 output_size = 10 max_latency_us = 50 # Model-specific latency requirement batch_size = 16 # Model-specific batch size enable_caching = true # Enable caching for this model [model_configs.risk_assessment] model_path = "./models/risk_assessment.onnx" model_type = "onnx" input_size = 30 output_size = 1 max_latency_us = 25 # Model-specific latency requirement batch_size = 32 # Model-specific batch size enable_caching = true # Enable caching for this model [model_configs.sentiment_analysis] model_path = "./models/sentiment.onnx" model_type = "onnx" input_size = 512 output_size = 3 max_latency_us = 200 # Model-specific latency requirement batch_size = 8 # Model-specific batch size enable_caching = true # Enable caching for this model [model_configs.mamba] model_path = "./models/mamba.onnx" model_type = "onnx" model_dim = 768 state_size = 64 max_latency_us = 10 # Ultra-low latency for HFT batch_size = 1 # Single inference for minimum latency enable_caching = false # No caching for real-time HFT [model_configs.liquid] model_path = "./models/liquid.onnx" model_type = "onnx" model_type_liquid = "LFM1B" hidden_size = 768 max_latency_us = 10 # Ultra-low latency for HFT batch_size = 1 # Single inference for minimum latency enable_caching = false # No caching for real-time HFT [model_configs.tft] model_path = "./models/tft.onnx" model_type = "onnx" hidden_size = 128 num_encoder_layers = 2 max_latency_us = 100 # Medium latency acceptable batch_size = 16 # Batch for efficiency enable_caching = true # Enable caching [model_configs.tggn] model_path = "./models/tggn.onnx" model_type = "onnx" hidden_size = 256 max_nodes = 1000 max_latency_us = 100 # Medium latency acceptable batch_size = 8 # Batch for efficiency enable_caching = true # Enable caching [model_configs.nlp] model_path = "./models/nlp.onnx" model_type = "onnx" model_type_nlp = "FinBERT" max_sequence_length = 512 max_latency_us = 1000 # Higher latency acceptable for NLP batch_size = 4 # Small batch for NLP enable_caching = true # Enable caching [preprocessing] # Input preprocessing enable_normalization = true # Enable input normalization normalization_type = "z_score" # Normalization type: z_score, min_max, robust feature_scaling = true # Enable feature scaling outlier_detection = true # Enable outlier detection outlier_threshold = 3.0 # Outlier detection threshold (std deviations) [postprocessing] # Output postprocessing enable_postprocessing = true # Enable output postprocessing confidence_calibration = false # Enable confidence calibration ensemble_voting = false # Enable ensemble voting output_smoothing = false # Enable output smoothing smoothing_window = 5 # Smoothing window size