1745 lines
58 KiB
Rust
1745 lines
58 KiB
Rust
#![deny(clippy::unwrap_used, clippy::expect_used)]
|
|
#![cfg_attr(test, allow(clippy::unwrap_used, clippy::expect_used))]
|
|
#![allow(dead_code)] // 10 ML model implementations with internal architecture not yet fully wired
|
|
#![allow(missing_docs)] // Internal implementation details don't require documentation
|
|
#![allow(missing_debug_implementations)] // Not all types need Debug
|
|
#![allow(unused_crate_dependencies)] // Dev dependencies not used in lib.rs
|
|
#![allow(clippy::float_arithmetic)] // ML operations require float arithmetic
|
|
// ML-specific lint overrides: these are intentional domain patterns, not safety issues
|
|
#![allow(clippy::non_ascii_literal)] // Box-drawing chars for tables, Greek letters for math
|
|
#![allow(clippy::str_to_string)] // Pervasive in ML config/display code, not a safety concern
|
|
#![allow(clippy::partial_pub_fields)] // ML model structs mix pub config with private state
|
|
#![allow(clippy::multiple_inherent_impl)] // Split impls for readability (core vs Display vs Builder)
|
|
#![allow(clippy::same_name_method)] // Trait methods intentionally shadow inherent methods
|
|
#![allow(clippy::shadow_reuse)] // Tensor code naturally shadows: let x = relu(x)
|
|
#![allow(clippy::shadow_unrelated)] // ML pipeline variable reuse across phases
|
|
#![allow(clippy::shadow_same)] // Rebinding after narrowing in match/if-let
|
|
// ML domain pedantic lints: these are noise in numerical/ML code, not safety issues
|
|
// Individual pedantic lints are managed at workspace level in Cargo.toml
|
|
#![allow(clippy::doc_markdown)] // Technical terms (AVX2, SIMD, HFT, etc.) in doc comments
|
|
#![allow(clippy::indexing_slicing)] // Tensor/array indexing is pervasive and bounds-checked in context
|
|
#![allow(clippy::missing_const_for_fn)] // Const fn not critical for ML model code
|
|
#![allow(clippy::module_name_repetitions)] // Module-prefixed types provide clarity (DQNAgent, TFTTrainer)
|
|
#![allow(clippy::integer_division)] // Integer division is intentional in batch/epoch calculations
|
|
#![allow(clippy::cognitive_complexity)] // ML training loops and model architectures are inherently complex
|
|
#![allow(clippy::similar_names)] // ML variables often have similar names (x, xs, x_hat, x_norm)
|
|
#![allow(clippy::clone_on_ref_ptr)] // Arc::clone is intentional for shared model state
|
|
#![allow(clippy::too_many_lines)] // Complex model implementations need many lines
|
|
#![allow(clippy::as_conversions)] // Necessary for tensor dimension/type conversions
|
|
#![allow(clippy::cast_precision_loss)] // Acceptable in ML with f32/f64 conversions
|
|
#![allow(clippy::cast_possible_truncation)] // Type conversions validated in context
|
|
#![allow(clippy::default_numeric_fallback)] // Float/int literals are contextually typed in ML code
|
|
#![allow(clippy::arithmetic_side_effects)] // ML math uses checked/saturating where needed
|
|
#![allow(clippy::needless_range_loop)] // Index-based loops often clearer for tensor operations
|
|
#![allow(clippy::into_iter_on_ref)] // .iter() vs .into_iter() on refs is stylistic
|
|
#![allow(clippy::new_without_default)] // Many ML types need config params, Default is inappropriate
|
|
#![allow(clippy::manual_let_else)] // if-let pattern preferred in many ML error paths
|
|
#![allow(clippy::unnecessary_wraps)] // Result/Option wrapping needed for trait consistency
|
|
#![allow(clippy::too_many_arguments)] // ML functions often need many hyperparameters
|
|
#![allow(clippy::must_use_candidate)] // Not all ML functions need must_use
|
|
#![allow(clippy::missing_errors_doc)] // Internal ML APIs don't need full error documentation
|
|
#![allow(clippy::cast_sign_loss)] // Sign loss validated in context
|
|
#![allow(clippy::cast_possible_wrap)] // Wrap validated in context
|
|
#![allow(clippy::cast_lossless)] // as casts are intentional for ML numeric conversions
|
|
#![allow(clippy::unused_async)] // Async needed for trait implementations
|
|
#![allow(clippy::match_same_arms)] // Explicit match arms preferred for clarity in ML code
|
|
#![allow(clippy::unused_self)] // Self parameter needed for trait consistency
|
|
#![allow(clippy::map_err_ignore)] // Error conversion doesn't need original context in ML pipelines
|
|
#![allow(clippy::single_match_else)] // Explicit match preferred for clarity in ML code
|
|
#![allow(clippy::wildcard_imports)] // Prelude-style imports common in ML modules
|
|
#![allow(clippy::unnecessary_cast)] // Explicit casts for tensor dimension/type clarity
|
|
#![allow(clippy::undocumented_unsafe_blocks)] // Unsafe blocks documented at usage site
|
|
#![allow(clippy::redundant_clone)] // Clones needed for ownership in async/parallel ML pipelines
|
|
#![allow(clippy::redundant_closure)] // Explicit closures preferred for readability
|
|
#![allow(clippy::type_complexity)] // Complex types unavoidable in ML generics
|
|
#![allow(clippy::manual_clamp)] // Explicit min/max preferred in numerical code
|
|
#![allow(clippy::clone_on_copy)] // Explicit clone for clarity on Copy types
|
|
#![allow(clippy::should_implement_trait)] // Custom builder patterns don't need std traits
|
|
#![allow(clippy::derivable_impls)] // Custom Default impls with domain-specific values
|
|
#![allow(clippy::useless_conversion)] // Explicit conversions for type clarity
|
|
#![allow(clippy::get_first)] // .get(0) preferred for consistency with .get(n)
|
|
#![allow(clippy::len_zero)] // .len() == 0 preferred for readability in some contexts
|
|
#![allow(clippy::assign_op_pattern)] // Explicit assignment preferred in numerical code
|
|
#![allow(clippy::if_same_then_else)] // Intentional identical branches for documentation
|
|
#![allow(clippy::unused_enumerate_index)] // Index used in debug/logging contexts
|
|
#![allow(clippy::doc_lazy_continuation)] // Doc formatting acceptable
|
|
#![allow(clippy::doc_overindented_list_items)] // Doc formatting acceptable
|
|
#![allow(clippy::single_char_add_str)] // String building patterns
|
|
#![allow(clippy::let_and_return)] // Named return values for clarity
|
|
#![allow(clippy::useless_format)] // Explicit format for consistency
|
|
#![allow(clippy::manual_div_ceil)] // Explicit ceiling division for clarity
|
|
#![allow(clippy::io_other_error)] // IO error construction patterns
|
|
#![allow(clippy::manual_range_contains)] // Explicit range checks for clarity
|
|
#![allow(clippy::unwrap_or_default)] // Explicit unwrap_or preferred in some contexts
|
|
#![allow(clippy::used_underscore_binding)] // Underscore-prefixed bindings used intentionally
|
|
#![allow(clippy::trivially_copy_pass_by_ref)] // Pass-by-ref for trait consistency
|
|
#![allow(clippy::needless_borrows_for_generic_args)] // Explicit borrows for clarity
|
|
#![allow(clippy::needless_borrow)] // Explicit borrows for clarity
|
|
#![allow(clippy::missing_safety_doc)] // Safety documented at usage site
|
|
#![allow(clippy::module_inception)] // Module name matches parent for re-export patterns
|
|
#![allow(clippy::if_not_else)] // Negated conditions preferred in some ML logic
|
|
#![allow(clippy::empty_line_after_doc_comments)] // Doc comment formatting acceptable
|
|
#![allow(clippy::unnecessary_lazy_evaluations)] // Explicit lazy evaluation for side effects
|
|
#![allow(clippy::collapsible_if)] // Separate ifs preferred for readability
|
|
#![allow(clippy::question_mark)] // Explicit error handling preferred
|
|
#![allow(clippy::op_ref)] // Explicit ref operations for numeric types
|
|
#![allow(clippy::iter_kv_map)] // Map iteration patterns
|
|
#![allow(clippy::ptr_arg)] // Ptr args for trait consistency
|
|
#![allow(clippy::needless_question_mark)] // Explicit ? for error propagation clarity
|
|
#![allow(clippy::explicit_auto_deref)] // Explicit derefs for clarity
|
|
#![allow(clippy::bind_instead_of_map)] // Bind preferred in some combinator chains
|
|
#![allow(clippy::upper_case_acronyms)] // ML acronyms (DQN, PPO, TFT, etc.)
|
|
#![allow(clippy::large_types_passed_by_value)] // Large types passed by value for ownership
|
|
#![allow(clippy::large_enum_variant)] // Large variants unavoidable in ML model enums
|
|
#![allow(clippy::collapsible_else_if)] // Separate else-if preferred for readability
|
|
#![allow(clippy::vec_init_then_push)] // Vec init then push for conditional building
|
|
#![allow(clippy::implicit_saturating_sub)] // Explicit subtraction preferred
|
|
#![allow(clippy::missing_fields_in_debug)] // Custom Debug impls for large ML structs
|
|
#![allow(clippy::field_reassign_with_default)] // Field reassignment after Default::default()
|
|
#![allow(clippy::len_without_is_empty)] // len() without is_empty() for ML containers
|
|
#![allow(clippy::iter_nth)] // .nth() for indexed iteration
|
|
#![allow(clippy::items_after_statements)] // Items after statements for local helpers
|
|
#![allow(clippy::uninlined_format_args)] // Non-inlined format args for readability
|
|
#![allow(clippy::manual_memcpy)] // Explicit loop copy for tensor operations
|
|
#![allow(clippy::single_char_lifetime_names)] // Standard Rust lifetime conventions in ODE solvers
|
|
#![allow(clippy::same_item_push)] // Intentional repeated pushes for sequence padding
|
|
#![allow(clippy::multiple_unsafe_ops_per_block)] // SIMD/hardware ops grouped for clarity
|
|
#![allow(clippy::arc_with_non_send_sync)] // Arc used for local-thread inference contexts
|
|
#![allow(clippy::format_in_format_args)] // Nested format for dynamic message building
|
|
#![allow(clippy::inherent_to_string)] // Custom to_string for display types
|
|
#![allow(clippy::redundant_locals)] // Rebinding for clarity in model pipelines
|
|
#![allow(clippy::to_string_trait_impl)] // Direct ToString impls for versioning types
|
|
//! Shared ML types, traits, and infrastructure for Foxhunt
|
|
//!
|
|
//! This crate provides the core type definitions, error types, and traits
|
|
//! used across all ML crates in the Foxhunt workspace. It is the foundation
|
|
//! layer that other ml-* crates depend on.
|
|
|
|
#![warn(missing_debug_implementations)]
|
|
#![warn(rust_2018_idioms)]
|
|
#![deny(clippy::panic, clippy::unimplemented, clippy::unreachable)]
|
|
|
|
// ========== STATE LAYOUT — SINGLE SOURCE OF TRUTH ==========
|
|
pub mod state_layout;
|
|
|
|
// ========== MOVED MODULES FROM ML CRATE ==========
|
|
pub mod error;
|
|
pub mod types;
|
|
pub mod common;
|
|
pub mod config;
|
|
pub mod model;
|
|
pub mod traits;
|
|
|
|
// ========== COMPUTE PRIMITIVES (moved from ml in task 5b) ==========
|
|
pub mod tensor_ops;
|
|
pub mod gpu;
|
|
pub mod safety;
|
|
pub mod memory_optimization;
|
|
pub mod batch_size_resolver;
|
|
|
|
// cuda_compile module removed — nvrtc replaced by build.rs cubins
|
|
|
|
// ========== CUDA-NATIVE AUTOGRAD (replaces Candle backward/VarMap/GradStore) ==========
|
|
#[cfg(feature = "cuda")]
|
|
pub mod cuda_autograd;
|
|
|
|
// ========== NATIVE TYPE REPLACEMENTS (Candle-free Device/DType/Tensor) ==========
|
|
pub mod native_types;
|
|
|
|
// ========== DEVICE ABSTRACTION (replaces candle_core::Device) ==========
|
|
pub mod device;
|
|
|
|
// ========== SAFETENSORS CHECKPOINT (replaces candle safetensors wrapper) ==========
|
|
pub mod checkpoint;
|
|
|
|
// ========== PROFILING (NVTX markers for Nsight Systems) ==========
|
|
pub mod nvtx;
|
|
|
|
// ========== SHARED INFRASTRUCTURE (moved from ml in task 5e) ==========
|
|
pub mod trading_action;
|
|
pub mod action_space;
|
|
pub mod xavier_init;
|
|
pub mod order_router;
|
|
pub mod fill_simulator;
|
|
pub mod portfolio_tracker;
|
|
|
|
// ========== METRICS & PERFORMANCE (moved from ml in task 5g) ==========
|
|
pub mod metrics;
|
|
pub mod performance;
|
|
|
|
// ========== TRAINING TRAIT (moved from ml to unblock trainers extraction) ==========
|
|
pub mod training;
|
|
|
|
use async_trait::async_trait;
|
|
use chrono::{DateTime, Utc};
|
|
use futures::future::join_all;
|
|
use rust_decimal::Decimal;
|
|
use serde::{Deserialize, Serialize};
|
|
use std::collections::HashMap;
|
|
use std::sync::Arc;
|
|
use thiserror::Error;
|
|
use tokio::sync::RwLock;
|
|
|
|
// ========== COMMON TYPE ERROR ==========
|
|
|
|
/// Common type errors that can occur during ML operations
|
|
///
|
|
/// This enum represents various type-related errors that can occur when working
|
|
/// with different data types across the ML pipeline, including type conversions,
|
|
/// validation errors, and compatibility issues.
|
|
#[derive(Debug, Clone, Error, Serialize, Deserialize)]
|
|
pub enum CommonTypeError {
|
|
/// Generic type error with descriptive message
|
|
#[error("Type error: {0}")]
|
|
Error(String),
|
|
}
|
|
|
|
// ========== MARKET REGIME ==========
|
|
|
|
/// Market regime classification for algorithmic trading strategies
|
|
///
|
|
/// This enum represents different market conditions that can be detected through
|
|
/// statistical analysis and machine learning models. Market regime detection is
|
|
/// crucial for adaptive trading strategies that adjust their behavior based on
|
|
/// current market conditions.
|
|
///
|
|
/// # Examples
|
|
///
|
|
/// ```rust
|
|
/// use ml_core::MarketRegime;
|
|
///
|
|
/// let regime = MarketRegime::Trending;
|
|
/// match regime {
|
|
/// MarketRegime::Bull => println!("Use momentum strategies"),
|
|
/// MarketRegime::Bear => println!("Use defensive strategies"),
|
|
/// MarketRegime::Crisis => println!("Implement risk controls"),
|
|
/// _ => println!("Use balanced approach"),
|
|
/// }
|
|
/// ```
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
|
pub enum MarketRegime {
|
|
/// Normal market conditions with typical volatility and volume
|
|
Normal,
|
|
/// Strong directional movement with clear trends
|
|
Trending,
|
|
/// Range-bound market with limited directional movement
|
|
Sideways,
|
|
/// Bullish market with rising prices and positive sentiment
|
|
Bull,
|
|
/// Bearish market with falling prices and negative sentiment
|
|
Bear,
|
|
/// Crisis conditions with extreme volatility and risk
|
|
Crisis,
|
|
}
|
|
|
|
// ========== COMMON ERROR ==========
|
|
|
|
/// Common errors that can occur across the ML system
|
|
///
|
|
/// This enum provides a unified error type that can be used throughout the ML
|
|
/// pipeline to ensure consistent error handling and reporting. It serves as a
|
|
/// bridge between different subsystems and provides appropriate error categorization.
|
|
#[derive(Debug, Clone, Error, Serialize, Deserialize)]
|
|
pub enum CommonError {
|
|
/// General error with descriptive message
|
|
#[error("Error: {0}")]
|
|
General(String),
|
|
}
|
|
|
|
impl CommonError {
|
|
/// Create a validation error with a descriptive message
|
|
pub fn validation<S: Into<String>>(msg: S) -> Self {
|
|
Self::General(msg.into())
|
|
}
|
|
|
|
/// Create a configuration error with a descriptive message
|
|
pub fn config<S: Into<String>>(msg: S) -> Self {
|
|
Self::General(msg.into())
|
|
}
|
|
|
|
/// Create a service error with category and descriptive message
|
|
pub fn service<S: Into<String>>(category: ErrorCategory, msg: S) -> Self {
|
|
Self::General(format!("{:?}: {}", category, msg.into()))
|
|
}
|
|
}
|
|
|
|
// Re-export canonical ErrorCategory from common crate (24 variants)
|
|
pub use ::common::error::ErrorCategory;
|
|
|
|
// ========== CORE ML TYPES ==========
|
|
|
|
/// Represents a financial trade for ML model training and analysis
|
|
///
|
|
/// This structure contains the essential information about a trade that is used
|
|
/// by ML models for pattern recognition, market analysis, and strategy optimization.
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct Trade {
|
|
/// Trading symbol (e.g., "AAPL", "MSFT")
|
|
pub symbol: String,
|
|
/// Trade price in decimal format for precision
|
|
pub price: Decimal,
|
|
/// Trade quantity in decimal format for precision
|
|
pub quantity: Decimal,
|
|
/// Timestamp in microseconds since Unix epoch
|
|
pub timestamp: u64,
|
|
/// Trade side: "buy" or "sell"
|
|
pub side: String,
|
|
}
|
|
|
|
/// Health status for ensemble models and ML system components
|
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
|
pub enum HealthStatus {
|
|
/// Component is operating within normal parameters
|
|
Healthy,
|
|
/// Component is operational but performance is below optimal
|
|
Degraded,
|
|
/// Component is not functioning properly and requires intervention
|
|
Unhealthy,
|
|
}
|
|
|
|
/// Market data snapshot for ML model input
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct MarketDataSnapshot {
|
|
/// Timestamp of the market data snapshot
|
|
pub timestamp: DateTime<Utc>,
|
|
/// Trading symbol (e.g., "AAPL", "MSFT")
|
|
pub symbol: String,
|
|
/// Current market price
|
|
pub price: Decimal,
|
|
/// Trading volume at this timestamp
|
|
pub volume: Decimal,
|
|
}
|
|
|
|
/// Feature vector for ML model input
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct FeatureVector(pub Vec<f64>);
|
|
|
|
impl FeatureVector {
|
|
/// Get the length of the feature vector
|
|
pub fn len(&self) -> usize {
|
|
self.0.len()
|
|
}
|
|
|
|
/// Check if the feature vector is empty
|
|
pub fn is_empty(&self) -> bool {
|
|
self.0.is_empty()
|
|
}
|
|
}
|
|
|
|
/// Integer tensor for discrete ML model operations
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct IntegerTensor(pub Vec<i64>);
|
|
|
|
/// Summary of model update operations
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct UpdateSummary {
|
|
/// Number of models successfully updated
|
|
pub updated_models: usize,
|
|
/// Total number of models in the update operation
|
|
pub total_models: usize,
|
|
}
|
|
|
|
// ========== ML ERROR (DEDUPED) ==========
|
|
|
|
/// Machine Learning specific errors
|
|
///
|
|
/// Note: `ConfigError` and `ConfigurationError` have been consolidated into a
|
|
/// single `ConfigError(String)` tuple variant.
|
|
#[derive(Debug, Clone, Error, Serialize, Deserialize)]
|
|
pub enum MLError {
|
|
/// Configuration error (consolidated from ConfigError{reason} + ConfigurationError)
|
|
#[error("Configuration error: {0}")]
|
|
ConfigError(String),
|
|
|
|
/// Dimension mismatch error
|
|
#[error("Dimension mismatch: expected {expected}, got {actual}")]
|
|
DimensionMismatch { expected: usize, actual: usize },
|
|
|
|
/// Graph-related error
|
|
#[error("Graph error: {message}")]
|
|
GraphError { message: String },
|
|
|
|
/// Resource limit exceeded
|
|
#[error("Resource limit exceeded: {resource} limit {limit}")]
|
|
ResourceLimit { resource: String, limit: usize },
|
|
|
|
/// Serialization error
|
|
#[error("Serialization error: {reason}")]
|
|
SerializationError { reason: String },
|
|
|
|
/// Validation error
|
|
#[error("Validation error: {message}")]
|
|
ValidationError { message: String },
|
|
|
|
/// Concurrency error
|
|
#[error("Concurrency error in operation: {operation}")]
|
|
ConcurrencyError { operation: String },
|
|
|
|
/// Invalid input error
|
|
#[error("Invalid input: {0}")]
|
|
InvalidInput(String),
|
|
|
|
/// Initialization error
|
|
#[error("Initialization error in {component}: {message}")]
|
|
InitializationError { component: String, message: String },
|
|
|
|
/// Training error
|
|
#[error("Training error: {0}")]
|
|
TrainingError(String),
|
|
|
|
/// Inference error
|
|
#[error("Inference error: {0}")]
|
|
InferenceError(String),
|
|
|
|
/// Model error
|
|
#[error("Model error: {0}")]
|
|
ModelError(String),
|
|
|
|
/// Model not trained error
|
|
#[error("Model not trained: {0}")]
|
|
NotTrained(String),
|
|
|
|
/// Anyhow error wrapping
|
|
#[error("General error: {0}")]
|
|
AnyhowError(String),
|
|
|
|
/// Tensor creation error
|
|
#[error("Tensor creation error in {operation}: {reason}")]
|
|
TensorCreationError { operation: String, reason: String },
|
|
|
|
/// Tensor operation error
|
|
#[error("Tensor operation error: {0}")]
|
|
TensorOperationError(String),
|
|
|
|
/// Lock error
|
|
#[error("Lock error: {0}")]
|
|
LockError(String),
|
|
|
|
/// Model not found error
|
|
#[error("Model not found: {0}")]
|
|
ModelNotFound(String),
|
|
|
|
/// Insufficient data error
|
|
#[error("Insufficient data: {0}")]
|
|
InsufficientData(String),
|
|
|
|
/// Checkpoint error
|
|
#[error("Checkpoint error: {0}")]
|
|
CheckpointError(String),
|
|
|
|
/// Device error (GPU/CUDA unavailable)
|
|
#[error("Device error: {0}")]
|
|
DeviceError(String),
|
|
}
|
|
|
|
// ========== FROM IMPLS ==========
|
|
|
|
// candle_core::Error conversion removed — candle eliminated from ml-core.
|
|
|
|
// Convert anyhow::Error to MLError
|
|
impl From<anyhow::Error> for MLError {
|
|
fn from(err: anyhow::Error) -> Self {
|
|
MLError::AnyhowError(err.to_string())
|
|
}
|
|
}
|
|
|
|
// Convert std::io::Error to MLError
|
|
impl From<std::io::Error> for MLError {
|
|
fn from(err: std::io::Error) -> Self {
|
|
MLError::ModelError(format!("IO error: {}", err))
|
|
}
|
|
}
|
|
|
|
// Convert common type errors to MLError
|
|
impl From<CommonTypeError> for MLError {
|
|
fn from(err: CommonTypeError) -> Self {
|
|
MLError::ModelError(format!("Common type error: {}", err))
|
|
}
|
|
}
|
|
|
|
// Convert serde_json::Error to MLError
|
|
impl From<serde_json::Error> for MLError {
|
|
fn from(err: serde_json::Error) -> Self {
|
|
MLError::SerializationError {
|
|
reason: err.to_string(),
|
|
}
|
|
}
|
|
}
|
|
|
|
// UNIFIED ERROR HANDLING: Convert all ML errors to CommonError for workspace consistency
|
|
impl From<MLError> for CommonError {
|
|
fn from(err: MLError) -> Self {
|
|
match err {
|
|
MLError::ConfigError(reason) => {
|
|
CommonError::config(format!("ML configuration error: {}", reason))
|
|
},
|
|
MLError::InitializationError { component, message } => CommonError::service(
|
|
ErrorCategory::System,
|
|
format!("ML initialization error in {}: {}", component, message),
|
|
),
|
|
MLError::DimensionMismatch { expected, actual } => CommonError::validation(format!(
|
|
"ML dimension mismatch: expected {}, got {}",
|
|
expected, actual
|
|
)),
|
|
MLError::GraphError { message } => CommonError::service(
|
|
ErrorCategory::System,
|
|
format!("ML graph error: {}", message),
|
|
),
|
|
MLError::ResourceLimit { resource, limit } => CommonError::service(
|
|
ErrorCategory::System,
|
|
format!("ML resource limit exceeded: {} limit {}", resource, limit),
|
|
),
|
|
MLError::SerializationError { reason } => CommonError::service(
|
|
ErrorCategory::System,
|
|
format!("ML serialization error: {}", reason),
|
|
),
|
|
MLError::ValidationError { message } => {
|
|
CommonError::validation(format!("ML validation error: {}", message))
|
|
},
|
|
MLError::ConcurrencyError { operation } => CommonError::service(
|
|
ErrorCategory::System,
|
|
format!("ML concurrency error in operation: {}", operation),
|
|
),
|
|
MLError::InvalidInput(msg) => {
|
|
CommonError::validation(format!("ML invalid input: {}", msg))
|
|
},
|
|
MLError::TrainingError(msg) => {
|
|
CommonError::service(ErrorCategory::System, format!("ML training error: {}", msg))
|
|
},
|
|
MLError::InferenceError(msg) => CommonError::service(
|
|
ErrorCategory::System,
|
|
format!("ML inference error: {}", msg),
|
|
),
|
|
MLError::ModelError(msg) => {
|
|
CommonError::service(ErrorCategory::System, format!("ML model error: {}", msg))
|
|
},
|
|
MLError::CheckpointError(msg) => CommonError::service(
|
|
ErrorCategory::System,
|
|
format!("ML checkpoint error: {}", msg),
|
|
),
|
|
MLError::NotTrained(msg) => CommonError::service(
|
|
ErrorCategory::System,
|
|
format!("ML model not trained: {}", msg),
|
|
),
|
|
MLError::AnyhowError(msg) => {
|
|
CommonError::service(ErrorCategory::System, format!("ML error: {}", msg))
|
|
},
|
|
MLError::TensorCreationError { operation, reason } => CommonError::service(
|
|
ErrorCategory::System,
|
|
format!("ML tensor creation error in {}: {}", operation, reason),
|
|
),
|
|
MLError::TensorOperationError(msg) => CommonError::service(
|
|
ErrorCategory::System,
|
|
format!("ML tensor operation error: {}", msg),
|
|
),
|
|
MLError::LockError(msg) => {
|
|
CommonError::service(ErrorCategory::System, format!("ML lock error: {}", msg))
|
|
},
|
|
MLError::ModelNotFound(msg) => CommonError::service(
|
|
ErrorCategory::System,
|
|
format!("ML model not found: {}", msg),
|
|
),
|
|
MLError::InsufficientData(msg) => {
|
|
CommonError::validation(format!("ML insufficient data: {}", msg))
|
|
},
|
|
MLError::DeviceError(msg) => CommonError::service(
|
|
ErrorCategory::System,
|
|
format!("ML device error: {}", msg),
|
|
),
|
|
}
|
|
}
|
|
}
|
|
|
|
// ========== RESULT TYPES ==========
|
|
|
|
/// Result type for ML operations
|
|
pub type MLResult<T> = Result<T, MLError>;
|
|
|
|
/// New unified result type using CommonError for better integration
|
|
pub type UnifiedMLResult<T> = Result<T, CommonError>;
|
|
|
|
// ========== CONSTANTS ==========
|
|
|
|
/// Precision factor for fixed-point arithmetic
|
|
pub const PRECISION_FACTOR: i64 = 100_000_000;
|
|
|
|
/// Maximum inference latency target in microseconds
|
|
pub const MAX_INFERENCE_LATENCY_US: u64 = 100;
|
|
|
|
// ========== DEVICE MANAGEMENT ==========
|
|
|
|
/// Get mandatory CUDA device for training.
|
|
///
|
|
/// Returns an `MlDevice::Cuda` with ordinal 0.
|
|
///
|
|
/// # Errors
|
|
///
|
|
/// Returns `MLError::DeviceError` if CUDA GPU is not available.
|
|
#[cfg(feature = "cuda")]
|
|
pub fn get_training_device() -> Result<device::MlDevice, MLError> {
|
|
device::MlDevice::cuda(0)
|
|
}
|
|
|
|
/// Get CUDA device with index (for multi-GPU setups)
|
|
///
|
|
/// # Errors
|
|
///
|
|
/// Returns `MLError::DeviceError` if CUDA GPU at specified index is not available.
|
|
#[cfg(feature = "cuda")]
|
|
pub fn get_training_device_at(device_id: usize) -> Result<device::MlDevice, MLError> {
|
|
device::MlDevice::cuda(device_id)
|
|
}
|
|
|
|
// ========== ML APP RESULT ==========
|
|
|
|
/// Application result wrapper for ML operations
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct MLAppResult<T> {
|
|
pub data: T,
|
|
pub success: bool,
|
|
pub message: Option<String>,
|
|
pub execution_time_ms: u64,
|
|
pub metadata: HashMap<String, String>,
|
|
}
|
|
|
|
impl<T> MLAppResult<T> {
|
|
/// Create a successful result
|
|
pub fn success(data: T) -> Self {
|
|
Self {
|
|
data,
|
|
success: true,
|
|
message: None,
|
|
execution_time_ms: 0,
|
|
metadata: HashMap::new(),
|
|
}
|
|
}
|
|
|
|
/// Create a failed result with message
|
|
pub fn error(data: T, message: String) -> Self {
|
|
Self {
|
|
data,
|
|
success: false,
|
|
message: Some(message),
|
|
execution_time_ms: 0,
|
|
metadata: HashMap::new(),
|
|
}
|
|
}
|
|
|
|
/// Set execution time
|
|
pub fn with_timing(mut self, execution_time_ms: u64) -> Self {
|
|
self.execution_time_ms = execution_time_ms;
|
|
self
|
|
}
|
|
|
|
/// Add metadata
|
|
pub fn with_metadata(mut self, key: String, value: String) -> Self {
|
|
self.metadata.insert(key, value);
|
|
self
|
|
}
|
|
}
|
|
|
|
// ========== HFT PERFORMANCE PROFILE ==========
|
|
|
|
/// Performance profile configuration for HFT models
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct HFTPerformanceProfile {
|
|
pub max_latency_us: u64,
|
|
pub target_throughput: u32,
|
|
pub memory_limit_mb: u64,
|
|
pub cpu_affinity: Option<Vec<usize>>,
|
|
pub gpu_enabled: bool,
|
|
pub batch_size: u32,
|
|
pub optimization_level: OptimizationLevel,
|
|
}
|
|
|
|
/// Optimization levels for HFT performance
|
|
#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
|
|
pub enum OptimizationLevel {
|
|
/// Maximum speed, minimal safety checks
|
|
UltraLow,
|
|
/// Balanced speed and safety
|
|
Low,
|
|
/// Standard optimization
|
|
Medium,
|
|
/// Conservative with full validation
|
|
High,
|
|
}
|
|
|
|
impl Default for HFTPerformanceProfile {
|
|
fn default() -> Self {
|
|
Self {
|
|
max_latency_us: 100, // 100 microseconds target
|
|
target_throughput: 10000, // 10k operations per second
|
|
memory_limit_mb: 1024, // 1GB memory limit
|
|
cpu_affinity: None,
|
|
gpu_enabled: false,
|
|
batch_size: 1,
|
|
optimization_level: OptimizationLevel::Medium,
|
|
}
|
|
}
|
|
}
|
|
|
|
impl HFTPerformanceProfile {
|
|
/// Create HFT performance profile with custom latency target
|
|
pub fn with_latency(max_latency_us: u64) -> Self {
|
|
Self {
|
|
max_latency_us,
|
|
..Default::default()
|
|
}
|
|
}
|
|
|
|
/// Create HFT performance profile optimized for ultra-low latency
|
|
pub fn ultra_low_latency() -> Self {
|
|
Self {
|
|
max_latency_us: 10, // 10 microseconds target
|
|
target_throughput: 50000, // 50k operations per second
|
|
memory_limit_mb: 512, // Reduced memory for cache efficiency
|
|
gpu_enabled: true, // Enable GPU acceleration
|
|
batch_size: 1, // No batching for minimal latency
|
|
optimization_level: OptimizationLevel::UltraLow,
|
|
..Default::default()
|
|
}
|
|
}
|
|
}
|
|
|
|
// ========== FEATURES, PREDICTION, FEEDBACK ==========
|
|
|
|
/// Features vector for ML model input
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct Features {
|
|
/// Raw feature values
|
|
pub values: Vec<f64>,
|
|
/// Feature names for debugging
|
|
pub names: Vec<String>,
|
|
/// Timestamp of features
|
|
pub timestamp: u64,
|
|
/// Symbol these features are for
|
|
pub symbol: Option<String>,
|
|
}
|
|
|
|
impl Features {
|
|
pub fn new(values: Vec<f64>, names: Vec<String>) -> Self {
|
|
Self {
|
|
values,
|
|
names,
|
|
timestamp: std::time::SystemTime::now()
|
|
.duration_since(std::time::UNIX_EPOCH)
|
|
.unwrap_or_default()
|
|
.as_micros() as u64,
|
|
symbol: None,
|
|
}
|
|
}
|
|
|
|
/// Set the symbol for this market data point
|
|
pub fn with_symbol(mut self, symbol: String) -> Self {
|
|
self.symbol = Some(symbol);
|
|
self
|
|
}
|
|
}
|
|
|
|
/// Model prediction result
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct ModelPrediction {
|
|
/// Predicted value (price direction, probability, etc.)
|
|
pub value: f64,
|
|
/// Model confidence (0.0 to 1.0)
|
|
pub confidence: f64,
|
|
/// Additional model-specific metadata
|
|
pub metadata: HashMap<String, serde_json::Value>,
|
|
/// Prediction timestamp
|
|
pub timestamp: u64,
|
|
/// Model identifier
|
|
pub model_id: String,
|
|
}
|
|
|
|
impl ModelPrediction {
|
|
pub fn new(model_id: String, value: f64, confidence: f64) -> Self {
|
|
Self {
|
|
value,
|
|
confidence,
|
|
metadata: HashMap::new(),
|
|
timestamp: std::time::SystemTime::now()
|
|
.duration_since(std::time::UNIX_EPOCH)
|
|
.unwrap_or_default()
|
|
.as_micros() as u64,
|
|
model_id,
|
|
}
|
|
}
|
|
|
|
/// Add metadata to the model prediction
|
|
pub fn with_metadata(mut self, key: String, value: serde_json::Value) -> Self {
|
|
self.metadata.insert(key, value);
|
|
self
|
|
}
|
|
}
|
|
|
|
/// Feedback for model weight updates
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct Feedback {
|
|
/// Actual outcome (for supervised learning)
|
|
pub actual_value: Option<f64>,
|
|
/// Reward signal (for reinforcement learning)
|
|
pub reward: Option<f64>,
|
|
/// Trading performance metrics
|
|
pub performance_metrics: HashMap<String, f64>,
|
|
/// Timestamp of feedback
|
|
pub timestamp: u64,
|
|
}
|
|
|
|
impl Feedback {
|
|
pub fn new() -> Self {
|
|
Self {
|
|
actual_value: None,
|
|
reward: None,
|
|
performance_metrics: HashMap::new(),
|
|
timestamp: std::time::SystemTime::now()
|
|
.duration_since(std::time::UNIX_EPOCH)
|
|
.unwrap_or_default()
|
|
.as_micros() as u64,
|
|
}
|
|
}
|
|
|
|
/// Set the actual outcome value for supervised learning feedback
|
|
pub fn with_actual(mut self, actual: f64) -> Self {
|
|
self.actual_value = Some(actual);
|
|
self
|
|
}
|
|
|
|
/// Set the reward signal for reinforcement learning feedback
|
|
pub fn with_reward(mut self, reward: f64) -> Self {
|
|
self.reward = Some(reward);
|
|
self
|
|
}
|
|
}
|
|
|
|
impl Default for Feedback {
|
|
fn default() -> Self {
|
|
Self::new()
|
|
}
|
|
}
|
|
|
|
// ========== ML MODEL TRAIT ==========
|
|
|
|
/// Unified interface for all ML models in the system
|
|
#[async_trait]
|
|
pub trait MLModel: Send + Sync + std::fmt::Debug {
|
|
/// Get unique model identifier
|
|
fn name(&self) -> &str;
|
|
|
|
/// Get model type
|
|
fn model_type(&self) -> ModelType;
|
|
|
|
/// Make prediction based on features
|
|
async fn predict(&self, features: &Features) -> MLResult<ModelPrediction>;
|
|
|
|
/// Get current model confidence score (0.0 to 1.0)
|
|
fn get_confidence(&self) -> f64;
|
|
|
|
/// Update model weights based on feedback (optional - not all models support online learning)
|
|
async fn update_weights(&mut self, _feedback: &Feedback) -> MLResult<()> {
|
|
// Default implementation does nothing (for immutable models)
|
|
Ok(())
|
|
}
|
|
|
|
/// Check if model is ready for predictions
|
|
fn is_ready(&self) -> bool {
|
|
true // Default to ready
|
|
}
|
|
|
|
/// Get model metadata
|
|
fn get_metadata(&self) -> ModelMetadata;
|
|
|
|
/// Validate input features
|
|
fn validate_features(&self, features: &Features) -> MLResult<()> {
|
|
// Default validation - check for empty features
|
|
if features.values.is_empty() {
|
|
return Err(MLError::ValidationError {
|
|
message: "Empty feature vector".to_owned(),
|
|
});
|
|
}
|
|
Ok(())
|
|
}
|
|
}
|
|
|
|
// ========== CHECKPOINT TRAIT ==========
|
|
|
|
/// Trait that models must implement to support checkpointing
|
|
#[async_trait]
|
|
pub trait Checkpointable: Send + Sync {
|
|
/// Get model type
|
|
fn model_type(&self) -> ModelType;
|
|
|
|
/// Get model name/identifier
|
|
fn model_name(&self) -> &str;
|
|
|
|
/// Get model version
|
|
fn model_version(&self) -> &str;
|
|
|
|
/// Serialize model weights and state to bytes
|
|
async fn serialize_state(&self) -> Result<Vec<u8>, MLError>;
|
|
|
|
/// Deserialize model weights and state from bytes
|
|
async fn deserialize_state(&mut self, data: &[u8]) -> Result<(), MLError>;
|
|
|
|
/// Get current training state (epoch, step, loss, accuracy)
|
|
fn get_training_state(&self) -> (Option<u64>, Option<u64>, Option<f64>, Option<f64>) {
|
|
(None, None, None, None)
|
|
}
|
|
|
|
/// Get hyperparameters for metadata
|
|
fn get_hyperparameters(&self) -> HashMap<String, serde_json::Value> {
|
|
HashMap::new()
|
|
}
|
|
|
|
/// Get current metrics for metadata
|
|
fn get_metrics(&self) -> HashMap<String, f64> {
|
|
HashMap::new()
|
|
}
|
|
|
|
/// Get architecture information
|
|
fn get_architecture_info(&self) -> HashMap<String, serde_json::Value> {
|
|
HashMap::new()
|
|
}
|
|
|
|
/// Validate loaded state (optional override for custom validation)
|
|
fn validate_loaded_state(&self) -> Result<(), MLError> {
|
|
Ok(())
|
|
}
|
|
}
|
|
|
|
// ========== MODEL REGISTRY ==========
|
|
|
|
/// Thread-safe model registry using DashMap for high-performance concurrent access
|
|
pub struct ModelRegistry {
|
|
/// Models stored by name
|
|
models: dashmap::DashMap<String, Arc<dyn MLModel>>,
|
|
/// Registry metadata
|
|
metadata: Arc<RwLock<RegistryMetadata>>,
|
|
}
|
|
|
|
impl std::fmt::Debug for ModelRegistry {
|
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
|
f.debug_struct("ModelRegistry")
|
|
.field(
|
|
"models",
|
|
&format_args!("<DashMap with {} models>", self.models.len()),
|
|
)
|
|
.field("metadata", &self.metadata)
|
|
.finish()
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Clone)]
|
|
struct RegistryMetadata {
|
|
created_at: std::time::SystemTime,
|
|
total_registrations: u64,
|
|
last_access: std::time::SystemTime,
|
|
}
|
|
|
|
impl ModelRegistry {
|
|
/// Create new model registry
|
|
pub fn new() -> Self {
|
|
Self {
|
|
models: dashmap::DashMap::new(),
|
|
metadata: Arc::new(RwLock::new(RegistryMetadata {
|
|
created_at: std::time::SystemTime::now(),
|
|
total_registrations: 0,
|
|
last_access: std::time::SystemTime::now(),
|
|
})),
|
|
}
|
|
}
|
|
|
|
/// Register a model in the registry
|
|
pub async fn register(&self, model: Arc<dyn MLModel>) -> MLResult<()> {
|
|
let name = model.name().to_string();
|
|
|
|
// Check if model is ready
|
|
if !model.is_ready() {
|
|
return Err(MLError::ModelError(format!("Model {} is not ready", name)));
|
|
}
|
|
|
|
self.models.insert(name.clone(), model);
|
|
|
|
// Update metadata
|
|
{
|
|
let mut meta = self.metadata.write().await;
|
|
meta.total_registrations += 1;
|
|
meta.last_access = std::time::SystemTime::now();
|
|
}
|
|
|
|
tracing::info!("Registered ML model: {}", name);
|
|
Ok(())
|
|
}
|
|
|
|
/// Get model by name
|
|
pub async fn get(&self, name: &str) -> Option<Arc<dyn MLModel>> {
|
|
// Update last access time
|
|
{
|
|
let mut meta = self.metadata.write().await;
|
|
meta.last_access = std::time::SystemTime::now();
|
|
}
|
|
|
|
self.models.get(name).map(|entry| entry.value().clone())
|
|
}
|
|
|
|
/// Get all registered models
|
|
pub fn get_all(&self) -> Vec<Arc<dyn MLModel>> {
|
|
self.models
|
|
.iter()
|
|
.map(|entry| entry.value().clone())
|
|
.collect()
|
|
}
|
|
|
|
/// Get model names
|
|
pub fn get_model_names(&self) -> Vec<String> {
|
|
self.models
|
|
.iter()
|
|
.map(|entry| entry.key().clone())
|
|
.collect()
|
|
}
|
|
|
|
/// Remove model from registry
|
|
pub async fn remove(&self, name: &str) -> Option<Arc<dyn MLModel>> {
|
|
let result = self.models.remove(name).map(|(_, model)| model);
|
|
|
|
if result.is_some() {
|
|
tracing::info!("Removed ML model: {}", name);
|
|
}
|
|
|
|
result
|
|
}
|
|
|
|
/// Get registry statistics
|
|
pub async fn get_stats(&self) -> RegistryStats {
|
|
let meta = self.metadata.read().await;
|
|
RegistryStats {
|
|
total_models: self.models.len(),
|
|
total_registrations: meta.total_registrations,
|
|
created_at: meta.created_at,
|
|
last_access: meta.last_access,
|
|
}
|
|
}
|
|
|
|
/// Parallel prediction across all models
|
|
pub async fn predict_all(&self, features: &Features) -> Vec<MLResult<ModelPrediction>> {
|
|
let models = self.get_all();
|
|
let futures = models.iter().map(|model| {
|
|
let features = features.clone();
|
|
async move { model.predict(&features).await }
|
|
});
|
|
|
|
join_all(futures).await
|
|
}
|
|
|
|
/// Parallel prediction across specific models
|
|
pub async fn predict_selected(
|
|
&self,
|
|
model_names: &[String],
|
|
features: &Features,
|
|
) -> Vec<MLResult<ModelPrediction>> {
|
|
let futures = model_names.iter().map(|name| {
|
|
let name = name.clone();
|
|
let features = features.clone();
|
|
async move {
|
|
if let Some(model) = self.get(&name).await {
|
|
model.predict(&features).await
|
|
} else {
|
|
Err(MLError::ModelNotFound(name))
|
|
}
|
|
}
|
|
});
|
|
|
|
join_all(futures).await
|
|
}
|
|
}
|
|
|
|
impl Default for ModelRegistry {
|
|
fn default() -> Self {
|
|
Self::new()
|
|
}
|
|
}
|
|
|
|
/// Registry statistics
|
|
#[derive(Debug, Clone)]
|
|
pub struct RegistryStats {
|
|
pub total_models: usize,
|
|
pub total_registrations: u64,
|
|
pub created_at: std::time::SystemTime,
|
|
pub last_access: std::time::SystemTime,
|
|
}
|
|
|
|
// ========== GLOBAL REGISTRY ==========
|
|
|
|
/// Global model registry instance (singleton pattern)
|
|
static GLOBAL_REGISTRY: once_cell::sync::Lazy<Arc<ModelRegistry>> =
|
|
once_cell::sync::Lazy::new(|| Arc::new(ModelRegistry::new()));
|
|
|
|
/// Get global model registry
|
|
pub fn get_global_registry() -> Arc<ModelRegistry> {
|
|
GLOBAL_REGISTRY.clone()
|
|
}
|
|
|
|
// ========== PARALLEL EXECUTOR ==========
|
|
|
|
/// High-performance parallel executor for ML models optimized for sub-50us latency
|
|
#[derive(Debug)]
|
|
pub struct ParallelExecutor {
|
|
/// Performance profile
|
|
profile: HFTPerformanceProfile,
|
|
/// CPU affinity settings
|
|
cpu_affinity: Option<Vec<usize>>,
|
|
/// Thread pool for CPU-bound operations
|
|
cpu_pool: Arc<rayon::ThreadPool>,
|
|
/// Async runtime handle
|
|
runtime_handle: tokio::runtime::Handle,
|
|
}
|
|
|
|
impl ParallelExecutor {
|
|
/// Create new parallel executor with HFT performance profile
|
|
pub fn new(profile: HFTPerformanceProfile) -> Result<Self, MLError> {
|
|
// Create dedicated thread pool based on profile
|
|
let cpu_pool = rayon::ThreadPoolBuilder::new()
|
|
.num_threads(
|
|
profile
|
|
.cpu_affinity
|
|
.as_ref()
|
|
.map(|v| v.len())
|
|
.unwrap_or(num_cpus::get()),
|
|
)
|
|
.thread_name(|i| format!("ml-cpu-{}", i))
|
|
.build()
|
|
.map_err(|e| MLError::ModelError(format!("Failed to create thread pool: {}", e)))?;
|
|
|
|
let runtime_handle = tokio::runtime::Handle::try_current()
|
|
.map_err(|e| MLError::ModelError(format!("No tokio runtime available: {}", e)))?;
|
|
|
|
let cpu_affinity = profile.cpu_affinity.clone();
|
|
|
|
Ok(Self {
|
|
profile,
|
|
cpu_affinity,
|
|
cpu_pool: Arc::new(cpu_pool),
|
|
runtime_handle,
|
|
})
|
|
}
|
|
|
|
/// Create an HFT-optimized parallel executor
|
|
pub fn hft() -> Result<Self, MLError> {
|
|
let profile = HFTPerformanceProfile::ultra_low_latency();
|
|
Self::new(profile)
|
|
}
|
|
|
|
/// Execute parallel predictions with latency optimization
|
|
pub async fn execute_parallel_predictions(
|
|
&self,
|
|
models: Vec<Arc<dyn MLModel>>,
|
|
features: Features,
|
|
) -> Vec<MLResult<ModelPrediction>> {
|
|
let start_time = std::time::Instant::now();
|
|
|
|
// Determine execution strategy based on performance profile
|
|
let results = match self.profile.optimization_level {
|
|
OptimizationLevel::UltraLow => {
|
|
// Ultra-low latency: parallel execution with minimal overhead
|
|
self.execute_ultra_low_latency(models, features).await
|
|
},
|
|
OptimizationLevel::Low => {
|
|
// Low latency: parallel with basic batching
|
|
self.execute_low_latency(models, features).await
|
|
},
|
|
OptimizationLevel::Medium => {
|
|
// Medium: balanced parallel execution
|
|
self.execute_balanced(models, features).await
|
|
},
|
|
OptimizationLevel::High => {
|
|
// High: conservative with full validation
|
|
self.execute_conservative(models, features).await
|
|
},
|
|
};
|
|
|
|
let execution_time = start_time.elapsed();
|
|
|
|
// Log performance if exceeding target latency
|
|
if execution_time.as_micros() > self.profile.max_latency_us as u128 {
|
|
tracing::warn!(
|
|
"Parallel execution exceeded target latency: {}us > {}us",
|
|
execution_time.as_micros(),
|
|
self.profile.max_latency_us
|
|
);
|
|
}
|
|
|
|
results
|
|
}
|
|
|
|
/// Ultra-low latency execution (<10us target)
|
|
async fn execute_ultra_low_latency(
|
|
&self,
|
|
models: Vec<Arc<dyn MLModel>>,
|
|
features: Features,
|
|
) -> Vec<MLResult<ModelPrediction>> {
|
|
// Use futures::future::join_all for minimal overhead
|
|
let futures = models.into_iter().map(|model| {
|
|
let features = features.clone();
|
|
async move { model.predict(&features).await }
|
|
});
|
|
|
|
join_all(futures).await
|
|
}
|
|
|
|
/// Low latency execution with basic optimizations
|
|
async fn execute_low_latency(
|
|
&self,
|
|
models: Vec<Arc<dyn MLModel>>,
|
|
features: Features,
|
|
) -> Vec<MLResult<ModelPrediction>> {
|
|
// Group models by type for potential batching
|
|
let mut model_groups: HashMap<ModelType, Vec<Arc<dyn MLModel>>> = HashMap::new();
|
|
|
|
for model in models {
|
|
let model_type = model.model_type();
|
|
model_groups.entry(model_type).or_default().push(model);
|
|
}
|
|
|
|
let mut all_futures = Vec::new();
|
|
|
|
for (_, group_models) in model_groups {
|
|
for model in group_models {
|
|
let features = features.clone();
|
|
all_futures.push(async move { model.predict(&features).await });
|
|
}
|
|
}
|
|
|
|
join_all(all_futures).await
|
|
}
|
|
|
|
/// Balanced execution with moderate optimizations
|
|
async fn execute_balanced(
|
|
&self,
|
|
models: Vec<Arc<dyn MLModel>>,
|
|
features: Features,
|
|
) -> Vec<MLResult<ModelPrediction>> {
|
|
// Validate features once for all models
|
|
for model in &models {
|
|
if let Err(e) = model.validate_features(&features) {
|
|
tracing::debug!(
|
|
"Feature validation failed for model {}: {}",
|
|
model.name(),
|
|
e
|
|
);
|
|
}
|
|
}
|
|
|
|
let futures = models.into_iter().map(|model| {
|
|
let features = features.clone();
|
|
async move {
|
|
if model.is_ready() {
|
|
model.predict(&features).await
|
|
} else {
|
|
Err(MLError::ModelError(format!(
|
|
"Model {} not ready",
|
|
model.name()
|
|
)))
|
|
}
|
|
}
|
|
});
|
|
|
|
join_all(futures).await
|
|
}
|
|
|
|
/// Conservative execution with full validation
|
|
async fn execute_conservative(
|
|
&self,
|
|
models: Vec<Arc<dyn MLModel>>,
|
|
features: Features,
|
|
) -> Vec<MLResult<ModelPrediction>> {
|
|
let mut results = Vec::new();
|
|
|
|
for model in models {
|
|
// Comprehensive validation
|
|
if !model.is_ready() {
|
|
results.push(Err(MLError::ModelError(format!(
|
|
"Model {} not ready",
|
|
model.name()
|
|
))));
|
|
continue;
|
|
}
|
|
|
|
if let Err(e) = model.validate_features(&features) {
|
|
results.push(Err(e));
|
|
continue;
|
|
}
|
|
|
|
// Execute with timeout
|
|
let prediction_future = model.predict(&features);
|
|
let timeout_duration = std::time::Duration::from_micros(self.profile.max_latency_us);
|
|
|
|
match tokio::time::timeout(timeout_duration, prediction_future).await {
|
|
Ok(result) => results.push(result),
|
|
Err(_) => results.push(Err(MLError::ModelError(format!(
|
|
"Model {} prediction timed out after {}us",
|
|
model.name(),
|
|
self.profile.max_latency_us
|
|
)))),
|
|
}
|
|
}
|
|
|
|
results
|
|
}
|
|
|
|
/// Get execution statistics
|
|
pub fn get_stats(&self) -> ExecutorStats {
|
|
ExecutorStats {
|
|
optimization_level: self.profile.optimization_level,
|
|
target_latency_us: self.profile.max_latency_us,
|
|
cpu_threads: self.cpu_pool.current_num_threads(),
|
|
cpu_affinity: self.cpu_affinity.clone(),
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Executor performance statistics
|
|
#[derive(Debug, Clone)]
|
|
pub struct ExecutorStats {
|
|
pub optimization_level: OptimizationLevel,
|
|
pub target_latency_us: u64,
|
|
pub cpu_threads: usize,
|
|
pub cpu_affinity: Option<Vec<usize>>,
|
|
}
|
|
|
|
// ========== LATENCY OPTIMIZER ==========
|
|
|
|
/// Latency optimizer for ML inference pipelines
|
|
#[derive(Debug)]
|
|
pub struct LatencyOptimizer {
|
|
/// Target latency in microseconds
|
|
target_latency_us: u64,
|
|
/// Performance history
|
|
performance_history: Arc<RwLock<Vec<PerformancePoint>>>,
|
|
/// Optimization parameters
|
|
optimization_params: OptimizationParams,
|
|
}
|
|
|
|
#[derive(Debug, Clone)]
|
|
struct PerformancePoint {
|
|
timestamp: std::time::Instant,
|
|
latency_us: u64,
|
|
model_count: usize,
|
|
batch_size: u32,
|
|
success: bool,
|
|
}
|
|
|
|
#[derive(Debug, Clone)]
|
|
struct OptimizationParams {
|
|
max_batch_size: u32,
|
|
adaptive_batching: bool,
|
|
prefetch_enabled: bool,
|
|
cache_predictions: bool,
|
|
}
|
|
|
|
impl Default for OptimizationParams {
|
|
fn default() -> Self {
|
|
Self {
|
|
max_batch_size: 8,
|
|
adaptive_batching: true,
|
|
prefetch_enabled: true,
|
|
cache_predictions: false, // Disabled for real-time trading
|
|
}
|
|
}
|
|
}
|
|
|
|
impl LatencyOptimizer {
|
|
/// Create new latency optimizer
|
|
pub fn new(target_latency_us: u64) -> Self {
|
|
Self {
|
|
target_latency_us,
|
|
performance_history: Arc::new(RwLock::new(Vec::new())),
|
|
optimization_params: OptimizationParams::default(),
|
|
}
|
|
}
|
|
|
|
/// Create latency optimizer with HFT targets
|
|
pub fn hft() -> Self {
|
|
Self::new(50) // 50 microsecond target
|
|
}
|
|
|
|
/// Record performance measurement
|
|
pub async fn record_performance(
|
|
&self,
|
|
latency_us: u64,
|
|
model_count: usize,
|
|
batch_size: u32,
|
|
success: bool,
|
|
) {
|
|
let point = PerformancePoint {
|
|
timestamp: std::time::Instant::now(),
|
|
latency_us,
|
|
model_count,
|
|
batch_size,
|
|
success,
|
|
};
|
|
|
|
{
|
|
let mut history = self.performance_history.write().await;
|
|
history.push(point);
|
|
|
|
// Keep only recent history (last 1000 measurements)
|
|
if history.len() > 1000 {
|
|
let excess = history.len() - 1000;
|
|
history.drain(0..excess);
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Get optimization recommendations
|
|
pub async fn get_recommendations(&self) -> OptimizationRecommendations {
|
|
let history = self.performance_history.read().await;
|
|
|
|
if history.is_empty() {
|
|
return OptimizationRecommendations::default();
|
|
}
|
|
|
|
let recent_points: Vec<&PerformancePoint> = history.iter().rev().take(100).collect();
|
|
|
|
let avg_latency =
|
|
recent_points.iter().map(|p| p.latency_us).sum::<u64>() / recent_points.len() as u64;
|
|
|
|
let success_rate =
|
|
recent_points.iter().filter(|p| p.success).count() as f64 / recent_points.len() as f64;
|
|
|
|
OptimizationRecommendations {
|
|
current_avg_latency_us: avg_latency,
|
|
target_latency_us: self.target_latency_us,
|
|
success_rate,
|
|
meets_target: avg_latency <= self.target_latency_us,
|
|
recommended_batch_size: self.calculate_optimal_batch_size(&recent_points),
|
|
recommended_model_limit: self.calculate_optimal_model_limit(&recent_points),
|
|
}
|
|
}
|
|
|
|
fn calculate_optimal_batch_size(&self, points: &[&PerformancePoint]) -> u32 {
|
|
// Simple heuristic: find batch size with best latency/success ratio
|
|
let mut batch_performance: HashMap<u32, (u64, f64)> = HashMap::new();
|
|
|
|
for point in points {
|
|
let entry = batch_performance
|
|
.entry(point.batch_size)
|
|
.or_insert((0, 0.0));
|
|
entry.0 += point.latency_us;
|
|
entry.1 += if point.success { 1.0 } else { 0.0 };
|
|
}
|
|
|
|
batch_performance
|
|
.into_iter()
|
|
.filter(|(_, (_, success_count))| *success_count > 0.0)
|
|
.min_by_key(|(_, (latency, success_count))| {
|
|
// Optimize for latency with success rate weighting
|
|
((*latency as f64) / success_count) as u64
|
|
})
|
|
.map(|(batch_size, _)| batch_size)
|
|
.unwrap_or(1)
|
|
}
|
|
|
|
fn calculate_optimal_model_limit(&self, points: &[&PerformancePoint]) -> usize {
|
|
// Find the sweet spot where adding more models doesn't improve latency
|
|
let mut model_performance: HashMap<usize, u64> = HashMap::new();
|
|
|
|
for point in points {
|
|
if point.success {
|
|
let entry = model_performance.entry(point.model_count).or_insert(0);
|
|
*entry += point.latency_us;
|
|
}
|
|
}
|
|
|
|
model_performance
|
|
.into_iter()
|
|
.filter(|(_, avg_latency)| *avg_latency <= self.target_latency_us)
|
|
.max_by_key(|(model_count, _)| *model_count)
|
|
.map(|(model_count, _)| model_count)
|
|
.unwrap_or(1)
|
|
}
|
|
}
|
|
|
|
/// Optimization recommendations from latency analysis
|
|
#[derive(Debug, Clone)]
|
|
pub struct OptimizationRecommendations {
|
|
pub current_avg_latency_us: u64,
|
|
pub target_latency_us: u64,
|
|
pub success_rate: f64,
|
|
pub meets_target: bool,
|
|
pub recommended_batch_size: u32,
|
|
pub recommended_model_limit: usize,
|
|
}
|
|
|
|
impl Default for OptimizationRecommendations {
|
|
fn default() -> Self {
|
|
Self {
|
|
current_avg_latency_us: 0,
|
|
target_latency_us: 50,
|
|
success_rate: 0.0,
|
|
meets_target: false,
|
|
recommended_batch_size: 1,
|
|
recommended_model_limit: 1,
|
|
}
|
|
}
|
|
}
|
|
|
|
// ========== TRAINING AND VALIDATION METRICS ==========
|
|
|
|
/// Canonical training metrics used throughout ML module
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct TrainingMetrics {
|
|
/// Training loss value
|
|
pub loss: f64,
|
|
/// Training accuracy (0.0 to 1.0)
|
|
pub accuracy: f64,
|
|
/// Training precision (0.0 to 1.0)
|
|
pub precision: f64,
|
|
/// Training recall (0.0 to 1.0)
|
|
pub recall: f64,
|
|
/// Training F1 score (0.0 to 1.0)
|
|
pub f1_score: f64,
|
|
/// Total training time in seconds
|
|
pub training_time_seconds: f64,
|
|
/// Number of epochs trained
|
|
pub epochs_trained: u32,
|
|
/// Whether convergence was achieved
|
|
pub convergence_achieved: bool,
|
|
/// Additional model-specific metrics
|
|
pub additional_metrics: HashMap<String, f64>,
|
|
}
|
|
|
|
impl TrainingMetrics {
|
|
/// Create new training metrics
|
|
pub fn new() -> Self {
|
|
Self {
|
|
loss: 0.0,
|
|
accuracy: 0.0,
|
|
precision: 0.0,
|
|
recall: 0.0,
|
|
f1_score: 0.0,
|
|
training_time_seconds: 0.0,
|
|
epochs_trained: 0,
|
|
convergence_achieved: false,
|
|
additional_metrics: HashMap::new(),
|
|
}
|
|
}
|
|
|
|
/// Add an additional metric
|
|
pub fn add_metric(&mut self, name: &str, value: f64) {
|
|
self.additional_metrics.insert(name.to_string(), value);
|
|
}
|
|
|
|
/// Check if training was successful
|
|
pub fn is_successful(&self) -> bool {
|
|
self.convergence_achieved && self.accuracy > 0.5
|
|
}
|
|
}
|
|
|
|
impl Default for TrainingMetrics {
|
|
fn default() -> Self {
|
|
Self::new()
|
|
}
|
|
}
|
|
|
|
/// Canonical validation metrics used throughout ML module
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct ValidationMetrics {
|
|
/// Validation loss value
|
|
pub validation_loss: f64,
|
|
/// Validation accuracy (0.0 to 1.0)
|
|
pub validation_accuracy: f64,
|
|
/// Validation precision (0.0 to 1.0)
|
|
pub validation_precision: f64,
|
|
/// Validation recall (0.0 to 1.0)
|
|
pub validation_recall: f64,
|
|
/// Validation F1 score (0.0 to 1.0)
|
|
pub validation_f1_score: f64,
|
|
/// Number of samples validated
|
|
pub samples_validated: usize,
|
|
/// Additional model-specific validation metrics
|
|
pub additional_metrics: HashMap<String, f64>,
|
|
}
|
|
|
|
impl ValidationMetrics {
|
|
/// Create new validation metrics
|
|
pub fn new() -> Self {
|
|
Self {
|
|
validation_loss: 0.0,
|
|
validation_accuracy: 0.0,
|
|
validation_precision: 0.0,
|
|
validation_recall: 0.0,
|
|
validation_f1_score: 0.0,
|
|
samples_validated: 0,
|
|
additional_metrics: HashMap::new(),
|
|
}
|
|
}
|
|
|
|
/// Add an additional validation metric
|
|
pub fn add_metric(&mut self, name: &str, value: f64) {
|
|
self.additional_metrics.insert(name.to_string(), value);
|
|
}
|
|
|
|
/// Check if validation was successful
|
|
pub fn is_successful(&self) -> bool {
|
|
self.validation_accuracy > 0.5 && self.samples_validated > 0
|
|
}
|
|
}
|
|
|
|
impl Default for ValidationMetrics {
|
|
fn default() -> Self {
|
|
Self::new()
|
|
}
|
|
}
|
|
|
|
// ========== INFERENCE RESULT & MODEL METADATA ==========
|
|
|
|
/// Canonical inference result used throughout ML module
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct InferenceResult {
|
|
/// Model identifier
|
|
pub model_id: String,
|
|
/// Prediction value (primary prediction)
|
|
pub prediction_value: f64,
|
|
/// Confidence score (0.0 to 1.0)
|
|
pub confidence: f64,
|
|
/// Latency in microseconds
|
|
pub latency_us: u64,
|
|
/// Timestamp in microseconds since UNIX epoch
|
|
pub timestamp: u64,
|
|
/// Model metadata
|
|
pub metadata: ModelMetadata,
|
|
}
|
|
|
|
impl InferenceResult {
|
|
/// Create new inference result
|
|
pub fn new(
|
|
model_id: String,
|
|
prediction_value: f64,
|
|
confidence: f64,
|
|
latency_us: u64,
|
|
timestamp: u64,
|
|
metadata: ModelMetadata,
|
|
) -> Self {
|
|
Self {
|
|
model_id,
|
|
prediction_value,
|
|
confidence,
|
|
latency_us,
|
|
timestamp,
|
|
metadata,
|
|
}
|
|
}
|
|
|
|
/// Extract prediction as float value
|
|
pub fn prediction_as_float(&self) -> f64 {
|
|
self.prediction_value
|
|
}
|
|
|
|
/// Get the model identifier
|
|
pub fn model_id(&self) -> &str {
|
|
&self.model_id
|
|
}
|
|
}
|
|
|
|
/// Canonical model metadata used throughout ML module
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct ModelMetadata {
|
|
/// Type of the model
|
|
pub model_type: ModelType,
|
|
/// Model version
|
|
pub version: String,
|
|
/// Number of features used for inference
|
|
pub features_used: usize,
|
|
/// Memory usage in megabytes
|
|
pub memory_usage_mb: f64,
|
|
/// Additional metadata key-value pairs
|
|
pub additional_metadata: HashMap<String, String>,
|
|
}
|
|
|
|
impl ModelMetadata {
|
|
/// Create new model metadata
|
|
pub fn new(
|
|
model_type: ModelType,
|
|
version: String,
|
|
features_used: usize,
|
|
memory_usage_mb: f64,
|
|
) -> Self {
|
|
Self {
|
|
model_type,
|
|
version,
|
|
features_used,
|
|
memory_usage_mb,
|
|
additional_metadata: HashMap::new(),
|
|
}
|
|
}
|
|
|
|
/// Add additional metadata
|
|
pub fn add_metadata(&mut self, key: &str, value: String) {
|
|
self.additional_metadata.insert(key.to_string(), value);
|
|
}
|
|
|
|
/// Mark the model as trained
|
|
pub fn mark_trained(&mut self) {
|
|
self.add_metadata("training_status", "trained".to_owned());
|
|
self.add_metadata(
|
|
"training_timestamp",
|
|
std::time::SystemTime::now()
|
|
.duration_since(std::time::UNIX_EPOCH)
|
|
.unwrap_or_default()
|
|
.as_secs()
|
|
.to_string(),
|
|
);
|
|
}
|
|
}
|
|
|
|
// ========== RE-EXPORTS ==========
|
|
|
|
// Re-export canonical ModelType from common crate
|
|
pub use ::common::model_types::ModelType;
|
|
|
|
// ========== PRELUDE ==========
|
|
|
|
/// Prelude module for convenient imports of commonly used ML types
|
|
///
|
|
/// This module re-exports the most commonly used types and traits from the ML crate
|
|
/// to allow users to import everything they need with a single `use ml_core::prelude::*;`
|
|
pub mod prelude {
|
|
// Core ML types
|
|
pub use crate::{
|
|
CommonError, CommonTypeError, ErrorCategory, FeatureVector, Features, Feedback,
|
|
HealthStatus, InferenceResult, IntegerTensor, MarketDataSnapshot, MarketRegime,
|
|
ModelMetadata, ModelPrediction, ModelType, Trade, TrainingMetrics, UpdateSummary,
|
|
ValidationMetrics,
|
|
};
|
|
|
|
// Error types
|
|
pub use crate::{MLError, MLResult, UnifiedMLResult};
|
|
|
|
// ML Model trait
|
|
pub use crate::MLModel;
|
|
|
|
// Model registry
|
|
pub use crate::{get_global_registry, ModelRegistry, RegistryStats};
|
|
|
|
// Performance types
|
|
pub use crate::{
|
|
ExecutorStats, HFTPerformanceProfile, LatencyOptimizer, OptimizationLevel,
|
|
OptimizationRecommendations, ParallelExecutor,
|
|
};
|
|
|
|
// Constants
|
|
pub use crate::{MAX_INFERENCE_LATENCY_US, PRECISION_FACTOR};
|
|
|
|
// Device abstraction (Candle-free)
|
|
pub use crate::device::MlDevice;
|
|
|
|
// Native type replacements (Candle-free)
|
|
pub use crate::native_types::{NativeDevice, NativeDType};
|
|
#[cfg(feature = "cuda")]
|
|
pub use crate::native_types::NativeTensor;
|
|
|
|
// Common external types
|
|
pub use rust_decimal::Decimal;
|
|
pub use serde::{Deserialize, Serialize};
|
|
}
|