From 7d1b0f232f32771bce4fb7a57fb5aecff4ad72f5 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sat, 7 Mar 2026 22:57:50 +0100 Subject: [PATCH] refactor(ml): move core types to ml-core + dedup MLError (5a) Move all inline type definitions from ml/src/lib.rs to ml-core: MLError, MLResult, Trade, MarketRegime, HealthStatus, Features, MLModel trait, ModelRegistry, ParallelExecutor, LatencyOptimizer, TrainingMetrics, ValidationMetrics, InferenceResult, ModelMetadata. Dedup: consolidate ConfigError{reason}/ConfigurationError(msg) into single ConfigError(String) tuple variant (was 2 variants, 174 refs). Cleanup: convert create_hft_* free functions to associated methods (HFTPerformanceProfile::ultra_low_latency(), ParallelExecutor::hft()). ml facade re-exports via `pub use ml_core::*`. Co-Authored-By: Claude Opus 4.6 --- Cargo.lock | 61 + crates/ml-core/Cargo.toml | 43 +- crates/ml-core/src/lib.rs | 1709 ++++++++++++++- crates/ml/Cargo.toml | 1 + crates/ml/src/asset_selection/mod.rs | 12 +- crates/ml/src/batch_processing.rs | 4 +- crates/ml/src/checkpoint/signer.rs | 10 +- crates/ml/src/data_pipeline/cache.rs | 20 +- crates/ml/src/data_pipeline/manager.rs | 6 +- crates/ml/src/diffusion/denoiser.rs | 6 +- crates/ml/src/diffusion/noise.rs | 8 +- crates/ml/src/diffusion/trainable.rs | 4 +- crates/ml/src/dqn/attention.rs | 6 +- crates/ml/src/dqn/dqn.rs | 8 +- crates/ml/src/dqn/ensemble.rs | 4 +- crates/ml/src/dqn/multi_step.rs | 12 +- crates/ml/src/dqn/reward.rs | 6 +- crates/ml/src/ensemble/mod.rs | 2 +- crates/ml/src/features/sample_weights.rs | 26 +- crates/ml/src/flash_attention/mod.rs | 2 +- crates/ml/src/gpu/mod.rs | 2 +- .../src/hyperopt/adapters/continuous_ppo.rs | 8 +- crates/ml/src/hyperopt/adapters/diffusion.rs | 8 +- crates/ml/src/hyperopt/adapters/dqn.rs | 18 +- crates/ml/src/hyperopt/adapters/ensemble.rs | 12 +- crates/ml/src/hyperopt/adapters/kan.rs | 8 +- crates/ml/src/hyperopt/adapters/liquid.rs | 42 +- crates/ml/src/hyperopt/adapters/mamba2.rs | 8 +- crates/ml/src/hyperopt/adapters/ppo.rs | 48 +- crates/ml/src/hyperopt/adapters/tft.rs | 8 +- crates/ml/src/hyperopt/adapters/tggn.rs | 8 +- crates/ml/src/hyperopt/adapters/tlob.rs | 8 +- crates/ml/src/hyperopt/adapters/xlstm.rs | 8 +- crates/ml/src/hyperopt/optimizer.rs | 12 +- crates/ml/src/hyperopt/tests_argmin.rs | 8 +- crates/ml/src/hyperopt/traits.rs | 4 +- crates/ml/src/integration/coordinator.rs | 6 +- crates/ml/src/integration/inference_engine.rs | 4 +- crates/ml/src/kan/network.rs | 16 +- crates/ml/src/kan/spline.rs | 16 +- .../labeling/meta_labeling/primary_model.rs | 11 +- .../labeling/meta_labeling/secondary_model.rs | 16 +- crates/ml/src/lib.rs | 1861 +---------------- crates/ml/src/liquid/candle_cfc.rs | 14 +- crates/ml/src/liquid/mod.rs | 4 +- crates/ml/src/mamba/mod.rs | 8 +- .../memory_optimization/auto_batch_size.rs | 22 +- crates/ml/src/observability/metrics.rs | 5 +- crates/ml/src/ppo/adaptive_entropy.rs | 14 +- crates/ml/src/ppo/ppo.rs | 10 +- crates/ml/src/regime_detection/hmm.rs | 6 +- crates/ml/src/tests/ml_tests.rs | 4 +- crates/ml/src/tft/hft_optimizations.rs | 2 +- crates/ml/src/tft/mod.rs | 12 +- crates/ml/src/tft/temporal_attention.rs | 2 +- crates/ml/src/tft/training.rs | 4 +- crates/ml/src/tgnn/gating.rs | 6 +- crates/ml/src/tgnn/mod.rs | 4 +- crates/ml/src/tgnn/trainable_adapter.rs | 8 +- crates/ml/src/tlob/trainable_adapter.rs | 6 +- crates/ml/src/trainers/liquid.rs | 8 +- crates/ml/src/trainers/ppo.rs | 24 +- crates/ml/src/trainers/tft/tests.rs | 4 +- crates/ml/src/trainers/tft/trainer.rs | 4 +- crates/ml/src/training/unified_data_loader.rs | 8 +- crates/ml/src/validation/ppo_adapter.rs | 2 +- crates/ml/src/xlstm/mlstm.rs | 6 +- crates/ml/src/xlstm/network.rs | 10 +- .../src/ensemble_risk_manager.rs | 2 +- 69 files changed, 2068 insertions(+), 2221 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 13162576f..d9ed64e13 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6262,6 +6262,7 @@ dependencies = [ "lru", "memmap2", "mimalloc", + "ml-core", "nalgebra 0.33.2", "ndarray", "num", @@ -6305,6 +6306,35 @@ dependencies = [ "zstd", ] +[[package]] +name = "ml-core" +version = "1.0.0" +dependencies = [ + "anyhow", + "async-trait", + "aws-config", + "aws-credential-types", + "aws-sdk-s3", + "aws-types", + "candle-core", + "candle-nn", + "chrono", + "common", + "dashmap 6.1.0", + "futures", + "mimalloc", + "num_cpus", + "once_cell", + "rayon", + "rust_decimal", + "serde", + "serde_json", + "thiserror 1.0.69", + "tokio", + "tracing", + "urlencoding", +] + [[package]] name = "ml-data" version = "0.1.0" @@ -6331,6 +6361,37 @@ dependencies = [ "uuid", ] +[[package]] +name = "ml-dqn" +version = "1.0.0" +dependencies = [ + "ml-core", +] + +[[package]] +name = "ml-infra" +version = "1.0.0" +dependencies = [ + "ml-core", + "ml-dqn", + "ml-ppo", + "ml-supervised", +] + +[[package]] +name = "ml-ppo" +version = "1.0.0" +dependencies = [ + "ml-core", +] + +[[package]] +name = "ml-supervised" +version = "1.0.0" +dependencies = [ + "ml-core", +] + [[package]] name = "ml-training-service" version = "1.0.0" diff --git a/crates/ml-core/Cargo.toml b/crates/ml-core/Cargo.toml index a3ba8f22b..266605f33 100644 --- a/crates/ml-core/Cargo.toml +++ b/crates/ml-core/Cargo.toml @@ -11,9 +11,50 @@ documentation.workspace = true publish.workspace = true keywords.workspace = true categories.workspace = true -description = "Shared ML types, traits, features, and data loading" +description = "Shared ML types, traits, and infrastructure for Foxhunt" + +[features] +default = ["cuda"] +cuda = ["candle-core/cuda", "candle-core/cudnn", "candle-nn/cuda", "candle-nn/cudnn"] +s3-storage = ["aws-config", "aws-sdk-s3", "aws-types", "aws-credential-types", "urlencoding"] +high-precision = ["rust_decimal/serde-float"] +mimalloc-allocator = ["mimalloc"] +simd = [] +gc = [] +financial = [] +minimal-inference = [] +nccl = ["cuda"] [dependencies] +# Core (needed by types in lib.rs) +serde = { workspace = true, features = ["derive"] } +serde_json.workspace = true +thiserror.workspace = true +anyhow.workspace = true +chrono.workspace = true +tracing.workspace = true +rust_decimal.workspace = true +async-trait.workspace = true +futures.workspace = true +tokio.workspace = true +once_cell = "1.19" +dashmap = { workspace = true } +rayon.workspace = true +num_cpus = "1.16" +candle-core = { git = "https://github.com/huggingface/candle", rev = "671de1db" } +candle-nn = { git = "https://github.com/huggingface/candle", rev = "671de1db" } +common.workspace = true + +# Optional +mimalloc = { version = "0.1", optional = true } +aws-config = { version = "1.1", optional = true } +aws-sdk-s3 = { version = "1.14", optional = true } +aws-types = { version = "1.1", optional = true } +aws-credential-types = { version = "1.1", optional = true } +urlencoding = { version = "2.1", optional = true } + +[dev-dependencies] +tokio = { workspace = true, features = ["test-util", "macros"] } [lints] workspace = true diff --git a/crates/ml-core/src/lib.rs b/crates/ml-core/src/lib.rs index 6ef21459c..707a48d29 100644 --- a/crates/ml-core/src/lib.rs +++ b/crates/ml-core/src/lib.rs @@ -1 +1,1708 @@ -// Modules will be moved here from ml crate +#![deny(clippy::unwrap_used, clippy::expect_used)] +#![cfg_attr(test, allow(clippy::unwrap_used, clippy::expect_used))] +#![allow(dead_code)] // 10 ML model implementations with internal architecture not yet fully wired +#![allow(missing_docs)] // Internal implementation details don't require documentation +#![allow(missing_debug_implementations)] // Not all types need Debug +#![allow(unused_crate_dependencies)] // Dev dependencies not used in lib.rs +#![allow(clippy::float_arithmetic)] // ML operations require float arithmetic +// ML-specific lint overrides: these are intentional domain patterns, not safety issues +#![allow(clippy::non_ascii_literal)] // Box-drawing chars for tables, Greek letters for math +#![allow(clippy::str_to_string)] // Pervasive in ML config/display code, not a safety concern +#![allow(clippy::partial_pub_fields)] // ML model structs mix pub config with private state +#![allow(clippy::multiple_inherent_impl)] // Split impls for readability (core vs Display vs Builder) +#![allow(clippy::same_name_method)] // Trait methods intentionally shadow inherent methods +#![allow(clippy::shadow_reuse)] // Tensor code naturally shadows: let x = relu(x) +#![allow(clippy::shadow_unrelated)] // ML pipeline variable reuse across phases +#![allow(clippy::shadow_same)] // Rebinding after narrowing in match/if-let +// ML domain pedantic lints: these are noise in numerical/ML code, not safety issues +// Individual pedantic lints are managed at workspace level in Cargo.toml +#![allow(clippy::doc_markdown)] // Technical terms (AVX2, SIMD, HFT, etc.) in doc comments +#![allow(clippy::indexing_slicing)] // Tensor/array indexing is pervasive and bounds-checked in context +#![allow(clippy::missing_const_for_fn)] // Const fn not critical for ML model code +#![allow(clippy::module_name_repetitions)] // Module-prefixed types provide clarity (DQNAgent, TFTTrainer) +#![allow(clippy::integer_division)] // Integer division is intentional in batch/epoch calculations +#![allow(clippy::cognitive_complexity)] // ML training loops and model architectures are inherently complex +#![allow(clippy::similar_names)] // ML variables often have similar names (x, xs, x_hat, x_norm) +#![allow(clippy::clone_on_ref_ptr)] // Arc::clone is intentional for shared model state +#![allow(clippy::too_many_lines)] // Complex model implementations need many lines +#![allow(clippy::as_conversions)] // Necessary for tensor dimension/type conversions +#![allow(clippy::cast_precision_loss)] // Acceptable in ML with f32/f64 conversions +#![allow(clippy::cast_possible_truncation)] // Type conversions validated in context +#![allow(clippy::default_numeric_fallback)] // Float/int literals are contextually typed in ML code +#![allow(clippy::arithmetic_side_effects)] // ML math uses checked/saturating where needed +#![allow(clippy::needless_range_loop)] // Index-based loops often clearer for tensor operations +#![allow(clippy::into_iter_on_ref)] // .iter() vs .into_iter() on refs is stylistic +#![allow(clippy::new_without_default)] // Many ML types need config params, Default is inappropriate +#![allow(clippy::manual_let_else)] // if-let pattern preferred in many ML error paths +#![allow(clippy::unnecessary_wraps)] // Result/Option wrapping needed for trait consistency +#![allow(clippy::too_many_arguments)] // ML functions often need many hyperparameters +#![allow(clippy::must_use_candidate)] // Not all ML functions need must_use +#![allow(clippy::missing_errors_doc)] // Internal ML APIs don't need full error documentation +#![allow(clippy::cast_sign_loss)] // Sign loss validated in context +#![allow(clippy::cast_possible_wrap)] // Wrap validated in context +#![allow(clippy::cast_lossless)] // as casts are intentional for ML numeric conversions +#![allow(clippy::unused_async)] // Async needed for trait implementations +#![allow(clippy::match_same_arms)] // Explicit match arms preferred for clarity in ML code +#![allow(clippy::unused_self)] // Self parameter needed for trait consistency +#![allow(clippy::map_err_ignore)] // Error conversion doesn't need original context in ML pipelines +#![allow(clippy::single_match_else)] // Explicit match preferred for clarity in ML code +#![allow(clippy::wildcard_imports)] // Prelude-style imports common in ML modules +#![allow(clippy::unnecessary_cast)] // Explicit casts for tensor dimension/type clarity +#![allow(clippy::undocumented_unsafe_blocks)] // Unsafe blocks documented at usage site +#![allow(clippy::redundant_clone)] // Clones needed for ownership in async/parallel ML pipelines +#![allow(clippy::redundant_closure)] // Explicit closures preferred for readability +#![allow(clippy::type_complexity)] // Complex types unavoidable in ML generics +#![allow(clippy::manual_clamp)] // Explicit min/max preferred in numerical code +#![allow(clippy::clone_on_copy)] // Explicit clone for clarity on Copy types +#![allow(clippy::should_implement_trait)] // Custom builder patterns don't need std traits +#![allow(clippy::derivable_impls)] // Custom Default impls with domain-specific values +#![allow(clippy::useless_conversion)] // Explicit conversions for type clarity +#![allow(clippy::get_first)] // .get(0) preferred for consistency with .get(n) +#![allow(clippy::len_zero)] // .len() == 0 preferred for readability in some contexts +#![allow(clippy::assign_op_pattern)] // Explicit assignment preferred in numerical code +#![allow(clippy::if_same_then_else)] // Intentional identical branches for documentation +#![allow(clippy::unused_enumerate_index)] // Index used in debug/logging contexts +#![allow(clippy::doc_lazy_continuation)] // Doc formatting acceptable +#![allow(clippy::doc_overindented_list_items)] // Doc formatting acceptable +#![allow(clippy::single_char_add_str)] // String building patterns +#![allow(clippy::let_and_return)] // Named return values for clarity +#![allow(clippy::useless_format)] // Explicit format for consistency +#![allow(clippy::manual_div_ceil)] // Explicit ceiling division for clarity +#![allow(clippy::io_other_error)] // IO error construction patterns +#![allow(clippy::manual_range_contains)] // Explicit range checks for clarity +#![allow(clippy::unwrap_or_default)] // Explicit unwrap_or preferred in some contexts +#![allow(clippy::used_underscore_binding)] // Underscore-prefixed bindings used intentionally +#![allow(clippy::trivially_copy_pass_by_ref)] // Pass-by-ref for trait consistency +#![allow(clippy::needless_borrows_for_generic_args)] // Explicit borrows for clarity +#![allow(clippy::needless_borrow)] // Explicit borrows for clarity +#![allow(clippy::missing_safety_doc)] // Safety documented at usage site +#![allow(clippy::module_inception)] // Module name matches parent for re-export patterns +#![allow(clippy::if_not_else)] // Negated conditions preferred in some ML logic +#![allow(clippy::empty_line_after_doc_comments)] // Doc comment formatting acceptable +#![allow(clippy::unnecessary_lazy_evaluations)] // Explicit lazy evaluation for side effects +#![allow(clippy::collapsible_if)] // Separate ifs preferred for readability +#![allow(clippy::question_mark)] // Explicit error handling preferred +#![allow(clippy::op_ref)] // Explicit ref operations for numeric types +#![allow(clippy::iter_kv_map)] // Map iteration patterns +#![allow(clippy::ptr_arg)] // Ptr args for trait consistency +#![allow(clippy::needless_question_mark)] // Explicit ? for error propagation clarity +#![allow(clippy::explicit_auto_deref)] // Explicit derefs for clarity +#![allow(clippy::bind_instead_of_map)] // Bind preferred in some combinator chains +#![allow(clippy::upper_case_acronyms)] // ML acronyms (DQN, PPO, TFT, etc.) +#![allow(clippy::large_types_passed_by_value)] // Large types passed by value for ownership +#![allow(clippy::large_enum_variant)] // Large variants unavoidable in ML model enums +#![allow(clippy::collapsible_else_if)] // Separate else-if preferred for readability +#![allow(clippy::vec_init_then_push)] // Vec init then push for conditional building +#![allow(clippy::implicit_saturating_sub)] // Explicit subtraction preferred +#![allow(clippy::missing_fields_in_debug)] // Custom Debug impls for large ML structs +#![allow(clippy::field_reassign_with_default)] // Field reassignment after Default::default() +#![allow(clippy::len_without_is_empty)] // len() without is_empty() for ML containers +#![allow(clippy::iter_nth)] // .nth() for indexed iteration +#![allow(clippy::items_after_statements)] // Items after statements for local helpers +#![allow(clippy::uninlined_format_args)] // Non-inlined format args for readability +#![allow(clippy::manual_memcpy)] // Explicit loop copy for tensor operations +#![allow(clippy::single_char_lifetime_names)] // Standard Rust lifetime conventions in ODE solvers +#![allow(clippy::same_item_push)] // Intentional repeated pushes for sequence padding +#![allow(clippy::multiple_unsafe_ops_per_block)] // SIMD/hardware ops grouped for clarity +#![allow(clippy::arc_with_non_send_sync)] // Arc used for local-thread inference contexts +#![allow(clippy::format_in_format_args)] // Nested format for dynamic message building +#![allow(clippy::inherent_to_string)] // Custom to_string for display types +#![allow(clippy::redundant_locals)] // Rebinding for clarity in model pipelines +#![allow(clippy::to_string_trait_impl)] // Direct ToString impls for versioning types +//! Shared ML types, traits, and infrastructure for Foxhunt +//! +//! This crate provides the core type definitions, error types, and traits +//! used across all ML crates in the Foxhunt workspace. It is the foundation +//! layer that other ml-* crates depend on. + +#![warn(missing_debug_implementations)] +#![warn(rust_2018_idioms)] +#![deny(clippy::panic, clippy::unimplemented, clippy::unreachable)] + +use async_trait::async_trait; +use chrono::{DateTime, Utc}; +use futures::future::join_all; +use rust_decimal::Decimal; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::sync::Arc; +use thiserror::Error; +use tokio::sync::RwLock; + +// ========== COMMON TYPE ERROR ========== + +/// Common type errors that can occur during ML operations +/// +/// This enum represents various type-related errors that can occur when working +/// with different data types across the ML pipeline, including type conversions, +/// validation errors, and compatibility issues. +#[derive(Debug, Clone, Error, Serialize, Deserialize)] +pub enum CommonTypeError { + /// Generic type error with descriptive message + #[error("Type error: {0}")] + Error(String), +} + +// ========== MARKET REGIME ========== + +/// Market regime classification for algorithmic trading strategies +/// +/// This enum represents different market conditions that can be detected through +/// statistical analysis and machine learning models. Market regime detection is +/// crucial for adaptive trading strategies that adjust their behavior based on +/// current market conditions. +/// +/// # Examples +/// +/// ```rust +/// use ml_core::MarketRegime; +/// +/// let regime = MarketRegime::Trending; +/// match regime { +/// MarketRegime::Bull => println!("Use momentum strategies"), +/// MarketRegime::Bear => println!("Use defensive strategies"), +/// MarketRegime::Crisis => println!("Implement risk controls"), +/// _ => println!("Use balanced approach"), +/// } +/// ``` +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum MarketRegime { + /// Normal market conditions with typical volatility and volume + Normal, + /// Strong directional movement with clear trends + Trending, + /// Range-bound market with limited directional movement + Sideways, + /// Bullish market with rising prices and positive sentiment + Bull, + /// Bearish market with falling prices and negative sentiment + Bear, + /// Crisis conditions with extreme volatility and risk + Crisis, +} + +// ========== COMMON ERROR ========== + +/// Common errors that can occur across the ML system +/// +/// This enum provides a unified error type that can be used throughout the ML +/// pipeline to ensure consistent error handling and reporting. It serves as a +/// bridge between different subsystems and provides appropriate error categorization. +#[derive(Debug, Clone, Error, Serialize, Deserialize)] +pub enum CommonError { + /// General error with descriptive message + #[error("Error: {0}")] + General(String), +} + +impl CommonError { + /// Create a validation error with a descriptive message + pub fn validation>(msg: S) -> Self { + Self::General(msg.into()) + } + + /// Create a configuration error with a descriptive message + pub fn config>(msg: S) -> Self { + Self::General(msg.into()) + } + + /// Create a service error with category and descriptive message + pub fn service>(category: ErrorCategory, msg: S) -> Self { + Self::General(format!("{:?}: {}", category, msg.into())) + } +} + +// Re-export canonical ErrorCategory from common crate (24 variants) +pub use ::common::error::ErrorCategory; + +// ========== CORE ML TYPES ========== + +/// Represents a financial trade for ML model training and analysis +/// +/// This structure contains the essential information about a trade that is used +/// by ML models for pattern recognition, market analysis, and strategy optimization. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Trade { + /// Trading symbol (e.g., "AAPL", "MSFT") + pub symbol: String, + /// Trade price in decimal format for precision + pub price: Decimal, + /// Trade quantity in decimal format for precision + pub quantity: Decimal, + /// Timestamp in microseconds since Unix epoch + pub timestamp: u64, + /// Trade side: "buy" or "sell" + pub side: String, +} + +/// Health status for ensemble models and ML system components +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub enum HealthStatus { + /// Component is operating within normal parameters + Healthy, + /// Component is operational but performance is below optimal + Degraded, + /// Component is not functioning properly and requires intervention + Unhealthy, +} + +/// Market data snapshot for ML model input +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MarketDataSnapshot { + /// Timestamp of the market data snapshot + pub timestamp: DateTime, + /// Trading symbol (e.g., "AAPL", "MSFT") + pub symbol: String, + /// Current market price + pub price: Decimal, + /// Trading volume at this timestamp + pub volume: Decimal, +} + +/// Feature vector for ML model input +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct FeatureVector(pub Vec); + +impl FeatureVector { + /// Get the length of the feature vector + pub fn len(&self) -> usize { + self.0.len() + } + + /// Check if the feature vector is empty + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } +} + +/// Integer tensor for discrete ML model operations +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct IntegerTensor(pub Vec); + +/// Summary of model update operations +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct UpdateSummary { + /// Number of models successfully updated + pub updated_models: usize, + /// Total number of models in the update operation + pub total_models: usize, +} + +// ========== ML ERROR (DEDUPED) ========== + +/// Machine Learning specific errors +/// +/// Note: `ConfigError` and `ConfigurationError` have been consolidated into a +/// single `ConfigError(String)` tuple variant. +#[derive(Debug, Clone, Error, Serialize, Deserialize)] +pub enum MLError { + /// Configuration error (consolidated from ConfigError{reason} + ConfigurationError) + #[error("Configuration error: {0}")] + ConfigError(String), + + /// Dimension mismatch error + #[error("Dimension mismatch: expected {expected}, got {actual}")] + DimensionMismatch { expected: usize, actual: usize }, + + /// Graph-related error + #[error("Graph error: {message}")] + GraphError { message: String }, + + /// Resource limit exceeded + #[error("Resource limit exceeded: {resource} limit {limit}")] + ResourceLimit { resource: String, limit: usize }, + + /// Serialization error + #[error("Serialization error: {reason}")] + SerializationError { reason: String }, + + /// Validation error + #[error("Validation error: {message}")] + ValidationError { message: String }, + + /// Concurrency error + #[error("Concurrency error in operation: {operation}")] + ConcurrencyError { operation: String }, + + /// Invalid input error + #[error("Invalid input: {0}")] + InvalidInput(String), + + /// Initialization error + #[error("Initialization error in {component}: {message}")] + InitializationError { component: String, message: String }, + + /// Training error + #[error("Training error: {0}")] + TrainingError(String), + + /// Inference error + #[error("Inference error: {0}")] + InferenceError(String), + + /// Model error + #[error("Model error: {0}")] + ModelError(String), + + /// Model not trained error + #[error("Model not trained: {0}")] + NotTrained(String), + + /// Anyhow error wrapping + #[error("General error: {0}")] + AnyhowError(String), + + /// Tensor creation error + #[error("Tensor creation error in {operation}: {reason}")] + TensorCreationError { operation: String, reason: String }, + + /// Tensor operation error + #[error("Tensor operation error: {0}")] + TensorOperationError(String), + + /// Lock error + #[error("Lock error: {0}")] + LockError(String), + + /// Model not found error + #[error("Model not found: {0}")] + ModelNotFound(String), + + /// Insufficient data error + #[error("Insufficient data: {0}")] + InsufficientData(String), + + /// Checkpoint error + #[error("Checkpoint error: {0}")] + CheckpointError(String), + + /// Device error (GPU/CUDA unavailable) + #[error("Device error: {0}")] + DeviceError(String), +} + +// ========== FROM IMPLS ========== + +// Convert candle_core::Error to MLError +impl From for MLError { + fn from(err: candle_core::Error) -> Self { + MLError::ModelError(format!("Candle error: {}", err)) + } +} + +// Convert anyhow::Error to MLError +impl From for MLError { + fn from(err: anyhow::Error) -> Self { + MLError::AnyhowError(err.to_string()) + } +} + +// Convert std::io::Error to MLError +impl From for MLError { + fn from(err: std::io::Error) -> Self { + MLError::ModelError(format!("IO error: {}", err)) + } +} + +// Convert common type errors to MLError +impl From for MLError { + fn from(err: CommonTypeError) -> Self { + MLError::ModelError(format!("Common type error: {}", err)) + } +} + +// Convert serde_json::Error to MLError +impl From for MLError { + fn from(err: serde_json::Error) -> Self { + MLError::SerializationError { + reason: err.to_string(), + } + } +} + +// UNIFIED ERROR HANDLING: Convert all ML errors to CommonError for workspace consistency +impl From for CommonError { + fn from(err: MLError) -> Self { + match err { + MLError::ConfigError(reason) => { + CommonError::config(format!("ML configuration error: {}", reason)) + }, + MLError::InitializationError { component, message } => CommonError::service( + ErrorCategory::System, + format!("ML initialization error in {}: {}", component, message), + ), + MLError::DimensionMismatch { expected, actual } => CommonError::validation(format!( + "ML dimension mismatch: expected {}, got {}", + expected, actual + )), + MLError::GraphError { message } => CommonError::service( + ErrorCategory::System, + format!("ML graph error: {}", message), + ), + MLError::ResourceLimit { resource, limit } => CommonError::service( + ErrorCategory::System, + format!("ML resource limit exceeded: {} limit {}", resource, limit), + ), + MLError::SerializationError { reason } => CommonError::service( + ErrorCategory::System, + format!("ML serialization error: {}", reason), + ), + MLError::ValidationError { message } => { + CommonError::validation(format!("ML validation error: {}", message)) + }, + MLError::ConcurrencyError { operation } => CommonError::service( + ErrorCategory::System, + format!("ML concurrency error in operation: {}", operation), + ), + MLError::InvalidInput(msg) => { + CommonError::validation(format!("ML invalid input: {}", msg)) + }, + MLError::TrainingError(msg) => { + CommonError::service(ErrorCategory::System, format!("ML training error: {}", msg)) + }, + MLError::InferenceError(msg) => CommonError::service( + ErrorCategory::System, + format!("ML inference error: {}", msg), + ), + MLError::ModelError(msg) => { + CommonError::service(ErrorCategory::System, format!("ML model error: {}", msg)) + }, + MLError::CheckpointError(msg) => CommonError::service( + ErrorCategory::System, + format!("ML checkpoint error: {}", msg), + ), + MLError::NotTrained(msg) => CommonError::service( + ErrorCategory::System, + format!("ML model not trained: {}", msg), + ), + MLError::AnyhowError(msg) => { + CommonError::service(ErrorCategory::System, format!("ML error: {}", msg)) + }, + MLError::TensorCreationError { operation, reason } => CommonError::service( + ErrorCategory::System, + format!("ML tensor creation error in {}: {}", operation, reason), + ), + MLError::TensorOperationError(msg) => CommonError::service( + ErrorCategory::System, + format!("ML tensor operation error: {}", msg), + ), + MLError::LockError(msg) => { + CommonError::service(ErrorCategory::System, format!("ML lock error: {}", msg)) + }, + MLError::ModelNotFound(msg) => CommonError::service( + ErrorCategory::System, + format!("ML model not found: {}", msg), + ), + MLError::InsufficientData(msg) => { + CommonError::validation(format!("ML insufficient data: {}", msg)) + }, + MLError::DeviceError(msg) => CommonError::service( + ErrorCategory::System, + format!("ML device error: {}", msg), + ), + } + } +} + +// ========== RESULT TYPES ========== + +/// Result type for ML operations +pub type MLResult = Result; + +/// New unified result type using CommonError for better integration +pub type UnifiedMLResult = Result; + +// ========== CONSTANTS ========== + +/// Precision factor for fixed-point arithmetic +pub const PRECISION_FACTOR: i64 = 100_000_000; + +/// Maximum inference latency target in microseconds +pub const MAX_INFERENCE_LATENCY_US: u64 = 100; + +// ========== DEVICE MANAGEMENT ========== + +/// Get mandatory CUDA device for training +/// +/// # Errors +/// +/// Returns `MLError::DeviceError` if CUDA GPU is not available +pub fn get_training_device() -> Result { + match candle_core::Device::new_cuda(0) { + Ok(device) => Ok(device), + Err(e) => { + tracing::error!( + "CUDA GPU not available: {}. \ + Troubleshooting: (1) nvidia-smi, (2) nvcc --version, \ + (3) check LD_LIBRARY_PATH, (4) cargo build --features cuda, \ + (5) check CUDA_HOME", + e + ); + Err(MLError::DeviceError(format!( + "CUDA GPU required but unavailable: {}", + e + ))) + } + } +} + +/// Get CUDA device with index (for multi-GPU setups) +/// +/// # Errors +/// +/// Returns `MLError::DeviceError` if CUDA GPU at specified index is not available +pub fn get_training_device_at(device_id: usize) -> Result { + match candle_core::Device::new_cuda(device_id) { + Ok(device) => Ok(device), + Err(e) => { + tracing::error!( + "CUDA GPU {} not available: {}. Check available GPUs with: nvidia-smi", + device_id, + e + ); + Err(MLError::DeviceError(format!( + "CUDA GPU {} required but unavailable: {}", + device_id, e + ))) + } + } +} + +// ========== ML APP RESULT ========== + +/// Application result wrapper for ML operations +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MLAppResult { + pub data: T, + pub success: bool, + pub message: Option, + pub execution_time_ms: u64, + pub metadata: HashMap, +} + +impl MLAppResult { + /// Create a successful result + pub fn success(data: T) -> Self { + Self { + data, + success: true, + message: None, + execution_time_ms: 0, + metadata: HashMap::new(), + } + } + + /// Create a failed result with message + pub fn error(data: T, message: String) -> Self { + Self { + data, + success: false, + message: Some(message), + execution_time_ms: 0, + metadata: HashMap::new(), + } + } + + /// Set execution time + pub fn with_timing(mut self, execution_time_ms: u64) -> Self { + self.execution_time_ms = execution_time_ms; + self + } + + /// Add metadata + pub fn with_metadata(mut self, key: String, value: String) -> Self { + self.metadata.insert(key, value); + self + } +} + +// ========== HFT PERFORMANCE PROFILE ========== + +/// Performance profile configuration for HFT models +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct HFTPerformanceProfile { + pub max_latency_us: u64, + pub target_throughput: u32, + pub memory_limit_mb: u64, + pub cpu_affinity: Option>, + pub gpu_enabled: bool, + pub batch_size: u32, + pub optimization_level: OptimizationLevel, +} + +/// Optimization levels for HFT performance +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +pub enum OptimizationLevel { + /// Maximum speed, minimal safety checks + UltraLow, + /// Balanced speed and safety + Low, + /// Standard optimization + Medium, + /// Conservative with full validation + High, +} + +impl Default for HFTPerformanceProfile { + fn default() -> Self { + Self { + max_latency_us: 100, // 100 microseconds target + target_throughput: 10000, // 10k operations per second + memory_limit_mb: 1024, // 1GB memory limit + cpu_affinity: None, + gpu_enabled: false, + batch_size: 1, + optimization_level: OptimizationLevel::Medium, + } + } +} + +impl HFTPerformanceProfile { + /// Create HFT performance profile with custom latency target + pub fn with_latency(max_latency_us: u64) -> Self { + Self { + max_latency_us, + ..Default::default() + } + } + + /// Create HFT performance profile optimized for ultra-low latency + pub fn ultra_low_latency() -> Self { + Self { + max_latency_us: 10, // 10 microseconds target + target_throughput: 50000, // 50k operations per second + memory_limit_mb: 512, // Reduced memory for cache efficiency + gpu_enabled: true, // Enable GPU acceleration + batch_size: 1, // No batching for minimal latency + optimization_level: OptimizationLevel::UltraLow, + ..Default::default() + } + } +} + +// ========== FEATURES, PREDICTION, FEEDBACK ========== + +/// Features vector for ML model input +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Features { + /// Raw feature values + pub values: Vec, + /// Feature names for debugging + pub names: Vec, + /// Timestamp of features + pub timestamp: u64, + /// Symbol these features are for + pub symbol: Option, +} + +impl Features { + pub fn new(values: Vec, names: Vec) -> Self { + Self { + values, + names, + timestamp: std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_micros() as u64, + symbol: None, + } + } + + /// Set the symbol for this market data point + pub fn with_symbol(mut self, symbol: String) -> Self { + self.symbol = Some(symbol); + self + } +} + +/// Model prediction result +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ModelPrediction { + /// Predicted value (price direction, probability, etc.) + pub value: f64, + /// Model confidence (0.0 to 1.0) + pub confidence: f64, + /// Additional model-specific metadata + pub metadata: HashMap, + /// Prediction timestamp + pub timestamp: u64, + /// Model identifier + pub model_id: String, +} + +impl ModelPrediction { + pub fn new(model_id: String, value: f64, confidence: f64) -> Self { + Self { + value, + confidence, + metadata: HashMap::new(), + timestamp: std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_micros() as u64, + model_id, + } + } + + /// Add metadata to the model prediction + pub fn with_metadata(mut self, key: String, value: serde_json::Value) -> Self { + self.metadata.insert(key, value); + self + } +} + +/// Feedback for model weight updates +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Feedback { + /// Actual outcome (for supervised learning) + pub actual_value: Option, + /// Reward signal (for reinforcement learning) + pub reward: Option, + /// Trading performance metrics + pub performance_metrics: HashMap, + /// Timestamp of feedback + pub timestamp: u64, +} + +impl Feedback { + pub fn new() -> Self { + Self { + actual_value: None, + reward: None, + performance_metrics: HashMap::new(), + timestamp: std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_micros() as u64, + } + } + + /// Set the actual outcome value for supervised learning feedback + pub fn with_actual(mut self, actual: f64) -> Self { + self.actual_value = Some(actual); + self + } + + /// Set the reward signal for reinforcement learning feedback + pub fn with_reward(mut self, reward: f64) -> Self { + self.reward = Some(reward); + self + } +} + +impl Default for Feedback { + fn default() -> Self { + Self::new() + } +} + +// ========== ML MODEL TRAIT ========== + +/// Unified interface for all ML models in the system +#[async_trait] +pub trait MLModel: Send + Sync + std::fmt::Debug { + /// Get unique model identifier + fn name(&self) -> &str; + + /// Get model type + fn model_type(&self) -> ModelType; + + /// Make prediction based on features + async fn predict(&self, features: &Features) -> MLResult; + + /// Get current model confidence score (0.0 to 1.0) + fn get_confidence(&self) -> f64; + + /// Update model weights based on feedback (optional - not all models support online learning) + async fn update_weights(&mut self, _feedback: &Feedback) -> MLResult<()> { + // Default implementation does nothing (for immutable models) + Ok(()) + } + + /// Check if model is ready for predictions + fn is_ready(&self) -> bool { + true // Default to ready + } + + /// Get model metadata + fn get_metadata(&self) -> ModelMetadata; + + /// Validate input features + fn validate_features(&self, features: &Features) -> MLResult<()> { + // Default validation - check for empty features + if features.values.is_empty() { + return Err(MLError::ValidationError { + message: "Empty feature vector".to_owned(), + }); + } + Ok(()) + } +} + +// ========== MODEL REGISTRY ========== + +/// Thread-safe model registry using DashMap for high-performance concurrent access +pub struct ModelRegistry { + /// Models stored by name + models: dashmap::DashMap>, + /// Registry metadata + metadata: Arc>, +} + +impl std::fmt::Debug for ModelRegistry { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ModelRegistry") + .field( + "models", + &format_args!("", self.models.len()), + ) + .field("metadata", &self.metadata) + .finish() + } +} + +#[derive(Debug, Clone)] +struct RegistryMetadata { + created_at: std::time::SystemTime, + total_registrations: u64, + last_access: std::time::SystemTime, +} + +impl ModelRegistry { + /// Create new model registry + pub fn new() -> Self { + Self { + models: dashmap::DashMap::new(), + metadata: Arc::new(RwLock::new(RegistryMetadata { + created_at: std::time::SystemTime::now(), + total_registrations: 0, + last_access: std::time::SystemTime::now(), + })), + } + } + + /// Register a model in the registry + pub async fn register(&self, model: Arc) -> MLResult<()> { + let name = model.name().to_string(); + + // Check if model is ready + if !model.is_ready() { + return Err(MLError::ModelError(format!("Model {} is not ready", name))); + } + + self.models.insert(name.clone(), model); + + // Update metadata + { + let mut meta = self.metadata.write().await; + meta.total_registrations += 1; + meta.last_access = std::time::SystemTime::now(); + } + + tracing::info!("Registered ML model: {}", name); + Ok(()) + } + + /// Get model by name + pub async fn get(&self, name: &str) -> Option> { + // Update last access time + { + let mut meta = self.metadata.write().await; + meta.last_access = std::time::SystemTime::now(); + } + + self.models.get(name).map(|entry| entry.value().clone()) + } + + /// Get all registered models + pub fn get_all(&self) -> Vec> { + self.models + .iter() + .map(|entry| entry.value().clone()) + .collect() + } + + /// Get model names + pub fn get_model_names(&self) -> Vec { + self.models + .iter() + .map(|entry| entry.key().clone()) + .collect() + } + + /// Remove model from registry + pub async fn remove(&self, name: &str) -> Option> { + let result = self.models.remove(name).map(|(_, model)| model); + + if result.is_some() { + tracing::info!("Removed ML model: {}", name); + } + + result + } + + /// Get registry statistics + pub async fn get_stats(&self) -> RegistryStats { + let meta = self.metadata.read().await; + RegistryStats { + total_models: self.models.len(), + total_registrations: meta.total_registrations, + created_at: meta.created_at, + last_access: meta.last_access, + } + } + + /// Parallel prediction across all models + pub async fn predict_all(&self, features: &Features) -> Vec> { + let models = self.get_all(); + let futures = models.iter().map(|model| { + let features = features.clone(); + async move { model.predict(&features).await } + }); + + join_all(futures).await + } + + /// Parallel prediction across specific models + pub async fn predict_selected( + &self, + model_names: &[String], + features: &Features, + ) -> Vec> { + let futures = model_names.iter().map(|name| { + let name = name.clone(); + let features = features.clone(); + async move { + if let Some(model) = self.get(&name).await { + model.predict(&features).await + } else { + Err(MLError::ModelNotFound(name)) + } + } + }); + + join_all(futures).await + } +} + +impl Default for ModelRegistry { + fn default() -> Self { + Self::new() + } +} + +/// Registry statistics +#[derive(Debug, Clone)] +pub struct RegistryStats { + pub total_models: usize, + pub total_registrations: u64, + pub created_at: std::time::SystemTime, + pub last_access: std::time::SystemTime, +} + +// ========== GLOBAL REGISTRY ========== + +/// Global model registry instance (singleton pattern) +static GLOBAL_REGISTRY: once_cell::sync::Lazy> = + once_cell::sync::Lazy::new(|| Arc::new(ModelRegistry::new())); + +/// Get global model registry +pub fn get_global_registry() -> Arc { + GLOBAL_REGISTRY.clone() +} + +// ========== PARALLEL EXECUTOR ========== + +/// High-performance parallel executor for ML models optimized for sub-50us latency +#[derive(Debug)] +pub struct ParallelExecutor { + /// Performance profile + profile: HFTPerformanceProfile, + /// CPU affinity settings + cpu_affinity: Option>, + /// Thread pool for CPU-bound operations + cpu_pool: Arc, + /// Async runtime handle + runtime_handle: tokio::runtime::Handle, +} + +impl ParallelExecutor { + /// Create new parallel executor with HFT performance profile + pub fn new(profile: HFTPerformanceProfile) -> Result { + // Create dedicated thread pool based on profile + let cpu_pool = rayon::ThreadPoolBuilder::new() + .num_threads( + profile + .cpu_affinity + .as_ref() + .map(|v| v.len()) + .unwrap_or(num_cpus::get()), + ) + .thread_name(|i| format!("ml-cpu-{}", i)) + .build() + .map_err(|e| MLError::ModelError(format!("Failed to create thread pool: {}", e)))?; + + let runtime_handle = tokio::runtime::Handle::try_current() + .map_err(|e| MLError::ModelError(format!("No tokio runtime available: {}", e)))?; + + let cpu_affinity = profile.cpu_affinity.clone(); + + Ok(Self { + profile, + cpu_affinity, + cpu_pool: Arc::new(cpu_pool), + runtime_handle, + }) + } + + /// Create an HFT-optimized parallel executor + pub fn hft() -> Result { + let profile = HFTPerformanceProfile::ultra_low_latency(); + Self::new(profile) + } + + /// Execute parallel predictions with latency optimization + pub async fn execute_parallel_predictions( + &self, + models: Vec>, + features: Features, + ) -> Vec> { + let start_time = std::time::Instant::now(); + + // Determine execution strategy based on performance profile + let results = match self.profile.optimization_level { + OptimizationLevel::UltraLow => { + // Ultra-low latency: parallel execution with minimal overhead + self.execute_ultra_low_latency(models, features).await + }, + OptimizationLevel::Low => { + // Low latency: parallel with basic batching + self.execute_low_latency(models, features).await + }, + OptimizationLevel::Medium => { + // Medium: balanced parallel execution + self.execute_balanced(models, features).await + }, + OptimizationLevel::High => { + // High: conservative with full validation + self.execute_conservative(models, features).await + }, + }; + + let execution_time = start_time.elapsed(); + + // Log performance if exceeding target latency + if execution_time.as_micros() > self.profile.max_latency_us as u128 { + tracing::warn!( + "Parallel execution exceeded target latency: {}us > {}us", + execution_time.as_micros(), + self.profile.max_latency_us + ); + } + + results + } + + /// Ultra-low latency execution (<10us target) + async fn execute_ultra_low_latency( + &self, + models: Vec>, + features: Features, + ) -> Vec> { + // Use futures::future::join_all for minimal overhead + let futures = models.into_iter().map(|model| { + let features = features.clone(); + async move { model.predict(&features).await } + }); + + join_all(futures).await + } + + /// Low latency execution with basic optimizations + async fn execute_low_latency( + &self, + models: Vec>, + features: Features, + ) -> Vec> { + // Group models by type for potential batching + let mut model_groups: HashMap>> = HashMap::new(); + + for model in models { + let model_type = model.model_type(); + model_groups.entry(model_type).or_default().push(model); + } + + let mut all_futures = Vec::new(); + + for (_, group_models) in model_groups { + for model in group_models { + let features = features.clone(); + all_futures.push(async move { model.predict(&features).await }); + } + } + + join_all(all_futures).await + } + + /// Balanced execution with moderate optimizations + async fn execute_balanced( + &self, + models: Vec>, + features: Features, + ) -> Vec> { + // Validate features once for all models + for model in &models { + if let Err(e) = model.validate_features(&features) { + tracing::debug!( + "Feature validation failed for model {}: {}", + model.name(), + e + ); + } + } + + let futures = models.into_iter().map(|model| { + let features = features.clone(); + async move { + if model.is_ready() { + model.predict(&features).await + } else { + Err(MLError::ModelError(format!( + "Model {} not ready", + model.name() + ))) + } + } + }); + + join_all(futures).await + } + + /// Conservative execution with full validation + async fn execute_conservative( + &self, + models: Vec>, + features: Features, + ) -> Vec> { + let mut results = Vec::new(); + + for model in models { + // Comprehensive validation + if !model.is_ready() { + results.push(Err(MLError::ModelError(format!( + "Model {} not ready", + model.name() + )))); + continue; + } + + if let Err(e) = model.validate_features(&features) { + results.push(Err(e)); + continue; + } + + // Execute with timeout + let prediction_future = model.predict(&features); + let timeout_duration = std::time::Duration::from_micros(self.profile.max_latency_us); + + match tokio::time::timeout(timeout_duration, prediction_future).await { + Ok(result) => results.push(result), + Err(_) => results.push(Err(MLError::ModelError(format!( + "Model {} prediction timed out after {}us", + model.name(), + self.profile.max_latency_us + )))), + } + } + + results + } + + /// Get execution statistics + pub fn get_stats(&self) -> ExecutorStats { + ExecutorStats { + optimization_level: self.profile.optimization_level, + target_latency_us: self.profile.max_latency_us, + cpu_threads: self.cpu_pool.current_num_threads(), + cpu_affinity: self.cpu_affinity.clone(), + } + } +} + +/// Executor performance statistics +#[derive(Debug, Clone)] +pub struct ExecutorStats { + pub optimization_level: OptimizationLevel, + pub target_latency_us: u64, + pub cpu_threads: usize, + pub cpu_affinity: Option>, +} + +// ========== LATENCY OPTIMIZER ========== + +/// Latency optimizer for ML inference pipelines +#[derive(Debug)] +pub struct LatencyOptimizer { + /// Target latency in microseconds + target_latency_us: u64, + /// Performance history + performance_history: Arc>>, + /// Optimization parameters + optimization_params: OptimizationParams, +} + +#[derive(Debug, Clone)] +struct PerformancePoint { + timestamp: std::time::Instant, + latency_us: u64, + model_count: usize, + batch_size: u32, + success: bool, +} + +#[derive(Debug, Clone)] +struct OptimizationParams { + max_batch_size: u32, + adaptive_batching: bool, + prefetch_enabled: bool, + cache_predictions: bool, +} + +impl Default for OptimizationParams { + fn default() -> Self { + Self { + max_batch_size: 8, + adaptive_batching: true, + prefetch_enabled: true, + cache_predictions: false, // Disabled for real-time trading + } + } +} + +impl LatencyOptimizer { + /// Create new latency optimizer + pub fn new(target_latency_us: u64) -> Self { + Self { + target_latency_us, + performance_history: Arc::new(RwLock::new(Vec::new())), + optimization_params: OptimizationParams::default(), + } + } + + /// Create latency optimizer with HFT targets + pub fn hft() -> Self { + Self::new(50) // 50 microsecond target + } + + /// Record performance measurement + pub async fn record_performance( + &self, + latency_us: u64, + model_count: usize, + batch_size: u32, + success: bool, + ) { + let point = PerformancePoint { + timestamp: std::time::Instant::now(), + latency_us, + model_count, + batch_size, + success, + }; + + { + let mut history = self.performance_history.write().await; + history.push(point); + + // Keep only recent history (last 1000 measurements) + if history.len() > 1000 { + let excess = history.len() - 1000; + history.drain(0..excess); + } + } + } + + /// Get optimization recommendations + pub async fn get_recommendations(&self) -> OptimizationRecommendations { + let history = self.performance_history.read().await; + + if history.is_empty() { + return OptimizationRecommendations::default(); + } + + let recent_points: Vec<&PerformancePoint> = history.iter().rev().take(100).collect(); + + let avg_latency = + recent_points.iter().map(|p| p.latency_us).sum::() / recent_points.len() as u64; + + let success_rate = + recent_points.iter().filter(|p| p.success).count() as f64 / recent_points.len() as f64; + + OptimizationRecommendations { + current_avg_latency_us: avg_latency, + target_latency_us: self.target_latency_us, + success_rate, + meets_target: avg_latency <= self.target_latency_us, + recommended_batch_size: self.calculate_optimal_batch_size(&recent_points), + recommended_model_limit: self.calculate_optimal_model_limit(&recent_points), + } + } + + fn calculate_optimal_batch_size(&self, points: &[&PerformancePoint]) -> u32 { + // Simple heuristic: find batch size with best latency/success ratio + let mut batch_performance: HashMap = HashMap::new(); + + for point in points { + let entry = batch_performance + .entry(point.batch_size) + .or_insert((0, 0.0)); + entry.0 += point.latency_us; + entry.1 += if point.success { 1.0 } else { 0.0 }; + } + + batch_performance + .into_iter() + .filter(|(_, (_, success_count))| *success_count > 0.0) + .min_by_key(|(_, (latency, success_count))| { + // Optimize for latency with success rate weighting + ((*latency as f64) / success_count) as u64 + }) + .map(|(batch_size, _)| batch_size) + .unwrap_or(1) + } + + fn calculate_optimal_model_limit(&self, points: &[&PerformancePoint]) -> usize { + // Find the sweet spot where adding more models doesn't improve latency + let mut model_performance: HashMap = HashMap::new(); + + for point in points { + if point.success { + let entry = model_performance.entry(point.model_count).or_insert(0); + *entry += point.latency_us; + } + } + + model_performance + .into_iter() + .filter(|(_, avg_latency)| *avg_latency <= self.target_latency_us) + .max_by_key(|(model_count, _)| *model_count) + .map(|(model_count, _)| model_count) + .unwrap_or(1) + } +} + +/// Optimization recommendations from latency analysis +#[derive(Debug, Clone)] +pub struct OptimizationRecommendations { + pub current_avg_latency_us: u64, + pub target_latency_us: u64, + pub success_rate: f64, + pub meets_target: bool, + pub recommended_batch_size: u32, + pub recommended_model_limit: usize, +} + +impl Default for OptimizationRecommendations { + fn default() -> Self { + Self { + current_avg_latency_us: 0, + target_latency_us: 50, + success_rate: 0.0, + meets_target: false, + recommended_batch_size: 1, + recommended_model_limit: 1, + } + } +} + +// ========== TRAINING AND VALIDATION METRICS ========== + +/// Canonical training metrics used throughout ML module +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TrainingMetrics { + /// Training loss value + pub loss: f64, + /// Training accuracy (0.0 to 1.0) + pub accuracy: f64, + /// Training precision (0.0 to 1.0) + pub precision: f64, + /// Training recall (0.0 to 1.0) + pub recall: f64, + /// Training F1 score (0.0 to 1.0) + pub f1_score: f64, + /// Total training time in seconds + pub training_time_seconds: f64, + /// Number of epochs trained + pub epochs_trained: u32, + /// Whether convergence was achieved + pub convergence_achieved: bool, + /// Additional model-specific metrics + pub additional_metrics: HashMap, +} + +impl TrainingMetrics { + /// Create new training metrics + pub fn new() -> Self { + Self { + loss: 0.0, + accuracy: 0.0, + precision: 0.0, + recall: 0.0, + f1_score: 0.0, + training_time_seconds: 0.0, + epochs_trained: 0, + convergence_achieved: false, + additional_metrics: HashMap::new(), + } + } + + /// Add an additional metric + pub fn add_metric(&mut self, name: &str, value: f64) { + self.additional_metrics.insert(name.to_string(), value); + } + + /// Check if training was successful + pub fn is_successful(&self) -> bool { + self.convergence_achieved && self.accuracy > 0.5 + } +} + +impl Default for TrainingMetrics { + fn default() -> Self { + Self::new() + } +} + +/// Canonical validation metrics used throughout ML module +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ValidationMetrics { + /// Validation loss value + pub validation_loss: f64, + /// Validation accuracy (0.0 to 1.0) + pub validation_accuracy: f64, + /// Validation precision (0.0 to 1.0) + pub validation_precision: f64, + /// Validation recall (0.0 to 1.0) + pub validation_recall: f64, + /// Validation F1 score (0.0 to 1.0) + pub validation_f1_score: f64, + /// Number of samples validated + pub samples_validated: usize, + /// Additional model-specific validation metrics + pub additional_metrics: HashMap, +} + +impl ValidationMetrics { + /// Create new validation metrics + pub fn new() -> Self { + Self { + validation_loss: 0.0, + validation_accuracy: 0.0, + validation_precision: 0.0, + validation_recall: 0.0, + validation_f1_score: 0.0, + samples_validated: 0, + additional_metrics: HashMap::new(), + } + } + + /// Add an additional validation metric + pub fn add_metric(&mut self, name: &str, value: f64) { + self.additional_metrics.insert(name.to_string(), value); + } + + /// Check if validation was successful + pub fn is_successful(&self) -> bool { + self.validation_accuracy > 0.5 && self.samples_validated > 0 + } +} + +impl Default for ValidationMetrics { + fn default() -> Self { + Self::new() + } +} + +// ========== INFERENCE RESULT & MODEL METADATA ========== + +/// Canonical inference result used throughout ML module +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct InferenceResult { + /// Model identifier + pub model_id: String, + /// Prediction value (primary prediction) + pub prediction_value: f64, + /// Confidence score (0.0 to 1.0) + pub confidence: f64, + /// Latency in microseconds + pub latency_us: u64, + /// Timestamp in microseconds since UNIX epoch + pub timestamp: u64, + /// Model metadata + pub metadata: ModelMetadata, +} + +impl InferenceResult { + /// Create new inference result + pub fn new( + model_id: String, + prediction_value: f64, + confidence: f64, + latency_us: u64, + timestamp: u64, + metadata: ModelMetadata, + ) -> Self { + Self { + model_id, + prediction_value, + confidence, + latency_us, + timestamp, + metadata, + } + } + + /// Extract prediction as float value + pub fn prediction_as_float(&self) -> f64 { + self.prediction_value + } + + /// Get the model identifier + pub fn model_id(&self) -> &str { + &self.model_id + } +} + +/// Canonical model metadata used throughout ML module +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ModelMetadata { + /// Type of the model + pub model_type: ModelType, + /// Model version + pub version: String, + /// Number of features used for inference + pub features_used: usize, + /// Memory usage in megabytes + pub memory_usage_mb: f64, + /// Additional metadata key-value pairs + pub additional_metadata: HashMap, +} + +impl ModelMetadata { + /// Create new model metadata + pub fn new( + model_type: ModelType, + version: String, + features_used: usize, + memory_usage_mb: f64, + ) -> Self { + Self { + model_type, + version, + features_used, + memory_usage_mb, + additional_metadata: HashMap::new(), + } + } + + /// Add additional metadata + pub fn add_metadata(&mut self, key: &str, value: String) { + self.additional_metadata.insert(key.to_string(), value); + } + + /// Mark the model as trained + pub fn mark_trained(&mut self) { + self.add_metadata("training_status", "trained".to_owned()); + self.add_metadata( + "training_timestamp", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs() + .to_string(), + ); + } +} + +// ========== RE-EXPORTS ========== + +// Re-export canonical ModelType from common crate +pub use ::common::model_types::ModelType; + +// ========== DEPRECATED FREE FUNCTIONS ========== +// These are backward-compatible wrappers; prefer the associated methods. + +/// Create HFT performance profile with default settings +#[deprecated(note = "Use HFTPerformanceProfile::default() instead")] +pub fn create_hft_performance_profile() -> HFTPerformanceProfile { + HFTPerformanceProfile::default() +} + +/// Create HFT performance profile with custom latency target +#[deprecated(note = "Use HFTPerformanceProfile::with_latency(us) instead")] +pub fn create_hft_performance_profile_with_latency(max_latency_us: u64) -> HFTPerformanceProfile { + HFTPerformanceProfile::with_latency(max_latency_us) +} + +/// Create HFT performance profile optimized for ultra-low latency +#[deprecated(note = "Use HFTPerformanceProfile::ultra_low_latency() instead")] +pub fn create_ultra_low_latency_profile() -> HFTPerformanceProfile { + HFTPerformanceProfile::ultra_low_latency() +} + +/// Create optimized parallel executor for HFT scenarios +#[deprecated(note = "Use ParallelExecutor::hft() instead")] +pub fn create_hft_parallel_executor() -> Result { + ParallelExecutor::hft() +} + +/// Create latency optimizer with HFT targets +#[deprecated(note = "Use LatencyOptimizer::hft() instead")] +pub fn create_hft_latency_optimizer() -> LatencyOptimizer { + LatencyOptimizer::hft() +} + +// ========== PRELUDE ========== + +/// Prelude module for convenient imports of commonly used ML types +/// +/// This module re-exports the most commonly used types and traits from the ML crate +/// to allow users to import everything they need with a single `use ml_core::prelude::*;` +pub mod prelude { + // Core ML types + pub use crate::{ + CommonError, CommonTypeError, ErrorCategory, FeatureVector, Features, Feedback, + HealthStatus, InferenceResult, IntegerTensor, MarketDataSnapshot, MarketRegime, + ModelMetadata, ModelPrediction, ModelType, Trade, TrainingMetrics, UpdateSummary, + ValidationMetrics, + }; + + // Error types + pub use crate::{MLError, MLResult, UnifiedMLResult}; + + // ML Model trait + pub use crate::MLModel; + + // Model registry + pub use crate::{get_global_registry, ModelRegistry, RegistryStats}; + + // Performance types (both new associated methods and deprecated free functions) + #[allow(deprecated)] + pub use crate::{ + create_hft_latency_optimizer, create_hft_parallel_executor, create_hft_performance_profile, + create_hft_performance_profile_with_latency, create_ultra_low_latency_profile, + ExecutorStats, HFTPerformanceProfile, LatencyOptimizer, OptimizationLevel, + OptimizationRecommendations, ParallelExecutor, + }; + + // Constants + pub use crate::{MAX_INFERENCE_LATENCY_US, PRECISION_FACTOR}; + + // Tensor types from candle + pub use candle_core::{Device, Tensor}; + pub use candle_nn::{Module, VarBuilder, VarMap}; + + // Common external types + pub use rust_decimal::Decimal; + pub use serde::{Deserialize, Serialize}; +} diff --git a/crates/ml/Cargo.toml b/crates/ml/Cargo.toml index 11bb0cb0d..469c0b6ab 100644 --- a/crates/ml/Cargo.toml +++ b/crates/ml/Cargo.toml @@ -68,6 +68,7 @@ reqwest.workspace = true colored = "2.1" # Terminal color output for evaluation reports # Internal workspace crates +ml-core.workspace = true config.workspace = true common = { workspace = true, features = ["questdb"] } risk = { path = "../risk" } diff --git a/crates/ml/src/asset_selection/mod.rs b/crates/ml/src/asset_selection/mod.rs index d4d45723c..e4e77a2e5 100644 --- a/crates/ml/src/asset_selection/mod.rs +++ b/crates/ml/src/asset_selection/mod.rs @@ -107,14 +107,10 @@ impl AssetUniverse { /// # Errors /// Returns `MLError::ConfigError` on I/O or parse failures. pub fn from_config(path: &Path) -> Result { - let contents = std::fs::read_to_string(path).map_err(|e| MLError::ConfigError { - reason: format!("Failed to read universe config {}: {e}", path.display()), - })?; + let contents = std::fs::read_to_string(path).map_err(|e| MLError::ConfigError(format!("Failed to read universe config {}: {e}", path.display())))?; let config: UniverseConfigFile = - toml::from_str(&contents).map_err(|e| MLError::ConfigError { - reason: format!("Failed to parse universe config {}: {e}", path.display()), - })?; + toml::from_str(&contents).map_err(|e| MLError::ConfigError(format!("Failed to parse universe config {}: {e}", path.display())))?; let assets = config .symbols @@ -125,9 +121,7 @@ impl AssetUniverse { "Equity" | "equity" => AssetClass::Equity, "ETF" | "etf" => AssetClass::ETF, other => { - return Err(MLError::ConfigError { - reason: format!("Unknown asset class: {other}"), - }) + return Err(MLError::ConfigError(format!("Unknown asset class: {other}"))) } }; Ok(UniverseAsset { diff --git a/crates/ml/src/batch_processing.rs b/crates/ml/src/batch_processing.rs index b8d6499fd..e0a9fa061 100644 --- a/crates/ml/src/batch_processing.rs +++ b/crates/ml/src/batch_processing.rs @@ -134,9 +134,7 @@ pub struct AlignedBuffer { impl AlignedBuffer { pub fn new(capacity: usize, alignment: usize) -> Result { if alignment == 0 || !alignment.is_power_of_two() { - return Err(MLError::ConfigError { - reason: "Alignment must be a power of two".to_owned(), - }); + return Err(MLError::ConfigError("Alignment must be a power of two".to_owned())); } let mut data = Vec::with_capacity(capacity + alignment); diff --git a/crates/ml/src/checkpoint/signer.rs b/crates/ml/src/checkpoint/signer.rs index 3afb31c7e..d5b636b64 100644 --- a/crates/ml/src/checkpoint/signer.rs +++ b/crates/ml/src/checkpoint/signer.rs @@ -265,18 +265,14 @@ impl CheckpointSigner { match std::env::var(&env_key) { Ok(key_hex) => { - let key_data = hex::decode(&key_hex).map_err(|e| MLError::ConfigError { - reason: format!("Invalid key hex in {}: {}", env_key, e), - })?; + let key_data = hex::decode(&key_hex).map_err(|e| MLError::ConfigError(format!("Invalid key hex in {}: {}", env_key, e)))?; if key_data.len() != 32 { - return Err(MLError::ConfigError { - reason: format!( + return Err(MLError::ConfigError(format!( "Invalid key length in {} (expected 32 bytes, got {})", env_key, key_data.len() - ), - }); + ))); } info!( diff --git a/crates/ml/src/data_pipeline/cache.rs b/crates/ml/src/data_pipeline/cache.rs index 7d42807c6..e283232cb 100644 --- a/crates/ml/src/data_pipeline/cache.rs +++ b/crates/ml/src/data_pipeline/cache.rs @@ -84,13 +84,9 @@ impl CacheManifest { pub fn load(cache_dir: &Path) -> Result { let path = cache_dir.join("manifest.json"); match std::fs::read_to_string(&path) { - Ok(contents) => serde_json::from_str(&contents).map_err(|e| MLError::ConfigError { - reason: format!("Failed to parse manifest at {}: {e}", path.display()), - }), + Ok(contents) => serde_json::from_str(&contents).map_err(|e| MLError::ConfigError(format!("Failed to parse manifest at {}: {e}", path.display()))), Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(Self::new()), - Err(e) => Err(MLError::ConfigError { - reason: format!("Failed to read manifest at {}: {e}", path.display()), - }), + Err(e) => Err(MLError::ConfigError(format!("Failed to read manifest at {}: {e}", path.display()))), } } @@ -101,19 +97,13 @@ impl CacheManifest { /// # Errors /// Returns `MLError::ConfigError` on I/O or serialization failures. pub fn save(&mut self, cache_dir: &Path) -> Result<(), MLError> { - std::fs::create_dir_all(cache_dir).map_err(|e| MLError::ConfigError { - reason: format!("Failed to create cache dir {}: {e}", cache_dir.display()), - })?; + std::fs::create_dir_all(cache_dir).map_err(|e| MLError::ConfigError(format!("Failed to create cache dir {}: {e}", cache_dir.display())))?; self.last_updated = Utc::now(); - let json = serde_json::to_string_pretty(self).map_err(|e| MLError::ConfigError { - reason: format!("Failed to serialize manifest: {e}"), - })?; + let json = serde_json::to_string_pretty(self).map_err(|e| MLError::ConfigError(format!("Failed to serialize manifest: {e}")))?; let path = cache_dir.join("manifest.json"); - std::fs::write(&path, json).map_err(|e| MLError::ConfigError { - reason: format!("Failed to write manifest to {}: {e}", path.display()), - })?; + std::fs::write(&path, json).map_err(|e| MLError::ConfigError(format!("Failed to write manifest to {}: {e}", path.display())))?; Ok(()) } diff --git a/crates/ml/src/data_pipeline/manager.rs b/crates/ml/src/data_pipeline/manager.rs index 4c486e893..7c7c8da6f 100644 --- a/crates/ml/src/data_pipeline/manager.rs +++ b/crates/ml/src/data_pipeline/manager.rs @@ -151,12 +151,10 @@ impl DatasetManager { } if all_features.is_empty() { - return Err(MLError::ConfigError { - reason: format!( + return Err(MLError::ConfigError(format!( "No data loaded for dataset '{}'. Check cache at {:?}", spec.name, self.cache_dir - ), - }); + ))); } let total_bars = all_features.len(); diff --git a/crates/ml/src/diffusion/denoiser.rs b/crates/ml/src/diffusion/denoiser.rs index db87e5012..33faeea6e 100644 --- a/crates/ml/src/diffusion/denoiser.rs +++ b/crates/ml/src/diffusion/denoiser.rs @@ -165,12 +165,10 @@ impl Denoiser { device: &Device, ) -> Result { if data_dim == 0 || hidden_dim == 0 { - return Err(MLError::ConfigError { - reason: format!( + return Err(MLError::ConfigError(format!( "Diffusion denoiser requires data_dim > 0 and hidden_dim > 0 (got {}x{})", data_dim, hidden_dim - ), - }); + ))); } let time_embed = TimeEmbedding::new(time_embed_dim, hidden_dim, vb.pp("time_embed"))?; diff --git a/crates/ml/src/diffusion/noise.rs b/crates/ml/src/diffusion/noise.rs index 07d7a53c2..bf6e4e5a8 100644 --- a/crates/ml/src/diffusion/noise.rs +++ b/crates/ml/src/diffusion/noise.rs @@ -33,9 +33,7 @@ impl NoiseScheduler { device: &Device, ) -> Result { if num_timesteps == 0 { - return Err(MLError::ConfigError { - reason: "num_timesteps must be > 0".to_owned(), - }); + return Err(MLError::ConfigError("num_timesteps must be > 0".to_owned())); } let alpha_bars = match schedule { @@ -90,9 +88,7 @@ impl NoiseScheduler { self.alpha_bars .get(t) .copied() - .ok_or_else(|| MLError::ConfigError { - reason: format!("Timestep {} out of range (max {})", t, self.num_timesteps), - }) + .ok_or_else(|| MLError::ConfigError(format!("Timestep {} out of range (max {})", t, self.num_timesteps))) } /// Forward process: add noise to clean data x0 at timestep t. diff --git a/crates/ml/src/diffusion/trainable.rs b/crates/ml/src/diffusion/trainable.rs index b5c61e5ea..50f8837ea 100644 --- a/crates/ml/src/diffusion/trainable.rs +++ b/crates/ml/src/diffusion/trainable.rs @@ -265,9 +265,7 @@ impl UnifiedTrainable for DiffusionTrainableAdapter { fn validate(&mut self, val_data: &[(Tensor, Tensor)]) -> Result { if val_data.is_empty() { - return Err(MLError::ConfigError { - reason: "Validation data is empty".to_owned(), - }); + return Err(MLError::ConfigError("Validation data is empty".to_owned())); } // Accumulate per-batch loss tensors on GPU, then stack+mean once diff --git a/crates/ml/src/dqn/attention.rs b/crates/ml/src/dqn/attention.rs index 2be4e7142..43ff77028 100644 --- a/crates/ml/src/dqn/attention.rs +++ b/crates/ml/src/dqn/attention.rs @@ -60,17 +60,17 @@ impl MultiHeadAttentionConfig { /// Create a new configuration with validation pub fn new(embed_dim: usize, num_heads: usize) -> Result { if embed_dim == 0 { - return Err(MLError::ConfigurationError( + return Err(MLError::ConfigError( "embed_dim must be greater than 0".to_owned(), )); } if num_heads == 0 { - return Err(MLError::ConfigurationError( + return Err(MLError::ConfigError( "num_heads must be greater than 0".to_owned(), )); } if embed_dim % num_heads != 0 { - return Err(MLError::ConfigurationError(format!( + return Err(MLError::ConfigError(format!( "embed_dim ({}) must be divisible by num_heads ({})", embed_dim, num_heads ))); diff --git a/crates/ml/src/dqn/dqn.rs b/crates/ml/src/dqn/dqn.rs index 282b75e84..1b2156c26 100644 --- a/crates/ml/src/dqn/dqn.rs +++ b/crates/ml/src/dqn/dqn.rs @@ -1095,14 +1095,10 @@ impl DQN { /// Create new `DQN` pub fn new(config: DQNConfig) -> Result { if config.state_dim == 0 { - return Err(MLError::ConfigError { - reason: "DQN requires state_dim > 0".to_owned(), - }); + return Err(MLError::ConfigError("DQN requires state_dim > 0".to_owned())); } if config.num_actions == 0 { - return Err(MLError::ConfigError { - reason: "DQN requires num_actions > 0".to_owned(), - }); + return Err(MLError::ConfigError("DQN requires num_actions > 0".to_owned())); } let device = Device::cuda_if_available(0)?; // Use GPU if available, fallback to CPU diff --git a/crates/ml/src/dqn/ensemble.rs b/crates/ml/src/dqn/ensemble.rs index 04eee1971..7b6d40e4b 100644 --- a/crates/ml/src/dqn/ensemble.rs +++ b/crates/ml/src/dqn/ensemble.rs @@ -94,9 +94,7 @@ impl EnsembleConfig { /// * `voting_strategy` - Voting strategy pub fn new(num_agents: usize, state_dim: usize, voting_strategy: VotingStrategy) -> Result { if !(3..=5).contains(&num_agents) { - return Err(MLError::ConfigError { - reason: format!("num_agents must be 3-5, got {}", num_agents), - }); + return Err(MLError::ConfigError(format!("num_agents must be 3-5, got {}", num_agents))); } Ok(Self { diff --git a/crates/ml/src/dqn/multi_step.rs b/crates/ml/src/dqn/multi_step.rs index 2665fd282..951a1ae88 100644 --- a/crates/ml/src/dqn/multi_step.rs +++ b/crates/ml/src/dqn/multi_step.rs @@ -98,21 +98,15 @@ pub struct MultiStepCalculator { impl MultiStepCalculator { pub fn new(config: MultiStepConfig) -> Result { if config.n_steps == 0 { - return Err(MLError::ConfigError { - reason: "n_steps must be greater than 0".to_owned(), - }); + return Err(MLError::ConfigError("n_steps must be greater than 0".to_owned())); } if config.gamma <= 0.0 || config.gamma > 1.0 { - return Err(MLError::ConfigError { - reason: "gamma must be in (0, 1]".to_owned(), - }); + return Err(MLError::ConfigError("gamma must be in (0, 1]".to_owned())); } if !config.enabled { - return Err(MLError::ConfigError { - reason: "multi-step learning must be enabled".to_owned(), - }); + return Err(MLError::ConfigError("multi-step learning must be enabled".to_owned())); } Ok(Self { diff --git a/crates/ml/src/dqn/reward.rs b/crates/ml/src/dqn/reward.rs index 3ce8e4c6c..b2489626e 100644 --- a/crates/ml/src/dqn/reward.rs +++ b/crates/ml/src/dqn/reward.rs @@ -357,11 +357,9 @@ impl RewardConfig { pub fn validate(&self) -> Result<(), MLError> { // MANDATORY VALIDATION: use_percentage_pnl MUST be true if !self.use_percentage_pnl { - return Err(MLError::ConfigError { - reason: "FATAL: use_percentage_pnl=false causes gradient explosion. \ + return Err(MLError::ConfigError("FATAL: use_percentage_pnl=false causes gradient explosion. \ Q-values explode to ±50,000, gradients to millions. \ - MUST use percentage_pnl=true for numerical stability.".to_string() - }); + MUST use percentage_pnl=true for numerical stability.".to_string())); } // Warn if normalization disabled (suboptimal but not fatal) diff --git a/crates/ml/src/ensemble/mod.rs b/crates/ml/src/ensemble/mod.rs index 88630f784..ddef09b74 100644 --- a/crates/ml/src/ensemble/mod.rs +++ b/crates/ml/src/ensemble/mod.rs @@ -101,7 +101,7 @@ pub enum EnsembleError { impl From for crate::MLError { fn from(err: EnsembleError) -> Self { match err { - EnsembleError::InvalidConfiguration(msg) => crate::MLError::ConfigurationError(msg), + EnsembleError::InvalidConfiguration(msg) => crate::MLError::ConfigError(msg), EnsembleError::ModelNotFound(msg) => crate::MLError::ModelNotFound(msg), EnsembleError::InsufficientModels { expected, actual } => { crate::MLError::ValidationError { diff --git a/crates/ml/src/features/sample_weights.rs b/crates/ml/src/features/sample_weights.rs index cde29e1bb..2f5edcc99 100644 --- a/crates/ml/src/features/sample_weights.rs +++ b/crates/ml/src/features/sample_weights.rs @@ -130,25 +130,19 @@ impl SampleWeightCalculator { ) -> Result, MLError> { // Validate inputs if labels.is_empty() || timestamps.is_empty() { - return Err(MLError::ConfigError { - reason: "Labels and timestamps cannot be empty".to_owned(), - }); + return Err(MLError::ConfigError("Labels and timestamps cannot be empty".to_owned())); } if labels.len() != timestamps.len() { - return Err(MLError::ConfigError { - reason: format!( + return Err(MLError::ConfigError(format!( "Labels length ({}) must match timestamps length ({})", labels.len(), timestamps.len() - ), - }); + ))); } if self.decay_factor <= 0.0 { - return Err(MLError::ConfigError { - reason: format!("Decay factor must be positive, got {}", self.decay_factor), - }); + return Err(MLError::ConfigError(format!("Decay factor must be positive, got {}", self.decay_factor))); } let n = labels.len(); @@ -188,9 +182,7 @@ impl SampleWeightCalculator { let latest_time = timestamps .iter() .max() - .ok_or_else(|| MLError::ConfigError { - reason: "No timestamps provided".to_owned(), - })?; + .ok_or_else(|| MLError::ConfigError("No timestamps provided".to_owned()))?; // Apply exponential decay based on age in days for (weight, timestamp) in weights.iter_mut().zip(timestamps.iter()) { @@ -227,9 +219,7 @@ impl SampleWeightCalculator { for (weight, label) in weights.iter_mut().zip(labels.iter()) { let count = label_counts .get(label) - .ok_or_else(|| MLError::ConfigError { - reason: format!("Label {:?} not found in counts", label), - })?; + .ok_or_else(|| MLError::ConfigError(format!("Label {:?} not found in counts", label)))?; let balance_factor = 1.0 / (*count as f64); *weight *= balance_factor; @@ -243,9 +233,7 @@ impl SampleWeightCalculator { let sum: f64 = weights.iter().sum(); if sum <= 0.0 { - return Err(MLError::ConfigError { - reason: format!("Weight sum must be positive, got {}", sum), - }); + return Err(MLError::ConfigError(format!("Weight sum must be positive, got {}", sum))); } // Normalize: divide each weight by the sum diff --git a/crates/ml/src/flash_attention/mod.rs b/crates/ml/src/flash_attention/mod.rs index 7ff51647d..068afd4ee 100644 --- a/crates/ml/src/flash_attention/mod.rs +++ b/crates/ml/src/flash_attention/mod.rs @@ -340,7 +340,7 @@ mod tests { #[test] fn test_flash_attention_creation() -> Result<(), MLError> { let device = Device::cuda_if_available(0).map_err(|e| { - MLError::ConfigurationError(format!("GPU required for flash attention: {}", e)) + MLError::ConfigError(format!("GPU required for flash attention: {}", e)) })?; let config = FlashAttention3Config::default(); let _attention = FlashAttention3::new(config, device)?; diff --git a/crates/ml/src/gpu/mod.rs b/crates/ml/src/gpu/mod.rs index 16a92315e..b44109ef4 100644 --- a/crates/ml/src/gpu/mod.rs +++ b/crates/ml/src/gpu/mod.rs @@ -33,7 +33,7 @@ impl DeviceConfig { match self { DeviceConfig::Cpu => Ok(Device::Cpu), DeviceConfig::Cuda(id) => Device::new_cuda(*id).map_err(|e| { - MLError::ConfigurationError(format!( + MLError::ConfigError(format!( "CUDA device {} required but unavailable: {}", id, e )) diff --git a/crates/ml/src/hyperopt/adapters/continuous_ppo.rs b/crates/ml/src/hyperopt/adapters/continuous_ppo.rs index 3b0a3a0ff..f77418e7f 100644 --- a/crates/ml/src/hyperopt/adapters/continuous_ppo.rs +++ b/crates/ml/src/hyperopt/adapters/continuous_ppo.rs @@ -135,9 +135,7 @@ impl ParameterSpace for ContinuousPPOParams { fn from_continuous(x: &[f64]) -> Result { if x.len() != 12 { - return Err(MLError::ConfigError { - reason: format!("Expected 12 continuous parameters, got {}", x.len()), - }); + return Err(MLError::ConfigError(format!("Expected 12 continuous parameters, got {}", x.len()))); } Ok(Self { @@ -283,9 +281,7 @@ impl ContinuousPPOTrainer { let parquet_file = parquet_file.into(); if !parquet_file.exists() { - return Err(MLError::ConfigError { - reason: format!("Parquet file not found: {}", parquet_file.display()), - } + return Err(MLError::ConfigError(format!("Parquet file not found: {}", parquet_file.display())) .into()); } diff --git a/crates/ml/src/hyperopt/adapters/diffusion.rs b/crates/ml/src/hyperopt/adapters/diffusion.rs index 3b2e39a19..6f7ecf77c 100644 --- a/crates/ml/src/hyperopt/adapters/diffusion.rs +++ b/crates/ml/src/hyperopt/adapters/diffusion.rs @@ -79,9 +79,7 @@ impl ParameterSpace for DiffusionParams { fn from_continuous(x: &[f64]) -> Result { if x.len() != 9 { - return Err(MLError::ConfigError { - reason: format!("Expected 9 params, got {}", x.len()), - }); + return Err(MLError::ConfigError(format!("Expected 9 params, got {}", x.len()))); } Ok(Self { learning_rate: x.first().copied().unwrap_or(-9.21).exp(), @@ -187,9 +185,7 @@ impl DiffusionTrainer { pub fn new>(data_dir: P, epochs: usize) -> Result { let data_dir = data_dir.into(); if !data_dir.exists() { - return Err(MLError::ConfigError { - reason: format!("Data directory not found: {}", data_dir.display()), - }); + return Err(MLError::ConfigError(format!("Data directory not found: {}", data_dir.display()))); } let device = Device::new_cuda(0).unwrap_or_else(|e| { diff --git a/crates/ml/src/hyperopt/adapters/dqn.rs b/crates/ml/src/hyperopt/adapters/dqn.rs index d3ae74c70..7ea96ba9d 100644 --- a/crates/ml/src/hyperopt/adapters/dqn.rs +++ b/crates/ml/src/hyperopt/adapters/dqn.rs @@ -968,9 +968,7 @@ impl DQNTrainer { let dbn_data_dir = dbn_data_dir.into(); if !dbn_data_dir.exists() { - return Err(MLError::ConfigError { - reason: format!("DBN data directory not found: {}", dbn_data_dir.display()), - } + return Err(MLError::ConfigError(format!("DBN data directory not found: {}", dbn_data_dir.display())) .into()); } @@ -981,7 +979,7 @@ impl DQNTrainer { // Initialize CUDA device at construction time — GPU is required for RL hyperopt let device = candle_core::Device::new_cuda(0) - .map_err(|e| MLError::ConfigError { reason: format!("CUDA GPU required for DQN hyperopt: {}", e) })?; + .map_err(|e| MLError::ConfigError(format!("CUDA GPU required for DQN hyperopt: {}", e)))?; info!(" Device: CUDA GPU"); // Try to reuse existing Tokio runtime, create new one if needed @@ -1184,9 +1182,7 @@ impl DQNTrainer { return Ok(()); } - let data_path_str = self.dbn_data_dir.to_str().ok_or_else(|| MLError::ConfigError { - reason: "Invalid UTF-8 in data path".to_owned(), - })?; + let data_path_str = self.dbn_data_dir.to_str().ok_or_else(|| MLError::ConfigError("Invalid UTF-8 in data path".to_owned()))?; info!("Preloading training data from: {} ...", data_path_str); let preload_start = std::time::Instant::now(); @@ -2235,9 +2231,7 @@ impl HyperparameterOptimizable for DQNTrainer { // Create all training directories self.training_paths .create_all() - .map_err(|e| MLError::ConfigError { - reason: format!("Failed to create training directories: {}", e), - })?; + .map_err(|e| MLError::ConfigError(format!("Failed to create training directories: {}", e)))?; info!("Training directories created:"); info!(" Checkpoints: {:?}", self.training_paths.checkpoints_dir()); @@ -2521,9 +2515,7 @@ impl HyperparameterOptimizable for DQNTrainer { let data_path_str = self .dbn_data_dir.to_str() - .ok_or_else(|| MLError::ConfigError { - reason: "Invalid UTF-8 in data path".to_owned(), - })?; + .ok_or_else(|| MLError::ConfigError("Invalid UTF-8 in data path".to_owned()))?; // Check if path is a parquet file let is_parquet_file = diff --git a/crates/ml/src/hyperopt/adapters/ensemble.rs b/crates/ml/src/hyperopt/adapters/ensemble.rs index 94b762f3e..b3faeccf9 100644 --- a/crates/ml/src/hyperopt/adapters/ensemble.rs +++ b/crates/ml/src/hyperopt/adapters/ensemble.rs @@ -156,10 +156,8 @@ fn with_config(f: impl FnOnce(&EnsembleSpaceConfig) -> T) -> Result Ok(f(cfg)), - None => Err(MLError::ConfigError { - reason: "EnsembleSpaceConfig not installed. Call config.install() first." - .to_string(), - }), + None => Err(MLError::ConfigError("EnsembleSpaceConfig not installed. Call config.install() first." + .to_string())), } }) } @@ -238,13 +236,11 @@ impl ParameterSpace for EnsembleParameterSpace { fn from_continuous(x: &[f64]) -> Result { let expected_dim = with_config(|cfg| cfg.total_dim())?; if x.len() != expected_dim { - return Err(MLError::ConfigError { - reason: format!( + return Err(MLError::ConfigError(format!( "EnsembleParameterSpace: expected {} params, got {}", expected_dim, x.len() - ), - }); + ))); } Ok(Self { values: x.to_vec(), diff --git a/crates/ml/src/hyperopt/adapters/kan.rs b/crates/ml/src/hyperopt/adapters/kan.rs index 834eeb990..323703480 100644 --- a/crates/ml/src/hyperopt/adapters/kan.rs +++ b/crates/ml/src/hyperopt/adapters/kan.rs @@ -79,9 +79,7 @@ impl ParameterSpace for KANParams { fn from_continuous(x: &[f64]) -> Result { if x.len() != NUM_PARAMS { - return Err(MLError::ConfigError { - reason: format!("Expected {} continuous parameters, got {}", NUM_PARAMS, x.len()), - }); + return Err(MLError::ConfigError(format!("Expected {} continuous parameters, got {}", NUM_PARAMS, x.len()))); } Ok(Self { learning_rate: x.first().copied().unwrap_or(0.0).exp(), @@ -177,9 +175,7 @@ impl KANTrainer { pub fn new>(data_dir: P, epochs: usize) -> Result { let data_dir = data_dir.into(); if !data_dir.exists() { - return Err(MLError::ConfigError { - reason: format!("Data directory not found: {}", data_dir.display()), - }); + return Err(MLError::ConfigError(format!("Data directory not found: {}", data_dir.display()))); } let device = Device::new_cuda(0).unwrap_or_else(|e| { diff --git a/crates/ml/src/hyperopt/adapters/liquid.rs b/crates/ml/src/hyperopt/adapters/liquid.rs index 242195598..dafaf6bbf 100644 --- a/crates/ml/src/hyperopt/adapters/liquid.rs +++ b/crates/ml/src/hyperopt/adapters/liquid.rs @@ -97,59 +97,41 @@ impl ParameterSpace for LiquidParams { fn from_continuous(x: &[f64]) -> Result { if x.len() != NUM_PARAMS { - return Err(MLError::ConfigError { - reason: format!( + return Err(MLError::ConfigError(format!( "LiquidParams: expected {} dimensions, got {}", NUM_PARAMS, x.len() - ), - }); + ))); } let learning_rate = x .get(0) - .ok_or_else(|| MLError::ConfigError { - reason: "LiquidParams: missing learning_rate at index 0".to_owned(), - })? + .ok_or_else(|| MLError::ConfigError("LiquidParams: missing learning_rate at index 0".to_owned()))? .exp(); let hidden_size = x .get(1) - .ok_or_else(|| MLError::ConfigError { - reason: "LiquidParams: missing hidden_size at index 1".to_owned(), - })? + .ok_or_else(|| MLError::ConfigError("LiquidParams: missing hidden_size at index 1".to_owned()))? .round() as usize; let backbone_depth = x .get(2) - .ok_or_else(|| MLError::ConfigError { - reason: "LiquidParams: missing backbone_depth at index 2".to_owned(), - })? + .ok_or_else(|| MLError::ConfigError("LiquidParams: missing backbone_depth at index 2".to_owned()))? .round() as usize; let tau_min = x .get(3) - .ok_or_else(|| MLError::ConfigError { - reason: "LiquidParams: missing tau_min at index 3".to_owned(), - })? + .ok_or_else(|| MLError::ConfigError("LiquidParams: missing tau_min at index 3".to_owned()))? .exp(); let tau_max = x .get(4) - .ok_or_else(|| MLError::ConfigError { - reason: "LiquidParams: missing tau_max at index 4".to_owned(), - })? + .ok_or_else(|| MLError::ConfigError("LiquidParams: missing tau_max at index 4".to_owned()))? .exp(); - let dropout = *x.get(5).ok_or_else(|| MLError::ConfigError { - reason: "LiquidParams: missing dropout at index 5".to_owned(), - })?; + let dropout = *x.get(5).ok_or_else(|| MLError::ConfigError("LiquidParams: missing dropout at index 5".to_owned()))?; let gradient_clip = x .get(6) - .ok_or_else(|| MLError::ConfigError { - reason: "LiquidParams: missing gradient_clip at index 6".to_owned(), - })? + .ok_or_else(|| MLError::ConfigError("LiquidParams: missing gradient_clip at index 6".to_owned()))? .exp(); let batch_size = x .get(7) - .ok_or_else(|| MLError::ConfigError { - reason: "LiquidParams: missing batch_size at index 7".to_owned(), - })? + .ok_or_else(|| MLError::ConfigError("LiquidParams: missing batch_size at index 7".to_owned()))? .round() as usize; Ok(Self { @@ -246,9 +228,7 @@ impl LiquidTrainer { pub fn new>(data_dir: P, epochs: usize) -> Result { let data_dir = data_dir.into(); if !data_dir.exists() { - return Err(MLError::ConfigError { - reason: format!("Data directory not found: {}", data_dir.display()), - }); + return Err(MLError::ConfigError(format!("Data directory not found: {}", data_dir.display()))); } let device = Device::new_cuda(0).unwrap_or_else(|e| { diff --git a/crates/ml/src/hyperopt/adapters/mamba2.rs b/crates/ml/src/hyperopt/adapters/mamba2.rs index da6ef3f53..9e5e326de 100644 --- a/crates/ml/src/hyperopt/adapters/mamba2.rs +++ b/crates/ml/src/hyperopt/adapters/mamba2.rs @@ -134,9 +134,7 @@ impl ParameterSpace for Mamba2Params { fn from_continuous(x: &[f64]) -> Result { if x.len() != 12 { - return Err(MLError::ConfigError { - reason: format!("Expected 12 parameters, got {}", x.len()), - }); + return Err(MLError::ConfigError(format!("Expected 12 parameters, got {}", x.len()))); } Ok(Self { @@ -275,9 +273,7 @@ impl Mamba2Trainer { let data_dir = data_dir.into(); if !data_dir.exists() { - return Err(MLError::ConfigError { - reason: format!("Data directory not found: {}", data_dir.display()), - } + return Err(MLError::ConfigError(format!("Data directory not found: {}", data_dir.display())) .into()); } diff --git a/crates/ml/src/hyperopt/adapters/ppo.rs b/crates/ml/src/hyperopt/adapters/ppo.rs index 0fbbe277b..6199f7e9c 100644 --- a/crates/ml/src/hyperopt/adapters/ppo.rs +++ b/crates/ml/src/hyperopt/adapters/ppo.rs @@ -158,26 +158,24 @@ impl ParameterSpace for PPOParams { fn from_continuous(x: &[f64]) -> Result { if x.len() != 14 { - return Err(MLError::ConfigError { - reason: format!("Expected 14 parameters, got {}", x.len()), - }); + return Err(MLError::ConfigError(format!("Expected 14 parameters, got {}", x.len()))); } Ok(Self { - policy_learning_rate: x.get(0).copied().ok_or(MLError::ConfigError { reason: "Missing policy_learning_rate".to_owned() })?.exp(), - value_learning_rate: x.get(1).copied().ok_or(MLError::ConfigError { reason: "Missing value_learning_rate".to_owned() })?.exp(), - clip_epsilon: x.get(2).copied().ok_or(MLError::ConfigError { reason: "Missing clip_epsilon".to_owned() })?.clamp(0.1, 0.3), - value_loss_coeff: x.get(3).copied().ok_or(MLError::ConfigError { reason: "Missing value_loss_coeff".to_owned() })?.clamp(0.5, 2.0), - entropy_coeff: x.get(4).copied().ok_or(MLError::ConfigError { reason: "Missing entropy_coeff".to_owned() })?.exp(), - batch_size: x.get(5).copied().ok_or(MLError::ConfigError { reason: "Missing batch_size".to_owned() })?.round() as usize, - hidden_dim_base: ((x.get(6).copied().ok_or(MLError::ConfigError { reason: "Missing hidden_dim_base".to_owned() })?.round() / 64.0).round() * 64.0).clamp(64.0, 4096.0) as usize, - gae_gamma: x.get(7).copied().ok_or(MLError::ConfigError { reason: "Missing gae_gamma".to_owned() })?.clamp(0.95, 0.999), - gae_lambda: x.get(8).copied().ok_or(MLError::ConfigError { reason: "Missing gae_lambda".to_owned() })?.clamp(0.8, 1.0), - mini_batch_size: x.get(9).copied().ok_or(MLError::ConfigError { reason: "Missing mini_batch_size".to_owned() })?.round() as usize, - max_grad_norm: x.get(10).copied().ok_or(MLError::ConfigError { reason: "Missing max_grad_norm".to_owned() })?.clamp(0.1, 1.0), - max_position_absolute: x.get(11).copied().ok_or(MLError::ConfigError { reason: "Missing max_position_absolute".to_owned() })?.clamp(0.5, 3.0), - clip_epsilon_high: x.get(12).copied().ok_or(MLError::ConfigError { reason: "Missing clip_epsilon_high".to_owned() })?.clamp(0.0, 0.5), - curiosity_weight: x.get(13).copied().ok_or(MLError::ConfigError { reason: "Missing curiosity_weight".to_owned() })?.clamp(0.0, 0.5), + policy_learning_rate: x.get(0).copied().ok_or(MLError::ConfigError("Missing policy_learning_rate".to_owned()))?.exp(), + value_learning_rate: x.get(1).copied().ok_or(MLError::ConfigError("Missing value_learning_rate".to_owned()))?.exp(), + clip_epsilon: x.get(2).copied().ok_or(MLError::ConfigError("Missing clip_epsilon".to_owned()))?.clamp(0.1, 0.3), + value_loss_coeff: x.get(3).copied().ok_or(MLError::ConfigError("Missing value_loss_coeff".to_owned()))?.clamp(0.5, 2.0), + entropy_coeff: x.get(4).copied().ok_or(MLError::ConfigError("Missing entropy_coeff".to_owned()))?.exp(), + batch_size: x.get(5).copied().ok_or(MLError::ConfigError("Missing batch_size".to_owned()))?.round() as usize, + hidden_dim_base: ((x.get(6).copied().ok_or(MLError::ConfigError("Missing hidden_dim_base".to_owned()))?.round() / 64.0).round() * 64.0).clamp(64.0, 4096.0) as usize, + gae_gamma: x.get(7).copied().ok_or(MLError::ConfigError("Missing gae_gamma".to_owned()))?.clamp(0.95, 0.999), + gae_lambda: x.get(8).copied().ok_or(MLError::ConfigError("Missing gae_lambda".to_owned()))?.clamp(0.8, 1.0), + mini_batch_size: x.get(9).copied().ok_or(MLError::ConfigError("Missing mini_batch_size".to_owned()))?.round() as usize, + max_grad_norm: x.get(10).copied().ok_or(MLError::ConfigError("Missing max_grad_norm".to_owned()))?.clamp(0.1, 1.0), + max_position_absolute: x.get(11).copied().ok_or(MLError::ConfigError("Missing max_position_absolute".to_owned()))?.clamp(0.5, 3.0), + clip_epsilon_high: x.get(12).copied().ok_or(MLError::ConfigError("Missing clip_epsilon_high".to_owned()))?.clamp(0.0, 0.5), + curiosity_weight: x.get(13).copied().ok_or(MLError::ConfigError("Missing curiosity_weight".to_owned()))?.clamp(0.0, 0.5), }) } @@ -405,9 +403,7 @@ impl PPOTrainer { let dbn_data_dir = dbn_data_dir.into(); if !dbn_data_dir.exists() { - return Err(MLError::ConfigError { - reason: format!("DBN data directory not found: {}", dbn_data_dir.display()), - } + return Err(MLError::ConfigError(format!("DBN data directory not found: {}", dbn_data_dir.display())) .into()); } @@ -942,12 +938,10 @@ impl HyperparameterOptimizable for PPOTrainer { // Validate minimum trajectory count for 80/20 split if total_trajectories < 10 { - return Err(MLError::ConfigError { - reason: format!( + return Err(MLError::ConfigError(format!( "Insufficient trajectories for train/val split: {} < 10 minimum", total_trajectories - ), - }); + ))); } // Split trajectories 80/20 for train/val @@ -957,12 +951,10 @@ impl HyperparameterOptimizable for PPOTrainer { // Validate non-empty validation set if num_val < 1 { - return Err(MLError::ConfigError { - reason: format!( + return Err(MLError::ConfigError(format!( "Validation set is empty after 80/20 split (train={}, val={})", num_train, num_val - ), - }); + ))); } // Warn if validation set is too small diff --git a/crates/ml/src/hyperopt/adapters/tft.rs b/crates/ml/src/hyperopt/adapters/tft.rs index 55906f763..b067211cd 100644 --- a/crates/ml/src/hyperopt/adapters/tft.rs +++ b/crates/ml/src/hyperopt/adapters/tft.rs @@ -107,9 +107,7 @@ impl ParameterSpace for TFTParams { fn from_continuous(x: &[f64]) -> Result { if x.len() != 5 { - return Err(MLError::ConfigError { - reason: format!("Expected 5 parameters, got {}", x.len()), - }); + return Err(MLError::ConfigError(format!("Expected 5 parameters, got {}", x.len()))); } // Map continuous indices to discrete values @@ -252,9 +250,7 @@ impl TFTTrainer { let data_dir = data_dir.into(); if !data_dir.exists() { - return Err(MLError::ConfigError { - reason: format!("Data directory not found: {}", data_dir.display()), - } + return Err(MLError::ConfigError(format!("Data directory not found: {}", data_dir.display())) .into()); } diff --git a/crates/ml/src/hyperopt/adapters/tggn.rs b/crates/ml/src/hyperopt/adapters/tggn.rs index df3ef3c60..7e8e3a422 100644 --- a/crates/ml/src/hyperopt/adapters/tggn.rs +++ b/crates/ml/src/hyperopt/adapters/tggn.rs @@ -93,9 +93,7 @@ impl ParameterSpace for TGGNParams { fn from_continuous(x: &[f64]) -> Result { if x.len() != NUM_PARAMS { - return Err(MLError::ConfigError { - reason: format!("Expected {} continuous parameters, got {}", NUM_PARAMS, x.len()), - }); + return Err(MLError::ConfigError(format!("Expected {} continuous parameters, got {}", NUM_PARAMS, x.len()))); } Ok(Self { learning_rate: x.first().copied().unwrap_or(0.0).exp(), @@ -202,9 +200,7 @@ impl TGGNTrainer { pub fn new>(data_dir: P, epochs: usize) -> Result { let data_dir = data_dir.into(); if !data_dir.exists() { - return Err(MLError::ConfigError { - reason: format!("Data directory not found: {}", data_dir.display()), - }); + return Err(MLError::ConfigError(format!("Data directory not found: {}", data_dir.display()))); } let device = Device::new_cuda(0).unwrap_or_else(|e| { diff --git a/crates/ml/src/hyperopt/adapters/tlob.rs b/crates/ml/src/hyperopt/adapters/tlob.rs index 528ad1ec7..b5addb888 100644 --- a/crates/ml/src/hyperopt/adapters/tlob.rs +++ b/crates/ml/src/hyperopt/adapters/tlob.rs @@ -87,9 +87,7 @@ impl ParameterSpace for TLOBParams { fn from_continuous(x: &[f64]) -> Result { if x.len() != NUM_PARAMS { - return Err(MLError::ConfigError { - reason: format!("Expected {} continuous parameters, got {}", NUM_PARAMS, x.len()), - }); + return Err(MLError::ConfigError(format!("Expected {} continuous parameters, got {}", NUM_PARAMS, x.len()))); } Ok(Self { learning_rate: x.first().copied().unwrap_or(0.0).exp(), @@ -191,9 +189,7 @@ impl TLOBTrainer { pub fn new>(data_dir: P, epochs: usize) -> Result { let data_dir = data_dir.into(); if !data_dir.exists() { - return Err(MLError::ConfigError { - reason: format!("Data directory not found: {}", data_dir.display()), - }); + return Err(MLError::ConfigError(format!("Data directory not found: {}", data_dir.display()))); } let device = Device::new_cuda(0).unwrap_or_else(|e| { diff --git a/crates/ml/src/hyperopt/adapters/xlstm.rs b/crates/ml/src/hyperopt/adapters/xlstm.rs index 4c35729c5..ca5987087 100644 --- a/crates/ml/src/hyperopt/adapters/xlstm.rs +++ b/crates/ml/src/hyperopt/adapters/xlstm.rs @@ -70,9 +70,7 @@ impl ParameterSpace for XLSTMParams { fn from_continuous(x: &[f64]) -> Result { if x.len() != 9 { - return Err(MLError::ConfigError { - reason: format!("Expected 9 params, got {}", x.len()), - }); + return Err(MLError::ConfigError(format!("Expected 9 params, got {}", x.len()))); } let hidden_dim = x.get(1).copied().unwrap_or(64.0).round().max(32.0) as usize; let num_heads = x.get(3).copied().unwrap_or(4.0).round().max(1.0) as usize; @@ -166,9 +164,7 @@ impl XLSTMTrainer { pub fn new>(data_dir: P, epochs: usize) -> Result { let data_dir = data_dir.into(); if !data_dir.exists() { - return Err(MLError::ConfigError { - reason: format!("Data directory not found: {}", data_dir.display()), - }); + return Err(MLError::ConfigError(format!("Data directory not found: {}", data_dir.display()))); } let device = Device::new_cuda(0).unwrap_or_else(|e| { diff --git a/crates/ml/src/hyperopt/optimizer.rs b/crates/ml/src/hyperopt/optimizer.rs index 18d2f111e..d774b6510 100644 --- a/crates/ml/src/hyperopt/optimizer.rs +++ b/crates/ml/src/hyperopt/optimizer.rs @@ -253,9 +253,7 @@ impl ArgminOptimizer { let n_params = bounds.len(); if n_params == 0 { - return Err(MLError::ConfigError { - reason: "Parameter space has zero dimensions".to_owned(), - } + return Err(MLError::ConfigError("Parameter space has zero dimensions".to_owned()) .into()); } @@ -630,9 +628,7 @@ impl ArgminOptimizer { let n_params = bounds.len(); if n_params == 0 { - return Err(MLError::ConfigError { - reason: "Parameter space has zero dimensions".to_owned(), - } + return Err(MLError::ConfigError("Parameter space has zero dimensions".to_owned()) .into()); } @@ -891,9 +887,7 @@ where let n_params = bounds.len(); if n_params == 0 { - return Err(MLError::ConfigError { - reason: "Parameter space has zero dimensions".to_owned(), - } + return Err(MLError::ConfigError("Parameter space has zero dimensions".to_owned()) .into()); } diff --git a/crates/ml/src/hyperopt/tests_argmin.rs b/crates/ml/src/hyperopt/tests_argmin.rs index aeae04c2c..85228a9f9 100644 --- a/crates/ml/src/hyperopt/tests_argmin.rs +++ b/crates/ml/src/hyperopt/tests_argmin.rs @@ -41,9 +41,7 @@ mod tests { fn from_continuous(x: &[f64]) -> Result { if x.len() != 2 { - return Err(MLError::ConfigError { - reason: format!("Expected 2 params, got {}", x.len()), - }); + return Err(MLError::ConfigError(format!("Expected 2 params, got {}", x.len()))); } Ok(Self { x: x[0], y: x[1] }) } @@ -93,9 +91,7 @@ mod tests { fn from_continuous(x: &[f64]) -> Result { if x.len() != 2 { - return Err(MLError::ConfigError { - reason: format!("Expected 2 params, got {}", x.len()), - }); + return Err(MLError::ConfigError(format!("Expected 2 params, got {}", x.len()))); } Ok(Self { x: x[0], y: x[1] }) } diff --git a/crates/ml/src/hyperopt/traits.rs b/crates/ml/src/hyperopt/traits.rs index 5db4af37a..671911fab 100644 --- a/crates/ml/src/hyperopt/traits.rs +++ b/crates/ml/src/hyperopt/traits.rs @@ -737,9 +737,7 @@ mod tests { fn from_continuous(x: &[f64]) -> Result { if x.len() != 2 { - return Err(MLError::ConfigError { - reason: format!("Expected 2 params, got {}", x.len()), - }); + return Err(MLError::ConfigError(format!("Expected 2 params, got {}", x.len()))); } Ok(Self { diff --git a/crates/ml/src/integration/coordinator.rs b/crates/ml/src/integration/coordinator.rs index 0a1d5c8b4..dbdfa618c 100644 --- a/crates/ml/src/integration/coordinator.rs +++ b/crates/ml/src/integration/coordinator.rs @@ -223,10 +223,8 @@ impl EnsembleCoordinator { candidates .into_iter() .next() - .ok_or_else(|| MLError::ConfigError { - reason: "No candidate models available for UltraLowLatency mode" - .to_string(), - })?; + .ok_or_else(|| MLError::ConfigError("No candidate models available for UltraLowLatency mode" + .to_string()))?; let timeout = (budget_us / 2).min(50); // Conservative timeout ( EnsembleStrategy::SingleModel, diff --git a/crates/ml/src/integration/inference_engine.rs b/crates/ml/src/integration/inference_engine.rs index d3363ff9c..e77339060 100644 --- a/crates/ml/src/integration/inference_engine.rs +++ b/crates/ml/src/integration/inference_engine.rs @@ -284,9 +284,7 @@ impl InferenceEngine { "load_onnx_model called for {} but ONNX Runtime is disabled", model_id ); - Err(MLError::ConfigError { - reason: "ONNX runtime not available (removed from dependencies)".to_owned(), - }) + Err(MLError::ConfigError("ONNX runtime not available (removed from dependencies)".to_owned())) } /// Load micro model for ultra-fast inference diff --git a/crates/ml/src/kan/network.rs b/crates/ml/src/kan/network.rs index cbebdfffb..27d1f60f2 100644 --- a/crates/ml/src/kan/network.rs +++ b/crates/ml/src/kan/network.rs @@ -25,24 +25,16 @@ impl KANNetwork { /// 51->32, 32->16, 16->1. pub fn new(config: &KANConfig, vb: VarBuilder<'_>) -> Result { if config.layer_widths.len() < 2 { - return Err(MLError::ConfigError { - reason: "layer_widths must have at least 2 entries".to_owned(), - }); + return Err(MLError::ConfigError("layer_widths must have at least 2 entries".to_owned())); } if config.layer_widths.contains(&0) { - return Err(MLError::ConfigError { - reason: "KAN requires all layer_widths > 0".to_owned(), - }); + return Err(MLError::ConfigError("KAN requires all layer_widths > 0".to_owned())); } let mut layers = Vec::with_capacity(config.layer_widths.len() - 1); for i in 0..config.layer_widths.len() - 1 { - let in_dim = *config.layer_widths.get(i).ok_or_else(|| MLError::ConfigError { - reason: format!("Missing layer_widths index {}", i), - })?; - let out_dim = *config.layer_widths.get(i + 1).ok_or_else(|| MLError::ConfigError { - reason: format!("Missing layer_widths index {}", i + 1), - })?; + let in_dim = *config.layer_widths.get(i).ok_or_else(|| MLError::ConfigError(format!("Missing layer_widths index {}", i)))?; + let out_dim = *config.layer_widths.get(i + 1).ok_or_else(|| MLError::ConfigError(format!("Missing layer_widths index {}", i + 1)))?; let layer = KANLayer::new( in_dim, diff --git a/crates/ml/src/kan/spline.rs b/crates/ml/src/kan/spline.rs index da032b6b2..b49f696c0 100644 --- a/crates/ml/src/kan/spline.rs +++ b/crates/ml/src/kan/spline.rs @@ -50,14 +50,10 @@ impl BSplineBasis { /// * `x_max` - Right boundary of the domain pub fn new(grid_size: usize, order: usize, x_min: f64, x_max: f64) -> Result { if grid_size == 0 || order == 0 { - return Err(MLError::ConfigError { - reason: "grid_size and order must be > 0".to_owned(), - }); + return Err(MLError::ConfigError("grid_size and order must be > 0".to_owned())); } if x_max <= x_min { - return Err(MLError::ConfigError { - reason: format!("x_max ({}) must be > x_min ({})", x_max, x_min), - }); + return Err(MLError::ConfigError(format!("x_max ({}) must be > x_min ({})", x_max, x_min))); } let num_bases = grid_size + order - 1; @@ -105,18 +101,14 @@ impl BSplineBasis { device: &Device, ) -> Result<(), MLError> { if resolution < 2 { - return Err(MLError::ConfigError { - reason: "GPU grid resolution must be >= 2".to_owned(), - }); + return Err(MLError::ConfigError("GPU grid resolution must be >= 2".to_owned())); } let grid_min = self.knots.first().copied().unwrap_or(0.0); let grid_max = self.knots.last().copied().unwrap_or(1.0); let range = grid_max - grid_min; if range.abs() < 1e-10 { - return Err(MLError::ConfigError { - reason: "Knot range is degenerate (all knots equal)".to_owned(), - }); + return Err(MLError::ConfigError("Knot range is degenerate (all knots equal)".to_owned())); } let step = range / (resolution - 1).max(1) as f32; diff --git a/crates/ml/src/labeling/meta_labeling/primary_model.rs b/crates/ml/src/labeling/meta_labeling/primary_model.rs index 01912daf3..404fac5e1 100644 --- a/crates/ml/src/labeling/meta_labeling/primary_model.rs +++ b/crates/ml/src/labeling/meta_labeling/primary_model.rs @@ -96,12 +96,10 @@ impl PrimaryModelConfig { /// Validate configuration pub fn validate(&self) -> Result<(), MLError> { if self.threshold < 0.0 || self.threshold > 1.0 { - return Err(MLError::ConfigError { - reason: format!( + return Err(MLError::ConfigError(format!( "Threshold must be in range [0.0, 1.0], got {}", self.threshold - ), - }); + ))); } Ok(()) } @@ -250,9 +248,8 @@ impl From for LabelingError { expected, actual )), MLError::InvalidInput(msg) => LabelingError::InvalidInput(msg), - MLError::ConfigError { reason } => LabelingError::ConfigError(reason), - MLError::ConfigurationError(_) - | MLError::GraphError { .. } + MLError::ConfigError(reason) => LabelingError::ConfigError(reason), + MLError::GraphError { .. } | MLError::ResourceLimit { .. } | MLError::SerializationError { .. } | MLError::ValidationError { .. } diff --git a/crates/ml/src/labeling/meta_labeling/secondary_model.rs b/crates/ml/src/labeling/meta_labeling/secondary_model.rs index db88d50e6..250cae664 100644 --- a/crates/ml/src/labeling/meta_labeling/secondary_model.rs +++ b/crates/ml/src/labeling/meta_labeling/secondary_model.rs @@ -54,27 +54,19 @@ impl SecondaryModelConfig { /// Validate configuration parameters pub fn validate(&self) -> Result<(), MLError> { if self.min_confidence >= self.max_confidence { - return Err(MLError::ConfigError { - reason: "min_confidence must be less than max_confidence".to_owned(), - }); + return Err(MLError::ConfigError("min_confidence must be less than max_confidence".to_owned())); } if self.min_confidence < 0.0 || self.max_confidence > 1.0 { - return Err(MLError::ConfigError { - reason: "Confidence thresholds must be in [0.0, 1.0]".to_owned(), - }); + return Err(MLError::ConfigError("Confidence thresholds must be in [0.0, 1.0]".to_owned())); } if self.min_bet_size < 0.0 || self.max_bet_size > 1.0 { - return Err(MLError::ConfigError { - reason: "Bet sizes must be in [0.0, 1.0]".to_owned(), - }); + return Err(MLError::ConfigError("Bet sizes must be in [0.0, 1.0]".to_owned())); } if self.min_bet_size >= self.max_bet_size { - return Err(MLError::ConfigError { - reason: "min_bet_size must be less than max_bet_size".to_owned(), - }); + return Err(MLError::ConfigError("min_bet_size must be less than max_bet_size".to_owned())); } Ok(()) diff --git a/crates/ml/src/lib.rs b/crates/ml/src/lib.rs index 1dede44b2..adadc2825 100644 --- a/crates/ml/src/lib.rs +++ b/crates/ml/src/lib.rs @@ -150,8 +150,9 @@ // are already "deny" at workspace level -- kept here for explicitness. #![deny(clippy::panic, clippy::unimplemented, clippy::unreachable)] -// Import common types properly - NO ALIASES THAT CONFLICT! -use serde::{Deserialize, Serialize}; +// Re-export all core types from ml-core +#[allow(deprecated)] +pub use ml_core::*; // Silence unused crate warnings for dependencies used in tests or feature-gated code use approx as _; @@ -163,623 +164,9 @@ use num_traits as _; use semver as _; use tempfile as _; -// Direct type imports - no compatibility aliases -use rust_decimal::Decimal; - -/// Common type errors that can occur during ML operations -/// -/// This enum represents various type-related errors that can occur when working -/// with different data types across the ML pipeline, including type conversions, -/// validation errors, and compatibility issues. -#[derive(Debug, Clone, thiserror::Error, serde::Serialize, serde::Deserialize)] -pub enum CommonTypeError { - /// Generic type error with descriptive message - #[error("Type error: {0}")] - Error(String), -} - -/// Market regime classification for algorithmic trading strategies -/// -/// This enum represents different market conditions that can be detected through -/// statistical analysis and machine learning models. Market regime detection is -/// crucial for adaptive trading strategies that adjust their behavior based on -/// current market conditions. -/// -/// # Examples -/// -/// ```rust -/// use ml::MarketRegime; -/// -/// let regime = MarketRegime::Trending; -/// match regime { -/// MarketRegime::Bull => println!("Use momentum strategies"), -/// MarketRegime::Bear => println!("Use defensive strategies"), -/// MarketRegime::Crisis => println!("Implement risk controls"), -/// _ => println!("Use balanced approach"), -/// } -/// ``` -#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] -pub enum MarketRegime { - /// Normal market conditions with typical volatility and volume - Normal, - /// Strong directional movement with clear trends - Trending, - /// Range-bound market with limited directional movement - Sideways, - /// Bullish market with rising prices and positive sentiment - Bull, - /// Bearish market with falling prices and negative sentiment - Bear, - /// Crisis conditions with extreme volatility and risk - Crisis, -} - -/// Common errors that can occur across the ML system -/// -/// This enum provides a unified error type that can be used throughout the ML -/// pipeline to ensure consistent error handling and reporting. It serves as a -/// bridge between different subsystems and provides appropriate error categorization. -#[derive(Debug, Clone, thiserror::Error, serde::Serialize, serde::Deserialize)] -pub enum CommonError { - /// General error with descriptive message - #[error("Error: {0}")] - General(String), -} - -impl CommonError { - /// Create a validation error with a descriptive message - /// - /// # Arguments - /// - /// * `msg` - A descriptive message explaining the validation failure - /// - /// # Returns - /// - /// Returns a `CommonError::General` variant with the validation message - /// - /// # Examples - /// - /// ```rust - /// use ml::CommonError; - /// - /// let error = CommonError::validation("Invalid input range"); - /// assert!(error.to_string().contains("Invalid input range")); - /// ``` - pub fn validation>(msg: S) -> Self { - Self::General(msg.into()) - } - - /// Create a configuration error with a descriptive message - /// - /// # Arguments - /// - /// * `msg` - A descriptive message explaining the configuration issue - /// - /// # Returns - /// - /// Returns a `CommonError::General` variant with the configuration message - /// - /// # Examples - /// - /// ```rust - /// use ml::CommonError; - /// - /// let error = CommonError::config("Missing required parameter"); - /// assert!(error.to_string().contains("Missing required parameter")); - /// ``` - pub fn config>(msg: S) -> Self { - Self::General(msg.into()) - } - - /// Create a service error with category and descriptive message - /// - /// # Arguments - /// - /// * `category` - The error category to classify the service error - /// * `msg` - A descriptive message explaining the service issue - /// - /// # Returns - /// - /// Returns a `CommonError::General` variant with the categorized service message - /// - /// # Examples - /// - /// ```rust - /// use ml::{CommonError, ErrorCategory}; - /// - /// let error = CommonError::service(ErrorCategory::System, "Database connection failed"); - /// assert!(error.to_string().contains("System")); - /// assert!(error.to_string().contains("Database connection failed")); - /// ``` - pub fn service>(category: ErrorCategory, msg: S) -> Self { - Self::General(format!("{:?}: {}", category, msg.into())) - } -} - -// Re-export canonical ErrorCategory from common crate (24 variants) -pub use ::common::error::ErrorCategory; - // Re-export Adam optimizer (moved from inline to optimizers/adam.rs) pub use optimizers::Adam; -// Now using real types from common crate - -// Core ML types -/// Represents a financial trade for ML model training and analysis -/// -/// This structure contains the essential information about a trade that is used -/// by ML models for pattern recognition, market analysis, and strategy optimization. -/// The structure is optimized for both in-memory processing and database storage. -/// -/// # Examples -/// -/// ```rust -/// use ml::Trade; -/// use rust_decimal::Decimal; -/// -/// let trade = Trade { -/// symbol: "AAPL".to_owned(), -/// price: Decimal::new(15000, 2), // $150.00 -/// quantity: Decimal::new(100, 0), // 100 shares -/// timestamp: 1640995200000000, // microseconds since epoch -/// side: "buy".to_owned(), -/// }; -/// ``` -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct Trade { - /// Trading symbol (e.g., "AAPL", "MSFT") - pub symbol: String, - /// Trade price in decimal format for precision - pub price: Decimal, - /// Trade quantity in decimal format for precision - pub quantity: Decimal, - /// Timestamp in microseconds since Unix epoch - pub timestamp: u64, - /// Trade side: "buy" or "sell" - pub side: String, -} - -/// Health status for ensemble models and ML system components -/// -/// This enum tracks the operational status of ML models and system components, -/// enabling automated health monitoring, alerting, and failover mechanisms. -/// Health status is crucial for maintaining system reliability in production. -/// -/// # Examples -/// -/// ```rust -/// use ml::HealthStatus; -/// -/// let status = HealthStatus::Healthy; -/// match status { -/// HealthStatus::Healthy => println!("System operating normally"), -/// HealthStatus::Degraded => println!("Performance below optimal"), -/// HealthStatus::Unhealthy => println!("System requires intervention"), -/// } -/// ``` -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -pub enum HealthStatus { - /// Component is operating within normal parameters - Healthy, - /// Component is operational but performance is below optimal - Degraded, - /// Component is not functioning properly and requires intervention - Unhealthy, -} - -// Using Decimal for financial types - -/// Market data snapshot for ML model input -/// -/// Represents a point-in-time snapshot of market data that serves as input -/// for ML models. This structure contains the essential market information -/// needed for real-time trading decisions and model inference. -/// -/// # Examples -/// -/// ```rust -/// use ml::MarketDataSnapshot; -/// use rust_decimal::Decimal; -/// use chrono::Utc; -/// -/// let snapshot = MarketDataSnapshot { -/// timestamp: Utc::now(), -/// symbol: "AAPL".to_owned(), -/// price: Decimal::new(15000, 2), // $150.00 -/// volume: Decimal::new(1000000, 0), // 1M shares -/// }; -/// ``` -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct MarketDataSnapshot { - /// Timestamp of the market data snapshot - pub timestamp: DateTime, - /// Trading symbol (e.g., "AAPL", "MSFT") - pub symbol: String, - /// Current market price - pub price: Decimal, - /// Trading volume at this timestamp - pub volume: Decimal, -} - -/// Feature vector for ML model input -/// -/// A wrapper around a vector of f64 values that represents extracted features -/// for machine learning models. Features are numerical representations of -/// market data, indicators, and other relevant information. -/// -/// # Examples -/// -/// ```rust -/// use ml::FeatureVector; -/// -/// let features = FeatureVector(vec![1.0, 2.5, -0.3, 4.2]); -/// assert_eq!(features.0.len(), 4); -/// ``` -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct FeatureVector(pub Vec); - -impl FeatureVector { - /// Get the length of the feature vector - pub fn len(&self) -> usize { - self.0.len() - } - - /// Check if the feature vector is empty - pub fn is_empty(&self) -> bool { - self.0.is_empty() - } -} - -/// Integer tensor for discrete ML model operations -/// -/// A wrapper around a vector of i64 values used for discrete operations -/// such as classification labels, indices, and categorical data in ML models. -/// -/// # Examples -/// -/// ```rust -/// use ml::IntegerTensor; -/// -/// let tensor = IntegerTensor(vec![0, 1, 2, 1, 0]); -/// assert_eq!(tensor.0.len(), 5); -/// ``` -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct IntegerTensor(pub Vec); - -/// Summary of model update operations -/// -/// Provides information about batch update operations on ML models, -/// including success counts and overall statistics. Used for monitoring -/// and logging model maintenance operations. -/// -/// # Examples -/// -/// ```rust -/// use ml::UpdateSummary; -/// -/// let summary = UpdateSummary { -/// updated_models: 5, -/// total_models: 10, -/// }; -/// -/// let success_rate = summary.updated_models as f64 / summary.total_models as f64; -/// println!("Update success rate: {:.1}%", success_rate * 100.0); -/// ``` -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct UpdateSummary { - /// Number of models successfully updated - pub updated_models: usize, - /// Total number of models in the update operation - pub total_models: usize, -} - -use thiserror::Error; - -/// Machine Learning specific errors -#[derive(Debug, Clone, Error, Serialize, Deserialize)] -pub enum MLError { - /// Configuration error - #[error("Configuration error: {reason}")] - ConfigError { reason: String }, - - /// Configuration error (alternative naming) - #[error("Configuration error: {0}")] - ConfigurationError(String), - - /// Dimension mismatch error - #[error("Dimension mismatch: expected {expected}, got {actual}")] - DimensionMismatch { expected: usize, actual: usize }, - - /// Graph-related error - #[error("Graph error: {message}")] - GraphError { message: String }, - - /// Resource limit exceeded - #[error("Resource limit exceeded: {resource} limit {limit}")] - ResourceLimit { resource: String, limit: usize }, - - /// Serialization error - #[error("Serialization error: {reason}")] - SerializationError { reason: String }, - - /// Validation error - #[error("Validation error: {message}")] - ValidationError { message: String }, - - /// Concurrency error - #[error("Concurrency error in operation: {operation}")] - ConcurrencyError { operation: String }, - - /// Invalid input error - #[error("Invalid input: {0}")] - InvalidInput(String), - - /// Initialization error - #[error("Initialization error in {component}: {message}")] - InitializationError { component: String, message: String }, - - /// Training error - #[error("Training error: {0}")] - TrainingError(String), - - /// Inference error - #[error("Inference error: {0}")] - InferenceError(String), - - /// Model error - #[error("Model error: {0}")] - ModelError(String), - - /// Model not trained error - #[error("Model not trained: {0}")] - NotTrained(String), - - /// Anyhow error wrapping - #[error("General error: {0}")] - AnyhowError(String), - - /// Tensor creation error - #[error("Tensor creation error in {operation}: {reason}")] - TensorCreationError { operation: String, reason: String }, - - /// Tensor operation error - #[error("Tensor operation error: {0}")] - TensorOperationError(String), - - /// Lock error - #[error("Lock error: {0}")] - LockError(String), - - /// Model not found error - #[error("Model not found: {0}")] - ModelNotFound(String), - - /// Insufficient data error - #[error("Insufficient data: {0}")] - InsufficientData(String), - - /// Checkpoint error - #[error("Checkpoint error: {0}")] - CheckpointError(String), - - /// Device error (GPU/CUDA unavailable) - #[error("Device error: {0}")] - DeviceError(String), -} - -// Implement From trait for candle_core::Error -impl From for MLError { - fn from(err: candle_core::Error) -> Self { - MLError::ModelError(format!("Candle error: {}", err)) - } -} - -// Implement From trait for LabelingError -impl From for MLError { - fn from(err: labeling::gpu_acceleration::LabelingError) -> Self { - MLError::InferenceError(err.to_string()) - } -} - -// Implement From trait for anyhow::Error -impl From for MLError { - fn from(err: anyhow::Error) -> Self { - MLError::AnyhowError(err.to_string()) - } -} - -// Implement From trait for std::io::Error -impl From for MLError { - fn from(err: std::io::Error) -> Self { - MLError::ModelError(format!("IO error: {}", err)) - } -} - -// UNIFIED ERROR HANDLING: Convert all ML errors to CommonError for workspace consistency -impl From for CommonError { - fn from(err: MLError) -> Self { - match err { - MLError::ConfigError { reason } => { - CommonError::config(format!("ML configuration error: {}", reason)) - }, - MLError::ConfigurationError(msg) => { - CommonError::config(format!("ML configuration error: {}", msg)) - }, - MLError::InitializationError { component, message } => CommonError::service( - ErrorCategory::System, - format!("ML initialization error in {}: {}", component, message), - ), - MLError::DimensionMismatch { expected, actual } => CommonError::validation(format!( - "ML dimension mismatch: expected {}, got {}", - expected, actual - )), - MLError::GraphError { message } => CommonError::service( - ErrorCategory::System, - format!("ML graph error: {}", message), - ), - MLError::ResourceLimit { resource, limit } => CommonError::service( - ErrorCategory::System, - format!("ML resource limit exceeded: {} limit {}", resource, limit), - ), - MLError::SerializationError { reason } => CommonError::service( - ErrorCategory::System, - format!("ML serialization error: {}", reason), - ), - MLError::ValidationError { message } => { - CommonError::validation(format!("ML validation error: {}", message)) - }, - MLError::ConcurrencyError { operation } => CommonError::service( - ErrorCategory::System, - format!("ML concurrency error in operation: {}", operation), - ), - MLError::InvalidInput(msg) => { - CommonError::validation(format!("ML invalid input: {}", msg)) - }, - MLError::TrainingError(msg) => { - CommonError::service(ErrorCategory::System, format!("ML training error: {}", msg)) - }, - MLError::InferenceError(msg) => CommonError::service( - ErrorCategory::System, - format!("ML inference error: {}", msg), - ), - MLError::ModelError(msg) => { - CommonError::service(ErrorCategory::System, format!("ML model error: {}", msg)) - }, - MLError::CheckpointError(msg) => CommonError::service( - ErrorCategory::System, - format!("ML checkpoint error: {}", msg), - ), - MLError::NotTrained(msg) => CommonError::service( - ErrorCategory::System, - format!("ML model not trained: {}", msg), - ), - MLError::AnyhowError(msg) => { - CommonError::service(ErrorCategory::System, format!("ML error: {}", msg)) - }, - MLError::TensorCreationError { operation, reason } => CommonError::service( - ErrorCategory::System, - format!("ML tensor creation error in {}: {}", operation, reason), - ), - MLError::TensorOperationError(msg) => CommonError::service( - ErrorCategory::System, - format!("ML tensor operation error: {}", msg), - ), - MLError::LockError(msg) => { - CommonError::service(ErrorCategory::System, format!("ML lock error: {}", msg)) - }, - MLError::ModelNotFound(msg) => CommonError::service( - ErrorCategory::System, - format!("ML model not found: {}", msg), - ), - MLError::InsufficientData(msg) => { - CommonError::validation(format!("ML insufficient data: {}", msg)) - }, - MLError::DeviceError(msg) => CommonError::service( - ErrorCategory::System, - format!("ML device error: {}", msg), - ), - } - } -} - -// Convert common type errors to MLError -impl From for MLError { - fn from(err: CommonTypeError) -> Self { - MLError::ModelError(format!("Common type error: {}", err)) - } -} - -impl From for MLError { - fn from(err: serde_json::Error) -> Self { - MLError::SerializationError { - reason: err.to_string(), - } - } -} - -impl From for MLError { - fn from(err: inference::RealInferenceError) -> Self { - match err { - inference::RealInferenceError::GpuRequired { reason } => { - MLError::ModelError(format!("GPU required: {}", reason)) - }, - inference::RealInferenceError::ComputationFailed { reason } => { - MLError::InferenceError(reason) - }, - inference::RealInferenceError::FeatureMismatch { expected, actual } => { - MLError::DimensionMismatch { expected, actual } - }, - inference::RealInferenceError::PredictionValidation { reason } => { - MLError::ValidationError { message: reason } - }, - inference::RealInferenceError::HardwareError { reason } => { - MLError::ModelError(format!("Hardware error: {}", reason)) - }, - other @ inference::RealInferenceError::ModelNotLoaded { .. } - | other @ inference::RealInferenceError::ArchitectureError { .. } - | other @ inference::RealInferenceError::TimeoutExceeded { .. } - | other @ inference::RealInferenceError::ModelDrift { .. } => { - MLError::InferenceError(other.to_string()) - }, - } - } -} - -// Implement From for MLError -impl From for MLError { - fn from(err: training_pipeline::ProductionTrainingError) -> Self { - match err { - training_pipeline::ProductionTrainingError::ConfigError { reason } => { - MLError::ConfigError { reason } - }, - training_pipeline::ProductionTrainingError::ArchitectureError { reason } => { - MLError::ModelError(format!("Architecture error: {}", reason)) - }, - training_pipeline::ProductionTrainingError::DataError { reason } => { - MLError::ValidationError { - message: format!("Data error: {}", reason), - } - }, - training_pipeline::ProductionTrainingError::OptimizationError { reason } => { - MLError::TrainingError(format!("Optimization error: {}", reason)) - }, - training_pipeline::ProductionTrainingError::FinancialError { reason } => { - MLError::ValidationError { - message: format!("Financial error: {}", reason), - } - }, - training_pipeline::ProductionTrainingError::SafetyViolation { reason } => { - MLError::ValidationError { - message: format!("Safety violation: {}", reason), - } - }, - training_pipeline::ProductionTrainingError::ConvergenceError { reason } => { - MLError::TrainingError(format!("Convergence error: {}", reason)) - }, - training_pipeline::ProductionTrainingError::ResourceError { reason } => { - MLError::ModelError(format!("Resource error: {}", reason)) - }, - training_pipeline::ProductionTrainingError::GpuRequired { reason } => { - MLError::ModelError(format!("GPU required: {}", reason)) - }, - } - } -} - -// Note: From trait for liquid::LiquidError is implemented in the liquid module to avoid conflicts - -/// Result type for ML operations -pub type MLResult = Result; - -/// New unified result type using CommonError for better integration -pub type UnifiedMLResult = Result; - -/// Precision factor for fixed-point arithmetic -pub const PRECISION_FACTOR: i64 = 100_000_000; - -/// Maximum inference latency target in microseconds -pub const MAX_INFERENCE_LATENCY_US: u64 = 100; - // ========== SHARED INFRASTRUCTURE (extracted from dqn/) ========== pub mod action_space; // Factored action space (5 DQN exposure actions + deterministic order routing) pub mod mixed_precision; // AMP utilities for 2x speedup (FP16/BF16) @@ -835,73 +222,6 @@ pub mod types; // Canonical shared types (OHLCVBar, etc.) pub mod transformers; pub mod universe; -// ============================================================================ -// MANDATORY CUDA TRAINING DEVICE -// ============================================================================ -// -// ALL ML training requires CUDA GPU acceleration. CPU fallback wastes time. -// -// This module provides a fail-fast device initialization function that: -// 1. Requires CUDA GPU (no silent CPU fallback) -// 2. Provides helpful error messages for common setup issues -// 3. Ensures consistent device initialization across all trainers -// -// Training scripts MUST use this function instead of Device::cuda_if_available() - -/// Get mandatory CUDA device for training -/// -/// # Errors -/// -/// Returns `MLError::DeviceError` if CUDA GPU is not available -/// -/// # Example -/// -/// ```no_run -/// use ml::get_training_device; -/// -/// let device = get_training_device().expect("GPU required"); -/// ``` -pub fn get_training_device() -> Result { - match candle_core::Device::new_cuda(0) { - Ok(device) => Ok(device), - Err(e) => { - tracing::error!( - "CUDA GPU not available: {}. \ - Troubleshooting: (1) nvidia-smi, (2) nvcc --version, \ - (3) check LD_LIBRARY_PATH, (4) cargo build --features cuda, \ - (5) check CUDA_HOME", - e - ); - Err(MLError::DeviceError(format!( - "CUDA GPU required but unavailable: {}", - e - ))) - } - } -} - -/// Get CUDA device with index (for multi-GPU setups) -/// -/// # Errors -/// -/// Returns `MLError::DeviceError` if CUDA GPU at specified index is not available -pub fn get_training_device_at(device_id: usize) -> Result { - match candle_core::Device::new_cuda(device_id) { - Ok(device) => Ok(device), - Err(e) => { - tracing::error!( - "CUDA GPU {} not available: {}. Check available GPUs with: nvidia-smi", - device_id, - e - ); - Err(MLError::DeviceError(format!( - "CUDA GPU {} required but unavailable: {}", - device_id, e - ))) - } - } -} - // ========== INFRASTRUCTURE MODULES ========== // Infrastructure pub mod benchmark; @@ -945,1137 +265,104 @@ pub mod explainability; pub mod model_registry; pub mod data_pipeline; pub mod asset_selection; -pub mod registry; // Operational maturity: model lifecycle (Candidate → Staging → Production → Archived) +pub mod registry; // Operational maturity: model lifecycle (Candidate -> Staging -> Production -> Archived) -// ========== MISSING TYPES STUBS ========== +// ========== FROM IMPLS FOR MODULE-LOCAL ERROR TYPES ========== +// These reference modules that remain in ml (not moved to ml-core) -/// Application result wrapper for ML operations -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct MLAppResult { - pub data: T, - pub success: bool, - pub message: Option, - pub execution_time_ms: u64, - pub metadata: HashMap, -} - -impl MLAppResult { - /// Create a successful result - pub fn success(data: T) -> Self { - Self { - data, - success: true, - message: None, - execution_time_ms: 0, - metadata: HashMap::new(), - } - } - - /// Create a failed result with message - pub fn error(data: T, message: String) -> Self { - Self { - data, - success: false, - message: Some(message), - execution_time_ms: 0, - metadata: HashMap::new(), - } - } - - /// Set execution time - pub fn with_timing(mut self, execution_time_ms: u64) -> Self { - self.execution_time_ms = execution_time_ms; - self - } - - /// Add metadata - pub fn with_metadata(mut self, key: String, value: String) -> Self { - self.metadata.insert(key, value); - self +// Implement From trait for LabelingError +impl From for MLError { + fn from(err: labeling::gpu_acceleration::LabelingError) -> Self { + MLError::InferenceError(err.to_string()) } } -/// Performance profile configuration for HFT models -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct HFTPerformanceProfile { - pub max_latency_us: u64, - pub target_throughput: u32, - pub memory_limit_mb: u64, - pub cpu_affinity: Option>, - pub gpu_enabled: bool, - pub batch_size: u32, - pub optimization_level: OptimizationLevel, -} - -/// Optimization levels for HFT performance -#[derive(Debug, Clone, Copy, Serialize, Deserialize)] -pub enum OptimizationLevel { - /// Maximum speed, minimal safety checks - UltraLow, - /// Balanced speed and safety - Low, - /// Standard optimization - Medium, - /// Conservative with full validation - High, -} - -impl Default for HFTPerformanceProfile { - fn default() -> Self { - Self { - max_latency_us: 100, // 100 microseconds target - target_throughput: 10000, // 10k operations per second - memory_limit_mb: 1024, // 1GB memory limit - cpu_affinity: None, - gpu_enabled: false, - batch_size: 1, - optimization_level: OptimizationLevel::Medium, +impl From for MLError { + fn from(err: inference::RealInferenceError) -> Self { + match err { + inference::RealInferenceError::GpuRequired { reason } => { + MLError::ModelError(format!("GPU required: {}", reason)) + }, + inference::RealInferenceError::ComputationFailed { reason } => { + MLError::InferenceError(reason) + }, + inference::RealInferenceError::FeatureMismatch { expected, actual } => { + MLError::DimensionMismatch { expected, actual } + }, + inference::RealInferenceError::PredictionValidation { reason } => { + MLError::ValidationError { message: reason } + }, + inference::RealInferenceError::HardwareError { reason } => { + MLError::ModelError(format!("Hardware error: {}", reason)) + }, + other @ inference::RealInferenceError::ModelNotLoaded { .. } + | other @ inference::RealInferenceError::ArchitectureError { .. } + | other @ inference::RealInferenceError::TimeoutExceeded { .. } + | other @ inference::RealInferenceError::ModelDrift { .. } => { + MLError::InferenceError(other.to_string()) + }, } } } -/// Create HFT performance profile with default settings -pub fn create_hft_performance_profile() -> HFTPerformanceProfile { - HFTPerformanceProfile::default() -} - -/// Create HFT performance profile with custom latency target -pub fn create_hft_performance_profile_with_latency(max_latency_us: u64) -> HFTPerformanceProfile { - HFTPerformanceProfile { - max_latency_us, - ..Default::default() - } -} - -/// Create HFT performance profile optimized for ultra-low latency -pub fn create_ultra_low_latency_profile() -> HFTPerformanceProfile { - HFTPerformanceProfile { - max_latency_us: 10, // 10 microseconds target - target_throughput: 50000, // 50k operations per second - memory_limit_mb: 512, // Reduced memory for cache efficiency - gpu_enabled: true, // Enable GPU acceleration - batch_size: 1, // No batching for minimal latency - optimization_level: OptimizationLevel::UltraLow, - ..Default::default() - } -} - -// ========== UNIFIED ML MODEL INTERFACE ========== - -use async_trait::async_trait; -use chrono::{DateTime, Utc}; -use futures::future::join_all; -use std::collections::HashMap; -use std::sync::Arc; -use tokio::sync::RwLock; - -/// Features vector for ML model input -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct Features { - /// Raw feature values - pub values: Vec, - /// Feature names for debugging - pub names: Vec, - /// Timestamp of features - pub timestamp: u64, - /// Symbol these features are for - pub symbol: Option, -} - -impl Features { - pub fn new(values: Vec, names: Vec) -> Self { - Self { - values, - names, - timestamp: std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_micros() as u64, - symbol: None, - } - } - - /// Set the symbol for this market data point - /// - /// # Arguments - /// * `symbol` - The trading symbol (e.g., "AAPL", "MSFT") - /// - /// # Returns - /// Modified MarketData instance with symbol set - pub fn with_symbol(mut self, symbol: String) -> Self { - self.symbol = Some(symbol); - self - } -} - -/// Model prediction result -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ModelPrediction { - /// Predicted value (price direction, probability, etc.) - pub value: f64, - /// Model confidence (0.0 to 1.0) - pub confidence: f64, - /// Additional model-specific metadata - pub metadata: HashMap, - /// Prediction timestamp - pub timestamp: u64, - /// Model identifier - pub model_id: String, -} - -impl ModelPrediction { - pub fn new(model_id: String, value: f64, confidence: f64) -> Self { - Self { - value, - confidence, - metadata: HashMap::new(), - timestamp: std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_micros() as u64, - model_id, - } - } - - /// Add metadata to the model prediction - /// - /// # Arguments - /// * `key` - Metadata key identifier - /// * `value` - JSON value containing metadata - /// - /// # Returns - /// Modified ModelPrediction with additional metadata - pub fn with_metadata(mut self, key: String, value: serde_json::Value) -> Self { - self.metadata.insert(key, value); - self - } -} - -/// Feedback for model weight updates -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct Feedback { - /// Actual outcome (for supervised learning) - pub actual_value: Option, - /// Reward signal (for reinforcement learning) - pub reward: Option, - /// Trading performance metrics - pub performance_metrics: HashMap, - /// Timestamp of feedback - pub timestamp: u64, -} - -impl Feedback { - pub fn new() -> Self { - Self { - actual_value: None, - reward: None, - performance_metrics: HashMap::new(), - timestamp: std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_micros() as u64, - } - } - - /// Set the actual outcome value for supervised learning feedback - /// - /// # Arguments - /// * `actual` - The actual observed value - /// - /// # Returns - /// Modified Feedback with actual value set - pub fn with_actual(mut self, actual: f64) -> Self { - self.actual_value = Some(actual); - self - } - - /// Set the reward signal for reinforcement learning feedback - /// - /// # Arguments - /// * `reward` - The reward value (positive for good outcomes, negative for bad) - /// - /// # Returns - /// Modified Feedback with reward signal set - pub fn with_reward(mut self, reward: f64) -> Self { - self.reward = Some(reward); - self - } -} - -/// Unified interface for all ML models in the system -#[async_trait] -pub trait MLModel: Send + Sync + std::fmt::Debug { - /// Get unique model identifier - fn name(&self) -> &str; - - /// Get model type - fn model_type(&self) -> ModelType; - - /// Make prediction based on features - async fn predict(&self, features: &Features) -> MLResult; - - /// Get current model confidence score (0.0 to 1.0) - fn get_confidence(&self) -> f64; - - /// Update model weights based on feedback (optional - not all models support online learning) - async fn update_weights(&mut self, _feedback: &Feedback) -> MLResult<()> { - // Default implementation does nothing (for immutable models) - Ok(()) - } - - /// Check if model is ready for predictions - fn is_ready(&self) -> bool { - true // Default to ready - } - - /// Get model metadata - fn get_metadata(&self) -> ModelMetadata; - - /// Validate input features - fn validate_features(&self, features: &Features) -> MLResult<()> { - // Default validation - check for empty features - if features.values.is_empty() { - return Err(MLError::ValidationError { - message: "Empty feature vector".to_owned(), - }); - } - Ok(()) - } -} - -/// Thread-safe model registry using DashMap for high-performance concurrent access -pub struct ModelRegistry { - /// Models stored by name - models: dashmap::DashMap>, - /// Registry metadata - metadata: Arc>, -} - -impl std::fmt::Debug for ModelRegistry { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("ModelRegistry") - .field( - "models", - &format_args!("", self.models.len()), - ) - .field("metadata", &self.metadata) - .finish() - } -} - -#[derive(Debug, Clone)] -struct RegistryMetadata { - created_at: std::time::SystemTime, - total_registrations: u64, - last_access: std::time::SystemTime, -} - -impl ModelRegistry { - /// Create new model registry - pub fn new() -> Self { - Self { - models: dashmap::DashMap::new(), - metadata: Arc::new(RwLock::new(RegistryMetadata { - created_at: std::time::SystemTime::now(), - total_registrations: 0, - last_access: std::time::SystemTime::now(), - })), - } - } - - /// Register a model in the registry - pub async fn register(&self, model: Arc) -> MLResult<()> { - let name = model.name().to_string(); - - // Check if model is ready - if !model.is_ready() { - return Err(MLError::ModelError(format!("Model {} is not ready", name))); - } - - self.models.insert(name.clone(), model); - - // Update metadata - { - let mut meta = self.metadata.write().await; - meta.total_registrations += 1; - meta.last_access = std::time::SystemTime::now(); - } - - tracing::info!("Registered ML model: {}", name); - Ok(()) - } - - /// Get model by name - pub async fn get(&self, name: &str) -> Option> { - // Update last access time - { - let mut meta = self.metadata.write().await; - meta.last_access = std::time::SystemTime::now(); - } - - self.models.get(name).map(|entry| entry.value().clone()) - } - - /// Get all registered models - pub fn get_all(&self) -> Vec> { - self.models - .iter() - .map(|entry| entry.value().clone()) - .collect() - } - - /// Get model names - pub fn get_model_names(&self) -> Vec { - self.models - .iter() - .map(|entry| entry.key().clone()) - .collect() - } - - /// Remove model from registry - pub async fn remove(&self, name: &str) -> Option> { - let result = self.models.remove(name).map(|(_, model)| model); - - if result.is_some() { - tracing::info!("Removed ML model: {}", name); - } - - result - } - - /// Get registry statistics - pub async fn get_stats(&self) -> RegistryStats { - let meta = self.metadata.read().await; - RegistryStats { - total_models: self.models.len(), - total_registrations: meta.total_registrations, - created_at: meta.created_at, - last_access: meta.last_access, - } - } - - /// Parallel prediction across all models - pub async fn predict_all(&self, features: &Features) -> Vec> { - let models = self.get_all(); - let futures = models.iter().map(|model| { - let features = features.clone(); - async move { model.predict(&features).await } - }); - - join_all(futures).await - } - - /// Parallel prediction across specific models - pub async fn predict_selected( - &self, - model_names: &[String], - features: &Features, - ) -> Vec> { - let futures = model_names.iter().map(|name| { - let name = name.clone(); - let features = features.clone(); - async move { - if let Some(model) = self.get(&name).await { - model.predict(&features).await - } else { - Err(MLError::ModelNotFound(name)) +// Implement From for MLError +impl From for MLError { + fn from(err: training_pipeline::ProductionTrainingError) -> Self { + match err { + training_pipeline::ProductionTrainingError::ConfigError { reason } => { + MLError::ConfigError(reason) + }, + training_pipeline::ProductionTrainingError::ArchitectureError { reason } => { + MLError::ModelError(format!("Architecture error: {}", reason)) + }, + training_pipeline::ProductionTrainingError::DataError { reason } => { + MLError::ValidationError { + message: format!("Data error: {}", reason), } - } - }); - - join_all(futures).await - } -} - -impl Default for ModelRegistry { - fn default() -> Self { - Self::new() - } -} - -/// Registry statistics -#[derive(Debug, Clone)] -pub struct RegistryStats { - pub total_models: usize, - pub total_registrations: u64, - pub created_at: std::time::SystemTime, - pub last_access: std::time::SystemTime, -} - -/// Global model registry instance (singleton pattern) -static GLOBAL_REGISTRY: once_cell::sync::Lazy> = - once_cell::sync::Lazy::new(|| Arc::new(ModelRegistry::new())); - -/// Get global model registry -pub fn get_global_registry() -> Arc { - GLOBAL_REGISTRY.clone() -} - -// ========== PARALLEL EXECUTION OPTIMIZATIONS ========== - -/// High-performance parallel executor for ML models optimized for sub-50μs latency -#[derive(Debug)] -pub struct ParallelExecutor { - /// Performance profile - profile: HFTPerformanceProfile, - /// CPU affinity settings - cpu_affinity: Option>, - /// Thread pool for CPU-bound operations - cpu_pool: Arc, - /// Async runtime handle - runtime_handle: tokio::runtime::Handle, -} - -impl ParallelExecutor { - /// Create new parallel executor with HFT performance profile - pub fn new(profile: HFTPerformanceProfile) -> Result { - // Create dedicated thread pool based on profile - let cpu_pool = rayon::ThreadPoolBuilder::new() - .num_threads( - profile - .cpu_affinity - .as_ref() - .map(|v| v.len()) - .unwrap_or(num_cpus::get()), - ) - .thread_name(|i| format!("ml-cpu-{}", i)) - .build() - .map_err(|e| MLError::ModelError(format!("Failed to create thread pool: {}", e)))?; - - let runtime_handle = tokio::runtime::Handle::try_current() - .map_err(|e| MLError::ModelError(format!("No tokio runtime available: {}", e)))?; - - let cpu_affinity = profile.cpu_affinity.clone(); - - Ok(Self { - profile, - cpu_affinity, - cpu_pool: Arc::new(cpu_pool), - runtime_handle, - }) - } - - /// Execute parallel predictions with latency optimization - pub async fn execute_parallel_predictions( - &self, - models: Vec>, - features: Features, - ) -> Vec> { - let start_time = std::time::Instant::now(); - - // Determine execution strategy based on performance profile - let results = match self.profile.optimization_level { - OptimizationLevel::UltraLow => { - // Ultra-low latency: parallel execution with minimal overhead - self.execute_ultra_low_latency(models, features).await }, - OptimizationLevel::Low => { - // Low latency: parallel with basic batching - self.execute_low_latency(models, features).await + training_pipeline::ProductionTrainingError::OptimizationError { reason } => { + MLError::TrainingError(format!("Optimization error: {}", reason)) }, - OptimizationLevel::Medium => { - // Medium: balanced parallel execution - self.execute_balanced(models, features).await - }, - OptimizationLevel::High => { - // High: conservative with full validation - self.execute_conservative(models, features).await - }, - }; - - let execution_time = start_time.elapsed(); - - // Log performance if exceeding target latency - if execution_time.as_micros() > self.profile.max_latency_us as u128 { - tracing::warn!( - "Parallel execution exceeded target latency: {}μs > {}μs", - execution_time.as_micros(), - self.profile.max_latency_us - ); - } - - results - } - - /// Ultra-low latency execution (<10μs target) - async fn execute_ultra_low_latency( - &self, - models: Vec>, - features: Features, - ) -> Vec> { - // Use futures::future::join_all for minimal overhead - let futures = models.into_iter().map(|model| { - let features = features.clone(); - async move { model.predict(&features).await } - }); - - join_all(futures).await - } - - /// Low latency execution with basic optimizations - async fn execute_low_latency( - &self, - models: Vec>, - features: Features, - ) -> Vec> { - // Group models by type for potential batching - let mut model_groups: HashMap>> = HashMap::new(); - - for model in models { - let model_type = model.model_type(); - model_groups.entry(model_type).or_default().push(model); - } - - let mut all_futures = Vec::new(); - - for (_, group_models) in model_groups { - for model in group_models { - let features = features.clone(); - all_futures.push(async move { model.predict(&features).await }); - } - } - - join_all(all_futures).await - } - - /// Balanced execution with moderate optimizations - async fn execute_balanced( - &self, - models: Vec>, - features: Features, - ) -> Vec> { - // Validate features once for all models - for model in &models { - if let Err(e) = model.validate_features(&features) { - tracing::debug!( - "Feature validation failed for model {}: {}", - model.name(), - e - ); - } - } - - let futures = models.into_iter().map(|model| { - let features = features.clone(); - async move { - if model.is_ready() { - model.predict(&features).await - } else { - Err(MLError::ModelError(format!( - "Model {} not ready", - model.name() - ))) + training_pipeline::ProductionTrainingError::FinancialError { reason } => { + MLError::ValidationError { + message: format!("Financial error: {}", reason), } - } - }); - - join_all(futures).await - } - - /// Conservative execution with full validation - async fn execute_conservative( - &self, - models: Vec>, - features: Features, - ) -> Vec> { - let mut results = Vec::new(); - - for model in models { - // Comprehensive validation - if !model.is_ready() { - results.push(Err(MLError::ModelError(format!( - "Model {} not ready", - model.name() - )))); - continue; - } - - if let Err(e) = model.validate_features(&features) { - results.push(Err(e)); - continue; - } - - // Execute with timeout - let prediction_future = model.predict(&features); - let timeout_duration = std::time::Duration::from_micros(self.profile.max_latency_us); - - match tokio::time::timeout(timeout_duration, prediction_future).await { - Ok(result) => results.push(result), - Err(_) => results.push(Err(MLError::ModelError(format!( - "Model {} prediction timed out after {}μs", - model.name(), - self.profile.max_latency_us - )))), - } - } - - results - } - - /// Get execution statistics - pub fn get_stats(&self) -> ExecutorStats { - ExecutorStats { - optimization_level: self.profile.optimization_level, - target_latency_us: self.profile.max_latency_us, - cpu_threads: self.cpu_pool.current_num_threads(), - cpu_affinity: self.cpu_affinity.clone(), + }, + training_pipeline::ProductionTrainingError::SafetyViolation { reason } => { + MLError::ValidationError { + message: format!("Safety violation: {}", reason), + } + }, + training_pipeline::ProductionTrainingError::ConvergenceError { reason } => { + MLError::TrainingError(format!("Convergence error: {}", reason)) + }, + training_pipeline::ProductionTrainingError::ResourceError { reason } => { + MLError::ModelError(format!("Resource error: {}", reason)) + }, + training_pipeline::ProductionTrainingError::GpuRequired { reason } => { + MLError::ModelError(format!("GPU required: {}", reason)) + }, } } } -/// Executor performance statistics -#[derive(Debug, Clone)] -pub struct ExecutorStats { - pub optimization_level: OptimizationLevel, - pub target_latency_us: u64, - pub cpu_threads: usize, - pub cpu_affinity: Option>, -} - -/// Latency optimizer for ML inference pipelines -#[derive(Debug)] -pub struct LatencyOptimizer { - /// Target latency in microseconds - target_latency_us: u64, - /// Performance history - performance_history: Arc>>, - /// Optimization parameters - optimization_params: OptimizationParams, -} - -#[derive(Debug, Clone)] -struct PerformancePoint { - timestamp: std::time::Instant, - latency_us: u64, - model_count: usize, - batch_size: u32, - success: bool, -} - -#[derive(Debug, Clone)] -struct OptimizationParams { - max_batch_size: u32, - adaptive_batching: bool, - prefetch_enabled: bool, - cache_predictions: bool, -} - -impl Default for OptimizationParams { - fn default() -> Self { - Self { - max_batch_size: 8, - adaptive_batching: true, - prefetch_enabled: true, - cache_predictions: false, // Disabled for real-time trading - } - } -} - -impl LatencyOptimizer { - /// Create new latency optimizer - pub fn new(target_latency_us: u64) -> Self { - Self { - target_latency_us, - performance_history: Arc::new(RwLock::new(Vec::new())), - optimization_params: OptimizationParams::default(), - } - } - - /// Record performance measurement - pub async fn record_performance( - &self, - latency_us: u64, - model_count: usize, - batch_size: u32, - success: bool, - ) { - let point = PerformancePoint { - timestamp: std::time::Instant::now(), - latency_us, - model_count, - batch_size, - success, - }; - - { - let mut history = self.performance_history.write().await; - history.push(point); - - // Keep only recent history (last 1000 measurements) - if history.len() > 1000 { - let excess = history.len() - 1000; - history.drain(0..excess); - } - } - } - - /// Get optimization recommendations - pub async fn get_recommendations(&self) -> OptimizationRecommendations { - let history = self.performance_history.read().await; - - if history.is_empty() { - return OptimizationRecommendations::default(); - } - - let recent_points: Vec<&PerformancePoint> = history.iter().rev().take(100).collect(); - - let avg_latency = - recent_points.iter().map(|p| p.latency_us).sum::() / recent_points.len() as u64; - - let success_rate = - recent_points.iter().filter(|p| p.success).count() as f64 / recent_points.len() as f64; - - OptimizationRecommendations { - current_avg_latency_us: avg_latency, - target_latency_us: self.target_latency_us, - success_rate, - meets_target: avg_latency <= self.target_latency_us, - recommended_batch_size: self.calculate_optimal_batch_size(&recent_points), - recommended_model_limit: self.calculate_optimal_model_limit(&recent_points), - } - } - - fn calculate_optimal_batch_size(&self, points: &[&PerformancePoint]) -> u32 { - // Simple heuristic: find batch size with best latency/success ratio - let mut batch_performance: HashMap = HashMap::new(); - - for point in points { - let entry = batch_performance - .entry(point.batch_size) - .or_insert((0, 0.0)); - entry.0 += point.latency_us; - entry.1 += if point.success { 1.0 } else { 0.0 }; - } - - batch_performance - .into_iter() - .filter(|(_, (_, success_count))| *success_count > 0.0) - .min_by_key(|(_, (latency, success_count))| { - // Optimize for latency with success rate weighting - ((*latency as f64) / success_count) as u64 - }) - .map(|(batch_size, _)| batch_size) - .unwrap_or(1) - } - - fn calculate_optimal_model_limit(&self, points: &[&PerformancePoint]) -> usize { - // Find the sweet spot where adding more models doesn't improve latency - let mut model_performance: HashMap = HashMap::new(); - - for point in points { - if point.success { - let entry = model_performance.entry(point.model_count).or_insert(0); - *entry += point.latency_us; - } - } - - model_performance - .into_iter() - .filter(|(_, avg_latency)| *avg_latency <= self.target_latency_us) - .max_by_key(|(model_count, _)| *model_count) - .map(|(model_count, _)| model_count) - .unwrap_or(1) - } -} - -/// Optimization recommendations from latency analysis -#[derive(Debug, Clone)] -pub struct OptimizationRecommendations { - pub current_avg_latency_us: u64, - pub target_latency_us: u64, - pub success_rate: f64, - pub meets_target: bool, - pub recommended_batch_size: u32, - pub recommended_model_limit: usize, -} - -impl Default for OptimizationRecommendations { - fn default() -> Self { - Self { - current_avg_latency_us: 0, - target_latency_us: 50, - success_rate: 0.0, - meets_target: false, - recommended_batch_size: 1, - recommended_model_limit: 1, - } - } -} - -/// Create optimized parallel executor for HFT scenarios -pub fn create_hft_parallel_executor() -> Result { - let profile = create_ultra_low_latency_profile(); - ParallelExecutor::new(profile) -} - -/// Create latency optimizer with HFT targets -pub fn create_hft_latency_optimizer() -> LatencyOptimizer { - LatencyOptimizer::new(50) // 50 microsecond target -} - -// ========== CANONICAL TRAINING AND VALIDATION METRICS ========== -// These are the unified types that all ML modules must use to prevent type conflicts - -/// Canonical training metrics used throughout ML module -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct TrainingMetrics { - /// Training loss value - pub loss: f64, - /// Training accuracy (0.0 to 1.0) - pub accuracy: f64, - /// Training precision (0.0 to 1.0) - pub precision: f64, - /// Training recall (0.0 to 1.0) - pub recall: f64, - /// Training F1 score (0.0 to 1.0) - pub f1_score: f64, - /// Total training time in seconds - pub training_time_seconds: f64, - /// Number of epochs trained - pub epochs_trained: u32, - /// Whether convergence was achieved - pub convergence_achieved: bool, - /// Additional model-specific metrics - pub additional_metrics: HashMap, -} - -impl TrainingMetrics { - /// Create new training metrics - pub fn new() -> Self { - Self { - loss: 0.0, - accuracy: 0.0, - precision: 0.0, - recall: 0.0, - f1_score: 0.0, - training_time_seconds: 0.0, - epochs_trained: 0, - convergence_achieved: false, - additional_metrics: HashMap::new(), - } - } - - /// Add an additional metric - pub fn add_metric(&mut self, name: &str, value: f64) { - self.additional_metrics.insert(name.to_string(), value); - } - - /// Check if training was successful - pub fn is_successful(&self) -> bool { - self.convergence_achieved && self.accuracy > 0.5 - } -} - -impl Default for TrainingMetrics { - fn default() -> Self { - Self::new() - } -} - -/// Canonical validation metrics used throughout ML module -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ValidationMetrics { - /// Validation loss value - pub validation_loss: f64, - /// Validation accuracy (0.0 to 1.0) - pub validation_accuracy: f64, - /// Validation precision (0.0 to 1.0) - pub validation_precision: f64, - /// Validation recall (0.0 to 1.0) - pub validation_recall: f64, - /// Validation F1 score (0.0 to 1.0) - pub validation_f1_score: f64, - /// Number of samples validated - pub samples_validated: usize, - /// Additional model-specific validation metrics - pub additional_metrics: HashMap, -} - -impl ValidationMetrics { - /// Create new validation metrics - pub fn new() -> Self { - Self { - validation_loss: 0.0, - validation_accuracy: 0.0, - validation_precision: 0.0, - validation_recall: 0.0, - validation_f1_score: 0.0, - samples_validated: 0, - additional_metrics: HashMap::new(), - } - } - - /// Add an additional validation metric - pub fn add_metric(&mut self, name: &str, value: f64) { - self.additional_metrics.insert(name.to_string(), value); - } - - /// Check if validation was successful - pub fn is_successful(&self) -> bool { - self.validation_accuracy > 0.5 && self.samples_validated > 0 - } -} - -impl Default for ValidationMetrics { - fn default() -> Self { - Self::new() - } -} - -// Public exports moved to end of file after all type definitions - -// ========== CANONICAL ML TYPES ========== -// These are the unified types that all ML modules must use to prevent type conflicts - -/// Canonical inference result used throughout ML module -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct InferenceResult { - /// Model identifier - pub model_id: String, - /// Prediction value (primary prediction) - pub prediction_value: f64, - /// Confidence score (0.0 to 1.0) - pub confidence: f64, - /// Latency in microseconds - pub latency_us: u64, - /// Timestamp in microseconds since UNIX epoch - pub timestamp: u64, - /// Model metadata - pub metadata: ModelMetadata, -} - -impl InferenceResult { - /// Create new inference result - pub fn new( - model_id: String, - prediction_value: f64, - confidence: f64, - latency_us: u64, - timestamp: u64, - metadata: ModelMetadata, - ) -> Self { - Self { - model_id, - prediction_value, - confidence, - latency_us, - timestamp, - metadata, - } - } - - /// Extract prediction as float value - pub fn prediction_as_float(&self) -> f64 { - self.prediction_value - } - - /// Get the model identifier - pub fn model_id(&self) -> &str { - &self.model_id - } -} - -/// Canonical model metadata used throughout ML module -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ModelMetadata { - /// Type of the model - pub model_type: ModelType, - /// Model version - pub version: String, - /// Number of features used for inference - pub features_used: usize, - /// Memory usage in megabytes - pub memory_usage_mb: f64, - /// Additional metadata key-value pairs - pub additional_metadata: HashMap, -} - -impl ModelMetadata { - /// Create new model metadata - pub fn new( - model_type: ModelType, - version: String, - features_used: usize, - memory_usage_mb: f64, - ) -> Self { - Self { - model_type, - version, - features_used, - memory_usage_mb, - additional_metadata: HashMap::new(), - } - } - - /// Add additional metadata - pub fn add_metadata(&mut self, key: &str, value: String) { - self.additional_metadata.insert(key.to_string(), value); - } - - /// Mark the model as trained - pub fn mark_trained(&mut self) { - self.add_metadata("training_status", "trained".to_owned()); - self.add_metadata( - "training_timestamp", - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_secs() - .to_string(), - ); - } -} - -// Re-export canonical ModelType from common crate -pub use ::common::model_types::ModelType; +// Note: From trait for liquid::LiquidError is implemented in the liquid module to avoid conflicts +// ========== ML PRELUDE (extends ml-core prelude with ml-specific items) ========== /// Prelude module for convenient imports of commonly used ML types /// /// This module re-exports the most commonly used types and traits from the ML crate /// to allow users to import everything they need with a single `use ml::prelude::*;` pub mod prelude { - // Core ML types - pub use crate::{ - CommonError, CommonTypeError, ErrorCategory, FeatureVector, Features, Feedback, - HealthStatus, InferenceResult, IntegerTensor, MarketDataSnapshot, MarketRegime, - ModelMetadata, ModelPrediction, ModelType, Trade, TrainingMetrics, UpdateSummary, - ValidationMetrics, - }; + // Re-export everything from ml-core's prelude + pub use ml_core::prelude::*; - // Error types - pub use crate::{MLError, MLResult, UnifiedMLResult}; - - // ML Model trait - pub use crate::MLModel; - - // Model registry - pub use crate::{get_global_registry, ModelRegistry, RegistryStats}; - - // Performance types - pub use crate::{ - create_hft_latency_optimizer, create_hft_parallel_executor, create_hft_performance_profile, - create_hft_performance_profile_with_latency, create_ultra_low_latency_profile, - ExecutorStats, HFTPerformanceProfile, LatencyOptimizer, OptimizationLevel, - OptimizationRecommendations, ParallelExecutor, - }; - - // Constants - pub use crate::{MAX_INFERENCE_LATENCY_US, PRECISION_FACTOR}; - - // GPU device management + // GPU device management (ml-specific) pub use crate::gpu::{capabilities::GpuCapabilities, DeviceConfig}; - // Tensor types from candle - pub use candle_core::{Device, Tensor}; - pub use candle_nn::{Module, VarBuilder, VarMap}; - - // Common external types - pub use rust_decimal::Decimal; - pub use serde::{Deserialize, Serialize}; - - // Data pipeline + // Data pipeline (ml-specific) pub use crate::data_pipeline::{DatasetManager, DatasetMode, DatasetSpec, PreparedDataset}; - // Asset selection + // Asset selection (ml-specific) pub use crate::asset_selection::{ActiveSetSelector, AssetUniverse, PredictabilityScorer}; } diff --git a/crates/ml/src/liquid/candle_cfc.rs b/crates/ml/src/liquid/candle_cfc.rs index 9c8a6754d..049d57a27 100644 --- a/crates/ml/src/liquid/candle_cfc.rs +++ b/crates/ml/src/liquid/candle_cfc.rs @@ -101,7 +101,7 @@ impl BackboneMLP { } let last_hidden = hidden_sizes.last().copied().ok_or_else(|| { - MLError::ConfigurationError("backbone_hidden_sizes cannot be empty".to_owned()) + MLError::ConfigError("backbone_hidden_sizes cannot be empty".to_owned()) })?; let f_head = candle_nn::linear(last_hidden, last_hidden, vb.pp("f_head")) @@ -170,9 +170,9 @@ impl CfCCell { let backbone = BackboneMLP::new(concat_dim, &config.backbone_hidden_sizes, vb)?; let last_backbone = config.backbone_hidden_sizes.last().copied() - .ok_or_else(|| MLError::ConfigurationError("backbone_hidden_sizes cannot be empty".to_owned()))?; + .ok_or_else(|| MLError::ConfigError("backbone_hidden_sizes cannot be empty".to_owned()))?; if last_backbone != config.hidden_size { - return Err(MLError::ConfigurationError(format!( + return Err(MLError::ConfigError(format!( "Last backbone hidden size ({}) must equal hidden_size ({})", last_backbone, config.hidden_size ))); @@ -247,14 +247,10 @@ pub struct CandleCfCNetwork { impl CandleCfCNetwork { pub fn new(config: &CfCTrainConfig, vb: &VarBuilder<'_>) -> Result { if config.input_size == 0 { - return Err(MLError::ConfigError { - reason: "Liquid CfC requires input_size > 0".to_owned(), - }); + return Err(MLError::ConfigError("Liquid CfC requires input_size > 0".to_owned())); } if config.hidden_size == 0 { - return Err(MLError::ConfigError { - reason: "Liquid CfC requires hidden_size > 0".to_owned(), - }); + return Err(MLError::ConfigError("Liquid CfC requires hidden_size > 0".to_owned())); } let cell = CfCCell::new(config, &vb.pp("cfc"))?; diff --git a/crates/ml/src/liquid/mod.rs b/crates/ml/src/liquid/mod.rs index 097b03e2b..15e6bd0e3 100644 --- a/crates/ml/src/liquid/mod.rs +++ b/crates/ml/src/liquid/mod.rs @@ -145,7 +145,7 @@ impl Error for LiquidError {} impl From for MLError { fn from(err: LiquidError) -> Self { match err { - LiquidError::InvalidConfiguration(msg) => MLError::ConfigurationError(msg), + LiquidError::InvalidConfiguration(msg) => MLError::ConfigError(msg), LiquidError::InvalidInput(msg) => MLError::InvalidInput(msg), LiquidError::InferenceError(msg) => MLError::InferenceError(msg), LiquidError::TrainingError(msg) => MLError::TrainingError(msg), @@ -159,7 +159,7 @@ impl From for MLError { impl From for LiquidError { fn from(err: MLError) -> Self { match err { - MLError::ConfigurationError(msg) | MLError::ConfigError { reason: msg } => { + MLError::ConfigError(msg) => { LiquidError::InvalidConfiguration(msg) }, MLError::InvalidInput(msg) => LiquidError::InvalidInput(msg), diff --git a/crates/ml/src/mamba/mod.rs b/crates/ml/src/mamba/mod.rs index 8d2949ba2..0cad442e0 100644 --- a/crates/ml/src/mamba/mod.rs +++ b/crates/ml/src/mamba/mod.rs @@ -632,9 +632,7 @@ impl Mamba2SSM { /// - SSD layer initialization fails pub fn new(config: Mamba2Config, device: &Device) -> Result { if config.d_model == 0 { - return Err(MLError::ConfigError { - reason: "Mamba2 requires d_model > 0".to_owned(), - }); + return Err(MLError::ConfigError("Mamba2 requires d_model > 0".to_owned())); } let vs = Arc::new(candle_nn::VarMap::new()); @@ -1300,7 +1298,7 @@ impl Mamba2SSM { }; let path_str = checkpoint_path.to_str().ok_or_else(|| { - MLError::ConfigError { reason: "Checkpoint path contains invalid UTF-8".to_owned() } + MLError::ConfigError("Checkpoint path contains invalid UTF-8".to_owned()) })?; self.save_checkpoint(path_str).await?; info!( @@ -1484,7 +1482,7 @@ impl Mamba2SSM { }; let path_str = checkpoint_path.to_str().ok_or_else(|| { - MLError::ConfigError { reason: "Checkpoint path contains invalid UTF-8".to_owned() } + MLError::ConfigError("Checkpoint path contains invalid UTF-8".to_owned()) })?; self.save_checkpoint(path_str).await?; info!( diff --git a/crates/ml/src/memory_optimization/auto_batch_size.rs b/crates/ml/src/memory_optimization/auto_batch_size.rs index d35ac7078..401164fde 100644 --- a/crates/ml/src/memory_optimization/auto_batch_size.rs +++ b/crates/ml/src/memory_optimization/auto_batch_size.rs @@ -251,12 +251,10 @@ impl AutoBatchSizer { // Check if we have enough memory for minimum batch size let total_overhead_mb = fixed_overhead_mb + batch_overhead_mb; if usable_memory_mb < total_overhead_mb { - return Err(MLError::ConfigError { - reason: format!( + return Err(MLError::ConfigError(format!( "Insufficient GPU memory: {:.1}MB available, {:.1}MB required (model: {:.1}MB + batch overhead: {:.1}MB). Consider enabling gradient_checkpointing, using INT8 quantization, or using a larger GPU.", usable_memory_mb, total_overhead_mb, fixed_overhead_mb, batch_overhead_mb - ) - }); + ))); } // Calculate available memory for batch data (after fixed overhead + batch overhead) @@ -450,29 +448,21 @@ pub fn detect_gpu_memory() -> MLResult<(f64, f64, String)> { parts[0] .trim() .parse::() - .map_err(|e| MLError::ConfigError { - reason: format!("Failed to parse total memory: {}", e), - })?; + .map_err(|e| MLError::ConfigError(format!("Failed to parse total memory: {}", e)))?; let free_mb = parts[1] .trim() .parse::() - .map_err(|e| MLError::ConfigError { - reason: format!("Failed to parse free memory: {}", e), - })?; + .map_err(|e| MLError::ConfigError(format!("Failed to parse free memory: {}", e)))?; let device_name = parts[2].trim().to_string(); return Ok((total_mb, free_mb, device_name)); } - Err(MLError::ConfigError { - reason: format!("Unexpected nvidia-smi output format: {}", stdout), - }) + Err(MLError::ConfigError(format!("Unexpected nvidia-smi output format: {}", stdout))) }, Ok(output) => { let stderr = String::from_utf8_lossy(&output.stderr); - Err(MLError::ConfigError { - reason: format!("nvidia-smi failed: {}", stderr), - }) + Err(MLError::ConfigError(format!("nvidia-smi failed: {}", stderr))) }, Err(e) => { // nvidia-smi not available, return conservative defaults diff --git a/crates/ml/src/observability/metrics.rs b/crates/ml/src/observability/metrics.rs index 4bfd1d9c3..9343f3fc8 100644 --- a/crates/ml/src/observability/metrics.rs +++ b/crates/ml/src/observability/metrics.rs @@ -394,10 +394,9 @@ impl MLMetricsCollector { MLError::ValidationError { .. } => "validation", MLError::InferenceError(..) => "inference", MLError::ModelError(..) => "model", - MLError::ConfigError { .. } => "config", + MLError::ConfigError(..) => "config", MLError::TensorCreationError { .. } => "tensor", - MLError::ConfigurationError(_) - | MLError::DimensionMismatch { .. } + MLError::DimensionMismatch { .. } | MLError::GraphError { .. } | MLError::ResourceLimit { .. } | MLError::SerializationError { .. } diff --git a/crates/ml/src/ppo/adaptive_entropy.rs b/crates/ml/src/ppo/adaptive_entropy.rs index a45e7ec6c..6623e4138 100644 --- a/crates/ml/src/ppo/adaptive_entropy.rs +++ b/crates/ml/src/ppo/adaptive_entropy.rs @@ -100,17 +100,13 @@ impl AdaptiveEntropyCoeff { /// Returns [`MLError`] if the initial alpha is not positive or parameter creation fails. pub fn new(config: &AdaptiveEntropyConfig, device: &Device) -> Result { if config.initial_alpha <= 0.0 { - return Err(MLError::ConfigError { - reason: format!( + return Err(MLError::ConfigError(format!( "initial_alpha must be positive, got {}", config.initial_alpha - ), - }); + ))); } if config.num_actions == 0 { - return Err(MLError::ConfigError { - reason: "num_actions must be > 0".to_owned(), - }); + return Err(MLError::ConfigError("num_actions must be > 0".to_owned())); } // Discrete SAC convention: positive target entropy @@ -285,9 +281,7 @@ impl AdaptiveEntropyCoeff { })?; let log_alpha_var = binding .get("log_alpha") - .ok_or_else(|| MLError::ConfigError { - reason: "log_alpha parameter not found in VarMap".to_owned(), - })?; + .ok_or_else(|| MLError::ConfigError("log_alpha parameter not found in VarMap".to_owned()))?; let tensor = log_alpha_var.as_tensor().clone(); drop(binding); Ok(tensor) diff --git a/crates/ml/src/ppo/ppo.rs b/crates/ml/src/ppo/ppo.rs index dd612f021..66724fdd0 100644 --- a/crates/ml/src/ppo/ppo.rs +++ b/crates/ml/src/ppo/ppo.rs @@ -795,14 +795,10 @@ impl PPO { /// 4. `HiddenStateManager` is initialized automatically pub fn with_device(config: PPOConfig, device: Device) -> Result { if config.state_dim == 0 { - return Err(MLError::ConfigError { - reason: "PPO requires state_dim > 0".to_owned(), - }); + return Err(MLError::ConfigError("PPO requires state_dim > 0".to_owned())); } if config.num_actions == 0 { - return Err(MLError::ConfigError { - reason: "PPO requires num_actions > 0".to_owned(), - }); + return Err(MLError::ConfigError("PPO requires num_actions > 0".to_owned())); } // Extract values before moving config @@ -1420,7 +1416,7 @@ impl PPO { // Extract LSTM networks (we already validated both are LSTM in match) let (actor_lstm, critic_lstm) = match (&self.actor, &self.critic) { (ActorNetwork::LSTM(actor), CriticNetwork::LSTM(critic)) => (actor, critic), - _ => return Err(MLError::ConfigError { reason: "Expected both actor and critic to be LSTM".to_owned() }), + _ => return Err(MLError::ConfigError("Expected both actor and critic to be LSTM".to_owned())), }; // Track mean log probability for adaptive entropy update diff --git a/crates/ml/src/regime_detection/hmm.rs b/crates/ml/src/regime_detection/hmm.rs index 90d9a469a..6edfee5a4 100644 --- a/crates/ml/src/regime_detection/hmm.rs +++ b/crates/ml/src/regime_detection/hmm.rs @@ -92,12 +92,10 @@ impl HiddenMarkovModel { let t = observations.len(); if t < n { - return Err(MLError::ConfigError { - reason: format!( + return Err(MLError::ConfigError(format!( "Need at least {} observations for {} states, got {}", n, n, t - ), - }); + ))); } // -- K-means initialization for emission parameters -- diff --git a/crates/ml/src/tests/ml_tests.rs b/crates/ml/src/tests/ml_tests.rs index bf954fe3f..49842c9ad 100644 --- a/crates/ml/src/tests/ml_tests.rs +++ b/crates/ml/src/tests/ml_tests.rs @@ -19,9 +19,7 @@ mod comprehensive_ml_tests { #[test] fn test_ml_error_creation_and_formatting() { - let config_error = MLError::ConfigError { - reason: "Invalid parameter".to_owned() - }; + let config_error = MLError::ConfigError("Invalid parameter".to_owned()); assert_eq!(config_error.to_string(), "Configuration error: Invalid parameter"); let dimension_error = MLError::DimensionMismatch { diff --git a/crates/ml/src/tft/hft_optimizations.rs b/crates/ml/src/tft/hft_optimizations.rs index 1a5399e2c..5377ea113 100644 --- a/crates/ml/src/tft/hft_optimizations.rs +++ b/crates/ml/src/tft/hft_optimizations.rs @@ -536,7 +536,7 @@ impl HFTOptimizedTFT { .num_threads(config.max_threads) .build() .map_err(|e| { - MLError::ConfigurationError(format!("Failed to create thread pool: {}", e)) + MLError::ConfigError(format!("Failed to create thread pool: {}", e)) }) }) .transpose()?; diff --git a/crates/ml/src/tft/mod.rs b/crates/ml/src/tft/mod.rs index 6ef663f4e..04c9903d6 100644 --- a/crates/ml/src/tft/mod.rs +++ b/crates/ml/src/tft/mod.rs @@ -198,7 +198,7 @@ impl TFTState { pub fn zeros(_config: &TFTConfig) -> Result { // MAX_CACHE_ENTRIES (2000) is non-zero by construction let capacity = NonZeroUsize::new(Self::MAX_CACHE_ENTRIES) - .ok_or_else(|| MLError::ConfigError { reason: "MAX_CACHE_ENTRIES must be non-zero".to_owned() })?; + .ok_or_else(|| MLError::ConfigError("MAX_CACHE_ENTRIES must be non-zero".to_owned()))?; Ok(Self { hidden_state: None, @@ -311,21 +311,17 @@ impl TemporalFusionTransformer { let total_features = config.num_static_features + config.num_known_features + config.num_unknown_features; if config.num_unknown_features == 0 { - return Err(MLError::ConfigError { - reason: "TFT requires num_unknown_features > 0 (temporal input)".to_owned(), - }); + return Err(MLError::ConfigError("TFT requires num_unknown_features > 0 (temporal input)".to_owned())); } if total_features != config.input_dim { - return Err(MLError::ConfigError { - reason: format!( + return Err(MLError::ConfigError(format!( "Feature count mismatch: static({}) + known({}) + unknown({}) = {} != input_dim({})", config.num_static_features, config.num_known_features, config.num_unknown_features, total_features, config.input_dim - ) - }); + ))); } // Log configuration for debugging diff --git a/crates/ml/src/tft/temporal_attention.rs b/crates/ml/src/tft/temporal_attention.rs index f1878defd..725cb0331 100644 --- a/crates/ml/src/tft/temporal_attention.rs +++ b/crates/ml/src/tft/temporal_attention.rs @@ -280,7 +280,7 @@ impl TemporalSelfAttention { // Ensure hidden_dim is divisible by num_heads if hidden_dim % num_heads != 0 { - return Err(MLError::ConfigurationError(format!( + return Err(MLError::ConfigError(format!( "Hidden dimension {} must be divisible by number of heads {}", hidden_dim, num_heads ))); diff --git a/crates/ml/src/tft/training.rs b/crates/ml/src/tft/training.rs index 96704dc6e..9e1a4c8d3 100644 --- a/crates/ml/src/tft/training.rs +++ b/crates/ml/src/tft/training.rs @@ -236,9 +236,7 @@ impl TFTDataLoader { /// This method is primarily used for dynamic OOM recovery scenarios. pub fn update_batch_size(&mut self, new_batch_size: usize) -> Result<(), MLError> { if new_batch_size < 1 { - return Err(MLError::ConfigError { - reason: format!("Batch size must be >= 1, got {}", new_batch_size), - }); + return Err(MLError::ConfigError(format!("Batch size must be >= 1, got {}", new_batch_size))); } self.batch_size = new_batch_size; diff --git a/crates/ml/src/tgnn/gating.rs b/crates/ml/src/tgnn/gating.rs index 3dca69a82..bb0b3a499 100644 --- a/crates/ml/src/tgnn/gating.rs +++ b/crates/ml/src/tgnn/gating.rs @@ -532,12 +532,10 @@ impl MultiHeadGating { /// Create new multi-head gating mechanism pub fn new(hidden_dim: usize, num_heads: usize) -> Result { if hidden_dim % num_heads != 0 { - return Err(MLError::ConfigError { - reason: format!( + return Err(MLError::ConfigError(format!( "Hidden dimension {} must be divisible by number of heads {}", hidden_dim, num_heads - ), - }); + ))); } let head_dim = hidden_dim / num_heads; diff --git a/crates/ml/src/tgnn/mod.rs b/crates/ml/src/tgnn/mod.rs index b58487cc7..ad2ebf141 100644 --- a/crates/ml/src/tgnn/mod.rs +++ b/crates/ml/src/tgnn/mod.rs @@ -1021,9 +1021,7 @@ impl MLModel for TGGN { } fn set_config(&mut self, config: Self::Config) -> Result<(), MLError> { - self.config = serde_json::from_value(config).map_err(|e| MLError::ConfigError { - reason: e.to_string(), - })?; + self.config = serde_json::from_value(config).map_err(|e| MLError::ConfigError(e.to_string()))?; Ok(()) } } diff --git a/crates/ml/src/tgnn/trainable_adapter.rs b/crates/ml/src/tgnn/trainable_adapter.rs index 4f92f2b30..b375e7200 100644 --- a/crates/ml/src/tgnn/trainable_adapter.rs +++ b/crates/ml/src/tgnn/trainable_adapter.rs @@ -74,14 +74,10 @@ impl TGGNTrainableAdapter { /// Initialized adapter ready for training pub fn new(config: TGGNConfig, device: &Device) -> Result { if config.node_dim == 0 { - return Err(MLError::ConfigError { - reason: "TGGN requires node_dim > 0".to_owned(), - }); + return Err(MLError::ConfigError("TGGN requires node_dim > 0".to_owned())); } if config.hidden_dim == 0 { - return Err(MLError::ConfigError { - reason: "TGGN requires hidden_dim > 0".to_owned(), - }); + return Err(MLError::ConfigError("TGGN requires hidden_dim > 0".to_owned())); } let var_map = VarMap::new(); diff --git a/crates/ml/src/tlob/trainable_adapter.rs b/crates/ml/src/tlob/trainable_adapter.rs index e801b6666..2bc69c230 100644 --- a/crates/ml/src/tlob/trainable_adapter.rs +++ b/crates/ml/src/tlob/trainable_adapter.rs @@ -105,12 +105,10 @@ impl TLOBTrainableAdapter { /// Initialized adapter ready for training pub fn new(config: TLOBAdapterConfig, device: &Device) -> Result { if config.seq_len == 0 || config.feature_dim == 0 { - return Err(MLError::ConfigError { - reason: format!( + return Err(MLError::ConfigError(format!( "TLOB requires seq_len > 0 and feature_dim > 0 (got {}x{})", config.seq_len, config.feature_dim - ), - }); + ))); } let var_map = VarMap::new(); diff --git a/crates/ml/src/trainers/liquid.rs b/crates/ml/src/trainers/liquid.rs index 427314c39..71df0ef98 100644 --- a/crates/ml/src/trainers/liquid.rs +++ b/crates/ml/src/trainers/liquid.rs @@ -370,18 +370,14 @@ impl LiquidTrainer { // Create checkpoint directory if it doesn't exist if let Some(parent) = checkpoint_path.parent() { - std::fs::create_dir_all(parent).map_err(|e| MLError::ConfigError { - reason: format!("Failed to create checkpoint directory: {}", e), - })?; + std::fs::create_dir_all(parent).map_err(|e| MLError::ConfigError(format!("Failed to create checkpoint directory: {}", e)))?; } // Save VarMap to safetensors self.adapter .varmap() .save(&checkpoint_path) - .map_err(|e| MLError::ConfigError { - reason: format!("Failed to save Liquid CfC checkpoint: {}", e), - })?; + .map_err(|e| MLError::ConfigError(format!("Failed to save Liquid CfC checkpoint: {}", e)))?; let file_size = std::fs::metadata(&checkpoint_path) .map(|m| m.len() / 1024) diff --git a/crates/ml/src/trainers/ppo.rs b/crates/ml/src/trainers/ppo.rs index 7c54a974b..8fa0b8aaa 100644 --- a/crates/ml/src/trainers/ppo.rs +++ b/crates/ml/src/trainers/ppo.rs @@ -1505,9 +1505,7 @@ impl PpoTrainer { if let Some(parent) = checkpoint_path.parent() { tokio::fs::create_dir_all(parent) .await - .map_err(|e| MLError::ConfigError { - reason: format!("Failed to create checkpoint directory: {}", e), - })?; + .map_err(|e| MLError::ConfigError(format!("Failed to create checkpoint directory: {}", e)))?; } // Serialize both actor and critic networks to SafeTensors format @@ -1521,9 +1519,7 @@ impl PpoTrainer { .actor .vars() .save(&actor_path) - .map_err(|e| MLError::ConfigError { - reason: format!("Failed to save actor network: {}", e), - })?; + .map_err(|e| MLError::ConfigError(format!("Failed to save actor network: {}", e)))?; // Save critic (value) network let critic_path = self @@ -1533,23 +1529,17 @@ impl PpoTrainer { .critic .vars() .save(&critic_path) - .map_err(|e| MLError::ConfigError { - reason: format!("Failed to save critic network: {}", e), - })?; + .map_err(|e| MLError::ConfigError(format!("Failed to save critic network: {}", e)))?; // Verify checkpoint files exist and have reasonable sizes let actor_metadata = tokio::fs::metadata(&actor_path) .await - .map_err(|e| MLError::ConfigError { - reason: format!("Failed to verify actor checkpoint: {}", e), - })?; + .map_err(|e| MLError::ConfigError(format!("Failed to verify actor checkpoint: {}", e)))?; let critic_metadata = tokio::fs::metadata(&critic_path) .await - .map_err(|e| MLError::ConfigError { - reason: format!("Failed to verify critic checkpoint: {}", e), - })?; + .map_err(|e| MLError::ConfigError(format!("Failed to verify critic checkpoint: {}", e)))?; let actor_size_kb = actor_metadata.len() / 1024; let critic_size_kb = critic_metadata.len() / 1024; @@ -1570,9 +1560,7 @@ impl PpoTrainer { ); tokio::fs::write(&checkpoint_path, metadata.as_bytes()) .await - .map_err(|e| MLError::ConfigError { - reason: format!("Failed to save checkpoint metadata: {}", e), - })?; + .map_err(|e| MLError::ConfigError(format!("Failed to save checkpoint metadata: {}", e)))?; debug!("Checkpoint metadata saved to {:?}", checkpoint_path); Ok(()) diff --git a/crates/ml/src/trainers/tft/tests.rs b/crates/ml/src/trainers/tft/tests.rs index 5950c6d9d..f1cba19f5 100644 --- a/crates/ml/src/trainers/tft/tests.rs +++ b/crates/ml/src/trainers/tft/tests.rs @@ -241,9 +241,7 @@ async fn test_is_oom_error() { "Should not detect regular errors" ); - let not_oom2 = MLError::ConfigError { - reason: "Missing parameter".to_owned(), - }; + let not_oom2 = MLError::ConfigError("Missing parameter".to_owned()); assert!( !TFTTrainer::is_oom_error(¬_oom2), "Should not detect config errors" diff --git a/crates/ml/src/trainers/tft/trainer.rs b/crates/ml/src/trainers/tft/trainer.rs index 1cab17bfb..c62ac908a 100644 --- a/crates/ml/src/trainers/tft/trainer.rs +++ b/crates/ml/src/trainers/tft/trainer.rs @@ -186,9 +186,7 @@ impl TFTTrainer { // Select device (GPU if available and requested) let device = if config.use_gpu { - Device::cuda_if_available(0).map_err(|e| MLError::ConfigError { - reason: format!("GPU requested but not available: {}", e), - })? + Device::cuda_if_available(0).map_err(|e| MLError::ConfigError(format!("GPU requested but not available: {}", e)))? } else { Device::Cpu }; diff --git a/crates/ml/src/training/unified_data_loader.rs b/crates/ml/src/training/unified_data_loader.rs index 84b65dae7..dc503f669 100644 --- a/crates/ml/src/training/unified_data_loader.rs +++ b/crates/ml/src/training/unified_data_loader.rs @@ -652,9 +652,7 @@ impl DatabentoHistoricalProvider { let client = reqwest::Client::builder() .timeout(Duration::from_secs(config.timeout_seconds)) .build() - .map_err(|e| MLError::ConfigError { - reason: format!("Failed to create HTTP client: {}", e), - })?; + .map_err(|e| MLError::ConfigError(format!("Failed to create HTTP client: {}", e)))?; Ok(Self { config, client }) } @@ -684,9 +682,7 @@ impl BenzingaHistoricalProvider { let client = reqwest::Client::builder() .timeout(Duration::from_secs(config.timeout_seconds)) .build() - .map_err(|e| MLError::ConfigError { - reason: format!("Failed to create HTTP client: {}", e), - })?; + .map_err(|e| MLError::ConfigError(format!("Failed to create HTTP client: {}", e)))?; Ok(Self { config, client }) } diff --git a/crates/ml/src/validation/ppo_adapter.rs b/crates/ml/src/validation/ppo_adapter.rs index e599503da..4b15bc761 100644 --- a/crates/ml/src/validation/ppo_adapter.rs +++ b/crates/ml/src/validation/ppo_adapter.rs @@ -220,7 +220,7 @@ impl PpoLstmStrategy { /// Create a new `PpoLstmStrategy` on a specific device. pub fn with_device(config: PPOConfig, device: Device) -> Result { if !config.use_lstm { - return Err(MLError::ConfigurationError( + return Err(MLError::ConfigError( "PpoLstmStrategy requires config.use_lstm = true".to_owned(), )); } diff --git a/crates/ml/src/xlstm/mlstm.rs b/crates/ml/src/xlstm/mlstm.rs index 11c989244..1ddd3f477 100644 --- a/crates/ml/src/xlstm/mlstm.rs +++ b/crates/ml/src/xlstm/mlstm.rs @@ -44,11 +44,9 @@ impl MLSTMCell { vb: VarBuilder<'_>, ) -> Result { if hidden_dim % num_heads != 0 { - return Err(MLError::ConfigError { - reason: format!( + return Err(MLError::ConfigError(format!( "hidden_dim ({hidden_dim}) must be divisible by num_heads ({num_heads})" - ), - }); + ))); } let head_dim = hidden_dim / num_heads; diff --git a/crates/ml/src/xlstm/network.rs b/crates/ml/src/xlstm/network.rs index 96027d117..d29c7664f 100644 --- a/crates/ml/src/xlstm/network.rs +++ b/crates/ml/src/xlstm/network.rs @@ -28,17 +28,13 @@ impl XLSTMNetwork { /// Build an xLSTM network from config. pub fn new(config: &XLSTMConfig, vb: VarBuilder<'_>) -> Result { if config.num_blocks == 0 { - return Err(MLError::ConfigError { - reason: "num_blocks must be >= 1".to_owned(), - }); + return Err(MLError::ConfigError("num_blocks must be >= 1".to_owned())); } if config.input_dim == 0 || config.hidden_dim == 0 { - return Err(MLError::ConfigError { - reason: format!( + return Err(MLError::ConfigError(format!( "xLSTM requires input_dim > 0 and hidden_dim > 0 (got {}x{})", config.input_dim, config.hidden_dim - ), - }); + ))); } let num_slstm = ((config.num_blocks as f64) * config.slstm_ratio).round() as usize; diff --git a/services/trading_service/src/ensemble_risk_manager.rs b/services/trading_service/src/ensemble_risk_manager.rs index ad13f1e5d..6fcc03464 100644 --- a/services/trading_service/src/ensemble_risk_manager.rs +++ b/services/trading_service/src/ensemble_risk_manager.rs @@ -197,7 +197,7 @@ impl EnsembleRiskManager { let circuit_breaker = RealCircuitBreaker::new(circuit_breaker_config, broker_service) .await .map_err(|e| { - MLError::ConfigurationError(format!("Circuit breaker init failed: {}", e)) + MLError::ConfigError(format!("Circuit breaker init failed: {}", e)) })?; Ok(Self {