7d1b0f232f
Move all inline type definitions from ml/src/lib.rs to ml-core:
MLError, MLResult, Trade, MarketRegime, HealthStatus, Features,
MLModel trait, ModelRegistry, ParallelExecutor, LatencyOptimizer,
TrainingMetrics, ValidationMetrics, InferenceResult, ModelMetadata.
Dedup: consolidate ConfigError{reason}/ConfigurationError(msg) into
single ConfigError(String) tuple variant (was 2 variants, 174 refs).
Cleanup: convert create_hft_* free functions to associated methods
(HFTPerformanceProfile::ultra_low_latency(), ParallelExecutor::hft()).
ml facade re-exports via `pub use ml_core::*`.
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
369 lines
19 KiB
Rust
369 lines
19 KiB
Rust
#![deny(clippy::unwrap_used, clippy::expect_used)]
|
|
#![cfg_attr(test, allow(clippy::unwrap_used, clippy::expect_used))]
|
|
#![allow(dead_code)] // 10 ML model implementations with internal architecture not yet fully wired
|
|
#![allow(missing_docs)] // Internal implementation details don't require documentation
|
|
#![allow(missing_debug_implementations)] // Not all types need Debug
|
|
#![allow(unused_crate_dependencies)] // Dev dependencies not used in lib.rs
|
|
#![allow(clippy::float_arithmetic)] // ML operations require float arithmetic
|
|
// ML-specific lint overrides: these are intentional domain patterns, not safety issues
|
|
#![allow(clippy::non_ascii_literal)] // Box-drawing chars for tables, Greek letters for math
|
|
#![allow(clippy::str_to_string)] // Pervasive in ML config/display code, not a safety concern
|
|
#![allow(clippy::partial_pub_fields)] // ML model structs mix pub config with private state
|
|
#![allow(clippy::multiple_inherent_impl)] // Split impls for readability (core vs Display vs Builder)
|
|
#![allow(clippy::same_name_method)] // Trait methods intentionally shadow inherent methods
|
|
#![allow(clippy::shadow_reuse)] // Tensor code naturally shadows: let x = relu(x)
|
|
#![allow(clippy::shadow_unrelated)] // ML pipeline variable reuse across phases
|
|
#![allow(clippy::shadow_same)] // Rebinding after narrowing in match/if-let
|
|
// ML domain pedantic lints: these are noise in numerical/ML code, not safety issues
|
|
// Individual pedantic lints are managed at workspace level in Cargo.toml
|
|
#![allow(clippy::doc_markdown)] // Technical terms (AVX2, SIMD, HFT, etc.) in doc comments
|
|
#![allow(clippy::indexing_slicing)] // Tensor/array indexing is pervasive and bounds-checked in context
|
|
#![allow(clippy::missing_const_for_fn)] // Const fn not critical for ML model code
|
|
#![allow(clippy::module_name_repetitions)] // Module-prefixed types provide clarity (DQNAgent, TFTTrainer)
|
|
#![allow(clippy::integer_division)] // Integer division is intentional in batch/epoch calculations
|
|
#![allow(clippy::cognitive_complexity)] // ML training loops and model architectures are inherently complex
|
|
#![allow(clippy::similar_names)] // ML variables often have similar names (x, xs, x_hat, x_norm)
|
|
#![allow(clippy::clone_on_ref_ptr)] // Arc::clone is intentional for shared model state
|
|
#![allow(clippy::too_many_lines)] // Complex model implementations need many lines
|
|
#![allow(clippy::as_conversions)] // Necessary for tensor dimension/type conversions
|
|
#![allow(clippy::cast_precision_loss)] // Acceptable in ML with f32/f64 conversions
|
|
#![allow(clippy::cast_possible_truncation)] // Type conversions validated in context
|
|
#![allow(clippy::default_numeric_fallback)] // Float/int literals are contextually typed in ML code
|
|
#![allow(clippy::arithmetic_side_effects)] // ML math uses checked/saturating where needed
|
|
#![allow(clippy::needless_range_loop)] // Index-based loops often clearer for tensor operations
|
|
#![allow(clippy::into_iter_on_ref)] // .iter() vs .into_iter() on refs is stylistic
|
|
#![allow(clippy::new_without_default)] // Many ML types need config params, Default is inappropriate
|
|
#![allow(clippy::manual_let_else)] // if-let pattern preferred in many ML error paths
|
|
#![allow(clippy::unnecessary_wraps)] // Result/Option wrapping needed for trait consistency
|
|
#![allow(clippy::too_many_arguments)] // ML functions often need many hyperparameters
|
|
#![allow(clippy::must_use_candidate)] // Not all ML functions need must_use
|
|
#![allow(clippy::missing_errors_doc)] // Internal ML APIs don't need full error documentation
|
|
#![allow(clippy::cast_sign_loss)] // Sign loss validated in context
|
|
#![allow(clippy::cast_possible_wrap)] // Wrap validated in context
|
|
#![allow(clippy::cast_lossless)] // as casts are intentional for ML numeric conversions
|
|
#![allow(clippy::unused_async)] // Async needed for trait implementations
|
|
#![allow(clippy::match_same_arms)] // Explicit match arms preferred for clarity in ML code
|
|
#![allow(clippy::unused_self)] // Self parameter needed for trait consistency
|
|
#![allow(clippy::map_err_ignore)] // Error conversion doesn't need original context in ML pipelines
|
|
#![allow(clippy::single_match_else)] // Explicit match preferred for clarity in ML code
|
|
#![allow(clippy::wildcard_imports)] // Prelude-style imports common in ML modules
|
|
#![allow(clippy::unnecessary_cast)] // Explicit casts for tensor dimension/type clarity
|
|
#![allow(clippy::undocumented_unsafe_blocks)] // Unsafe blocks documented at usage site
|
|
#![allow(clippy::redundant_clone)] // Clones needed for ownership in async/parallel ML pipelines
|
|
#![allow(clippy::redundant_closure)] // Explicit closures preferred for readability
|
|
#![allow(clippy::type_complexity)] // Complex types unavoidable in ML generics
|
|
#![allow(clippy::manual_clamp)] // Explicit min/max preferred in numerical code
|
|
#![allow(clippy::clone_on_copy)] // Explicit clone for clarity on Copy types
|
|
#![allow(clippy::should_implement_trait)] // Custom builder patterns don't need std traits
|
|
#![allow(clippy::derivable_impls)] // Custom Default impls with domain-specific values
|
|
#![allow(clippy::useless_conversion)] // Explicit conversions for type clarity
|
|
#![allow(clippy::get_first)] // .get(0) preferred for consistency with .get(n)
|
|
#![allow(clippy::len_zero)] // .len() == 0 preferred for readability in some contexts
|
|
#![allow(clippy::assign_op_pattern)] // Explicit assignment preferred in numerical code
|
|
#![allow(clippy::if_same_then_else)] // Intentional identical branches for documentation
|
|
#![allow(clippy::unused_enumerate_index)] // Index used in debug/logging contexts
|
|
#![allow(clippy::doc_lazy_continuation)] // Doc formatting acceptable
|
|
#![allow(clippy::doc_overindented_list_items)] // Doc formatting acceptable
|
|
#![allow(clippy::single_char_add_str)] // String building patterns
|
|
#![allow(clippy::let_and_return)] // Named return values for clarity
|
|
#![allow(clippy::useless_format)] // Explicit format for consistency
|
|
#![allow(clippy::manual_div_ceil)] // Explicit ceiling division for clarity
|
|
#![allow(clippy::io_other_error)] // IO error construction patterns
|
|
#![allow(clippy::manual_range_contains)] // Explicit range checks for clarity
|
|
#![allow(clippy::unwrap_or_default)] // Explicit unwrap_or preferred in some contexts
|
|
#![allow(clippy::used_underscore_binding)] // Underscore-prefixed bindings used intentionally
|
|
#![allow(clippy::trivially_copy_pass_by_ref)] // Pass-by-ref for trait consistency
|
|
#![allow(clippy::needless_borrows_for_generic_args)] // Explicit borrows for clarity
|
|
#![allow(clippy::needless_borrow)] // Explicit borrows for clarity
|
|
#![allow(clippy::missing_safety_doc)] // Safety documented at usage site
|
|
#![allow(clippy::module_inception)] // Module name matches parent for re-export patterns
|
|
#![allow(clippy::if_not_else)] // Negated conditions preferred in some ML logic
|
|
#![allow(clippy::empty_line_after_doc_comments)] // Doc comment formatting acceptable
|
|
#![allow(clippy::unnecessary_lazy_evaluations)] // Explicit lazy evaluation for side effects
|
|
#![allow(clippy::collapsible_if)] // Separate ifs preferred for readability
|
|
#![allow(clippy::question_mark)] // Explicit error handling preferred
|
|
#![allow(clippy::op_ref)] // Explicit ref operations for numeric types
|
|
#![allow(clippy::iter_kv_map)] // Map iteration patterns
|
|
#![allow(clippy::ptr_arg)] // Ptr args for trait consistency
|
|
#![allow(clippy::needless_question_mark)] // Explicit ? for error propagation clarity
|
|
#![allow(clippy::explicit_auto_deref)] // Explicit derefs for clarity
|
|
#![allow(clippy::bind_instead_of_map)] // Bind preferred in some combinator chains
|
|
#![allow(clippy::upper_case_acronyms)] // ML acronyms (DQN, PPO, TFT, etc.)
|
|
#![allow(clippy::large_types_passed_by_value)] // Large types passed by value for ownership
|
|
#![allow(clippy::large_enum_variant)] // Large variants unavoidable in ML model enums
|
|
#![allow(clippy::collapsible_else_if)] // Separate else-if preferred for readability
|
|
#![allow(clippy::vec_init_then_push)] // Vec init then push for conditional building
|
|
#![allow(clippy::implicit_saturating_sub)] // Explicit subtraction preferred
|
|
#![allow(clippy::missing_fields_in_debug)] // Custom Debug impls for large ML structs
|
|
#![allow(clippy::field_reassign_with_default)] // Field reassignment after Default::default()
|
|
#![allow(clippy::len_without_is_empty)] // len() without is_empty() for ML containers
|
|
#![allow(clippy::iter_nth)] // .nth() for indexed iteration
|
|
#![allow(clippy::items_after_statements)] // Items after statements for local helpers
|
|
#![allow(clippy::uninlined_format_args)] // Non-inlined format args for readability
|
|
#![allow(clippy::manual_memcpy)] // Explicit loop copy for tensor operations
|
|
#![allow(clippy::single_char_lifetime_names)] // Standard Rust lifetime conventions in ODE solvers
|
|
#![allow(clippy::same_item_push)] // Intentional repeated pushes for sequence padding
|
|
#![allow(clippy::multiple_unsafe_ops_per_block)] // SIMD/hardware ops grouped for clarity
|
|
#![allow(clippy::arc_with_non_send_sync)] // Arc used for local-thread inference contexts
|
|
#![allow(clippy::format_in_format_args)] // Nested format for dynamic message building
|
|
#![allow(clippy::inherent_to_string)] // Custom to_string for display types
|
|
#![allow(clippy::redundant_locals)] // Rebinding for clarity in model pipelines
|
|
#![allow(clippy::to_string_trait_impl)] // Direct ToString impls for versioning types
|
|
#![recursion_limit = "256"] // Required for complex TFT quantile operations
|
|
//! Machine Learning Models for Foxhunt
|
|
//!
|
|
//! This crate provides comprehensive machine learning models and algorithms
|
|
//! for the Foxhunt high-frequency trading system. All ML operations use
|
|
//! enterprise-grade safety controls to prevent system failures.
|
|
//!
|
|
//! ## Safety Features
|
|
//!
|
|
//! - **Comprehensive mathematical safety**: All operations handle NaN/Infinity gracefully
|
|
//! - **Tensor bounds checking**: Prevents buffer overflows and memory issues
|
|
//! - **Model drift detection**: Automatic monitoring of model performance degradation
|
|
//! - **Financial validation**: Ensures all predictions use unified financial types
|
|
//! - **Memory management**: Prevents OOM conditions and memory leaks
|
|
//! - **Timeout handling**: Prevents hanging operations
|
|
//!
|
|
//! ## Usage
|
|
//!
|
|
//! ```no_run
|
|
//! // ML safety manager usage example
|
|
//! // Note: This is a conceptual example - actual implementation may vary
|
|
//! use ml::safety::MLSafetyConfig;
|
|
//!
|
|
//! #[tokio::main]
|
|
//! async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
|
//! // Initialize safety with custom configuration
|
|
//! let _config = MLSafetyConfig::default();
|
|
//!
|
|
//! // ML operations would use the safety manager
|
|
//! // (actual implementation details depend on the safety module structure)
|
|
//! Ok(())
|
|
//! }
|
|
//! ```
|
|
|
|
#![warn(missing_debug_implementations)]
|
|
#![warn(rust_2018_idioms)]
|
|
// NOTE: clippy::unwrap_used, clippy::expect_used, clippy::indexing_slicing
|
|
// are governed by workspace lints at "warn" level. The remaining lints below
|
|
// are already "deny" at workspace level -- kept here for explicitness.
|
|
#![deny(clippy::panic, clippy::unimplemented, clippy::unreachable)]
|
|
|
|
// Re-export all core types from ml-core
|
|
#[allow(deprecated)]
|
|
pub use ml_core::*;
|
|
|
|
// Silence unused crate warnings for dependencies used in tests or feature-gated code
|
|
use approx as _;
|
|
use bincode as _;
|
|
use half as _;
|
|
use memmap2 as _;
|
|
use num as _;
|
|
use num_traits as _;
|
|
use semver as _;
|
|
use tempfile as _;
|
|
|
|
// Re-export Adam optimizer (moved from inline to optimizers/adam.rs)
|
|
pub use optimizers::Adam;
|
|
|
|
// ========== SHARED INFRASTRUCTURE (extracted from dqn/) ==========
|
|
pub mod action_space; // Factored action space (5 DQN exposure actions + deterministic order routing)
|
|
pub mod mixed_precision; // AMP utilities for 2x speedup (FP16/BF16)
|
|
pub mod order_router; // Deterministic order routing: exposure -> (order_type, urgency) from microstructure
|
|
pub mod portfolio_tracker; // Portfolio state tracking for training
|
|
pub mod trading_action; // TradingAction enum (Buy/Sell/Hold)
|
|
pub mod xavier_init; // Xavier/Glorot weight initialization
|
|
|
|
// ========== CORE ML MODULES ==========
|
|
// Core ML modules
|
|
pub mod backtesting; // Backtesting framework for barrier optimization
|
|
pub mod checkpoint;
|
|
pub mod config; // Configuration module for feature extraction
|
|
pub mod cuda_compat; // CUDA-compatible operations (manual sigmoid, etc.)
|
|
pub mod cuda_pipeline; // GPU data pre-upload pipeline for DQN/PPO trainers
|
|
pub mod data_loaders; // Data loaders for ML training
|
|
pub mod deployment; // Model deployment, A/B testing, versioning, hot-swap
|
|
pub mod dqn;
|
|
pub mod gradient_accumulation; // Gradient accumulation for mini-batch training
|
|
pub mod gradient_utils; // Gradient utilities (clipping, monitoring)
|
|
pub mod gpu;
|
|
pub mod diffusion;
|
|
pub mod ensemble;
|
|
pub mod evaluation; // DQN evaluation engine (backtest metrics, Sharpe ratio)
|
|
pub mod flash_attention;
|
|
pub mod hyperopt; // Bayesian hyperparameter optimization (egobox)
|
|
pub mod integration;
|
|
pub mod kan;
|
|
pub mod labeling;
|
|
pub mod liquid;
|
|
pub mod mamba;
|
|
pub mod memory_optimization; // Memory optimization utilities (lazy loading, quantization, precision)
|
|
pub mod microstructure;
|
|
pub mod optimizers;
|
|
pub mod paper_trading;
|
|
pub mod ppo;
|
|
pub mod preprocessing; // Data preprocessing (log returns, normalization, outlier clipping)
|
|
pub mod risk;
|
|
pub mod safety;
|
|
pub mod security; // ML security (prediction validation, anomaly detection)
|
|
pub mod tft;
|
|
pub mod tgnn;
|
|
pub mod tlob;
|
|
pub mod xlstm;
|
|
|
|
// Re-export quantized TFT types (Wave 9.12)
|
|
pub use tft::{
|
|
QuantizedGatedResidualNetwork, QuantizedLSTMEncoder, QuantizedTemporalAttention,
|
|
QuantizedTemporalFusionTransformer, QuantizedVariableSelectionNetwork,
|
|
};
|
|
pub mod trainers; // ML model trainers with gRPC integration
|
|
pub mod types; // Canonical shared types (OHLCVBar, etc.)
|
|
pub mod transformers;
|
|
pub mod universe;
|
|
|
|
// ========== INFRASTRUCTURE MODULES ==========
|
|
// Infrastructure
|
|
pub mod benchmark;
|
|
pub mod benchmarks;
|
|
pub mod common;
|
|
pub mod metrics; // Performance metrics (Sharpe ratio, etc.)
|
|
pub mod training;
|
|
|
|
// ========== MODEL DEPLOYMENT AND FACTORY ==========
|
|
pub mod model_factory;
|
|
|
|
// ========== CORE EXPORTS ==========
|
|
// Core exports
|
|
pub mod error;
|
|
pub mod features; // Feature cache and extraction (Parquet + MinIO)
|
|
pub mod feature_cache; // MBP-10 OFI feature caching for hyperopt speedup
|
|
|
|
pub mod inference;
|
|
pub mod model;
|
|
pub mod performance;
|
|
pub mod validation;
|
|
|
|
// ========== ADDITIONAL MODULES ==========
|
|
// Additional ML processing modules
|
|
pub mod batch_processing; // Batch processing for ML operations
|
|
pub mod bridge; // Type system bridge for ML-Financial integration
|
|
pub mod portfolio_transformer; // Portfolio-specific transformer
|
|
pub mod regime; // Wave D: Structural breaks and regime classification
|
|
pub mod regime_detection; // Market regime detection
|
|
pub mod tensor_ops;
|
|
pub mod examples;
|
|
pub mod observability;
|
|
pub mod stress_testing; // Stress testing framework
|
|
pub mod training_pipeline; // Complete training pipeline system
|
|
pub mod traits; // Common traits for ML models // Production observability and monitoring // Integration with model_loader crate
|
|
|
|
pub mod data_loader;
|
|
pub mod walk_forward;
|
|
pub mod data_validation;
|
|
pub mod explainability;
|
|
pub mod model_registry;
|
|
pub mod data_pipeline;
|
|
pub mod asset_selection;
|
|
pub mod registry; // Operational maturity: model lifecycle (Candidate -> Staging -> Production -> Archived)
|
|
|
|
// ========== FROM IMPLS FOR MODULE-LOCAL ERROR TYPES ==========
|
|
// These reference modules that remain in ml (not moved to ml-core)
|
|
|
|
// Implement From trait for LabelingError
|
|
impl From<labeling::gpu_acceleration::LabelingError> for MLError {
|
|
fn from(err: labeling::gpu_acceleration::LabelingError) -> Self {
|
|
MLError::InferenceError(err.to_string())
|
|
}
|
|
}
|
|
|
|
impl From<inference::RealInferenceError> for MLError {
|
|
fn from(err: inference::RealInferenceError) -> Self {
|
|
match err {
|
|
inference::RealInferenceError::GpuRequired { reason } => {
|
|
MLError::ModelError(format!("GPU required: {}", reason))
|
|
},
|
|
inference::RealInferenceError::ComputationFailed { reason } => {
|
|
MLError::InferenceError(reason)
|
|
},
|
|
inference::RealInferenceError::FeatureMismatch { expected, actual } => {
|
|
MLError::DimensionMismatch { expected, actual }
|
|
},
|
|
inference::RealInferenceError::PredictionValidation { reason } => {
|
|
MLError::ValidationError { message: reason }
|
|
},
|
|
inference::RealInferenceError::HardwareError { reason } => {
|
|
MLError::ModelError(format!("Hardware error: {}", reason))
|
|
},
|
|
other @ inference::RealInferenceError::ModelNotLoaded { .. }
|
|
| other @ inference::RealInferenceError::ArchitectureError { .. }
|
|
| other @ inference::RealInferenceError::TimeoutExceeded { .. }
|
|
| other @ inference::RealInferenceError::ModelDrift { .. } => {
|
|
MLError::InferenceError(other.to_string())
|
|
},
|
|
}
|
|
}
|
|
}
|
|
|
|
// Implement From<ProductionTrainingError> for MLError
|
|
impl From<training_pipeline::ProductionTrainingError> for MLError {
|
|
fn from(err: training_pipeline::ProductionTrainingError) -> Self {
|
|
match err {
|
|
training_pipeline::ProductionTrainingError::ConfigError { reason } => {
|
|
MLError::ConfigError(reason)
|
|
},
|
|
training_pipeline::ProductionTrainingError::ArchitectureError { reason } => {
|
|
MLError::ModelError(format!("Architecture error: {}", reason))
|
|
},
|
|
training_pipeline::ProductionTrainingError::DataError { reason } => {
|
|
MLError::ValidationError {
|
|
message: format!("Data error: {}", reason),
|
|
}
|
|
},
|
|
training_pipeline::ProductionTrainingError::OptimizationError { reason } => {
|
|
MLError::TrainingError(format!("Optimization error: {}", reason))
|
|
},
|
|
training_pipeline::ProductionTrainingError::FinancialError { reason } => {
|
|
MLError::ValidationError {
|
|
message: format!("Financial error: {}", reason),
|
|
}
|
|
},
|
|
training_pipeline::ProductionTrainingError::SafetyViolation { reason } => {
|
|
MLError::ValidationError {
|
|
message: format!("Safety violation: {}", reason),
|
|
}
|
|
},
|
|
training_pipeline::ProductionTrainingError::ConvergenceError { reason } => {
|
|
MLError::TrainingError(format!("Convergence error: {}", reason))
|
|
},
|
|
training_pipeline::ProductionTrainingError::ResourceError { reason } => {
|
|
MLError::ModelError(format!("Resource error: {}", reason))
|
|
},
|
|
training_pipeline::ProductionTrainingError::GpuRequired { reason } => {
|
|
MLError::ModelError(format!("GPU required: {}", reason))
|
|
},
|
|
}
|
|
}
|
|
}
|
|
|
|
// Note: From trait for liquid::LiquidError is implemented in the liquid module to avoid conflicts
|
|
|
|
// ========== ML PRELUDE (extends ml-core prelude with ml-specific items) ==========
|
|
/// Prelude module for convenient imports of commonly used ML types
|
|
///
|
|
/// This module re-exports the most commonly used types and traits from the ML crate
|
|
/// to allow users to import everything they need with a single `use ml::prelude::*;`
|
|
pub mod prelude {
|
|
// Re-export everything from ml-core's prelude
|
|
pub use ml_core::prelude::*;
|
|
|
|
// GPU device management (ml-specific)
|
|
pub use crate::gpu::{capabilities::GpuCapabilities, DeviceConfig};
|
|
|
|
// Data pipeline (ml-specific)
|
|
pub use crate::data_pipeline::{DatasetManager, DatasetMode, DatasetSpec, PreparedDataset};
|
|
|
|
// Asset selection (ml-specific)
|
|
pub use crate::asset_selection::{ActiveSetSelector, AssetUniverse, PredictabilityScorer};
|
|
}
|