diff --git a/Cargo.lock b/Cargo.lock index 98f703a24..73c51ac73 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6262,10 +6262,19 @@ dependencies = [ "lru", "memmap2", "mimalloc", + "ml-checkpoint", "ml-core", + "ml-data-validation", "ml-dqn", + "ml-ensemble", + "ml-features", + "ml-hyperopt", + "ml-labeling", "ml-ppo", + "ml-regime", + "ml-risk", "ml-supervised", + "ml-validation", "nalgebra 0.33.2", "ndarray", "num", @@ -6309,6 +6318,32 @@ dependencies = [ "zstd", ] +[[package]] +name = "ml-checkpoint" +version = "1.0.0" +dependencies = [ + "async-trait", + "aws-config", + "aws-credential-types", + "aws-sdk-s3", + "aws-types", + "chrono", + "dashmap 6.1.0", + "flate2", + "fs2", + "hex", + "hmac", + "ml-core", + "serde", + "serde_json", + "sha2", + "tempfile", + "tokio", + "tracing", + "urlencoding", + "uuid", +] + [[package]] name = "ml-core" version = "1.0.0" @@ -6371,6 +6406,18 @@ dependencies = [ "uuid", ] +[[package]] +name = "ml-data-validation" +version = "1.0.0" +dependencies = [ + "anyhow", + "chrono", + "ml-core", + "serde", + "serde_json", + "tokio", +] + [[package]] name = "ml-dqn" version = "1.0.0" @@ -6403,6 +6450,94 @@ dependencies = [ "tracing", ] +[[package]] +name = "ml-ensemble" +version = "1.0.0" +dependencies = [ + "anyhow", + "async-trait", + "candle-core", + "candle-nn", + "chrono", + "chrono-tz", + "common", + "crossbeam", + "dashmap 6.1.0", + "ml-core", + "ndarray", + "once_cell", + "parking_lot 0.12.5", + "prometheus", + "rand 0.8.5", + "rayon", + "serde", + "serde_json", + "statrs", + "thiserror 1.0.69", + "tokio", + "tracing", + "uuid", +] + +[[package]] +name = "ml-features" +version = "1.0.0" +dependencies = [ + "anyhow", + "approx", + "chrono", + "chrono-tz", + "common", + "data", + "dbn 0.42.0", + "ml-core", + "once_cell", + "rand 0.8.5", + "rayon", + "serde", + "serde_json", + "tokio", + "tracing", + "zstd", +] + +[[package]] +name = "ml-hyperopt" +version = "1.0.0" +dependencies = [ + "anyhow", + "approx", + "argmin", + "argmin-math", + "candle-core", + "chrono", + "common", + "ml-core", + "ndarray", + "rand 0.8.5", + "rand_chacha 0.3.1", + "rand_distr 0.4.3", + "rayon", + "serde", + "serde_json", + "statrs", + "tokio", + "tracing", +] + +[[package]] +name = "ml-labeling" +version = "1.0.0" +dependencies = [ + "candle-core", + "dashmap 6.1.0", + "ml-core", + "serde", + "tokio", + "tracing", + "uuid", +] + [[package]] name = "ml-ppo" version = "1.0.0" @@ -6427,6 +6562,40 @@ dependencies = [ "tracing", ] +[[package]] +name = "ml-regime" +version = "1.0.0" +dependencies = [ + "anyhow", + "approx", + "chrono", + "ml-core", + "ml-ensemble", + "rand 0.8.5", + "serde", + "serde_json", + "sqlx", + "thiserror 1.0.69", + "tokio", + "tracing", +] + +[[package]] +name = "ml-risk" +version = "1.0.0" +dependencies = [ + "chrono", + "common", + "ml-core", + "ndarray", + "rand 0.8.5", + "risk", + "rust_decimal", + "serde", + "tokio", + "tracing", +] + [[package]] name = "ml-supervised" version = "1.0.0" @@ -6531,6 +6700,19 @@ dependencies = [ "zeroize", ] +[[package]] +name = "ml-validation" +version = "1.0.0" +dependencies = [ + "approx", + "chrono", + "ml-core", + "rand 0.8.5", + "rand_chacha 0.3.1", + "serde", + "tokio", +] + [[package]] name = "mockito" version = "1.7.0" diff --git a/Cargo.toml b/Cargo.toml index 94b9aa5d5..7e4e6d16e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -110,10 +110,19 @@ members = [ "crates/trading-data", "crates/ml", "crates/ml-core", + "crates/ml-ensemble", "crates/ml-dqn", "crates/ml-ppo", "crates/ml-supervised", "crates/ml-data", + "crates/ml-hyperopt", + "crates/ml-features", + "crates/ml-labeling", + "crates/ml-regime", + "crates/ml-checkpoint", + "crates/ml-data-validation", + "crates/ml-validation", + "crates/ml-risk", "crates/data", "crates/backtesting", "crates/common", @@ -389,9 +398,18 @@ risk-data = { path = "crates/risk-data" } backtesting = { path = "crates/backtesting" } ml = { path = "crates/ml", default-features = false } ml-core = { path = "crates/ml-core" } +ml-ensemble = { path = "crates/ml-ensemble", default-features = false } ml-dqn = { path = "crates/ml-dqn" } ml-ppo = { path = "crates/ml-ppo" } ml-supervised = { path = "crates/ml-supervised" } +ml-hyperopt = { path = "crates/ml-hyperopt", default-features = false } +ml-features = { path = "crates/ml-features" } +ml-labeling = { path = "crates/ml-labeling", default-features = false } +ml-regime = { path = "crates/ml-regime" } +ml-checkpoint = { path = "crates/ml-checkpoint" } +ml-data-validation = { path = "crates/ml-data-validation" } +ml-validation = { path = "crates/ml-validation" } +ml-risk = { path = "crates/ml-risk" } common = { path = "crates/common" } storage = { path = "crates/storage" } market-data = { path = "crates/market-data" } diff --git a/crates/ml-checkpoint/Cargo.toml b/crates/ml-checkpoint/Cargo.toml new file mode 100644 index 000000000..bc77512e3 --- /dev/null +++ b/crates/ml-checkpoint/Cargo.toml @@ -0,0 +1,49 @@ +[package] +name = "ml-checkpoint" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +authors.workspace = true +license.workspace = true +repository.workspace = true +homepage.workspace = true +documentation.workspace = true +publish.workspace = true +keywords.workspace = true +categories.workspace = true +description = "ML model checkpoint persistence — compression, signing, storage, validation, versioning" + +[features] +default = [] +s3-storage = ["aws-config", "aws-sdk-s3", "aws-types", "aws-credential-types", "urlencoding"] + +[dependencies] +ml-core = { path = "../ml-core", default-features = false } + +async-trait.workspace = true +chrono.workspace = true +dashmap.workspace = true +flate2.workspace = true +fs2.workspace = true +hex.workspace = true +hmac = "0.12" +serde = { workspace = true, features = ["derive"] } +serde_json.workspace = true +sha2 = "0.10" +tokio.workspace = true +tracing.workspace = true +uuid.workspace = true + +# Optional S3 storage backend +aws-config = { version = "1.1", optional = true } +aws-sdk-s3 = { version = "1.14", optional = true } +aws-types = { version = "1.1", optional = true } +aws-credential-types = { version = "1.1", optional = true } +urlencoding = { version = "2.1", optional = true } + +[dev-dependencies] +tempfile.workspace = true +tokio = { workspace = true, features = ["test-util", "macros"] } + +[lints] +workspace = true diff --git a/crates/ml/src/checkpoint/compression.rs b/crates/ml-checkpoint/src/compression.rs similarity index 99% rename from crates/ml/src/checkpoint/compression.rs rename to crates/ml-checkpoint/src/compression.rs index 531dc69ba..608c6dafe 100644 --- a/crates/ml/src/checkpoint/compression.rs +++ b/crates/ml-checkpoint/src/compression.rs @@ -7,7 +7,7 @@ use std::io::{Read, Write}; use flate2::{read::GzDecoder, write::GzEncoder, Compression}; use tracing::debug; -use super::CompressionType; +use crate::CompressionType; use crate::MLError; /// Compression manager for checkpoint data diff --git a/crates/ml/src/checkpoint/integration_tests.rs b/crates/ml-checkpoint/src/integration_tests.rs similarity index 99% rename from crates/ml/src/checkpoint/integration_tests.rs rename to crates/ml-checkpoint/src/integration_tests.rs index 0c20e1900..ef44261b5 100644 --- a/crates/ml/src/checkpoint/integration_tests.rs +++ b/crates/ml-checkpoint/src/integration_tests.rs @@ -5,7 +5,7 @@ #[cfg(test)] mod tests { use super::super::*; - use crate::checkpoint::versioning::CompatibilityRisk; + use crate::versioning::CompatibilityRisk; use std::sync::Arc; use tempfile::{tempdir, TempDir}; diff --git a/crates/ml-checkpoint/src/lib.rs b/crates/ml-checkpoint/src/lib.rs new file mode 100644 index 000000000..a58246970 --- /dev/null +++ b/crates/ml-checkpoint/src/lib.rs @@ -0,0 +1,1036 @@ +//! # Unified Model Weight Persistence System +//! +//! Comprehensive checkpoint system for all AI models +//! with versioning, metadata, compression, and validation. +//! +//! ## Key Features +//! +//! - **Unified Interface**: Single API for all model checkpointing +//! - **Model Versioning**: Semantic versioning with compatibility checks +//! - **Metadata Management**: Training metrics, hyperparameters, performance stats +//! - **Compression**: Optional LZ4/Zstd compression for large models +//! - **Validation**: Checksum verification and corruption detection +//! - **Incremental Saves**: Delta checkpoints for memory efficiency +//! - **Async I/O**: Non-blocking checkpoint operations +//! - **Multi-format**: Binary, JSON, and custom formats + +use std::collections::HashMap; +use std::fs::{self}; +use std::path::PathBuf; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; + +use async_trait::async_trait; +use chrono::{DateTime, Utc}; +use dashmap::DashMap; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use tokio::sync::RwLock; +use tracing::{info, instrument, warn}; +use uuid::Uuid; + +// Re-export foundation types so sub-modules can use `crate::MLError` / `crate::ModelType` +pub use ml_core::MLError; +pub use ml_core::ModelType; + +pub mod compression; +pub mod signer; +pub mod storage; +pub mod validation; +pub mod versioning; + +#[cfg(test)] +pub mod integration_tests; + +// Re-export key types for external usage +pub use compression::CompressionManager; +pub use signer::{CheckpointSigner, SignatureInfo}; +#[cfg(feature = "s3-storage")] +pub use storage::S3CheckpointStorage; +pub use storage::{CheckpointStorage, FileSystemStorage, MemoryStorage, StorageStats}; +pub use validation::{verify_checksum, write_checksum, ValidationManager}; +pub use versioning::VersionManager; + +/// Checkpoint format options +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum CheckpointFormat { + /// Binary format (fastest) + Binary, + /// JSON format (human-readable) + JSON, + /// MessagePack format (compact) + MessagePack, + /// Custom optimized format + Custom, +} + +/// Compression algorithm options +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum CompressionType { + /// No compression + None, + /// LZ4 - fast compression + LZ4, + /// Zstandard - balanced compression + Zstd, + /// Gzip - high compression + Gzip, +} + +/// Checkpoint metadata containing model information and training state +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CheckpointMetadata { + /// Unique checkpoint identifier + pub checkpoint_id: String, + + /// Model type + pub model_type: ModelType, + + /// Model name/identifier + pub model_name: String, + + /// Model version (semantic versioning) + pub version: String, + + /// Creation timestamp + pub created_at: DateTime, + + /// Training epoch when checkpoint was created + pub epoch: Option, + + /// Training step when checkpoint was created + pub step: Option, + + /// Training loss at checkpoint time + pub loss: Option, + + /// Validation accuracy at checkpoint time + pub accuracy: Option, + + /// Model hyperparameters + pub hyperparameters: HashMap, + + /// Training metrics and statistics + pub metrics: HashMap, + + /// Model architecture information + pub architecture: HashMap, + + /// Checkpoint file format + pub format: CheckpointFormat, + + /// Compression algorithm used + pub compression: CompressionType, + + /// File size in bytes + pub file_size: u64, + + /// Compressed file size (if compressed) + pub compressed_size: Option, + + /// SHA-256 checksum for validation + pub checksum: String, + + /// Tags for organizing checkpoints + pub tags: Vec, + + /// Additional custom metadata + pub custom_metadata: HashMap, + + // Security fields (Agent 122 - SEC-001 fix) + /// HMAC-SHA256 signature (hex-encoded) + pub signature: Option, + /// Signing algorithm identifier + pub signature_algorithm: String, + /// Signing key ID for rotation support + pub signing_key_id: String, + /// Signature timestamp + pub signed_at: Option>, +} + +impl Default for CheckpointMetadata { + fn default() -> Self { + Self { + checkpoint_id: Uuid::new_v4().to_string(), + model_type: ModelType::DQN, // Default to first variant + model_name: String::new(), + version: "1.0.0".to_owned(), + created_at: Utc::now(), + epoch: None, + step: None, + loss: None, + accuracy: None, + hyperparameters: HashMap::new(), + metrics: HashMap::new(), + architecture: HashMap::new(), + format: CheckpointFormat::Binary, + compression: CompressionType::None, + file_size: 0, + compressed_size: None, + checksum: String::new(), + tags: Vec::new(), + custom_metadata: HashMap::new(), + signature: None, + signature_algorithm: String::new(), + signing_key_id: String::new(), + signed_at: None, + } + } +} + +impl CheckpointMetadata { + /// Create new checkpoint metadata + pub fn new(model_type: ModelType, model_name: String, version: String) -> Self { + Self { + checkpoint_id: Uuid::new_v4().to_string(), + model_type, + model_name, + version, + created_at: Utc::now(), + epoch: None, + step: None, + loss: None, + accuracy: None, + hyperparameters: HashMap::new(), + metrics: HashMap::new(), + architecture: HashMap::new(), + format: CheckpointFormat::Binary, + compression: CompressionType::None, + file_size: 0, + compressed_size: None, + checksum: String::new(), + tags: Vec::new(), + custom_metadata: HashMap::new(), + signature: None, + signature_algorithm: String::new(), + signing_key_id: String::new(), + signed_at: None, + } + } + + /// Add training state information + pub fn with_training_state( + mut self, + epoch: Option, + step: Option, + loss: Option, + accuracy: Option, + ) -> Self { + self.epoch = epoch; + self.step = step; + self.loss = loss; + self.accuracy = accuracy; + self + } + + /// Add hyperparameters + pub fn with_hyperparameters(mut self, hyperparams: HashMap) -> Self { + self.hyperparameters = hyperparams; + self + } + + /// Add metrics + pub fn with_metrics(mut self, metrics: HashMap) -> Self { + self.metrics = metrics; + self + } + + /// Add tags + pub fn with_tags(mut self, tags: Vec) -> Self { + self.tags = tags; + self + } + + /// Check if this is a newer version than another metadata + pub fn is_newer_than(&self, other: &CheckpointMetadata) -> bool { + if self.model_type != other.model_type || self.model_name != other.model_name { + return false; + } + + // Compare by epoch if available + if let (Some(self_epoch), Some(other_epoch)) = (self.epoch, other.epoch) { + return self_epoch > other_epoch; + } + + // Compare by step if available + if let (Some(self_step), Some(other_step)) = (self.step, other.step) { + return self_step > other_step; + } + + // Compare by timestamp + self.created_at > other.created_at + } + + /// Generate filename for this checkpoint + pub fn generate_filename(&self) -> String { + let timestamp = self.created_at.format("%Y%m%d_%H%M%S%.3f"); + let epoch_str = self.epoch.map(|e| format!("_e{}", e)).unwrap_or_default(); + let step_str = self.step.map(|s| format!("_s{}", s)).unwrap_or_default(); + let ext = self.model_type.file_extension(); + + format!( + "{}_{}_v{}{}{}_{}.{}", + self.model_type.file_extension(), + self.model_name, + self.version, + epoch_str, + step_str, + timestamp, + ext + ) + } +} + +/// Configuration for checkpoint operations +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CheckpointConfig { + /// Base directory for checkpoints + pub base_dir: PathBuf, + + /// Default compression type + pub compression: CompressionType, + + /// Default checkpoint format + pub format: CheckpointFormat, + + /// Maximum number of checkpoints to keep per model + pub max_checkpoints_per_model: usize, + + /// Automatic cleanup of old checkpoints + pub auto_cleanup: bool, + + /// Enable checksum validation + pub validate_checksums: bool, + + /// Enable incremental checkpoints (delta saves) + pub incremental_checkpoints: bool, + + /// Compression level (0-9, algorithm dependent) + pub compression_level: u32, + + /// Enable async I/O operations + pub async_io: bool, + + /// Buffer size for I/O operations + pub buffer_size: usize, +} + +impl Default for CheckpointConfig { + fn default() -> Self { + Self { + base_dir: PathBuf::from("./checkpoints"), + compression: CompressionType::LZ4, + format: CheckpointFormat::Binary, + max_checkpoints_per_model: 10, + auto_cleanup: true, + validate_checksums: true, + incremental_checkpoints: true, + compression_level: 3, + async_io: true, + buffer_size: 64 * 1024, // 64KB + } + } +} + +/// Statistics for checkpoint operations +#[derive(Debug, Default, Serialize, Deserialize)] +pub struct CheckpointStats { + /// Total checkpoints saved + pub total_saved: AtomicU64, + + /// Total checkpoints loaded + pub total_loaded: AtomicU64, + + /// Total bytes saved + pub total_bytes_saved: AtomicU64, + + /// Total bytes loaded + pub total_bytes_loaded: AtomicU64, + + /// Total compression savings (bytes) + pub compression_savings: AtomicU64, + + /// Average save time (microseconds) + pub avg_save_time_us: AtomicU64, + + /// Average load time (microseconds) + pub avg_load_time_us: AtomicU64, + + /// Failed operations + pub failed_operations: AtomicU64, +} + +impl CheckpointStats { + /// Record a save operation + pub fn record_save(&self, bytes_saved: u64, save_time_us: u64, compression_savings: u64) { + self.total_saved.fetch_add(1, Ordering::Relaxed); + self.total_bytes_saved + .fetch_add(bytes_saved, Ordering::Relaxed); + self.compression_savings + .fetch_add(compression_savings, Ordering::Relaxed); + + // Update moving average + let count = self.total_saved.load(Ordering::Relaxed); + let current_avg = self.avg_save_time_us.load(Ordering::Relaxed); + let new_avg = ((current_avg * (count - 1)) + save_time_us) / count; + self.avg_save_time_us.store(new_avg, Ordering::Relaxed); + } + + /// Record a load operation + pub fn record_load(&self, bytes_loaded: u64, load_time_us: u64) { + self.total_loaded.fetch_add(1, Ordering::Relaxed); + self.total_bytes_loaded + .fetch_add(bytes_loaded, Ordering::Relaxed); + + // Update moving average + let count = self.total_loaded.load(Ordering::Relaxed); + let current_avg = self.avg_load_time_us.load(Ordering::Relaxed); + let new_avg = ((current_avg * (count - 1)) + load_time_us) / count; + self.avg_load_time_us.store(new_avg, Ordering::Relaxed); + } + + /// Record a failed operation + pub fn record_failure(&self) { + self.failed_operations.fetch_add(1, Ordering::Relaxed); + } + + /// Get statistics as a map + pub fn to_map(&self) -> HashMap { + let mut map = HashMap::new(); + map.insert( + "total_saved".to_owned(), + self.total_saved.load(Ordering::Relaxed), + ); + map.insert( + "total_loaded".to_owned(), + self.total_loaded.load(Ordering::Relaxed), + ); + map.insert( + "total_bytes_saved".to_owned(), + self.total_bytes_saved.load(Ordering::Relaxed), + ); + map.insert( + "total_bytes_loaded".to_owned(), + self.total_bytes_loaded.load(Ordering::Relaxed), + ); + map.insert( + "compression_savings".to_owned(), + self.compression_savings.load(Ordering::Relaxed), + ); + map.insert( + "avg_save_time_us".to_owned(), + self.avg_save_time_us.load(Ordering::Relaxed), + ); + map.insert( + "avg_load_time_us".to_owned(), + self.avg_load_time_us.load(Ordering::Relaxed), + ); + map.insert( + "failed_operations".to_owned(), + self.failed_operations.load(Ordering::Relaxed), + ); + map + } +} + +// Checkpointable trait is defined in ml-core to avoid orphan rule issues +pub use ml_core::Checkpointable; + +/// Main checkpoint manager for all AI models +#[derive(Debug)] +pub struct CheckpointManager { + /// Configuration + config: CheckpointConfig, + + /// Storage backend + storage: Arc, + + /// Checkpoint metadata index + metadata_index: Arc>>, + + /// Statistics + stats: Arc, + + /// Version manager + version_manager: Arc, + + /// Compression manager + compression_manager: Arc, + + /// Validation manager + validation_manager: Arc, +} + +impl CheckpointManager { + /// Create a new checkpoint manager + pub fn new(config: CheckpointConfig) -> Result { + // Create base directory if it doesn't exist + if !config.base_dir.exists() { + fs::create_dir_all(&config.base_dir).map_err(|e| { + MLError::ModelError(format!("Failed to create checkpoint directory: {}", e)) + })?; + } + + let storage: Arc = + Arc::new(FileSystemStorage::new(config.base_dir.clone())); + let metadata_index = Arc::new(RwLock::new(DashMap::new())); + let stats = Arc::new(CheckpointStats::default()); + let version_manager = Arc::new(VersionManager::new()); + let compression_manager = Arc::new(CompressionManager::new()); + let validation_manager = Arc::new(ValidationManager::new()); + + Ok(Self { + config, + storage, + metadata_index, + stats, + version_manager, + compression_manager, + validation_manager, + }) + } + + /// Save a checkpoint for a model + #[instrument(skip(self, model))] + pub async fn save_checkpoint( + &self, + model: &M, + tags: Option>, + ) -> Result { + let start_time = std::time::Instant::now(); + + info!("Saving checkpoint for model: {}", model.model_name()); + + // Create metadata + let (epoch, step, loss, accuracy) = model.get_training_state(); + let mut metadata = CheckpointMetadata::new( + model.model_type(), + model.model_name().to_string(), + model.model_version().to_string(), + ) + .with_training_state(epoch, step, loss, accuracy) + .with_hyperparameters(model.get_hyperparameters()) + .with_metrics(model.get_metrics()); + + if let Some(tags) = tags { + metadata = metadata.with_tags(tags); + } + + metadata.format = self.config.format; + metadata.compression = self.config.compression; + + // Add architecture info + metadata.architecture = model.get_architecture_info(); + + // Serialize model state + let model_data = model.serialize_state().await?; + let original_size = model_data.len() as u64; + + // Compress if needed + let (final_data, compressed_size) = if self.config.compression != CompressionType::None { + let compressed = self.compression_manager.compress( + &model_data, + self.config.compression, + self.config.compression_level, + )?; + let comp_size = compressed.len() as u64; + (compressed, Some(comp_size)) + } else { + (model_data, None) + }; + + // Calculate checksum + let mut hasher = Sha256::new(); + hasher.update(&final_data); + let checksum = format!("{:x}", hasher.finalize()); + + // Update metadata + metadata.file_size = original_size; + metadata.compressed_size = compressed_size; + metadata.checksum = checksum; + + // Generate filename + let filename = metadata.generate_filename(); + + // Save to storage + self.storage + .save_checkpoint(&filename, &final_data, &metadata) + .await?; + + // Update index + { + let index = self.metadata_index.write().await; + index.insert(metadata.checkpoint_id.clone(), metadata.clone()); + } + + // Cleanup old checkpoints if needed + if self.config.auto_cleanup { + self.cleanup_old_checkpoints(model.model_type(), model.model_name()) + .await?; + } + + // Record statistics + let save_time_us = start_time.elapsed().as_micros() as u64; + let compression_savings = compressed_size + .map(|cs| original_size.saturating_sub(cs)) + .unwrap_or(0); + self.stats + .record_save(original_size, save_time_us, compression_savings); + + info!( + "Checkpoint saved: {} ({} bytes, {}µs, {:.1}% compression)", + filename, + final_data.len(), + save_time_us, + if compressed_size.is_some() { + (compression_savings as f64 / original_size as f64) * 100.0 + } else { + 0.0 + } + ); + + Ok(metadata.checkpoint_id) + } + + /// Load a checkpoint into a model + #[instrument(skip(self, model))] + pub async fn load_checkpoint( + &self, + model: &mut M, + checkpoint_id: &str, + ) -> Result { + let start_time = std::time::Instant::now(); + + info!("Loading checkpoint: {}", checkpoint_id); + + // Get metadata + let metadata = { + let index = self.metadata_index.read().await; + index.get(checkpoint_id).map(|entry| entry.clone()) + }; + + let metadata = metadata.ok_or_else(|| { + MLError::ModelError(format!("Checkpoint not found: {}", checkpoint_id)) + })?; + + // Verify model compatibility + if metadata.model_type != model.model_type() { + return Err(MLError::ModelError(format!( + "Model type mismatch: expected {:?}, got {:?}", + model.model_type(), + metadata.model_type + ))); + } + + // Load data from storage + let filename = metadata.generate_filename(); + let data = self.storage.load_checkpoint(&filename).await?; + + // Validate checksum if enabled + if self.config.validate_checksums { + self.validation_manager + .validate_checksum(&data, &metadata.checksum)?; + } + + // Decompress if needed + let final_data = if metadata.compression != CompressionType::None { + self.compression_manager + .decompress(&data, metadata.compression)? + } else { + data + }; + + // Deserialize into model + model.deserialize_state(&final_data).await?; + + // Validate loaded state + model.validate_loaded_state()?; + + // Record statistics + let load_time_us = start_time.elapsed().as_micros() as u64; + self.stats + .record_load(final_data.len() as u64, load_time_us); + + info!( + "Checkpoint loaded: {} ({} bytes, {}µs)", + filename, + final_data.len(), + load_time_us + ); + + Ok(metadata) + } + + /// Load the latest checkpoint for a model + pub async fn load_latest_checkpoint( + &self, + model: &mut M, + ) -> Result, MLError> { + let model_type = model.model_type(); + let model_name = model.model_name(); + + // Find latest checkpoint + let latest_checkpoint = { + let index = self.metadata_index.read().await; + index + .iter() + .filter(|entry| { + let metadata = entry.value(); + metadata.model_type == model_type && metadata.model_name == model_name + }) + .max_by(|a, b| a.value().created_at.cmp(&b.value().created_at)) + .map(|entry| entry.value().clone()) + }; + + if let Some(metadata) = latest_checkpoint { + let loaded_metadata = self.load_checkpoint(model, &metadata.checkpoint_id).await?; + Ok(Some(loaded_metadata)) + } else { + Ok(None) + } + } + + /// List all checkpoints for a model + pub async fn list_checkpoints( + &self, + model_type: ModelType, + model_name: &str, + ) -> Vec { + let index = self.metadata_index.read().await; + let mut checkpoints: Vec<_> = index + .iter() + .filter(|entry| { + let metadata = entry.value(); + // Empty string matches all model names (wildcard behavior) + metadata.model_type == model_type + && (model_name.is_empty() || metadata.model_name == model_name) + }) + .map(|entry| entry.value().clone()) + .collect(); + + // Sort by creation time (newest first) + checkpoints.sort_by(|a, b| b.created_at.cmp(&a.created_at)); + checkpoints + } + + /// Delete a checkpoint + pub async fn delete_checkpoint(&self, checkpoint_id: &str) -> Result<(), MLError> { + // Get metadata + let metadata = { + let index = self.metadata_index.read().await; + index.get(checkpoint_id).map(|entry| entry.clone()) + }; + + let metadata = metadata.ok_or_else(|| { + MLError::ModelError(format!("Checkpoint not found: {}", checkpoint_id)) + })?; + + // Delete from storage + let filename = metadata.generate_filename(); + self.storage.delete_checkpoint(&filename).await?; + + // Remove from index + { + let index = self.metadata_index.write().await; + index.remove(checkpoint_id); + } + + info!("Checkpoint deleted: {}", checkpoint_id); + Ok(()) + } + + /// Cleanup old checkpoints for a model + async fn cleanup_old_checkpoints( + &self, + model_type: ModelType, + model_name: &str, + ) -> Result<(), MLError> { + let mut checkpoints = self.list_checkpoints(model_type, model_name).await; + + if checkpoints.len() <= self.config.max_checkpoints_per_model { + return Ok(()); + } + + // Remove oldest checkpoints + checkpoints.sort_by(|a, b| a.created_at.cmp(&b.created_at)); + let to_remove = checkpoints.len() - self.config.max_checkpoints_per_model; + + for checkpoint in checkpoints.into_iter().take(to_remove) { + if let Err(e) = self.delete_checkpoint(&checkpoint.checkpoint_id).await { + warn!( + "Failed to delete old checkpoint {}: {}", + checkpoint.checkpoint_id, e + ); + } + } + + info!( + "Cleaned up {} old checkpoints for {}:{}", + to_remove, + model_type.file_extension(), + model_name + ); + Ok(()) + } + + /// Get checkpoint statistics + pub fn get_stats(&self) -> HashMap { + self.stats.to_map() + } + + /// Refresh metadata index from storage + pub async fn refresh_index(&self) -> Result<(), MLError> { + let all_metadata = self.storage.list_all_checkpoints().await?; + + let index = self.metadata_index.write().await; + index.clear(); + + for metadata in all_metadata { + index.insert(metadata.checkpoint_id.clone(), metadata); + } + + info!( + "Refreshed checkpoint index with {} checkpoints", + index.len() + ); + Ok(()) + } + + /// Get checkpoint by tags + pub async fn find_checkpoints_by_tags(&self, tags: &[String]) -> Vec { + let index = self.metadata_index.read().await; + index + .iter() + .filter(|entry| { + let metadata = entry.value(); + tags.iter().all(|tag| metadata.tags.contains(tag)) + }) + .map(|entry| entry.value().clone()) + .collect() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::AtomicBool; + + + // Mock model for testing + struct MockModel { + model_type: ModelType, + model_name: String, + version: String, + data: Vec, + trained: AtomicBool, + } + + impl MockModel { + fn new(model_type: ModelType, name: String, version: String) -> Self { + Self { + model_type, + model_name: name, + version, + data: vec![1, 2, 3, 4, 5], + trained: AtomicBool::new(false), + } + } + } + + #[async_trait] + impl Checkpointable for MockModel { + fn model_type(&self) -> ModelType { + self.model_type + } + + fn model_name(&self) -> &str { + &self.model_name + } + + fn model_version(&self) -> &str { + &self.version + } + + async fn serialize_state(&self) -> Result, MLError> { + Ok(self.data.clone()) + } + + async fn deserialize_state(&mut self, data: &[u8]) -> Result<(), MLError> { + self.data = data.to_vec(); + self.trained.store(true, Ordering::Relaxed); + Ok(()) + } + + fn get_training_state(&self) -> (Option, Option, Option, Option) { + (Some(10), Some(1000), Some(0.1), Some(0.95)) + } + } + + #[tokio::test] + async fn test_checkpoint_save_load() -> Result<(), MLError> { + let temp_dir = tempfile::tempdir()?; + let config = CheckpointConfig { + base_dir: temp_dir.path().to_path_buf(), + compression: CompressionType::None, + ..Default::default() + }; + + let manager = CheckpointManager::new(config)?; + let mut model = MockModel::new( + ModelType::DQN, + "test_model".to_owned(), + "1.0.0".to_owned(), + ); + + // Save checkpoint + let checkpoint_id = manager + .save_checkpoint(&model, Some(vec!["test".to_owned()])) + .await?; + + // Modify model data + model.data = vec![9, 8, 7, 6, 5]; + + // Load checkpoint + let metadata = manager.load_checkpoint(&mut model, &checkpoint_id).await?; + + // Verify data was restored + assert_eq!(model.data, vec![1, 2, 3, 4, 5]); + assert_eq!(metadata.model_type, ModelType::DQN); + assert_eq!(metadata.model_name, "test_model"); + assert_eq!(metadata.tags, vec!["test".to_owned()]); + assert!(model.trained.load(Ordering::Relaxed)); + Ok(()) + } + + #[tokio::test] + async fn test_checkpoint_compression() -> Result<(), MLError> { + let temp_dir = tempfile::tempdir()?; + let config = CheckpointConfig { + base_dir: temp_dir.path().to_path_buf(), + compression: CompressionType::LZ4, + ..Default::default() + }; + + let manager = CheckpointManager::new(config)?; + let mut model = MockModel::new( + ModelType::MAMBA, + "test_model".to_owned(), + "1.0.0".to_owned(), + ); + + // Create larger data for compression test + model.data = vec![42; 1000]; + + let checkpoint_id = manager.save_checkpoint(&model, None).await?; + + // Clear data + model.data.clear(); + + // Load and verify + manager.load_checkpoint(&mut model, &checkpoint_id).await?; + assert_eq!(model.data, vec![42; 1000]); + Ok(()) + } + + #[tokio::test] + async fn test_checkpoint_metadata() -> Result<(), MLError> { + let metadata = CheckpointMetadata::new( + ModelType::TFT, + "transformer_model".to_owned(), + "2.1.0".to_owned(), + ) + .with_training_state(Some(50), Some(5000), Some(0.05), Some(0.98)) + .with_tags(vec!["production".to_owned(), "validated".to_owned()]); + + assert_eq!(metadata.model_type, ModelType::TFT); + assert_eq!(metadata.epoch, Some(50)); + assert_eq!(metadata.accuracy, Some(0.98)); + assert!(metadata.tags.contains(&"production".to_owned())); + + let filename = metadata.generate_filename(); + assert!(filename.contains("tft")); + assert!(filename.contains("transformer_model")); + assert!(filename.contains("v2.1.0")); + assert!(filename.contains("e50")); + assert!(filename.contains("s5000")); + Ok(()) + } + + #[tokio::test] + async fn test_list_and_cleanup_checkpoints() -> Result<(), MLError> { + let temp_dir = tempfile::tempdir()?; + let config = CheckpointConfig { + base_dir: temp_dir.path().to_path_buf(), + max_checkpoints_per_model: 2, + auto_cleanup: false, + ..Default::default() + }; + + let manager = CheckpointManager::new(config)?; + let model = MockModel::new( + ModelType::TGGN, + "graph_model".to_owned(), + "1.0.0".to_owned(), + ); + + // Save multiple checkpoints + let id1 = manager.save_checkpoint(&model, None).await?; + tokio::time::sleep(tokio::time::Duration::from_millis(10)).await; + let id2 = manager.save_checkpoint(&model, None).await?; + tokio::time::sleep(tokio::time::Duration::from_millis(10)).await; + let id3 = manager.save_checkpoint(&model, None).await?; + + // List checkpoints + let checkpoints = manager + .list_checkpoints(ModelType::TGGN, "graph_model") + .await; + assert_eq!(checkpoints.len(), 3); + + // Manual cleanup + manager + .cleanup_old_checkpoints(ModelType::TGGN, "graph_model") + .await?; + + // Should have only 2 checkpoints now + let checkpoints = manager + .list_checkpoints(ModelType::TGGN, "graph_model") + .await; + assert_eq!(checkpoints.len(), 2); + + // The oldest checkpoint should be gone + assert!(manager + .load_checkpoint( + &mut MockModel::new( + ModelType::TGGN, + "graph_model".to_owned(), + "1.0.0".to_owned() + ), + &id1 + ) + .await + .is_err()); + assert!(manager + .load_checkpoint( + &mut MockModel::new( + ModelType::TGGN, + "graph_model".to_owned(), + "1.0.0".to_owned() + ), + &id2 + ) + .await + .is_ok()); + assert!(manager + .load_checkpoint( + &mut MockModel::new( + ModelType::TGGN, + "graph_model".to_owned(), + "1.0.0".to_owned() + ), + &id3 + ) + .await + .is_ok()); + Ok(()) + } +} diff --git a/crates/ml/src/checkpoint/signer.rs b/crates/ml-checkpoint/src/signer.rs similarity index 93% rename from crates/ml/src/checkpoint/signer.rs rename to crates/ml-checkpoint/src/signer.rs index d5b636b64..7f29e7072 100644 --- a/crates/ml/src/checkpoint/signer.rs +++ b/crates/ml-checkpoint/src/signer.rs @@ -13,7 +13,7 @@ use sha2::Sha256; use tokio::sync::RwLock; use tracing::{debug, error, info, warn}; -use crate::checkpoint::ModelType; +use crate::ModelType; use crate::MLError; type HmacSha256 = Hmac; @@ -79,17 +79,6 @@ impl CheckpointSigner { } /// Sign checkpoint data with HMAC-SHA256 - /// - /// # Arguments - /// * `data` - Checkpoint binary data to sign - /// * `model_type` - Model type for key selection - /// - /// # Returns - /// SignatureInfo containing signature and metadata - /// - /// # Performance - /// - Cache hit: ~1μs (no Vault call) - /// - Cache miss: ~10ms (Vault HTTP request + ~50μs HMAC computation) pub async fn sign_checkpoint( &self, data: &[u8], @@ -131,16 +120,6 @@ impl CheckpointSigner { /// Verify checkpoint signature /// - /// # Arguments - /// * `data` - Checkpoint binary data to verify - /// * `signature` - Expected HMAC-SHA256 signature (hex-encoded) - /// * `key_id` - Key ID used for signing - /// * `model_type` - Model type for key selection - /// - /// # Returns - /// Ok(()) if signature is valid, Err otherwise - /// - /// # Security /// Uses constant-time comparison to prevent timing attacks pub async fn verify_signature( &self, @@ -247,9 +226,6 @@ impl CheckpointSigner { /// /// # Key Path Format /// `secret/foxhunt/ml/checkpoint-signing/{model_type}/{key_id}` - /// - /// # Example - /// `secret/foxhunt/ml/checkpoint-signing/DQN/2024-Q4` async fn fetch_key_from_vault( &self, key_id: &str, @@ -311,9 +287,6 @@ impl CheckpointSigner { } /// Rotate signing keys (quarterly maintenance task) - /// - /// This should be called manually or via scheduled task to generate - /// new keys for the next quarter. pub async fn rotate_keys(&self) -> Result<(), MLError> { info!("Starting signing key rotation"); @@ -324,7 +297,6 @@ impl CheckpointSigner { info!("Current key: {}, Next key: {}", current_key_id, next_key_id); // In production, this would generate new keys in Vault - // For now, just log the rotation intent warn!("Key rotation not fully implemented - would generate keys in Vault"); // Clear cache to force reload diff --git a/crates/ml/src/checkpoint/storage.rs b/crates/ml-checkpoint/src/storage.rs similarity index 91% rename from crates/ml/src/checkpoint/storage.rs rename to crates/ml-checkpoint/src/storage.rs index 292448bf7..cde755c61 100644 --- a/crates/ml/src/checkpoint/storage.rs +++ b/crates/ml-checkpoint/src/storage.rs @@ -16,8 +16,6 @@ use crate::MLError; // S3 dependencies for AWS SDK #[cfg(feature = "s3-storage")] -use aws_config::meta::credentials::CredentialsProviderChain; -#[cfg(feature = "s3-storage")] use aws_config::BehaviorVersion; #[cfg(feature = "s3-storage")] use aws_credential_types::Credentials; @@ -587,12 +585,7 @@ impl S3CheckpointStorage { bucket_name, key_prefix, region ); - let client = Self::create_s3_client_from_env().await.map_err(|e| { - MLError::ModelError(format!( - "Failed to create S3 client from environment: {}", - e - )) - })?; + let client = Self::create_s3_client_from_env().await?; // Test bucket access client @@ -681,7 +674,7 @@ impl S3CheckpointStorage { } /// Create AWS S3 client using environment variables or AWS credential chain - async fn create_s3_client_from_env() -> Result { + async fn create_s3_client_from_env() -> Result { // Try to use explicit credentials first, then fall back to AWS credential chain let region_name = std::env::var("AWS_REGION").unwrap_or_else(|_| "us-east-1".to_owned()); let aws_region = aws_types::region::Region::new(region_name); @@ -1150,91 +1143,3 @@ impl CheckpointStorage for S3CheckpointStorage { }) } } - -// DISABLED: Tests -// // #[cfg(test)] -// mod tests { -// use super::*; -// use tempfile::tempdir; -// // use crate::safe_operations; // DISABLED - module not found -// -// #[tokio::test] -// async fn test_filesystem_storage() { -// let temp_dir = tempdir()?; -// let storage = FileSystemStorage::new(temp_dir.path().to_path_buf()); -// -// let metadata = CheckpointMetadata::new( -// super::super::ModelType::DQN, -// "test_model".to_owned(), -// "1.0.0".to_owned(), -// ); -// -// let test_data = vec![1, 2, 3, 4, 5]; -// let filename = "test_checkpoint.dqn"; -// -// // Save checkpoint -// storage -// .save_checkpoint(filename, &test_data, &metadata) -// .await?; -// -// // Check existence -// assert!(storage.has_checkpoint(filename).await); -// -// // Load checkpoint -// let loaded_data = storage.load_checkpoint(filename).await?; -// assert_eq!(loaded_data, test_data); -// -// // List checkpoints -// let checkpoints = storage.list_all_checkpoints().await?; -// assert_eq!(checkpoints.len(), 1); -// assert_eq!(checkpoints[0].checkpoint_id, metadata.checkpoint_id); -// -// // Get stats -// let stats = storage.get_storage_stats().await?; -// assert_eq!(stats.total_checkpoints, 1); -// assert!(stats.total_bytes > 0); -// -// // Delete checkpoint -// storage.delete_checkpoint(filename).await?; -// assert!(!storage.has_checkpoint(filename).await); -// } -// -// #[tokio::test] -// async fn test_memory_storage() { -// let storage = MemoryStorage::new(); -// -// let metadata = CheckpointMetadata::new( -// super::super::ModelType::MAMBA, -// "test_model".to_owned(), -// "2.0.0".to_owned(), -// ); -// -// let test_data = vec![10, 20, 30, 40, 50]; -// let filename = "test_checkpoint.mamba"; -// -// // Save checkpoint -// storage -// .save_checkpoint(filename, &test_data, &metadata) -// .await?; -// -// // Check existence -// assert!(storage.has_checkpoint(filename).await); -// -// // Load checkpoint -// let loaded_data = storage.load_checkpoint(filename).await?; -// assert_eq!(loaded_data, test_data); -// -// // List checkpoints -// let checkpoints = storage.list_all_checkpoints().await?; -// assert_eq!(checkpoints.len(), 1); -// -// // Get stats -// let stats = storage.get_storage_stats().await?; -// assert_eq!(stats.total_checkpoints, 1); -// assert_eq!(stats.backend_type, "memory"); -// -// // Delete checkpoint -// storage.delete_checkpoint(filename).await?; -// assert!(!storage.has_checkpoint(filename).await); -// } -// } diff --git a/crates/ml/src/checkpoint/validation.rs b/crates/ml-checkpoint/src/validation.rs similarity index 100% rename from crates/ml/src/checkpoint/validation.rs rename to crates/ml-checkpoint/src/validation.rs diff --git a/crates/ml/src/checkpoint/versioning.rs b/crates/ml-checkpoint/src/versioning.rs similarity index 100% rename from crates/ml/src/checkpoint/versioning.rs rename to crates/ml-checkpoint/src/versioning.rs diff --git a/crates/ml-core/src/lib.rs b/crates/ml-core/src/lib.rs index 7f4161946..e8804c51e 100644 --- a/crates/ml-core/src/lib.rs +++ b/crates/ml-core/src/lib.rs @@ -870,6 +870,52 @@ pub trait MLModel: Send + Sync + std::fmt::Debug { } } +// ========== CHECKPOINT TRAIT ========== + +/// Trait that models must implement to support checkpointing +#[async_trait] +pub trait Checkpointable: Send + Sync { + /// Get model type + fn model_type(&self) -> ModelType; + + /// Get model name/identifier + fn model_name(&self) -> &str; + + /// Get model version + fn model_version(&self) -> &str; + + /// Serialize model weights and state to bytes + async fn serialize_state(&self) -> Result, MLError>; + + /// Deserialize model weights and state from bytes + async fn deserialize_state(&mut self, data: &[u8]) -> Result<(), MLError>; + + /// Get current training state (epoch, step, loss, accuracy) + fn get_training_state(&self) -> (Option, Option, Option, Option) { + (None, None, None, None) + } + + /// Get hyperparameters for metadata + fn get_hyperparameters(&self) -> HashMap { + HashMap::new() + } + + /// Get current metrics for metadata + fn get_metrics(&self) -> HashMap { + HashMap::new() + } + + /// Get architecture information + fn get_architecture_info(&self) -> HashMap { + HashMap::new() + } + + /// Validate loaded state (optional override for custom validation) + fn validate_loaded_state(&self) -> Result<(), MLError> { + Ok(()) + } +} + // ========== MODEL REGISTRY ========== /// Thread-safe model registry using DashMap for high-performance concurrent access diff --git a/crates/ml-data-validation/Cargo.toml b/crates/ml-data-validation/Cargo.toml new file mode 100644 index 000000000..209decf6c --- /dev/null +++ b/crates/ml-data-validation/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "ml-data-validation" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +authors.workspace = true +license.workspace = true +repository.workspace = true +homepage.workspace = true +documentation.workspace = true +publish.workspace = true +keywords.workspace = true +categories.workspace = true +description = "Data validation and quality checks for Foxhunt ML training data" + +[features] +default = [] + +[dependencies] +ml-core.workspace = true +serde = { workspace = true, features = ["derive"] } +anyhow.workspace = true +chrono.workspace = true + +[dev-dependencies] +tokio = { workspace = true, features = ["test-util", "macros"] } +serde_json.workspace = true + +[lints] +workspace = true diff --git a/crates/ml/src/data_validation/corrector.rs b/crates/ml-data-validation/src/corrector.rs similarity index 99% rename from crates/ml/src/data_validation/corrector.rs rename to crates/ml-data-validation/src/corrector.rs index a992c0afe..901b4c839 100644 --- a/crates/ml/src/data_validation/corrector.rs +++ b/crates/ml-data-validation/src/corrector.rs @@ -10,7 +10,7 @@ //! All corrections are conservative and preserve data integrity. //! Original data is never modified in-place. -use crate::types::OHLCVBar; +use crate::OHLCVBar; use anyhow::Result; /// Data corrector for automatic fixes @@ -219,6 +219,7 @@ impl Default for DataCorrector { // Helper functions /// Calculate mean and standard deviation +#[cfg_attr(not(test), allow(dead_code))] fn calculate_mean_std(values: &[f64]) -> (f64, f64) { if values.is_empty() { return (0.0, 0.0); diff --git a/crates/ml/src/data_validation/cpcv.rs b/crates/ml-data-validation/src/cpcv.rs similarity index 99% rename from crates/ml/src/data_validation/cpcv.rs rename to crates/ml-data-validation/src/cpcv.rs index 43ee41338..016dbff27 100644 --- a/crates/ml/src/data_validation/cpcv.rs +++ b/crates/ml-data-validation/src/cpcv.rs @@ -20,7 +20,7 @@ //! ## Usage //! //! ```rust,no_run -//! use ml::data_validation::cpcv::{CPCVConfig, CPCVValidator}; +//! use ml_data_validation::cpcv::{CPCVConfig, CPCVValidator}; //! //! let config = CPCVConfig::default(); //! let validator = CPCVValidator::new(config)?; @@ -33,7 +33,7 @@ //! //! println!("Mean return: {:.4}", result.mean_return); //! println!("P(loss): {:.2}%", result.probability_of_loss * 100.0); -//! # Ok::<(), ml::MLError>(()) +//! # Ok::<(), ml_data_validation::MLError>(()) //! ``` use crate::MLError; diff --git a/crates/ml/src/data_validation/fdr.rs b/crates/ml-data-validation/src/fdr.rs similarity index 99% rename from crates/ml/src/data_validation/fdr.rs rename to crates/ml-data-validation/src/fdr.rs index 86945cbdd..929eb112a 100644 --- a/crates/ml/src/data_validation/fdr.rs +++ b/crates/ml-data-validation/src/fdr.rs @@ -20,7 +20,7 @@ //! ## Usage //! //! ```rust,no_run -//! use ml::data_validation::fdr::{FDRConfig, FDRCorrector, FDRMethod}; +//! use ml_data_validation::fdr::{FDRConfig, FDRCorrector, FDRMethod}; //! //! let config = FDRConfig { //! alpha: 0.05, diff --git a/crates/ml-data-validation/src/lib.rs b/crates/ml-data-validation/src/lib.rs new file mode 100644 index 000000000..3e93c6ea3 --- /dev/null +++ b/crates/ml-data-validation/src/lib.rs @@ -0,0 +1,70 @@ +#![deny(clippy::unwrap_used, clippy::expect_used)] +#![cfg_attr(test, allow(clippy::unwrap_used, clippy::expect_used))] +//! # ML Data Validation +//! +//! Data quality validation, cross-validation, and statistical correction +//! for Foxhunt ML training data. +//! +//! ## Modules +//! +//! - [`corrector`] -- Automatic correction of data quality issues +//! - [`cpcv`] -- Combinatorial Purged Cross-Validation (Lopez de Prado) +//! - [`fdr`] -- False Discovery Rate correction for multiple hypothesis testing +//! - [`rules`] -- Composable validation rules for OHLCV data +//! - [`validator`] -- Orchestrator that runs rules and generates reports + +pub mod corrector; +pub mod cpcv; +pub mod fdr; +pub mod rules; +pub mod validator; + +// Re-export core types used by this crate +pub use ml_core::types::OHLCVBar; +pub use ml_core::MLError; + +/// Technical indicators (10 essential ones). +/// +/// All indicators are calculated with standard parameters for 1-minute OHLCV data: +/// - RSI(14): Relative Strength Index +/// - MACD(12,26,9): Moving Average Convergence Divergence +/// - Bollinger Bands(20, 2.0): Price envelope +/// - ATR(14): Average True Range +/// - EMA(12, 26): Exponential moving averages +/// - Volume MA(20): Volume moving average +/// +/// This type mirrors `ml::data_loader::Indicators` so that the validation crate +/// can operate independently of the full `ml` crate. Conversion between the two +/// is trivial (both are plain `pub` field structs with identical layout). +#[derive(Debug, Clone)] +pub struct Indicators { + /// RSI(14) - values 0-100 + pub rsi: Vec, + /// MACD line (12,26) + pub macd: Vec, + /// MACD signal line (9) + pub macd_signal: Vec, + /// Bollinger upper band (20, 2.0) + pub bb_upper: Vec, + /// Bollinger middle band (SMA 20) + pub bb_middle: Vec, + /// Bollinger lower band (20, 2.0) + pub bb_lower: Vec, + /// ATR(14) - volatility measure + pub atr: Vec, + /// EMA(12) - fast exponential moving average + pub ema_fast: Vec, + /// EMA(26) - slow exponential moving average + pub ema_slow: Vec, + /// Volume MA(20) - volume moving average + pub volume_ma: Vec, +} + +// Re-export main types +pub use corrector::DataCorrector; +pub use cpcv::{CPCVConfig, CPCVResult, CPCVSplit, CPCVValidator}; +pub use fdr::{FDRConfig, FDRCorrector, FDRMethod, FDRResult}; +pub use rules::{ + CompletenessRule, ContinuityRule, IndicatorRule, IntegrityRule, TimestampRule, ValidationRule, +}; +pub use validator::{DataValidator, ValidationReport, ValidationResult}; diff --git a/crates/ml/src/data_validation/rules.rs b/crates/ml-data-validation/src/rules.rs similarity index 99% rename from crates/ml/src/data_validation/rules.rs rename to crates/ml-data-validation/src/rules.rs index 2c2980e5e..e12a6d77b 100644 --- a/crates/ml/src/data_validation/rules.rs +++ b/crates/ml-data-validation/src/rules.rs @@ -4,8 +4,8 @@ //! Each rule implements the `ValidationRule` trait and can be composed //! into a comprehensive validation pipeline. -use crate::data_loader::Indicators; -use crate::types::OHLCVBar; +use crate::Indicators; +use crate::OHLCVBar; use anyhow::Result; /// Validation error information diff --git a/crates/ml/src/data_validation/validator.rs b/crates/ml-data-validation/src/validator.rs similarity index 99% rename from crates/ml/src/data_validation/validator.rs rename to crates/ml-data-validation/src/validator.rs index 10f0245a7..cfaad5e8e 100644 --- a/crates/ml/src/data_validation/validator.rs +++ b/crates/ml-data-validation/src/validator.rs @@ -5,8 +5,8 @@ use std::fmt::Write as _; use super::rules::{Severity, ValidationError, ValidationRule}; -use crate::data_loader::Indicators; -use crate::types::OHLCVBar; +use crate::Indicators; +use crate::OHLCVBar; use anyhow::Result; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -386,7 +386,7 @@ impl From for ValidationReport { #[cfg(test)] mod tests { use super::*; - use crate::data_validation::rules::IntegrityRule; + use crate::rules::IntegrityRule; #[test] fn test_validator_creation() { diff --git a/crates/ml-dqn/src/checkpoint.rs b/crates/ml-dqn/src/checkpoint.rs new file mode 100644 index 000000000..59916a93f --- /dev/null +++ b/crates/ml-dqn/src/checkpoint.rs @@ -0,0 +1,189 @@ +//! DQN checkpoint implementation — serialization/deserialization of DQN agent state. + +use std::collections::HashMap; + +use async_trait::async_trait; +use ml_core::{Checkpointable, MLError, ModelType}; +use rust_decimal::Decimal; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use tracing::{debug, warn}; + +use crate::agent::DQNAgent; +use crate::DQNConfig; + +/// Serializable state for DQN model +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DQNCheckpointState { + pub config: DQNConfig, + pub epoch: Option, + pub step: Option, + pub total_episodes: u64, + pub total_steps: u64, + pub q_network_weights: Vec, + pub target_network_weights: Vec, + pub replay_buffer_size: usize, + pub replay_buffer_capacity: usize, + pub average_reward: f64, + pub epsilon: f64, + pub loss_history: Vec, + pub total_inferences: u64, + pub avg_inference_time_us: f64, +} + +#[async_trait] +impl Checkpointable for DQNAgent { + fn model_type(&self) -> ModelType { + ModelType::DQN + } + + fn model_name(&self) -> &str { + "dqn_agent" + } + + fn model_version(&self) -> &str { + "1.0.0" + } + + async fn serialize_state(&self) -> Result, MLError> { + let state = DQNCheckpointState { + config: self.config.clone(), + epoch: None, + step: None, + total_episodes: self.metrics.total_episodes, + total_steps: 0, + q_network_weights: self + .extract_network_weights("q_network") + .unwrap_or_default(), + target_network_weights: vec![], + replay_buffer_size: self.get_replay_buffer_size(), + replay_buffer_capacity: self.config.replay_buffer_capacity, + average_reward: self.get_average_reward(), + epsilon: self.config.epsilon_start as f64, + loss_history: vec![], + total_inferences: 0, + avg_inference_time_us: 0.0, + }; + + let serialized = serde_json::to_vec(&state) + .map_err(|e| MLError::ModelError(format!("DQN serialization failed: {}", e)))?; + + debug!("Serialized DQN state: {} bytes", serialized.len()); + Ok(serialized) + } + + async fn deserialize_state(&mut self, data: &[u8]) -> Result<(), MLError> { + let state: DQNCheckpointState = serde_json::from_slice(data) + .map_err(|e| MLError::ModelError(format!("DQN deserialization failed: {}", e)))?; + + self.config = state.config; + if !state.q_network_weights.is_empty() { + if let Err(e) = self.restore_network_weights("q_network", &state.q_network_weights) { + warn!("Failed to restore Q-network weights: {}", e); + } + } + + if !state.target_network_weights.is_empty() { + if let Err(e) = + self.restore_network_weights("target_network", &state.target_network_weights) + { + warn!("Failed to restore target network weights: {}", e); + } + } + + self.metrics.total_episodes = state.total_episodes; + self.metrics.avg_reward = Decimal::try_from(state.average_reward).unwrap_or(Decimal::ZERO); + + debug!("Deserialized DQN state from {} bytes", data.len()); + Ok(()) + } + + fn get_training_state(&self) -> (Option, Option, Option, Option) { + let current_epoch = None; + let total_episodes = Some(self.metrics.total_episodes); + let average_reward = + Some(TryInto::::try_into(self.metrics.avg_reward).unwrap_or(0.0)); + let loss = Some(TryInto::::try_into(self.metrics.current_loss).unwrap_or(0.0)); + (current_epoch, total_episodes, average_reward, loss) + } + + fn get_hyperparameters(&self) -> HashMap { + let mut params = HashMap::new(); + params.insert( + "learning_rate".to_owned(), + Value::from(self.config.learning_rate), + ); + params.insert( + "discount_factor".to_owned(), + Value::from(self.config.gamma), + ); + params.insert( + "epsilon".to_owned(), + Value::from(self.config.epsilon_start), + ); + params.insert( + "epsilon_decay".to_owned(), + Value::from(self.config.epsilon_decay), + ); + params.insert( + "epsilon_min".to_owned(), + Value::from(self.config.epsilon_end), + ); + params.insert( + "replay_buffer_size".to_owned(), + Value::from(self.config.replay_buffer_capacity), + ); + params.insert( + "batch_size".to_owned(), + Value::from(self.config.batch_size), + ); + params.insert( + "target_update_frequency".to_owned(), + Value::from(self.config.target_update_freq), + ); + params + } + + fn get_metrics(&self) -> HashMap { + let mut metrics = HashMap::new(); + metrics.insert( + "total_episodes".to_owned(), + self.metrics.total_episodes as f64, + ); + metrics.insert( + "average_reward".to_owned(), + TryInto::::try_into(self.metrics.avg_reward).unwrap_or(0.0), + ); + metrics.insert( + "current_loss".to_owned(), + TryInto::::try_into(self.metrics.current_loss).unwrap_or(0.0), + ); + metrics.insert("epsilon".to_owned(), self.config.epsilon_start as f64); + metrics.insert( + "replay_buffer_size".to_owned(), + self.get_replay_buffer_size() as f64, + ); + metrics.insert( + "exploration_rate".to_owned(), + self.config.epsilon_start as f64, + ); + metrics + } + + fn get_architecture_info(&self) -> HashMap { + let mut info = HashMap::new(); + info.insert("network_type".to_owned(), Value::from("DQN")); + info.insert("input_size".to_owned(), Value::from(self.config.state_dim)); + info.insert( + "hidden_size".to_owned(), + Value::from(self.config.hidden_dims.len()), + ); + info.insert( + "output_size".to_owned(), + Value::from(self.config.num_actions), + ); + info.insert("num_hidden_layers".to_owned(), Value::from(2)); + info.insert("activation".to_owned(), Value::from("ReLU")); + info + } +} diff --git a/crates/ml/src/evaluation/engine.rs b/crates/ml-dqn/src/evaluation/engine.rs similarity index 99% rename from crates/ml/src/evaluation/engine.rs rename to crates/ml-dqn/src/evaluation/engine.rs index c50211426..91b03e348 100644 --- a/crates/ml/src/evaluation/engine.rs +++ b/crates/ml-dqn/src/evaluation/engine.rs @@ -219,7 +219,7 @@ impl EvaluationEngine { &mut self, bar_idx: usize, bar: &OHLCVBarF32, - action: &crate::common::action::FactoredAction, + action: &crate::action_space::FactoredAction, ) { let target = action.target_exposure(); // -1.0, -0.5, 0.0, +0.5, +1.0 let delta = target - self.current_exposure; @@ -367,7 +367,7 @@ pub struct ActionDistribution { #[cfg(test)] mod tests { use super::*; - use crate::common::action::{ExposureLevel, FactoredAction, OrderType, Urgency}; + use crate::action_space::{ExposureLevel, FactoredAction, OrderType, Urgency}; fn bar(close: f32) -> OHLCVBarF32 { OHLCVBarF32 { diff --git a/crates/ml/src/evaluation/metrics.rs b/crates/ml-dqn/src/evaluation/metrics.rs similarity index 99% rename from crates/ml/src/evaluation/metrics.rs rename to crates/ml-dqn/src/evaluation/metrics.rs index 732c01254..3f3b9910a 100644 --- a/crates/ml/src/evaluation/metrics.rs +++ b/crates/ml-dqn/src/evaluation/metrics.rs @@ -258,7 +258,7 @@ fn calculate_calmar_ratio(annualized_return: f64, max_drawdown_pct: f64) -> f64 /// /// VaR: Maximum expected loss at given confidence level /// CVaR: Average loss beyond VaR threshold (tail risk) -pub(crate) fn calculate_var_cvar(returns: &[f64], confidence_level: f64) -> (f64, f64) { +pub fn calculate_var_cvar(returns: &[f64], confidence_level: f64) -> (f64, f64) { if returns.is_empty() { return (0.0, 0.0); } diff --git a/crates/ml-dqn/src/evaluation/mod.rs b/crates/ml-dqn/src/evaluation/mod.rs new file mode 100644 index 000000000..7722123a6 --- /dev/null +++ b/crates/ml-dqn/src/evaluation/mod.rs @@ -0,0 +1,12 @@ +//! DQN Evaluation Module +//! +//! Backtesting evaluation for DQN models: position tracking, trade recording, +//! performance metrics (Sharpe, win rate, drawdown), and report generation. + +pub mod engine; +pub mod metrics; +pub mod report; + +pub use engine::{EvaluationEngine, Position, Trade}; +pub use metrics::PerformanceMetrics; +pub use report::BacktestReport; diff --git a/crates/ml/src/evaluation/report.rs b/crates/ml-dqn/src/evaluation/report.rs similarity index 100% rename from crates/ml/src/evaluation/report.rs rename to crates/ml-dqn/src/evaluation/report.rs diff --git a/crates/ml-dqn/src/lib.rs b/crates/ml-dqn/src/lib.rs index b67153144..0d2fada11 100644 --- a/crates/ml-dqn/src/lib.rs +++ b/crates/ml-dqn/src/lib.rs @@ -79,6 +79,12 @@ pub mod rmsnorm; #[cfg(feature = "cuda")] pub mod gpu_replay_buffer; +// Checkpoint (Checkpointable impl for DQNAgent) +pub mod checkpoint; + +// DQN evaluation (backtesting engine, metrics, reports) +pub mod evaluation; + // Re-export core DQN types for public usage pub use action_space::{ExposureLevel, FactoredAction, OrderType, Urgency}; pub use order_router::OrderRouter; diff --git a/crates/ml-ensemble/Cargo.toml b/crates/ml-ensemble/Cargo.toml new file mode 100644 index 000000000..e168eb5b1 --- /dev/null +++ b/crates/ml-ensemble/Cargo.toml @@ -0,0 +1,49 @@ +[package] +name = "ml-ensemble" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +authors.workspace = true +license.workspace = true +repository.workspace = true +homepage.workspace = true +documentation.workspace = true +publish.workspace = true +keywords.workspace = true +categories.workspace = true +description = "ML ensemble coordination — voting, confidence, gating, hot-swap, inference pipeline" + +[features] +default = ["cuda"] +cuda = ["candle-core/cuda", "candle-core/cudnn"] + +[dependencies] +ml-core = { path = "../ml-core", default-features = false } +common.workspace = true +async-trait.workspace = true +candle-core = { git = "https://github.com/huggingface/candle", rev = "671de1db" } +candle-nn = { git = "https://github.com/huggingface/candle", rev = "671de1db" } +serde = { workspace = true, features = ["derive"] } +serde_json.workspace = true +chrono.workspace = true +chrono-tz = "0.10" +tracing.workspace = true +uuid.workspace = true +tokio.workspace = true +anyhow.workspace = true +once_cell = "1.19" +prometheus.workspace = true +statrs.workspace = true +thiserror.workspace = true +parking_lot = { version = "0.12", features = ["hardware-lock-elision"] } +dashmap = { workspace = true } +rand.workspace = true +ndarray = { workspace = true, features = ["rayon"] } +crossbeam = { version = "0.8", features = ["std"] } +rayon.workspace = true + +[dev-dependencies] +tokio = { workspace = true, features = ["test-util", "macros"] } + +[lints] +workspace = true diff --git a/crates/ml/src/ensemble/ab_testing.rs b/crates/ml-ensemble/src/ab_testing.rs similarity index 99% rename from crates/ml/src/ensemble/ab_testing.rs rename to crates/ml-ensemble/src/ab_testing.rs index bf674059c..5be69e493 100644 --- a/crates/ml/src/ensemble/ab_testing.rs +++ b/crates/ml-ensemble/src/ab_testing.rs @@ -11,7 +11,7 @@ use thiserror::Error; use tokio::sync::RwLock; use uuid::Uuid; -use crate::ensemble::EnsembleError; +use crate::EnsembleError; /// A/B test group assignment #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Hash)] diff --git a/crates/ml/src/ensemble/adaptive_ml_integration.rs b/crates/ml-ensemble/src/adaptive_ml_integration.rs similarity index 99% rename from crates/ml/src/ensemble/adaptive_ml_integration.rs rename to crates/ml-ensemble/src/adaptive_ml_integration.rs index 8cb57a0c7..bebde01c9 100644 --- a/crates/ml/src/ensemble/adaptive_ml_integration.rs +++ b/crates/ml-ensemble/src/adaptive_ml_integration.rs @@ -4,7 +4,7 @@ //! This module bridges the ML ensemble coordinator with market regime detection to //! dynamically adjust model weights based on current market conditions. -use crate::ensemble::EnsembleDecision; +use crate::EnsembleDecision; use crate::{MLError, MLResult, ModelPrediction}; use std::collections::{HashMap, VecDeque}; use std::sync::Arc; diff --git a/crates/ml/src/ensemble/aggregator.rs b/crates/ml-ensemble/src/aggregator.rs similarity index 96% rename from crates/ml/src/ensemble/aggregator.rs rename to crates/ml-ensemble/src/aggregator.rs index c2a967e39..0e27d1772 100644 --- a/crates/ml/src/ensemble/aggregator.rs +++ b/crates/ml-ensemble/src/aggregator.rs @@ -8,8 +8,8 @@ use std::sync::Arc; use parking_lot::RwLock; use serde::{Deserialize, Serialize}; -use crate::ensemble::confidence::ConfidenceCalculator; -use crate::ensemble::weights::ModelWeights; +use crate::confidence::ConfidenceCalculator; +use crate::weights::ModelWeights; use crate::MLError; /// Model signal structure diff --git a/crates/ml/src/ensemble/confidence.rs b/crates/ml-ensemble/src/confidence.rs similarity index 100% rename from crates/ml/src/ensemble/confidence.rs rename to crates/ml-ensemble/src/confidence.rs diff --git a/crates/ml/src/ensemble/conviction_gates.rs b/crates/ml-ensemble/src/conviction_gates.rs similarity index 100% rename from crates/ml/src/ensemble/conviction_gates.rs rename to crates/ml-ensemble/src/conviction_gates.rs diff --git a/crates/ml/src/ensemble/coordinator.rs b/crates/ml-ensemble/src/coordinator.rs similarity index 96% rename from crates/ml/src/ensemble/coordinator.rs rename to crates/ml-ensemble/src/coordinator.rs index b802d7978..63159e7f3 100644 --- a/crates/ml/src/ensemble/coordinator.rs +++ b/crates/ml-ensemble/src/coordinator.rs @@ -4,11 +4,11 @@ //! from multiple ML models (DQN, PPO, TFT, MAMBA-2, Liquid, TLOB) for production trading decisions. //! Supports dynamic weighting based on performance and model diversity metrics. -use crate::ensemble::conviction_gates::{ +use crate::conviction_gates::{ ConvictionGateConfig, ConvictionGateEvaluator, ConvictionGateOutcome, GateInput, TradingSession, }; -use crate::ensemble::inference_adapter::{FeatureVector, ModelInferenceAdapter}; -use crate::ensemble::{EnsembleDecision, ModelVote, ModelWeight, TradingAction}; +use crate::inference_adapter::{FeatureVector, ModelInferenceAdapter}; +use crate::{EnsembleDecision, ModelVote, ModelWeight, TradingAction}; use crate::{Features, MLError, MLResult, ModelPrediction}; use chrono::{DateTime, Timelike, Utc}; use chrono_tz::America::New_York; @@ -744,7 +744,7 @@ mod tests { #[tokio::test] async fn test_ensemble_prediction() { - use crate::ensemble::inference_adapter::{ + use crate::inference_adapter::{ EnsemblePrediction, FeatureVector, ModelInferenceAdapter, PredictionMeta, }; @@ -864,7 +864,7 @@ mod tests { #[tokio::test] async fn test_ensemble_with_real_adapter() { - use crate::ensemble::inference_adapter::{ + use crate::inference_adapter::{ EnsemblePrediction, FeatureVector, ModelInferenceAdapter, PredictionMeta, }; @@ -911,47 +911,11 @@ mod tests { assert_eq!(decision.model_count(), 1); } - #[tokio::test] - async fn test_full_ensemble_with_dqn_adapter() { - use crate::ensemble::adapters::DqnInferenceAdapter; - use crate::dqn::dqn::DQNConfig; - - let dqn_config = DQNConfig { - state_dim: 56, - num_actions: 45, - hidden_dims: vec![64, 64], - ..Default::default() - }; - - let adapter = DqnInferenceAdapter::new(dqn_config).unwrap(); - - let mut coordinator = EnsembleCoordinator::new(); - coordinator.add_adapter(Box::new(adapter)); - coordinator - .register_model("DQN".to_owned(), 1.0) - .await - .unwrap(); - - let features = Features::new( - vec![ - 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 0.1, 0.2, 0.3, 0.4, 0.5, - 0.6, 0.7, 0.8, 0.9, 1.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, - 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 0.1, 0.2, 0.3, 0.4, 0.5, - 0.6, 0.7, 0.8, 0.9, 1.0, 0.55, 0.0, 0.0, 0.0, 0.0, 0.0, - ], - (0..56).map(|i| format!("f{}", i)).collect(), - ); - - let decision = coordinator.predict(&features).await.unwrap(); - - assert!(decision.confidence > 0.0); - assert!(decision.signal >= -1.0 && decision.signal <= 1.0); - assert_eq!(decision.model_count(), 1); - } + // NOTE: test_full_ensemble_with_dqn_adapter lives in ml crate (depends on DqnInferenceAdapter) #[tokio::test] async fn test_ensemble_filters_nan_inf_predictions() { - use crate::ensemble::inference_adapter::{ + use crate::inference_adapter::{ EnsemblePrediction, FeatureVector, ModelInferenceAdapter, PredictionMeta, }; @@ -1070,7 +1034,7 @@ mod tests { #[tokio::test] async fn test_ensemble_all_nan_returns_error() { - use crate::ensemble::inference_adapter::{ + use crate::inference_adapter::{ EnsemblePrediction, FeatureVector, ModelInferenceAdapter, PredictionMeta, }; diff --git a/crates/ml/src/ensemble/coordinator_extended.rs b/crates/ml-ensemble/src/coordinator_extended.rs similarity index 99% rename from crates/ml/src/ensemble/coordinator_extended.rs rename to crates/ml-ensemble/src/coordinator_extended.rs index 2e3fe394a..2be008b0a 100644 --- a/crates/ml/src/ensemble/coordinator_extended.rs +++ b/crates/ml-ensemble/src/coordinator_extended.rs @@ -3,7 +3,7 @@ //! This module extends the coordinator to support all 6 models (DQN, PPO, TFT, MAMBA-2, Liquid, TLOB) //! with dynamic weighting, correlation analysis, and performance attribution. -use crate::ensemble::{EnsembleDecision, ModelVote, TradingAction}; +use crate::{EnsembleDecision, ModelVote, TradingAction}; use crate::{MLError, MLResult, ModelPrediction}; use std::collections::HashMap; use std::sync::Arc; diff --git a/crates/ml/src/ensemble/cuda_streams.rs b/crates/ml-ensemble/src/cuda_streams.rs similarity index 100% rename from crates/ml/src/ensemble/cuda_streams.rs rename to crates/ml-ensemble/src/cuda_streams.rs diff --git a/crates/ml/src/ensemble/decision.rs b/crates/ml-ensemble/src/decision.rs similarity index 100% rename from crates/ml/src/ensemble/decision.rs rename to crates/ml-ensemble/src/decision.rs diff --git a/crates/ml/src/ensemble/gate_optimizer.rs b/crates/ml-ensemble/src/gate_optimizer.rs similarity index 100% rename from crates/ml/src/ensemble/gate_optimizer.rs rename to crates/ml-ensemble/src/gate_optimizer.rs diff --git a/crates/ml/src/ensemble/hot_swap.rs b/crates/ml-ensemble/src/hot_swap.rs similarity index 100% rename from crates/ml/src/ensemble/hot_swap.rs rename to crates/ml-ensemble/src/hot_swap.rs diff --git a/crates/ml/src/ensemble/inference_adapter.rs b/crates/ml-ensemble/src/inference_adapter.rs similarity index 100% rename from crates/ml/src/ensemble/inference_adapter.rs rename to crates/ml-ensemble/src/inference_adapter.rs diff --git a/crates/ml/src/ensemble/inference_ensemble.rs b/crates/ml-ensemble/src/inference_ensemble.rs similarity index 99% rename from crates/ml/src/ensemble/inference_ensemble.rs rename to crates/ml-ensemble/src/inference_ensemble.rs index b6c5d0fa5..69ec795f4 100644 --- a/crates/ml/src/ensemble/inference_ensemble.rs +++ b/crates/ml-ensemble/src/inference_ensemble.rs @@ -10,7 +10,7 @@ use rayon::prelude::*; use candle_core::Tensor; -use crate::ensemble::inference_adapter::{ +use crate::inference_adapter::{ EnsemblePrediction, FeatureVector, ModelInferenceAdapter, PredictionMeta, RawPrediction, }; use crate::{MLError, MLResult}; diff --git a/crates/ml-ensemble/src/lib.rs b/crates/ml-ensemble/src/lib.rs new file mode 100644 index 000000000..ddc711123 --- /dev/null +++ b/crates/ml-ensemble/src/lib.rs @@ -0,0 +1,171 @@ +#![deny(clippy::unwrap_used, clippy::expect_used)] +#![cfg_attr(test, allow(clippy::unwrap_used, clippy::expect_used))] +#![allow(dead_code)] +#![allow(missing_docs)] +#![allow(missing_debug_implementations)] +#![allow(unused_crate_dependencies)] +#![allow(clippy::float_arithmetic)] +#![allow(clippy::non_ascii_literal)] +#![allow(clippy::str_to_string)] +#![allow(clippy::partial_pub_fields)] +#![allow(clippy::multiple_inherent_impl)] +#![allow(clippy::same_name_method)] +#![allow(clippy::shadow_reuse)] +#![allow(clippy::shadow_unrelated)] +#![allow(clippy::shadow_same)] +#![allow(clippy::doc_markdown)] +#![allow(clippy::indexing_slicing)] +#![allow(clippy::missing_const_for_fn)] +#![allow(clippy::module_name_repetitions)] +#![allow(clippy::integer_division)] +#![allow(clippy::cognitive_complexity)] +#![allow(clippy::similar_names)] +#![allow(clippy::clone_on_ref_ptr)] +#![allow(clippy::too_many_lines)] +#![allow(clippy::as_conversions)] +#![allow(clippy::cast_precision_loss)] +#![allow(clippy::cast_possible_truncation)] +#![allow(clippy::default_numeric_fallback)] +#![allow(clippy::arithmetic_side_effects)] +#![allow(clippy::needless_range_loop)] +#![allow(clippy::into_iter_on_ref)] +#![allow(clippy::new_without_default)] +#![allow(clippy::manual_let_else)] +#![allow(clippy::unnecessary_wraps)] +#![allow(clippy::too_many_arguments)] +#![allow(clippy::must_use_candidate)] +#![allow(clippy::missing_errors_doc)] +#![allow(clippy::cast_sign_loss)] +#![allow(clippy::cast_possible_wrap)] +#![allow(clippy::cast_lossless)] +#![allow(clippy::unused_async)] +#![allow(clippy::match_same_arms)] +#![allow(clippy::unused_self)] +#![allow(clippy::map_err_ignore)] +#![allow(clippy::single_match_else)] +#![allow(clippy::wildcard_imports)] +#![allow(clippy::unnecessary_cast)] + +//! Ensemble signal aggregation for trading models + +use thiserror::Error; + +// Re-export core types so sub-modules can use `crate::X` +pub use ml_core::{Features, HealthStatus, MLError, MLResult, ModelPrediction}; +pub use common::model_types::ModelType; + +pub mod ab_testing; +pub mod adaptive_ml_integration; +pub mod aggregator; +pub mod confidence; +pub mod conviction_gates; +pub mod coordinator; +pub mod coordinator_extended; +pub mod cuda_streams; +pub mod decision; +pub mod gate_optimizer; +pub mod hot_swap; +pub mod inference_adapter; +pub mod inference_ensemble; +pub mod metrics; +pub mod model; +pub mod signal; +pub mod stream_ensemble; +pub mod training_integration; +pub mod voting; +pub mod weight_optimizer; +pub mod weights; + +// Re-export key types that are used across ensemble modules +pub use ab_testing::{ + ABGroup, ABMetricsTracker, ABTestConfig, ABTestResults, ABTestRouter, GroupMetrics, + Recommendation, StatisticalTestResult, +}; +pub use adaptive_ml_integration::{ + AdaptiveMLEnsemble, AdaptiveMetrics, MarketRegime, PricePoint, RegimeConfig, +}; +pub use aggregator::{ModelSignal, SignalMetadata, SignalStatistics}; +pub use confidence::{ + AggregationConfig, AleatoricConfig, CalibrationParams, CombinationMethod, + ConfidenceAggregator, DisagreementRecord, DisagreementTracker, EnsemblePredictionWithUncertainty, + EpistemicConfig, IntervalCombiner, ModelContribution, PredictionInterval, ReliabilityRecord, + ReliabilityScorer, UncertaintyDecomposition, UncertaintyQuantifier, VarianceEstimationMethod, +}; +pub use conviction_gates::{ + ConvictionGateConfig, ConvictionGateEvaluator, ConvictionGateOutcome, GateEvaluation, + GateInput, GatePassResult, GateRejection, TradingSession, +}; +pub use coordinator::{EnsembleCoordinator, ModelRegistry, SignalAggregator}; +pub use coordinator_extended::{ + DiversityAnalyzer, DiversityMetrics, EnsembleConfig as ExtendedEnsembleConfig, + ExtendedEnsembleCoordinator, ModelPerformance, PerformanceAttribution, PerformanceTracker, + SupportedModel, WeightSnapshot, +}; +pub use decision::{EnsembleDecision, ModelVote, ModelWeight, PerformanceMetrics, TradingAction}; +pub use gate_optimizer::{ + GateBucketMetrics, GateOptimizationResult, GateOptimizer, GateOptimizerConfig, + ThresholdAdjustment, +}; +pub use hot_swap::{ + CanaryMetrics, CanaryResult, CheckpointModel, CheckpointValidator, HotSwapManager, + ModelBufferPair, RollbackPolicy, ValidationResult, +}; +pub use inference_adapter::{EnsemblePrediction, FeatureVector, ModelInferenceAdapter, PredictionMeta}; +pub use metrics::{ + EnsembleMetrics, CANARY_MONITORING_TOTAL, CHECKPOINT_SWAPS_TOTAL, + CHECKPOINT_SWAP_LATENCY_MICROSECONDS, CHECKPOINT_VALIDATION_TOTAL, +}; +pub use training_integration::EnsembleTrainingIntegration; +pub use weight_optimizer::{ + ModelRollingMetrics, OptimizationResult, WeightAdjustment, WeightOptimizer, + WeightOptimizerConfig, +}; + +/// Errors that can occur in ensemble operations +#[derive(Error, Debug)] +pub enum EnsembleError { + #[error("Failed to acquire lock: {0}")] + LockAcquisitionFailed(String), + + #[error("Invalid ensemble configuration: {0}")] + InvalidConfiguration(String), + + #[error("Model not found: {0}")] + ModelNotFound(String), + + #[error("Insufficient models for ensemble: expected {expected}, got {actual}")] + InsufficientModels { expected: usize, actual: usize }, + + #[error("Weight calculation failed: {0}")] + WeightCalculationFailed(String), + + #[error("Aggregation failed: {0}")] + AggregationFailed(String), +} + +// `From for MLError` lives here because ml-ensemble owns +// EnsembleError (local) and imports MLError from ml-core (dependency). +// This satisfies the orphan rule: EnsembleError is a local type parameter. +impl From for MLError { + fn from(err: EnsembleError) -> Self { + match err { + EnsembleError::InvalidConfiguration(msg) => MLError::ConfigError(msg), + EnsembleError::ModelNotFound(msg) => MLError::ModelNotFound(msg), + EnsembleError::InsufficientModels { expected, actual } => { + MLError::ValidationError { + message: format!( + "Insufficient models: expected {}, got {}", + expected, actual + ), + } + } + EnsembleError::LockAcquisitionFailed(msg) => MLError::LockError(msg), + EnsembleError::WeightCalculationFailed(msg) => { + MLError::ModelError(format!("Weight calculation failed: {}", msg)) + } + EnsembleError::AggregationFailed(msg) => { + MLError::InferenceError(format!("Aggregation failed: {}", msg)) + } + } + } +} diff --git a/crates/ml/src/ensemble/metrics.rs b/crates/ml-ensemble/src/metrics.rs similarity index 100% rename from crates/ml/src/ensemble/metrics.rs rename to crates/ml-ensemble/src/metrics.rs diff --git a/crates/ml/src/ensemble/model.rs b/crates/ml-ensemble/src/model.rs similarity index 99% rename from crates/ml/src/ensemble/model.rs rename to crates/ml-ensemble/src/model.rs index c2ccf1b13..e3cfd3a54 100644 --- a/crates/ml/src/ensemble/model.rs +++ b/crates/ml-ensemble/src/model.rs @@ -10,9 +10,8 @@ use std::time::{SystemTime, UNIX_EPOCH}; use crossbeam::atomic::AtomicCell; use serde::{Deserialize, Serialize}; -use super::aggregator::ModelSignal; +use crate::aggregator::ModelSignal; use crate::{HealthStatus, MLError}; -use super::*; use common::trading::MarketRegime; /// Configuration for ensemble models diff --git a/crates/ml/src/ensemble/signal.rs b/crates/ml-ensemble/src/signal.rs similarity index 95% rename from crates/ml/src/ensemble/signal.rs rename to crates/ml-ensemble/src/signal.rs index b8e1ae582..db59b3d37 100644 --- a/crates/ml/src/ensemble/signal.rs +++ b/crates/ml-ensemble/src/signal.rs @@ -1,6 +1,6 @@ //! Trade signal derived from ensemble predictions. -use crate::ensemble::inference_adapter::EnsemblePrediction; +use crate::inference_adapter::EnsemblePrediction; /// Trade action mapped from ensemble direction. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -58,7 +58,7 @@ impl std::fmt::Display for TradeAction { #[cfg(test)] mod tests { use super::*; - use crate::ensemble::inference_adapter::PredictionMeta; + use crate::inference_adapter::PredictionMeta; fn make_pred(direction: f64, confidence: f64) -> EnsemblePrediction { EnsemblePrediction { diff --git a/crates/ml/src/ensemble/stream_ensemble.rs b/crates/ml-ensemble/src/stream_ensemble.rs similarity index 99% rename from crates/ml/src/ensemble/stream_ensemble.rs rename to crates/ml-ensemble/src/stream_ensemble.rs index 6a48673b0..bab7e84c8 100644 --- a/crates/ml/src/ensemble/stream_ensemble.rs +++ b/crates/ml-ensemble/src/stream_ensemble.rs @@ -16,8 +16,8 @@ use rayon::prelude::*; use candle_core::Tensor; -use crate::ensemble::cuda_streams::CudaStreamPool; -use crate::ensemble::inference_adapter::{ +use crate::cuda_streams::CudaStreamPool; +use crate::inference_adapter::{ EnsemblePrediction, FeatureVector, ModelInferenceAdapter, PredictionMeta, RawPrediction, }; use crate::{MLError, MLResult}; diff --git a/crates/ml/src/ensemble/training_integration.rs b/crates/ml-ensemble/src/training_integration.rs similarity index 99% rename from crates/ml/src/ensemble/training_integration.rs rename to crates/ml-ensemble/src/training_integration.rs index e9c33e071..f34143f1b 100644 --- a/crates/ml/src/ensemble/training_integration.rs +++ b/crates/ml-ensemble/src/training_integration.rs @@ -15,7 +15,7 @@ use std::path::Path; use anyhow::{anyhow, Result}; use tracing::{debug, info}; -use crate::ensemble::EnsembleCoordinator; +use crate::EnsembleCoordinator; use crate::ModelPrediction; /// Training integration for ensemble system diff --git a/crates/ml/src/ensemble/voting.rs b/crates/ml-ensemble/src/voting.rs similarity index 97% rename from crates/ml/src/ensemble/voting.rs rename to crates/ml-ensemble/src/voting.rs index b872b921a..81e1f7c30 100644 --- a/crates/ml/src/ensemble/voting.rs +++ b/crates/ml-ensemble/src/voting.rs @@ -7,8 +7,7 @@ use std::collections::HashMap; use serde::{Deserialize, Serialize}; -use super::*; -use crate::ensemble::ModelSignal; +use crate::ModelSignal; use crate::MLError; /// Voting strategy for ensemble aggregation diff --git a/crates/ml/src/ensemble/weight_optimizer.rs b/crates/ml-ensemble/src/weight_optimizer.rs similarity index 100% rename from crates/ml/src/ensemble/weight_optimizer.rs rename to crates/ml-ensemble/src/weight_optimizer.rs diff --git a/crates/ml/src/ensemble/weights.rs b/crates/ml-ensemble/src/weights.rs similarity index 99% rename from crates/ml/src/ensemble/weights.rs rename to crates/ml-ensemble/src/weights.rs index a49f3c7b6..9f07f4afb 100644 --- a/crates/ml/src/ensemble/weights.rs +++ b/crates/ml-ensemble/src/weights.rs @@ -8,7 +8,6 @@ use std::collections::HashMap; // CIRCULAR DEPENDENCY FIX: Use MarketRegime from core types use crate::MLError; -use super::*; #[derive(Debug, Clone)] pub enum WeightUpdateMethod { diff --git a/crates/ml-features/Cargo.toml b/crates/ml-features/Cargo.toml new file mode 100644 index 000000000..02649358c --- /dev/null +++ b/crates/ml-features/Cargo.toml @@ -0,0 +1,48 @@ +[package] +name = "ml-features" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +authors.workspace = true +license.workspace = true +repository.workspace = true +homepage.workspace = true +documentation.workspace = true +publish.workspace = true +keywords.workspace = true +categories.workspace = true +description = "Feature engineering for Foxhunt ML models" + +[features] +default = [] + +[dependencies] +# Internal workspace crates +ml-core.workspace = true +common.workspace = true +data = { path = "../data" } + +# Serialization and error handling +serde = { workspace = true, features = ["derive"] } +serde_json.workspace = true +anyhow.workspace = true +chrono.workspace = true +chrono-tz = "0.10" +tracing.workspace = true +once_cell = "1.19" + +# Async and parallelism +tokio.workspace = true +rayon.workspace = true + +# Data loading (DBN/Databento) +dbn = "0.42" +zstd.workspace = true +rand.workspace = true + +[dev-dependencies] +tokio = { workspace = true, features = ["test-util", "macros"] } +approx.workspace = true + +[lints] +workspace = true diff --git a/crates/ml/src/features/adx_features.rs b/crates/ml-features/src/adx_features.rs similarity index 99% rename from crates/ml/src/features/adx_features.rs rename to crates/ml-features/src/adx_features.rs index 27e658f6d..e3905e3eb 100644 --- a/crates/ml/src/features/adx_features.rs +++ b/crates/ml-features/src/adx_features.rs @@ -45,7 +45,7 @@ use std::collections::VecDeque; -pub use crate::types::OHLCVBar; +pub use crate::OHLCVBar; /// ADX feature extractor using Wilder's 14-period algorithm /// diff --git a/crates/ml/src/features/alternative_bars.rs b/crates/ml-features/src/alternative_bars.rs similarity index 99% rename from crates/ml/src/features/alternative_bars.rs rename to crates/ml-features/src/alternative_bars.rs index 1ebc02679..8dccc61dc 100644 --- a/crates/ml/src/features/alternative_bars.rs +++ b/crates/ml-features/src/alternative_bars.rs @@ -11,7 +11,7 @@ use chrono::{DateTime, Utc}; -pub use crate::types::OHLCVBar; +pub use crate::OHLCVBar; /// Tick Bar Sampler - Aggregates every N ticks (PRIMARY IMPLEMENTATION - Agent B3) /// diff --git a/crates/ml/src/features/bar_resampler.rs b/crates/ml-features/src/bar_resampler.rs similarity index 99% rename from crates/ml/src/features/bar_resampler.rs rename to crates/ml-features/src/bar_resampler.rs index 67418c8b9..4b6d49767 100644 --- a/crates/ml/src/features/bar_resampler.rs +++ b/crates/ml-features/src/bar_resampler.rs @@ -3,7 +3,7 @@ //! Aggregates 1-minute OHLCV bars into higher timeframes (5m, 15m, 1h). //! Uses correct OHLC aggregation: open=first, high=max, low=min, close=last, volume=sum. -use crate::types::OHLCVBar; +use crate::OHLCVBar; /// Aggregates 1-minute OHLCV bars into 5m, 15m, and 1h bars. /// diff --git a/crates/ml/src/features/barrier_optimization.rs b/crates/ml-features/src/barrier_optimization.rs similarity index 100% rename from crates/ml/src/features/barrier_optimization.rs rename to crates/ml-features/src/barrier_optimization.rs diff --git a/crates/ml/src/features/config.rs b/crates/ml-features/src/config.rs similarity index 100% rename from crates/ml/src/features/config.rs rename to crates/ml-features/src/config.rs diff --git a/crates/ml/src/features/ewma.rs b/crates/ml-features/src/ewma.rs similarity index 100% rename from crates/ml/src/features/ewma.rs rename to crates/ml-features/src/ewma.rs diff --git a/crates/ml/src/features/feature_extraction.rs b/crates/ml-features/src/feature_extraction.rs similarity index 99% rename from crates/ml/src/features/feature_extraction.rs rename to crates/ml-features/src/feature_extraction.rs index a0434473b..d745b2f86 100644 --- a/crates/ml/src/features/feature_extraction.rs +++ b/crates/ml-features/src/feature_extraction.rs @@ -4,7 +4,7 @@ //! Phase 1: Implements 15 core features (5 OHLCV + 10 technical indicators) //! Phase 2-4: Will add 241 additional engineered features -pub use crate::types::OHLCVBar; +pub use crate::OHLCVBar; use crate::MLError; #[cfg(test)] use chrono::Utc; diff --git a/crates/ml-features/src/lib.rs b/crates/ml-features/src/lib.rs new file mode 100644 index 000000000..aaec53d03 --- /dev/null +++ b/crates/ml-features/src/lib.rs @@ -0,0 +1,63 @@ +//! Feature Engineering for Foxhunt ML models +//! +//! Core feature extractors: technical indicators, price patterns, volume analysis, +//! microstructure proxies, OFI (Order Flow Imbalance), normalization, and more. + +#![deny( + clippy::unwrap_used, + clippy::expect_used, + clippy::panic, + clippy::indexing_slicing +)] + +// Re-export core types at crate root for internal use +pub use ml_core::MLError; +pub use ml_core::types::OHLCVBar; + +// Public modules (21 total) +pub mod adx_features; +pub mod alternative_bars; +pub mod bar_resampler; +pub mod barrier_optimization; +pub mod config; +pub mod ewma; +pub mod feature_extraction; +pub mod mbp10_loader; +pub mod microstructure; +pub mod microstructure_features; +pub mod normalization; +pub mod ofi_calculator; +pub mod pipeline; +pub mod position_features; +pub mod price_features; +pub mod regime_adx; +pub mod statistical_features; +pub mod time_features; +pub mod trades_loader; +pub mod types; +pub mod volume_features; + +// Re-exports +pub use config::{FeatureConfig, FeatureGroup, FeatureIndices, FeaturePhase}; +pub use alternative_bars::{ + DollarBarSampler, ImbalanceBarSampler, RunBarSampler, TickBarSampler, VolumeBarSampler, +}; +pub use barrier_optimization::{BarrierOptimizer, BarrierParams, OptimizationResult}; +pub use ewma::{AdaptiveThreshold, EWMACalculator}; +pub use price_features::PriceFeatureExtractor; +pub use volume_features::VolumeFeatureExtractor; +pub use adx_features::AdxFeatureExtractor; +pub use regime_adx::RegimeADXFeatures; +pub use microstructure_features::{ + BuySellImbalance, HighLowSpread, InterArrivalTime, KyleLambda, MicrostructureFeature, + PriceImpact, TickCount, VarianceRatio, VolumeWeightedSpread, +}; +pub use time_features::TimeFeatureExtractor; +pub use statistical_features::StatisticalFeatureExtractor; +pub use normalization::{FeatureNormalizer, NormalizationStats, RingBuffer}; +pub use pipeline::{FeatureExtractionPipeline, PipelinePerformance}; +pub use position_features::PositionFeatures; +pub use ofi_calculator::{OFICalculator, OFIFeatures}; +pub use mbp10_loader::{get_recent_snapshots, get_snapshots_for_timestamp, load_mbp10_snapshots_sync}; +pub use trades_loader::{get_trades_for_bar, load_trades_sync, DbnTrade}; +pub use bar_resampler::BarResampler; diff --git a/crates/ml/src/features/mbp10_loader.rs b/crates/ml-features/src/mbp10_loader.rs similarity index 100% rename from crates/ml/src/features/mbp10_loader.rs rename to crates/ml-features/src/mbp10_loader.rs diff --git a/crates/ml/src/features/microstructure.rs b/crates/ml-features/src/microstructure.rs similarity index 100% rename from crates/ml/src/features/microstructure.rs rename to crates/ml-features/src/microstructure.rs diff --git a/crates/ml/src/features/microstructure_features.rs b/crates/ml-features/src/microstructure_features.rs similarity index 100% rename from crates/ml/src/features/microstructure_features.rs rename to crates/ml-features/src/microstructure_features.rs diff --git a/crates/ml/src/features/normalization.rs b/crates/ml-features/src/normalization.rs similarity index 100% rename from crates/ml/src/features/normalization.rs rename to crates/ml-features/src/normalization.rs diff --git a/crates/ml/src/features/ofi_calculator.rs b/crates/ml-features/src/ofi_calculator.rs similarity index 100% rename from crates/ml/src/features/ofi_calculator.rs rename to crates/ml-features/src/ofi_calculator.rs diff --git a/crates/ml/src/features/pipeline.rs b/crates/ml-features/src/pipeline.rs similarity index 99% rename from crates/ml/src/features/pipeline.rs rename to crates/ml-features/src/pipeline.rs index ce681cea4..310171036 100644 --- a/crates/ml/src/features/pipeline.rs +++ b/crates/ml-features/src/pipeline.rs @@ -52,14 +52,14 @@ use anyhow::{Context, Result}; use std::collections::VecDeque; -use crate::features::microstructure_features::{ +use crate::microstructure_features::{ BuySellImbalance, HighLowSpread, InterArrivalTime, KyleLambda, PriceImpact, TickCount, VarianceRatio, VolumeWeightedSpread, }; -use crate::features::price_features::PriceFeatureExtractor; -use crate::features::time_features::TimeFeatureExtractor; -use crate::features::volume_features::VolumeFeatureExtractor; -use crate::types::OHLCVBar; +use crate::price_features::PriceFeatureExtractor; +use crate::time_features::TimeFeatureExtractor; +use crate::volume_features::VolumeFeatureExtractor; +use crate::OHLCVBar; /// Wrapper around VecDeque that provides lazy allocation for bars /// diff --git a/crates/ml/src/features/position_features.rs b/crates/ml-features/src/position_features.rs similarity index 100% rename from crates/ml/src/features/position_features.rs rename to crates/ml-features/src/position_features.rs diff --git a/crates/ml/src/features/price_features.rs b/crates/ml-features/src/price_features.rs similarity index 99% rename from crates/ml/src/features/price_features.rs rename to crates/ml-features/src/price_features.rs index 8c1efb79f..6b1e488cb 100644 --- a/crates/ml/src/features/price_features.rs +++ b/crates/ml-features/src/price_features.rs @@ -18,7 +18,7 @@ use std::collections::VecDeque; -pub use crate::types::OHLCVBar; +pub use crate::OHLCVBar; /// Price-based feature extractor for all 15 features #[derive(Debug)] diff --git a/crates/ml/src/features/regime_adx.rs b/crates/ml-features/src/regime_adx.rs similarity index 99% rename from crates/ml/src/features/regime_adx.rs rename to crates/ml-features/src/regime_adx.rs index 4bbde33e8..8852efdcd 100644 --- a/crates/ml/src/features/regime_adx.rs +++ b/crates/ml-features/src/regime_adx.rs @@ -30,7 +30,7 @@ //! - Memory footprint: <200 bytes per symbol //! - Cache-friendly: Sequential access patterns -pub use crate::types::OHLCVBar; +pub use crate::OHLCVBar; /// ADX Feature Extractor with Wilder's Smoothing /// diff --git a/crates/ml/src/features/statistical_features.rs b/crates/ml-features/src/statistical_features.rs similarity index 99% rename from crates/ml/src/features/statistical_features.rs rename to crates/ml-features/src/statistical_features.rs index b75531b03..71f555a7d 100644 --- a/crates/ml/src/features/statistical_features.rs +++ b/crates/ml-features/src/statistical_features.rs @@ -24,7 +24,7 @@ use std::collections::VecDeque; -pub use crate::types::OHLCVBar; +pub use crate::OHLCVBar; /// Statistical feature extractor with efficient rolling window algorithms #[derive(Debug)] diff --git a/crates/ml/src/features/time_features.rs b/crates/ml-features/src/time_features.rs similarity index 100% rename from crates/ml/src/features/time_features.rs rename to crates/ml-features/src/time_features.rs diff --git a/crates/ml/src/features/trades_loader.rs b/crates/ml-features/src/trades_loader.rs similarity index 100% rename from crates/ml/src/features/trades_loader.rs rename to crates/ml-features/src/trades_loader.rs diff --git a/crates/ml/src/features/types.rs b/crates/ml-features/src/types.rs similarity index 100% rename from crates/ml/src/features/types.rs rename to crates/ml-features/src/types.rs diff --git a/crates/ml/src/features/volume_features.rs b/crates/ml-features/src/volume_features.rs similarity index 99% rename from crates/ml/src/features/volume_features.rs rename to crates/ml-features/src/volume_features.rs index 78c7fc96f..fa0d16111 100644 --- a/crates/ml/src/features/volume_features.rs +++ b/crates/ml-features/src/volume_features.rs @@ -30,7 +30,7 @@ use anyhow::Result; use std::collections::VecDeque; -pub use crate::types::OHLCVBar; +pub use crate::OHLCVBar; /// Volume feature extractor with stateful rolling windows (Wave G17 optimized) /// diff --git a/crates/ml-hyperopt/Cargo.toml b/crates/ml-hyperopt/Cargo.toml new file mode 100644 index 000000000..f059682d0 --- /dev/null +++ b/crates/ml-hyperopt/Cargo.toml @@ -0,0 +1,43 @@ +[package] +name = "ml-hyperopt" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +authors.workspace = true +license.workspace = true +repository.workspace = true +homepage.workspace = true +documentation.workspace = true +publish.workspace = true +keywords.workspace = true +categories.workspace = true +description = "ML hyperparameter optimization — PSO, TPE, campaigns, sensitivity analysis" + +[features] +default = ["cuda"] +cuda = ["candle-core/cuda", "candle-core/cudnn"] + +[dependencies] +ml-core = { path = "../ml-core", default-features = false } +common.workspace = true +candle-core = { git = "https://github.com/huggingface/candle", rev = "671de1db" } +serde = { workspace = true, features = ["derive"] } +serde_json.workspace = true +chrono.workspace = true +tracing.workspace = true +anyhow.workspace = true +rand.workspace = true +rand_distr.workspace = true +ndarray = { workspace = true, features = ["rayon"] } +rayon.workspace = true +statrs.workspace = true +argmin = { version = "0.8", features = ["rayon"] } +argmin-math = "0.3" + +[dev-dependencies] +tokio = { workspace = true, features = ["test-util", "macros"] } +rand_chacha = "0.3" +approx = "0.5" + +[lints] +workspace = true diff --git a/crates/ml/src/hyperopt/early_stopping.rs b/crates/ml-hyperopt/src/early_stopping.rs similarity index 100% rename from crates/ml/src/hyperopt/early_stopping.rs rename to crates/ml-hyperopt/src/early_stopping.rs diff --git a/crates/ml-hyperopt/src/lib.rs b/crates/ml-hyperopt/src/lib.rs new file mode 100644 index 000000000..b67408d1d --- /dev/null +++ b/crates/ml-hyperopt/src/lib.rs @@ -0,0 +1,45 @@ +//! Hyperparameter Optimization Library +//! +//! Production-ready hyperparameter optimization using argmin (PSO) and TPE. +//! +//! This crate provides: +//! - **Argmin Optimization**: Derivative-free optimization using Particle Swarm +//! - **TPE**: Tree-Parzen Estimator for Bayesian optimization +//! - **Latin Hypercube Sampling**: Smart initialization for exploration +//! - **Early Stopping**: Plateau detection, median pruning, percentile pruning +//! - **Sensitivity Analysis**: Fragility detection for hyperparameter robustness +//! +//! Model-specific adapters (DQN, PPO, TFT, Mamba2, etc.) live in the `ml` crate's +//! `hyperopt::adapters` module and depend on this crate for core infrastructure. + +#![deny(clippy::unwrap_used, clippy::expect_used)] +#![cfg_attr(test, allow(clippy::unwrap_used, clippy::expect_used))] +#![allow(clippy::doc_markdown)] +#![allow(clippy::float_arithmetic)] +#![allow(clippy::indexing_slicing)] +#![allow(clippy::missing_const_for_fn)] +#![allow(clippy::non_ascii_literal)] +#![allow(clippy::shadow_reuse)] +#![allow(clippy::shadow_unrelated)] +#![allow(clippy::shadow_same)] +#![allow(dead_code)] +#![allow(missing_docs)] +#![allow(unused_crate_dependencies)] + +pub use ml_core::{MLError, MLResult}; + +pub mod early_stopping; +pub mod observer; +pub mod optimizer; +pub mod paths; +pub mod sensitivity; +pub mod tpe; +pub mod traits; + +// Re-exports for convenience +pub use observer::TrialBudgetObserver; +pub use optimizer::{optimize_with_tpe, ArgminOptimizer, ArgminOptimizerBuilder, TwoPhaseObjective}; +pub use traits::{ + HardwareBudget, HyperoptStrategy, HyperparameterOptimizable, ObjectiveMode, + OptimizationResult, ParameterSpace, TrialResult, +}; diff --git a/crates/ml/src/hyperopt/observer.rs b/crates/ml-hyperopt/src/observer.rs similarity index 100% rename from crates/ml/src/hyperopt/observer.rs rename to crates/ml-hyperopt/src/observer.rs diff --git a/crates/ml/src/hyperopt/optimizer.rs b/crates/ml-hyperopt/src/optimizer.rs similarity index 98% rename from crates/ml/src/hyperopt/optimizer.rs rename to crates/ml-hyperopt/src/optimizer.rs index d774b6510..725bb38d2 100644 --- a/crates/ml/src/hyperopt/optimizer.rs +++ b/crates/ml-hyperopt/src/optimizer.rs @@ -51,8 +51,9 @@ use std::sync::{Arc, Mutex}; use std::time::Instant; use tracing::{info, warn}; -use super::traits::{HyperparameterOptimizable, OptimizationResult, ParameterSpace, TrialResult}; -use crate::hyperopt::adapters::dqn::ObjectiveMode; +use super::traits::{ + HyperparameterOptimizable, ObjectiveMode, OptimizationResult, ParameterSpace, TrialResult, +}; use crate::MLError; /// Bayesian optimizer using argmin library @@ -83,17 +84,17 @@ use crate::MLError; #[derive(Debug, Clone)] pub struct ArgminOptimizer { /// Maximum number of trials (evaluations) - pub(crate) max_trials: usize, + pub max_trials: usize, /// Number of initial samples via Latin Hypercube Sampling - pub(crate) n_initial: usize, + pub n_initial: usize, /// Number of particles in the swarm - pub(crate) n_particles: usize, + pub n_particles: usize, /// Random seed for reproducibility - pub(crate) seed: Option, + pub seed: Option, /// Maximum iterations per restart - pub(crate) max_iters_per_restart: usize, + pub max_iters_per_restart: usize, /// Model name for Prometheus metrics progress reporting (e.g. "dqn", "ppo") - pub(crate) model_name: Option, + pub model_name: Option, } impl Default for ArgminOptimizer { @@ -163,7 +164,7 @@ impl ArgminOptimizer { /// # Returns /// /// Array of shape (n_samples, n_params) with LHS samples - pub(crate) fn latin_hypercube_sampling( + pub fn latin_hypercube_sampling( n_samples: usize, bounds: &[(f64, f64)], rng: &mut R, @@ -247,7 +248,7 @@ impl ArgminOptimizer { info!("╚═══════════════════════════════════════════════════════════╝"); // Detect hardware budget and extract VRAM-aware parameter space bounds - let budget = crate::hyperopt::traits::HardwareBudget::detect(); + let budget = crate::traits::HardwareBudget::detect(); info!("Hardware budget: {}MB GPU memory", budget.gpu_memory_mb); let bounds = M::Params::continuous_bounds_for(&budget); let n_params = bounds.len(); @@ -309,7 +310,7 @@ impl ArgminOptimizer { // Create trial budget observer BEFORE creating cost function // This observer enforces strict trial limits (fixes PSO infinite iteration bug) - let observer = crate::hyperopt::TrialBudgetObserver::new(self.max_trials); + let observer = crate::TrialBudgetObserver::new(self.max_trials); // Create cost function wrapper let optimization_start = Instant::now(); @@ -589,7 +590,7 @@ impl ArgminOptimizer { info!("Parallel execution: {} rayon threads", rayon_threads); // Detect hardware budget and extract VRAM-aware parameter space bounds - let budget = crate::hyperopt::traits::HardwareBudget::detect(); + let budget = crate::traits::HardwareBudget::detect(); info!("Hardware budget: {}MB GPU memory", budget.gpu_memory_mb); // Compute VRAM-aware concurrency strategy @@ -678,7 +679,7 @@ impl ArgminOptimizer { for chunk in initial_vecs.chunks(initial_concurrency) { // OOM Layer 4: VRAM watchdog — check free memory before spawning - if let Ok((_total_mb, free_mb, _name)) = crate::memory_optimization::auto_batch_size::detect_gpu_memory() { + if let Ok((_total_mb, free_mb, _name)) = ml_core::memory_optimization::auto_batch_size::detect_gpu_memory() { let total_mb = budget.gpu_memory_mb as f64; if total_mb > 0.0 && free_mb / total_mb < 0.10 { warn!("VRAM watchdog: only {:.0}MB free ({:.0}%), reducing concurrency", @@ -725,7 +726,7 @@ impl ArgminOptimizer { } // Create trial budget observer BEFORE creating cost function - let observer = crate::hyperopt::TrialBudgetObserver::new(effective_max_trials); + let observer = crate::TrialBudgetObserver::new(effective_max_trials); // Create parallel cost function — clones model per evaluation instead of holding mutex let cost_fn = ParallelObjectiveFunction { @@ -876,13 +877,13 @@ where M: HyperparameterOptimizable + Send, M::Params: ParameterSpace + Send, { - use crate::hyperopt::tpe::{TpeConfig, TpeOptimizer}; + use crate::tpe::{TpeConfig, TpeOptimizer}; info!("╔═══════════════════════════════════════════════════════════╗"); info!("║ Bayesian Hyperparameter Optimization (TPE) ║"); info!("╚═══════════════════════════════════════════════════════════╝"); - let budget = crate::hyperopt::traits::HardwareBudget::detect(); + let budget = crate::traits::HardwareBudget::detect(); let bounds = M::Params::continuous_bounds_for(&budget); let n_params = bounds.len(); @@ -1036,7 +1037,7 @@ where trial_counter: Arc, param_names: Vec<&'static str>, bounds: Vec<(f64, f64)>, - observer: crate::hyperopt::TrialBudgetObserver, + observer: crate::TrialBudgetObserver, model_name: Option, max_trials: usize, optimization_start: Instant, @@ -1179,7 +1180,7 @@ where trial_counter: Arc, param_names: Vec<&'static str>, bounds: Vec<(f64, f64)>, - observer: crate::hyperopt::TrialBudgetObserver, + observer: crate::TrialBudgetObserver, model_name: Option, max_trials: usize, optimization_start: Instant, diff --git a/crates/ml/src/hyperopt/paths.rs b/crates/ml-hyperopt/src/paths.rs similarity index 100% rename from crates/ml/src/hyperopt/paths.rs rename to crates/ml-hyperopt/src/paths.rs diff --git a/crates/ml/src/hyperopt/sensitivity.rs b/crates/ml-hyperopt/src/sensitivity.rs similarity index 100% rename from crates/ml/src/hyperopt/sensitivity.rs rename to crates/ml-hyperopt/src/sensitivity.rs diff --git a/crates/ml/src/hyperopt/tpe.rs b/crates/ml-hyperopt/src/tpe.rs similarity index 100% rename from crates/ml/src/hyperopt/tpe.rs rename to crates/ml-hyperopt/src/tpe.rs diff --git a/crates/ml/src/hyperopt/traits.rs b/crates/ml-hyperopt/src/traits.rs similarity index 98% rename from crates/ml/src/hyperopt/traits.rs rename to crates/ml-hyperopt/src/traits.rs index 671911fab..4e9ae34ad 100644 --- a/crates/ml/src/hyperopt/traits.rs +++ b/crates/ml-hyperopt/src/traits.rs @@ -59,6 +59,24 @@ use tracing::info; use crate::MLError; +/// Hyperopt objective mode for two-phase optimization. +/// +/// In two-phase optimization, Phase A uses `EpisodeReward` for fast convergence, +/// then Phase B switches to `Sharpe` for financial quality refinement. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ObjectiveMode { + /// Phase A: Optimize episode reward (fast convergence signal) + EpisodeReward, + /// Phase B: Optimize Sharpe ratio (financial quality) + Sharpe, +} + +impl Default for ObjectiveMode { + fn default() -> Self { + ObjectiveMode::Sharpe + } +} + /// Strategy computed by HardwareBudget for optimal GPU utilization during hyperopt. #[derive(Debug, Clone)] pub struct HyperoptStrategy { @@ -91,7 +109,7 @@ impl HardwareBudget { /// Uses `nvidia-smi` via the existing `detect_gpu_memory()` utility. /// Falls back to `cpu_only()` if detection fails (no GPU, driver error, etc.). pub fn detect() -> Self { - match crate::memory_optimization::auto_batch_size::detect_gpu_memory() { + match ml_core::memory_optimization::auto_batch_size::detect_gpu_memory() { Ok((total_mb, _free_mb, name)) => { let gpu_memory_mb = total_mb as usize; info!("HardwareBudget: detected {} with {}MB VRAM", name, gpu_memory_mb); diff --git a/crates/ml-labeling/Cargo.toml b/crates/ml-labeling/Cargo.toml new file mode 100644 index 000000000..abe081710 --- /dev/null +++ b/crates/ml-labeling/Cargo.toml @@ -0,0 +1,40 @@ +[package] +name = "ml-labeling" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +authors.workspace = true +license.workspace = true +repository.workspace = true +homepage.workspace = true +documentation.workspace = true +publish.workspace = true +keywords.workspace = true +categories.workspace = true +description = "ML labeling algorithms for Foxhunt HFT training data" + +[features] +default = ["cuda"] +cuda = ["candle-core/cuda", "candle-core/cudnn"] + +[dependencies] +ml-core.workspace = true + +# ML frameworks (gpu_acceleration.rs uses candle tensors) +candle-core = { git = "https://github.com/huggingface/candle", rev = "671de1db" } + +# Serialization +serde = { workspace = true, features = ["derive"] } + +# Core utilities +tracing.workspace = true +uuid.workspace = true + +# Concurrency +dashmap = { workspace = true } + +[dev-dependencies] +tokio = { workspace = true, features = ["test-util", "macros"] } + +[lints] +workspace = true diff --git a/crates/ml/src/labeling/benchmarks.rs b/crates/ml-labeling/src/benchmarks.rs similarity index 97% rename from crates/ml/src/labeling/benchmarks.rs rename to crates/ml-labeling/src/benchmarks.rs index 3a76911e7..a655607e6 100644 --- a/crates/ml/src/labeling/benchmarks.rs +++ b/crates/ml-labeling/src/benchmarks.rs @@ -235,7 +235,7 @@ mod tests { let latency = result?; assert!(latency > 0.0); - info!("Triple barrier latency: {:.2} μs", latency); + info!("Triple barrier latency: {:.2} us", latency); // Performance target check assert!(latency <= MAX_TRIPLE_BARRIER_LATENCY_US as f64 * 2.0); // Allow 2x slack for CI @@ -251,7 +251,7 @@ mod tests { let latency = result?; // Allow >= 0.0 for CI environments where timing may round to zero assert!(latency >= 0.0); - info!("Meta-labeling latency: {:.2} μs", latency); + info!("Meta-labeling latency: {:.2} us", latency); Ok(()) } @@ -262,7 +262,7 @@ mod tests { let latency = result?; assert!(latency > 0.0); - info!("Concurrent tracking latency: {:.2} μs", latency); + info!("Concurrent tracking latency: {:.2} us", latency); Ok(()) } diff --git a/crates/ml/src/labeling/concurrent_tracking.rs b/crates/ml-labeling/src/concurrent_tracking.rs similarity index 100% rename from crates/ml/src/labeling/concurrent_tracking.rs rename to crates/ml-labeling/src/concurrent_tracking.rs diff --git a/crates/ml/src/labeling/fractional_diff.rs b/crates/ml-labeling/src/fractional_diff.rs similarity index 96% rename from crates/ml/src/labeling/fractional_diff.rs rename to crates/ml-labeling/src/fractional_diff.rs index af5be88e8..bda8795df 100644 --- a/crates/ml/src/labeling/fractional_diff.rs +++ b/crates/ml-labeling/src/fractional_diff.rs @@ -1,6 +1,6 @@ //! Fractional differentiation for stationarity with memory preservation //! -//! Implements streaming fractional differentiation with <1μs latency target. +//! Implements streaming fractional differentiation with <1us latency target. //! Based on the fractional differentiation concepts from financial machine learning. use std::collections::VecDeque; @@ -266,7 +266,7 @@ impl FractionalDifferentiator { #[cfg(test)] mod tests { use super::*; - use crate::labeling::constants::MAX_FRACTIONAL_DIFF_LATENCY_US; + use crate::constants::MAX_FRACTIONAL_DIFF_LATENCY_US; #[test] fn test_fractional_coeffs() -> Result<(), Box> { @@ -283,7 +283,7 @@ mod tests { } #[test] - #[ignore] // Flaky: 1μs latency target is too tight for reliable CI + #[ignore] // Flaky: 1us latency target is too tight for reliable CI fn test_streaming_differentiator() -> Result<(), LabelingError> { let config = FractionalDiffConfig::standard(); let mut differentiator = StreamingDifferentiator::new(config)?; @@ -334,10 +334,10 @@ mod tests { } #[test] - #[ignore = "Performance benchmark: 1μs latency target too strict for CI. \ - Run manually with: cargo test -p ml test_differentiator_with_history -- --ignored"] + #[ignore = "Performance benchmark: 1us latency target too strict for CI. \ + Run manually with: cargo test -p ml-labeling test_differentiator_with_history -- --ignored"] /// Performance benchmark for fractional differentiation with history - /// Target: ≤1μs processing latency (MAX_FRACTIONAL_DIFF_LATENCY_US) + /// Target: <=1us processing latency (MAX_FRACTIONAL_DIFF_LATENCY_US) fn test_differentiator_with_history() -> Result<(), LabelingError> { let config = FractionalDiffConfig::standard(); let differentiator = FractionalDifferentiator::new(config)?; diff --git a/crates/ml/src/labeling/gpu_acceleration.rs b/crates/ml-labeling/src/gpu_acceleration.rs similarity index 96% rename from crates/ml/src/labeling/gpu_acceleration.rs rename to crates/ml-labeling/src/gpu_acceleration.rs index b3fe35ae1..f718162cd 100644 --- a/crates/ml/src/labeling/gpu_acceleration.rs +++ b/crates/ml-labeling/src/gpu_acceleration.rs @@ -77,7 +77,7 @@ impl GPULabelingEngine { 0, // neutral label 0, // no return 0.5, // medium quality - 10, // 10μs processing + 10, // 10us processing ); labels.push(label); } @@ -118,6 +118,12 @@ impl fmt::Display for LabelingError { impl Error for LabelingError {} +impl From for crate::MLError { + fn from(err: LabelingError) -> Self { + crate::MLError::InferenceError(err.to_string()) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/ml-labeling/src/lib.rs b/crates/ml-labeling/src/lib.rs new file mode 100644 index 000000000..c66890d5d --- /dev/null +++ b/crates/ml-labeling/src/lib.rs @@ -0,0 +1,156 @@ +//! # ML Labeling Module for Foxhunt HFT System +//! +//! This module provides high-performance machine learning labeling algorithms +//! optimized for ultra-low latency financial applications. All implementations +//! use FixedPoint arithmetic for financial precision and target sub-microsecond +//! performance. +//! +//! ## Core Features +//! +//! - **Triple Barrier Engine**: <80us latency for event labeling +//! - **Meta-Labeling**: Separates direction prediction from confidence/bet sizing +//! - **Fractional Differentiation**: Streaming transforms with <1us latency +//! - **Sample Weighting**: Volatility/return/time-based weighting algorithms +//! - **GPU Acceleration**: Batch processing with CUDA via candle integration +//! - **Concurrent Processing**: Lock-free barrier tracking with DashMap +//! +//! ## Performance Targets +//! +//! - Triple barrier labeling: <80us per event +//! - Meta-labeling: <50us per prediction +//! - Fractional differentiation: <1us per transform +//! - Sample weighting: <10us per sample +//! - Batch processing: 10K+ labels/second +//! +//! ## Architecture +//! +//! All components use integer arithmetic (cents, nanoseconds, basis points) +//! for financial precision, matching the Python reference implementation +//! patterns from the HFTTrendfollowing project. + +#![deny(clippy::unwrap_used, clippy::expect_used)] +#![cfg_attr(test, allow(clippy::unwrap_used, clippy::expect_used))] +// Overrides of workspace deny → allow (labeling code uses these patterns) +#![allow(clippy::shadow_reuse, clippy::shadow_same, clippy::shadow_unrelated)] +#![allow(clippy::non_ascii_literal, clippy::str_to_string)] +#![allow(clippy::partial_pub_fields, clippy::multiple_inherent_impl)] +#![allow(clippy::same_name_method, clippy::indexing_slicing)] + +// Re-export MLError so internal modules can use `crate::MLError` +pub use ml_core::MLError; + +pub mod benchmarks; +pub mod concurrent_tracking; +pub mod fractional_diff; +pub mod gpu_acceleration; + +// Meta-labeling engine (legacy interface) +pub mod meta_labeling_engine; + +// New meta-labeling module with secondary model +pub mod meta_labeling; + +pub mod sample_weights; +pub mod triple_barrier; +pub mod types; + +// DO NOT RE-EXPORT - Use explicit imports at usage sites + +// DO NOT RE-EXPORT - Benchmarks should be imported explicitly + +/// Labeling module constants matching Python reference precision +pub mod constants { + /// Cents per dollar for `price` precision + pub const CENTS_PER_DOLLAR: i64 = 100; + + /// Basis points per dollar for return precision + pub const BASIS_POINTS_PER_DOLLAR: i64 = 10_000; + + /// Nanoseconds per second for time precision + pub const NANOSECONDS_PER_SECOND: i64 = 1_000_000_000; + + /// Microseconds per second + pub const MICROSECONDS_PER_SECOND: i64 = 1_000_000; + + /// Maximum latency target for triple barrier labeling (80us) + pub const MAX_TRIPLE_BARRIER_LATENCY_US: u64 = 80; + + /// Maximum latency target for meta-labeling (50us) + pub const MAX_META_LABELING_LATENCY_US: u64 = 50; + + /// Maximum latency target for fractional differentiation (1us) + pub const MAX_FRACTIONAL_DIFF_LATENCY_US: u64 = 1; + + /// Minimum throughput for batch processing (labels/second) + pub const MIN_BATCH_THROUGHPUT_LPS: u64 = 10_000; +} + +/// Utility functions for labeling operations +pub mod utils { + use super::constants::*; + + /// Convert price to cents + pub fn price_to_cents(price: f64) -> u64 { + (price * CENTS_PER_DOLLAR as f64) as u64 + } + + /// Convert cents to price + pub fn cents_to_price(cents: u64) -> f64 { + cents as f64 / CENTS_PER_DOLLAR as f64 + } + + /// Convert ratio to basis points + pub fn ratio_to_bps(ratio: f64) -> i32 { + (ratio * BASIS_POINTS_PER_DOLLAR as f64) as i32 + } + + /// Convert basis points to ratio + pub fn bps_to_ratio(bps: i32) -> f64 { + bps as f64 / BASIS_POINTS_PER_DOLLAR as f64 + } + + /// Convert timestamp to nanoseconds + pub fn timestamp_to_ns(timestamp: f64) -> u64 { + (timestamp * NANOSECONDS_PER_SECOND as f64) as u64 + } + + /// Convert nanoseconds to timestamp + pub fn ns_to_timestamp(ns: u64) -> f64 { + ns as f64 / NANOSECONDS_PER_SECOND as f64 + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_price_conversions() { + let price = 123.45; + let cents = utils::price_to_cents(price); + let converted_back = utils::cents_to_price(cents); + + assert_eq!(cents, 12345); + assert!((converted_back - price).abs() < 1e-10); + } + + #[test] + fn test_ratio_conversions() { + let ratio = 0.0250; // 2.5% + let bps = utils::ratio_to_bps(ratio); + let converted_back = utils::bps_to_ratio(bps); + + assert_eq!(bps, 250); + assert!((converted_back - ratio).abs() < 1e-10); + } + + #[test] + fn test_timestamp_conversions() { + let timestamp = 1692000000.123456789; // Example timestamp with nanosecond precision + let ns = utils::timestamp_to_ns(timestamp); + let converted_back = utils::ns_to_timestamp(ns); + + // Should preserve millisecond precision + assert!((converted_back - timestamp).abs() < 1e-6); + } +} diff --git a/crates/ml/src/labeling/meta_labeling/mod.rs b/crates/ml-labeling/src/meta_labeling/mod.rs similarity index 100% rename from crates/ml/src/labeling/meta_labeling/mod.rs rename to crates/ml-labeling/src/meta_labeling/mod.rs diff --git a/crates/ml/src/labeling/meta_labeling/primary_model.rs b/crates/ml-labeling/src/meta_labeling/primary_model.rs similarity index 95% rename from crates/ml/src/labeling/meta_labeling/primary_model.rs rename to crates/ml-labeling/src/meta_labeling/primary_model.rs index 404fac5e1..13c7d77a8 100644 --- a/crates/ml/src/labeling/meta_labeling/primary_model.rs +++ b/crates/ml-labeling/src/meta_labeling/primary_model.rs @@ -7,13 +7,13 @@ //! ## Architecture //! //! ```text -//! Features (256-dim) → Primary Model → (Label, Confidence) -//! ↓ +//! Features (256-dim) -> Primary Model -> (Label, Confidence) +//! | //! BUY/SELL/HOLD + Confidence Score //! ``` //! //! ## Performance -//! - Target latency: <50μs per prediction +//! - Target latency: <50us per prediction //! - Confidence scores: 0.0 to 1.0 //! - Labels: {-1: SELL, 0: HOLD, 1: BUY} //! @@ -22,7 +22,7 @@ //! - Integrates with 256-dim feature extraction //! - Aligns with triple barrier labels for training -use crate::labeling::gpu_acceleration::LabelingError; +use crate::gpu_acceleration::LabelingError; use crate::MLError; use serde::{Deserialize, Serialize}; use std::time::Instant; @@ -97,9 +97,9 @@ impl PrimaryModelConfig { pub fn validate(&self) -> Result<(), MLError> { if self.threshold < 0.0 || self.threshold > 1.0 { return Err(MLError::ConfigError(format!( - "Threshold must be in range [0.0, 1.0], got {}", - self.threshold - ))); + "Threshold must be in range [0.0, 1.0], got {}", + self.threshold + ))); } Ok(()) } @@ -138,7 +138,7 @@ impl PrimaryDirectionalModel { /// - `(Label, f64)`: Direction label and confidence score /// /// ## Performance - /// - Target: <50μs latency + /// - Target: <50us latency /// /// ## Errors /// - Returns `DimensionMismatch` if features length != 256 @@ -205,8 +205,8 @@ impl PrimaryDirectionalModel { /// In production, this would call into DQN/PPO/MAMBA models. fn compute_raw_prediction(&self, features: &[f64]) -> f64 { // Simple weighted average of features - // Positive features → BUY signal - // Negative features → SELL signal + // Positive features -> BUY signal + // Negative features -> SELL signal // Weight recent price action more heavily (features 0-4 are OHLCV) let price_features_weight = 0.4; @@ -321,13 +321,13 @@ mod tests { let config = PrimaryModelConfig::default(); let model = PrimaryDirectionalModel::new(config).unwrap(); - // Positive features → BUY + // Positive features -> BUY let features = vec![1.0; 256]; let (label, confidence) = model.predict(&features).unwrap(); assert_eq!(label, Label::Buy); assert!(confidence > 0.0); - // Negative features → SELL + // Negative features -> SELL let features = vec![-1.0; 256]; let (label, confidence) = model.predict(&features).unwrap(); assert_eq!(label, Label::Sell); diff --git a/crates/ml/src/labeling/meta_labeling/secondary_model.rs b/crates/ml-labeling/src/meta_labeling/secondary_model.rs similarity index 94% rename from crates/ml/src/labeling/meta_labeling/secondary_model.rs rename to crates/ml-labeling/src/meta_labeling/secondary_model.rs index 250cae664..368129666 100644 --- a/crates/ml/src/labeling/meta_labeling/secondary_model.rs +++ b/crates/ml-labeling/src/meta_labeling/secondary_model.rs @@ -13,7 +13,7 @@ //! //! ## Performance Targets //! -//! - Latency: <50μs per prediction +//! - Latency: <50us per prediction //! - Throughput: >10K predictions/second //! - False positive reduction: 30-50% @@ -54,19 +54,27 @@ impl SecondaryModelConfig { /// Validate configuration parameters pub fn validate(&self) -> Result<(), MLError> { if self.min_confidence >= self.max_confidence { - return Err(MLError::ConfigError("min_confidence must be less than max_confidence".to_owned())); + return Err(MLError::ConfigError( + "min_confidence must be less than max_confidence".to_owned(), + )); } if self.min_confidence < 0.0 || self.max_confidence > 1.0 { - return Err(MLError::ConfigError("Confidence thresholds must be in [0.0, 1.0]".to_owned())); + return Err(MLError::ConfigError( + "Confidence thresholds must be in [0.0, 1.0]".to_owned(), + )); } if self.min_bet_size < 0.0 || self.max_bet_size > 1.0 { - return Err(MLError::ConfigError("Bet sizes must be in [0.0, 1.0]".to_owned())); + return Err(MLError::ConfigError( + "Bet sizes must be in [0.0, 1.0]".to_owned(), + )); } if self.min_bet_size >= self.max_bet_size { - return Err(MLError::ConfigError("min_bet_size must be less than max_bet_size".to_owned())); + return Err(MLError::ConfigError( + "min_bet_size must be less than max_bet_size".to_owned(), + )); } Ok(()) @@ -191,12 +199,13 @@ impl SecondaryBettingModel { let current_ema_fixed = self.confidence_ema.load(Ordering::Relaxed); let current_ema = current_ema_fixed as f64 / 1_000_000.0; let alpha = self.confidence_ema_alpha; - let new_ema = if current_ema_fixed == 0 && self.total_predictions.load(Ordering::Relaxed) == 1 { - // First prediction: initialize EMA to this confidence value - primary.confidence - } else { - alpha * current_ema + (1.0 - alpha) * primary.confidence - }; + let new_ema = + if current_ema_fixed == 0 && self.total_predictions.load(Ordering::Relaxed) == 1 { + // First prediction: initialize EMA to this confidence value + primary.confidence + } else { + alpha * current_ema + (1.0 - alpha) * primary.confidence + }; self.confidence_ema .store((new_ema * 1_000_000.0) as u64, Ordering::Relaxed); } diff --git a/crates/ml/src/labeling/meta_labeling_engine.rs b/crates/ml-labeling/src/meta_labeling_engine.rs similarity index 96% rename from crates/ml/src/labeling/meta_labeling_engine.rs rename to crates/ml-labeling/src/meta_labeling_engine.rs index aa2593741..6862703bf 100644 --- a/crates/ml/src/labeling/meta_labeling_engine.rs +++ b/crates/ml-labeling/src/meta_labeling_engine.rs @@ -11,7 +11,7 @@ use serde::{Deserialize, Serialize}; use super::gpu_acceleration::LabelingError; use super::types::{EventLabel, MetaLabel}; -/// Meta-labeling engine for advanced trading strategies +/// Meta-labeling engine for advanced trading strategies #[derive(Debug)] pub struct MetaLabelingEngine { config: MetaLabelConfig, @@ -70,7 +70,7 @@ impl MetaLabelingEngine { #[cfg(test)] mod tests { use super::*; - use crate::labeling::types::BarrierResult; + use crate::types::BarrierResult; #[test] fn test_meta_labeling_engine() -> Result<(), LabelingError> { diff --git a/crates/ml/src/labeling/sample_weights.rs b/crates/ml-labeling/src/sample_weights.rs similarity index 99% rename from crates/ml/src/labeling/sample_weights.rs rename to crates/ml-labeling/src/sample_weights.rs index dd96a9840..62eeb037b 100644 --- a/crates/ml/src/labeling/sample_weights.rs +++ b/crates/ml-labeling/src/sample_weights.rs @@ -104,7 +104,7 @@ impl SampleWeightCalculator { #[cfg(test)] mod tests { use super::*; - use crate::labeling::types::BarrierResult; + use crate::types::BarrierResult; #[test] fn test_sample_weight_calculator() -> Result<(), LabelingError> { diff --git a/crates/ml/src/labeling/triple_barrier.rs b/crates/ml-labeling/src/triple_barrier.rs similarity index 99% rename from crates/ml/src/labeling/triple_barrier.rs rename to crates/ml-labeling/src/triple_barrier.rs index 2b95a93c9..3832e7f36 100644 --- a/crates/ml/src/labeling/triple_barrier.rs +++ b/crates/ml-labeling/src/triple_barrier.rs @@ -1,6 +1,6 @@ //! Triple Barrier Engine implementation //! -//! High-performance triple barrier labeling with <80μs latency target. +//! High-performance triple barrier labeling with <80us latency target. //! Based on the Python reference implementation from HFTTrendfollowing //! with optimizations for ultra-low latency financial applications. diff --git a/crates/ml/src/labeling/types.rs b/crates/ml-labeling/src/types.rs similarity index 99% rename from crates/ml/src/labeling/types.rs rename to crates/ml-labeling/src/types.rs index e81ac5867..258c876c2 100644 --- a/crates/ml/src/labeling/types.rs +++ b/crates/ml-labeling/src/types.rs @@ -133,7 +133,7 @@ pub struct LabelingStatistics { pub total_events: usize, /// Number of positive labels pub positive_labels: usize, - /// Number of negative labels + /// Number of negative labels pub negative_labels: usize, /// Number of neutral labels pub neutral_labels: usize, @@ -363,7 +363,7 @@ mod tests { 1, // positive label 500, // 5% return 0.95, // high quality - 50, // 50μs processing + 50, // 50us processing ); assert_eq!(label.label_value, 1); diff --git a/crates/ml-regime/Cargo.toml b/crates/ml-regime/Cargo.toml new file mode 100644 index 000000000..d0e5b7959 --- /dev/null +++ b/crates/ml-regime/Cargo.toml @@ -0,0 +1,37 @@ +[package] +name = "ml-regime" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +authors.workspace = true +license.workspace = true +repository.workspace = true +homepage.workspace = true +documentation.workspace = true +publish.workspace = true +keywords.workspace = true +categories.workspace = true +description = "Regime detection and structural break analysis for Foxhunt ML" + +[features] +default = [] + +[dependencies] +ml-core.workspace = true +ml-ensemble = { workspace = true, default-features = false } + +serde = { workspace = true, features = ["derive"] } +serde_json.workspace = true +tracing.workspace = true +chrono.workspace = true +thiserror.workspace = true +anyhow.workspace = true +sqlx.workspace = true + +[dev-dependencies] +tokio = { workspace = true, features = ["test-util", "macros"] } +approx.workspace = true +rand.workspace = true + +[lints] +workspace = true diff --git a/crates/ml/src/regime/bayesian_changepoint.rs b/crates/ml-regime/src/bayesian_changepoint.rs similarity index 100% rename from crates/ml/src/regime/bayesian_changepoint.rs rename to crates/ml-regime/src/bayesian_changepoint.rs diff --git a/crates/ml/src/regime/cusum.rs b/crates/ml-regime/src/cusum.rs similarity index 100% rename from crates/ml/src/regime/cusum.rs rename to crates/ml-regime/src/cusum.rs diff --git a/crates/ml-regime/src/lib.rs b/crates/ml-regime/src/lib.rs new file mode 100644 index 000000000..315b0b4f1 --- /dev/null +++ b/crates/ml-regime/src/lib.rs @@ -0,0 +1,77 @@ +#![deny(clippy::unwrap_used, clippy::expect_used)] +#![cfg_attr(test, allow(clippy::unwrap_used, clippy::expect_used))] +#![allow(dead_code)] +#![allow(missing_docs)] +#![allow(missing_debug_implementations)] +#![allow(unused_crate_dependencies)] +#![allow(clippy::float_arithmetic)] +#![allow(clippy::non_ascii_literal)] +#![allow(clippy::str_to_string)] +#![allow(clippy::partial_pub_fields)] +#![allow(clippy::multiple_inherent_impl)] +#![allow(clippy::same_name_method)] +#![allow(clippy::shadow_reuse)] +#![allow(clippy::shadow_unrelated)] +#![allow(clippy::shadow_same)] +#![allow(clippy::doc_markdown)] +#![allow(clippy::indexing_slicing)] +#![allow(clippy::missing_const_for_fn)] +#![allow(clippy::module_name_repetitions)] +#![allow(clippy::integer_division)] +#![allow(clippy::cognitive_complexity)] +#![allow(clippy::similar_names)] +#![allow(clippy::clone_on_ref_ptr)] +#![allow(clippy::too_many_lines)] +#![allow(clippy::as_conversions)] +#![allow(clippy::cast_precision_loss)] +#![allow(clippy::cast_possible_truncation)] +#![allow(clippy::default_numeric_fallback)] +#![allow(clippy::arithmetic_side_effects)] +#![allow(clippy::needless_range_loop)] +#![allow(clippy::into_iter_on_ref)] +#![allow(clippy::new_without_default)] +#![allow(clippy::manual_let_else)] +#![allow(clippy::unnecessary_wraps)] +#![allow(clippy::too_many_arguments)] +#![allow(clippy::must_use_candidate)] +#![allow(clippy::missing_errors_doc)] +#![allow(clippy::cast_sign_loss)] +#![allow(clippy::cast_possible_wrap)] +#![allow(clippy::cast_lossless)] +#![allow(clippy::unused_async)] +#![allow(clippy::match_same_arms)] +#![allow(clippy::unused_self)] +#![allow(clippy::map_err_ignore)] +#![allow(clippy::single_match_else)] +#![allow(clippy::wildcard_imports)] +#![allow(clippy::unnecessary_cast)] + +//! Regime detection and structural break analysis for Foxhunt ML. +//! +//! This crate provides structural break detection and regime classification: +//! - CUSUM-based changepoint detection (mean, variance, multivariate) +//! - Bayesian online changepoint detection +//! - Regime classifiers (trending, ranging, volatile) +//! - Regime orchestrator (coordinates detection, classification, persistence) +//! - Transition matrix and probability features + +// Re-export core types so modules can use `crate::X` +pub use ml_core::types::OHLCVBar; +pub use ml_core::MLError; +pub use ml_ensemble::MarketRegime; + +// Wave D: Structural Breaks Detection (Agents D1-D4) +pub mod bayesian_changepoint; +pub mod cusum; +pub mod multi_cusum; +pub mod pages_test; + +// Wave D: Regime Classification (Agents D5-D8) +pub mod orchestrator; +pub mod ranging; +pub mod transition_matrix; +pub mod trending; +pub mod volatile; + +// Wave D: Transition Probability Features (Agent D15) +pub mod transition_probability_features; diff --git a/crates/ml/src/regime/multi_cusum.rs b/crates/ml-regime/src/multi_cusum.rs similarity index 100% rename from crates/ml/src/regime/multi_cusum.rs rename to crates/ml-regime/src/multi_cusum.rs diff --git a/crates/ml/src/regime/orchestrator.rs b/crates/ml-regime/src/orchestrator.rs similarity index 95% rename from crates/ml/src/regime/orchestrator.rs rename to crates/ml-regime/src/orchestrator.rs index e7289de03..eb8429766 100644 --- a/crates/ml/src/regime/orchestrator.rs +++ b/crates/ml-regime/src/orchestrator.rs @@ -34,13 +34,13 @@ //! # } //! ``` -use crate::regime::{ +use crate::{ cusum::CUSUMDetector, ranging::RangingClassifier, trending::{TrendingClassifier, TrendingSignal}, volatile::{VolatileClassifier, VolatileSignal}, }; -use crate::types::OHLCVBar; +use crate::OHLCVBar; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use sqlx::PgPool; @@ -291,7 +291,7 @@ impl RegimeOrchestrator { let ranging_signal = if let Some(&last_bar) = bars.last() { self.ranging_classifier.classify(last_bar) } else { - crate::regime::ranging::RangingSignal::NotRanging + crate::ranging::RangingSignal::NotRanging }; let volatile_signal = if let Some(&last_bar) = bars.last() { @@ -316,12 +316,12 @@ impl RegimeOrchestrator { TrendingSignal::WeakTrend { .. } => { // Weak trend, check if ranging match ranging_signal { - crate::regime::ranging::RangingSignal::StrongRanging - | crate::regime::ranging::RangingSignal::ModerateRanging => { + crate::ranging::RangingSignal::StrongRanging + | crate::ranging::RangingSignal::ModerateRanging => { "Ranging".to_owned() }, - crate::regime::ranging::RangingSignal::WeakRanging - | crate::regime::ranging::RangingSignal::NotRanging => { + crate::ranging::RangingSignal::WeakRanging + | crate::ranging::RangingSignal::NotRanging => { "Trending".to_owned() }, // Default to weak trend } @@ -329,12 +329,12 @@ impl RegimeOrchestrator { TrendingSignal::Ranging { .. } => { // Check ranging classifier match ranging_signal { - crate::regime::ranging::RangingSignal::StrongRanging - | crate::regime::ranging::RangingSignal::ModerateRanging => { + crate::ranging::RangingSignal::StrongRanging + | crate::ranging::RangingSignal::ModerateRanging => { "Ranging".to_owned() }, - crate::regime::ranging::RangingSignal::WeakRanging - | crate::regime::ranging::RangingSignal::NotRanging => { + crate::ranging::RangingSignal::WeakRanging + | crate::ranging::RangingSignal::NotRanging => { "Normal".to_owned() }, // Default to normal } diff --git a/crates/ml/src/regime/pages_test.rs b/crates/ml-regime/src/pages_test.rs similarity index 100% rename from crates/ml/src/regime/pages_test.rs rename to crates/ml-regime/src/pages_test.rs diff --git a/crates/ml/src/regime/ranging.rs b/crates/ml-regime/src/ranging.rs similarity index 99% rename from crates/ml/src/regime/ranging.rs rename to crates/ml-regime/src/ranging.rs index cdba6e761..0aa528edf 100644 --- a/crates/ml/src/regime/ranging.rs +++ b/crates/ml-regime/src/ranging.rs @@ -11,7 +11,7 @@ use serde::{Deserialize, Serialize}; use std::collections::VecDeque; -use crate::types::OHLCVBar; +use crate::OHLCVBar; /// Ranging regime signal #[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] diff --git a/crates/ml/src/regime/transition_matrix.rs b/crates/ml-regime/src/transition_matrix.rs similarity index 99% rename from crates/ml/src/regime/transition_matrix.rs rename to crates/ml-regime/src/transition_matrix.rs index f97bc123c..0896eaab3 100644 --- a/crates/ml/src/regime/transition_matrix.rs +++ b/crates/ml-regime/src/transition_matrix.rs @@ -9,7 +9,7 @@ //! Uses exponential moving average for online updates and Laplace smoothing //! to handle sparse transitions. -use crate::ensemble::MarketRegime; +use crate::MarketRegime; use std::collections::HashMap; /// Regime Transition Matrix diff --git a/crates/ml/src/regime/transition_probability_features.rs b/crates/ml-regime/src/transition_probability_features.rs similarity index 99% rename from crates/ml/src/regime/transition_probability_features.rs rename to crates/ml-regime/src/transition_probability_features.rs index 1b7508cda..93e39dff1 100644 --- a/crates/ml/src/regime/transition_probability_features.rs +++ b/crates/ml-regime/src/transition_probability_features.rs @@ -36,8 +36,8 @@ //! assert_eq!(result.len(), 5); //! ``` -use crate::ensemble::MarketRegime; -use crate::regime::transition_matrix::RegimeTransitionMatrix; +use crate::MarketRegime; +use crate::transition_matrix::RegimeTransitionMatrix; /// Transition Probability Feature Extractor /// diff --git a/crates/ml/src/regime/trending.rs b/crates/ml-regime/src/trending.rs similarity index 99% rename from crates/ml/src/regime/trending.rs rename to crates/ml-regime/src/trending.rs index ddcf8d8b8..0eaafc8e8 100644 --- a/crates/ml/src/regime/trending.rs +++ b/crates/ml-regime/src/trending.rs @@ -17,7 +17,7 @@ use std::collections::VecDeque; -use crate::types::OHLCVBar; +use crate::OHLCVBar; /// Trending signal output #[derive(Debug, Clone, PartialEq)] diff --git a/crates/ml/src/regime/volatile.rs b/crates/ml-regime/src/volatile.rs similarity index 99% rename from crates/ml/src/regime/volatile.rs rename to crates/ml-regime/src/volatile.rs index 7b165b019..010ac0932 100644 --- a/crates/ml/src/regime/volatile.rs +++ b/crates/ml-regime/src/volatile.rs @@ -22,7 +22,7 @@ use std::collections::VecDeque; -use crate::types::OHLCVBar; +use crate::OHLCVBar; /// Volatile signal output #[derive(Debug, Clone, Copy, PartialEq, Eq)] diff --git a/crates/ml-risk/Cargo.toml b/crates/ml-risk/Cargo.toml new file mode 100644 index 000000000..208490a8e --- /dev/null +++ b/crates/ml-risk/Cargo.toml @@ -0,0 +1,35 @@ +[package] +name = "ml-risk" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +authors.workspace = true +license.workspace = true +repository.workspace = true +homepage.workspace = true +documentation.workspace = true +publish.workspace = true +keywords.workspace = true +categories.workspace = true +description = "ML-driven risk management for Foxhunt HFT" + +[features] +default = [] + +[dependencies] +ml-core.workspace = true +common.workspace = true +risk = { path = "../risk" } +serde = { workspace = true, features = ["derive"] } +chrono.workspace = true +ndarray.workspace = true +tracing.workspace = true +rand.workspace = true +rust_decimal.workspace = true +tokio.workspace = true + +[dev-dependencies] +tokio = { workspace = true, features = ["test-util", "macros"] } + +[lints] +workspace = true diff --git a/crates/ml/src/risk/circuit_breakers.rs b/crates/ml-risk/src/circuit_breakers.rs similarity index 100% rename from crates/ml/src/risk/circuit_breakers.rs rename to crates/ml-risk/src/circuit_breakers.rs diff --git a/crates/ml/src/risk/graph_risk_model.rs b/crates/ml-risk/src/graph_risk_model.rs similarity index 100% rename from crates/ml/src/risk/graph_risk_model.rs rename to crates/ml-risk/src/graph_risk_model.rs diff --git a/crates/ml/src/risk/kelly_optimizer.rs b/crates/ml-risk/src/kelly_optimizer.rs similarity index 100% rename from crates/ml/src/risk/kelly_optimizer.rs rename to crates/ml-risk/src/kelly_optimizer.rs diff --git a/crates/ml/src/risk/kelly_position_sizing_service.rs b/crates/ml-risk/src/kelly_position_sizing_service.rs similarity index 99% rename from crates/ml/src/risk/kelly_position_sizing_service.rs rename to crates/ml-risk/src/kelly_position_sizing_service.rs index b8aec18a0..57b5b0b1a 100644 --- a/crates/ml/src/risk/kelly_position_sizing_service.rs +++ b/crates/ml-risk/src/kelly_position_sizing_service.rs @@ -45,7 +45,7 @@ use serde::{Deserialize, Serialize}; use tokio::sync::{broadcast, RwLock}; use tracing::{debug, info, warn}; -use crate::risk::{KellyCriterionOptimizer, KellyOptimizerConfig, KellyPositionRecommendation}; +use crate::{KellyCriterionOptimizer, KellyOptimizerConfig, KellyPositionRecommendation}; use crate::{MLError, MLResult as Result, MarketDataSnapshot}; use common::types::Price; use risk::position_tracker::{EnhancedRiskPosition, PositionUpdateEvent}; diff --git a/crates/ml-risk/src/lib.rs b/crates/ml-risk/src/lib.rs new file mode 100644 index 000000000..580e8aa73 --- /dev/null +++ b/crates/ml-risk/src/lib.rs @@ -0,0 +1,285 @@ +#![deny(clippy::unwrap_used, clippy::expect_used)] +#![cfg_attr(test, allow(clippy::unwrap_used, clippy::expect_used))] +#![allow(dead_code)] +#![allow(missing_docs)] +#![allow(missing_debug_implementations)] +#![allow(unused_crate_dependencies)] +#![allow(clippy::float_arithmetic)] +#![allow(clippy::non_ascii_literal)] +#![allow(clippy::str_to_string)] +#![allow(clippy::partial_pub_fields)] +#![allow(clippy::multiple_inherent_impl)] +#![allow(clippy::same_name_method)] +#![allow(clippy::shadow_reuse)] +#![allow(clippy::shadow_unrelated)] +#![allow(clippy::shadow_same)] +#![allow(clippy::doc_markdown)] +#![allow(clippy::indexing_slicing)] +#![allow(clippy::missing_const_for_fn)] +#![allow(clippy::module_name_repetitions)] +#![allow(clippy::integer_division)] +#![allow(clippy::cognitive_complexity)] +#![allow(clippy::similar_names)] +#![allow(clippy::clone_on_ref_ptr)] +#![allow(clippy::too_many_lines)] +#![allow(clippy::as_conversions)] +#![allow(clippy::cast_precision_loss)] +#![allow(clippy::cast_possible_truncation)] +#![allow(clippy::default_numeric_fallback)] +#![allow(clippy::arithmetic_side_effects)] +#![allow(clippy::needless_range_loop)] +#![allow(clippy::into_iter_on_ref)] +#![allow(clippy::new_without_default)] +#![allow(clippy::manual_let_else)] +#![allow(clippy::unnecessary_wraps)] +#![allow(clippy::too_many_arguments)] +#![allow(clippy::must_use_candidate)] +#![allow(clippy::missing_errors_doc)] +#![allow(clippy::cast_sign_loss)] +#![allow(clippy::cast_possible_wrap)] +#![allow(clippy::cast_lossless)] +#![allow(clippy::unused_async)] +#![allow(clippy::match_same_arms)] +#![allow(clippy::unused_self)] +#![allow(clippy::map_err_ignore)] +#![allow(clippy::single_match_else)] +#![allow(clippy::wildcard_imports)] +#![allow(clippy::unnecessary_cast)] +#![allow(clippy::undocumented_unsafe_blocks)] +#![allow(clippy::redundant_clone)] +#![allow(clippy::redundant_closure)] +#![allow(clippy::type_complexity)] +#![allow(clippy::manual_clamp)] +#![allow(clippy::clone_on_copy)] +#![allow(clippy::should_implement_trait)] +#![allow(clippy::derivable_impls)] +#![allow(clippy::useless_conversion)] +#![allow(clippy::get_first)] +#![allow(clippy::len_zero)] +#![allow(clippy::assign_op_pattern)] +#![allow(clippy::if_same_then_else)] +#![allow(clippy::unused_enumerate_index)] +#![allow(clippy::doc_lazy_continuation)] +#![allow(clippy::doc_overindented_list_items)] +#![allow(clippy::single_char_add_str)] +#![allow(clippy::let_and_return)] +#![allow(clippy::useless_format)] +#![allow(clippy::manual_div_ceil)] +#![allow(clippy::io_other_error)] +#![allow(clippy::manual_range_contains)] +#![allow(clippy::unwrap_or_default)] +#![allow(clippy::used_underscore_binding)] +#![allow(clippy::trivially_copy_pass_by_ref)] +#![allow(clippy::needless_borrows_for_generic_args)] +#![allow(clippy::needless_borrow)] +#![allow(clippy::missing_safety_doc)] +#![allow(clippy::module_inception)] +#![allow(clippy::if_not_else)] + +//! # Neural Risk Management System +//! +//! Advanced ML-driven risk management with neural VaR models, Kelly criterion optimization, +//! and real-time regime detection for production HFT systems using canonical types. + +pub mod circuit_breakers; +pub mod graph_risk_model; +pub mod kelly_optimizer; +pub mod kelly_position_sizing_service; +pub mod position_sizing; +pub mod slippage; +pub mod var_models; + +// Re-export key types from submodules +pub use circuit_breakers::MLCircuitBreaker; +pub use kelly_optimizer::{ + KellyCriterionOptimizer, KellyOptimizerConfig, KellyPositionRecommendation, +}; +pub use position_sizing::PositionSizingNetwork; +pub use slippage::{FixedCostModel, LinearImpactConfig, LinearImpactModel, MarketImpactModel}; +pub use var_models::{NeuralVarConfig, NeuralVarModel}; + +// Re-export core ML types used throughout risk modules +pub use ml_core::{MLError, MLResult, MarketDataSnapshot}; + +use std::collections::HashMap; + +use chrono::{DateTime, Utc}; +use ndarray::Array2; +use serde::{Deserialize, Serialize}; + +use common::{Price, Volume}; + +// AssetId type for risk management +#[derive(Debug, Clone, Serialize, Deserialize, Hash, PartialEq, Eq)] +pub struct AssetId(String); + +/// Risk assessment levels +/// RiskLevel component. +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, PartialOrd)] +pub enum RiskLevel { + VeryLow, + Low, + Medium, + High, + VeryHigh, + Critical, +} + +/// Portfolio risk profile +/// RiskProfile component. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RiskProfile { + pub total_var_95: f64, // 1-day 95% VaR + pub total_var_99: f64, // 1-day 99% VaR + pub expected_shortfall: f64, // Expected tail loss + pub maximum_drawdown: f64, // Historical maximum drawdown + pub current_drawdown: f64, // Current drawdown from peak + pub sharpe_ratio: f64, // Risk-adjusted return + pub sortino_ratio: f64, // Downside risk-adjusted return + pub calmar_ratio: f64, // Return over maximum drawdown + pub beta: f64, // Market beta + pub tracking_error: f64, // Volatility vs benchmark + pub concentration_risk: f64, // Single position concentration + pub correlation_risk: f64, // Average correlation risk + pub liquidity_risk: f64, // Liquidity-adjusted risk + pub regime_risk: f64, // Market regime risk factor + pub overall_risk_score: f64, // Combined ML risk score (0-1) + pub risk_level: RiskLevel, // Categorical risk assessment + pub timestamp: DateTime, +} + +/// Position-level risk metrics +/// PositionRisk component. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PositionRisk { + pub asset_id: AssetId, + pub position_size: f64, // Current position size + pub market_value: f64, // Current market value + pub var_contribution: f64, // Contribution to portfolio VaR + pub marginal_var: f64, // Marginal VaR (change in portfolio VaR) + pub component_var: f64, // Component VaR (allocation of portfolio VaR) + pub standalone_var: f64, // Position VaR in isolation + pub beta_to_portfolio: f64, // Beta relative to portfolio + pub correlation_risk: f64, // Correlation with other positions + pub liquidity_horizon: f64, // Days to liquidate position + pub concentration_weight: f64, // Weight in portfolio + pub stress_loss: f64, // Loss under stress scenarios + pub kelly_optimal_size: f64, // Kelly optimal position size + pub recommended_size: f64, // ML recommended position size + pub risk_score: f64, // Individual risk score (0-1) + pub timestamp: DateTime, +} + +/// `Market` data for risk calculations +/// MarketData component. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MarketData { + pub timestamp: DateTime, + pub prices: HashMap, + pub volumes: HashMap, + pub volatilities: HashMap, + pub correlations: Array2, + pub market_cap_weights: HashMap, + pub sector_exposures: HashMap, +} + +/// Risk limits and constraints +/// RiskLimits component. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RiskLimits { + pub max_portfolio_var: f64, // Maximum portfolio VaR + pub max_position_size: f64, // Maximum single position size + pub max_sector_exposure: f64, // Maximum sector exposure + pub max_correlation: f64, // Maximum position correlation + pub max_drawdown: f64, // Maximum allowed drawdown + pub min_liquidity_days: f64, // Minimum liquidity (days to exit) + pub max_leverage: f64, // Maximum portfolio leverage + pub var_limit_buffer: f64, // VaR limit buffer (e.g., 0.8 of limit) +} + +impl Default for RiskLimits { + fn default() -> Self { + Self { + max_portfolio_var: 0.02, // 2% daily VaR limit + max_position_size: 0.05, // 5% maximum position size + max_sector_exposure: 0.20, // 20% maximum sector exposure + max_correlation: 0.70, // 70% maximum correlation + max_drawdown: 0.10, // 10% maximum drawdown + min_liquidity_days: 2.0, // 2 days maximum liquidation time + max_leverage: 3.0, // 3x maximum leverage + var_limit_buffer: 0.80, // Use 80% of VaR limit + } + } +} + +/// Comprehensive risk configuration +/// RiskConfig component. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RiskConfig { + pub var_confidence_levels: Vec, + pub lookback_days: usize, + pub monte_carlo_simulations: usize, + pub stress_scenarios: usize, + pub regime_detection_window: usize, + pub update_frequency_seconds: u32, + pub enable_neural_var: bool, + pub enable_regime_detection: bool, + pub enable_ml_position_sizing: bool, + pub enable_dynamic_hedging: bool, + pub risk_limits: RiskLimits, +} + +impl Default for RiskConfig { + fn default() -> Self { + Self { + var_confidence_levels: vec![0.95, 0.99], + lookback_days: 252, + monte_carlo_simulations: 10_000, + stress_scenarios: 1_000, + regime_detection_window: 60, + update_frequency_seconds: 60, + enable_neural_var: true, + enable_regime_detection: true, + enable_ml_position_sizing: true, + enable_dynamic_hedging: true, + risk_limits: RiskLimits::default(), + } + } +} + +/// Main neural risk management system +#[derive(Debug)] +pub struct NeuralRiskManager { + config: RiskConfig, + var_model: NeuralVarModel, + kelly_optimizer: KellyCriterionOptimizer, + position_sizer: PositionSizingNetwork, + circuit_breaker: MLCircuitBreaker, +} + +impl NeuralRiskManager { + /// Create new neural risk management system + pub fn new(config: RiskConfig) -> MLResult { + let var_config = NeuralVarConfig { + confidence_levels: config.var_confidence_levels.clone(), + lookback_days: config.lookback_days, + lookback_period: config.lookback_days, + monte_carlo_simulations: config.monte_carlo_simulations, + enable_stress_testing: true, + hidden_layers: vec![128, 64, 32], + }; + + let var_model = NeuralVarModel::new(var_config)?; + let kelly_optimizer = KellyCriterionOptimizer::new(Default::default())?; + let position_sizer = PositionSizingNetwork::new(Default::default())?; + let circuit_breaker = MLCircuitBreaker::new(Default::default())?; + + Ok(Self { + config, + var_model, + kelly_optimizer, + position_sizer, + circuit_breaker, + }) + } +} diff --git a/crates/ml/src/risk/position_sizing.rs b/crates/ml-risk/src/position_sizing.rs similarity index 100% rename from crates/ml/src/risk/position_sizing.rs rename to crates/ml-risk/src/position_sizing.rs diff --git a/crates/ml/src/risk/slippage.rs b/crates/ml-risk/src/slippage.rs similarity index 99% rename from crates/ml/src/risk/slippage.rs rename to crates/ml-risk/src/slippage.rs index e2ecea3a4..d2eedddd9 100644 --- a/crates/ml/src/risk/slippage.rs +++ b/crates/ml-risk/src/slippage.rs @@ -6,7 +6,7 @@ use serde::{Deserialize, Serialize}; -use crate::common::action::OrderType; +use ml_core::common::action::OrderType; // --------------------------------------------------------------------------- // Trait diff --git a/crates/ml/src/risk/var_models.rs b/crates/ml-risk/src/var_models.rs similarity index 100% rename from crates/ml/src/risk/var_models.rs rename to crates/ml-risk/src/var_models.rs diff --git a/crates/ml-supervised/src/checkpoint.rs b/crates/ml-supervised/src/checkpoint.rs new file mode 100644 index 000000000..9d02cdc70 --- /dev/null +++ b/crates/ml-supervised/src/checkpoint.rs @@ -0,0 +1,319 @@ +//! Checkpoint implementations for supervised models (Mamba2, TGGN). + +use std::collections::HashMap; + +use async_trait::async_trait; +use ml_core::{Checkpointable, MLError, ModelType}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use tracing::debug; + +use crate::mamba::{Mamba2Config, Mamba2SSM}; +use crate::tgnn::{TGGNConfig, TGGN}; + +// ========== MAMBA2 ========== + +/// Serializable state for Mamba2 model +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MambaCheckpointState { + pub config: Mamba2Config, + pub epoch: Option, + pub step: Option, + pub training_loss: f64, + pub validation_loss: f64, + pub ssd_layer_weights: Vec>, + pub input_projection_weights: Vec, + pub output_projection_weights: Vec, + pub layer_norm_weights: Vec>, + pub ssm_a_matrices: Vec>, + pub ssm_b_matrices: Vec>, + pub ssm_c_matrices: Vec>, + pub ssm_delta_params: Vec, + pub total_inferences: u64, + pub avg_latency_us: f64, + pub throughput_pps: f64, +} + +#[async_trait] +impl Checkpointable for Mamba2SSM { + fn model_type(&self) -> ModelType { + ModelType::MAMBA + } + + fn model_name(&self) -> &str { + &self.metadata.model_id + } + + fn model_version(&self) -> &str { + &self.metadata.version + } + + async fn serialize_state(&self) -> Result, MLError> { + let (current_epoch, current_step) = self.get_current_training_state(); + let training_metrics = self.get_training_metrics(); + let performance_stats = self.get_inference_stats(); + + let state = MambaCheckpointState { + config: self.config.clone(), + epoch: current_epoch, + step: current_step, + training_loss: training_metrics + .get("training_loss") + .copied() + .unwrap_or(0.0), + validation_loss: training_metrics + .get("validation_loss") + .copied() + .unwrap_or(0.0), + ssd_layer_weights: self.extract_ssd_weights(), + input_projection_weights: self.extract_input_projection_weights(), + output_projection_weights: self.extract_output_projection_weights(), + layer_norm_weights: self.extract_layer_norm_weights(), + ssm_a_matrices: self.extract_ssm_matrices("A"), + ssm_b_matrices: self.extract_ssm_matrices("B"), + ssm_c_matrices: self.extract_ssm_matrices("C"), + ssm_delta_params: self.extract_delta_params(), + total_inferences: self + .total_inferences + .load(std::sync::atomic::Ordering::Relaxed), + avg_latency_us: performance_stats + .get("avg_latency_us") + .copied() + .unwrap_or(0.0), + throughput_pps: performance_stats + .get("throughput_pps") + .copied() + .unwrap_or(0.0), + }; + + let serialized = serde_json::to_vec(&state) + .map_err(|e| MLError::ModelError(format!("MAMBA serialization failed: {}", e)))?; + + debug!("Serialized MAMBA state: {} bytes", serialized.len()); + Ok(serialized) + } + + async fn deserialize_state(&mut self, data: &[u8]) -> Result<(), MLError> { + let state: MambaCheckpointState = serde_json::from_slice(data) + .map_err(|e| MLError::ModelError(format!("MAMBA deserialization failed: {}", e)))?; + + self.config = state.config; + self.restore_ssd_weights(&state.ssd_layer_weights); + self.restore_input_projection_weights(&state.input_projection_weights); + self.restore_output_projection_weights(&state.output_projection_weights); + self.restore_layer_norm_weights(&state.layer_norm_weights); + self.restore_ssm_matrices("A", &state.ssm_a_matrices); + self.restore_ssm_matrices("B", &state.ssm_b_matrices); + self.restore_ssm_matrices("C", &state.ssm_c_matrices); + self.restore_delta_params(&state.ssm_delta_params); + + debug!("Deserialized MAMBA state from {} bytes", data.len()); + Ok(()) + } + + fn get_training_state(&self) -> (Option, Option, Option, Option) { + if let Some(last_epoch) = self.metadata.training_history.back() { + let total_steps = self + .metadata + .training_history + .iter() + .map(|epoch| epoch.epoch as u64 * 1000) + .max() + .unwrap_or(0); + + ( + Some(last_epoch.epoch as u64), + Some(total_steps), + Some(last_epoch.loss), + Some(last_epoch.accuracy), + ) + } else { + (Some(0), Some(0), Some(f64::INFINITY), Some(0.0)) + } + } + + fn get_hyperparameters(&self) -> HashMap { + let mut params = HashMap::new(); + params.insert("d_model".to_owned(), Value::from(self.config.d_model)); + params.insert("d_state".to_owned(), Value::from(self.config.d_state)); + params.insert("d_head".to_owned(), Value::from(self.config.d_head)); + params.insert("num_heads".to_owned(), Value::from(self.config.num_heads)); + params.insert("expand".to_owned(), Value::from(self.config.expand)); + params.insert( + "num_layers".to_owned(), + Value::from(self.config.num_layers), + ); + params.insert("dropout".to_owned(), Value::from(self.config.dropout)); + params.insert( + "learning_rate".to_owned(), + Value::from(self.config.learning_rate), + ); + params.insert( + "target_latency_us".to_owned(), + Value::from(self.config.target_latency_us), + ); + params + } + + fn get_metrics(&self) -> HashMap { + self.get_performance_metrics() + } + + fn get_architecture_info(&self) -> HashMap { + let mut info = HashMap::new(); + info.insert("model_type".to_owned(), Value::from("MAMBA-2")); + info.insert("input_dim".to_owned(), Value::from(self.config.d_model)); + info.insert( + "hidden_dim".to_owned(), + Value::from(self.config.d_model * self.config.expand), + ); + info.insert("state_dim".to_owned(), Value::from(self.config.d_state)); + info.insert( + "num_layers".to_owned(), + Value::from(self.config.num_layers), + ); + info.insert("use_ssd".to_owned(), Value::from(self.config.use_ssd)); + info.insert( + "use_selective_state".to_owned(), + Value::from(self.config.use_selective_state), + ); + info.insert( + "hardware_aware".to_owned(), + Value::from(self.config.hardware_aware), + ); + info + } +} + +// ========== TGGN ========== + +/// Serializable state for TGGN model +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TGGNCheckpointState { + pub config: TGGNConfig, + pub epoch: Option, + pub step: Option, + pub training_loss: f64, + pub validation_loss: f64, + pub node_embeddings: HashMap>, + pub edge_embeddings: HashMap>, + pub graph_statistics: HashMap, + pub message_passing_weights: Vec>, + pub gating_weights: Vec, + pub total_inferences: u64, + pub graph_updates: u64, + pub avg_inference_latency_ns: u64, + pub avg_graph_update_latency_ns: u64, +} + +#[async_trait] +impl Checkpointable for TGGN { + fn model_type(&self) -> ModelType { + ModelType::TGGN + } + + fn model_name(&self) -> &str { + &self.metadata.version + } + + fn model_version(&self) -> &str { + &self.metadata.version + } + + async fn serialize_state(&self) -> Result, MLError> { + let node_embeddings: HashMap> = self + .node_embeddings() + .iter() + .map(|entry| { + let key = format!("{:?}", entry.key()); + let value = entry.value().iter().map(|&x| x as f32).collect(); + (key, value) + }) + .collect(); + + let edge_embeddings: HashMap> = self + .edge_embeddings() + .iter() + .map(|entry| { + let key = format!("{:?}", entry.key()); + let value = entry.value().iter().map(|&x| x as f32).collect(); + (key, value) + }) + .collect(); + + let state = TGGNCheckpointState { + config: self.config().clone(), + epoch: None, + step: None, + training_loss: 0.0, + validation_loss: 0.0, + node_embeddings, + edge_embeddings, + graph_statistics: HashMap::new(), + message_passing_weights: Vec::new(), + gating_weights: Vec::new(), + total_inferences: self + .inference_count() + .load(std::sync::atomic::Ordering::Relaxed), + graph_updates: self + .graph_updates() + .load(std::sync::atomic::Ordering::Relaxed), + avg_inference_latency_ns: self.calculate_avg_inference_latency(), + avg_graph_update_latency_ns: self.calculate_avg_graph_update_latency(), + }; + + let serialized = serde_json::to_vec(&state) + .map_err(|e| MLError::ModelError(format!("TGGN serialization failed: {}", e)))?; + + debug!("Serialized TGGN state: {} bytes", serialized.len()); + Ok(serialized) + } + + async fn deserialize_state(&mut self, data: &[u8]) -> Result<(), MLError> { + let state: TGGNCheckpointState = serde_json::from_slice(data) + .map_err(|e| MLError::ModelError(format!("TGGN deserialization failed: {}", e)))?; + + self.restore_node_embeddings(&state.node_embeddings)?; + self.restore_edge_embeddings(&state.edge_embeddings)?; + self.restore_graph_statistics(&state.graph_statistics)?; + self.restore_message_passing_weights(&state.message_passing_weights)?; + + self.inference_count() + .store(state.total_inferences, std::sync::atomic::Ordering::Relaxed); + self.graph_updates() + .store(state.graph_updates, std::sync::atomic::Ordering::Relaxed); + + debug!("Deserialized TGGN state from {} bytes", data.len()); + Ok(()) + } + + fn get_hyperparameters(&self) -> HashMap { + let mut params = HashMap::new(); + let config = self.config(); + params.insert("max_nodes".to_owned(), Value::from(config.max_nodes)); + params.insert("max_edges".to_owned(), Value::from(config.max_edges)); + params.insert("node_dim".to_owned(), Value::from(config.node_dim)); + params.insert("edge_dim".to_owned(), Value::from(config.edge_dim)); + params.insert("hidden_dim".to_owned(), Value::from(config.hidden_dim)); + params.insert("num_layers".to_owned(), Value::from(config.num_layers)); + params.insert( + "temporal_decay".to_owned(), + Value::from(config.temporal_decay), + ); + params.insert("use_simd".to_owned(), Value::from(config.use_simd)); + params + } + + fn get_architecture_info(&self) -> HashMap { + let mut info = HashMap::new(); + let config = self.config(); + info.insert("model_type".to_owned(), Value::from("TGGN")); + info.insert("max_nodes".to_owned(), Value::from(config.max_nodes)); + info.insert("max_edges".to_owned(), Value::from(config.max_edges)); + info.insert("node_feature_dim".to_owned(), Value::from(config.node_dim)); + info.insert("edge_feature_dim".to_owned(), Value::from(config.edge_dim)); + info.insert("gnn_layers".to_owned(), Value::from(config.num_layers)); + info.insert("supports_temporal_decay".to_owned(), Value::from(true)); + info + } +} diff --git a/crates/ml-supervised/src/lib.rs b/crates/ml-supervised/src/lib.rs index a287b4611..8b73cbdaa 100644 --- a/crates/ml-supervised/src/lib.rs +++ b/crates/ml-supervised/src/lib.rs @@ -2,7 +2,7 @@ //! //! Contains 8 model architectures: TFT, Mamba-2, Liquid, TGGN, TLOB, KAN, xLSTM, Diffusion. //! Each model directory contains the core network, layers, and self-contained training logic. -//! Bridge modules (UnifiedTrainable adapters, Checkpointable impls) live in the `ml` crate. +//! Checkpointable impls live in the `checkpoint` module. UnifiedTrainable adapters live in `ml`. // Re-export shared types from ml-core pub use ml_core::cuda_compat; @@ -17,3 +17,6 @@ pub mod tlob; pub mod kan; pub mod xlstm; pub mod diffusion; + +// Checkpoint impls (Checkpointable for Mamba2SSM, TGGN) +pub mod checkpoint; diff --git a/crates/ml-validation/Cargo.toml b/crates/ml-validation/Cargo.toml new file mode 100644 index 000000000..c3b5190e6 --- /dev/null +++ b/crates/ml-validation/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "ml-validation" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +authors.workspace = true +license.workspace = true +repository.workspace = true +homepage.workspace = true +documentation.workspace = true +publish.workspace = true +keywords.workspace = true +categories.workspace = true +description = "Statistical validation for ML trading strategies" + +[features] +default = [] + +[dependencies] +ml-core.workspace = true + +serde = { workspace = true, features = ["derive"] } +chrono.workspace = true +rand.workspace = true +rand_chacha = "0.3" + +[dev-dependencies] +tokio = { workspace = true, features = ["test-util", "macros"] } +approx.workspace = true + +[lints] +workspace = true diff --git a/crates/ml/src/validation/degradation.rs b/crates/ml-validation/src/degradation.rs similarity index 100% rename from crates/ml/src/validation/degradation.rs rename to crates/ml-validation/src/degradation.rs diff --git a/crates/ml-validation/src/lib.rs b/crates/ml-validation/src/lib.rs new file mode 100644 index 000000000..01bd4dea3 --- /dev/null +++ b/crates/ml-validation/src/lib.rs @@ -0,0 +1,37 @@ +#![deny(clippy::unwrap_used, clippy::expect_used)] +#![cfg_attr(test, allow(clippy::unwrap_used, clippy::expect_used))] +#![allow(clippy::float_arithmetic)] +#![allow(clippy::doc_markdown)] +#![allow(clippy::missing_const_for_fn)] + +//! Statistical and Financial Validation — core algorithms. +//! +//! Provides two categories of validation: +//! - Statistical validation (walk-forward, DSR, PBO, permutation tests) +//! - Temporal guards for preventing forward-looking data leakage +//! - Noise injection for robustness testing +//! - Fold-by-fold performance degradation tracking +//! +//! Bridge modules (DQN/PPO adapters, regime analysis, harness) stay in the `ml` +//! crate and re-export types from this crate. + +pub use ml_core::{MLError, MLResult}; + +pub mod degradation; +pub mod noise; +pub mod statistical; +pub mod temporal_guard; +pub mod types; +pub mod walk_forward; + +// Re-export core types at crate root for convenience. +pub use degradation::{DegradationReport, DegradationTracker, FoldMetrics}; +pub use noise::{compute_robustness_score, NoiseConfig, NoiseInjector}; +pub use statistical::{ + binomial_coefficient, deflated_sharpe_ratio, excess_kurtosis, normal_cdf, normal_ppf, + permutation_test, probability_of_backtest_overfitting, sharpe_ratio, skewness, DsrResult, + PboResult, PermutationResult, +}; +pub use temporal_guard::{LeakageAuditReport, NormalizationStats, TemporalGuard}; +pub use types::{TimeSeriesData, ValidatableStrategy}; +pub use walk_forward::{walk_forward_split, Fold, WalkForwardConfig}; diff --git a/crates/ml/src/validation/noise.rs b/crates/ml-validation/src/noise.rs similarity index 98% rename from crates/ml/src/validation/noise.rs rename to crates/ml-validation/src/noise.rs index b12059a8c..b7bfea0ef 100644 --- a/crates/ml/src/validation/noise.rs +++ b/crates/ml-validation/src/noise.rs @@ -4,7 +4,7 @@ //! to exact feature values by perturbing inputs and measuring degradation. use crate::MLError; -use crate::validation::TimeSeriesData; +use crate::types::TimeSeriesData; /// Noise injection methods for robustness testing. #[derive(Debug)] @@ -180,7 +180,7 @@ impl SimpleRng { mod tests { use super::*; use chrono::{TimeZone, Utc}; - use crate::validation::TimeSeriesData; + use crate::types::TimeSeriesData; fn make_data(n: usize) -> TimeSeriesData { let timestamps: Vec<_> = (0..n) diff --git a/crates/ml/src/validation/statistical.rs b/crates/ml-validation/src/statistical.rs similarity index 100% rename from crates/ml/src/validation/statistical.rs rename to crates/ml-validation/src/statistical.rs diff --git a/crates/ml/src/validation/temporal_guard.rs b/crates/ml-validation/src/temporal_guard.rs similarity index 99% rename from crates/ml/src/validation/temporal_guard.rs rename to crates/ml-validation/src/temporal_guard.rs index 9132d87a6..3fa2d4335 100644 --- a/crates/ml/src/validation/temporal_guard.rs +++ b/crates/ml-validation/src/temporal_guard.rs @@ -7,7 +7,7 @@ use chrono::{DateTime, Utc}; -use super::TimeSeriesData; +use crate::types::TimeSeriesData; use crate::MLError; // --------------------------------------------------------------------------- diff --git a/crates/ml/src/validation/types.rs b/crates/ml-validation/src/types.rs similarity index 100% rename from crates/ml/src/validation/types.rs rename to crates/ml-validation/src/types.rs diff --git a/crates/ml/src/validation/walk_forward.rs b/crates/ml-validation/src/walk_forward.rs similarity index 99% rename from crates/ml/src/validation/walk_forward.rs rename to crates/ml-validation/src/walk_forward.rs index 60ef78944..660129b9a 100644 --- a/crates/ml/src/validation/walk_forward.rs +++ b/crates/ml-validation/src/walk_forward.rs @@ -8,7 +8,7 @@ //! # Example //! //! ``` -//! use ml::validation::walk_forward::{WalkForwardConfig, walk_forward_split}; +//! use ml_validation::walk_forward::{WalkForwardConfig, walk_forward_split}; //! //! let cfg = WalkForwardConfig { //! train_bars: 100, diff --git a/crates/ml/Cargo.toml b/crates/ml/Cargo.toml index 7d675bf16..888c4c6b1 100644 --- a/crates/ml/Cargo.toml +++ b/crates/ml/Cargo.toml @@ -28,7 +28,7 @@ simd = [] # SIMD without heavy dependencies # Storage and memory management features gc = [] # Garbage collection features -s3-storage = ["aws-config", "aws-sdk-s3", "aws-types", "aws-credential-types", "urlencoding"] # S3 storage backend with AWS SDK +s3-storage = ["ml-checkpoint/s3-storage", "aws-config", "aws-sdk-s3", "aws-types", "aws-credential-types", "urlencoding"] # S3 storage backend with AWS SDK cuda = ["candle-core/cuda", "candle-core/cudnn", "candle-nn/cuda", "candle-nn/cudnn"] # CUDA support (includes LSTM sigmoid ops) - OPTIONAL for CI/Docker nccl = ["cuda"] # NCCL multi-GPU data parallelism (requires NCCL library + cudarc nccl feature) @@ -69,9 +69,18 @@ colored = "2.1" # Terminal color output for evaluation reports # Internal workspace crates ml-core.workspace = true +ml-ensemble.workspace = true ml-dqn.workspace = true ml-ppo.workspace = true ml-supervised.workspace = true +ml-hyperopt.workspace = true +ml-features.workspace = true +ml-labeling.workspace = true +ml-regime.workspace = true +ml-checkpoint.workspace = true +ml-data-validation.workspace = true +ml-validation.workspace = true +ml-risk.workspace = true config.workspace = true common = { workspace = true, features = ["questdb"] } risk = { path = "../risk" } diff --git a/crates/ml/src/checkpoint/enterprise_implementations.rs b/crates/ml/src/checkpoint/enterprise_implementations.rs deleted file mode 100644 index 2efb33027..000000000 --- a/crates/ml/src/checkpoint/enterprise_implementations.rs +++ /dev/null @@ -1,563 +0,0 @@ -//! Enterprise-grade implementations for ML model checkpoint operations -//! -//! This module provides production-ready implementations to complete all ML model -//! in the checkpoint system with comprehensive error handling, validation, and monitoring. - -use crate::MLError; -use std::collections::HashMap; -use tracing::{debug, error, info, warn}; -use std::sync::atomic::AtomicU64; -use std::time::Instant; - -/// Helper methods for Mamba2SSM checkpoint implementations -impl crate::mamba::Mamba2SSM { - /// Apply weights to SSD layer with enterprise-grade validation - pub fn apply_ssd_layer_weights(&mut self, layer_idx: usize, weights: &[f32]) -> Result<(), MLError> { - info!("Applying {} weights to SSD layer {}", weights.len(), layer_idx); - - // Validate layer index - if layer_idx >= self.config.num_layers as usize { - return Err(MLError::ModelError(format!( - "Layer index {} exceeds maximum layers {}", - layer_idx, self.config.num_layers - ))); - } - - // Validate weight dimensions - let expected_size = self.config.d_model * self.config.expand; - if weights.len() != expected_size { - return Err(MLError::ModelError(format!( - "Weight size mismatch for layer {}: expected {}, got {}", - layer_idx, expected_size, weights.len() - ))); - } - - // Perform numerical stability checks - let weight_stats = self.analyze_weight_statistics(weights); - if weight_stats.has_issues { - warn!("Layer {} weights have stability issues: {}", layer_idx, weight_stats.warning); - - // Apply stabilization if needed - let stabilized_weights = self.stabilize_weights(weights)?; - self.update_layer_weights(layer_idx, &stabilized_weights)?; - } else { - self.update_layer_weights(layer_idx, weights)?; - } - - debug!("Successfully applied weights to SSD layer {}", layer_idx); - Ok(()) - } - - /// Apply selective SSM delta parameters with validation - pub fn apply_selective_ssm_deltas(&mut self, layer_idx: usize, deltas: &[f32]) -> Result<(), MLError> { - info!("Applying {} delta parameters to selective SSM layer {}", deltas.len(), layer_idx); - - // Validate delta parameters are positive (required for SSM stability) - let invalid_deltas = deltas.iter().filter(|&&d| d <= 0.0 || !d.is_finite()).count(); - if invalid_deltas > 0 { - warn!("Found {} invalid delta parameters in layer {}, applying correction", - invalid_deltas, layer_idx); - - // Correct invalid delta parameters - let corrected_deltas: Vec = deltas.iter() - .map(|&d| if d > 0.0 && d.is_finite() { d } else { 1e-3 }) - .collect(); - - self.update_ssm_deltas(layer_idx, &corrected_deltas)?; - } else { - self.update_ssm_deltas(layer_idx, deltas)?; - } - - debug!("Successfully applied delta parameters to layer {}", layer_idx); - Ok(()) - } - - /// Apply input projection weights with comprehensive validation - pub fn apply_input_projection_weights(&mut self, weights: &[f32]) -> Result<(), MLError> { - info!("Applying {} input projection weights", weights.len()); - - // Validate dimensions - let expected_size = self.config.d_model * self.config.input_dim; - if weights.len() != expected_size { - return Err(MLError::ModelError(format!( - "Input projection weight size mismatch: expected {}, got {}", - expected_size, weights.len() - ))); - } - - // Check weight distribution for potential issues - let mean = weights.iter().sum::() / weights.len() as f32; - let variance = weights.iter().map(|&w| (w - mean).powi(2)).sum::() / weights.len() as f32; - let std_dev = variance.sqrt(); - - if std_dev > 10.0 || mean.abs() > 5.0 { - warn!("Input projection weights have unusual statistics: mean={:.4}, std={:.4}", mean, std_dev); - - // Apply normalization - let normalized_weights = self.normalize_projection_weights(weights)?; - self.update_input_projection(&normalized_weights)?; - } else { - self.update_input_projection(weights)?; - } - - debug!("Successfully applied input projection weights"); - Ok(()) - } - - /// Apply output projection weights with quantization support - pub fn apply_output_projection_weights(&mut self, weights: &[f32]) -> Result<(), MLError> { - info!("Applying {} output projection weights", weights.len()); - - // Validate dimensions - let expected_size = self.config.d_model * self.config.output_dim; - if weights.len() != expected_size { - return Err(MLError::ModelError(format!( - "Output projection weight size mismatch: expected {}, got {}", - expected_size, weights.len() - ))); - } - - // Apply quantization if enabled - let processed_weights = if self.config.use_quantization { - self.quantize_weights(weights, 8)? // 8-bit quantization - } else { - weights.to_vec() - }; - - self.update_output_projection(&processed_weights)?; - debug!("Successfully applied output projection weights"); - Ok(()) - } - - /// Apply layer normalization weights with stability checks - pub fn apply_layer_norm_weights(&mut self, layer_idx: usize, weights: &[f32]) -> Result<(), MLError> { - info!("Applying {} layer norm weights to layer {}", weights.len(), layer_idx); - - // Validate dimensions (should match d_model) - if weights.len() != self.config.d_model { - return Err(MLError::ModelError(format!( - "Layer norm weight size mismatch for layer {}: expected {}, got {}", - layer_idx, self.config.d_model, weights.len() - ))); - } - - // Check for numerical stability issues - let has_zeros = weights.iter().any(|&w| w == 0.0); - let has_extremes = weights.iter().any(|&w| w.abs() > 100.0); - - if has_zeros || has_extremes { - warn!("Layer norm {} has stability issues (zeros={}, extremes={})", - layer_idx, has_zeros, has_extremes); - - // Apply stabilization - let stabilized_weights = self.stabilize_layer_norm_weights(weights)?; - self.update_layer_norm(layer_idx, &stabilized_weights)?; - } else { - self.update_layer_norm(layer_idx, weights)?; - } - - debug!("Successfully applied layer norm weights to layer {}", layer_idx); - Ok(()) - } - - /// Apply SSM matrix weights with mathematical validation - pub fn apply_ssm_matrix_weights(&mut self, layer_idx: usize, matrix_type: &str, matrix: &[f32]) -> Result<(), MLError> { - info!("Applying {} matrix {} to SSM layer {}", matrix_type, matrix.len(), layer_idx); - - // Validate matrix dimensions based on type - let expected_size = match matrix_type { - "A" => self.config.d_state * self.config.d_state, - "B" => self.config.d_state * self.config.d_model, - "C" => self.config.d_model * self.config.d_state, - _ => return Err(MLError::ModelError(format!("Unknown matrix type: {}", matrix_type))), - }; - - if matrix.len() != expected_size { - return Err(MLError::ModelError(format!( - "SSM {} matrix size mismatch for layer {}: expected {}, got {}", - matrix_type, layer_idx, expected_size, matrix.len() - ))); - } - - // Perform matrix-specific validation - match matrix_type { - "A" => self.validate_state_transition_matrix(matrix)?, - "B" => self.validate_input_matrix(matrix)?, - "C" => self.validate_output_matrix(matrix)?, - _ => {} - } - - self.update_ssm_matrix(layer_idx, matrix_type, matrix)?; - debug!("Successfully applied {} matrix to layer {}", matrix_type, layer_idx); - Ok(()) - } - - // Private helper methods - - fn analyze_weight_statistics(&self, weights: &[f32]) -> WeightStatistics { - let mean = weights.iter().sum::() / weights.len() as f32; - let variance = weights.iter().map(|&w| (w - mean).powi(2)).sum::() / weights.len() as f32; - let std_dev = variance.sqrt(); - - let invalid_count = weights.iter().filter(|&&w| !w.is_finite()).count(); - let extreme_count = weights.iter().filter(|&&w| w.abs() > 100.0).count(); - - let has_issues = invalid_count > 0 || extreme_count > 0 || std_dev > 50.0; - let warning = if has_issues { - format!("invalid={}, extreme={}, std={:.2}", invalid_count, extreme_count, std_dev) - } else { - "stable".to_owned() - }; - - WeightStatistics { - mean, - std_dev, - invalid_count, - extreme_count, - has_issues, - warning, - } - } - - fn stabilize_weights(&self, weights: &[f32]) -> Result, MLError> { - let mut stabilized = weights.to_vec(); - - // Replace invalid values - for weight in &mut stabilized { - if !weight.is_finite() { - *weight = 0.0; - } else if weight.abs() > 10.0 { - *weight = weight.signum() * 10.0; // Clip extreme values - } - } - - Ok(stabilized) - } - - fn stabilize_layer_norm_weights(&self, weights: &[f32]) -> Result, MLError> { - let mut stabilized = weights.to_vec(); - - // Replace zeros with small positive values - for weight in &mut stabilized { - if *weight == 0.0 { - *weight = 1e-6; - } else if weight.abs() > 10.0 { - *weight = weight.signum() * 10.0; - } - } - - Ok(stabilized) - } - - fn normalize_projection_weights(&self, weights: &[f32]) -> Result, MLError> { - let mut normalized = weights.to_vec(); - - let mean = weights.iter().sum::() / weights.len() as f32; - let variance = weights.iter().map(|&w| (w - mean).powi(2)).sum::() / weights.len() as f32; - - if variance > 1e-8 { - let std_dev = variance.sqrt(); - for weight in &mut normalized { - *weight = (*weight - mean) / std_dev; - } - } - - Ok(normalized) - } - - fn quantize_weights(&self, weights: &[f32], bits: u8) -> Result, MLError> { - let max_val = (1 << (bits - 1)) as f32; - let min_val = -max_val; - - let quantized: Vec = weights.iter() - .map(|&w| { - let scaled = (w * max_val).round(); - scaled.clamp(min_val, max_val - 1.0) / max_val - }) - .collect(); - - Ok(quantized) - } - - fn validate_state_transition_matrix(&self, matrix: &[f32]) -> Result<(), MLError> { - // Check for numerical stability (eigenvalues should be < 1 for stability) - let spectral_norm = self.estimate_spectral_norm(matrix, self.config.d_state); - if spectral_norm > 1.0 { - warn!("State transition matrix has spectral norm {:.4} > 1.0, may be unstable", spectral_norm); - } - Ok(()) - } - - fn validate_input_matrix(&self, matrix: &[f32]) -> Result<(), MLError> { - let frobenius_norm = matrix.iter().map(|&x| x * x).sum::().sqrt(); - if frobenius_norm > 100.0 { - warn!("Input matrix has large Frobenius norm {:.4}", frobenius_norm); - } - Ok(()) - } - - fn validate_output_matrix(&self, matrix: &[f32]) -> Result<(), MLError> { - let max_element = matrix.iter().map(|&x| x.abs()).fold(0.0_f32, f32::max); - if max_element > 50.0 { - warn!("Output matrix has large maximum element {:.4}", max_element); - } - Ok(()) - } - - fn estimate_spectral_norm(&self, matrix: &[f32], size: usize) -> f32 { - // Simple power iteration for largest eigenvalue estimate - let mut x = vec![1.0; size]; - - for _ in 0..5 { // 5 iterations for rough estimate - let mut y = vec![0.0; size]; - - // Matrix vector multiply - for i in 0..size { - for j in 0..size { - y[i] += matrix[i * size + j] * x[j]; - } - } - - // Normalize - let norm = y.iter().map(|&yi| yi * yi).sum::().sqrt(); - if norm > 0.0 { - for yi in &mut y { - *yi /= norm; - } - } - - x = y; - } - - // Compute Rayleigh quotient - let mut numerator = 0.0; - let mut denominator = 0.0; - - for i in 0..size { - let mut ax_i = 0.0; - for j in 0..size { - ax_i += matrix[i * size + j] * x[j]; - } - numerator += x[i] * ax_i; - denominator += x[i] * x[i]; - } - - if denominator > 0.0 { - (numerator / denominator).abs() - } else { - 0.0 - } - } - - // Production methods for actual model updates (would interface with ML framework) - - fn update_layer_weights(&mut self, layer_idx: usize, weights: &[f32]) -> Result<(), MLError> { - debug!("Updating layer {} with {} weights", layer_idx, weights.len()); - // In production: self.layers[layer_idx].set_weights(weights) - Ok(()) - } - - fn update_ssm_deltas(&mut self, layer_idx: usize, deltas: &[f32]) -> Result<(), MLError> { - debug!("Updating SSM deltas for layer {} with {} parameters", layer_idx, deltas.len()); - // In production: self.ssm_layers[layer_idx].set_deltas(deltas) - Ok(()) - } - - fn update_input_projection(&mut self, weights: &[f32]) -> Result<(), MLError> { - debug!("Updating input projection with {} weights", weights.len()); - // In production: self.input_projection.set_weights(weights) - Ok(()) - } - - fn update_output_projection(&mut self, weights: &[f32]) -> Result<(), MLError> { - debug!("Updating output projection with {} weights", weights.len()); - // In production: self.output_projection.set_weights(weights) - Ok(()) - } - - fn update_layer_norm(&mut self, layer_idx: usize, weights: &[f32]) -> Result<(), MLError> { - debug!("Updating layer norm {} with {} weights", layer_idx, weights.len()); - // In production: self.layer_norms[layer_idx].set_weights(weights) - Ok(()) - } - - fn update_ssm_matrix(&mut self, layer_idx: usize, matrix_type: &str, matrix: &[f32]) -> Result<(), MLError> { - debug!("Updating SSM {} matrix for layer {} with {} values", matrix_type, layer_idx, matrix.len()); - // In production: self.ssm_layers[layer_idx].set_matrix(matrix_type, matrix) - Ok(()) - } -} - -/// Helper struct for weight statistics analysis -#[derive(Debug)] -struct WeightStatistics { - mean: f32, - std_dev: f32, - invalid_count: usize, - extreme_count: usize, - has_issues: bool, - warning: String, -} - -/// Helper methods for TGGN model -impl crate::tgnn::TGGN { - /// Extract comprehensive graph statistics for checkpointing - pub fn extract_graph_statistics(&self) -> Result, MLError> { - let mut stats = HashMap::new(); - - // Basic graph topology statistics - let node_count = self.node_embeddings().len() as f64; - let edge_count = self.edge_embeddings().len() as f64; - - stats.insert("node_count".to_owned(), node_count); - stats.insert("edge_count".to_owned(), edge_count); - stats.insert("avg_degree".to_owned(), if node_count > 0.0 { edge_count * 2.0 / node_count } else { 0.0 }); - - // Graph connectivity metrics - stats.insert("graph_density".to_owned(), self.calculate_graph_density()); - stats.insert("clustering_coefficient".to_owned(), self.calculate_clustering_coefficient()); - - // Performance metrics - stats.insert("avg_message_passing_time_ns".to_owned(), self.get_avg_message_passing_time()); - stats.insert("graph_update_frequency_hz".to_owned(), self.get_graph_update_frequency()); - - Ok(stats) - } - - /// Extract message passing weights with layer-wise organization - pub fn extract_message_passing_weights(&self) -> Result>, MLError> { - let config = self.config(); - let mut weights = Vec::new(); - - for layer_idx in 0..config.num_layers { - let layer_weights = self.extract_layer_message_passing_weights(layer_idx)?; - weights.push(layer_weights); - } - - Ok(weights) - } - - /// Calculate average inference latency from performance counters - pub fn calculate_average_inference_latency(&self) -> u64 { - let total_inferences = self.inference_count().load(std::sync::atomic::Ordering::Relaxed); - - if total_inferences > 0 { - // Get cumulative inference time from performance metrics - let total_time_ns = self.get_cumulative_inference_time_ns(); - total_time_ns / total_inferences - } else { - 0 - } - } - - // Private helper methods - - fn calculate_graph_density(&self) -> f64 { - let node_count = self.node_embeddings().len() as f64; - let edge_count = self.edge_embeddings().len() as f64; - - if node_count > 1.0 { - edge_count / (node_count * (node_count - 1.0) / 2.0) - } else { - 0.0 - } - } - - fn calculate_clustering_coefficient(&self) -> f64 { - // Simplified clustering coefficient calculation - // In production, would implement proper triangle counting - let density = self.calculate_graph_density(); - density.powf(1.5) // Rough approximation - } - - fn get_avg_message_passing_time(&self) -> f64 { - // Get from performance metrics - if let Some(metrics) = self.get_performance_metrics_ref() { - metrics.get("avg_message_passing_time_ns").copied().unwrap_or(0.0) - } else { - 0.0 - } - } - - fn get_graph_update_frequency(&self) -> f64 { - let updates = self.graph_updates().load(std::sync::atomic::Ordering::Relaxed) as f64; - let runtime_seconds = self.get_runtime_seconds(); - - if runtime_seconds > 0.0 { - updates / runtime_seconds - } else { - 0.0 - } - } - - fn extract_layer_message_passing_weights(&self, layer_idx: usize) -> Result, MLError> { - let config = self.config(); - let weight_size = config.hidden_dim * config.hidden_dim; - - // In production, would extract actual layer weights - // For now, generate representative weights based on layer index - let mut weights = Vec::with_capacity(weight_size); - let scale = 1.0 / (layer_idx + 1) as f32; - - for i in 0..weight_size { - let weight = scale * (i as f32 / weight_size as f32 - 0.5); - weights.push(weight); - } - - Ok(weights) - } - - fn get_cumulative_inference_time_ns(&self) -> u64 { - // Get from internal performance tracking - // In production, would maintain actual cumulative timing - self.inference_count().load(std::sync::atomic::Ordering::Relaxed) * 50_000 // Assume 50μs per inference - } - - fn get_runtime_seconds(&self) -> f64 { - // Calculate runtime from startup time - if let Some(start_time) = self.get_startup_time() { - start_time.elapsed().as_secs_f64() - } else { - 1.0 // Default to 1 second to avoid division by zero - } - } - - fn get_startup_time(&self) -> Option { - // In production, would track actual startup time - Some(Instant::now() - std::time::Duration::from_secs(3600)) // Fake 1 hour runtime - } - - fn get_performance_metrics_ref(&self) -> Option<&HashMap> { - // In production, would return reference to actual metrics - None - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_weight_statistics_analysis() { - // Test weight analysis with various scenarios - let stable_weights = vec![0.1, 0.2, 0.3, 0.4, 0.5]; - let unstable_weights = vec![f32::NAN, f32::INFINITY, 1000.0, -1000.0, 0.1]; - - // Would test with actual Mamba2SSM instance in production - assert!(stable_weights.iter().all(|&w| w.is_finite())); - assert!(!unstable_weights.iter().all(|&w| w.is_finite())); - } - - #[test] - fn test_quantization() { - let weights = vec![1.0, 0.5, -0.5, -1.0]; - // Test 8-bit quantization logic - let max_val = 128.0; - let quantized: Vec = weights.iter() - .map(|&w| (w * max_val).round().clamp(-max_val, max_val - 1.0) / max_val) - .collect(); - - assert_eq!(quantized.len(), weights.len()); - assert!(quantized.iter().all(|&w| w.abs() <= 1.0)); - } -} diff --git a/crates/ml/src/checkpoint/mod.rs b/crates/ml/src/checkpoint/mod.rs index 13cafb2ae..787d52c92 100644 --- a/crates/ml/src/checkpoint/mod.rs +++ b/crates/ml/src/checkpoint/mod.rs @@ -1,1093 +1,11 @@ -//! # Unified Model Weight Persistence System +//! Checkpoint system — thin facade re-exporting from ml-checkpoint crate. //! -//! Comprehensive checkpoint system for all 5 AI models (DQN, MAMBA, TFT, TGGN, LNN) -//! with versioning, metadata, compression, and validation. -//! -//! ## Key Features -//! -//! - **Unified Interface**: Single API for all model checkpointing -//! - **Model Versioning**: Semantic versioning with compatibility checks -//! - **Metadata Management**: Training metrics, hyperparameters, performance stats -//! - **Compression**: Optional LZ4/Zstd compression for large models -//! - **Validation**: Checksum verification and corruption detection -//! - **Incremental Saves**: Delta checkpoints for memory efficiency -//! - **Async I/O**: Non-blocking checkpoint operations -//! - **Multi-format**: Binary, JSON, and custom formats -//! -//! ## Architecture -//! -//! ```text -//! ┌─────────────────────────────────────────────────────────────┐ -//! │ CheckpointManager │ -//! ├─────────────────┬─────────────────┬─────────────────────────┤ -//! │ Versioning │ Compression │ Storage Backend │ -//! │ │ │ │ -//! │ • Semantic Ver │ • LZ4/Zstd │ • FileSystem │ -//! │ • Compatibility │ • Delta Saves │ • Cloud Storage │ -//! │ • Migration │ • Streaming │ • Database │ -//! └─────────────────┴─────────────────┴─────────────────────────┘ -//! ``` +//! Model-specific Checkpointable impls live in their respective crates: +//! - DQNAgent → ml-dqn::checkpoint +//! - Mamba2SSM, TGGN → ml-supervised::checkpoint -use std::collections::HashMap; -use std::fs::{self}; -use std::path::PathBuf; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::Arc; +pub use ml_checkpoint::*; -use async_trait::async_trait; -use chrono::{DateTime, Utc}; -use dashmap::DashMap; -use serde::{Deserialize, Serialize}; -use sha2::{Digest, Sha256}; -use tokio::sync::RwLock; -use tracing::{info, instrument, warn}; -use uuid::Uuid; - -use crate::MLError; - -// Re-export ModelType for test access -pub use crate::ModelType; - -pub mod compression; -pub mod model_implementations; -pub mod signer; -pub mod storage; -pub mod validation; -pub mod versioning; - -#[cfg(test)] -pub mod integration_tests; - -// Re-export key types for external usage -pub use compression::CompressionManager; -pub use signer::{CheckpointSigner, SignatureInfo}; -#[cfg(feature = "s3-storage")] -pub use storage::S3CheckpointStorage; -pub use storage::{CheckpointStorage, FileSystemStorage, MemoryStorage, StorageStats}; -pub use validation::{verify_checksum, write_checksum, ValidationManager}; -pub use versioning::VersionManager; - -/// Checkpoint format options -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -pub enum CheckpointFormat { - /// Binary format (fastest) - Binary, - /// JSON format (human-readable) - JSON, - /// MessagePack format (compact) - MessagePack, - /// Custom optimized format - Custom, -} - -/// Compression algorithm options -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -pub enum CompressionType { - /// No compression - None, - /// LZ4 - fast compression - LZ4, - /// Zstandard - balanced compression - Zstd, - /// Gzip - high compression - Gzip, -} - -/// Checkpoint metadata containing model information and training state -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct CheckpointMetadata { - /// Unique checkpoint identifier - pub checkpoint_id: String, - - /// Model type - pub model_type: ModelType, - - /// Model name/identifier - pub model_name: String, - - /// Model version (semantic versioning) - pub version: String, - - /// Creation timestamp - pub created_at: DateTime, - - /// Training epoch when checkpoint was created - pub epoch: Option, - - /// Training step when checkpoint was created - pub step: Option, - - /// Training loss at checkpoint time - pub loss: Option, - - /// Validation accuracy at checkpoint time - pub accuracy: Option, - - /// Model hyperparameters - pub hyperparameters: HashMap, - - /// Training metrics and statistics - pub metrics: HashMap, - - /// Model architecture information - pub architecture: HashMap, - - /// Checkpoint file format - pub format: CheckpointFormat, - - /// Compression algorithm used - pub compression: CompressionType, - - /// File size in bytes - pub file_size: u64, - - /// Compressed file size (if compressed) - pub compressed_size: Option, - - /// SHA-256 checksum for validation - pub checksum: String, - - /// Tags for organizing checkpoints - pub tags: Vec, - - /// Additional custom metadata - pub custom_metadata: HashMap, - - // Security fields (Agent 122 - SEC-001 fix) - /// HMAC-SHA256 signature (hex-encoded) - pub signature: Option, - /// Signing algorithm identifier - pub signature_algorithm: String, - /// Signing key ID for rotation support - pub signing_key_id: String, - /// Signature timestamp - pub signed_at: Option>, -} - -impl Default for CheckpointMetadata { - fn default() -> Self { - Self { - checkpoint_id: Uuid::new_v4().to_string(), - model_type: ModelType::DQN, // Default to first variant - model_name: String::new(), - version: "1.0.0".to_owned(), - created_at: Utc::now(), - epoch: None, - step: None, - loss: None, - accuracy: None, - hyperparameters: HashMap::new(), - metrics: HashMap::new(), - architecture: HashMap::new(), - format: CheckpointFormat::Binary, - compression: CompressionType::None, - file_size: 0, - compressed_size: None, - checksum: String::new(), - tags: Vec::new(), - custom_metadata: HashMap::new(), - signature: None, - signature_algorithm: String::new(), - signing_key_id: String::new(), - signed_at: None, - } - } -} - -impl CheckpointMetadata { - /// Create new checkpoint metadata - pub fn new(model_type: ModelType, model_name: String, version: String) -> Self { - Self { - checkpoint_id: Uuid::new_v4().to_string(), - model_type, - model_name, - version, - created_at: Utc::now(), - epoch: None, - step: None, - loss: None, - accuracy: None, - hyperparameters: HashMap::new(), - metrics: HashMap::new(), - architecture: HashMap::new(), - format: CheckpointFormat::Binary, - compression: CompressionType::None, - file_size: 0, - compressed_size: None, - checksum: String::new(), - tags: Vec::new(), - custom_metadata: HashMap::new(), - signature: None, - signature_algorithm: String::new(), - signing_key_id: String::new(), - signed_at: None, - } - } - - /// Add training state information - pub fn with_training_state( - mut self, - epoch: Option, - step: Option, - loss: Option, - accuracy: Option, - ) -> Self { - self.epoch = epoch; - self.step = step; - self.loss = loss; - self.accuracy = accuracy; - self - } - - /// Add hyperparameters - pub fn with_hyperparameters(mut self, hyperparams: HashMap) -> Self { - self.hyperparameters = hyperparams; - self - } - - /// Add metrics - pub fn with_metrics(mut self, metrics: HashMap) -> Self { - self.metrics = metrics; - self - } - - /// Add tags - pub fn with_tags(mut self, tags: Vec) -> Self { - self.tags = tags; - self - } - - /// Check if this is a newer version than another metadata - pub fn is_newer_than(&self, other: &CheckpointMetadata) -> bool { - if self.model_type != other.model_type || self.model_name != other.model_name { - return false; - } - - // Compare by epoch if available - if let (Some(self_epoch), Some(other_epoch)) = (self.epoch, other.epoch) { - return self_epoch > other_epoch; - } - - // Compare by step if available - if let (Some(self_step), Some(other_step)) = (self.step, other.step) { - return self_step > other_step; - } - - // Compare by timestamp - self.created_at > other.created_at - } - - /// Generate filename for this checkpoint - pub fn generate_filename(&self) -> String { - let timestamp = self.created_at.format("%Y%m%d_%H%M%S%.3f"); - let epoch_str = self.epoch.map(|e| format!("_e{}", e)).unwrap_or_default(); - let step_str = self.step.map(|s| format!("_s{}", s)).unwrap_or_default(); - let ext = self.model_type.file_extension(); - - format!( - "{}_{}_v{}{}{}_{}.{}", - self.model_type.file_extension(), - self.model_name, - self.version, - epoch_str, - step_str, - timestamp, - ext - ) - } -} - -/// Configuration for checkpoint operations -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct CheckpointConfig { - /// Base directory for checkpoints - pub base_dir: PathBuf, - - /// Default compression type - pub compression: CompressionType, - - /// Default checkpoint format - pub format: CheckpointFormat, - - /// Maximum number of checkpoints to keep per model - pub max_checkpoints_per_model: usize, - - /// Automatic cleanup of old checkpoints - pub auto_cleanup: bool, - - /// Enable checksum validation - pub validate_checksums: bool, - - /// Enable incremental checkpoints (delta saves) - pub incremental_checkpoints: bool, - - /// Compression level (0-9, algorithm dependent) - pub compression_level: u32, - - /// Enable async I/O operations - pub async_io: bool, - - /// Buffer size for I/O operations - pub buffer_size: usize, -} - -impl Default for CheckpointConfig { - fn default() -> Self { - Self { - base_dir: PathBuf::from("./checkpoints"), - compression: CompressionType::LZ4, - format: CheckpointFormat::Binary, - max_checkpoints_per_model: 10, - auto_cleanup: true, - validate_checksums: true, - incremental_checkpoints: true, - compression_level: 3, - async_io: true, - buffer_size: 64 * 1024, // 64KB - } - } -} - -/// Statistics for checkpoint operations -#[derive(Debug, Default, Serialize, Deserialize)] -pub struct CheckpointStats { - /// Total checkpoints saved - pub total_saved: AtomicU64, - - /// Total checkpoints loaded - pub total_loaded: AtomicU64, - - /// Total bytes saved - pub total_bytes_saved: AtomicU64, - - /// Total bytes loaded - pub total_bytes_loaded: AtomicU64, - - /// Total compression savings (bytes) - pub compression_savings: AtomicU64, - - /// Average save time (microseconds) - pub avg_save_time_us: AtomicU64, - - /// Average load time (microseconds) - pub avg_load_time_us: AtomicU64, - - /// Failed operations - pub failed_operations: AtomicU64, -} - -impl CheckpointStats { - /// Record a save operation - pub fn record_save(&self, bytes_saved: u64, save_time_us: u64, compression_savings: u64) { - self.total_saved.fetch_add(1, Ordering::Relaxed); - self.total_bytes_saved - .fetch_add(bytes_saved, Ordering::Relaxed); - self.compression_savings - .fetch_add(compression_savings, Ordering::Relaxed); - - // Update moving average - let count = self.total_saved.load(Ordering::Relaxed); - let current_avg = self.avg_save_time_us.load(Ordering::Relaxed); - let new_avg = ((current_avg * (count - 1)) + save_time_us) / count; - self.avg_save_time_us.store(new_avg, Ordering::Relaxed); - } - - /// Record a load operation - pub fn record_load(&self, bytes_loaded: u64, load_time_us: u64) { - self.total_loaded.fetch_add(1, Ordering::Relaxed); - self.total_bytes_loaded - .fetch_add(bytes_loaded, Ordering::Relaxed); - - // Update moving average - let count = self.total_loaded.load(Ordering::Relaxed); - let current_avg = self.avg_load_time_us.load(Ordering::Relaxed); - let new_avg = ((current_avg * (count - 1)) + load_time_us) / count; - self.avg_load_time_us.store(new_avg, Ordering::Relaxed); - } - - /// Record a failed operation - pub fn record_failure(&self) { - self.failed_operations.fetch_add(1, Ordering::Relaxed); - } - - /// Get statistics as a map - pub fn to_map(&self) -> HashMap { - let mut map = HashMap::new(); - map.insert( - "total_saved".to_owned(), - self.total_saved.load(Ordering::Relaxed), - ); - map.insert( - "total_loaded".to_owned(), - self.total_loaded.load(Ordering::Relaxed), - ); - map.insert( - "total_bytes_saved".to_owned(), - self.total_bytes_saved.load(Ordering::Relaxed), - ); - map.insert( - "total_bytes_loaded".to_owned(), - self.total_bytes_loaded.load(Ordering::Relaxed), - ); - map.insert( - "compression_savings".to_owned(), - self.compression_savings.load(Ordering::Relaxed), - ); - map.insert( - "avg_save_time_us".to_owned(), - self.avg_save_time_us.load(Ordering::Relaxed), - ); - map.insert( - "avg_load_time_us".to_owned(), - self.avg_load_time_us.load(Ordering::Relaxed), - ); - map.insert( - "failed_operations".to_owned(), - self.failed_operations.load(Ordering::Relaxed), - ); - map - } -} - -/// Trait that models must implement to support checkpointing -#[async_trait] -pub trait Checkpointable { - /// Get model type - fn model_type(&self) -> ModelType; - - /// Get model name/identifier - fn model_name(&self) -> &str; - - /// Get model version - fn model_version(&self) -> &str; - - /// Serialize model weights and state to bytes - async fn serialize_state(&self) -> Result, MLError>; - - /// Deserialize model weights and state from bytes - async fn deserialize_state(&mut self, data: &[u8]) -> Result<(), MLError>; - - /// Get current training state (epoch, step, loss, etc.) - fn get_training_state(&self) -> (Option, Option, Option, Option) { - (None, None, None, None) // Default implementation - } - - /// Get hyperparameters for metadata - fn get_hyperparameters(&self) -> HashMap { - HashMap::new() // Default implementation - } - - /// Get current metrics for metadata - fn get_metrics(&self) -> HashMap { - HashMap::new() // Default implementation - } - - /// Get architecture information - fn get_architecture_info(&self) -> HashMap { - HashMap::new() // Default implementation - } - - /// Validate loaded state (optional override for custom validation) - fn validate_loaded_state(&self) -> Result<(), MLError> { - Ok(()) // Default implementation - } -} - -/// Main checkpoint manager for all AI models -#[derive(Debug)] -pub struct CheckpointManager { - /// Configuration - config: CheckpointConfig, - - /// Storage backend - storage: Arc, - - /// Checkpoint metadata index - metadata_index: Arc>>, - - /// Statistics - stats: Arc, - - /// Version manager - version_manager: Arc, - - /// Compression manager - compression_manager: Arc, - - /// Validation manager - validation_manager: Arc, -} - -impl CheckpointManager { - /// Create a new checkpoint manager - pub fn new(config: CheckpointConfig) -> Result { - // Create base directory if it doesn't exist - if !config.base_dir.exists() { - fs::create_dir_all(&config.base_dir).map_err(|e| { - MLError::ModelError(format!("Failed to create checkpoint directory: {}", e)) - })?; - } - - let storage: Arc = - Arc::new(FileSystemStorage::new(config.base_dir.clone())); - let metadata_index = Arc::new(RwLock::new(DashMap::new())); - let stats = Arc::new(CheckpointStats::default()); - let version_manager = Arc::new(VersionManager::new()); - let compression_manager = Arc::new(CompressionManager::new()); - let validation_manager = Arc::new(ValidationManager::new()); - - Ok(Self { - config, - storage, - metadata_index, - stats, - version_manager, - compression_manager, - validation_manager, - }) - } - - /// Save a checkpoint for a model - #[instrument(skip(self, model))] - pub async fn save_checkpoint( - &self, - model: &M, - tags: Option>, - ) -> Result { - let start_time = std::time::Instant::now(); - - info!("Saving checkpoint for model: {}", model.model_name()); - - // Create metadata - let (epoch, step, loss, accuracy) = model.get_training_state(); - let mut metadata = CheckpointMetadata::new( - model.model_type(), - model.model_name().to_string(), - model.model_version().to_string(), - ) - .with_training_state(epoch, step, loss, accuracy) - .with_hyperparameters(model.get_hyperparameters()) - .with_metrics(model.get_metrics()); - - if let Some(tags) = tags { - metadata = metadata.with_tags(tags); - } - - metadata.format = self.config.format; - metadata.compression = self.config.compression; - - // Add architecture info - metadata.architecture = model.get_architecture_info(); - - // Serialize model state - let model_data = model.serialize_state().await?; - let original_size = model_data.len() as u64; - - // Compress if needed - let (final_data, compressed_size) = if self.config.compression != CompressionType::None { - let compressed = self.compression_manager.compress( - &model_data, - self.config.compression, - self.config.compression_level, - )?; - let comp_size = compressed.len() as u64; - (compressed, Some(comp_size)) - } else { - (model_data, None) - }; - - // Calculate checksum - let mut hasher = Sha256::new(); - hasher.update(&final_data); - let checksum = format!("{:x}", hasher.finalize()); - - // Update metadata - metadata.file_size = original_size; - metadata.compressed_size = compressed_size; - metadata.checksum = checksum; - - // Generate filename - let filename = metadata.generate_filename(); - - // Save to storage - self.storage - .save_checkpoint(&filename, &final_data, &metadata) - .await?; - - // Update index - { - let index = self.metadata_index.write().await; - index.insert(metadata.checkpoint_id.clone(), metadata.clone()); - } - - // Cleanup old checkpoints if needed - if self.config.auto_cleanup { - self.cleanup_old_checkpoints(model.model_type(), model.model_name()) - .await?; - } - - // Record statistics - let save_time_us = start_time.elapsed().as_micros() as u64; - let compression_savings = compressed_size - .map(|cs| original_size.saturating_sub(cs)) - .unwrap_or(0); - self.stats - .record_save(original_size, save_time_us, compression_savings); - - info!( - "Checkpoint saved: {} ({} bytes, {}µs, {:.1}% compression)", - filename, - final_data.len(), - save_time_us, - if compressed_size.is_some() { - (compression_savings as f64 / original_size as f64) * 100.0 - } else { - 0.0 - } - ); - - Ok(metadata.checkpoint_id) - } - - /// Load a checkpoint into a model - #[instrument(skip(self, model))] - pub async fn load_checkpoint( - &self, - model: &mut M, - checkpoint_id: &str, - ) -> Result { - let start_time = std::time::Instant::now(); - - info!("Loading checkpoint: {}", checkpoint_id); - - // Get metadata - let metadata = { - let index = self.metadata_index.read().await; - index.get(checkpoint_id).map(|entry| entry.clone()) - }; - - let metadata = metadata.ok_or_else(|| { - MLError::ModelError(format!("Checkpoint not found: {}", checkpoint_id)) - })?; - - // Verify model compatibility - if metadata.model_type != model.model_type() { - return Err(MLError::ModelError(format!( - "Model type mismatch: expected {:?}, got {:?}", - model.model_type(), - metadata.model_type - ))); - } - - // Load data from storage - let filename = metadata.generate_filename(); - let data = self.storage.load_checkpoint(&filename).await?; - - // Validate checksum if enabled - if self.config.validate_checksums { - self.validation_manager - .validate_checksum(&data, &metadata.checksum)?; - } - - // Decompress if needed - let final_data = if metadata.compression != CompressionType::None { - self.compression_manager - .decompress(&data, metadata.compression)? - } else { - data - }; - - // Deserialize into model - model.deserialize_state(&final_data).await?; - - // Validate loaded state - model.validate_loaded_state()?; - - // Record statistics - let load_time_us = start_time.elapsed().as_micros() as u64; - self.stats - .record_load(final_data.len() as u64, load_time_us); - - info!( - "Checkpoint loaded: {} ({} bytes, {}µs)", - filename, - final_data.len(), - load_time_us - ); - - Ok(metadata) - } - - /// Load the latest checkpoint for a model - pub async fn load_latest_checkpoint( - &self, - model: &mut M, - ) -> Result, MLError> { - let model_type = model.model_type(); - let model_name = model.model_name(); - - // Find latest checkpoint - let latest_checkpoint = { - let index = self.metadata_index.read().await; - index - .iter() - .filter(|entry| { - let metadata = entry.value(); - metadata.model_type == model_type && metadata.model_name == model_name - }) - .max_by(|a, b| a.value().created_at.cmp(&b.value().created_at)) - .map(|entry| entry.value().clone()) - }; - - if let Some(metadata) = latest_checkpoint { - let loaded_metadata = self.load_checkpoint(model, &metadata.checkpoint_id).await?; - Ok(Some(loaded_metadata)) - } else { - Ok(None) - } - } - - /// List all checkpoints for a model - pub async fn list_checkpoints( - &self, - model_type: ModelType, - model_name: &str, - ) -> Vec { - let index = self.metadata_index.read().await; - let mut checkpoints: Vec<_> = index - .iter() - .filter(|entry| { - let metadata = entry.value(); - // Empty string matches all model names (wildcard behavior) - metadata.model_type == model_type - && (model_name.is_empty() || metadata.model_name == model_name) - }) - .map(|entry| entry.value().clone()) - .collect(); - - // Sort by creation time (newest first) - checkpoints.sort_by(|a, b| b.created_at.cmp(&a.created_at)); - checkpoints - } - - /// Delete a checkpoint - pub async fn delete_checkpoint(&self, checkpoint_id: &str) -> Result<(), MLError> { - // Get metadata - let metadata = { - let index = self.metadata_index.read().await; - index.get(checkpoint_id).map(|entry| entry.clone()) - }; - - let metadata = metadata.ok_or_else(|| { - MLError::ModelError(format!("Checkpoint not found: {}", checkpoint_id)) - })?; - - // Delete from storage - let filename = metadata.generate_filename(); - self.storage.delete_checkpoint(&filename).await?; - - // Remove from index - { - let index = self.metadata_index.write().await; - index.remove(checkpoint_id); - } - - info!("Checkpoint deleted: {}", checkpoint_id); - Ok(()) - } - - /// Cleanup old checkpoints for a model - async fn cleanup_old_checkpoints( - &self, - model_type: ModelType, - model_name: &str, - ) -> Result<(), MLError> { - let mut checkpoints = self.list_checkpoints(model_type, model_name).await; - - if checkpoints.len() <= self.config.max_checkpoints_per_model { - return Ok(()); - } - - // Remove oldest checkpoints - checkpoints.sort_by(|a, b| a.created_at.cmp(&b.created_at)); - let to_remove = checkpoints.len() - self.config.max_checkpoints_per_model; - - for checkpoint in checkpoints.into_iter().take(to_remove) { - if let Err(e) = self.delete_checkpoint(&checkpoint.checkpoint_id).await { - warn!( - "Failed to delete old checkpoint {}: {}", - checkpoint.checkpoint_id, e - ); - } - } - - info!( - "Cleaned up {} old checkpoints for {}:{}", - to_remove, - model_type.file_extension(), - model_name - ); - Ok(()) - } - - /// Get checkpoint statistics - pub fn get_stats(&self) -> HashMap { - self.stats.to_map() - } - - /// Refresh metadata index from storage - pub async fn refresh_index(&self) -> Result<(), MLError> { - let all_metadata = self.storage.list_all_checkpoints().await?; - - let index = self.metadata_index.write().await; - index.clear(); - - for metadata in all_metadata { - index.insert(metadata.checkpoint_id.clone(), metadata); - } - - info!( - "Refreshed checkpoint index with {} checkpoints", - index.len() - ); - Ok(()) - } - - /// Get checkpoint by tags - pub async fn find_checkpoints_by_tags(&self, tags: &[String]) -> Vec { - let index = self.metadata_index.read().await; - index - .iter() - .filter(|entry| { - let metadata = entry.value(); - tags.iter().all(|tag| metadata.tags.contains(tag)) - }) - .map(|entry| entry.value().clone()) - .collect() - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::sync::atomic::AtomicBool; - - - // Mock model for testing - struct MockModel { - model_type: ModelType, - model_name: String, - version: String, - data: Vec, - trained: AtomicBool, - } - - impl MockModel { - fn new(model_type: ModelType, name: String, version: String) -> Self { - Self { - model_type, - model_name: name, - version, - data: vec![1, 2, 3, 4, 5], - trained: AtomicBool::new(false), - } - } - } - - #[async_trait] - impl Checkpointable for MockModel { - fn model_type(&self) -> ModelType { - self.model_type - } - - fn model_name(&self) -> &str { - &self.model_name - } - - fn model_version(&self) -> &str { - &self.version - } - - async fn serialize_state(&self) -> Result, MLError> { - Ok(self.data.clone()) - } - - async fn deserialize_state(&mut self, data: &[u8]) -> Result<(), MLError> { - self.data = data.to_vec(); - self.trained.store(true, Ordering::Relaxed); - Ok(()) - } - - fn get_training_state(&self) -> (Option, Option, Option, Option) { - (Some(10), Some(1000), Some(0.1), Some(0.95)) - } - } - - #[tokio::test] - async fn test_checkpoint_save_load() -> Result<(), MLError> { - let temp_dir = tempfile::tempdir()?; - let config = CheckpointConfig { - base_dir: temp_dir.path().to_path_buf(), - compression: CompressionType::None, - ..Default::default() - }; - - let manager = CheckpointManager::new(config)?; - let mut model = MockModel::new( - ModelType::DQN, - "test_model".to_owned(), - "1.0.0".to_owned(), - ); - - // Save checkpoint - let checkpoint_id = manager - .save_checkpoint(&model, Some(vec!["test".to_owned()])) - .await?; - - // Modify model data - model.data = vec![9, 8, 7, 6, 5]; - - // Load checkpoint - let metadata = manager.load_checkpoint(&mut model, &checkpoint_id).await?; - - // Verify data was restored - assert_eq!(model.data, vec![1, 2, 3, 4, 5]); - assert_eq!(metadata.model_type, ModelType::DQN); - assert_eq!(metadata.model_name, "test_model"); - assert_eq!(metadata.tags, vec!["test".to_owned()]); - assert!(model.trained.load(Ordering::Relaxed)); - Ok(()) - } - - #[tokio::test] - async fn test_checkpoint_compression() -> Result<(), MLError> { - let temp_dir = tempfile::tempdir()?; - let config = CheckpointConfig { - base_dir: temp_dir.path().to_path_buf(), - compression: CompressionType::LZ4, - ..Default::default() - }; - - let manager = CheckpointManager::new(config)?; - let mut model = MockModel::new( - ModelType::MAMBA, - "test_model".to_owned(), - "1.0.0".to_owned(), - ); - - // Create larger data for compression test - model.data = vec![42; 1000]; - - let checkpoint_id = manager.save_checkpoint(&model, None).await?; - - // Clear data - model.data.clear(); - - // Load and verify - manager.load_checkpoint(&mut model, &checkpoint_id).await?; - assert_eq!(model.data, vec![42; 1000]); - Ok(()) - } - - #[tokio::test] - async fn test_checkpoint_metadata() -> Result<(), MLError> { - let metadata = CheckpointMetadata::new( - ModelType::TFT, - "transformer_model".to_owned(), - "2.1.0".to_owned(), - ) - .with_training_state(Some(50), Some(5000), Some(0.05), Some(0.98)) - .with_tags(vec!["production".to_owned(), "validated".to_owned()]); - - assert_eq!(metadata.model_type, ModelType::TFT); - assert_eq!(metadata.epoch, Some(50)); - assert_eq!(metadata.accuracy, Some(0.98)); - assert!(metadata.tags.contains(&"production".to_owned())); - - let filename = metadata.generate_filename(); - assert!(filename.contains("tft")); - assert!(filename.contains("transformer_model")); - assert!(filename.contains("v2.1.0")); - assert!(filename.contains("e50")); - assert!(filename.contains("s5000")); - Ok(()) - } - - #[tokio::test] - async fn test_list_and_cleanup_checkpoints() -> Result<(), MLError> { - let temp_dir = tempfile::tempdir()?; - let config = CheckpointConfig { - base_dir: temp_dir.path().to_path_buf(), - max_checkpoints_per_model: 2, - auto_cleanup: false, - ..Default::default() - }; - - let manager = CheckpointManager::new(config)?; - let model = MockModel::new( - ModelType::TGGN, - "graph_model".to_owned(), - "1.0.0".to_owned(), - ); - - // Save multiple checkpoints - let id1 = manager.save_checkpoint(&model, None).await?; - tokio::time::sleep(tokio::time::Duration::from_millis(10)).await; - let id2 = manager.save_checkpoint(&model, None).await?; - tokio::time::sleep(tokio::time::Duration::from_millis(10)).await; - let id3 = manager.save_checkpoint(&model, None).await?; - - // List checkpoints - let checkpoints = manager - .list_checkpoints(ModelType::TGGN, "graph_model") - .await; - assert_eq!(checkpoints.len(), 3); - - // Manual cleanup - manager - .cleanup_old_checkpoints(ModelType::TGGN, "graph_model") - .await?; - - // Should have only 2 checkpoints now - let checkpoints = manager - .list_checkpoints(ModelType::TGGN, "graph_model") - .await; - assert_eq!(checkpoints.len(), 2); - - // The oldest checkpoint should be gone - assert!(manager - .load_checkpoint( - &mut MockModel::new( - ModelType::TGGN, - "graph_model".to_owned(), - "1.0.0".to_owned() - ), - &id1 - ) - .await - .is_err()); - assert!(manager - .load_checkpoint( - &mut MockModel::new( - ModelType::TGGN, - "graph_model".to_owned(), - "1.0.0".to_owned() - ), - &id2 - ) - .await - .is_ok()); - assert!(manager - .load_checkpoint( - &mut MockModel::new( - ModelType::TGGN, - "graph_model".to_owned(), - "1.0.0".to_owned() - ), - &id3 - ) - .await - .is_ok()); - Ok(()) - } -} +// Re-export model checkpoint types for convenience +pub use ml_dqn::checkpoint::DQNCheckpointState; +pub use ml_supervised::checkpoint::{MambaCheckpointState, TGGNCheckpointState}; diff --git a/crates/ml/src/checkpoint/model_implementations.rs b/crates/ml/src/checkpoint/model_implementations.rs deleted file mode 100644 index 88599a1b1..000000000 --- a/crates/ml/src/checkpoint/model_implementations.rs +++ /dev/null @@ -1,706 +0,0 @@ -//! Model-specific checkpoint implementations -//! -//! Implements the Checkpointable trait for all 5 AI models. - -use std::collections::HashMap; - -use async_trait::async_trait; -use candle_core::{Device, Tensor}; -use serde::{Deserialize, Serialize}; -use serde_json::Value; -use tracing::{debug, info, warn}; - -use super::{Checkpointable, ModelType}; -use crate::MLError; -use rust_decimal::Decimal; - -// Import all model types -use crate::dqn::agent::DQNAgent; -use crate::dqn::DQNConfig; -use crate::liquid::LiquidNetworkConfig; -use crate::mamba::{Mamba2Config, Mamba2SSM}; -use crate::tft::TFTConfig; -use crate::tgnn::{TGGNConfig, TGGN}; - -/// Serializable state for `DQN` model -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct DQNCheckpointState { - /// Model configuration - pub config: DQNConfig, - - /// Training state - pub epoch: Option, - pub step: Option, - pub total_episodes: u64, - pub total_steps: u64, - - /// Model weights (serialized as bytes) - pub q_network_weights: Vec, - pub target_network_weights: Vec, - - /// Replay buffer state - pub replay_buffer_size: usize, - pub replay_buffer_capacity: usize, - - /// Training metrics - pub average_reward: f64, - pub epsilon: f64, - pub loss_history: Vec, - - /// Performance stats - pub total_inferences: u64, - pub avg_inference_time_us: f64, -} - -#[async_trait] -impl Checkpointable for DQNAgent { - fn model_type(&self) -> ModelType { - ModelType::DQN - } - - fn model_name(&self) -> &str { - "dqn_agent" - } - - fn model_version(&self) -> &str { - "1.0.0" - } - - async fn serialize_state(&self) -> Result, MLError> { - let state = DQNCheckpointState { - config: self.config.clone(), - epoch: None, // DQN doesn't track epochs - step: None, - total_episodes: self.metrics.total_episodes, - total_steps: 0, - q_network_weights: self - .extract_network_weights("q_network") - .unwrap_or_default(), - target_network_weights: vec![], - replay_buffer_size: self.get_replay_buffer_size(), - replay_buffer_capacity: self.config.replay_buffer_capacity, - average_reward: self.get_average_reward(), - epsilon: self.config.epsilon_start as f64, // Use epsilon_start instead of epsilon - loss_history: vec![], - total_inferences: 0, - avg_inference_time_us: 0.0, - }; - - let serialized = serde_json::to_vec(&state) - .map_err(|e| MLError::ModelError(format!("DQN serialization failed: {}", e)))?; - - debug!("Serialized DQN state: {} bytes", serialized.len()); - Ok(serialized) - } - - async fn deserialize_state(&mut self, data: &[u8]) -> Result<(), MLError> { - let state: DQNCheckpointState = serde_json::from_slice(data) - .map_err(|e| MLError::ModelError(format!("DQN deserialization failed: {}", e)))?; - - // Restore model state - self.config = state.config; - // Restore network weights and training state - if !state.q_network_weights.is_empty() { - if let Err(e) = self.restore_network_weights("q_network", &state.q_network_weights) { - warn!("Failed to restore Q-network weights: {}", e); - } - } - - if !state.target_network_weights.is_empty() { - if let Err(e) = - self.restore_network_weights("target_network", &state.target_network_weights) - { - warn!("Failed to restore target network weights: {}", e); - } - } - - // Restore training metadata - // Note: DQNAgent doesn't have current_epoch tracking in metrics - self.metrics.total_episodes = state.total_episodes; - self.metrics.avg_reward = Decimal::try_from(state.average_reward).unwrap_or(Decimal::ZERO); - - debug!("Deserialized DQN state from {} bytes", data.len()); - Ok(()) - } - - fn get_training_state(&self) -> (Option, Option, Option, Option) { - // Get actual training state from the agent's metrics - let current_epoch = None; // DQN doesn't track epochs, only episodes - let total_episodes = Some(self.metrics.total_episodes); - let average_reward = Some(TryInto::::try_into(self.metrics.avg_reward).unwrap_or(0.0)); - let loss = Some(TryInto::::try_into(self.metrics.current_loss).unwrap_or(0.0)); - (current_epoch, total_episodes, average_reward, loss) - } - - fn get_hyperparameters(&self) -> HashMap { - let mut params = HashMap::new(); - params.insert( - "learning_rate".to_owned(), - Value::from(self.config.learning_rate), - ); - params.insert( - "discount_factor".to_owned(), - Value::from(self.config.gamma), - ); // Use gamma instead of discount_factor - params.insert( - "epsilon".to_owned(), - Value::from(self.config.epsilon_start), - ); - params.insert( - "epsilon_decay".to_owned(), - Value::from(self.config.epsilon_decay), - ); - params.insert( - "epsilon_min".to_owned(), - Value::from(self.config.epsilon_end), - ); - params.insert( - "replay_buffer_size".to_owned(), - Value::from(self.config.replay_buffer_capacity), - ); - params.insert( - "batch_size".to_owned(), - Value::from(self.config.batch_size), - ); - params.insert( - "target_update_frequency".to_owned(), - Value::from(self.config.target_update_freq), - ); - params - } - - fn get_metrics(&self) -> HashMap { - // Get actual metrics from the agent's training metadata - let mut metrics = HashMap::new(); - metrics.insert( - "total_episodes".to_owned(), - self.metrics.total_episodes as f64, - ); - metrics.insert( - "average_reward".to_owned(), - TryInto::::try_into(self.metrics.avg_reward).unwrap_or(0.0), - ); - metrics.insert( - "current_loss".to_owned(), - TryInto::::try_into(self.metrics.current_loss).unwrap_or(0.0), - ); - metrics.insert("epsilon".to_owned(), self.config.epsilon_start as f64); // Current epsilon value - metrics.insert( - "replay_buffer_size".to_owned(), - self.get_replay_buffer_size() as f64, - ); - metrics.insert("average_reward".to_owned(), 0.0); - metrics.insert("success_rate".to_owned(), 0.0); - metrics.insert("exploration_rate".to_owned(), self.config.epsilon_start as f64); - metrics - } - - fn get_architecture_info(&self) -> HashMap { - let mut info = HashMap::new(); - info.insert("network_type".to_owned(), Value::from("DQN")); - info.insert("input_size".to_owned(), Value::from(self.config.state_dim)); - info.insert( - "hidden_size".to_owned(), - Value::from(self.config.hidden_dims.len()), - ); // Use length of hidden dims vector - info.insert( - "output_size".to_owned(), - Value::from(self.config.num_actions), - ); - info.insert("num_hidden_layers".to_owned(), Value::from(2)); - info.insert("activation".to_owned(), Value::from("ReLU")); - info - } -} -// DQNAgent checkpoint helper methods (extract/restore_network_weights, get_replay_buffer_size, -// get_average_reward) are defined in ml-dqn::agent since DQNAgent lives in that crate. - -/// Serializable state for `MAMBA` model -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct MambaCheckpointState { - /// Model configuration - pub config: Mamba2Config, - - /// Training state - pub epoch: Option, - pub step: Option, - pub training_loss: f64, - pub validation_loss: f64, - - /// Model parameters (simplified) - pub ssd_layer_weights: Vec>, - pub input_projection_weights: Vec, - pub output_projection_weights: Vec, - pub layer_norm_weights: Vec>, - - /// State space matrices - pub ssm_a_matrices: Vec>, - pub ssm_b_matrices: Vec>, - pub ssm_c_matrices: Vec>, - pub ssm_delta_params: Vec, - - /// Performance metrics - pub total_inferences: u64, - pub avg_latency_us: f64, - pub throughput_pps: f64, -} - -#[async_trait] -impl Checkpointable for Mamba2SSM { - fn model_type(&self) -> ModelType { - ModelType::MAMBA - } - - fn model_name(&self) -> &str { - &self.metadata.model_id - } - - fn model_version(&self) -> &str { - &self.metadata.version - } - - async fn serialize_state(&self) -> Result, MLError> { - // Get actual training state from the model - let (current_epoch, current_step) = self.get_current_training_state(); - let training_metrics = self.get_training_metrics(); - let performance_stats = self.get_inference_stats(); - - let state = MambaCheckpointState { - config: self.config.clone(), - epoch: current_epoch, - step: current_step, - training_loss: training_metrics - .get("training_loss") - .copied() - .unwrap_or(0.0), - validation_loss: training_metrics - .get("validation_loss") - .copied() - .unwrap_or(0.0), - ssd_layer_weights: self.extract_ssd_weights(), - input_projection_weights: self.extract_input_projection_weights(), - output_projection_weights: self.extract_output_projection_weights(), - layer_norm_weights: self.extract_layer_norm_weights(), - ssm_a_matrices: self.extract_ssm_matrices("A"), - ssm_b_matrices: self.extract_ssm_matrices("B"), - ssm_c_matrices: self.extract_ssm_matrices("C"), - ssm_delta_params: self.extract_delta_params(), - total_inferences: self - .total_inferences - .load(std::sync::atomic::Ordering::Relaxed), - avg_latency_us: performance_stats - .get("avg_latency_us") - .copied() - .unwrap_or(0.0), - throughput_pps: performance_stats - .get("throughput_pps") - .copied() - .unwrap_or(0.0), - }; - - let serialized = serde_json::to_vec(&state) - .map_err(|e| MLError::ModelError(format!("MAMBA serialization failed: {}", e)))?; - - debug!("Serialized MAMBA state: {} bytes", serialized.len()); - Ok(serialized) - } - - async fn deserialize_state(&mut self, data: &[u8]) -> Result<(), MLError> { - let state: MambaCheckpointState = serde_json::from_slice(data) - .map_err(|e| MLError::ModelError(format!("MAMBA deserialization failed: {}", e)))?; - - // Restore model state - self.config = state.config; - - // Restore actual model weights and parameters - self.restore_ssd_weights(&state.ssd_layer_weights); - self.restore_input_projection_weights(&state.input_projection_weights); - self.restore_output_projection_weights(&state.output_projection_weights); - self.restore_layer_norm_weights(&state.layer_norm_weights); - self.restore_ssm_matrices("A", &state.ssm_a_matrices); - self.restore_ssm_matrices("B", &state.ssm_b_matrices); - self.restore_ssm_matrices("C", &state.ssm_c_matrices); - self.restore_delta_params(&state.ssm_delta_params); - - debug!("Deserialized MAMBA state from {} bytes", data.len()); - Ok(()) - } - - fn get_training_state(&self) -> (Option, Option, Option, Option) { - // Get from training history with comprehensive state information - if let Some(last_epoch) = self.metadata.training_history.back() { - // Calculate total steps from training history - let total_steps = self - .metadata - .training_history - .iter() - .map(|epoch| epoch.epoch as u64 * 1000) // Assume 1000 steps per epoch - .max() - .unwrap_or(0); - - ( - Some(last_epoch.epoch as u64), - Some(total_steps), - Some(last_epoch.loss), - Some(last_epoch.accuracy), - ) - } else { - // Return default training state for untrained models - (Some(0), Some(0), Some(f64::INFINITY), Some(0.0)) - } - } - - fn get_hyperparameters(&self) -> HashMap { - let mut params = HashMap::new(); - params.insert("d_model".to_owned(), Value::from(self.config.d_model)); - params.insert("d_state".to_owned(), Value::from(self.config.d_state)); - params.insert("d_head".to_owned(), Value::from(self.config.d_head)); - params.insert("num_heads".to_owned(), Value::from(self.config.num_heads)); - params.insert("expand".to_owned(), Value::from(self.config.expand)); - params.insert( - "num_layers".to_owned(), - Value::from(self.config.num_layers), - ); - params.insert("dropout".to_owned(), Value::from(self.config.dropout)); - params.insert( - "learning_rate".to_owned(), - Value::from(self.config.learning_rate), - ); - params.insert( - "target_latency_us".to_owned(), - Value::from(self.config.target_latency_us), - ); - params - } - - fn get_metrics(&self) -> HashMap { - self.get_performance_metrics() - } - - fn get_architecture_info(&self) -> HashMap { - let mut info = HashMap::new(); - info.insert("model_type".to_owned(), Value::from("MAMBA-2")); - info.insert("input_dim".to_owned(), Value::from(self.config.d_model)); - info.insert( - "hidden_dim".to_owned(), - Value::from(self.config.d_model * self.config.expand), - ); - info.insert("state_dim".to_owned(), Value::from(self.config.d_state)); - info.insert( - "num_layers".to_owned(), - Value::from(self.config.num_layers), - ); - info.insert("use_ssd".to_owned(), Value::from(self.config.use_ssd)); - info.insert( - "use_selective_state".to_owned(), - Value::from(self.config.use_selective_state), - ); - info.insert( - "hardware_aware".to_owned(), - Value::from(self.config.hardware_aware), - ); - info - } -} - -/// Serializable state for `TFT` model -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct TFTCheckpointState { - /// Model configuration - pub config: TFTConfig, - - /// Training state - pub epoch: Option, - pub step: Option, - pub training_loss: f64, - pub validation_loss: f64, - - /// Model weights (simplified) - pub encoder_weights: Vec, - pub decoder_weights: Vec, - pub attention_weights: Vec, - pub variable_selection_weights: Vec, - pub quantile_layer_weights: Vec, - - /// Performance metrics - pub total_inferences: u64, - pub avg_latency_us: f64, - pub max_latency_us: f64, - pub throughput_pps: f64, -} - -// Note: TFT implementation would be similar to MAMBA -// For brevity, showing the structure but not full implementation - -/// Serializable state for TGGN model -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct TGGNCheckpointState { - /// Model configuration - pub config: TGGNConfig, - - /// Training state - pub epoch: Option, - pub step: Option, - pub training_loss: f64, - pub validation_loss: f64, - - /// Graph structure state - pub node_embeddings: HashMap>, - pub edge_embeddings: HashMap>, - pub graph_statistics: HashMap, - - /// Message passing weights - pub message_passing_weights: Vec>, - pub gating_weights: Vec, - - /// Performance metrics - pub total_inferences: u64, - pub graph_updates: u64, - pub avg_inference_latency_ns: u64, - pub avg_graph_update_latency_ns: u64, -} - -#[async_trait] -impl Checkpointable for TGGN { - fn model_type(&self) -> ModelType { - ModelType::TGGN - } - - fn model_name(&self) -> &str { - &self.metadata.version - } - - fn model_version(&self) -> &str { - &self.metadata.version - } - - async fn serialize_state(&self) -> Result, MLError> { - // Extract node embeddings - let node_embeddings: HashMap> = self - .node_embeddings() - .iter() - .map(|entry| { - let key = format!("{:?}", entry.key()); - let value = entry.value().iter().map(|&x| x as f32).collect(); - (key, value) - }) - .collect(); - - // Extract edge embeddings - let edge_embeddings: HashMap> = self - .edge_embeddings() - .iter() - .map(|entry| { - let key = format!("{:?}", entry.key()); - let value = entry.value().iter().map(|&x| x as f32).collect(); - (key, value) - }) - .collect(); - - let state = TGGNCheckpointState { - config: self.config().clone(), - epoch: None, - step: None, - training_loss: 0.0, - validation_loss: 0.0, - node_embeddings, - edge_embeddings, - graph_statistics: HashMap::new(), // Production since extract method doesn't exist - message_passing_weights: Vec::new(), // Production since extract method doesn't exist - gating_weights: Vec::new(), // Production since extract method doesn't exist - total_inferences: self - .inference_count() - .load(std::sync::atomic::Ordering::Relaxed), - graph_updates: self - .graph_updates() - .load(std::sync::atomic::Ordering::Relaxed), - avg_inference_latency_ns: self.calculate_avg_inference_latency(), - avg_graph_update_latency_ns: self.calculate_avg_graph_update_latency(), - }; - - let serialized = serde_json::to_vec(&state) - .map_err(|e| MLError::ModelError(format!("TGGN serialization failed: {}", e)))?; - - debug!("Serialized TGGN state: {} bytes", serialized.len()); - Ok(serialized) - } - - async fn deserialize_state(&mut self, data: &[u8]) -> Result<(), MLError> { - let state: TGGNCheckpointState = serde_json::from_slice(data) - .map_err(|e| MLError::ModelError(format!("TGGN deserialization failed: {}", e)))?; - - // Restore model state - self.restore_node_embeddings(&state.node_embeddings)?; - self.restore_edge_embeddings(&state.edge_embeddings)?; - self.restore_graph_statistics(&state.graph_statistics)?; - self.restore_message_passing_weights(&state.message_passing_weights)?; - - // Update inference counters - self.inference_count() - .store(state.total_inferences, std::sync::atomic::Ordering::Relaxed); - self.graph_updates() - .store(state.graph_updates, std::sync::atomic::Ordering::Relaxed); - - debug!("Deserialized TGGN state from {} bytes", data.len()); - Ok(()) - } - - fn get_hyperparameters(&self) -> HashMap { - let mut params = HashMap::new(); - let config = self.config(); // Use public getter method - params.insert("max_nodes".to_owned(), Value::from(config.max_nodes)); - params.insert("max_edges".to_owned(), Value::from(config.max_edges)); - params.insert("node_dim".to_owned(), Value::from(config.node_dim)); - params.insert("edge_dim".to_owned(), Value::from(config.edge_dim)); - params.insert("hidden_dim".to_owned(), Value::from(config.hidden_dim)); - params.insert("num_layers".to_owned(), Value::from(config.num_layers)); - params.insert( - "temporal_decay".to_owned(), - Value::from(config.temporal_decay), - ); - params.insert("use_simd".to_owned(), Value::from(config.use_simd)); - params - } - - fn get_architecture_info(&self) -> HashMap { - let mut info = HashMap::new(); - let config = self.config(); // Use public getter method - info.insert("model_type".to_owned(), Value::from("TGGN")); - info.insert("max_nodes".to_owned(), Value::from(config.max_nodes)); - info.insert("max_edges".to_owned(), Value::from(config.max_edges)); - info.insert("node_feature_dim".to_owned(), Value::from(config.node_dim)); - info.insert("edge_feature_dim".to_owned(), Value::from(config.edge_dim)); - info.insert("gnn_layers".to_owned(), Value::from(config.num_layers)); - info.insert("supports_temporal_decay".to_owned(), Value::from(true)); - info - } -} - - -/// Serializable state for Liquid Neural Network -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct LiquidCheckpointState { - /// Model configuration - pub config: LiquidNetworkConfig, - - /// Training state - pub epoch: Option, - pub step: Option, - pub training_loss: f64, - pub validation_loss: f64, - - /// Neural ODE parameters - pub ltc_weights: Vec, - pub ltc_biases: Vec, - pub tau_values: Vec, - - /// CfC parameters (if used) - pub cfc_weights: Vec, - pub cfc_biases: Vec, - - /// Output layer weights - pub output_weights: Vec, - pub output_biases: Vec, - - /// Performance metrics - pub total_inferences: u64, - pub avg_inference_time_us: f64, - pub ode_solver_stats: HashMap, -} - -// Note: Liquid Neural Network implementation would be similar -// For brevity, showing structure but not full implementation - -/// Helper function to create checkpoint implementations for all models -pub fn register_all_checkpoint_implementations() { - info!("Registering checkpoint implementations for all 5 AI models"); - - // All implementations are handled via the trait implementations above - // This function can be used for any global registration if needed -} - -// DISABLED: Tests require DQNAgent, TGGN, DQNConfig types not available -// #[cfg(test)] -// mod tests { -// use super::*; -// use anyhow::anyhow; -// use std::sync::atomic::AtomicU64; -// // use crate::safe_operations; // DISABLED - module not found -// -// #[tokio::test] -// async fn test_dqn_checkpoint_serialization() { -// let config = DQNConfig::default(); -// let agent = -// DQNAgent::new(config).map_err(|e| anyhow!("Failed to create DQN agent: {:?}", e))?; -// -// // Test serialization -// let serialized = agent.serialize_state().await?; -// assert!(!serialized.is_empty()); -// -// // Test deserialization -// let mut agent2 = DQNAgent::new(DQNConfig::default()) -// .map_err(|e| anyhow!("Failed to create DQN agent: {:?}", e))?; -// agent2.deserialize_state(&serialized).await?; -// -// // Verify model type and metadata -// assert_eq!(agent.model_type(), ModelType::DQN); -// assert_eq!(agent.model_name(), "dqn_agent"); -// assert_eq!(agent.model_version(), "1.0.0"); -// } -// -// #[tokio::test] -// async fn test_tggn_checkpoint_serialization() { -// let config = TGGNConfig::default(); -// let tggn = TGGN::new(config)?; -// -// // Test serialization -// let serialized = tggn.serialize_state().await?; -// assert!(!serialized.is_empty()); -// -// // Test deserialization -// let mut tggn2 = TGGN::new(TGGNConfig::default())?; -// tggn2.deserialize_state(&serialized).await?; -// -// // Verify model type and metadata -// assert_eq!(tggn.model_type(), ModelType::TGGN); -// assert!(!tggn.model_name().is_empty()); -// } -// -// #[test] -// fn test_hyperparameter_extraction() { -// let config = DQNConfig::default(); -// let agent = -// DQNAgent::new(config).map_err(|e| anyhow!("Failed to create DQN agent: {:?}", e))?; -// -// let hyperparams = agent.get_hyperparameters(); -// -// assert!(hyperparams.contains_key("learning_rate")); -// assert!(hyperparams.contains_key("epsilon")); -// assert!(hyperparams.contains_key("replay_buffer_size")); -// -// // Verify types -// if let Some(Value::Number(lr)) = hyperparams.get("learning_rate") { -// assert!(lr.as_f64()? > 0.0); -// } else { -// return Err(anyhow!("Learning rate should be a number")); -// } -// } -// -// #[test] -// fn test_architecture_info_extraction() { -// let config = TGGNConfig::default(); -// let tggn = TGGN::new(config)?; -// -// let arch_info = tggn.get_architecture_info(); -// -// assert!(arch_info.contains_key("model_type")); -// assert!(arch_info.contains_key("max_nodes")); -// assert!(arch_info.contains_key("gnn_layers")); -// -// if let Some(Value::String(model_type)) = arch_info.get("model_type") { -// assert_eq!(model_type, "TGGN"); -// } else { -// return Err(anyhow!("Model type should be a string")); -// } -// } -// } diff --git a/crates/ml/src/checkpoint/quantized_checkpoint.rs b/crates/ml/src/checkpoint/quantized_checkpoint.rs deleted file mode 100644 index ae33c6fd2..000000000 --- a/crates/ml/src/checkpoint/quantized_checkpoint.rs +++ /dev/null @@ -1,890 +0,0 @@ -//! SafeTensors-compatible storage for quantized INT8 model weights -//! -//! This module provides serialization/deserialization for quantized models with metadata, -//! enabling 3-4x file size reduction (FP32: ~300MB → INT8: ~100MB). -//! -//! ## Format Specification -//! -//! SafeTensors file structure: -//! ```text -//! { -//! "fc1.weight": , // Quantized weights -//! "fc1.weight.scale": , // Scaling factors -//! "fc1.weight.zero_point": , // Zero points -//! "__metadata__": { -//! "quantization": "int8", -//! "method": "symmetric", -//! "model_type": "DQN", -//! "version": "1.0.0" -//! } -//! } -//! ``` -//! -//! ## Usage Example -//! -//! ```ignore -//! use ml::checkpoint::quantized_checkpoint::{save_quantized_checkpoint, load_quantized_checkpoint}; -//! use ml::memory_optimization::quantization::{Quantizer, QuantizationConfig}; -//! -//! // Save quantized model -//! let quantized_weights = quantize_model_weights(&varmap)?; -//! save_quantized_checkpoint( -//! "model_int8.safetensors", -//! &quantized_weights, -//! Some(metadata), -//! false, // compress -//! )?; -//! -//! // Load quantized model -//! let (weights, metadata) = load_quantized_checkpoint("model_int8.safetensors")?; -//! ``` - -use crate::memory_optimization::quantization::{QuantizedTensor, QuantizationType}; -use crate::MLError; -use candle_core::{Device, Tensor}; -use serde::{Deserialize, Serialize}; -use std::collections::HashMap; -use std::fs::File; -use std::io::{Read, Write}; -use std::path::Path; -use tracing::{debug, info, warn}; -use uuid::Uuid; - -/// Quantized weight with scale and zero-point metadata -#[derive(Debug, Clone)] -pub struct QuantizedWeight { - /// Quantized INT8 data - pub data: Tensor, - /// Scaling factor - pub scale: f32, - /// Zero point for asymmetric quantization - pub zero_point: i8, - /// Original tensor shape - pub shape: Vec, -} - -impl QuantizedWeight { - /// Create from QuantizedTensor - pub fn from_quantized_tensor(qt: &QuantizedTensor) -> Result { - Ok(Self { - data: qt.data.clone(), - scale: qt.scale, - zero_point: qt.zero_point, - shape: qt.data.dims().to_vec(), - }) - } - - /// Convert to QuantizedTensor - pub fn to_quantized_tensor(&self) -> QuantizedTensor { - QuantizedTensor { - data: self.data.clone(), - quant_type: QuantizationType::Int8, - scale: self.scale, - zero_point: self.zero_point, - } - } - - /// Get memory size in bytes - pub fn memory_bytes(&self) -> usize { - let elem_count: usize = self.shape.iter().product(); - elem_count // INT8 = 1 byte per element - } -} - -/// Metadata for quantized checkpoint -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct QuantizedCheckpointMetadata { - /// Quantization method (e.g., "symmetric", "asymmetric") - pub quantization_method: String, - /// Quantization type (always "int8" for this implementation) - pub quantization_type: String, - /// Model type (DQN, MAMBA, PPO, TFT, etc.) - pub model_type: String, - /// Model version - pub version: String, - /// Number of quantized layers - pub num_layers: usize, - /// Total size in bytes (compressed) - pub total_size_bytes: usize, - /// Original FP32 size estimate - pub original_size_bytes: usize, - /// Additional custom metadata - #[serde(flatten)] - pub custom: HashMap, -} - -impl Default for QuantizedCheckpointMetadata { - fn default() -> Self { - Self { - quantization_method: "symmetric".to_owned(), - quantization_type: "int8".to_owned(), - model_type: "unknown".to_owned(), - version: "1.0.0".to_owned(), - num_layers: 0, - total_size_bytes: 0, - original_size_bytes: 0, - custom: HashMap::new(), - } - } -} - -/// Save quantized model weights to SafeTensors format -/// -/// # Arguments -/// * `path` - Output file path (.safetensors extension recommended) -/// * `weights` - HashMap of layer names to quantized weights -/// * `metadata` - Optional checkpoint metadata -/// * `compress` - Enable gzip compression (experimental) -/// -/// # Returns -/// * File size in bytes -/// -/// # File Format -/// SafeTensors with quantization metadata: -/// - Each weight layer stored as 3 tensors: data (i8), scale (f32), zero_point (i8) -/// - Metadata embedded in __metadata__ field -/// - Optional gzip compression for additional ~30% size reduction -pub fn save_quantized_checkpoint>( - path: P, - weights: &HashMap, - metadata: Option, - compress: bool, -) -> Result { - let path = path.as_ref(); - info!( - "Saving quantized checkpoint: {} ({} layers, compress={})", - path.display(), - weights.len(), - compress - ); - - // Build tensor map for SafeTensors - let mut tensor_map: HashMap = HashMap::new(); - - for (layer_name, weight) in weights { - // Store quantized data as U8 tensor (Candle doesn't support I8) - let u8_data = weight.data.clone(); - tensor_map.insert(format!("{}", layer_name), u8_data); - - // Store scale as F32 tensor (single value) - let scale_tensor = Tensor::new(&[weight.scale], &Device::Cpu) - .map_err(|e| MLError::ModelError(format!("Failed to create scale tensor: {}", e)))?; - tensor_map.insert(format!("{}.scale", layer_name), scale_tensor); - - // Store zero_point as U8 tensor (convert i8 → u8, single value) - let zp_u8 = (weight.zero_point as i16 + 128) as u8; - let zp_tensor = Tensor::new(&[zp_u8], &Device::Cpu) - .map_err(|e| { - MLError::ModelError(format!("Failed to create zero_point tensor: {}", e)) - })?; - tensor_map.insert(format!("{}.zero_point", layer_name), zp_tensor); - } - - // Build metadata - let mut meta = metadata.unwrap_or_default(); - meta.num_layers = weights.len(); - meta.total_size_bytes = weights.values().map(|w| w.memory_bytes()).sum(); - meta.original_size_bytes = meta.total_size_bytes * 4; // FP32 = 4x larger - - let metadata_json = serde_json::to_string(&meta) - .map_err(|e| MLError::ModelError(format!("Failed to serialize metadata: {}", e)))?; - - // Save to SafeTensors format using VarMap - use candle_nn::VarMap; - use candle_core::Var; - let varmap = VarMap::new(); - { - let mut vars = varmap.data().lock().unwrap(); - for (name, tensor) in tensor_map { - vars.insert(name, Var::from_tensor(&tensor)?); - } - } - - // Create temp file for SafeTensors - let temp_path = std::env::temp_dir().join(format!("safetensors_{}.tmp", Uuid::new_v4())); - varmap.save(&temp_path) - .map_err(|e| MLError::ModelError(format!("Failed to save SafeTensors: {}", e)))?; - - // Read back the SafeTensors data - let mut safetensors_data = std::fs::read(&temp_path) - .map_err(|e| MLError::ModelError(format!("Failed to read temp file: {}", e)))?; - std::fs::remove_file(&temp_path).ok(); - - // Inject metadata into SafeTensors header (manual header modification) - safetensors_data = inject_metadata_into_safetensors(safetensors_data, &metadata_json)?; - - let final_data = if compress { - // Optional gzip compression - use flate2::write::GzEncoder; - use flate2::Compression; - - let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); - encoder - .write_all(&safetensors_data) - .map_err(|e| MLError::ModelError(format!("Failed to compress: {}", e)))?; - encoder - .finish() - .map_err(|e| MLError::ModelError(format!("Failed to finish compression: {}", e)))? - } else { - safetensors_data - }; - - // Write to file - let mut file = File::create(path) - .map_err(|e| MLError::ModelError(format!("Failed to create file: {}", e)))?; - file.write_all(&final_data) - .map_err(|e| MLError::ModelError(format!("Failed to write file: {}", e)))?; - - let file_size = final_data.len(); - info!( - "Checkpoint saved: {} bytes ({:.1}% of FP32 size)", - file_size, - (file_size as f64 / meta.original_size_bytes as f64) * 100.0 - ); - - Ok(file_size) -} - -/// Load quantized model weights from SafeTensors format -/// -/// # Arguments -/// * `path` - Input file path -/// -/// # Returns -/// * Tuple of (weights HashMap, metadata) -/// -/// # Compatibility -/// - Automatically detects and decompresses gzip files -/// - Backward compatible with FP32 SafeTensors (converts on-the-fly) -/// - Validates metadata format and quantization method -pub fn load_quantized_checkpoint>( - path: P, -) -> Result<(HashMap, QuantizedCheckpointMetadata), MLError> { - let path = path.as_ref(); - info!("Loading quantized checkpoint: {}", path.display()); - - // Read file - let mut file = File::open(path) - .map_err(|e| MLError::ModelError(format!("Failed to open file: {}", e)))?; - let mut data = Vec::new(); - file.read_to_end(&mut data) - .map_err(|e| MLError::ModelError(format!("Failed to read file: {}", e)))?; - - // Detect and decompress gzip if needed - let safetensors_data = if is_gzip_compressed(&data) { - use flate2::read::GzDecoder; - let mut decoder = GzDecoder::new(&data[..]); - let mut decompressed = Vec::new(); - decoder - .read_to_end(&mut decompressed) - .map_err(|e| MLError::ModelError(format!("Failed to decompress: {}", e)))?; - debug!("Decompressed checkpoint: {} → {} bytes", data.len(), decompressed.len()); - decompressed - } else { - data - }; - - // Load SafeTensors - let tensors = candle_core::safetensors::load_buffer(&safetensors_data, &Device::Cpu) - .map_err(|e| MLError::ModelError(format!("Failed to load SafeTensors: {}", e)))?; - - // Extract metadata - let metadata_str = extract_metadata(&safetensors_data)?; - let metadata: QuantizedCheckpointMetadata = if !metadata_str.is_empty() { - serde_json::from_str(&metadata_str).unwrap_or_else(|e| { - warn!("Failed to parse metadata: {}, using default", e); - QuantizedCheckpointMetadata::default() - }) - } else { - warn!("No metadata found, using default"); - QuantizedCheckpointMetadata::default() - }; - - // Validate format - if metadata.quantization_type != "int8" && metadata.quantization_type != "" { - return Err(MLError::ModelError(format!( - "Unsupported quantization type: {} (expected 'int8')", - metadata.quantization_type - ))); - } - - // Reconstruct quantized weights - let mut weights = HashMap::new(); - let mut processed_layers = std::collections::HashSet::new(); - - for (name, tensor) in tensors.iter() { - // Skip scale/zero_point tensors (will be processed with base tensor) - if name.ends_with(".scale") || name.ends_with(".zero_point") { - continue; - } - - let layer_name = name.clone(); - - // Check if this is a quantized checkpoint (has .scale and .zero_point) - let scale_name = format!("{}.scale", layer_name); - let zp_name = format!("{}.zero_point", layer_name); - - if let (Some(scale_tensor), Some(zp_tensor)) = - (tensors.get(&scale_name), tensors.get(&zp_name)) - { - // INT8 quantized checkpoint - let scale = scale_tensor - .to_vec1::() - .map_err(|e| MLError::ModelError(format!("Failed to extract scale: {}", e)))?[0]; - - // Extract zero_point (stored as U8, convert to i8) - let zp_u8 = zp_tensor - .to_vec1::() - .map_err(|e| { - MLError::ModelError(format!("Failed to extract zero_point: {}", e)) - })?[0]; - let zero_point = zp_u8.wrapping_sub(128) as i8; - - // Data is already U8, just clone - let shape = tensor.dims().to_vec(); - - weights.insert( - layer_name.clone(), - QuantizedWeight { - data: tensor.clone(), - scale, - zero_point, - shape, - }, - ); - - processed_layers.insert(layer_name); - } else { - // FP32 checkpoint (backward compatibility) - quantize on-the-fly - warn!( - "Layer {} is FP32 format, converting to INT8 (this may cause precision loss)", - layer_name - ); - - // Simple symmetric quantization - let f32_vec = tensor - .flatten_all() - .map_err(|e| MLError::ModelError(format!("Failed to flatten: {}", e)))? - .to_vec1::() - .map_err(|e| MLError::ModelError(format!("Failed to extract F32: {}", e)))?; - - let min_val = f32_vec.iter().cloned().fold(f32::INFINITY, f32::min); - let max_val = f32_vec.iter().cloned().fold(f32::NEG_INFINITY, f32::max); - let abs_max = min_val.abs().max(max_val.abs()); - let scale = abs_max / 127.0; - - let u8_vec: Vec = f32_vec - .iter() - .map(|&x| ((x / scale) + 127.0).clamp(0.0, 255.0) as u8) - .collect(); - - let shape = tensor.dims().to_vec(); - let u8_tensor = Tensor::from_vec(u8_vec, shape.as_slice(), &Device::Cpu) - .map_err(|e| MLError::ModelError(format!("Failed to create U8 tensor: {}", e)))?; - - weights.insert( - layer_name.clone(), - QuantizedWeight { - data: u8_tensor, - scale, - zero_point: 127, - shape, - }, - ); - } - } - - info!( - "Checkpoint loaded: {} layers ({} bytes total)", - weights.len(), - weights.values().map(|w| w.memory_bytes()).sum::() - ); - - Ok((weights, metadata)) -} - -/// Inject metadata into SafeTensors header -fn inject_metadata_into_safetensors( - data: Vec, - metadata_json: &str, -) -> Result, MLError> { - // SafeTensors format: 8-byte header size | header JSON | tensor data - if data.len() < 8 { - return Err(MLError::ModelError("Invalid SafeTensors file".to_owned())); - } - - let header_size = u64::from_le_bytes([ - data[0], data[1], data[2], data[3], data[4], data[5], data[6], data[7], - ]) as usize; - - if data.len() < 8 + header_size { - return Err(MLError::ModelError("Invalid SafeTensors header size".to_owned())); - } - - // Parse existing header - let header_json = std::str::from_utf8(&data[8..8 + header_size]) - .map_err(|e| MLError::ModelError(format!("Invalid UTF-8 in header: {}", e)))?; - - let mut header: serde_json::Value = serde_json::from_str(header_json) - .map_err(|e| MLError::ModelError(format!("Failed to parse header JSON: {}", e)))?; - - // Add __metadata__ field - let metadata: serde_json::Value = serde_json::from_str(metadata_json) - .map_err(|e| MLError::ModelError(format!("Failed to parse metadata JSON: {}", e)))?; - - if let Some(obj) = header.as_object_mut() { - obj.insert("__metadata__".to_owned(), metadata); - } - - // Serialize updated header - let new_header_json = serde_json::to_string(&header) - .map_err(|e| MLError::ModelError(format!("Failed to serialize header: {}", e)))?; - - let new_header_size = new_header_json.len(); - let new_header_size_bytes = (new_header_size as u64).to_le_bytes(); - - // Rebuild SafeTensors file - let mut new_data = Vec::new(); - new_data.extend_from_slice(&new_header_size_bytes); - new_data.extend_from_slice(new_header_json.as_bytes()); - new_data.extend_from_slice(&data[8 + header_size..]); // Append tensor data - - Ok(new_data) -} - -/// Check if data is gzip compressed -fn is_gzip_compressed(data: &[u8]) -> bool { - data.len() >= 2 && data[0] == 0x1f && data[1] == 0x8b -} - -/// Extract metadata string from SafeTensors buffer -fn extract_metadata(data: &[u8]) -> Result { - // SafeTensors format: 8-byte header size | header JSON | tensor data - if data.len() < 8 { - return Ok(String::new()); - } - - let header_size = u64::from_le_bytes([ - data[0], data[1], data[2], data[3], data[4], data[5], data[6], data[7], - ]) as usize; - - if data.len() < 8 + header_size { - return Ok(String::new()); - } - - let header_json = std::str::from_utf8(&data[8..8 + header_size]) - .map_err(|e| MLError::ModelError(format!("Invalid UTF-8 in header: {}", e)))?; - - // Parse header to extract __metadata__ field - let header: serde_json::Value = serde_json::from_str(header_json) - .map_err(|e| MLError::ModelError(format!("Failed to parse header JSON: {}", e)))?; - - if let Some(metadata) = header.get("__metadata__") { - Ok(metadata.to_string()) - } else { - Ok(String::new()) - } -} - -/// Calculate compression ratio -pub fn calculate_compression_ratio( - weights: &HashMap, -) -> f64 { - let int8_size: usize = weights.values().map(|w| w.memory_bytes()).sum(); - let fp32_size = int8_size * 4; - fp32_size as f64 / int8_size as f64 -} - -#[cfg(test)] -mod tests { - use super::*; - use tempfile::NamedTempFile; - - #[test] - fn test_save_load_round_trip() -> Result<(), MLError> { - let device = Device::Cpu; - - // Create test quantized weights - let mut weights = HashMap::new(); - - // Layer 1: Simple INT8 tensor - let data1 = Tensor::from_vec(vec![0_u8, 127, 255], &[3], &device) - .map_err(|e| MLError::ModelError(format!("Failed to create tensor: {}", e)))?; - weights.insert( - "layer1.weight".to_owned(), - QuantizedWeight { - data: data1, - scale: 0.1, - zero_point: 127, - shape: vec![3], - }, - ); - - // Layer 2: 2D INT8 tensor - let data2 = Tensor::from_vec(vec![50_u8; 12], &[3, 4], &device) - .map_err(|e| MLError::ModelError(format!("Failed to create tensor: {}", e)))?; - weights.insert( - "layer2.weight".to_owned(), - QuantizedWeight { - data: data2, - scale: 0.05, - zero_point: 100, - shape: vec![3, 4], - }, - ); - - // Save to temp file - let temp_file = NamedTempFile::new().unwrap(); - let temp_path = temp_file.path(); - - let metadata = QuantizedCheckpointMetadata { - model_type: "DQN".to_owned(), - version: "1.0.0".to_owned(), - ..Default::default() - }; - - let file_size = save_quantized_checkpoint(temp_path, &weights, Some(metadata), false)?; - - assert!(file_size > 0); - assert!(file_size < 1000); // Should be small for test data - - // Load checkpoint - let (loaded_weights, loaded_metadata) = load_quantized_checkpoint(temp_path)?; - - assert_eq!(loaded_weights.len(), 2); - assert_eq!(loaded_metadata.model_type, "DQN"); - assert_eq!(loaded_metadata.num_layers, 2); - - // Verify layer1 - let layer1 = loaded_weights.get("layer1.weight").unwrap(); - assert_eq!(layer1.scale, 0.1); - assert_eq!(layer1.zero_point, 127); - assert_eq!(layer1.shape, vec![3]); - - // Verify layer2 - let layer2 = loaded_weights.get("layer2.weight").unwrap(); - assert_eq!(layer2.scale, 0.05); - assert_eq!(layer2.zero_point, 100); - assert_eq!(layer2.shape, vec![3, 4]); - - Ok(()) - } - - #[test] - fn test_compression_ratio() -> Result<(), MLError> { - let device = Device::Cpu; - let mut weights = HashMap::new(); - - // 1MB of INT8 data - let data = Tensor::from_vec(vec![127_u8; 1_000_000], &[1_000_000], &device) - .map_err(|e| MLError::ModelError(format!("Failed to create tensor: {}", e)))?; - weights.insert( - "large_layer".to_owned(), - QuantizedWeight { - data, - scale: 1.0, - zero_point: 127, - shape: vec![1_000_000], - }, - ); - - let ratio = calculate_compression_ratio(&weights); - assert_eq!(ratio, 4.0); // FP32 / INT8 = 4x - - Ok(()) - } - - #[test] - fn test_gzip_compression() -> Result<(), MLError> { - let device = Device::Cpu; - let mut weights = HashMap::new(); - - // Repetitive data (compresses well) - let data = Tensor::from_vec(vec![42_u8; 10_000], &[10_000], &device) - .map_err(|e| MLError::ModelError(format!("Failed to create tensor: {}", e)))?; - weights.insert( - "compressible".to_owned(), - QuantizedWeight { - data, - scale: 1.0, - zero_point: 127, - shape: vec![10_000], - }, - ); - - let temp_file = NamedTempFile::new().unwrap(); - let temp_path = temp_file.path(); - - // Save with compression - let compressed_size = - save_quantized_checkpoint(temp_path, &weights, None, true)?; - - // Save without compression - let temp_file2 = NamedTempFile::new().unwrap(); - let uncompressed_size = - save_quantized_checkpoint(temp_file2.path(), &weights, None, false)?; - - // Compressed should be smaller (significantly for repetitive data) - assert!(compressed_size < uncompressed_size); - - // Load compressed checkpoint - let (loaded_weights, _) = load_quantized_checkpoint(temp_path)?; - assert_eq!(loaded_weights.len(), 1); - - Ok(()) - } - - #[test] - fn test_file_size_validation_75_percent_smaller() -> Result<(), MLError> { - let device = Device::Cpu; - let mut weights = HashMap::new(); - - // Create realistic-sized weight tensors (simulating a small DQN model) - // FC1: 225 input features × 128 hidden units = 28,800 weights - let fc1_data = Tensor::from_vec(vec![127_u8; 28_800], &[225, 128], &device) - .map_err(|e| MLError::ModelError(format!("Failed to create tensor: {}", e)))?; - weights.insert( - "fc1.weight".to_owned(), - QuantizedWeight { - data: fc1_data, - scale: 0.02, - zero_point: 127, - shape: vec![225, 128], - }, - ); - - // FC2: 128 × 64 = 8,192 weights - let fc2_data = Tensor::from_vec(vec![100_u8; 8_192], &[128, 64], &device) - .map_err(|e| MLError::ModelError(format!("Failed to create tensor: {}", e)))?; - weights.insert( - "fc2.weight".to_owned(), - QuantizedWeight { - data: fc2_data, - scale: 0.015, - zero_point: 100, - shape: vec![128, 64], - }, - ); - - // Output layer: 64 × 3 = 192 weights - let output_data = Tensor::from_vec(vec![150_u8; 192], &[64, 3], &device) - .map_err(|e| MLError::ModelError(format!("Failed to create tensor: {}", e)))?; - weights.insert( - "output.weight".to_owned(), - QuantizedWeight { - data: output_data, - scale: 0.01, - zero_point: 150, - shape: vec![64, 3], - }, - ); - - let temp_file = NamedTempFile::new().unwrap(); - let temp_path = temp_file.path(); - - let metadata = QuantizedCheckpointMetadata { - model_type: "DQN".to_owned(), - version: "1.0.0".to_owned(), - ..Default::default() - }; - - let file_size = save_quantized_checkpoint(temp_path, &weights, Some(metadata.clone()), false)?; - - // Calculate expected sizes - let int8_size: usize = weights.values().map(|w| w.memory_bytes()).sum(); - let fp32_size = int8_size * 4; // FP32 is 4 bytes per element - - // INT8 checkpoint should be approximately 75% smaller than FP32 - // File size includes metadata overhead, so we allow some tolerance - let size_ratio = file_size as f64 / fp32_size as f64; - - // Verify INT8 file is 20-30% of FP32 size (70-80% reduction) - assert!(size_ratio < 0.30, "INT8 checkpoint should be <30% of FP32 size, got {:.1}%", size_ratio * 100.0); - assert!(size_ratio > 0.15, "INT8 checkpoint unusually small: {:.1}% of FP32 size", size_ratio * 100.0); - - // Verify compression ratio calculation - let compression_ratio = calculate_compression_ratio(&weights); - assert_eq!(compression_ratio, 4.0); // FP32 / INT8 = 4x - - // Verify metadata reflects size savings - let (_, loaded_metadata) = load_quantized_checkpoint(temp_path)?; - assert_eq!(loaded_metadata.total_size_bytes, int8_size); - assert_eq!(loaded_metadata.original_size_bytes, fp32_size); - - let metadata_size_ratio = loaded_metadata.total_size_bytes as f64 - / loaded_metadata.original_size_bytes as f64; - assert_eq!(metadata_size_ratio, 0.25); // INT8 is 25% of FP32 size - - println!( - "✓ File size validation passed: INT8={} bytes ({}% of FP32={})", - file_size, - (size_ratio * 100.0) as u32, - fp32_size - ); - - Ok(()) - } - - #[test] - fn test_cpu_cuda_device_compatibility() -> Result<(), MLError> { - // Test 1: Save on CPU, load on CPU - let device = Device::Cpu; - let mut weights = HashMap::new(); - - let data = Tensor::from_vec(vec![42_u8; 1000], &[10, 100], &device) - .map_err(|e| MLError::ModelError(format!("Failed to create tensor: {}", e)))?; - weights.insert( - "test_layer".to_owned(), - QuantizedWeight { - data, - scale: 0.1, - zero_point: 127, - shape: vec![10, 100], - }, - ); - - let temp_file = NamedTempFile::new().unwrap(); - let temp_path = temp_file.path(); - - save_quantized_checkpoint(temp_path, &weights, None, false)?; - - // Load back on CPU - let (loaded_weights, _) = load_quantized_checkpoint(temp_path)?; - assert_eq!(loaded_weights.len(), 1); - - let loaded_layer = loaded_weights.get("test_layer").unwrap(); - assert_eq!(loaded_layer.scale, 0.1); - assert_eq!(loaded_layer.zero_point, 127); - assert_eq!(loaded_layer.shape, vec![10, 100]); - - // Verify data can be moved to CUDA if available (without panic) - if let Ok(cuda_device) = Device::cuda_if_available(0) { - let cuda_tensor = loaded_layer.data.to_device(&cuda_device); - assert!(cuda_tensor.is_ok(), "Failed to transfer INT8 tensor to CUDA"); - - println!("✓ CUDA transfer test passed: INT8 tensor successfully moved to GPU"); - } else { - println!("⚠ CUDA not available, skipping GPU transfer test"); - } - - Ok(()) - } - - #[test] - fn test_metadata_preservation() -> Result<(), MLError> { - let device = Device::Cpu; - let mut weights = HashMap::new(); - - let data = Tensor::from_vec(vec![127_u8; 100], &[10, 10], &device) - .map_err(|e| MLError::ModelError(format!("Failed to create tensor: {}", e)))?; - weights.insert( - "layer".to_owned(), - QuantizedWeight { - data, - scale: 0.05, - zero_point: 100, - shape: vec![10, 10], - }, - ); - - let temp_file = NamedTempFile::new().unwrap(); - let temp_path = temp_file.path(); - - // Create metadata with custom fields - let mut metadata = QuantizedCheckpointMetadata { - quantization_method: "symmetric".to_owned(), - quantization_type: "int8".to_owned(), - model_type: "MAMBA".to_owned(), - version: "2.0.0".to_owned(), - ..Default::default() - }; - - // Add custom calibration info - metadata.custom.insert( - "calibration_samples".to_owned(), - serde_json::json!(1000), - ); - metadata.custom.insert( - "quantization_date".to_owned(), - serde_json::json!("2025-10-21"), - ); - - save_quantized_checkpoint(temp_path, &weights, Some(metadata.clone()), false)?; - - // Load and verify all metadata is preserved - let (_, loaded_metadata) = load_quantized_checkpoint(temp_path)?; - - assert_eq!(loaded_metadata.quantization_method, "symmetric"); - assert_eq!(loaded_metadata.quantization_type, "int8"); - assert_eq!(loaded_metadata.model_type, "MAMBA"); - assert_eq!(loaded_metadata.version, "2.0.0"); - assert_eq!(loaded_metadata.num_layers, 1); - - // Verify custom fields - assert_eq!( - loaded_metadata.custom.get("calibration_samples"), - Some(&serde_json::json!(1000)) - ); - assert_eq!( - loaded_metadata.custom.get("quantization_date"), - Some(&serde_json::json!("2025-10-21")) - ); - - println!("✓ Metadata preservation test passed: All fields preserved correctly"); - - Ok(()) - } - - #[test] - fn test_per_channel_quantization_metadata() -> Result<(), MLError> { - let device = Device::Cpu; - let mut weights = HashMap::new(); - - // Simulate per-channel quantization with different scales - let data = Tensor::from_vec(vec![50_u8; 60], &[3, 20], &device) - .map_err(|e| MLError::ModelError(format!("Failed to create tensor: {}", e)))?; - weights.insert( - "conv.weight".to_owned(), - QuantizedWeight { - data, - scale: 0.02, // Global scale (not used in per-channel) - zero_point: 127, - shape: vec![3, 20], - }, - ); - - let temp_file = NamedTempFile::new().unwrap(); - let temp_path = temp_file.path(); - - let mut metadata = QuantizedCheckpointMetadata { - quantization_method: "per_channel".to_owned(), - quantization_type: "int8".to_owned(), - model_type: "TFT".to_owned(), - version: "1.0.0".to_owned(), - ..Default::default() - }; - - // Add per-channel info to custom metadata - metadata.custom.insert( - "per_channel".to_owned(), - serde_json::json!(true), - ); - metadata.custom.insert( - "symmetric".to_owned(), - serde_json::json!(true), - ); - - save_quantized_checkpoint(temp_path, &weights, Some(metadata), false)?; - - let (loaded_weights, loaded_metadata) = load_quantized_checkpoint(temp_path)?; - - assert_eq!(loaded_metadata.quantization_method, "per_channel"); - assert_eq!( - loaded_metadata.custom.get("per_channel"), - Some(&serde_json::json!(true)) - ); - assert_eq!(loaded_weights.len(), 1); - - Ok(()) - } -} diff --git a/crates/ml/src/data_validation/mod.rs b/crates/ml/src/data_validation/mod.rs index bdef2646c..86713a7e4 100644 --- a/crates/ml/src/data_validation/mod.rs +++ b/crates/ml/src/data_validation/mod.rs @@ -1,70 +1,18 @@ //! # Data Quality Validation Module //! -//! Automated data quality validation for DBN market data files. +//! Thin facade re-exporting from the `ml-data-validation` crate. //! -//! ## Features -//! -//! - **OHLCV Integrity**: Validates price relationships (high≥low, volume≥0) -//! - **Price Continuity**: Detects price spikes (>20% changes) -//! - **Technical Indicators**: Validates RSI range (0-100), detects NaN/Inf -//! - **Timestamp Alignment**: Ensures proper ordering and no gaps -//! - **Data Completeness**: Checks for missing bars in sequence -//! - **Automatic Correction**: Interpolates price spikes, removes outliers -//! -//! ## Usage -//! -//! ```rust,no_run -//! use ml::data_validation::validator::DataValidator; -//! use ml::data_validation::rules::{IntegrityRule, ContinuityRule}; -//! -//! # async fn example() -> anyhow::Result<()> { -//! let validator = DataValidator::new() -//! .with_rule(Box::new(IntegrityRule::new())) -//! .with_rule(Box::new(ContinuityRule::new(0.20))) -//! .with_metrics_enabled(true); -//! -//! let bars = vec![/* ... */]; -//! let result = validator.validate(&bars)?; -//! -//! if !result.is_valid() { -//! println!("Validation failed: {}", result.generate_report()); -//! } -//! # Ok(()) -//! # } -//! ``` -//! -//! ## Architecture -//! -//! ```text -//! ┌──────────────────┐ -//! │ DataValidator │ -//! │ (orchestrator) │ -//! └────────┬─────────┘ -//! │ -//! ├─────▶ IntegrityRule (OHLCV checks) -//! ├─────▶ ContinuityRule (spike detection) -//! ├─────▶ IndicatorRule (RSI, MACD, etc.) -//! ├─────▶ TimestampRule (ordering, gaps) -//! └─────▶ CompletenessRule (missing bars) -//! │ -//! ▼ -//! ┌──────────────┐ -//! │ DataCorrector│ -//! │ (auto-fix) │ -//! └──────────────┘ -//! ``` +//! See [`ml_data_validation`] for full documentation. -pub mod corrector; -pub mod cpcv; -pub mod fdr; -pub mod rules; -pub mod validator; +pub use ml_data_validation::corrector; +pub use ml_data_validation::cpcv; +pub use ml_data_validation::fdr; +pub use ml_data_validation::rules; +pub use ml_data_validation::validator; // Re-export main types -pub use corrector::DataCorrector; -pub use cpcv::{CPCVConfig, CPCVResult, CPCVSplit, CPCVValidator}; -pub use fdr::{FDRConfig, FDRCorrector, FDRMethod, FDRResult}; -pub use rules::{ - CompletenessRule, ContinuityRule, IndicatorRule, IntegrityRule, TimestampRule, ValidationRule, +pub use ml_data_validation::{ + CPCVConfig, CPCVResult, CPCVSplit, CPCVValidator, CompletenessRule, ContinuityRule, + DataCorrector, DataValidator, FDRConfig, FDRCorrector, FDRMethod, FDRResult, IndicatorRule, + Indicators, IntegrityRule, TimestampRule, ValidationReport, ValidationResult, ValidationRule, }; -pub use validator::{DataValidator, ValidationReport, ValidationResult}; diff --git a/crates/ml/src/ensemble/mod.rs b/crates/ml/src/ensemble/mod.rs index ddef09b74..dc06c1277 100644 --- a/crates/ml/src/ensemble/mod.rs +++ b/crates/ml/src/ensemble/mod.rs @@ -1,120 +1,17 @@ -//! Ensemble signal aggregation for trading models +//! Ensemble — thin facade re-exporting from ml-ensemble crate. +//! Model-specific adapters stay here. -use std; +pub use ml_ensemble::*; -use thiserror::Error; +// Re-export sub-modules so existing `crate::ensemble::X::Y` paths keep working +pub use ml_ensemble::{ + ab_testing, adaptive_ml_integration, aggregator, confidence, conviction_gates, coordinator, + coordinator_extended, cuda_streams, decision, gate_optimizer, hot_swap, inference_adapter, + inference_ensemble, metrics, model, signal, stream_ensemble, training_integration, voting, + weight_optimizer, weights, +}; -pub mod ab_testing; -pub mod adaptive_ml_integration; // Adaptive ML ensemble with regime detection -pub mod aggregator; -pub mod confidence; -pub mod coordinator; -pub mod coordinator_extended; // Extended 6-model coordinator -pub mod decision; -pub mod hot_swap; -pub mod metrics; -pub mod model; -pub mod training_integration; // Training integration for ML service -pub mod voting; -pub mod weights; -pub mod inference_adapter; -pub mod inference_ensemble; -pub mod signal; pub mod adapters; -pub mod conviction_gates; -pub mod weight_optimizer; -pub mod gate_optimizer; pub mod model_adapter; -pub mod cuda_streams; -pub mod stream_ensemble; -// Re-export key types that are used across ensemble modules pub use model_adapter::{EnsembleModelAdapter, build_production_strategy}; -pub use ab_testing::{ - ABGroup, ABMetricsTracker, ABTestConfig, ABTestResults, ABTestRouter, GroupMetrics, - Recommendation, StatisticalTestResult, -}; -pub use adaptive_ml_integration::{ - AdaptiveMLEnsemble, AdaptiveMetrics, MarketRegime, PricePoint, RegimeConfig, -}; -pub use aggregator::{ModelSignal, SignalMetadata, SignalStatistics}; -pub use coordinator::{EnsembleCoordinator, ModelRegistry, SignalAggregator}; -pub use coordinator_extended::{ - DiversityAnalyzer, DiversityMetrics, EnsembleConfig as ExtendedEnsembleConfig, - ExtendedEnsembleCoordinator, ModelPerformance, PerformanceAttribution, PerformanceTracker, - SupportedModel, WeightSnapshot, -}; -pub use decision::{EnsembleDecision, ModelVote, ModelWeight, PerformanceMetrics, TradingAction}; -pub use hot_swap::{ - CanaryMetrics, CanaryResult, CheckpointModel, CheckpointValidator, HotSwapManager, - ModelBufferPair, RollbackPolicy, ValidationResult, -}; -pub use metrics::{ - EnsembleMetrics, CANARY_MONITORING_TOTAL, CHECKPOINT_SWAPS_TOTAL, - CHECKPOINT_SWAP_LATENCY_MICROSECONDS, CHECKPOINT_VALIDATION_TOTAL, -}; -pub use training_integration::EnsembleTrainingIntegration; -pub use confidence::{ - AggregationConfig, AleatoricConfig, CalibrationParams, CombinationMethod, - ConfidenceAggregator, DisagreementRecord, DisagreementTracker, EnsemblePredictionWithUncertainty, - EpistemicConfig, IntervalCombiner, ModelContribution, PredictionInterval, ReliabilityRecord, - ReliabilityScorer, UncertaintyDecomposition, UncertaintyQuantifier, VarianceEstimationMethod, -}; -pub use inference_adapter::{EnsemblePrediction, FeatureVector, ModelInferenceAdapter, PredictionMeta}; -pub use conviction_gates::{ - ConvictionGateConfig, ConvictionGateEvaluator, ConvictionGateOutcome, GateEvaluation, - GateInput, GatePassResult, GateRejection, TradingSession, -}; -pub use weight_optimizer::{ - ModelRollingMetrics, OptimizationResult, WeightAdjustment, WeightOptimizer, - WeightOptimizerConfig, -}; -pub use gate_optimizer::{ - GateBucketMetrics, GateOptimizationResult, GateOptimizer, GateOptimizerConfig, - ThresholdAdjustment, -}; - -/// Errors that can occur in ensemble operations -#[derive(Error, Debug)] -/// `EnsembleError` component. -pub enum EnsembleError { - #[error("Failed to acquire lock: {0}")] - LockAcquisitionFailed(String), - - #[error("Invalid ensemble configuration: {0}")] - InvalidConfiguration(String), - - #[error("Model not found: {0}")] - ModelNotFound(String), - - #[error("Insufficient models for ensemble: expected {expected}, got {actual}")] - InsufficientModels { expected: usize, actual: usize }, - - #[error("Weight calculation failed: {0}")] - WeightCalculationFailed(String), - - #[error("Aggregation failed: {0}")] - AggregationFailed(String), -} - -// Implement From trait for EnsembleError to MLError conversion -impl From for crate::MLError { - fn from(err: EnsembleError) -> Self { - match err { - EnsembleError::InvalidConfiguration(msg) => crate::MLError::ConfigError(msg), - EnsembleError::ModelNotFound(msg) => crate::MLError::ModelNotFound(msg), - EnsembleError::InsufficientModels { expected, actual } => { - crate::MLError::ValidationError { - message: format!("Insufficient models: expected {}, got {}", expected, actual), - } - }, - EnsembleError::LockAcquisitionFailed(msg) => crate::MLError::LockError(msg), - EnsembleError::WeightCalculationFailed(msg) => { - crate::MLError::ModelError(format!("Weight calculation failed: {}", msg)) - }, - EnsembleError::AggregationFailed(msg) => { - crate::MLError::InferenceError(format!("Aggregation failed: {}", msg)) - }, - } - } -} diff --git a/crates/ml/src/evaluation/mod.rs b/crates/ml/src/evaluation/mod.rs index 832d31c65..b8935bdf7 100644 --- a/crates/ml/src/evaluation/mod.rs +++ b/crates/ml/src/evaluation/mod.rs @@ -1,16 +1,3 @@ -//! DQN Evaluation Module -//! -//! Provides comprehensive backtesting evaluation for DQN models including: -//! - Position tracking with buy/sell/hold actions -//! - Trade recording (entry/exit prices, PnL) -//! - Performance metrics (Sharpe, win rate, drawdown, total return) -//! - Report generation (JSON, markdown, console) +//! Evaluation — thin facade re-exporting from ml-dqn crate. -pub mod engine; -pub mod metrics; -pub mod report; - -// Re-exports for convenience -pub use engine::{EvaluationEngine, Position, Trade}; -pub use metrics::PerformanceMetrics; -pub use report::BacktestReport; +pub use ml_dqn::evaluation::*; diff --git a/crates/ml/src/features/cache_service.rs b/crates/ml/src/features/cache_service.rs deleted file mode 100644 index d93b0b027..000000000 --- a/crates/ml/src/features/cache_service.rs +++ /dev/null @@ -1,466 +0,0 @@ -//! Feature cache service with MinIO backend and LRU cache -//! -//! This service provides: -//! - MinIO storage for persistent feature caching -//! - LRU in-memory cache for hot data -//! - SHA-256-based cache invalidation -//! - Metadata tracking (cache hits/misses) - -use crate::features::feature_extraction::{extract_ml_features, OHLCVBar}; -use crate::features::parquet_io::{ - deserialize_features_from_bytes, serialize_features_to_bytes, -}; -use crate::features::types::{CacheMetadata, CacheStats, FeatureCache, FeatureMatrix, FeatureStats}; -use crate::MLError; -use chrono::Utc; -use lru::LruCache; -use sha2::{Digest, Sha256}; -use std::num::NonZeroUsize; -use std::sync::Arc; -use tokio::sync::RwLock; - -/// Feature cache service with MinIO backend -pub struct FeatureCacheService { - /// MinIO storage backend - storage: Option>, - /// LRU in-memory cache - cache: Arc>>, - /// Cache statistics - stats: Arc>, - /// MinIO bucket name - bucket: String, - /// Cache enabled flag - enabled: bool, -} - -impl FeatureCacheService { - /// Create new feature cache service - /// - /// # Arguments - /// * `storage` - Optional MinIO storage backend - /// * `cache_size` - LRU cache size (default 100) - /// * `bucket` - MinIO bucket name (default "feature-cache") - pub fn new( - storage: Option>, - cache_size: Option, - bucket: Option, - ) -> Self { - let cache_size = cache_size.unwrap_or(100); - let bucket = bucket.unwrap_or_else(|| "feature-cache".to_owned()); - - Self { - storage, - cache: Arc::new(RwLock::new( - LruCache::new(NonZeroUsize::new(cache_size).expect("Cache size must be > 0")), - )), - stats: Arc::new(RwLock::new(CacheStats::new())), - bucket, - enabled: true, - } - } - - /// Create disabled cache service (always computes features) - pub fn disabled() -> Self { - Self { - storage: None, - cache: Arc::new(RwLock::new( - LruCache::new(NonZeroUsize::new(1).expect("Cache size must be > 0")), - )), - stats: Arc::new(RwLock::new(CacheStats::new())), - bucket: "feature-cache".to_owned(), - enabled: false, - } - } - - /// Get or compute features for OHLCV bars - /// - /// Logic: - /// 1. Compute data hash (SHA-256) - /// 2. Check in-memory LRU cache - /// 3. If not found, check MinIO - /// 4. If not found or hash mismatch, compute features - /// 5. Cache result (memory + MinIO) - pub async fn get_or_compute( - &self, - symbol: &str, - bars: &[OHLCVBar], - ) -> Result { - if !self.enabled { - // Cache disabled, compute directly - let features = extract_ml_features(bars)?; - let mut stats = self.stats.write().await; - stats.record_computed(bars.len()); - return Ok(FeatureMatrix::new(features, symbol.to_string())); - } - - // Compute data hash for invalidation - let data_hash = self.compute_data_hash(bars); - let cache_key = format!("{}_{}", symbol, data_hash); - - // Check in-memory cache - { - let mut cache = self.cache.write().await; - if let Some(cached) = cache.get(&cache_key) { - // Verify hash matches - if cached.metadata.data_hash == data_hash { - tracing::debug!("Feature cache hit (memory): {}", symbol); - let mut stats = self.stats.write().await; - stats.record_hit(); - stats.record_loaded(bars.len()); - return Ok(cached.matrix.clone()); - } else { - // Hash mismatch, invalidate - cache.pop(&cache_key); - let mut stats = self.stats.write().await; - stats.record_invalidation(); - } - } - } - - // Check MinIO if available - if let Some(ref storage) = self.storage { - let object_key = self.build_object_key(symbol, &data_hash); - - match self.load_from_minio(storage, &object_key).await { - Ok(feature_matrix) => { - tracing::debug!("Feature cache hit (MinIO): {}", symbol); - - // Update in-memory cache - let metadata = CacheMetadata { - symbol: symbol.to_string(), - bar_count: bars.len(), - feature_dim: feature_matrix.feature_dim, - created_at: Utc::now(), - data_hash: data_hash.clone(), - object_key: object_key.clone(), - stats: Some(FeatureStats::from_features(&feature_matrix.features)), - }; - - let cache_entry = FeatureCache { - matrix: feature_matrix.clone(), - metadata, - last_access: Utc::now(), - }; - - let mut cache = self.cache.write().await; - cache.put(cache_key, cache_entry); - - let mut stats = self.stats.write().await; - stats.record_hit(); - stats.record_loaded(bars.len()); - - return Ok(feature_matrix); - } - Err(e) => { - tracing::debug!("MinIO cache miss: {}", e); - } - } - } - - // Cache miss - compute features - tracing::debug!("Feature cache miss, computing: {}", symbol); - let mut stats = self.stats.write().await; - stats.record_miss(); - drop(stats); - - let features = extract_ml_features(bars)?; - let feature_matrix = FeatureMatrix::new(features.clone(), symbol.to_string()); - - // Validate features - feature_matrix.validate().map_err(|e| MLError::ValidationError { - message: format!("Feature validation failed: {}", e), - })?; - - // Store in MinIO if available - if let Some(ref storage) = self.storage { - let object_key = self.build_object_key(symbol, &data_hash); - - if let Err(e) = self.save_to_minio(storage, &object_key, &features).await { - tracing::warn!("Failed to save features to MinIO: {}", e); - } else { - tracing::debug!("Saved features to MinIO: {}", object_key); - } - } - - // Update in-memory cache - let metadata = CacheMetadata { - symbol: symbol.to_string(), - bar_count: bars.len(), - feature_dim: feature_matrix.feature_dim, - created_at: Utc::now(), - data_hash: data_hash.clone(), - object_key: self.build_object_key(symbol, &data_hash), - stats: Some(FeatureStats::from_features(&features)), - }; - - let cache_entry = FeatureCache { - matrix: feature_matrix.clone(), - metadata, - last_access: Utc::now(), - }; - - { - let mut cache = self.cache.write().await; - cache.put(cache_key, cache_entry); - let mut stats = self.stats.write().await; - stats.record_computed(bars.len()); - stats.cache_size = cache.len(); - } - - Ok(feature_matrix) - } - - /// Invalidate cache for symbol (removes from memory and MinIO) - pub async fn invalidate(&self, symbol: &str) -> Result<(), MLError> { - // Remove from in-memory cache (all entries for this symbol) - { - let mut cache = self.cache.write().await; - let keys_to_remove: Vec = cache - .iter() - .filter(|(k, _)| k.starts_with(&format!("{}_", symbol))) - .map(|(k, _)| k.clone()) - .collect(); - - for key in keys_to_remove { - cache.pop(&key); - } - - let mut stats = self.stats.write().await; - stats.record_invalidation(); - stats.cache_size = cache.len(); - } - - // Remove from MinIO if available - if let Some(ref storage) = self.storage { - // List all objects for this symbol - let prefix = format!("features/{}/", symbol); - match storage.list(&prefix).await { - Ok(objects) => { - for obj_key in objects { - if let Err(e) = storage.delete(&obj_key).await { - tracing::warn!("Failed to delete {} from MinIO: {}", obj_key, e); - } - } - } - Err(e) => { - tracing::warn!("Failed to list objects for {}: {}", symbol, e); - } - } - } - - Ok(()) - } - - /// Get cache statistics - pub async fn get_stats(&self) -> CacheStats { - let mut stats = self.stats.read().await.clone(); - let cache = self.cache.read().await; - stats.cache_size = cache.len(); - stats - } - - /// Check if symbol is cached - pub async fn is_cached(&self, symbol: &str) -> bool { - let cache = self.cache.read().await; - cache - .iter() - .any(|(k, _)| k.starts_with(&format!("{}_", symbol))) - } - - /// List all cached symbols - pub async fn list_cached_symbols(&self) -> Vec { - let mut symbols = std::collections::HashSet::new(); - - // From memory cache - { - let cache = self.cache.read().await; - for (key, _) in cache.iter() { - if let Some(symbol) = key.split('_').next() { - symbols.insert(symbol.to_string()); - } - } - } - - // From MinIO if available - if let Some(ref storage) = self.storage { - if let Ok(objects) = storage.list("features/").await { - for obj_key in objects { - // Extract symbol from key like "features/SYMBOL/..." - let parts: Vec<&str> = obj_key.split('/').collect(); - if parts.len() >= 2 { - symbols.insert(parts[1].to_string()); - } - } - } - } - - symbols.into_iter().collect() - } - - /// Compute SHA-256 hash of OHLCV bars - fn compute_data_hash(&self, bars: &[OHLCVBar]) -> String { - let mut hasher = Sha256::new(); - - for bar in bars { - // Hash timestamp - hasher.update(bar.timestamp.to_rfc3339().as_bytes()); - - // Hash OHLCV values - hasher.update(&bar.open.to_le_bytes()); - hasher.update(&bar.high.to_le_bytes()); - hasher.update(&bar.low.to_le_bytes()); - hasher.update(&bar.close.to_le_bytes()); - hasher.update(&bar.volume.to_le_bytes()); - } - - format!("{:x}", hasher.finalize()) - } - - /// Build MinIO object key - fn build_object_key(&self, symbol: &str, data_hash: &str) -> String { - let date = Utc::now().format("%Y%m%d"); - format!("features/{}/{}_{}. parquet", symbol, date, &data_hash[..16]) - } - - /// Load features from MinIO - async fn load_from_minio( - &self, - storage: &Arc, - object_key: &str, - ) -> Result { - let bytes = storage - .download(object_key) - .await - .map_err(|e| MLError::ModelError(format!("MinIO download failed: {}", e)))?; - - let features = deserialize_features_from_bytes(&bytes)?; - - // Extract symbol from object key - let parts: Vec<&str> = object_key.split('/').collect(); - let symbol = parts.get(1).unwrap_or(&"unknown").to_string(); - - Ok(FeatureMatrix::new(features, symbol)) - } - - /// Save features to MinIO - async fn save_to_minio( - &self, - storage: &Arc, - object_key: &str, - features: &[Vec], - ) -> Result<(), MLError> { - let bytes = serialize_features_to_bytes(features)?; - - storage - .upload(object_key, bytes) - .await - .map_err(|e| MLError::ModelError(format!("MinIO upload failed: {}", e)))?; - - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use chrono::Duration; - - fn create_test_bars(count: usize) -> Vec { - let base_time = Utc::now(); - (0..count) - .map(|i| OHLCVBar { - timestamp: base_time + Duration::seconds(i as i64), - open: 100.0 + i as f64, - high: 105.0 + i as f64, - low: 95.0 + i as f64, - close: 102.0 + i as f64, - volume: 1000.0 + i as f64 * 10.0, - }) - .collect() - } - - #[tokio::test] - async fn test_cache_disabled() { - let service = FeatureCacheService::disabled(); - let bars = create_test_bars(50); - - let result = service.get_or_compute("TEST", &bars).await; - assert!(result.is_ok()); - - let stats = service.get_stats().await; - assert_eq!(stats.hits, 0); - assert_eq!(stats.misses, 0); - assert!(stats.features_computed > 0); - } - - #[tokio::test] - async fn test_in_memory_cache() { - let service = FeatureCacheService::new(None, Some(10), None); - let bars = create_test_bars(50); - - // First call - cache miss - let result1 = service.get_or_compute("TEST", &bars).await.unwrap(); - assert_eq!(result1.sample_count, 50); - - let stats = service.get_stats().await; - assert_eq!(stats.misses, 1); - assert_eq!(stats.hits, 0); - - // Second call - cache hit - let result2 = service.get_or_compute("TEST", &bars).await.unwrap(); - assert_eq!(result2.sample_count, 50); - - let stats = service.get_stats().await; - assert_eq!(stats.hits, 1); - assert_eq!(stats.misses, 1); - } - - #[tokio::test] - async fn test_cache_invalidation() { - let service = FeatureCacheService::new(None, Some(10), None); - let bars = create_test_bars(50); - - // Cache the features - service.get_or_compute("TEST", &bars).await.unwrap(); - assert!(service.is_cached("TEST").await); - - // Invalidate - service.invalidate("TEST").await.unwrap(); - assert!(!service.is_cached("TEST").await); - - let stats = service.get_stats().await; - assert_eq!(stats.invalidations, 1); - } - - #[tokio::test] - async fn test_data_hash() { - let service = FeatureCacheService::new(None, Some(10), None); - let bars1 = create_test_bars(50); - let bars2 = create_test_bars(50); - - let hash1 = service.compute_data_hash(&bars1); - let hash2 = service.compute_data_hash(&bars2); - - // Same data should produce same hash - assert_eq!(hash1, hash2); - - // Different data should produce different hash - let mut bars3 = bars1.clone(); - bars3[0].close = 999.0; - let hash3 = service.compute_data_hash(&bars3); - assert_ne!(hash1, hash3); - } - - #[tokio::test] - async fn test_list_cached_symbols() { - let service = FeatureCacheService::new(None, Some(10), None); - let bars = create_test_bars(50); - - service.get_or_compute("AAPL", &bars).await.unwrap(); - service.get_or_compute("MSFT", &bars).await.unwrap(); - - let symbols = service.list_cached_symbols().await; - assert!(symbols.contains(&"AAPL".to_owned())); - assert!(symbols.contains(&"MSFT".to_owned())); - } -} diff --git a/crates/ml/src/features/cache_storage.rs b/crates/ml/src/features/cache_storage.rs deleted file mode 100644 index c06bfd6db..000000000 --- a/crates/ml/src/features/cache_storage.rs +++ /dev/null @@ -1,594 +0,0 @@ -//! MinIO-backed feature cache storage -//! -//! Provides upload/download/list operations for pre-computed ML features. -//! Features are stored in Parquet format with SHA-256 hash-based cache keys. -//! -//! **Cache Key Format**: `{symbol}_{start_date}_{end_date}_{data_hash}.parquet` -//! -//! Example: `ZN.FUT_20250101_20250115_a1b2c3d4.parquet` - -use anyhow::{Context, Result}; -use arrow::array::{Float32Array, RecordBatch}; -use arrow::datatypes::{DataType, Field, Schema}; -use chrono::{DateTime, Utc}; -use parquet::arrow::arrow_writer::ArrowWriter; -use parquet::arrow::ParquetRecordBatchReader; -use parquet::file::properties::WriterProperties; -use serde::{Deserialize, Serialize}; -use sha2::{Digest, Sha256}; -use std::fs::File; -use std::path::Path; -use std::sync::Arc; -use storage::{ObjectStoreBackend, Storage}; -use tempfile::NamedTempFile; - -/// Feature cache storage using MinIO backend -pub struct FeatureCacheStorage { - /// MinIO storage backend - storage: Arc, - /// Bucket name for feature cache - bucket_prefix: String, -} - -impl FeatureCacheStorage { - /// Create new feature cache storage - /// - /// # Arguments - /// * `storage` - ObjectStoreBackend configured for MinIO - /// - /// # Returns - /// Feature cache storage instance - pub fn new(storage: Arc) -> Self { - Self { - storage, - bucket_prefix: "feature-cache".to_owned(), - } - } - - /// Upload features to MinIO - /// - /// # Arguments - /// * `features` - 2D feature matrix (N bars × M features) - /// * `metadata` - Metadata describing the features - /// - /// # Returns - /// Cache key used for storage - /// - /// # Errors - /// Returns error if serialization or upload fails - pub async fn upload_features( - &self, - features: &[Vec], - metadata: &FeatureMetadata, - ) -> Result { - // Generate cache key - let cache_key = self.generate_cache_key(metadata)?; - let parquet_path = format!("{}/{}", self.bucket_prefix, cache_key); - - // Serialize features to Parquet (in-memory via temp file) - let temp_file = self.serialize_to_parquet(features, metadata)?; - - // Read temp file as bytes - let parquet_bytes = std::fs::read(temp_file.path()) - .context("Failed to read temporary Parquet file")?; - - // Upload to MinIO - self.storage - .store(&parquet_path, &parquet_bytes) - .await - .context("Failed to upload features to MinIO")?; - - tracing::info!( - "Uploaded feature cache: {} ({} bars, {} features, {} bytes)", - cache_key, - features.len(), - features.first().map(|v| v.len()).unwrap_or(0), - parquet_bytes.len() - ); - - // Upload metadata as separate JSON file - let metadata_path = format!("{}/{}.meta.json", self.bucket_prefix, cache_key); - let metadata_json = serde_json::to_vec_pretty(metadata) - .context("Failed to serialize metadata")?; - - self.storage - .store(&metadata_path, &metadata_json) - .await - .context("Failed to upload metadata to MinIO")?; - - Ok(cache_key) - } - - /// Download features from MinIO - /// - /// # Arguments - /// * `cache_key` - Cache key returned from upload_features - /// - /// # Returns - /// 2D feature matrix (N bars × M features) - /// - /// # Errors - /// Returns error if download or deserialization fails - pub async fn download_features(&self, cache_key: &str) -> Result>> { - let parquet_path = format!("{}/{}", self.bucket_prefix, cache_key); - - // Download from MinIO - let parquet_bytes = self - .storage - .retrieve(&parquet_path) - .await - .context("Failed to download features from MinIO")?; - - tracing::info!( - "Downloaded feature cache: {} ({} bytes)", - cache_key, - parquet_bytes.len() - ); - - // Deserialize from Parquet - let features = self.deserialize_from_parquet(&parquet_bytes)?; - - Ok(features) - } - - /// Download metadata for cached features - /// - /// # Arguments - /// * `cache_key` - Cache key to retrieve metadata for - /// - /// # Returns - /// Feature metadata - /// - /// # Errors - /// Returns error if download or deserialization fails - pub async fn download_metadata(&self, cache_key: &str) -> Result { - let metadata_path = format!("{}/{}.meta.json", self.bucket_prefix, cache_key); - - let metadata_bytes = self - .storage - .retrieve(&metadata_path) - .await - .context("Failed to download metadata from MinIO")?; - - let metadata: FeatureMetadata = serde_json::from_slice(&metadata_bytes) - .context("Failed to deserialize metadata")?; - - Ok(metadata) - } - - /// List all cached features in MinIO - /// - /// # Returns - /// Vector of cache keys available in storage - /// - /// # Errors - /// Returns error if listing fails - pub async fn list_cached_features(&self) -> Result> { - let prefix = format!("{}/", self.bucket_prefix); - - let all_objects = self - .storage - .list(&prefix) - .await - .context("Failed to list cached features")?; - - // Filter for .parquet files only (exclude .meta.json) - let cache_keys: Vec = all_objects - .into_iter() - .filter(|path| path.ends_with(".parquet")) - .map(|path| { - // Extract cache key from full path - // e.g., "feature-cache/ZN.FUT_20250101_20250115_a1b2c3d4.parquet" - // -> "ZN.FUT_20250101_20250115_a1b2c3d4.parquet" - path.trim_start_matches(&prefix).to_string() - }) - .collect(); - - tracing::info!("Listed {} cached feature sets", cache_keys.len()); - - Ok(cache_keys) - } - - /// List cached features for a specific symbol - /// - /// # Arguments - /// * `symbol` - Trading symbol (e.g., "ZN.FUT") - /// - /// # Returns - /// Vector of cache keys for the specified symbol - /// - /// # Errors - /// Returns error if listing fails - pub async fn list_symbol_caches(&self, symbol: &str) -> Result> { - let all_caches = self.list_cached_features().await?; - - // Filter for symbol prefix - let symbol_caches: Vec = all_caches - .into_iter() - .filter(|key| key.starts_with(symbol)) - .collect(); - - Ok(symbol_caches) - } - - /// Check if features are cached - /// - /// # Arguments - /// * `cache_key` - Cache key to check - /// - /// # Returns - /// True if cache exists, false otherwise - pub async fn is_cached(&self, cache_key: &str) -> bool { - let parquet_path = format!("{}/{}", self.bucket_prefix, cache_key); - self.storage.exists(&parquet_path).await.unwrap_or(false) - } - - /// Delete cached features - /// - /// # Arguments - /// * `cache_key` - Cache key to delete - /// - /// # Returns - /// True if deletion succeeded - /// - /// # Errors - /// Returns error if deletion fails - pub async fn delete_cache(&self, cache_key: &str) -> Result { - let parquet_path = format!("{}/{}", self.bucket_prefix, cache_key); - let metadata_path = format!("{}/{}.meta.json", self.bucket_prefix, cache_key); - - // Delete both Parquet and metadata - let parquet_deleted = self - .storage - .delete(&parquet_path) - .await - .context("Failed to delete Parquet file")?; - - let _metadata_deleted = self.storage.delete(&metadata_path).await.unwrap_or(false); - - Ok(parquet_deleted) - } - - /// Generate cache key from metadata - /// - /// Format: `{symbol}_{start_date}_{end_date}_{data_hash}.parquet` - /// - /// # Arguments - /// * `metadata` - Feature metadata - /// - /// # Returns - /// Cache key string - fn generate_cache_key(&self, metadata: &FeatureMetadata) -> Result { - let start_date = metadata.start_date.format("%Y%m%d"); - let end_date = metadata.end_date.format("%Y%m%d"); - let hash_short = &metadata.data_hash[..8]; // First 8 chars of SHA-256 - - let cache_key = format!( - "{}_{}_{}_{}.parquet", - metadata.symbol, start_date, end_date, hash_short - ); - - Ok(cache_key) - } - - /// Serialize features to Parquet format - /// - /// # Arguments - /// * `features` - 2D feature matrix (N bars × M features) - /// * `metadata` - Feature metadata - /// - /// # Returns - /// Temporary file containing Parquet data - /// - /// # Errors - /// Returns error if serialization fails - fn serialize_to_parquet( - &self, - features: &[Vec], - metadata: &FeatureMetadata, - ) -> Result { - if features.is_empty() { - anyhow::bail!("Cannot serialize empty feature matrix"); - } - - let num_features = features[0].len(); - - // Build Arrow schema (feature_0, feature_1, ..., feature_N) - let mut fields = Vec::with_capacity(num_features); - for i in 0..num_features { - fields.push(Field::new( - format!("feature_{}", i), - DataType::Float32, - false, - )); - } - let schema = Arc::new(Schema::new(fields)); - - // Create temporary file for Parquet - let temp_file = NamedTempFile::new().context("Failed to create temporary file")?; - - // Create Parquet writer - let file = File::create(temp_file.path()) - .context("Failed to create file for Parquet writer")?; - - let props = WriterProperties::builder() - .set_compression(parquet::basic::Compression::SNAPPY) - .build(); - - let mut writer = ArrowWriter::try_new(file, schema.clone(), Some(props)) - .context("Failed to create Parquet writer")?; - - // Convert features to Arrow arrays (column-wise) - let mut arrays: Vec> = Vec::with_capacity(num_features); - - for feature_idx in 0..num_features { - let values: Vec = features - .iter() - .map(|bar_features| bar_features[feature_idx]) - .collect(); - - arrays.push(Arc::new(Float32Array::from(values))); - } - - // Create RecordBatch - let batch = RecordBatch::try_new(schema.clone(), arrays) - .context("Failed to create RecordBatch")?; - - // Write batch - writer - .write(&batch) - .context("Failed to write RecordBatch to Parquet")?; - - // Close writer - writer.close().context("Failed to close Parquet writer")?; - - tracing::debug!( - "Serialized {} bars × {} features to Parquet (symbol: {})", - features.len(), - num_features, - metadata.symbol - ); - - Ok(temp_file) - } - - /// Deserialize features from Parquet bytes - /// - /// # Arguments - /// * `parquet_bytes` - Raw Parquet file bytes - /// - /// # Returns - /// 2D feature matrix (N bars × M features) - /// - /// # Errors - /// Returns error if deserialization fails - fn deserialize_from_parquet(&self, parquet_bytes: &[u8]) -> Result>> { - // Write bytes to temporary file (Parquet reader needs a file) - let temp_file = NamedTempFile::new().context("Failed to create temporary file")?; - std::fs::write(temp_file.path(), parquet_bytes) - .context("Failed to write Parquet bytes to temp file")?; - - // Open Parquet file - let file = File::open(temp_file.path()).context("Failed to open Parquet file")?; - - let reader = - ParquetRecordBatchReader::try_new(file, 1024).context("Failed to create reader")?; - - let mut all_features: Vec> = Vec::new(); - - // Read all batches - for batch_result in reader { - let batch = batch_result.context("Failed to read RecordBatch")?; - - let num_rows = batch.num_rows(); - let num_cols = batch.num_columns(); - - // Initialize rows - for _ in 0..num_rows { - all_features.push(Vec::with_capacity(num_cols)); - } - - // Read each column (feature) - for col_idx in 0..num_cols { - let array = batch.column(col_idx); - let float_array = array - .as_any() - .downcast_ref::() - .context("Failed to downcast to Float32Array")?; - - // Populate rows with this column's values - for (row_idx, value) in float_array.iter().enumerate() { - let value = value.context("Null value in feature array")?; - all_features[row_idx].push(value); - } - } - } - - tracing::debug!( - "Deserialized {} bars × {} features from Parquet", - all_features.len(), - all_features.first().map(|v| v.len()).unwrap_or(0) - ); - - Ok(all_features) - } -} - -/// Cache key for identifying cached features -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct CacheKey { - /// Trading symbol - pub symbol: String, - /// Start date of data range - pub start_date: DateTime, - /// End date of data range - pub end_date: DateTime, - /// SHA-256 hash of raw OHLCV data (first 8 chars) - pub data_hash: String, -} - -impl CacheKey { - /// Create new cache key - pub fn new( - symbol: String, - start_date: DateTime, - end_date: DateTime, - data_hash: String, - ) -> Self { - Self { - symbol, - start_date, - end_date, - data_hash, - } - } - - /// Convert to string format - pub fn to_string(&self) -> String { - format!( - "{}_{}_{}_{}.parquet", - self.symbol, - self.start_date.format("%Y%m%d"), - self.end_date.format("%Y%m%d"), - &self.data_hash[..8] - ) - } -} - -/// Metadata for cached features -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct FeatureMetadata { - /// Trading symbol - pub symbol: String, - /// Number of bars - pub bar_count: usize, - /// Number of features per bar - pub feature_dim: usize, - /// Start date of data range - pub start_date: DateTime, - /// End date of data range - pub end_date: DateTime, - /// SHA-256 hash of raw input data - pub data_hash: String, - /// Timestamp when cache was created - pub created_at: DateTime, -} - -impl FeatureMetadata { - /// Create new feature metadata - pub fn new( - symbol: String, - bar_count: usize, - feature_dim: usize, - start_date: DateTime, - end_date: DateTime, - data_hash: String, - ) -> Self { - Self { - symbol, - bar_count, - feature_dim, - start_date, - end_date, - data_hash, - created_at: Utc::now(), - } - } - - /// Compute SHA-256 hash of OHLCV bar data - /// - /// # Arguments - /// * `bars` - OHLCV bars to hash - /// - /// # Returns - /// Hex-encoded SHA-256 hash - pub fn compute_data_hash(bars: &[T]) -> String { - let mut hasher = Sha256::new(); - - // Hash the debug representation (includes all fields) - for bar in bars { - hasher.update(format!("{:?}", bar).as_bytes()); - } - - format!("{:x}", hasher.finalize()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_cache_key_generation() { - let metadata = FeatureMetadata::new( - "ZN.FUT".to_owned(), - 1000, - 256, - DateTime::parse_from_rfc3339("2025-01-01T00:00:00Z") - .unwrap() - .with_timezone(&Utc), - DateTime::parse_from_rfc3339("2025-01-15T00:00:00Z") - .unwrap() - .with_timezone(&Utc), - "a1b2c3d4e5f6g7h8".to_owned(), - ); - - let cache_key = CacheKey::new( - metadata.symbol.clone(), - metadata.start_date, - metadata.end_date, - metadata.data_hash.clone(), - ); - - assert_eq!( - cache_key.to_string(), - "ZN.FUT_20250101_20250115_a1b2c3d4.parquet" - ); - } - - #[test] - fn test_metadata_serialization() { - let metadata = FeatureMetadata::new( - "ZN.FUT".to_owned(), - 1000, - 256, - Utc::now(), - Utc::now(), - "abc123".to_owned(), - ); - - let json = serde_json::to_string(&metadata).unwrap(); - let deserialized: FeatureMetadata = serde_json::from_str(&json).unwrap(); - - assert_eq!(metadata.symbol, deserialized.symbol); - assert_eq!(metadata.bar_count, deserialized.bar_count); - assert_eq!(metadata.feature_dim, deserialized.feature_dim); - } - - #[test] - fn test_data_hash_computation() { - #[derive(Debug)] - struct MockBar { - price: f64, - volume: f64, - } - - let bars = vec![ - MockBar { - price: 100.0, - volume: 1000.0, - }, - MockBar { - price: 101.0, - volume: 1100.0, - }, - ]; - - let hash1 = FeatureMetadata::compute_data_hash(&bars); - let hash2 = FeatureMetadata::compute_data_hash(&bars); - - // Same data should produce same hash - assert_eq!(hash1, hash2); - assert_eq!(hash1.len(), 64); // SHA-256 produces 64 hex characters - } -} diff --git a/crates/ml/src/features/extraction_wave_d_impl.rs b/crates/ml/src/features/extraction_wave_d_impl.rs deleted file mode 100644 index 4db00c6e3..000000000 --- a/crates/ml/src/features/extraction_wave_d_impl.rs +++ /dev/null @@ -1,74 +0,0 @@ -/// WAVE 8 AGENT 37: Extract Wave D regime detection features (24 total) -/// -/// ## Feature Breakdown (Indices 201-224) -/// - 201-210: CUSUM features (10) -/// - 211-215: ADX & directional indicators (5) -/// - 216-220: Transition probabilities (5) -/// - 221-224: Adaptive position/stop-loss (4) -fn extract_wave_d_features(&mut self, out: &mut [f64]) -> Result<()> { - let bar = self.bars.back().context("No current bar")?; - let mut idx = 0; - - // Features 201-210: CUSUM regime detection (10 features) - let return_value = if self.bars.len() > 1 { - let prev = &self.bars[self.bars.len() - 2]; - safe_log_return(bar.close, prev.close) - } else { - 0.0 - }; - let cusum_features = self.regime_cusum.update(return_value); - out[idx..idx + 10].copy_from_slice(&cusum_features); - idx += 10; - - // Features 211-215: ADX & directional indicators (5 features) - // Convert OHLCVBar to regime_adx OHLCVBar format - let adx_bar = crate::features::regime_adx::OHLCVBar { - timestamp: bar.timestamp.timestamp(), - open: bar.open, - high: bar.high, - low: bar.low, - close: bar.close, - volume: bar.volume, - }; - let adx_features = self.regime_adx.update(&adx_bar); - out[idx..idx + 5].copy_from_slice(&adx_features); - idx += 5; - - // Features 216-220: Transition probabilities (5 features) - // Determine current regime based on ADX and CUSUM - let adx_value = adx_features[0]; // ADX strength - let cusum_direction = cusum_features[3]; // Direction feature - let current_regime = if adx_value > 25.0 { - if cusum_direction > 0.5 { - MarketRegime::Bull - } else if cusum_direction < -0.5 { - MarketRegime::Bear - } else { - MarketRegime::Trending - } - } else if adx_value < 20.0 { - MarketRegime::Sideways - } else { - MarketRegime::Normal - }; - let transition_features = self.regime_transition.update(current_regime); - out[idx..idx + 5].copy_from_slice(&transition_features); - idx += 5; - - // Features 221-224: Adaptive position sizing & stop-loss (4 features) - // Convert bars to regime_adaptive format - let adaptive_bars: Vec = self.bars.iter().map(|b| { - crate::features::regime_adaptive::OHLCVBar { - timestamp: b.timestamp, - open: b.open, - high: b.high, - low: b.low, - close: b.close, - volume: b.volume, - } - }).collect(); - let adaptive_features = self.regime_adaptive.update(current_regime, return_value, 0.0, &adaptive_bars); - out[idx..idx + 4].copy_from_slice(&adaptive_features); - - Ok(()) -} diff --git a/crates/ml/src/features/minio_integration.rs b/crates/ml/src/features/minio_integration.rs deleted file mode 100644 index bdbc03b19..000000000 --- a/crates/ml/src/features/minio_integration.rs +++ /dev/null @@ -1,589 +0,0 @@ -//! MinIO Integration for Feature Caching -//! -//! This module provides upload/download functionality for pre-computed feature vectors -//! using MinIO (S3-compatible) object storage. Features are stored as compressed Parquet -//! files with metadata tags for efficient retrieval. -//! -//! ## Architecture -//! -//! ```text -//! Feature Extraction → Parquet Serialization → Compression → MinIO Upload -//! ↓ -//! ML Model Training ← Feature Deserialization ← Decompression ← MinIO Download -//! ``` -//! -//! ## Storage Structure -//! -//! ```text -//! feature-cache/ -//! ├── features/ -//! │ ├── ZN.FUT/ -//! │ │ ├── 20250115.parquet (feature vectors) -//! │ │ └── 20250115_metadata.json (cache metadata) -//! │ ├── 6E.FUT/ -//! │ │ └── ... -//! │ └── ES.FUT/ -//! │ └── ... -//! ``` -//! -//! ## Performance -//! -//! - Upload: ~10ms for 1000 bars (256 features × 1000 bars = 256KB compressed) -//! - Download: ~5ms for 1000 bars (10x faster than recomputation) -//! - Compression: Snappy (fast) or ZSTD (high ratio) -//! -//! ## Usage -//! -//! ```rust -//! use ml::features::minio_integration::{ -//! upload_features_to_minio, download_features_from_minio, list_cached_features -//! }; -//! use config::schemas::S3Config; -//! -//! // Upload features -//! let features = vec![vec![0.0; 256]; 1000]; // 1000 bars × 256 features -//! upload_features_to_minio(&features, "feature-cache", "ZN.FUT/20250115.parquet").await?; -//! -//! // Download features -//! let cached_features = download_features_from_minio("feature-cache", "ZN.FUT/20250115.parquet").await?; -//! -//! // List cached symbols -//! let symbols = list_cached_features("feature-cache").await?; -//! ``` - -use anyhow::{Context, Result}; -use chrono::{DateTime, Utc}; -use serde::{Deserialize, Serialize}; -use sha2::{Digest, Sha256}; -use std::collections::HashMap; -use storage::{ObjectStoreBackend, Storage}; -use tracing::{debug, info}; - -/// Feature cache metadata stored alongside Parquet files -/// -/// Tracks cache version, data hash for invalidation, and feature statistics. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct CacheMetadata { - /// Symbol name (e.g., "ZN.FUT") - pub symbol: String, - /// Number of bars in the feature cache - pub bar_count: usize, - /// Feature dimensionality (always 256 for this system) - pub feature_dim: usize, - /// Timestamp when cache was created - pub created_at: DateTime, - /// SHA-256 hash of input OHLCV data for invalidation - pub data_hash: String, - /// Feature extraction version (for migration) - pub extraction_version: String, -} - -impl CacheMetadata { - /// Create new cache metadata for a feature cache - pub fn new(symbol: String, bar_count: usize, data_hash: String) -> Self { - Self { - symbol, - bar_count, - feature_dim: 256, // Fixed for this system - created_at: Utc::now(), - data_hash, - extraction_version: "1.0.0".to_owned(), - } - } -} - -/// Upload feature matrix to MinIO as compressed Parquet file -/// -/// ## Arguments -/// -/// - `features`: Feature matrix (N bars × 256 features) -/// - `bucket`: MinIO bucket name (e.g., "feature-cache") -/// - `key`: Storage key (e.g., "features/ZN.FUT/20250115.parquet") -/// -/// ## Storage Format -/// -/// - Parquet schema: 256 float32 columns (feature_0, feature_1, ..., feature_255) -/// - Compression: Snappy (fast) by default -/// - Metadata: Stored as separate JSON object -/// -/// ## Performance -/// -/// - ~10ms for 1000 bars (256KB compressed) -/// - Automatic retry with exponential backoff -/// -/// ## Errors -/// -/// Returns error if: -/// - Feature dimensions are invalid (not 256-dim) -/// - MinIO connection fails -/// - Upload operation fails after retries -pub async fn upload_features_to_minio( - features: &[Vec], - bucket: &str, - key: &str, -) -> Result<()> { - info!( - "Uploading {} feature vectors to MinIO: {}/{}", - features.len(), - bucket, - key - ); - - // Validate feature dimensions - if let Some(first) = features.first() { - if first.len() != 256 { - anyhow::bail!( - "Invalid feature dimension: expected 256, got {}", - first.len() - ); - } - } - - // Serialize features to Parquet bytes (in-memory) - let parquet_bytes = serialize_features_to_parquet(features) - .context("Failed to serialize features to Parquet")?; - - // Create MinIO backend - let s3_config = config::schemas::S3Config::for_minio_testing(bucket); - let storage = ObjectStoreBackend::new(s3_config, None) - .await - .context("Failed to create MinIO backend")?; - - // Upload Parquet file - storage - .store(key, &parquet_bytes) - .await - .context("Failed to upload features to MinIO")?; - - info!( - "Successfully uploaded {} bytes to MinIO: {}/{}", - parquet_bytes.len(), - bucket, - key - ); - - Ok(()) -} - -/// Download feature matrix from MinIO -/// -/// ## Arguments -/// -/// - `bucket`: MinIO bucket name (e.g., "feature-cache") -/// - `key`: Storage key (e.g., "features/ZN.FUT/20250115.parquet") -/// -/// ## Returns -/// -/// Feature matrix (N bars × 256 features) as `Vec>` -/// -/// ## Performance -/// -/// - ~5ms for 1000 bars (10x faster than recomputation) -/// - Automatic decompression (Snappy/ZSTD) -/// -/// ## Errors -/// -/// Returns error if: -/// - Cache does not exist in MinIO -/// - Download operation fails -/// - Parquet deserialization fails -/// - Feature dimensions are invalid -pub async fn download_features_from_minio(bucket: &str, key: &str) -> Result>> { - debug!("Downloading features from MinIO: {}/{}", bucket, key); - - // Create MinIO backend - let s3_config = config::schemas::S3Config::for_minio_testing(bucket); - let storage = ObjectStoreBackend::new(s3_config, None) - .await - .context("Failed to create MinIO backend")?; - - // Download Parquet file - let parquet_bytes = storage - .retrieve(key) - .await - .context("Failed to download features from MinIO")?; - - info!( - "Downloaded {} bytes from MinIO: {}/{}", - parquet_bytes.len(), - bucket, - key - ); - - // Deserialize from Parquet - let features = deserialize_features_from_parquet(&parquet_bytes) - .context("Failed to deserialize features from Parquet")?; - - // Validate dimensions - if let Some(first) = features.first() { - if first.len() != 256 { - anyhow::bail!( - "Invalid cached feature dimension: expected 256, got {}", - first.len() - ); - } - } - - info!( - "Successfully loaded {} feature vectors from cache", - features.len() - ); - Ok(features) -} - -/// List all cached features in MinIO bucket -/// -/// ## Arguments -/// -/// - `bucket`: MinIO bucket name (e.g., "feature-cache") -/// -/// ## Returns -/// -/// Map of symbol → cache keys -/// Example: {"ZN.FUT" → ["20250115.parquet", "20250116.parquet"], "6E.FUT" → [...]} -/// -/// ## Usage -/// -/// ```rust -/// let cached = list_cached_features("feature-cache").await?; -/// if cached.contains_key("ZN.FUT") { -/// println!("ZN.FUT cache available: {:?}", cached["ZN.FUT"]); -/// } -/// ``` -/// -/// ## Performance -/// -/// - ~50ms for 1000 objects -/// - Results sorted by symbol -pub async fn list_cached_features(bucket: &str) -> Result>> { - debug!("Listing cached features in bucket: {}", bucket); - - // Create MinIO backend - let s3_config = config::schemas::S3Config::for_minio_testing(bucket); - let storage = ObjectStoreBackend::new(s3_config, None) - .await - .context("Failed to create MinIO backend")?; - - // List all objects with prefix "features/" - let objects = storage - .list("features/") - .await - .context("Failed to list cached features")?; - - // Parse symbol names from keys - // Example: "features/ZN.FUT/20250115.parquet" → "ZN.FUT" - let mut symbol_cache_map: HashMap> = HashMap::new(); - - for object_key in objects { - if object_key.ends_with(".parquet") { - // Extract symbol from key: "features/ZN.FUT/20250115.parquet" - let parts: Vec<&str> = object_key.split('/').collect(); - if parts.len() >= 3 && parts[0] == "features" { - let symbol = parts[1].to_string(); - let filename = parts[2].to_string(); - - symbol_cache_map - .entry(symbol) - .or_insert_with(Vec::new) - .push(filename); - } - } - } - - info!( - "Found {} cached symbols in bucket: {}", - symbol_cache_map.len(), - bucket - ); - Ok(symbol_cache_map) -} - -/// Upload cache metadata to MinIO -/// -/// Stores metadata as JSON alongside the Parquet feature file. -/// Metadata key: `_metadata.json` -/// -/// Example: `features/ZN.FUT/20250115.parquet` → `features/ZN.FUT/20250115_metadata.json` -pub async fn upload_cache_metadata( - bucket: &str, - parquet_key: &str, - metadata: &CacheMetadata, -) -> Result<()> { - let metadata_key = format!("{}_metadata.json", parquet_key.trim_end_matches(".parquet")); - - // Serialize metadata to JSON - let metadata_json = - serde_json::to_vec_pretty(metadata).context("Failed to serialize cache metadata")?; - - // Create MinIO backend - let s3_config = config::schemas::S3Config::for_minio_testing(bucket); - let storage = ObjectStoreBackend::new(s3_config, None) - .await - .context("Failed to create MinIO backend")?; - - // Upload metadata - storage - .store(&metadata_key, &metadata_json) - .await - .context("Failed to upload cache metadata")?; - - debug!("Uploaded cache metadata: {}/{}", bucket, metadata_key); - Ok(()) -} - -/// Download cache metadata from MinIO -/// -/// Retrieves metadata JSON from MinIO for cache validation. -pub async fn download_cache_metadata(bucket: &str, parquet_key: &str) -> Result { - let metadata_key = format!("{}_metadata.json", parquet_key.trim_end_matches(".parquet")); - - // Create MinIO backend - let s3_config = config::schemas::S3Config::for_minio_testing(bucket); - let storage = ObjectStoreBackend::new(s3_config, None) - .await - .context("Failed to create MinIO backend")?; - - // Download metadata - let metadata_json = storage - .retrieve(&metadata_key) - .await - .context("Failed to download cache metadata")?; - - // Deserialize from JSON - let metadata: CacheMetadata = - serde_json::from_slice(&metadata_json).context("Failed to deserialize cache metadata")?; - - debug!("Downloaded cache metadata: {}/{}", bucket, metadata_key); - Ok(metadata) -} - -/// Check if feature cache exists in MinIO -/// -/// ## Arguments -/// -/// - `bucket`: MinIO bucket name -/// - `key`: Storage key for Parquet file -/// -/// ## Returns -/// -/// `true` if cache exists, `false` otherwise -pub async fn cache_exists(bucket: &str, key: &str) -> Result { - let s3_config = config::schemas::S3Config::for_minio_testing(bucket); - let storage = ObjectStoreBackend::new(s3_config, None) - .await - .context("Failed to create MinIO backend")?; - - storage - .exists(key) - .await - .context("Failed to check cache existence") -} - -// ============================================================================ -// Parquet Serialization/Deserialization (In-Memory) -// ============================================================================ - -/// Serialize feature matrix to Parquet bytes (in-memory) -/// -/// ## Format -/// -/// - Schema: 256 float32 columns (feature_0, feature_1, ..., feature_255) -/// - Compression: Snappy (fast, ~3x compression ratio) -/// - Row groups: 1024 rows per group (optimized for 1000-10000 bar datasets) -fn serialize_features_to_parquet(features: &[Vec]) -> Result> { - use arrow::array::{ArrayRef, Float32Array}; - use arrow::datatypes::{DataType, Field, Schema}; - use arrow::record_batch::RecordBatch; - use parquet::arrow::arrow_writer::ArrowWriter; - use parquet::basic::Compression; - use parquet::file::properties::WriterProperties; - use std::sync::Arc; - - if features.is_empty() { - anyhow::bail!("Cannot serialize empty feature matrix"); - } - - // Validate all rows have 256 features - for (i, row) in features.iter().enumerate() { - if row.len() != 256 { - anyhow::bail!( - "Feature row {} has invalid dimension: expected 256, got {}", - i, - row.len() - ); - } - } - - // Build Arrow schema: 256 float32 columns - let mut fields = Vec::with_capacity(256); - for i in 0..256 { - fields.push(Field::new( - format!("feature_{}", i), - DataType::Float32, - false, // Not nullable - )); - } - let schema = Arc::new(Schema::new(fields)); - - // Transpose feature matrix: (N rows × 256 columns) → 256 columns of N elements - let mut columns: Vec = Vec::with_capacity(256); - for col_idx in 0..256 { - let column_data: Vec = features.iter().map(|row| row[col_idx]).collect(); - columns.push(Arc::new(Float32Array::from(column_data))); - } - - // Create Arrow RecordBatch - let batch = RecordBatch::try_new(schema.clone(), columns) - .context("Failed to create Arrow RecordBatch")?; - - // Create Parquet writer with Snappy compression - let mut buffer = Vec::new(); - let props = WriterProperties::builder() - .set_compression(Compression::SNAPPY) - .build(); - - let mut writer = ArrowWriter::try_new(&mut buffer, schema, Some(props)) - .context("Failed to create Parquet writer")?; - - writer - .write(&batch) - .context("Failed to write RecordBatch to Parquet")?; - writer.close().context("Failed to close Parquet writer")?; - - debug!( - "Serialized {} feature vectors to {} bytes (Parquet + Snappy)", - features.len(), - buffer.len() - ); - Ok(buffer) -} - -/// Deserialize feature matrix from Parquet bytes (in-memory) -/// -/// ## Returns -/// -/// Feature matrix as `Vec>` (N rows × 256 columns) -fn deserialize_features_from_parquet(parquet_bytes: &[u8]) -> Result>> { - use arrow::array::Float32Array; - use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; - - // Create Parquet reader from bytes - let reader = - ParquetRecordBatchReaderBuilder::try_new(bytes::Bytes::from(parquet_bytes.to_vec())) - .context("Failed to create Parquet reader")? - .build() - .context("Failed to build Parquet reader")?; - - let mut all_features = Vec::new(); - - // Read record batches - for batch_result in reader { - let batch = batch_result.context("Failed to read Parquet record batch")?; - let num_rows = batch.num_rows(); - let num_cols = batch.num_columns(); - - if num_cols != 256 { - anyhow::bail!( - "Invalid Parquet schema: expected 256 columns, got {}", - num_cols - ); - } - - // Extract columns and transpose back to row-major format - let mut columns_data: Vec> = Vec::with_capacity(256); - for col_idx in 0..256 { - let array = batch - .column(col_idx) - .as_any() - .downcast_ref::() - .context("Failed to downcast column to Float32Array")?; - - let column_vec: Vec = (0..array.len()).map(|i| array.value(i)).collect(); - columns_data.push(column_vec); - } - - // Transpose: 256 columns of N elements → N rows of 256 elements - for row_idx in 0..num_rows { - let row: Vec = columns_data.iter().map(|col| col[row_idx]).collect(); - all_features.push(row); - } - } - - debug!( - "Deserialized {} feature vectors from Parquet", - all_features.len() - ); - Ok(all_features) -} - -/// Compute SHA-256 hash of OHLCV data for cache invalidation -/// -/// ## Usage -/// -/// ```rust -/// let data_hash = compute_data_hash(&bars); -/// let metadata = CacheMetadata::new("ZN.FUT".to_owned(), bars.len(), data_hash); -/// ``` -pub fn compute_data_hash(bars: &[crate::features::extraction::OHLCVBar]) -> String { - let mut hasher = Sha256::new(); - - for bar in bars { - // Hash all OHLCV fields + timestamp - hasher.update(bar.timestamp.to_rfc3339().as_bytes()); - hasher.update(bar.open.to_le_bytes()); - hasher.update(bar.high.to_le_bytes()); - hasher.update(bar.low.to_le_bytes()); - hasher.update(bar.close.to_le_bytes()); - hasher.update(bar.volume.to_le_bytes()); - } - - format!("{:x}", hasher.finalize()) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_parquet_serialization_roundtrip() { - // Create mock feature matrix (100 rows × 256 columns) - let features: Vec> = (0..100) - .map(|i| { - let mut row = vec![0.0; 256]; - row[0] = i as f32; // First feature = row index - row - }) - .collect(); - - // Serialize to Parquet - let parquet_bytes = serialize_features_to_parquet(&features).unwrap(); - assert!(parquet_bytes.len() > 0); - println!("Parquet size: {} bytes", parquet_bytes.len()); - - // Deserialize from Parquet - let deserialized = deserialize_features_from_parquet(&parquet_bytes).unwrap(); - - // Validate roundtrip - assert_eq!(deserialized.len(), 100); - assert_eq!(deserialized[0].len(), 256); - for i in 0..100 { - assert_eq!(deserialized[i][0], i as f32); - } - } - - #[test] - fn test_cache_metadata_serialization() { - let metadata = CacheMetadata::new("ZN.FUT".to_owned(), 1000, "abc123".to_owned()); - - // Serialize to JSON - let json = serde_json::to_string(&metadata).unwrap(); - assert!(json.contains("ZN.FUT")); - assert!(json.contains("abc123")); - - // Deserialize from JSON - let deserialized: CacheMetadata = serde_json::from_str(&json).unwrap(); - assert_eq!(deserialized.symbol, "ZN.FUT"); - assert_eq!(deserialized.bar_count, 1000); - assert_eq!(deserialized.feature_dim, 256); - } -} diff --git a/crates/ml/src/features/mod.rs b/crates/ml/src/features/mod.rs index 2cc6a741e..436692921 100644 --- a/crates/ml/src/features/mod.rs +++ b/crates/ml/src/features/mod.rs @@ -1,138 +1,36 @@ -//! Feature Engineering Module +//! Feature Engineering Module — thin facade re-exporting from ml-features crate. //! -//! This module provides comprehensive feature extraction for ML models: -//! - Progressive feature engineering (Wave A: 26, Wave B: 36, Wave C: 65+) -//! - Technical indicators (RSI, MACD, Bollinger, ATR, EMA) -//! - Price patterns, volume analysis, microstructure proxies -//! - Time-based and statistical features -//! - MinIO integration for feature caching (10x faster loading) +//! Core feature extractors live in `ml-features`. This module keeps bridge files +//! that depend on types from multiple ml sub-crates (ensemble, regime, labeling, etc.). -// New feature system -pub mod adx_features; // Wave D: ADX directional indicators (5 features, indices 211-215) -pub mod alternative_bars; -pub mod bar_resampler; // Multi-timeframe: 1m → 5m/15m/1h OHLCV bar aggregation -pub mod barrier_optimization; -pub mod config; // Wave C: Feature configuration for progressive engineering -pub mod ewma; -pub mod extraction; -pub mod feature_extraction; // ATR and other technical indicator calculations -pub mod mbp10_loader; // MBP-10 data loader for OFI feature extraction -pub mod microstructure; -pub mod microstructure_features; // Wave C: Additional microstructure features (9 features) -pub mod minio_integration; -pub mod multi_timeframe; // Multi-timeframe LSTM encoder + fusion (1m/5m/15m/1h → 128-dim macro state) -pub mod ofi_calculator; // Order Flow Imbalance features (8 features, indices 226-233) -pub mod trades_loader; // DBN trades loader for OFI feed_trade() (VPIN, Kyle's Lambda) -pub mod normalization; // Wave C: Feature normalization pipeline (5 strategies) -pub mod position_features; // RL agent position-aware features (3 features: pnl, bars, cost basis) -pub mod pipeline; // Wave C: 5-stage feature assembly pipeline (orchestrates all extractors) -pub mod price_features; // Wave C: Price-based features (15 features) -pub mod production_adapter; // WAVE 10: Adapter for common::ml_strategy 225-feature injection -pub mod regime_adaptive; // Wave D: Regime-adaptive position sizing & stop-loss (4 features, indices 221-224) -pub mod regime_adx; // Wave D: ADX & directional indicators (5 features, indices 211-215) -pub mod regime_cusum; // Wave D: CUSUM regime detection features (10 features, indices 201-210) -pub mod regime_transition; // Wave D: Regime transition probabilities (5 features, indices 216-220) -pub mod sample_weights; -pub mod statistical_features; // Wave C: Statistical aggregate features (7 features) -pub mod time_features; // Wave C: Time-based features (8 features) -pub mod unified; -pub mod volume_features; // Wave C: Volume-based features (10 features) +// Re-export everything from ml-features +pub use ml_features::*; -// Feature configuration (Wave C) -pub use config::{FeatureConfig, FeatureGroup, FeatureIndices, FeaturePhase}; +// Bridge modules that stay in ml (cross-module dependencies) +pub mod extraction; // Depends on regime_adaptive/cusum/transition + microstructure + ofi_calculator +pub mod multi_timeframe; // Depends on dqn::mixed_precision::training_dtype +pub mod regime_adaptive; // Depends on ensemble::MarketRegime +pub mod regime_cusum; // Depends on regime::cusum +pub mod regime_transition; // Depends on ensemble::MarketRegime + regime::transition_matrix +pub mod sample_weights; // Depends on labeling::meta_labeling +pub mod unified; // Depends on safety module +pub mod production_adapter; // Depends on extraction::FeatureExtractor +// Re-export bridge module types pub use extraction::{extract_ml_features, FeatureVector}; -pub use crate::types::OHLCVBar; -pub use minio_integration::{ - cache_exists, compute_data_hash, download_cache_metadata, download_features_from_minio, - list_cached_features, upload_cache_metadata, upload_features_to_minio, CacheMetadata, -}; - -// WAVE 10: Production adapter for common::ml_strategy dependency injection -pub use production_adapter::ProductionFeatureExtractorAdapter; - -// Unified feature extraction (production system) +pub use regime_adaptive::RegimeAdaptiveFeatures; +pub use regime_cusum::RegimeCUSUMFeatures; +pub use regime_transition::RegimeTransitionFeatures; +pub use sample_weights::{SampleWeightCalculator, WeightingScheme}; pub use unified::{ FeatureExtractionConfig, FeatureQualityMetrics, OrderBookLevel, UnifiedFeatureExtractor, UnifiedFinancialFeatures, }; - -// Alternative bar sampling (tick, volume, dollar, imbalance, run bars) -pub use alternative_bars::{ - DollarBarSampler, ImbalanceBarSampler, RunBarSampler, TickBarSampler, - VolumeBarSampler, -}; - -// Barrier optimization for triple barrier labeling -pub use barrier_optimization::{BarrierOptimizer, BarrierParams, OptimizationResult}; - -// EWMA for adaptive thresholds -pub use ewma::{AdaptiveThreshold, EWMACalculator}; - -// Sample weights for label imbalance and temporal decay -pub use sample_weights::{SampleWeightCalculator, WeightingScheme}; - -// Price features (Wave C) -pub use price_features::PriceFeatureExtractor; - -// Volume features (Wave C) -pub use volume_features::VolumeFeatureExtractor; - -// ADX features (Wave D) -pub use adx_features::AdxFeatureExtractor; - -// Regime ADX features (Wave D) -pub use regime_adx::RegimeADXFeatures; - -// Microstructure features (Wave C) -pub use microstructure_features::{ - BuySellImbalance, HighLowSpread, InterArrivalTime, KyleLambda, MicrostructureFeature, - PriceImpact, TickCount, VarianceRatio, VolumeWeightedSpread, -}; - -// Time features (Wave C) -pub use time_features::TimeFeatureExtractor; - -// Statistical features (Wave C) -pub use statistical_features::StatisticalFeatureExtractor; - -// Normalization pipeline (Wave C) -pub use normalization::{FeatureNormalizer, NormalizationStats, RingBuffer}; - -// Feature assembly pipeline (Wave C) -pub use pipeline::{FeatureExtractionPipeline, PipelinePerformance}; - -// Regime detection features (Wave D) -pub use regime_adaptive::RegimeAdaptiveFeatures; -pub use regime_cusum::RegimeCUSUMFeatures; -pub use regime_transition::RegimeTransitionFeatures; - -// Position-aware features for RL agents (3 features) -pub use position_features::PositionFeatures; - -// OFI features (Order Flow Imbalance) -pub use ofi_calculator::{OFICalculator, OFIFeatures}; - -// MBP-10 data loader for OFI integration -pub use mbp10_loader::{ - compute_ofi_from_file, compute_ofi_with_trades, get_recent_snapshots, - get_snapshots_for_timestamp, load_mbp10_snapshots_sync, load_ofi_features_parallel, -}; - -// DBN trades loader for OFI feed_trade() -pub use trades_loader::{get_trades_for_bar, load_trades_sync, DbnTrade}; - -// Bar resampler (1m → 5m/15m/1h) -pub use bar_resampler::BarResampler; - -// Multi-timeframe encoder (LSTM fusion → 128-dim macro state) +pub use production_adapter::ProductionFeatureExtractorAdapter; pub use multi_timeframe::{ bar_to_features, LstmEncoder, MultiTimeframeConfig, MultiTimeframeEncoder, }; -// Legacy features_old module removed in Wave D Phase 6 cleanup (3,513 lines) -// Add mock features helper to features module - // Test helper function #[cfg(test)] pub fn create_mock_features() -> FeatureVector { diff --git a/crates/ml/src/features/parquet_io.rs b/crates/ml/src/features/parquet_io.rs deleted file mode 100644 index 48f3b921a..000000000 --- a/crates/ml/src/features/parquet_io.rs +++ /dev/null @@ -1,244 +0,0 @@ -//! Parquet serialization for feature vectors -//! -//! This module provides efficient serialization/deserialization of feature matrices -//! using Apache Parquet format with Snappy compression. - -use crate::MLError; -use std::fs::File; -use std::io::{BufReader, BufWriter, Read, Write}; -use std::path::PathBuf; - -/// Write feature matrix to Parquet file -/// -/// # Arguments -/// * `features` - Feature matrix (rows = samples, cols = features) -/// * `path` - Output file path -/// -/// # Returns -/// * `Ok(())` on success -/// * `Err(MLError)` on serialization failure -/// -/// # Note -/// Currently uses bincode for serialization. Will be upgraded to proper Parquet -/// format with Arrow integration in Phase 2. -pub fn write_features_to_parquet(features: &[Vec], path: &PathBuf) -> Result<(), MLError> { - if features.is_empty() { - return Err(MLError::ValidationError { - message: "Cannot write empty feature matrix".to_owned(), - }); - } - - // Validate all rows have same dimension - let feature_dim = features[0].len(); - for (i, row) in features.iter().enumerate() { - if row.len() != feature_dim { - return Err(MLError::ValidationError { - message: format!( - "Row {} has {} features, expected {}", - i, - row.len(), - feature_dim - ), - }); - } - } - - // Create parent directory if it doesn't exist - if let Some(parent) = path.parent() { - std::fs::create_dir_all(parent).map_err(|e| MLError::ModelError(format!( - "Failed to create parent directory: {}", - e - )))?; - } - - // Serialize using bincode (fast and efficient) - let file = File::create(path).map_err(|e| { - MLError::ModelError(format!("Failed to create file {}: {}", path.display(), e)) - })?; - - let writer = BufWriter::new(file); - - bincode::serialize_into(writer, features).map_err(|e| MLError::SerializationError { - reason: format!("Failed to serialize features: {}", e), - })?; - - Ok(()) -} - -/// Read feature matrix from Parquet file -/// -/// # Arguments -/// * `path` - Input file path -/// -/// # Returns -/// * `Ok(Vec>)` - Feature matrix on success -/// * `Err(MLError)` - On deserialization failure -/// -/// # Note -/// Currently uses bincode for deserialization. Will be upgraded to proper Parquet -/// format with Arrow integration in Phase 2. -pub fn read_features_from_parquet(path: &PathBuf) -> Result>, MLError> { - if !path.exists() { - return Err(MLError::ModelError(format!( - "Feature file not found: {}", - path.display() - ))); - } - - let file = File::open(path).map_err(|e| { - MLError::ModelError(format!("Failed to open file {}: {}", path.display(), e)) - })?; - - let reader = BufReader::new(file); - - let features: Vec> = bincode::deserialize_from(reader).map_err(|e| { - MLError::SerializationError { - reason: format!("Failed to deserialize features: {}", e), - } - })?; - - // Validate features - if features.is_empty() { - return Err(MLError::ValidationError { - message: "Deserialized empty feature matrix".to_owned(), - }); - } - - let feature_dim = features[0].len(); - for (i, row) in features.iter().enumerate() { - if row.len() != feature_dim { - return Err(MLError::ValidationError { - message: format!( - "Row {} has {} features, expected {}", - i, - row.len(), - feature_dim - ), - }); - } - - // Check for NaN/Inf - for (j, &value) in row.iter().enumerate() { - if !value.is_finite() { - return Err(MLError::ValidationError { - message: format!( - "Row {} col {} contains non-finite value: {}", - i, j, value - ), - }); - } - } - } - - Ok(features) -} - -/// Serialize features to bytes (for MinIO upload) -pub fn serialize_features_to_bytes(features: &[Vec]) -> Result, MLError> { - bincode::serialize(features).map_err(|e| MLError::SerializationError { - reason: format!("Failed to serialize features to bytes: {}", e), - }) -} - -/// Deserialize features from bytes (for MinIO download) -pub fn deserialize_features_from_bytes(bytes: &[u8]) -> Result>, MLError> { - bincode::deserialize(bytes).map_err(|e| MLError::SerializationError { - reason: format!("Failed to deserialize features from bytes: {}", e), - }) -} - -/// Write bytes to file with compression -pub fn write_compressed(data: &[u8], path: &PathBuf) -> Result<(), MLError> { - use flate2::write::GzEncoder; - use flate2::Compression; - - let file = File::create(path).map_err(|e| { - MLError::ModelError(format!("Failed to create file {}: {}", path.display(), e)) - })?; - - let mut encoder = GzEncoder::new(file, Compression::default()); - encoder.write_all(data).map_err(|e| { - MLError::ModelError(format!("Failed to write compressed data: {}", e)) - })?; - - encoder.finish().map_err(|e| { - MLError::ModelError(format!("Failed to finish compression: {}", e)) - })?; - - Ok(()) -} - -/// Read compressed file -pub fn read_compressed(path: &PathBuf) -> Result, MLError> { - use flate2::read::GzDecoder; - - let file = File::open(path).map_err(|e| { - MLError::ModelError(format!("Failed to open file {}: {}", path.display(), e)) - })?; - - let mut decoder = GzDecoder::new(file); - let mut data = Vec::new(); - decoder.read_to_end(&mut data).map_err(|e| { - MLError::ModelError(format!("Failed to read compressed data: {}", e)) - })?; - - Ok(data) -} - -#[cfg(test)] -mod tests { - use super::*; - use tempfile::tempdir; - - #[test] - fn test_write_read_roundtrip() { - let dir = tempdir().unwrap(); - let path = dir.path().join("features.parquet"); - - let features = vec![ - vec![1.0, 2.0, 3.0], - vec![4.0, 5.0, 6.0], - vec![7.0, 8.0, 9.0], - ]; - - // Write - write_features_to_parquet(&features, &path).unwrap(); - - // Read - let loaded = read_features_from_parquet(&path).unwrap(); - - // Verify - assert_eq!(loaded.len(), features.len()); - for (i, row) in loaded.iter().enumerate() { - assert_eq!(row.len(), features[i].len()); - for (j, &value) in row.iter().enumerate() { - assert!((value - features[i][j]).abs() < 1e-6); - } - } - } - - #[test] - fn test_bytes_roundtrip() { - let features = vec![ - vec![1.0, 2.0, 3.0], - vec![4.0, 5.0, 6.0], - ]; - - let bytes = serialize_features_to_bytes(&features).unwrap(); - let loaded = deserialize_features_from_bytes(&bytes).unwrap(); - - assert_eq!(loaded, features); - } - - #[test] - fn test_compression() { - let dir = tempdir().unwrap(); - let path = dir.path().join("compressed.gz"); - - let data = b"Hello, World!"; - write_compressed(data, &path).unwrap(); - - let loaded = read_compressed(&path).unwrap(); - assert_eq!(loaded, data); - } -} diff --git a/crates/ml/src/hyperopt/adapters/dqn.rs b/crates/ml/src/hyperopt/adapters/dqn.rs index 7ea96ba9d..5dfd3b8cd 100644 --- a/crates/ml/src/hyperopt/adapters/dqn.rs +++ b/crates/ml/src/hyperopt/adapters/dqn.rs @@ -60,23 +60,8 @@ use crate::features::extraction::FeatureVector; use crate::trainers::dqn::{DQNHyperparameters, DQNTrainer as InternalDQNTrainer}; use crate::MLError; -/// Hyperopt objective mode for two-phase optimization. -/// -/// In two-phase optimization, Phase A uses `EpisodeReward` for fast convergence, -/// then Phase B switches to `Sharpe` for financial quality refinement. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum ObjectiveMode { - /// Phase A: Optimize episode reward (fast convergence signal) - EpisodeReward, - /// Phase B: Optimize Sharpe ratio (financial quality) - Sharpe, -} - -impl Default for ObjectiveMode { - fn default() -> Self { - ObjectiveMode::Sharpe - } -} +// Re-export ObjectiveMode from ml-hyperopt (canonical definition) +pub use crate::hyperopt::ObjectiveMode; /// Backtest metrics from EvaluationEngine /// diff --git a/crates/ml/src/hyperopt/mod.rs b/crates/ml/src/hyperopt/mod.rs index e36eef1af..dddf07272 100644 --- a/crates/ml/src/hyperopt/mod.rs +++ b/crates/ml/src/hyperopt/mod.rs @@ -1,19 +1,7 @@ //! Hyperparameter Optimization Module //! -//! Production-ready hyperparameter optimization using argmin (Nelder-Mead). -//! -//! This module provides: -//! - **Argmin Optimization**: Derivative-free optimization using Nelder-Mead simplex -//! - **Latin Hypercube Sampling**: Smart initialization for exploration -//! - **Multi-restart**: Escape local minima with strategic restarts -//! - **Model Adapters**: MAMBA-2, DQN, PPO, TFT support -//! -//! ## Features -//! -//! - **Fast Convergence**: Finds optimal hyperparameters in 20-50 trials -//! - **Log-Scale Support**: Proper handling of learning rates and weight decay -//! - **Production Ready**: Integrates with existing training pipelines -//! - **GPU Accelerated**: Leverages CUDA for fast evaluations +//! Core optimization infrastructure is provided by the `ml-hyperopt` crate. +//! Model-specific adapters stay here in `adapters/`. //! //! ## Example //! @@ -38,24 +26,13 @@ //! # } //! ``` +// Re-export everything from ml-hyperopt crate +pub use ml_hyperopt::*; + +// Modules that stay in ml (depend on ml-internal types) pub mod adapters; pub mod campaign; -pub mod early_stopping; -pub mod observer; -pub mod optimizer; -pub mod paths; -pub mod sensitivity; pub mod shared_data; -pub mod tpe; -pub mod traits; #[cfg(test)] mod tests_argmin; - -// Re-exports for convenience -pub use observer::TrialBudgetObserver; -pub use optimizer::{optimize_with_tpe, ArgminOptimizer, ArgminOptimizerBuilder, TwoPhaseObjective}; -pub use traits::{ - HardwareBudget, HyperoptStrategy, HyperparameterOptimizable, OptimizationResult, - ParameterSpace, TrialResult, -}; diff --git a/crates/ml/src/labeling/constants.rs b/crates/ml/src/labeling/constants.rs deleted file mode 100644 index 7d8a1148b..000000000 --- a/crates/ml/src/labeling/constants.rs +++ /dev/null @@ -1,34 +0,0 @@ -//! Constants for ML labeling operations - -/// Default profit target in basis points (1%) -pub const DEFAULT_PROFIT_TARGET_BPS: u32 = 100; - -/// Default stop loss in basis points (0.5%) -pub const DEFAULT_STOP_LOSS_BPS: u32 = 50; - -/// Default maximum holding period (1 hour in nanoseconds) -pub const DEFAULT_MAX_HOLDING_PERIOD_NS: u64 = 3600_000_000_000; - -/// Minimum return threshold in basis points -pub const MIN_RETURN_THRESHOLD_BPS: i32 = 5; - -/// Maximum batch size for `GPU` processing -pub const MAX_GPU_BATCH_SIZE: usize = 8192; - -/// Maximum batch size for CPU processing -pub const MAX_CPU_BATCH_SIZE: usize = 1024; - -/// Nanoseconds per microsecond -pub const NANOS_PER_MICRO: u64 = 1_000; - -/// Nanoseconds per millisecond -pub const NANOS_PER_MILLI: u64 = 1_000_000; - -/// Nanoseconds per second -pub const NANOS_PER_SECOND: u64 = 1_000_000_000; - -/// Basis points per unit (10,000) -pub const BASIS_POINTS_PER_UNIT: u32 = 10_000; - -/// Cents per dollar -pub const CENTS_PER_DOLLAR: u32 = 100; \ No newline at end of file diff --git a/crates/ml/src/labeling/mod.rs b/crates/ml/src/labeling/mod.rs index acb203326..3d844963a 100644 --- a/crates/ml/src/labeling/mod.rs +++ b/crates/ml/src/labeling/mod.rs @@ -1,146 +1,2 @@ -//! # ML Labeling Module for Foxhunt HFT System -//! -//! This module provides high-performance machine learning labeling algorithms -//! optimized for ultra-low latency financial applications. All implementations -//! use FixedPoint arithmetic for financial precision and target sub-microsecond -//! performance. -//! -//! ## Core Features -//! -//! - **Triple Barrier Engine**: <80μs latency for event labeling -//! - **Meta-Labeling**: Separates direction prediction from confidence/bet sizing -//! - **Fractional Differentiation**: Streaming transforms with <1μs latency -//! - **Sample Weighting**: Volatility/return/time-based weighting algorithms -//! - **GPU Acceleration**: Batch processing with CUDA via candle integration -//! - **Concurrent Processing**: Lock-free barrier tracking with DashMap -//! -//! ## Performance Targets -//! -//! - Triple barrier labeling: <80μs per event -//! - Meta-labeling: <50μs per prediction -//! - Fractional differentiation: <1μs per transform -//! - Sample weighting: <10μs per sample -//! - Batch processing: 10K+ labels/second -//! -//! ## Architecture -//! -//! All components use integer arithmetic (cents, nanoseconds, basis points) -//! for financial precision, matching the Python reference implementation -//! patterns from the HFTTrendfollowing project. - -pub mod benchmarks; -pub mod concurrent_tracking; -pub mod fractional_diff; -pub mod gpu_acceleration; - -// Meta-labeling engine (legacy interface) -pub mod meta_labeling_engine; - -// New meta-labeling module with secondary model -pub mod meta_labeling; - -pub mod sample_weights; -pub mod triple_barrier; -pub mod types; -// validation_test moved to tests/ directory - -// DO NOT RE-EXPORT - Use explicit imports at usage sites - -// DO NOT RE-EXPORT - Benchmarks should be imported explicitly - -/// Labeling module constants matching Python reference precision -pub mod constants { - /// Cents per dollar for `price` precision - pub const CENTS_PER_DOLLAR: i64 = 100; - - /// Basis points per dollar for return precision - pub const BASIS_POINTS_PER_DOLLAR: i64 = 10_000; - - /// Nanoseconds per second for time precision - pub const NANOSECONDS_PER_SECOND: i64 = 1_000_000_000; - - /// Microseconds per second - pub const MICROSECONDS_PER_SECOND: i64 = 1_000_000; - - /// Maximum latency target for triple barrier labeling (80μs) - pub const MAX_TRIPLE_BARRIER_LATENCY_US: u64 = 80; - - /// Maximum latency target for meta-labeling (50μs) - pub const MAX_META_LABELING_LATENCY_US: u64 = 50; - - /// Maximum latency target for fractional differentiation (1μs) - pub const MAX_FRACTIONAL_DIFF_LATENCY_US: u64 = 1; - - /// Minimum throughput for batch processing (labels/second) - pub const MIN_BATCH_THROUGHPUT_LPS: u64 = 10_000; -} - -/// Utility functions for labeling operations -pub mod utils { - use super::constants::*; - - /// Convert price to cents - pub fn price_to_cents(price: f64) -> u64 { - (price * CENTS_PER_DOLLAR as f64) as u64 - } - - /// Convert cents to price - pub fn cents_to_price(cents: u64) -> f64 { - cents as f64 / CENTS_PER_DOLLAR as f64 - } - - /// Convert ratio to basis points - pub fn ratio_to_bps(ratio: f64) -> i32 { - (ratio * BASIS_POINTS_PER_DOLLAR as f64) as i32 - } - - /// Convert basis points to ratio - pub fn bps_to_ratio(bps: i32) -> f64 { - bps as f64 / BASIS_POINTS_PER_DOLLAR as f64 - } - - /// Convert timestamp to nanoseconds - pub fn timestamp_to_ns(timestamp: f64) -> u64 { - (timestamp * NANOSECONDS_PER_SECOND as f64) as u64 - } - - /// Convert nanoseconds to timestamp - pub fn ns_to_timestamp(ns: u64) -> f64 { - ns as f64 / NANOSECONDS_PER_SECOND as f64 - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_price_conversions() { - let price = 123.45; - let cents = utils::price_to_cents(price); - let converted_back = utils::cents_to_price(cents); - - assert_eq!(cents, 12345); - assert!((converted_back - price).abs() < 1e-10); - } - - #[test] - fn test_ratio_conversions() { - let ratio = 0.0250; // 2.5% - let bps = utils::ratio_to_bps(ratio); - let converted_back = utils::bps_to_ratio(bps); - - assert_eq!(bps, 250); - assert!((converted_back - ratio).abs() < 1e-10); - } - - #[test] - fn test_timestamp_conversions() { - let timestamp = 1692000000.123456789; // Example timestamp with nanosecond precision - let ns = utils::timestamp_to_ns(timestamp); - let converted_back = utils::ns_to_timestamp(ns); - - // Should preserve millisecond precision - assert!((converted_back - timestamp).abs() < 1e-6); - } -} +//! Labeling -- thin facade re-exporting from ml-labeling crate. +pub use ml_labeling::*; diff --git a/crates/ml/src/lib.rs b/crates/ml/src/lib.rs index 56b32895d..20e664eab 100644 --- a/crates/ml/src/lib.rs +++ b/crates/ml/src/lib.rs @@ -296,12 +296,7 @@ pub mod registry; // Operational maturity: model lifecycle (Candidate -> Staging // ========== FROM IMPLS FOR MODULE-LOCAL ERROR TYPES ========== // These reference modules that remain in ml (not moved to ml-core) -// Implement From trait for LabelingError -impl From for MLError { - fn from(err: labeling::gpu_acceleration::LabelingError) -> Self { - MLError::InferenceError(err.to_string()) - } -} +// LabelingError → MLError impl moved to ml-labeling crate (orphan rule) impl From for MLError { fn from(err: inference::InferenceError) -> Self { diff --git a/crates/ml/src/microstructure/advanced_models.rs b/crates/ml/src/microstructure/advanced_models.rs deleted file mode 100644 index 4cc1a5d91..000000000 --- a/crates/ml/src/microstructure/advanced_models.rs +++ /dev/null @@ -1,94 +0,0 @@ -//! # Advanced Market Microstructure Models for HFT Alpha Generation -//! -//! Implements state-of-the-art machine learning models for market microstructure analysis -//! targeting <25μs inference latency. All models are optimized for real-time trading. -//! -//! ## Model Portfolio -//! -//! 1. **Order Flow Imbalance Prediction** - Predicts OFI using LSTM-Transformer hybrid -//! 2. **Liquidity Provision Optimization** - Optimal spread and size determination -//! 3. **Spread Prediction Models** - Real-time bid-ask spread forecasting -//! 4. **Market Impact Estimation** - Dynamic impact modeling with neural networks -//! 5. **Adverse Selection Detection** - Real-time toxic flow identification -//! 6. **Price Discovery Models** - Information incorporation efficiency analysis -//! 7. **Hidden Liquidity Detection** - Dark pool and iceberg order identification - -use std::collections::{HashMap, VecDeque}; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::time::{Duration, Instant}; - -use candle_core::Device; -use candle_core::{Tensor, Device, DType, Result as CandleResult}; -use candle_nn::{Linear, LayerNorm, Dropout, Module, VarBuilder}; -use ndarray::{Array1, Array2, Array3, ArrayView1, ArrayView2, s!}; -use serde::{Deserialize, Serialize}; - -use crate::{MLAppResult, InferenceResult, ModelMetadata}; -use super::*; -use super::{ - - - #[test] - fn test_feature_extractor_creation() { - let extractor = MicrostructureFeatureExtractor::new(64, OFI_FEATURE_DIM); - assert_eq!(extractor.window_size, 64); - assert_eq!(extractor.feature_dim, OFI_FEATURE_DIM); - } - - #[test] - fn test_feature_extraction() { - let mut extractor = MicrostructureFeatureExtractor::new(10, 16); - - let update = MarketDataUpdate { - timestamp: 1000000000, - symbol: "AAPL".to_owned(), - price: 150_00000000, // $150.00 in scaled format - volume: 1000, - bid: 149_95000000, // $149.95 - ask: 150_05000000, // $150.05 - bid_size: 500, - ask_size: 600, - direction: Some(TradeDirection::Buy), - }; - - let features = extractor.extract_features(&update)?; - assert_eq!(features.len(), 16); - - // Test feature values are reasonable - assert!(features[0] > 0.0); // Price feature - assert!(features[3] > 0.0); // Relative spread - } - - #[tokio::test] - async fn test_liquidity_optimization_structure() { - let optimization = LiquidityOptimization { - optimal_bid_spread_bps: 10.0, - optimal_ask_spread_bps: 10.0, - optimal_bid_size: 1000.0, - optimal_ask_size: 1000.0, - expected_profitability: 0.001, - risk_score: 0.2, - confidence: 0.8, - inference_time_us: 20, - }; - - assert_eq!(optimization.optimal_bid_spread_bps, 10.0); - assert!(optimization.inference_time_us <= TARGET_INFERENCE_LATENCY_US); - } - - #[test] - fn test_spread_prediction_structure() { - let prediction = SpreadPrediction { - current_spread_bps: 8.5, - predicted_spread_bps: 9.2, - spread_change_pct: 8.2, - prediction_horizon_seconds: 30, - spread_volatility: 0.15, - confidence: 0.75, - inference_time_us: 18, - }; - - assert!(prediction.predicted_spread_bps > prediction.current_spread_bps); - assert!(prediction.confidence > 0.0 && prediction.confidence < 1.0); - } -} \ No newline at end of file diff --git a/crates/ml/src/microstructure/advanced_models_extended.rs b/crates/ml/src/microstructure/advanced_models_extended.rs deleted file mode 100644 index 06a9bba35..000000000 --- a/crates/ml/src/microstructure/advanced_models_extended.rs +++ /dev/null @@ -1,249 +0,0 @@ -//! # Extended Advanced Microstructure Models (4-7) -//! -//! Continuation of advanced ML models for HFT alpha generation: -//! 4. Market Impact Estimation -//! 5. Adverse Selection Detection -//! 6. Price Discovery Models -//! 7. Hidden Liquidity Detection - -use chrono::{DateTime, Duration, Utc}; -use std::collections::{HashMap, VecDeque}; -use std::sync::atomic::{AtomicU64, AtomicI64, Ordering}; -use std::time::{Duration, Instant}; - -use candle_core::{Tensor, Device, DType, Result as CandleResult}; -use candle_nn::{Linear, LayerNorm, Dropout, Module, VarBuilder}; -use ndarray::{Array1, Array2, Array3, ArrayView1, ArrayView2, s!}; -use serde::{Deserialize, Serialize}; - -use crate::{MLAppResult, InferenceResult, ModelMetadata}; -use super::*; -use super::advanced_models::{ -use super::{MicrostructureResult, MarketDataUpdate, TradeDirection, MAX_CALCULATION_LATENCY_US}; - - - #[test] - fn test_market_regime_encoding() { - assert_eq!(MarketRegime::Normal as usize, 7); - assert_eq!(MarketRegime::Normal as usize, 0); - } - - #[test] - fn test_toxicity_type_classification() { - let toxic_types = [ - ToxicityType::InformedTrading, - ToxicityType::MomentumIgnition, - ToxicityType::Spoofing, - ToxicityType::Layering, - ToxicityType::Benign, - ToxicityType::Unknown, - ]; - - assert_eq!(toxic_types.len(), 6); - } - - #[test] - fn test_impact_measurement_structure() { - let measurement = ImpactMeasurement { - timestamp: 1000000000, - symbol: "AAPL".to_owned(), - trade_size: 1000, - pre_trade_price: 150_00000000, - execution_price: 150_01000000, - post_trade_price_1s: Some(150_02000000), - post_trade_price_5s: Some(150_01500000), - post_trade_price_30s: Some(150_00500000), - predicted_impact: 2.5, - actual_impact_1s: Some(2.0), - actual_impact_5s: Some(1.5), - actual_impact_30s: Some(0.5), - regime: MarketRegime::Normal, - }; - - assert_eq!(measurement.trade_size, 1000); - assert!(measurement.actual_impact_1s? > 0.0); - } -} - -// ============================================================================ -// Supporting Types for Models 6 and 7 -// ============================================================================ - -/// Price impact prediction result -/// PriceImpactPrediction component. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct PriceImpactPrediction { - pub total_impact: f64, - pub permanent_impact: f64, - pub temporary_impact: f64, - pub impact_duration_ms: u64, - pub confidence: f64, -} - -/// `Market` efficiency classification levels -/// EfficiencyLevel component. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub enum EfficiencyLevel { - High, - Medium, - Low, -} - -/// Information regime classification -/// InformationRegime component. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub enum InformationRegime { - NewsRiven, - TechnicalDriven, - Balanced, -} - -/// Efficiency classification result -/// EfficiencyClassification component. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct EfficiencyClassification { - pub efficiency_level: EfficiencyLevel, - pub efficiency_score: f64, - pub anomaly_detected: bool, - pub information_regime: InformationRegime, - pub processing_speed_ms: u64, -} - -/// Price formation dynamics analysis -/// PriceFormationDynamics component. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct PriceFormationDynamics { - pub formation_speed: f64, - pub price_efficiency: f64, - pub volatility_prediction: f64, - pub liquidity_depth: f64, - pub market_participation: f64, -} - -/// Information cascade analysis -/// CascadeAnalysis component. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct CascadeAnalysis { - pub cascade_detected: bool, - pub cascade_strength: f64, - pub cascade_duration_ms: u64, - pub participants_count: u32, -} - -/// Comprehensive `price` discovery analysis result -/// PriceDiscoveryAnalysis component. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct PriceDiscoveryAnalysis { - pub information_incorporation_speed: u64, - pub price_impact_prediction: PriceImpactPrediction, - pub efficiency_classification: EfficiencyClassification, - pub price_formation_dynamics: PriceFormationDynamics, - pub information_cascade_detected: bool, - pub cascade_strength: f64, - pub market_depth_impact: f64, - pub liquidity_impact: f64, - pub inference_time_us: u64, - pub confidence_score: f64, - pub timestamp: DateTime, -} - -/// Iceberg execution strategy types -/// IcebergStrategy component. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub enum IcebergStrategy { - None, - Simple, - TimeWeighted, - VolumeWeighted, -} - -/// Iceberg order detection result -/// IcebergDetection component. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct IcebergDetection { - pub iceberg_detected: bool, - pub confidence: f64, - pub estimated_total_size: f64, - pub revealed_portion: f64, - pub execution_strategy: IcebergStrategy, -} - -/// Dark pool detection result -/// DarkPoolDetection component. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct DarkPoolDetection { - pub dark_pool_detected: bool, - pub confidence: f64, - pub estimated_dark_volume: f64, - pub dark_pool_percentage: f64, - pub venue_estimates: HashMap, -} - -/// Stealth trading strategy types -/// StealthStrategy component. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub enum StealthStrategy { - None, - TWAP, - VWAP, - Implementation, - Iceberg, -} - -/// Stealth trading detection result -/// StealthTradingDetection component. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct StealthTradingDetection { - pub stealth_detected: bool, - pub confidence: f64, - pub execution_style: StealthStrategy, - pub stealth_score: f64, -} - -/// Volume pattern analysis result -/// VolumePatternAnalysis component. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct VolumePatternAnalysis { - pub pattern_strength: f64, - pub clustering_detected: bool, - pub unusual_patterns: bool, - pub volume_consistency: f64, -} - -/// Price action analysis result -/// PriceActionAnalysis component. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct PriceActionAnalysis { - pub liquidity_footprint_strength: f64, - pub hidden_support_resistance: bool, - pub price_memory_effect: f64, - pub estimated_hidden_depth: f64, -} - -/// Comprehensive hidden liquidity analysis result -/// HiddenLiquidityAnalysis component. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct HiddenLiquidityAnalysis { - pub iceberg_detection: IcebergDetection, - pub dark_pool_detection: DarkPoolDetection, - pub stealth_trading_detection: StealthTradingDetection, - pub volume_pattern_analysis: VolumePatternAnalysis, - pub price_action_analysis: PriceActionAnalysis, - pub overall_hidden_liquidity_score: f64, - pub estimated_hidden_volume: f64, - pub liquidity_sources: Vec, - pub detection_confidence: f64, - pub inference_time_us: u64, - pub timestamp: DateTime, -} - -/// Hidden liquidity detection performance metrics -/// HiddenLiquidityMetrics component. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct HiddenLiquidityMetrics { - pub total_detections: u64, - pub avg_inference_time_us: u64, - pub detection_accuracy: f64, - pub false_positive_rate: f64, - pub true_positive_rate: f64, -} \ No newline at end of file diff --git a/crates/ml/src/microstructure/amihud.rs b/crates/ml/src/microstructure/amihud.rs deleted file mode 100644 index 1a6f88cda..000000000 --- a/crates/ml/src/microstructure/amihud.rs +++ /dev/null @@ -1,169 +0,0 @@ -//! # Amihud Illiquidity Measure -//! -//! Implementation of the Amihud (2002) illiquidity measure for quantifying -//! the price impact per unit of trading volume. -//! -//! ## Algorithm -//! -//! ILLIQ = (1/T) × Σ(|Return_t| / DollarVolume_t) -//! -//! - Measures average ratio of absolute return to dollar volume -//! - Higher values indicate greater illiquidity (larger price impact) -//! - Can be calculated for different time horizons (daily, intraday) -//! -//! ## Performance -//! -//! - Target latency: <25μs per calculation -//! - Rolling window calculations with efficient updates -//! - Integer arithmetic for financial precision - -use std::collections::VecDeque; -use std::sync::atomic::{AtomicI64, AtomicU64, Ordering}; - -use serde::{Deserialize, Serialize}; - -use super::*; -use super::{ - - - #[test] - fn test_amihud_measure_creation() { - let measure = AmihudIlliquidityMeasure::default(); - assert_eq!(measure.get_illiquidity(), 0.0); - assert_eq!(measure.get_period_count(), 0); - assert_eq!(measure.get_liquidity_score(), 0.0); - } - - #[test] - fn test_trading_period() { - let mut period = TradingPeriod::new(0, 1000000, 2000000); - - let update1 = MarketDataUpdate { - timestamp: 1500000, - symbol: "AAPL".to_owned(), - price: 100000, // $10.00 - volume: 1000, - bid: 99000, - ask: 101000, - bid_size: 100, - ask_size: 100, - direction: None, - }; - - let update2 = MarketDataUpdate { - timestamp: 1600000, - symbol: "AAPL".to_owned(), - price: 105000, // $10.50 (5% increase) - volume: 1000, - bid: 104000, - ask: 106000, - bid_size: 100, - ask_size: 100, - direction: None, - }; - - period.add_trade(&update1); - period.add_trade(&update2); - - assert_eq!(period.trade_count, 2); - assert_eq!(period.open_price, 100000); - assert_eq!(period.close_price, 105000); - - period.finalize(10000 * PRECISION_FACTOR, 1000000); // Min $10k volume, 100% return cap - - assert!(period.is_valid()); - assert!(period.period_return > 0); // Positive return - assert!(period.illiquidity_ratio > 0); // Some illiquidity - } - - #[test] - fn test_amihud_calculation() { - let config = AmihudConfig { - period_duration_ns: 1000000, // 1ms for testing - window_size: 5, - min_dollar_volume: 1000 * PRECISION_FACTOR, // $1k minimum - ..Default::default() - }; - - let mut measure = AmihudIlliquidityMeasure::new(config); - - // Add trades with varying price impact - let base_price = 100000; // $10.00 - for i in 0..10 { - let price_change = if i % 2 == 0 { 500 } else { -500 }; // ±$0.05 - let price = base_price + price_change; - - let update = MarketDataUpdate { - timestamp: (i * 2000000) as u64, // 2ms intervals - symbol: "AAPL".to_owned(), - price, - volume: 1000 + (i * 100), // Varying volume - bid: price - 500, - ask: price + 500, - bid_size: 100, - ask_size: 100, - direction: None, - }; - - measure.update(&update)?; - } - - let result = measure.get_result(); - assert!(result.period_count > 0); - - // Should have some measurable illiquidity - println!("Illiquidity: {}, Liquidity Score: {}", - result.illiquidity, result.liquidity_score); - } - - #[test] - fn test_intraday_measure() { - let measure = AmihudIlliquidityMeasure::intraday(20, 5); // 20 periods of 5 minutes - - let config = measure.get_config(); - assert_eq!(config.time_horizon as u8, TimeHorizon::Intraday as u8); - assert_eq!(config.period_duration_ns, 300_000_000_000); // 5 minutes - assert_eq!(config.window_size, 20); - } - - #[test] - fn test_liquidity_classification() { - let mut measure = AmihudIlliquidityMeasure::default(); - - // High volume, low return changes = liquid - let update = MarketDataUpdate { - timestamp: 1000000, - symbol: "AAPL".to_owned(), - price: 100000, - volume: 100000, // Large volume - bid: 99950, - ask: 100050, - bid_size: 1000, - ask_size: 1000, - direction: None, - }; - - measure.update(&update)?; - - // Small price change with large volume should indicate liquidity - let update2 = MarketDataUpdate { - timestamp: 86400_000_000_000 + 1000000, // Next day - symbol: "AAPL".to_owned(), - price: 100010, // Tiny change - volume: 100000, - bid: 99960, - ask: 100060, - bid_size: 1000, - ask_size: 1000, - direction: None, - }; - - measure.update(&update2)?; - - let result = measure.get_result(); - - // Low illiquidity (high liquidity) due to small price change and large volume - assert!(result.illiquidity >= 0.0); - println!("Illiquidity: {}", result.illiquidity); - } -} \ No newline at end of file diff --git a/crates/ml/src/microstructure/benchmarks.rs b/crates/ml/src/microstructure/benchmarks.rs deleted file mode 100644 index 05f90b915..000000000 --- a/crates/ml/src/microstructure/benchmarks.rs +++ /dev/null @@ -1,193 +0,0 @@ -//! # Microstructure Analytics Performance Benchmarks -//! -//! Comprehensive benchmarks for all microstructure analytics components -//! to validate <25μs latency targets and throughput requirements. - -use chrono::{DateTime, Duration, Utc}; -use std::sync::Arc; -use std::thread; -use std::time::{Duration, Instant}; - -use criterion::{black_box, criterion_group, criterion_main, Criterion, Throughput, BenchmarkId}; -use tokio; - -use super::*; -use super::{ - - -/// Generate realistic market data for testing (replaces synthetic generation) -/// Based on realistic market microstructure patterns -fn generate_market_data(count: usize, symbol: &str) -> Vec { - let mut data = Vec::with_capacity(count); - let mut timestamp = Utc::now(); - let mut base_price = 150.0; // Realistic base price - let tick_size = 0.01; - - for i in 0..count { - // Create realistic price movement - let time_factor = i as f64 / count as f64; - let trend = (time_factor * 6.28).sin() * 0.005; // Small trend - let noise = (fastrand::f64() - 0.5) * 0.002; // Realistic noise - - let price_change = trend + noise; - base_price *= 1.0 + price_change; - - // Round to tick size - base_price = (base_price / tick_size).round() * tick_size; - - // Realistic bid-ask spread (0.01-0.03) - let spread = tick_size + (fastrand::f64() * 0.02); - let bid = base_price - spread / 2.0; - let ask = base_price + spread / 2.0; - - // Realistic volume patterns - let base_volume = 1000.0; - let volume_factor = 1.0 + (time_factor * 3.14).sin() * 0.5; // Volume cycles - let volume = (base_volume * volume_factor * (0.5 + fastrand::f64())).round(); - - // Market order probability based on time - let is_market_order = fastrand::f64() < 0.3; // 30% market orders - - data.push(MarketUpdate { - symbol: symbol.to_string(), - timestamp, - price: base_price, - volume, - bid, - ask, - trade_type: if is_market_order { TradeType::Market } else { TradeType::Limit }, - side: if fastrand::bool() { OrderSide::Buy } else { OrderSide::Sell }, - }); - - // Increment timestamp by realistic intervals (1-100ms) - timestamp += Duration::milliseconds(1 + fastrand::i64(0..100)); - } - - data -} - - #[test] - fn test_performance_targets() { - // Test that all components meet <25μs latency target - let data = generate_market_data(100, "AAPL"); - let mut violations = 0; - let mut total_tests = 0; - - // Test VPIN - { - let config = VPINConfig::default(); - let mut calculator = VPINCalculator::new(config); - - for update in &data { - let start = Instant::now(); - calculator.update(update)?; - let elapsed = start.elapsed().as_micros() as u64; - - if elapsed > MAX_CALCULATION_LATENCY_US { - violations += 1; - } - total_tests += 1; - } - } - - // Test Kyle's Lambda - { - let config = KyleLambdaConfig::default(); - let mut estimator = KyleLambdaEstimator::new(config); - - for update in &data { - let start = Instant::now(); - estimator.update(update)?; - let elapsed = start.elapsed().as_micros() as u64; - - if elapsed > MAX_CALCULATION_LATENCY_US { - violations += 1; - } - total_tests += 1; - } - } - - // Test Amihud - { - let config = AmihudConfig::default(); - let mut measure = AmihudIlliquidityMeasure::new(config); - - for update in &data { - let start = Instant::now(); - measure.update(update)?; - let elapsed = start.elapsed().as_micros() as u64; - - if elapsed > MAX_CALCULATION_LATENCY_US { - violations += 1; - } - total_tests += 1; - } - } - - let violation_rate = violations as f64 / total_tests as f64; - println!("Latency violations: {}/{} ({:.2}%)", violations, total_tests, violation_rate * 100.0); - - // Allow up to 5% violations for acceptable performance - assert!(violation_rate < 0.05, "Too many latency violations: {:.2}%", violation_rate * 100.0); - } - - #[tokio::test] - async fn test_engine_performance() { - let mut engine = MicrostructureEngine::new("AAPL".to_owned()); - let data = generate_market_data(50, "AAPL"); - - let start = Instant::now(); - for update in &data { - engine.update(update).await?; - } - let total_elapsed = start.elapsed(); - - let avg_latency = total_elapsed.as_micros() as f64 / data.len() as f64; - println!("Average engine update latency: {:.2}μs", avg_latency); - - // Should be much faster than 1ms per update - assert!(avg_latency < 1000.0, "Engine too slow: {:.2}μs per update", avg_latency); - } - - #[test] - fn test_data_generation_quality() { - let data = generate_market_data(1000, "AAPL"); - - // Verify data quality - assert_eq!(data.len(), 1000); - assert!(data.iter().all(|d| d.price > 0)); - assert!(data.iter().all(|d| d.volume > 0)); - assert!(data.iter().all(|d| d.bid < d.ask)); - assert!(data.iter().all(|d| d.symbol == "AAPL")); - - // Check timestamp progression - for i in 1..data.len() { - assert!(data[i].timestamp > data[i-1].timestamp); - } - - println!("Generated {} high-quality market data samples", data.len()); - } - - #[test] - fn test_memory_efficiency() { - // Test that components don't grow unboundedly - let config = VPINConfig::default(); - let mut calculator = VPINCalculator::new(config); - let data = generate_market_data(10000, "AAPL"); // Large dataset - - let initial_size = std::mem::size_of_val(&calculator); - - // Process many updates - for update in &data { - calculator.update(update)?; - } - - let final_size = std::mem::size_of_val(&calculator); - - // Size should remain bounded (within reasonable growth) - let growth_ratio = final_size as f64 / initial_size as f64; - println!("Memory growth ratio: {:.2}x", growth_ratio); - - assert!(growth_ratio < 2.0, "Excessive memory growth: {:.2}x", growth_ratio); - } -} \ No newline at end of file diff --git a/crates/ml/src/microstructure/hasbrouck.rs b/crates/ml/src/microstructure/hasbrouck.rs deleted file mode 100644 index d75a6a52e..000000000 --- a/crates/ml/src/microstructure/hasbrouck.rs +++ /dev/null @@ -1,138 +0,0 @@ -//! # Hasbrouck Information Share -//! -//! Implementation of Hasbrouck (1995) information share measure for quantifying -//! the contribution of each market or quote source to price discovery. -//! -//! ## Algorithm -//! -//! 1. Estimate Vector Error Correction Model (VECM) on price series -//! 2. Decompose variance of innovations to common efficient price -//! 3. Attribute variance shares to each source -//! 4. Information share = proportion of price discovery by each source -//! -//! ## Performance -//! -//! - Target latency: <25μs per calculation -//! - Simplified VECM estimation for real-time use -//! - Multiple market/source support - -use std::collections::{HashMap, VecDeque}; -use std::sync::atomic::{AtomicI64, AtomicU64, Ordering}; - -use serde::{Deserialize, Serialize}; - -use super::*; -use super::{ - - - #[test] - fn test_hasbrouck_creation() { - let hasbrouck = HasbrouckInformationShare::default(); - assert_eq!(hasbrouck.get_active_source_count(), 0); - assert_eq!(hasbrouck.get_dominant_source(), ""); - assert_eq!(hasbrouck.get_efficient_price(), 0.0); - } - - #[test] - fn test_price_observation() { - let mut hasbrouck = HasbrouckInformationShare::default(); - - let obs = PriceObservation { - timestamp: 1000000, - source: "NYSE".to_owned(), - price: 100000, // $10.00 - quote_type: QuoteType::Trade, - size: 1000, - sequence: 1, - }; - - hasbrouck.add_observation(obs)?; - - assert_eq!(hasbrouck.get_active_source_count(), 1); - assert!(hasbrouck.get_information_share("NYSE") >= 0.0); - } - - #[test] - fn test_multiple_sources() { - let config = HasbrouckConfig { - window_size: 50, - min_observations_per_source: 5, - update_frequency: 1, - ..Default::default() - }; - - let mut hasbrouck = HasbrouckInformationShare::new(config); - - // Add observations from multiple sources - let sources = vec!["NYSE", "NASDAQ", "BATS"]; - let base_price = 100000; - - for i in 0..30 { - for (j, &source) in sources.into_iter().enumerate() { - let price_offset = if source == "NYSE" { 0 } else { j as i64 * 10 }; // NYSE leads - - let obs = PriceObservation { - timestamp: (i * 1000000) as u64, - source: source.to_string(), - price: base_price + price_offset + (i as i64 * 100), - quote_type: QuoteType::Trade, - size: 1000, - sequence: (i * sources.len() + j) as u64, - }; - - hasbrouck.add_observation(obs)?; - } - } - - let result = hasbrouck.get_result(); - - assert_eq!(result.active_source_count, 3); - assert!(!result.dominant_source.is_empty()); - - // NYSE should have higher information share since it "leads" price discovery - let nyse_share = result.source_shares.get("NYSE").unwrap_or(&0.0); - println!("NYSE information share: {:.4}", nyse_share); - - // Sum of all shares should be approximately 1.0 - let total_share: f64 = result.source_shares.values().sum(); - assert!((total_share - 1.0).abs() < 0.1); - - println!("Concentration index: {:.4}", result.concentration_index); - println!("Fragmentation index: {:.4}", result.fragmentation_index); - } - - #[test] - fn test_market_data_integration() { - let mut hasbrouck = HasbrouckInformationShare::default(); - - // Simulate market data from different exchanges - for i in 0..20 { - let update = MarketDataUpdate { - timestamp: (i * 1000000) as u64, - symbol: "AAPL".to_owned(), - price: 150000 + (i * 50), // Trending price - volume: 1000, - bid: 149950, - ask: 150050, - bid_size: 100, - ask_size: 100, - direction: None, - }; - - // Alternate between exchanges - let exchange = if i % 2 == 0 { "NYSE" } else { "NASDAQ" }; - hasbrouck.update(&update, exchange)?; - } - - let result = hasbrouck.get_result(); - assert!(result.active_source_count > 0); - assert!(result.efficient_price > 0.0); - } - - #[test] - fn test_quote_types() { - assert_eq!(QuoteType::Trade.price_discovery_weight(), 1.0); - assert_eq!(QuoteType::Best.price_discovery_weight(), 0.9); - assert!(QuoteType::Mid.price_discovery_weight() < QuoteType::Best.price_discovery_weight()); - } -} \ No newline at end of file diff --git a/crates/ml/src/microstructure/integration.rs b/crates/ml/src/microstructure/integration.rs deleted file mode 100644 index d9eb9f1a5..000000000 --- a/crates/ml/src/microstructure/integration.rs +++ /dev/null @@ -1,161 +0,0 @@ -//! # Microstructure Integration Layer -//! -//! Integration of all microstructure analytics with ML models and risk management -//! for unified real-time analysis and decision support. - -use std::collections::HashMap; -use std::sync::Arc; -use std::time::Duration; - -use serde::{Deserialize, Serialize}; -use tokio::sync::RwLock; -use tokio; - -use super::*; -use super::{ - - - #[tokio::test] - async fn test_microstructure_engine_creation() { - let test_symbol = crate::test_fixtures::get_test_symbol(0); - let engine = MicrostructureEngine::new(test_symbol.symbol.to_string()); - - assert_eq!(engine.symbol, test_symbol.symbol); - assert!(engine.get_analytics().await.is_none()); - assert!(engine.get_signals().await.is_none()); - } - - #[tokio::test] - async fn test_engine_update() { - let test_symbol = crate::test_fixtures::get_test_symbol(0); - let mut engine = MicrostructureEngine::new(test_symbol.symbol.to_string()); - - let update = MarketDataUpdate { - timestamp: 1000000, - symbol: test_symbol.symbol.to_string(), - price: 150000, - volume: 1000, - bid: 149950, - ask: 150050, - bid_size: 100, - ask_size: 100, - direction: None, - }; - - engine.update(&update).await?; - - // Force calculation to generate analytics - engine.force_calculation(update.timestamp).await?; - - let analytics = engine.get_analytics().await; - assert!(analytics.is_some()); - - let signals = engine.get_signals().await; - assert!(signals.is_some()); - - let quality = engine.get_quality().await; - assert!(quality.is_some()); - } - - #[tokio::test] - async fn test_multi_source_update() { - let test_symbol = crate::test_fixtures::get_test_symbol(0); - let mut engine = MicrostructureEngine::new(test_symbol.symbol.to_string()); - - let updates = vec![ - (MarketDataUpdate { - timestamp: 1000000, - symbol: test_symbol.symbol.to_string(), - price: 150000, - volume: 1000, - bid: 149950, - ask: 150050, - bid_size: 100, - ask_size: 100, - direction: None, - }, "NYSE".to_owned()), - (MarketDataUpdate { - timestamp: 1000001, - symbol: test_symbol.symbol.to_string(), - price: 150010, - volume: 800, - bid: 149960, - ask: 150060, - bid_size: 80, - ask_size: 80, - direction: None, - }, "NASDAQ".to_owned()), - ]; - - engine.update_multi_source(&updates).await?; - engine.force_calculation(1000001).await?; - - let analytics = engine.get_analytics().await?; - assert_eq!(analytics.hasbrouck.active_source_count, 2); - } - - #[tokio::test] - async fn test_alert_generation() { - let test_symbol = crate::test_fixtures::get_test_symbol(0); - let mut engine = MicrostructureEngine::new(test_symbol.symbol.to_string()); - - // Create conditions that should trigger alerts - for i in 0..50 { - let update = MarketDataUpdate { - timestamp: (i * 1000000) as u64, - symbol: test_symbol.symbol.to_string(), - price: 150000 + (i % 2) * 1000, // High volatility to trigger toxicity - volume: 100, // Low volume to trigger liquidity alerts - bid: 149000, - ask: 151000, // Wide spread - bid_size: 10, - ask_size: 10, - direction: None, - }; - - engine.update(&update).await?; - } - - engine.force_calculation(50000000).await?; - - let alerts = engine.get_alerts().await; - println!("Generated {} alerts", alerts.len()); - - for alert in &alerts { - println!("Alert: {:?} - {}", alert.alert_type, alert.message); - } - } - - #[tokio::test] - async fn test_performance_metrics() { - let test_symbol = crate::test_fixtures::get_test_symbol(0); - let mut engine = MicrostructureEngine::new(test_symbol.symbol.to_string()); - - // Add several updates to generate metrics - for i in 0..10 { - let update = MarketDataUpdate { - timestamp: (i * 1000000) as u64, - symbol: "AAPL".to_owned(), - price: 150000, - volume: 1000, - bid: 149950, - ask: 150050, - bid_size: 100, - ask_size: 100, - direction: None, - }; - - engine.update(&update).await?; - } - - let metrics = engine.get_performance_metrics(); - let (avg_latency, max_latency, violations) = metrics.get_overall_latency_stats(); - - assert!(avg_latency >= 0.0); - assert!(max_latency < 1000); // Should be fast - assert!(violations == 0); // No violations expected for simple test - - println!("Avg latency: {:.2}μs, Max: {}μs, Violations: {}", - avg_latency, max_latency, violations); - } -} \ No newline at end of file diff --git a/crates/ml/src/microstructure/kyle_lambda.rs b/crates/ml/src/microstructure/kyle_lambda.rs deleted file mode 100644 index f83dcf959..000000000 --- a/crates/ml/src/microstructure/kyle_lambda.rs +++ /dev/null @@ -1,127 +0,0 @@ -//! # Kyle's Lambda Estimator -//! -//! Implementation of Kyle's Lambda for measuring price impact and -//! information asymmetry in financial markets. -//! -//! ## Algorithm -//! -//! Kyle's Lambda (λ) measures the price impact per unit of signed order flow: -//! - Returns = λ × SignedOrderFlow + ε -//! - λ is estimated via regression of returns on signed square-root dollar volume -//! - Higher λ indicates greater price impact (lower liquidity) -//! -//! ## Performance -//! -//! - Target latency: <25μs per calculation -//! - Rolling regression with fixed-point arithmetic -//! - Efficient covariance calculation updates - -use std::collections::VecDeque; -use std::sync::atomic::{AtomicI64, AtomicU64, Ordering}; - -use serde::{Deserialize, Serialize}; - -use super::*; -use super::{ - - - #[test] - fn test_kyle_lambda_estimator_creation() { - let estimator = KyleLambdaEstimator::default(); - assert_eq!(estimator.get_lambda(), 0.0); - assert_eq!(estimator.get_interval_count(), 0); - assert_eq!(estimator.get_r_squared(), 0.0); - } - - #[test] - fn test_trading_interval() { - let mut interval = TradingInterval::new(0, 1000000, 2000000); - - let update = MarketDataUpdate { - timestamp: 1500000, - symbol: "AAPL".to_owned(), - price: 150000, - volume: 1000, - bid: 149000, - ask: 151000, - bid_size: 100, - ask_size: 100, - direction: Some(TradeDirection::Buy), - }; - - interval.add_trade(&update); - - assert_eq!(interval.trade_count, 1); - assert_eq!(interval.open_price, 150000); - assert_eq!(interval.close_price, 150000); - assert!(interval.signed_sqrt_dollar_volume > 0); // Buy trade - - interval.finalize(); - assert!(interval.is_valid()); - } - - #[test] - fn test_lambda_calculation() { - let config = KyleLambdaConfig { - interval_duration_ns: 1000000, // 1ms for testing - min_trades_per_interval: 1, - regression_window: 5, - ..Default::default() - }; - - let mut estimator = KyleLambdaEstimator::new(config); - - // Add trades with price impact pattern - for i in 0..20 { - let price_impact = if i % 2 == 0 { 100 } else { -100 }; - let direction = if i % 2 == 0 { TradeDirection::Buy } else { TradeDirection::Sell }; - - let update = MarketDataUpdate { - timestamp: (i * 2000000) as u64, // 2ms intervals - symbol: "AAPL".to_owned(), - price: 150000 + price_impact, - volume: 1000, - bid: 149000, - ask: 151000, - bid_size: 100, - ask_size: 100, - direction: Some(direction), - }; - - estimator.update(&update)?; - } - - // Should have calculated lambda - let result = estimator.get_result(); - assert!(result.interval_count > 0); - - // Lambda should be non-zero if there's a price impact pattern - // (exact value depends on the specific pattern) - println!("Lambda: {}, R²: {}", result.lambda, result.r_squared); - } - - #[test] - fn test_information_asymmetry() { - let mut estimator = KyleLambdaEstimator::default(); - - // Add persistent positive returns (trend) - for i in 0..10 { - let update = MarketDataUpdate { - timestamp: (i * 300_000_000_000) as u64, // 5 min intervals - symbol: "AAPL".to_owned(), - price: 150000 + (i * 100), // Trending up - volume: 1000, - bid: 149000, - ask: 151000, - bid_size: 100, - ask_size: 100, - direction: Some(TradeDirection::Buy), - }; - - estimator.update(&update)?; - } - - let info_asymmetry = estimator.get_information_asymmetry(); - assert!(info_asymmetry >= 0.0); // Should detect some persistence - } -} \ No newline at end of file diff --git a/crates/ml/src/microstructure/ml_integration.rs b/crates/ml/src/microstructure/ml_integration.rs deleted file mode 100644 index ef0d86238..000000000 --- a/crates/ml/src/microstructure/ml_integration.rs +++ /dev/null @@ -1,51 +0,0 @@ -//! # ML Microstructure Integration Module -//! -//! Orchestrates all advanced microstructure ML models for unified HFT alpha generation. -//! Provides a single interface for all models with ensemble prediction capabilities. - -use std::collections::{HashMap, VecDeque}; -use std::sync::Arc; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::time::Instant; - -use candle_core::Device; -use candle_core::{Device, VarBuilder}; -use serde::{Deserialize, Serialize}; -use tokio::sync::{RwLock, Mutex}; - -use crate::{MLAppResult, InferenceResult, ModelMetadata}; -use super::*; -use super::advanced_models::{ -use super::advanced_models_extended::{ -use super::{MarketDataUpdate, TradeDirection, MicrostructureResult}; - - #[tokio::test] - async fn test_ensemble_creation() { - let device = Device::Cpu; - let ensemble = MicrostructureMLEnsemble::new(device).await; - - // Note: This will fail without proper model files, but tests the structure - assert!(ensemble.is_err()); // Expected without trained models - } - - #[test] - fn test_ensemble_weights() { - let weights = EnsembleWeights::default(); - let total_weight = weights.ofi_weight + weights.liquidity_weight + - weights.spread_weight + weights.impact_weight + - weights.toxicity_weight; - - assert!((total_weight - 1.0).abs() < 0.01); // Should sum to approximately 1.0 - } - - #[test] - fn test_trading_action_determination() { - // Test various alpha/confidence combinations - let action_strong_buy = TradingAction::StrongBuy; - let action_hold = TradingAction::Hold; - - // These would be tested with actual ensemble logic - assert!(matches!(action_strong_buy, TradingAction::StrongBuy)); - assert!(matches!(action_hold, TradingAction::Hold)); - } -} \ No newline at end of file diff --git a/crates/ml/src/microstructure/portfolio_integration.rs b/crates/ml/src/microstructure/portfolio_integration.rs deleted file mode 100644 index c06b7d319..000000000 --- a/crates/ml/src/microstructure/portfolio_integration.rs +++ /dev/null @@ -1,159 +0,0 @@ -//! # Portfolio Integration for Microstructure ML Models -//! -//! This module provides seamless integration between the advanced microstructure models -//! and the Portfolio Transformer, creating a unified ML system for HFT alpha generation. -//! -//! ## Key Features -//! -//! - **Unified Interface**: Single entry point for all microstructure ML predictions -//! - **Portfolio Enhancement**: Integrates microstructure signals with portfolio optimization -//! - **Real-time Processing**: Sub-25μs microstructure signal generation for portfolio decisions -//! - **Risk-Aware Integration**: Combines microstructure risk signals with portfolio risk management -//! - **Performance Tracking**: Comprehensive metrics for microstructure contribution to alpha - -use std::{ - -use candle_core::Device; -use candle_core::{Device, Tensor}; -use chrono::{DateTime, Utc}; -use portfolio_management::advanced_black_litterman::{ -use serde::{Serialize, Deserialize}; -use tokio::sync::RwLock; -use tracing::{debug, info, warn, instrument}; - -use crate::portfolio_transformer::{PortfolioTransformer, PortfolioState, PortfolioOptimizationResult}; -use crate::portfolio_transformer::{PortfolioTransformerConfig}; -use super::*; -use super::{ - - - async fn create_test_integrator() -> Result> { - let portfolio_config = PortfolioTransformerConfig::nano(); - let device = Device::Cpu; - let portfolio_transformer = Arc::new( - PortfolioTransformer::new(portfolio_config, device)? - ); - - let integration_config = PortfolioMicrostructureConfig::default(); - let bl_config = AdvancedBlackLittermanConfig::default(); - - PortfolioMicrostructureIntegrator::new( - integration_config, - portfolio_transformer, - bl_config, - ).await - } - - fn create_test_portfolio_state() -> PortfolioState { - PortfolioState { - weights: vec![0.25, 0.25, 0.25, 0.25], - expected_returns: vec![0.08, 0.12, 0.06, 0.10], - volatilities: vec![0.15, 0.25, 0.12, 0.18], - correlations: vec![0.6, 0.3, 0.4, 0.7, 0.5, 0.2], - market_regime: vec![1.0, 0.0, 0.0, 0.0], - risk_metrics: vec![0.05, 0.08, 0.03, 0.15], - confidence_scores: vec![0.8, 0.7, 0.9, 0.6], - alpha_signals: vec![0.02, -0.01, 0.03, 0.01], - timestamp: Utc::now(), - } - } - - fn create_test_asset_symbols() -> Vec { - vec![ - Symbol::from_static("AAPL"), - Symbol::from_static("MSFT"), - Symbol::from_static("GOOGL"), - Symbol::from_static("AMZN"), - ] - } - - #[tokio::test] - async fn test_integrator_creation() { - let integrator = create_test_integrator().await; - assert!(integrator.is_ok()); - } - - #[tokio::test] - async fn test_portfolio_optimization_integration() { - let integrator = create_test_integrator().await?; - let portfolio_state = create_test_portfolio_state(); - let asset_symbols = create_test_asset_symbols(); - - let result = integrator.optimize_portfolio_integrated( - &portfolio_state, - None, - &asset_symbols, - ).await; - - assert!(result.is_ok()); - let optimization_result = result?; - assert_eq!(optimization_result.integrated_weights.len(), 4); - assert!(optimization_result.integration_confidence > 0.0); - assert!(optimization_result.total_optimization_time_us > 0); - } - - #[tokio::test] - async fn test_signal_combination() { - let integrator = create_test_integrator().await?; - let portfolio_state = create_test_portfolio_state(); - - let portfolio_result = PortfolioOptimizationResult { - optimal_weights: vec![0.3, 0.3, 0.2, 0.2], - expected_return: 0.08, - expected_volatility: 0.15, - sharpe_ratio: 0.8, - optimization_confidence: 0.9, - inference_time_us: 1000, - model_components: HashMap::new(), - }; - - let microstructure_prediction = EnsemblePrediction { - ensemble_alpha: 0.02, - ensemble_confidence: 0.8, - recommended_action: crate::ml_integration::TradingAction::Buy, - total_inference_time_us: 500, - model_predictions: HashMap::new(), - risk_metrics: HashMap::new(), - market_quality_score: 0.9, - signal_strength: 0.7, - execution_recommendation: "Execute gradually".to_owned(), - }; - - let integrated_weights = integrator.combine_signals( - &portfolio_result, - µstructure_prediction, - None, - &portfolio_state, - ).await; - - assert!(integrated_weights.is_ok()); - let weights = integrated_weights?; - assert_eq!(weights.len(), 4); - - // Check that weights sum to approximately 1 - let total_weight: f64 = weights.iter().sum(); - assert!((total_weight - 1.0).abs() < 1e-6); - } - - #[tokio::test] - async fn test_metrics_tracking() { - let integrator = create_test_integrator().await?; - - integrator.update_metrics(1500, 0.85).await; - - let metrics = integrator.get_metrics().await; - assert_eq!(metrics.total_optimizations.load(Ordering::Relaxed), 1); - assert_eq!(metrics.average_optimization_time_us, 1500.0); - assert_eq!(metrics.average_integration_confidence, 0.85); - assert!(metrics.last_optimization_time.is_some()); - } - - #[test] - fn test_config_defaults() { - let config = PortfolioMicrostructureConfig::default(); - assert_eq!(config.microstructure_weight, 0.3); - assert_eq!(config.portfolio_weight, 0.7); - assert!(config.risk_adjustment_config.enable_adverse_selection_adjustment); - assert!(config.execution_config.enable_execution_optimization); - } -} \ No newline at end of file diff --git a/crates/ml/src/microstructure/roll_spread.rs b/crates/ml/src/microstructure/roll_spread.rs deleted file mode 100644 index 0abe3c0c2..000000000 --- a/crates/ml/src/microstructure/roll_spread.rs +++ /dev/null @@ -1,217 +0,0 @@ -//! # Roll Spread Estimator -//! -//! Implementation of Roll (1984) spread estimator that infers bid-ask spread -//! from serial covariance in price changes. -//! -//! ## Algorithm -//! -//! Roll Spread = 2 × √(-Cov(ΔP_t, ΔP_{t-1})) -//! -//! - Uses negative autocovariance in price changes -//! - Assumes price changes alternate due to bid-ask bounce -//! - Provides spread estimate when quotes are not available -//! -//! ## Performance -//! -//! - Target latency: <25μs per calculation -//! - Efficient rolling covariance calculation -//! - Handles missing or invalid data gracefully - -use std::collections::VecDeque; -use std::sync::atomic::{AtomicI64, AtomicU64, Ordering}; - -use serde::{Deserialize, Serialize}; -use common::types::Price; - -use super::*; -use super::{ - - - #[test] - fn test_roll_spread_estimator_creation() { - let estimator = RollSpreadEstimator::default(); - assert_eq!(estimator.get_spread(), 0.0); - assert_eq!(estimator.get_price_change_count(), 0); - assert!(!estimator.is_valid_estimate()); - } - - #[test] - fn test_price_change_tracking() { - let mut estimator = RollSpreadEstimator::default(); - - // Add series of price changes that alternate (bid-ask bounce) - let base_price = 100000; // $10.00 - for i in 0..50 { - let price = if i % 2 == 0 { - base_price + 50 // Ask side - } else { - base_price - 50 // Bid side - }; - - let update = MarketDataUpdate { - timestamp: (i * 1000000) as u64, - symbol: "AAPL".to_owned(), - price, - volume: 1000, - bid: base_price - 50, - ask: base_price + 50, - bid_size: 100, - ask_size: 100, - direction: None, - }; - - estimator.update(&update)?; - } - - let result = estimator.get_result(); - assert!(result.price_change_count > 0); - - // Should detect negative autocovariance from alternating prices - if result.is_valid_estimate { - assert!(result.autocovariance < 0.0); - assert!(result.spread > 0.0); - println!("Roll spread: {:.4}, Autocovariance: {:.4}", - result.spread, result.autocovariance); - } - } - - #[test] - fn test_high_frequency_estimator() { - let estimator = RollSpreadEstimator::high_frequency(200); - - let config = estimator.get_config(); - assert_eq!(config.window_size, 200); - assert_eq!(config.update_frequency, 1); // Every trade - assert_eq!(config.min_price_change, 0); // No minimum - } - - #[test] - fn test_spread_calculation() { - let config = RollSpreadConfig { - window_size: 20, - min_price_changes: 10, - update_frequency: 1, - ..Default::default() - }; - - let mut estimator = RollSpreadEstimator::new(config); - - // Create alternating price pattern (classic bid-ask bounce) - let prices = [100050, 99950, 100050, 99950, 100050, 99950, - 100050, 99950, 100050, 99950, 100050, 99950, - 100050, 99950, 100050, 99950, 100050, 99950]; - - for (i, &price) in prices.into_iter().enumerate() { - let update = MarketDataUpdate { - timestamp: (i * 1000000) as u64, - symbol: "AAPL".to_owned(), - price, - volume: 1000, - bid: 99950, - ask: 100050, - bid_size: 100, - ask_size: 100, - direction: None, - }; - - estimator.update(&update)?; - } - - let result = estimator.get_result(); - - // Perfect bid-ask bounce should give negative autocovariance - if result.is_valid_estimate { - assert!(result.autocovariance < 0.0); - assert!(result.spread > 0.0); - assert!(result.spread_bps > 0.0); - - // The spread should be close to the actual bid-ask spread (100 basis points) - println!("Detected spread: {:.4} ({:.2} bps), Expected: 0.01 (100 bps)", - result.spread, result.spread_bps); - } - } - - #[test] - fn test_data_quality_score() { - let mut estimator = RollSpreadEstimator::default(); - - // Initially no data - assert_eq!(estimator.get_data_quality_score(), 0.0); - - // Add some price changes - for i in 0..10 { - let update = MarketDataUpdate { - timestamp: (i * 1000000) as u64, - symbol: "AAPL".to_owned(), - price: 100000 + (i % 2) * 100, // Alternating - volume: 1000, - bid: 99950, - ask: 100050, - bid_size: 100, - ask_size: 100, - direction: None, - }; - - estimator.update(&update)?; - } - - let quality_score = estimator.get_data_quality_score(); - assert!(quality_score > 0.0); - assert!(quality_score <= 1.0); - - println!("Data quality score: {:.4}", quality_score); - } - - #[test] - fn test_midpoint_vs_transaction_prices() { - // Test with transaction prices - let mut est_transaction = RollSpreadEstimator::new(RollSpreadConfig { - use_transaction_prices: true, - window_size: 20, - min_price_changes: 10, - update_frequency: 1, - ..Default::default() - }); - - // Test with midpoint prices - let mut est_midpoint = RollSpreadEstimator::new(RollSpreadConfig { - use_transaction_prices: false, - window_size: 20, - min_price_changes: 10, - update_frequency: 1, - ..Default::default() - }); - - // Add same data to both - for i in 0..20 { - let update = MarketDataUpdate { - timestamp: (i * 1000000) as u64, - symbol: "AAPL".to_owned(), - price: if i % 2 == 0 { 100050 } else { 99950 }, // Transaction price alternates - volume: 1000, - bid: 99950, - ask: 100050, // Midpoint = 100000 (constant) - bid_size: 100, - ask_size: 100, - direction: None, - }; - - est_transaction.update(&update)?; - est_midpoint.update(&update)?; - } - - let result_trans = est_transaction.get_result(); - let result_mid = est_midpoint.get_result(); - - // Transaction prices should show bid-ask bounce, midpoint should not - println!("Transaction spread: {:.4}, Midpoint spread: {:.4}", - result_trans.spread, result_mid.spread); - - if result_trans.is_valid_estimate { - assert!(result_trans.spread > 0.0); - } - - // Midpoint prices are constant, so no spread detected - assert_eq!(result_mid.spread, 0.0); - } -} \ No newline at end of file diff --git a/crates/ml/src/microstructure/training_pipeline.rs b/crates/ml/src/microstructure/training_pipeline.rs deleted file mode 100644 index 0477fc523..000000000 --- a/crates/ml/src/microstructure/training_pipeline.rs +++ /dev/null @@ -1,81 +0,0 @@ -//! # Microstructure ML Training Pipeline -//! -//! This module implements a comprehensive training pipeline that integrates: -//! - Polygon historical data ingestion -//! - Feature engineering for microstructure models -//! - Model training and validation -//! - Performance monitoring and model selection -//! - Real-time model deployment and updates -//! -//! The pipeline is designed for <25μs inference latency with continuous learning -//! capabilities for high-frequency trading environments. - -use std::collections::HashMap; -use std::{ - -use candle_core::{Device, Tensor, DType}; -use candle_nn::{VarBuilder, VarMap}; -use chrono::{DateTime, Duration, Duration as ChronoDuration, NaiveDate, Utc}; -use ndarray::Array2; -// REMOVED: Polygon imports - replaced with Databento{ -use serde::{Serialize, Deserialize}; -use tokio::{ -use tracing::{debug, info, warn, error, instrument}; - -use super::*; -use super::{ - - - #[tokio::test] - async fn test_training_pipeline_creation() { - let config = TrainingPipelineConfig::default(); - let pipeline = MicrostructureTrainingPipeline::new(config).await; - assert!(pipeline.is_ok()); - } - - #[test] - fn test_training_config_defaults() { - let config = TrainingPipelineConfig::default(); - assert_eq!(config.training_data_config.split_ratios, (0.7, 0.15, 0.15)); - assert_eq!(config.model_training_config.batch_size, 256); - assert!(config.monitoring_config.retraining_triggers.enable_automatic_retraining); - } - - #[tokio::test] - async fn test_pipeline_lifecycle() { - let config = TrainingPipelineConfig::default(); - let pipeline = MicrostructureTrainingPipeline::new(config).await?; - - // Test stop without start - let result = pipeline.stop().await; - assert!(result.is_ok()); - - // Test metrics access - let metrics = pipeline.get_metrics().await; - assert_eq!(metrics.total_samples_processed.load(Ordering::Relaxed), 0); - } - - #[tokio::test] - async fn test_retraining_trigger() { - let config = TrainingPipelineConfig::default(); - let pipeline = MicrostructureTrainingPipeline::new(config).await?; - - let result = pipeline.trigger_retraining().await; - assert!(result.is_ok()); - - let should_retrain = *pipeline.should_retrain.read().await; - assert!(should_retrain); - } - - #[test] - fn test_target_type_serialization() { - let target_type = TargetType::PriceDirection; - let serialized = serde_json::to_string(&target_type)?; - let deserialized: TargetType = serde_json::from_str(&serialized)?; - - match (target_type, deserialized) { - (TargetType::PriceDirection, TargetType::PriceDirection) => {}, - _ => return Err(anyhow!("Serialization roundtrip failed")), - } - } -} \ No newline at end of file diff --git a/crates/ml/src/microstructure/types.rs b/crates/ml/src/microstructure/types.rs deleted file mode 100644 index e9aa55ca4..000000000 --- a/crates/ml/src/microstructure/types.rs +++ /dev/null @@ -1,134 +0,0 @@ -//! # Microstructure Types -//! -//! Common types and data structures for microstructure analytics. - -use std::collections::HashMap; - -use serde::{Deserialize, Serialize}; -// use trading_engine::prelude::*; // REMOVED - prelude does not exist // For AlertSeverity and other types - -use super::*; -use super::{PRECISION_FACTOR, TradeDirection}; - - #[test] - fn test_market_quality_indicators() { - let analytics = MicrostructureAnalytics { - symbol: "AAPL".to_owned(), - timestamp: 1000000, - vpin: VPINMetrics { - vpin: 0.3, - order_flow_imbalance: 0.1, - toxicity_score: 0.4, - is_toxic: false, - bucket_count: 20, - current_bucket_fill: 0.5, - }, - kyle_lambda: KyleLambdaMetrics { - lambda: 0.5, - price_impact_coefficient: 0.01, - r_squared: 0.8, - information_asymmetry: 0.2, - adverse_selection: 0.1, - interval_count: 30, - }, - amihud: AmihudMetrics { - illiquidity: 0.1, - avg_absolute_return: 0.02, - avg_dollar_volume: 1000000.0, - price_impact_per_million: 50.0, - liquidity_score: 0.8, - period_count: 15, - }, - roll_spread: RollSpreadMetrics { - spread: 0.001, - spread_bps: 10.0, - autocovariance: -0.0001, - effective_spread: 0.0012, - spread_volatility: 0.0002, - is_valid_estimate: true, - data_quality_score: 0.9, - }, - hasbrouck: HasbrouckMetrics { - source_shares: [("NYSE".to_owned(), 0.6), ("NASDAQ".to_owned(), 0.4)].iter().cloned().collect(), - dominant_source: "NYSE".to_owned(), - concentration_index: 0.52, - fragmentation_index: 0.48, - active_source_count: 2, - efficient_price: 150.0, - }, - liquidity_score: 0.7, - toxicity_indicator: 0.4, - price_discovery_quality: 0.8, - }; - - let quality = MarketQualityIndicators::from_analytics(&analytics); - - assert!(quality.liquidity_score > 0.0); - assert!(quality.liquidity_score <= 1.0); - assert!(quality.toxicity_score >= 0.0); - assert!(quality.overall_quality > 0.0); - - println!("Quality indicators: {:#?}", quality); - } - - #[test] - fn test_microstructure_signals() { - let analytics = MicrostructureAnalytics { - symbol: "AAPL".to_owned(), - timestamp: 1000000, - vpin: VPINMetrics { - vpin: 0.2, - order_flow_imbalance: 0.05, - toxicity_score: 0.3, - is_toxic: false, - bucket_count: 25, - current_bucket_fill: 0.7, - }, - kyle_lambda: KyleLambdaMetrics { - lambda: 0.3, - price_impact_coefficient: 0.005, - r_squared: 0.85, - information_asymmetry: 0.15, - adverse_selection: 0.08, - interval_count: 40, - }, - amihud: AmihudMetrics { - illiquidity: 0.05, - avg_absolute_return: 0.015, - avg_dollar_volume: 2000000.0, - price_impact_per_million: 30.0, - liquidity_score: 0.9, - period_count: 20, - }, - roll_spread: RollSpreadMetrics { - spread: 0.0008, - spread_bps: 8.0, - autocovariance: -0.00015, - effective_spread: 0.001, - spread_volatility: 0.0001, - is_valid_estimate: true, - data_quality_score: 0.95, - }, - hasbrouck: HasbrouckMetrics { - source_shares: [("NYSE".to_owned(), 0.7), ("NASDAQ".to_owned(), 0.3)].iter().cloned().collect(), - dominant_source: "NYSE".to_owned(), - concentration_index: 0.58, - fragmentation_index: 0.42, - active_source_count: 2, - efficient_price: 150.5, - }, - liquidity_score: 0.85, - toxicity_indicator: 0.3, - price_discovery_quality: 0.85, - }; - - let quality = MarketQualityIndicators::from_analytics(&analytics); - let signals = MicrostructureSignals::from_analytics(&analytics, &quality); - - assert!(signals.confidence > 0.0); - assert!(signals.overall_score >= -1.0 && signals.overall_score <= 1.0); - assert!(signals.liquidity_signal >= -1.0 && signals.liquidity_signal <= 1.0); - - println!("Microstructure signals: {:#?}", signals); - } -} \ No newline at end of file diff --git a/crates/ml/src/microstructure/vpin.rs b/crates/ml/src/microstructure/vpin.rs deleted file mode 100644 index 15a3c7661..000000000 --- a/crates/ml/src/microstructure/vpin.rs +++ /dev/null @@ -1,156 +0,0 @@ -//! # VPIN Calculator -//! -//! Volume-Synchronized Probability of Informed Trading implementation -//! for detecting order flow toxicity and informed trading activity. -//! -//! ## Algorithm -//! -//! 1. **Volume Bucketing**: Divide trading into fixed-volume buckets -//! 2. **Trade Classification**: Classify trades as buyer/seller initiated -//! 3. **Imbalance Calculation**: Calculate |BuyVol - SellVol| / TotalVol per bucket -//! 4. **VPIN Calculation**: Rolling average of normalized imbalances -//! -//! ## Performance -//! -//! - Target latency: <25μs per calculation -//! - Memory: Ring buffer with zero allocations in hot path -//! - Precision: Integer arithmetic with 10,000x scaling - -use std::sync::atomic::{AtomicI64, AtomicU64, Ordering}; -use std::time::Duration; - -use serde::{Deserialize, Serialize}; - -use super::*; -use super::{ - - - #[test] - fn test_vpin_calculator_creation() { - let calculator = VPINCalculator::default(); - assert_eq!(calculator.get_vpin(), 0.0); - assert_eq!(calculator.get_bucket_count(), 0); - assert!(!calculator.is_toxic()); - } - - #[test] - fn test_vpin_calculation() { - let mut calculator = VPINCalculator::default(); - - // Add buy trades - for i in 0..10 { - let update = MarketDataUpdate { - timestamp: 1000000 + i, - symbol: "AAPL".to_owned(), - price: 150000 + (i as i64 * 100), // Increasing prices - volume: 1000, - bid: 149000, - ask: 151000, - bid_size: 100, - ask_size: 100, - direction: Some(TradeDirection::Buy), - }; - - calculator.update(&update)?; - } - - // Add sell trades - for i in 0..10 { - let update = MarketDataUpdate { - timestamp: 1000000 + 10 + i, - symbol: "AAPL".to_owned(), - price: 150000 - (i as i64 * 100), // Decreasing prices - volume: 1000, - bid: 149000, - ask: 151000, - bid_size: 100, - ask_size: 100, - direction: Some(TradeDirection::Sell), - }; - - calculator.update(&update)?; - } - - // Should have completed 2 buckets (20k volume, 10k per bucket) - assert_eq!(calculator.get_bucket_count(), 2); - - // VPIN should be high due to imbalanced flow - let result = calculator.get_result(); - assert!(result.vpin > 0.0); - assert!(result.order_flow_imbalance >= 0.0); - } - - #[test] - fn test_trade_classification() { - let calculator = VPINCalculator::default(); - - let update = MarketDataUpdate { - timestamp: 1000000, - symbol: "AAPL".to_owned(), - price: 155000, // Above midpoint (150000) - volume: 1000, - bid: 149000, - ask: 151000, - bid_size: 100, - ask_size: 100, - direction: None, // Will be classified - }; - - let direction = calculator.classify_trade(&update); - assert_eq!(direction, TradeDirection::Buy); - } - - #[test] - fn test_performance_metrics() { - let mut calculator = VPINCalculator::default(); - - let update = MarketDataUpdate { - timestamp: 1000000, - symbol: "AAPL".to_owned(), - price: 150000, - volume: 1000, - bid: 149000, - ask: 151000, - bid_size: 100, - ask_size: 100, - direction: Some(TradeDirection::Buy), - }; - - calculator.update(&update)?; - - let metrics = calculator.get_metrics(); - assert_eq!(metrics.total_calculations, 1); - assert!(metrics.avg_latency_us >= 0.0); - assert!(metrics.max_latency_us < 1000); // Should be very fast - } - - #[test] - fn test_bucket_filling() { - let config = VPINConfig { - bucket_volume: 5000, // Small bucket for testing - ..Default::default() - }; - let mut calculator = VPINCalculator::new(config); - - // Add trades that exactly fill one bucket - for i in 0..5 { - let update = MarketDataUpdate { - timestamp: 1000000 + i, - symbol: "AAPL".to_owned(), - price: 150000, - volume: 1000, - bid: 149000, - ask: 151000, - bid_size: 100, - ask_size: 100, - direction: Some(TradeDirection::Buy), - }; - - calculator.update(&update)?; - } - - // Should have completed 1 bucket - assert_eq!(calculator.get_bucket_count(), 1); - assert_eq!(calculator.get_current_bucket_fill(), 0.0); // New bucket started - } -} \ No newline at end of file diff --git a/crates/ml/src/regime/mod.rs b/crates/ml/src/regime/mod.rs index 16edb4809..d14bccb34 100644 --- a/crates/ml/src/regime/mod.rs +++ b/crates/ml/src/regime/mod.rs @@ -1,25 +1,2 @@ -//! Regime Detection Module -//! -//! This module provides structural break detection and regime classification: -//! - CUSUM-based changepoint detection (mean, variance, multivariate) -//! - Bayesian online changepoint detection -//! - Regime classifiers (trending, ranging, volatile) -//! - Adaptive strategy components (position sizing, dynamic stops) -//! - Performance tracking per regime - -// Wave D: Structural Breaks Detection (Agents D1-D4) -pub mod bayesian_changepoint; -pub mod cusum; -pub mod multi_cusum; -pub mod pages_test; - -// Wave D: Regime Classification (Agents D5-D8) -pub mod orchestrator; -pub mod ranging; -pub mod transition_matrix; -pub mod trending; -pub mod volatile; - -// Wave D: Transition Probability Features (Agent D15) -pub mod transition_probability_features; - +//! Regime Detection — thin facade re-exporting from ml-regime crate. +pub use ml_regime::*; diff --git a/crates/ml/src/risk/advanced_risk_engine.rs b/crates/ml/src/risk/advanced_risk_engine.rs deleted file mode 100644 index 7d4dfabdf..000000000 --- a/crates/ml/src/risk/advanced_risk_engine.rs +++ /dev/null @@ -1,725 +0,0 @@ -//! Advanced Risk Management Engine -//! -//! Production-ready risk management system implementing 2025 best practices for HFT systems. -//! Features real-time VaR calculation, stress testing, position monitoring, and regulatory compliance. - -use std::collections::HashMap; -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; -use std::sync::{Arc, RwLock, Mutex}; - -use chrono::{DateTime, Utc, Duration}; -use crossbeam::queue::ArrayQueue; -use ndarray::{Array1, Array2}; -use rand::Rng; -use rand_distr::{Distribution, StandardNormal}; -use rayon::prelude::*; -use serde::{Deserialize, Serialize}; -use thiserror::Error; -use tokio::sync::mpsc; - -use crate::{MLResult, MLError}; -/// Monte Carlo simulation for VaR calculation -pub fn simulate_random_shock(volatility: f64) -> f64 { - let mut rng = rand::thread_rng(); - let z: f64 = rng.sample(StandardNormal); - z * volatility -} - -/// Stress testing engine with parallel execution -/// StressTestEngine component. -#[derive(Debug)] -pub struct StressTestEngine { - scenarios: Vec, - thread_pool: rayon::ThreadPool, - results_queue: Arc>, -} - -/// StressScenario component. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct StressScenario { - pub name: String, - pub description: String, - pub shock_magnitude: f64, - pub affected_assets: Vec, - pub correlation_shock: Option, - pub volatility_multiplier: f64, - pub duration_days: u32, -} - -/// StressTestResult component. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct StressTestResult { - pub scenario_name: String, - pub portfolio_pnl: f64, - pub max_drawdown: f64, - pub var_breach_probability: f64, - pub liquidity_shortfall: f64, - pub recovery_time_days: u32, - pub passed: bool, - pub confidence_level: f64, - pub timestamp: DateTime, -} - -impl StressTestEngine { - /// Create new stress testing engine - pub fn new(scenarios: Vec) -> Self { - let thread_pool = rayon::ThreadPoolBuilder::new() - .num_threads(num_cpus::get()) - .build() - .map_err(|e| anyhow!("Failed to create thread pool: {:?}", e))?; - - let results_queue = Arc::new(ArrayQueue::new(1000)); - - Self { - scenarios, - thread_pool, - results_queue, - } - } - - /// Run all stress scenarios in parallel - pub fn run_stress_tests( - &self, - portfolio: &HashMap, - var_engine: &RealTimeVarEngine, - ) -> Vec { - let results: Vec<_> = self.scenarios - .par_iter() - .map(|scenario| self.execute_scenario(scenario, portfolio, var_engine)) - .collect(); - - results.into_iter().filter_map(Result::ok).collect() - } - - fn execute_scenario( - &self, - scenario: &StressScenario, - portfolio: &HashMap, - var_engine: &RealTimeVarEngine, - ) -> Result { - // Apply scenario shocks to portfolio - let mut stressed_portfolio = portfolio.clone(); - - for asset_id in &scenario.affected_assets { - if let Some(position) = stressed_portfolio.get_mut(asset_id) { - *position *= 1.0 + scenario.shock_magnitude; - } - } - - // Calculate stressed VaR - let stressed_var = var_engine.calculate_portfolio_var(&stressed_portfolio, 10000)?; - - // Calculate portfolio P&L under stress - let portfolio_pnl = self.calculate_stressed_pnl(portfolio, scenario); - - // Determine if scenario passed - let max_acceptable_loss = portfolio.values().sum::() * 0.05; // 5% max loss - let passed = portfolio_pnl.abs() <= max_acceptable_loss; - - Ok(StressTestResult { - scenario_name: scenario.name.clone(), - portfolio_pnl, - max_drawdown: portfolio_pnl.min(0.0), - var_breach_probability: if stressed_var.var_95 > 0.0 { 0.05 } else { 0.0 }, - liquidity_shortfall: 0.0, // Would calculate based on liquidation costs - recovery_time_days: if passed { 0 } else { scenario.duration_days }, - passed, - confidence_level: 0.95, - timestamp: Utc::now(), - }) - } - - fn calculate_stressed_pnl( - &self, - portfolio: &HashMap, - scenario: &StressScenario, - ) -> f64 { - let mut total_pnl = 0.0; - - for (asset_id, position) in portfolio { - if scenario.affected_assets.contains(asset_id) { - total_pnl += position * scenario.shock_magnitude; - } - } - - total_pnl - } -} - -/// Position monitoring system with hierarchical limits -/// PositionMonitor component. -#[derive(Debug)] -pub struct PositionMonitor { - account_limits: Arc>>, - strategy_limits: Arc>>, - instrument_limits: Arc>>, - current_positions: Arc>>, - circuit_breaker: Arc, - limit_breach_sender: mpsc::Sender, -} - -/// AccountLimits component. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct AccountLimits { - pub max_gross_exposure: f64, - pub max_net_exposure: f64, - pub max_var_95: f64, - pub max_concentration: f64, - pub max_leverage: f64, -} - -/// StrategyLimits component. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct StrategyLimits { - pub max_position_size: f64, - pub max_daily_pnl_loss: f64, - pub max_drawdown: f64, - pub enabled: bool, -} - -/// InstrumentLimits component. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct InstrumentLimits { - pub max_position: f64, - pub max_order_size: f64, - pub min_price: f64, - pub max_price: f64, - pub trading_enabled: bool, -} - -/// LimitBreach component. -#[derive(Debug, Clone)] -pub struct LimitBreach { - pub limit_type: String, - pub current_value: f64, - pub limit_value: f64, - pub asset_id: Option, - pub timestamp: DateTime, -} - -impl PositionMonitor { - /// Create new position monitoring system - pub fn new() -> (Self, mpsc::Receiver) { - let (sender, receiver) = mpsc::channel(1000); - - (Self { - account_limits: Arc::new(RwLock::new(HashMap::new())), - strategy_limits: Arc::new(RwLock::new(HashMap::new())), - instrument_limits: Arc::new(RwLock::new(HashMap::new())), - current_positions: Arc::new(RwLock::new(HashMap::new())), - circuit_breaker: Arc::new(AtomicBool::new(false)), - limit_breach_sender, - }, receiver) - } - - /// Check if order passes all risk limits - pub async fn check_order_limits( - &self, - asset_id: AssetId, - order_size: f64, - account_id: AccountId, - strategy_id: StrategyId, - ) -> Result<(), AdvancedRiskError> { - // Check circuit breaker - if self.circuit_breaker.load(Ordering::Relaxed) { - return Err(AdvancedRiskError::CircuitBreakerError { - reason: "Global circuit breaker activated".to_owned(), - }); - } - - // Check instrument limits - self.check_instrument_limits(asset_id, order_size).await?; - - // Check strategy limits - self.check_strategy_limits(strategy_id, asset_id, order_size).await?; - - // Check account limits - self.check_account_limits(account_id, asset_id, order_size).await?; - - Ok(()) - } - - async fn check_instrument_limits( - &self, - asset_id: AssetId, - order_size: f64, - ) -> Result<(), AdvancedRiskError> { - let limits = self.instrument_limits.read()?; - let positions = self.current_positions.read()?; - - if let Some(limit) = limits.get(&asset_id) { - if !limit.trading_enabled { - return Err(AdvancedRiskError::PositionLimitError { - limit_type: "Trading disabled".to_owned(), - }); - } - - if order_size.abs() > limit.max_order_size { - return Err(AdvancedRiskError::PositionLimitError { - limit_type: "Order size limit exceeded".to_owned(), - }); - } - - let current_position = positions.get(&asset_id).copied().unwrap_or(0.0); - let projected_position = current_position + order_size; - - if projected_position.abs() > limit.max_position { - let _ = self.limit_breach_sender.try_send(LimitBreach { - limit_type: "Position limit breach".to_owned(), - current_value: projected_position.abs(), - limit_value: limit.max_position, - asset_id: Some(asset_id), - timestamp: Utc::now(), - }); - - return Err(AdvancedRiskError::PositionLimitError { - limit_type: "Position limit exceeded".to_owned(), - }); - } - } - - Ok(()) - } - - async fn check_strategy_limits( - &self, - strategy_id: StrategyId, - asset_id: AssetId, - order_size: f64, - ) -> Result<(), AdvancedRiskError> { - let limits = self.strategy_limits.read()?; - - if let Some(limit) = limits.get(&strategy_id) { - if !limit.enabled { - return Err(AdvancedRiskError::PositionLimitError { - limit_type: "Strategy disabled".to_owned(), - }); - } - - if order_size.abs() > limit.max_position_size { - return Err(AdvancedRiskError::PositionLimitError { - limit_type: "Strategy position size exceeded".to_owned(), - }); - } - } - - Ok(()) - } - - async fn check_account_limits( - &self, - account_id: AccountId, - asset_id: AssetId, - order_size: f64, - ) -> Result<(), AdvancedRiskError> { - let limits = self.account_limits.read()?; - let positions = self.current_positions.read()?; - - if let Some(limit) = limits.get(&account_id) { - // Calculate projected exposure - let current_gross_exposure: f64 = positions.values().map(|p| p.abs()).sum(); - let projected_gross_exposure = current_gross_exposure + order_size.abs(); - - if projected_gross_exposure > limit.max_gross_exposure { - return Err(AdvancedRiskError::PositionLimitError { - limit_type: "Account gross exposure exceeded".to_owned(), - }); - } - } - - Ok(()) - } - - /// Trigger emergency circuit breaker - pub fn activate_circuit_breaker(&self, reason: String) { - self.circuit_breaker.store(true, Ordering::Relaxed); - - let _ = self.limit_breach_sender.try_send(LimitBreach { - limit_type: "Circuit breaker activated".to_owned(), - current_value: 1.0, - limit_value: 0.0, - asset_id: None, - timestamp: Utc::now(), - }); - } - - /// Deactivate circuit breaker - pub fn deactivate_circuit_breaker(&self) { - self.circuit_breaker.store(false, Ordering::Relaxed); - } -} - -/// Portfolio optimization engine with dynamic correlation analysis -/// PortfolioOptimizer component. -#[derive(Debug)] -pub struct PortfolioOptimizer { - correlation_estimator: OnlineCorrelationEstimator, - expected_returns: Arc>>, - risk_aversion: f64, -} - -/// OnlineCorrelationEstimizer component. -#[derive(Debug)] -pub struct OnlineCorrelationEstimizer { - correlation_matrix: Arc>>, - means: Arc>>, - n_observations: Arc>, - decay_factor: f64, -} - -impl OnlineCorrelationEstimator { - pub fn new(decay_factor: f64) -> Self { - Self { - correlation_matrix: Arc::new(RwLock::new(Array2::zeros((0, 0)))), - means: Arc::new(RwLock::new(HashMap::new())), - n_observations: Arc::new(Mutex::new(0)), - decay_factor, - } - } -} - -/// OptimizationResult component. -#[derive(Debug, Clone)] -pub struct OptimizationResult { - pub optimal_weights: HashMap, - pub expected_return: f64, - pub expected_risk: f64, - pub sharpe_ratio: f64, - pub timestamp: DateTime, -} - -impl PortfolioOptimizer { - /// Create new portfolio optimizer - pub fn new(risk_aversion: f64) -> Self { - Self { - correlation_estimator: OnlineCorrelationEstimizer::new(0.94), - expected_returns: Arc::new(RwLock::new(HashMap::new())), - risk_aversion, - } - } - - /// Optimize portfolio using mean-variance optimization - pub fn optimize_portfolio( - &self, - current_positions: &HashMap, - target_return: Option, - ) -> Result { - let returns = self.expected_returns.read()?; - let correlations = self.correlation_estimator.correlation_matrix.read()?; - - // Simple mean-variance optimization (simplified) - let mut optimal_weights = HashMap::new(); - let num_assets = current_positions.len(); - let equal_weight = 1.0 / num_assets as f64; - - for asset_id in current_positions.keys() { - optimal_weights.insert(*asset_id, equal_weight); - } - - // Calculate expected portfolio return and risk - let expected_return = self.calculate_portfolio_return(&optimal_weights, &returns); - let expected_risk = self.calculate_portfolio_risk(&optimal_weights, &correlations); - - let sharpe_ratio = if expected_risk > 0.0 { - expected_return / expected_risk - } else { - 0.0 - }; - - Ok(OptimizationResult { - optimal_weights, - expected_return, - expected_risk, - sharpe_ratio, - timestamp: Utc::now(), - }) - } - - fn calculate_portfolio_return( - &self, - weights: &HashMap, - returns: &HashMap, - ) -> f64 { - weights.iter() - .map(|(asset_id, weight)| weight * returns.get(asset_id).copied().unwrap_or(0.0)) - .sum() - } - - fn calculate_portfolio_risk( - &self, - weights: &HashMap, - correlations: &Array2, - ) -> f64 { - // Simplified risk calculation - // In practice, this would involve full covariance matrix multiplication - weights.values().map(|w| w.powi(2)).sum::().sqrt() - } -} - -impl OnlineCorrelationEstimator { - /// Create new online correlation estimator - pub fn new(decay_factor: f64) -> Self { - Self { - correlation_matrix: Arc::new(RwLock::new(Array2::zeros((100, 100)))), - means: Arc::new(RwLock::new(HashMap::new())), - n_observations: Arc::new(Mutex::new(0)), - decay_factor, - } - } - - /// Update correlations with new return data - pub fn update_correlations(&self, returns: &HashMap) { - let mut means = self.means.write()?; - let mut n_obs = self.n_observations.lock()?; - *n_obs += 1; - - // Update running means using EWMA - for (asset_id, return_value) in returns { - let current_mean = means.get(asset_id).copied().unwrap_or(0.0); - let updated_mean = self.decay_factor * current_mean + (1.0 - self.decay_factor) * return_value; - means.insert(*asset_id, updated_mean); - } - } -} - -/// Regulatory compliance engine -/// ComplianceEngine component. -#[derive(Debug)] -pub struct ComplianceEngine { - position_limits: HashMap, - reporting_buffer: Arc>>, - violation_count: Arc, -} - -/// ComplianceEvent component. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ComplianceEvent { - pub event_type: String, - pub description: String, - pub severity: ComplianceSeverity, - pub asset_id: Option, - pub value: f64, - pub timestamp: DateTime, -} - -/// ComplianceSeverity component. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub enum ComplianceSeverity { - Info, - Warning, - Critical, - Breach, -} - -impl ComplianceEngine { - /// Create new compliance engine - pub fn new() -> Self { - Self { - position_limits: HashMap::new(), - reporting_buffer: Arc::new(Mutex::new(Vec::new())), - violation_count: Arc::new(AtomicU64::new(0)), - } - } - - /// Check order for regulatory compliance - pub fn check_compliance( - &self, - asset_id: AssetId, - order_size: f64, - current_positions: &HashMap, - ) -> Result<(), AdvancedRiskError> { - // Example: Large trader reporting threshold - if order_size.abs() > 1_000_000.0 { - self.log_compliance_event(ComplianceEvent { - event_type: "Large Trade".to_owned(), - description: format!("Large order size: {}", order_size), - severity: ComplianceSeverity::Info, - asset_id: Some(asset_id), - value: order_size, - timestamp: Utc::now(), - }); - } - - // Example: Position concentration check - let total_exposure: f64 = current_positions.values().map(|p| p.abs()).sum(); - let current_position = current_positions.get(&asset_id).copied().unwrap_or(0.0); - let concentration = (current_position + order_size).abs() / total_exposure; - - if concentration > 0.1 { // 10% concentration limit - self.violation_count.fetch_add(1, Ordering::Relaxed); - return Err(AdvancedRiskError::ComplianceError { - rule: "Position concentration limit exceeded".to_owned(), - }); - } - - Ok(()) - } - - fn log_compliance_event(&self, event: ComplianceEvent) { - if let Ok(mut buffer) = self.reporting_buffer.lock() { - buffer.push(event); - - // Maintain buffer size - if buffer.len() > 10000 { - buffer.drain(0..1000); - } - } - } - - /// Get compliance report - pub fn generate_compliance_report(&self) -> Vec { - self.reporting_buffer.lock() - .map(|buffer| buffer.clone()) - .unwrap_or_default() - } -} - -/// Main advanced risk management system -/// AdvancedRiskManagementSystem component. -#[derive(Debug)] -pub struct AdvancedRiskManagementSystem { - var_engine: RealTimeVarEngine, - stress_engine: StressTestEngine, - position_monitor: PositionMonitor, - portfolio_optimizer: PortfolioOptimizer, - compliance_engine: ComplianceEngine, - config: AdvancedRiskConfig, -} - -/// AdvancedRiskConfig component. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct AdvancedRiskConfig { - pub var_confidence_levels: Vec, - pub stress_scenarios_enabled: bool, - pub position_monitoring_enabled: bool, - pub portfolio_optimization_enabled: bool, - pub compliance_checking_enabled: bool, - pub update_frequency_ms: u64, - pub max_portfolio_var: f64, - pub max_concentration: f64, -} - -impl AdvancedRiskManagementSystem { - /// Create new advanced risk management system - pub fn new(config: AdvancedRiskConfig) -> (Self, mpsc::Receiver) { - let var_engine = RealTimeVarEngine::new(config.var_confidence_levels.clone(), 252); - - // Default stress scenarios - let stress_scenarios = vec![ - StressScenario { - name: "Market Crash".to_owned(), - description: "20% market decline".to_owned(), - shock_magnitude: -0.20, - affected_assets: vec![], // Would be populated with relevant assets - correlation_shock: Some(0.8), // High correlation during crisis - volatility_multiplier: 2.0, - duration_days: 5, - }, - StressScenario { - name: "Liquidity Crisis".to_owned(), - description: "Severe liquidity constraints".to_owned(), - shock_magnitude: -0.10, - affected_assets: vec![], - correlation_shock: None, - volatility_multiplier: 1.5, - duration_days: 10, - }, - ]; - - let stress_engine = StressTestEngine::new(stress_scenarios); - let (position_monitor, limit_receiver) = PositionMonitor::new(); - let portfolio_optimizer = PortfolioOptimizer::new(1.0); // Moderate risk aversion - let compliance_engine = ComplianceEngine::new(); - - (Self { - var_engine, - stress_engine, - position_monitor, - portfolio_optimizer, - compliance_engine, - config, - }, limit_receiver) - } - - /// Comprehensive risk check for new order - pub async fn check_order_risk( - &self, - asset_id: AssetId, - order_size: f64, - account_id: AccountId, - strategy_id: StrategyId, - current_positions: &HashMap, - ) -> Result { - let start_time = std::time::Instant::now(); - - // 1. Position limit checks - if self.config.position_monitoring_enabled { - self.position_monitor - .check_order_limits(asset_id, order_size, account_id, strategy_id) - .await?; - } - - // 2. Compliance checks - if self.config.compliance_checking_enabled { - self.compliance_engine - .check_compliance(asset_id, order_size, current_positions)?; - } - - // 3. VaR impact assessment - let var_impact = if current_positions.contains_key(&asset_id) { - self.var_engine - .calculate_incremental_var( - asset_id, - order_size, - self.config.max_portfolio_var, - )? - } else { - 0.0 - }; - - // 4. Portfolio optimization impact (optional, for advisory) - let optimization_advice = if self.config.portfolio_optimization_enabled { - Some(self.portfolio_optimizer.optimize_portfolio(current_positions, None)?) - } else { - None - }; - - let processing_time = start_time.elapsed(); - - Ok(OrderRiskAssessment { - approved: true, - var_impact, - optimization_advice, - processing_time_nanos: processing_time.as_nanos() as u64, - timestamp: Utc::now(), - }) - } - - /// Run comprehensive stress testing - pub fn run_stress_tests( - &self, - current_positions: &HashMap, - ) -> Vec { - if self.config.stress_scenarios_enabled { - self.stress_engine.run_stress_tests(current_positions, &self.var_engine) - } else { - vec![] - } - } - - /// Update risk models with new market data - pub fn update_risk_models(&self, market_data: &HashMap) { - self.var_engine.update_volatility_estimates(market_data); - self.portfolio_optimizer.correlation_estimator.update_correlations(market_data); - } -} - -/// OrderRiskAssessment component. -#[derive(Debug, Clone)] -pub struct OrderRiskAssessment { - pub approved: bool, - pub var_impact: f64, - pub optimization_advice: Option, - pub processing_time_nanos: u64, - pub timestamp: DateTime, -} \ No newline at end of file diff --git a/crates/ml/src/risk/bayesian_risk_models.rs b/crates/ml/src/risk/bayesian_risk_models.rs deleted file mode 100644 index b2b860373..000000000 --- a/crates/ml/src/risk/bayesian_risk_models.rs +++ /dev/null @@ -1,128 +0,0 @@ -//! Bayesian Neural Networks for Risk Uncertainty Quantification -//! -//! Implements variational Bayesian neural networks for uncertainty quantification in risk models. -//! Provides confidence intervals and uncertainty estimates required for regulatory compliance -//! and model risk management frameworks (SR 11-7). -//! -//! # Features -//! - Variational inference for weight uncertainty -//! - Epistemic and aleatoric uncertainty decomposition -//! - Monte Carlo dropout approximation -//! - Bayesian VaR with confidence intervals -//! - Uncertainty-aware risk predictions -//! - Model confidence scoring for regulatory reporting -//! -//! # Performance Targets -//! - Inference latency: <200μs (with uncertainty sampling) -//! - Training convergence: <100 epochs -//! - Uncertainty calibration: >95% coverage -//! - Memory efficiency: <512MB model size - -use std::sync::Arc; - -use chrono::{DateTime, Utc}; -use serde::{Deserialize, Serialize}; - -use super::*; - -// Bayesian neural network implementations would go here -// Currently only tests are implemented - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_bayesian_layer_creation() { - let layer = BayesianLinearLayer::new(10, 5, 1.0); - - assert_eq!(layer.input_dim, 10); - assert_eq!(layer.output_dim, 5); - assert_eq!(layer.weight_mean.len(), 5); - assert_eq!(layer.weight_mean[0].len(), 10); - assert_eq!(layer.bias_mean.len(), 5); - } - - #[test] - fn test_bayesian_network_prediction() { - let config = BayesianNetworkConfig::default(); - let network = BayesianRiskNetwork::new(&[5, 10, 1], config); - - let features = vec![0.1, 0.2, 0.3, 0.4, 0.5]; - let prediction = network.predict_with_uncertainty(&features); - - assert!(prediction.inference_time_us > 0); - assert!(prediction.uncertainty_std >= 0.0); - assert!(prediction.model_confidence >= 0.0 && prediction.model_confidence <= 1.0); - assert_eq!(prediction.prediction_samples.len(), 100); // default mc_samples - } - - #[test] - fn test_var_calculation() { - let samples = vec![-2.0, -1.5, -1.0, -0.5, 0.0, 0.5, 1.0, 1.5, 2.0]; - let var_95 = calculate_var_from_samples(&samples, 0.95); - - // For 95% confidence, we expect the 5th percentile (worst 5%) - assert!(var_95 < 0.0); // Should be negative for losses - } - - #[test] - fn test_uncertainty_decomposition() { - let prediction = RiskPredictionWithUncertainty { - mean_prediction: 0.5, - uncertainty_std: 0.2, - confidence_interval_95: (0.1, 0.9), - confidence_interval_90: (0.2, 0.8), - epistemic_uncertainty: 0.14, - aleatoric_uncertainty: 0.06, - prediction_samples: vec![0.4, 0.5, 0.6], - model_confidence: 0.8, - inference_time_us: 100, - timestamp: Timestamp::now(), - }; - - // Check uncertainty decomposition - let total_uncertainty_approx = (prediction.epistemic_uncertainty.powi(2) + - prediction.aleatoric_uncertainty.powi(2)).sqrt(); - - assert!((total_uncertainty_approx - prediction.uncertainty_std).abs() < 0.1); - assert!(prediction.meets_confidence_threshold(0.7)); - assert!(!prediction.meets_confidence_threshold(0.9)); - } - - #[test] - fn test_risk_categorization() { - let high_risk_pred = RiskPredictionWithUncertainty { - mean_prediction: 0.9, - uncertainty_std: 0.1, - confidence_interval_95: (0.7, 1.0), - confidence_interval_90: (0.75, 0.95), - epistemic_uncertainty: 0.07, - aleatoric_uncertainty: 0.03, - prediction_samples: vec![0.85, 0.9, 0.95], - model_confidence: 0.9, - inference_time_us: 80, - timestamp: Timestamp::now(), - }; - - assert!(matches!(high_risk_pred.risk_category(), RiskCategory::High)); - } - - #[test] - fn test_performance_requirements() { - let config = BayesianNetworkConfig { - mc_samples: 50, // Reduced for performance test - ..Default::default() - }; - let network = BayesianRiskNetwork::new(&[10, 20, 10, 1], config); - - let features = vec![0.1; 10]; - let start = std::time::Instant::now(); - let prediction = network.predict_with_uncertainty(&features); - let elapsed = start.elapsed(); - - // Should complete in <200μs (with 50 samples) - assert!(elapsed.as_micros() < 200); - assert!(prediction.inference_time_us < 200); - } -} \ No newline at end of file diff --git a/crates/ml/src/risk/copula_dependency_models.rs b/crates/ml/src/risk/copula_dependency_models.rs deleted file mode 100644 index 57d90ea3f..000000000 --- a/crates/ml/src/risk/copula_dependency_models.rs +++ /dev/null @@ -1,187 +0,0 @@ -//! Copula-Based Dependency Modeling with Machine Learning -//! -//! Implements ML-enhanced copula models for capturing complex dependencies between -//! financial risk factors. Supports various copula families with neural network -//! parameter estimation for dynamic dependency modeling. -//! -//! # Features -//! - Dynamic copula parameter estimation using neural networks -//! - Multiple copula families: Gaussian, t-Copula, Clayton, Gumbel, Frank -//! - Vine copulas for high-dimensional dependencies -//! - Time-varying copula parameters -//! - Tail dependency modeling for extreme events -//! - Conditional copulas for regime-dependent dependencies -//! -//! # Performance Targets -//! - Parameter estimation: <500μs -//! - Dependency simulation: <1ms for 1000 samples -//! - Real-time parameter updates: <100μs -//! - Memory efficiency: <128MB for 100 assets - -use std::sync::Arc; - -use chrono::{DateTime, Utc}; -use serde::{Deserialize, Serialize}; - -use super::*; - - #[test] - fn test_neural_copula_creation() { - let config = CopulaModelConfig::default(); - let copula_family = CopulaFamily::Gaussian { - correlation_matrix: vec![vec![1.0, 0.5], vec![0.5, 1.0]] - }; - - let model = NeuralCopulaModel::new(copula_family, config); - - assert_eq!(model.config.n_factors, 10); - assert!(model.parameter_history.is_empty()); - } - - #[test] - fn test_parameter_constraint() { - let config = CopulaModelConfig::default(); - let copula_family = CopulaFamily::Clayton { theta: 1.0 }; - let model = NeuralCopulaModel::new(copula_family, config); - - // Test Clayton theta constraint (must be > 0) - let constrained = model.constrain_parameters(vec![-0.5]); - assert!(constrained[0] > 0.0); - - let constrained = model.constrain_parameters(vec![2.0]); - assert_eq!(constrained[0], 2.0); - } - - #[test] - fn test_tail_dependence_estimation() { - let config = CopulaModelConfig::default(); - let copula_family = CopulaFamily::Gaussian { - correlation_matrix: vec![vec![1.0, 0.5], vec![0.5, 1.0]] - }; - let model = NeuralCopulaModel::new(copula_family, config); - - // Create mock market data - let asset_data = vec![ - AssetMarketData { - asset_id: "AAPL".to_owned(), - returns: vec![0.01, -0.02, 0.015, -0.01, 0.005], - mean_return: 0.001, - volatility: 0.02, - skewness: 0.1, - kurtosis: 3.0, - }, - AssetMarketData { - asset_id: "MSFT".to_owned(), - returns: vec![0.008, -0.015, 0.012, -0.008, 0.003], - mean_return: 0.0005, - volatility: 0.018, - skewness: -0.1, - kurtosis: 2.8, - }, - ]; - - let market_data = MarketDataWindow { - asset_data, - window_start: Utc::now(), - window_end: Utc::now(), - n_observations: 5, - }; - - let tail_dep = model.estimate_tail_dependence(&market_data); - - assert!(tail_dep.upper_tail >= 0.0 && tail_dep.upper_tail <= 1.0); - assert!(tail_dep.lower_tail >= 0.0 && tail_dep.lower_tail <= 1.0); - } - - #[test] - fn test_copula_simulation() { - let config = CopulaModelConfig::default(); - let copula_family = CopulaFamily::Clayton { theta: 2.0 }; - let model = NeuralCopulaModel::new(copula_family, config); - - let parameters = CopulaParameters { - parameters: vec![2.0], - confidence_intervals: vec![(1.5, 2.5)], - tail_dependence: TailDependence { - upper_tail: 0.0, - lower_tail: 0.5, - asymmetry: -0.5, - upper_tail_ci: (0.0, 0.1), - lower_tail_ci: (0.4, 0.6), - }, - gof_statistics: GoodnessOfFitStats { - cramer_von_mises: 0.1, - anderson_darling: 0.8, - kolmogorov_smirnov: 0.05, - aic: 100.0, - bic: 105.0, - p_value: 0.15, - }, - timestamp: Utc::now(), - confidence_score: 0.85, - }; - - let simulation_result = model.simulate_dependencies(1000, ¶meters); - - assert_eq!(simulation_result.simulated_uniforms.len(), 1000); - assert_eq!(simulation_result.simulated_uniforms[0].len(), model.config.n_factors); - assert!(simulation_result.simulation_time_us > 0); - - // Check that all values are in [0, 1] - for sample in &simulation_result.simulated_uniforms { - for &value in sample { - assert!(value >= 0.0 && value <= 1.0); - } - } - } - - #[test] - fn test_performance_requirements() { - let config = CopulaModelConfig::default(); - let copula_family = CopulaFamily::Gaussian { - correlation_matrix: vec![vec![1.0, 0.3], vec![0.3, 1.0]] - }; - let mut model = NeuralCopulaModel::new(copula_family, config); - - // Create minimal market data for performance test - let asset_data = vec![ - AssetMarketData { - asset_id: "TEST1".to_owned(), - returns: vec![0.01; 100], - mean_return: 0.01, - volatility: 0.02, - skewness: 0.0, - kurtosis: 3.0, - }, - AssetMarketData { - asset_id: "TEST2".to_owned(), - returns: vec![0.008; 100], - mean_return: 0.008, - volatility: 0.018, - skewness: 0.0, - kurtosis: 3.0, - }, - ]; - - let market_data = MarketDataWindow { - asset_data, - window_start: Utc::now(), - window_end: Utc::now(), - n_observations: 100, - }; - - let start = std::time::Instant::now(); - let parameters = model.estimate_parameters(&market_data); - let estimation_time = start.elapsed(); - - // Should complete parameter estimation in <500μs - assert!(estimation_time.as_micros() < 500); - - let start = std::time::Instant::now(); - let simulation = model.simulate_dependencies(1000, ¶meters); - let simulation_time = start.elapsed(); - - // Should complete simulation in <1ms - assert!(simulation_time.as_millis() < 1); - assert!(simulation.simulation_time_us < 1000); - } \ No newline at end of file diff --git a/crates/ml/src/risk/extreme_value_models.rs b/crates/ml/src/risk/extreme_value_models.rs deleted file mode 100644 index 522b7f89b..000000000 --- a/crates/ml/src/risk/extreme_value_models.rs +++ /dev/null @@ -1,228 +0,0 @@ -//! Extreme Value Theory (EVT) Integration with Machine Learning -//! -//! Implements ML-enhanced extreme value models for tail risk estimation and extreme -//! event modeling. Combines classical EVT with neural networks for dynamic parameter -//! estimation and regime-dependent tail behavior modeling. -//! -//! # Features -//! - Generalized Extreme Value (GEV) distribution modeling -//! - Generalized Pareto Distribution (GPD) for peaks-over-threshold -//! - Neural network-based parameter estimation -//! - Time-varying EVT parameters with regime detection -//! - Extreme quantile estimation with uncertainty -//! - Tail risk measures: VaR, Expected Shortfall, Tail Value-at-Risk -//! - Block maxima and peaks-over-threshold approaches -//! -//! # Performance Targets -//! - Parameter estimation: <1ms -//! - Extreme quantile calculation: <100μs -//! - Real-time threshold updates: <200μs -//! - Memory efficiency: <64MB for large datasets - -use std::sync::Arc; - -use chrono::{DateTime, Utc}; -use serde::{Deserialize, Serialize}; - -use super::*; - - - #[test] - fn test_neural_evt_model_creation() { - let config = EVTModelConfig::default(); - let distribution = ExtremeValueDistribution::GEV { - location: 0.0, - scale: 1.0, - shape: 0.1, - }; - - let model = NeuralEVTModel::new(distribution, config); - - assert_eq!(model.parameter_network.architecture.output_size, 3); // GEV has 3 parameters - assert!(model.parameter_history.is_empty()); - } - - #[test] - fn test_block_maxima_extraction() { - let config = EVTModelConfig { - block_size: 5, - ..Default::default() - }; - let distribution = ExtremeValueDistribution::GEV { - location: 0.0, - scale: 1.0, - shape: 0.0, - }; - let model = NeuralEVTModel::new(distribution, config); - - let data = vec![1.0, 3.0, 2.0, 5.0, 1.0, 2.0, 4.0, 1.0, 3.0, 2.0]; - let maxima = model.extract_block_maxima(&data); - - assert_eq!(maxima.len(), 2); - assert_eq!(maxima[0], 5.0); - assert_eq!(maxima[1], 4.0); - } - - #[test] - fn test_threshold_estimation_and_exceedances() { - let config = EVTModelConfig { - threshold_quantile: 0.8, - ..Default::default() - }; - let distribution = ExtremeValueDistribution::GPD { - scale: 1.0, - shape: 0.1, - threshold: 0.0, - }; - let model = NeuralEVTModel::new(distribution, config); - - let data = vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0]; - let threshold = model.estimate_threshold(&data); - let exceedances = model.extract_exceedances(&data, threshold); - - assert!(threshold > 8.0); // 80th percentile should be around 8.8 - assert!(exceedances.len() <= 2); // Only values above threshold - } - - #[test] - fn test_parameter_constraints() { - let config = EVTModelConfig::default(); - let distribution = ExtremeValueDistribution::GEV { - location: 0.0, - scale: 1.0, - shape: 0.0, - }; - let model = NeuralEVTModel::new(distribution, config); - - // Test GEV parameter constraints - let raw_params = vec![-1.0, -0.5, 2.0]; // location, scale, shape - let constrained = model.constrain_parameters(raw_params); - - assert_eq!(constrained[0], -1.0); // location unconstrained - assert!(constrained[1] > 0.0); // scale must be positive - assert!(constrained[2] >= -0.5 && constrained[2] <= 0.5); // shape bounded - } - - #[test] - fn test_quantile_calculation() { - let config = EVTModelConfig::default(); - let distribution = ExtremeValueDistribution::Gumbel { - location: 0.0, - scale: 1.0, - }; - let model = NeuralEVTModel::new(distribution, config); - - let parameters = vec![0.0, 1.0]; // location, scale - let quantile_99 = model.calculate_quantile(0.99, ¶meters, None); - - // For Gumbel(0,1), 99th percentile should be approximately 4.6 - assert!(quantile_99 > 4.0 && quantile_99 < 5.0); - } - - #[test] - fn test_parameter_estimation_with_sufficient_data() { - let config = EVTModelConfig { - min_exceedances: 10, - ..Default::default() - }; - let distribution = ExtremeValueDistribution::GPD { - scale: 1.0, - shape: 0.1, - threshold: 0.0, - }; - let mut model = NeuralEVTModel::new(distribution, config); - - // Generate data with enough exceedances - let data: Vec = (0..1000) - .map(|i| (i as f32 / 100.0).exp()) // Exponential-like data - .collect(); - - let result = model.estimate_parameters(&data); - - assert!(result.is_ok()); - let parameters = result?; - assert_eq!(parameters.parameters.len(), 2); // GPD has 2 parameters - assert!(parameters.threshold.is_some()); - assert!(parameters.n_exceedances.is_some()); - } - - #[test] - fn test_extreme_quantile_calculation() { - let config = EVTModelConfig { - mc_samples: 100, // Reduced for test speed - ..Default::default() - }; - let distribution = ExtremeValueDistribution::GEV { - location: 0.0, - scale: 1.0, - shape: 0.1, - }; - let model = NeuralEVTModel::new(distribution, config); - - let parameters = EVTParameters { - parameters: vec![0.0, 1.0, 0.1], - confidence_intervals: vec![(-0.1, 0.1), (0.9, 1.1), (0.05, 0.15)], - threshold: None, - n_exceedances: None, - gof_statistics: EVTGoodnessOfFit { - anderson_darling: 0.5, - kolmogorov_smirnov: 0.08, - cramer_von_mises: 0.12, - p_value: 0.25, - aic: 100.0, - bic: 105.0, - }, - tail_index: 0.1, - return_levels: vec![], - timestamp: Utc::now().timestamp_nanos_opt().unwrap_or(0) as u64, - confidence_score: 0.8, - }; - - let probabilities = vec![0.99, 0.995, 0.999]; - let quantiles = model.calculate_extreme_quantiles(&probabilities, ¶meters); - - assert_eq!(quantiles.len(), 3); - - // Check that extreme quantiles are increasing - assert!(quantiles[0].value < quantiles[1].value); - assert!(quantiles[1].value < quantiles[2].value); - - // Check that return periods are reasonable - assert!(quantiles[0].return_period < quantiles[1].return_period); - assert!(quantiles[2].return_period > 1000.0); // 99.9th percentile - } - - #[test] - fn test_performance_requirements() { - let config = EVTModelConfig { - mc_samples: 100, // Reduced for performance test - ..Default::default() - }; - let distribution = ExtremeValueDistribution::GEV { - location: 0.0, - scale: 1.0, - shape: 0.1, - }; - let mut model = NeuralEVTModel::new(distribution, config); - - // Generate test data - let data: Vec = (0..1000).map(|i| (i as f32).sin()).collect(); - - let start = std::time::Instant::now(); - let result = model.estimate_parameters(&data); - let estimation_time = start.elapsed(); - - // Should complete parameter estimation in <1ms - assert!(estimation_time.as_millis() < 1); - assert!(result.is_ok()); - - let parameters = result?; - - let start = std::time::Instant::now(); - let quantiles = model.calculate_extreme_quantiles(&[0.99], ¶meters); - let quantile_time = start.elapsed(); - - // Should complete quantile calculation in <100μs - assert!(quantile_time.as_micros() < 100); - assert_eq!(quantiles.len(), 1); - } \ No newline at end of file diff --git a/crates/ml/src/risk/lstm_gan_scenarios.rs b/crates/ml/src/risk/lstm_gan_scenarios.rs deleted file mode 100644 index 08b50420e..000000000 --- a/crates/ml/src/risk/lstm_gan_scenarios.rs +++ /dev/null @@ -1,87 +0,0 @@ -//! # LSTM-GAN Stress Scenario Generation -//! -//! Enterprise-grade LSTM-GAN implementation for generating realistic financial stress scenarios. -//! Combines Long Short-Term Memory networks with Generative Adversarial Networks to create -//! plausible, yet unseen, market stress scenarios for comprehensive risk testing. -//! -//! ## Features -//! - Conditional LSTM-GAN for scenario-specific generation -//! - Wasserstein GAN with Gradient Penalty for training stability -//! - Multi-asset, multi-factor scenario generation -//! - Real-time scenario validation and quality scoring -//! - Regulatory-compliant scenario documentation -//! - Production-grade performance: <50ms generation time - -use std::collections::HashMap; - -use chrono::{DateTime, Utc, Duration}; -use ndarray::{Array1, Array2, Array3, ArrayView1, ArrayView2, Axis, concatenate}; -use serde::{Deserialize, Serialize}; - -use crate::{MLResult, MLError}; -use super::*; - -#[cfg(test)] -mod tests { - use super::*; - - - #[test] - fn test_lstm_layer_creation() { - let layer = LSTMLayer::new(100, 128, 0.1); - assert_eq!(layer.input_size, 100); - assert_eq!(layer.hidden_size, 128); - assert_eq!(layer.weight_ih.dim(), (4 * 128, 100)); - assert_eq!(layer.weight_hh.dim(), (4 * 128, 128)); - } - - #[test] - fn test_generator_creation() { - let config = GeneratorConfig::default(); - let generator = LSTMGenerator::new(config); - assert_eq!(generator.lstm_layers.len(), 3); - assert_eq!(generator.config.hidden_size, 256); - } - - #[test] - fn test_condition_encoding() { - let config = GeneratorConfig::default(); - let generator = LSTMGenerator::new(config); - - let condition = ScenarioCondition { - scenario_type: ScenarioType::MarketCrash, - severity: SeverityLevel::Severe, - duration_days: 30, - asset_ids: vec![AssetId("AAPL".to_owned()), AssetId("MSFT".to_owned())], - macro_factors: HashMap::new(), - target_correlations: None, - custom_constraints: HashMap::new(), - }; - - let encoded = generator.encode_condition(&condition)?; - assert_eq!(encoded.len(), generator.config.condition_dim); - assert_eq!(encoded[0], 1.0); // Market crash should be first type - } - - #[test] - fn test_correlation_calculation() { - let config = GeneratorConfig::default(); - let generator = LSTMGenerator::new(config); - - let x = Array1::from_vec(vec![1.0, 2.0, 3.0, 4.0, 5.0]); - let y = Array1::from_vec(vec![2.0, 4.0, 6.0, 8.0, 10.0]); - - let correlation = generator.calculate_correlation(&x.view(), &y.view())?; - assert!((correlation - 1.0).abs() < 1e-10); // Perfect positive correlation - } - - #[test] - fn test_stress_testing_engine() { - let gen_config = GeneratorConfig::default(); - let disc_config = DiscriminatorConfig::default(); - let engine = StressTestingEngine::new(gen_config, disc_config); - - assert_eq!(engine.scenario_library.len(), 0); - assert!(engine.config.quality_threshold > 0.0); - } -} \ No newline at end of file diff --git a/crates/ml/src/risk/metrics.rs b/crates/ml/src/risk/metrics.rs deleted file mode 100644 index f7ac3497e..000000000 --- a/crates/ml/src/risk/metrics.rs +++ /dev/null @@ -1,46 +0,0 @@ -//! Risk metrics and portfolio analytics - -use chrono::{DateTime, Utc}; -use serde::{Deserialize, Serialize}; - -/// Portfolio-level risk metrics -#[derive(Debug, Clone, Serialize, Deserialize)] -/// PortfolioMetrics component. -pub struct PortfolioMetrics { - pub max_drawdown: f64, - pub current_drawdown: f64, - pub sharpe_ratio: f64, - pub sortino_ratio: f64, - pub calmar_ratio: f64, - pub beta: f64, - pub tracking_error: f64, - pub concentration_risk: f64, - pub correlation_risk: f64, - pub liquidity_risk: f64, -} - -/// Real-time risk metrics for monitoring -#[derive(Debug, Clone, Serialize, Deserialize)] -/// RealTimeMetrics component. -pub struct RealTimeMetrics { - pub portfolio_var_estimate: f64, - pub max_position_risk: f64, - pub correlation_risk: f64, - pub liquidity_risk: f64, - pub circuit_breaker_triggered: bool, - pub active_triggers: Vec, - pub processing_time_micros: u64, - pub timestamp: DateTime, -} - -/// General risk metrics -#[derive(Debug, Clone, Serialize, Deserialize)] -/// RiskMetrics component. -pub struct RiskMetrics { - pub var_95: f64, - pub var_99: f64, - pub expected_shortfall: f64, - pub volatility: f64, - pub correlation: f64, - pub timestamp: DateTime, -} \ No newline at end of file diff --git a/crates/ml/src/risk/mod.rs b/crates/ml/src/risk/mod.rs index 12cd12e8a..5d600a9f9 100644 --- a/crates/ml/src/risk/mod.rs +++ b/crates/ml/src/risk/mod.rs @@ -1,210 +1,8 @@ -//! # Neural Risk Management System -//! -//! Advanced ML-driven risk management with neural VaR models, Kelly criterion optimization, -//! and real-time regime detection for production HFT systems using canonical types. +//! Neural Risk Management — thin facade re-exporting from ml-risk crate. -pub mod circuit_breakers; -pub mod graph_risk_model; -pub mod kelly_optimizer; -pub mod kelly_position_sizing_service; -pub mod position_sizing; -pub mod slippage; -pub mod var_models; +pub use ml_risk::*; -// Export types from modules that actually exist -pub use circuit_breakers::MLCircuitBreaker; -pub use kelly_optimizer::{ - KellyCriterionOptimizer, KellyOptimizerConfig, KellyPositionRecommendation, -}; -pub use position_sizing::PositionSizingNetwork; -pub use slippage::{FixedCostModel, LinearImpactConfig, LinearImpactModel, MarketImpactModel}; -pub use var_models::{NeuralVarConfig, NeuralVarModel}; - -// Export graph risk model types from TGNN module +// Bridge re-exports from tgnn (stays in ml, can't move to ml-risk) pub use crate::tgnn::gating::GatingMechanism; pub use crate::tgnn::graph::MarketGraph; pub use crate::tgnn::message_passing::MessagePassing; - -use std::collections::HashMap; - -use chrono::{DateTime, Utc}; -use ndarray::Array2; -use serde::{Deserialize, Serialize}; - -use crate::MLResult; -use common::{Price, Volume}; - -// AssetId type for risk management -#[derive(Debug, Clone, Serialize, Deserialize, Hash, PartialEq, Eq)] -pub struct AssetId(String); - -/// Risk assessment levels -/// RiskLevel component. -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, PartialOrd)] -pub enum RiskLevel { - VeryLow, - Low, - Medium, - High, - VeryHigh, - Critical, -} - -/// Portfolio risk profile -/// RiskProfile component. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct RiskProfile { - pub total_var_95: f64, // 1-day 95% VaR - pub total_var_99: f64, // 1-day 99% VaR - pub expected_shortfall: f64, // Expected tail loss - pub maximum_drawdown: f64, // Historical maximum drawdown - pub current_drawdown: f64, // Current drawdown from peak - pub sharpe_ratio: f64, // Risk-adjusted return - pub sortino_ratio: f64, // Downside risk-adjusted return - pub calmar_ratio: f64, // Return over maximum drawdown - pub beta: f64, // Market beta - pub tracking_error: f64, // Volatility vs benchmark - pub concentration_risk: f64, // Single position concentration - pub correlation_risk: f64, // Average correlation risk - pub liquidity_risk: f64, // Liquidity-adjusted risk - pub regime_risk: f64, // Market regime risk factor - pub overall_risk_score: f64, // Combined ML risk score (0-1) - pub risk_level: RiskLevel, // Categorical risk assessment - pub timestamp: DateTime, -} - -/// Position-level risk metrics -/// PositionRisk component. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct PositionRisk { - pub asset_id: AssetId, - pub position_size: f64, // Current position size - pub market_value: f64, // Current market value - pub var_contribution: f64, // Contribution to portfolio VaR - pub marginal_var: f64, // Marginal VaR (change in portfolio VaR) - pub component_var: f64, // Component VaR (allocation of portfolio VaR) - pub standalone_var: f64, // Position VaR in isolation - pub beta_to_portfolio: f64, // Beta relative to portfolio - pub correlation_risk: f64, // Correlation with other positions - pub liquidity_horizon: f64, // Days to liquidate position - pub concentration_weight: f64, // Weight in portfolio - pub stress_loss: f64, // Loss under stress scenarios - pub kelly_optimal_size: f64, // Kelly optimal position size - pub recommended_size: f64, // ML recommended position size - pub risk_score: f64, // Individual risk score (0-1) - pub timestamp: DateTime, -} - -/// `Market` data for risk calculations -/// MarketData component. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct MarketData { - pub timestamp: DateTime, - pub prices: HashMap, - pub volumes: HashMap, - pub volatilities: HashMap, - pub correlations: Array2, - pub market_cap_weights: HashMap, - pub sector_exposures: HashMap, -} - -/// Risk limits and constraints -/// RiskLimits component. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct RiskLimits { - pub max_portfolio_var: f64, // Maximum portfolio VaR - pub max_position_size: f64, // Maximum single position size - pub max_sector_exposure: f64, // Maximum sector exposure - pub max_correlation: f64, // Maximum position correlation - pub max_drawdown: f64, // Maximum allowed drawdown - pub min_liquidity_days: f64, // Minimum liquidity (days to exit) - pub max_leverage: f64, // Maximum portfolio leverage - pub var_limit_buffer: f64, // VaR limit buffer (e.g., 0.8 of limit) -} - -impl Default for RiskLimits { - fn default() -> Self { - Self { - max_portfolio_var: 0.02, // 2% daily VaR limit - max_position_size: 0.05, // 5% maximum position size - max_sector_exposure: 0.20, // 20% maximum sector exposure - max_correlation: 0.70, // 70% maximum correlation - max_drawdown: 0.10, // 10% maximum drawdown - min_liquidity_days: 2.0, // 2 days maximum liquidation time - max_leverage: 3.0, // 3x maximum leverage - var_limit_buffer: 0.80, // Use 80% of VaR limit - } - } -} - -/// Comprehensive risk configuration -/// RiskConfig component. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct RiskConfig { - pub var_confidence_levels: Vec, - pub lookback_days: usize, - pub monte_carlo_simulations: usize, - pub stress_scenarios: usize, - pub regime_detection_window: usize, - pub update_frequency_seconds: u32, - pub enable_neural_var: bool, - pub enable_regime_detection: bool, - pub enable_ml_position_sizing: bool, - pub enable_dynamic_hedging: bool, - pub risk_limits: RiskLimits, -} - -impl Default for RiskConfig { - fn default() -> Self { - Self { - var_confidence_levels: vec![0.95, 0.99], - lookback_days: 252, - monte_carlo_simulations: 10_000, - stress_scenarios: 1_000, - regime_detection_window: 60, - update_frequency_seconds: 60, - enable_neural_var: true, - enable_regime_detection: true, - enable_ml_position_sizing: true, - enable_dynamic_hedging: true, - risk_limits: RiskLimits::default(), - } - } -} - -/// Main neural risk management system -#[derive(Debug)] -pub struct NeuralRiskManager { - config: RiskConfig, - var_model: NeuralVarModel, - kelly_optimizer: KellyCriterionOptimizer, - position_sizer: PositionSizingNetwork, - circuit_breaker: MLCircuitBreaker, -} - -impl NeuralRiskManager { - /// Create new neural risk management system - pub fn new(config: RiskConfig) -> MLResult { - let var_config = NeuralVarConfig { - confidence_levels: config.var_confidence_levels.clone(), - lookback_days: config.lookback_days, - lookback_period: config.lookback_days, - monte_carlo_simulations: config.monte_carlo_simulations, - enable_stress_testing: true, - hidden_layers: vec![128, 64, 32], - }; - - let var_model = NeuralVarModel::new(var_config)?; - let kelly_optimizer = KellyCriterionOptimizer::new(Default::default())?; - let position_sizer = PositionSizingNetwork::new(Default::default())?; - let circuit_breaker = MLCircuitBreaker::new(Default::default())?; - - Ok(Self { - config, - var_model, - kelly_optimizer, - position_sizer, - circuit_breaker, - }) - } -} diff --git a/crates/ml/src/risk/monitor.rs b/crates/ml/src/risk/monitor.rs deleted file mode 100644 index b47b0dc16..000000000 --- a/crates/ml/src/risk/monitor.rs +++ /dev/null @@ -1,184 +0,0 @@ -//! Real-time Position and Risk Monitoring -//! -//! High-frequency monitoring system for positions, exposures, and risk metrics -//! with sub-millisecond update capabilities and automatic alerting. - -use std::collections::HashMap; -use std::sync::Arc; -use std::sync::atomic::{AtomicU64, AtomicBool, Ordering}; - -use chrono::{DateTime, Utc}; -use serde::{Deserialize, Serialize}; -use tokio::sync::{RwLock, mpsc, broadcast}; -use tokio::time::{interval, Duration, Instant}; -use tracing::{info, warn, error, debug}; - -// CIRCULAR DEPENDENCY FIX: Remove risk module dependency -// IMPLEMENTED: Local type definitions below (Symbol and Position) -use crate::MLError; - -// Use MLError directly - no compatibility wrapper needed -pub type VarResult = Result; - -// Local type definitions for monitor-specific usage -// These are simplified versions focused on monitoring needs -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub struct Symbol { - value: [u8; 16], // Fixed-size for performance -} - -impl Symbol { - pub fn new(s: String) -> Self { - let mut value = [0_u8; 16]; - let bytes = s.as_bytes(); - let len = bytes.len().min(16); - value[..len].copy_from_slice(&bytes[..len]); - Self { value } - } -} - -#[derive(Debug, Clone)] -pub struct Position { - pub quantity: f64, - pub current_price: f64, - pub avg_price: Option, -} - -/// Real-time monitor configuration -#[derive(Debug, Clone)] -pub struct MonitorConfig { - pub update_interval_ms: u64, - pub alert_threshold: f64, -} - -impl Default for MonitorConfig { - fn default() -> Self { - Self { - update_interval_ms: 100, - alert_threshold: 0.05, - } - } -} - -// NO RE-EXPORTS - Use explicit imports: common::Position - -/// Exposure metrics -#[derive(Debug, Clone)] -pub struct ExposureMetrics { - pub total_gross_exposure: f64, - pub total_net_exposure: f64, -} - -/// Real-time monitor -#[derive(Debug)] -pub struct RealTimeMonitor { - config: MonitorConfig, - positions: Arc>>, - is_active: AtomicBool, -} - -impl RealTimeMonitor { - pub fn new(config: MonitorConfig) -> (Self, mpsc::Receiver) { - let (tx, rx) = mpsc::channel(100); - let monitor = Self { - config, - positions: Arc::new(RwLock::new(HashMap::new())), - is_active: AtomicBool::new(false), - }; - (monitor, rx) - } - - pub fn is_monitoring_active(&self) -> bool { - self.is_active.load(Ordering::Relaxed) - } - - pub async fn update_position( - &self, - asset_id: Symbol, - quantity: f64, - current_price: f64, - avg_price: Option - ) -> Result<(), MLError> { - let mut positions = self.positions.write().await; - positions.insert(asset_id, Position { - quantity, - current_price, - avg_price, - }); - Ok(()) - } - - pub async fn get_position(&self, asset_id: Symbol) -> Option { - let positions = self.positions.read().await; - positions.get(&asset_id).cloned() - } - - pub async fn trigger_immediate_risk_update(&self) { - // Trigger risk calculations - } - - pub async fn get_exposure_metrics(&self) -> ExposureMetrics { - let positions = self.positions.read().await; - let gross_exposure: f64 = positions.values() - .map(|p| (p.quantity * p.current_price).abs()) - .sum(); - let net_exposure: f64 = positions.values() - .map(|p| p.quantity * p.current_price) - .sum(); - - ExposureMetrics { - total_gross_exposure: gross_exposure, - total_net_exposure: net_exposure, - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use tokio::test; - - - #[test] - async fn test_monitor_creation() { - let config = MonitorConfig::default(); - let (monitor, _receiver) = RealTimeMonitor::new(config); - assert!(!monitor.is_monitoring_active()); - } - - #[test] - async fn test_position_update() { - let config = MonitorConfig::default(); - let (monitor, _receiver) = RealTimeMonitor::new(config); - - let asset_id = Symbol::new("TEST".to_owned()); - let result = monitor.update_position(asset_id, 100.0, 50.0, Some(49.0)).await; - assert!(result.is_ok()); - - let position = monitor.get_position(asset_id).await; - assert!(position.is_some()); - - let pos = position?; - assert_eq!(pos.quantity, 100.0); - assert_eq!(pos.current_price, 50.0); - } - - #[test] - async fn test_exposure_calculation() { - let config = MonitorConfig::default(); - let (monitor, _receiver) = RealTimeMonitor::new(config); - - // Add some test positions - let asset1 = Symbol::new("TEST1".to_owned()); - let asset2 = Symbol::new("TEST2".to_owned()); - - monitor.update_position(asset1, 100.0, 50.0, Some(49.0)).await?; - monitor.update_position(asset2, -50.0, 100.0, Some(102.0)).await?; - - // Trigger metrics update - monitor.trigger_immediate_risk_update().await; - - let exposures = monitor.get_exposure_metrics().await; - assert!(exposures.total_gross_exposure > 0.0); - } -} \ No newline at end of file diff --git a/crates/ml/src/risk/tests/kelly_warmup_tests.rs b/crates/ml/src/risk/tests/kelly_warmup_tests.rs deleted file mode 100644 index e7067d62e..000000000 --- a/crates/ml/src/risk/tests/kelly_warmup_tests.rs +++ /dev/null @@ -1,383 +0,0 @@ -//! Tests for Kelly Criterion warmup signal leakage prevention -//! -//! These tests verify that the concentration penalty logic: -//! 1. Does not use hardcoded thresholds that models can memorize -//! 2. Respects Kelly's warmup period and statistical confidence -//! 3. Prevents signal leakage from future-looking information - -use super::super::kelly_position_sizing_service::{ - KellyPositionSizingService, KellyServiceConfig, PositionSizingRequest, PositionTracker, - RiskTolerance, -}; -use crate::risk::KellyOptimizerConfig; -use common::types::Price; - -/// Create a test service with default configuration -fn create_test_service() -> KellyPositionSizingService { - let config = KellyServiceConfig::default(); - let position_tracker = PositionTracker::new(); - - tokio::runtime::Runtime::new() - .unwrap() - .block_on(KellyPositionSizingService::new(config, position_tracker)) - .expect("Failed to create test service") -} - -/// Create a test service with custom warmup configuration -fn create_custom_test_service( - warmup_sample_size: usize, - warmup_min_penalty: f64, - confidence_penalty_range: f64, -) -> KellyPositionSizingService { - let mut config = KellyServiceConfig::default(); - config.kelly_warmup_sample_size = warmup_sample_size; - config.warmup_min_penalty = warmup_min_penalty; - config.confidence_penalty_range = confidence_penalty_range; - - let position_tracker = PositionTracker::new(); - - tokio::runtime::Runtime::new() - .unwrap() - .block_on(KellyPositionSizingService::new(config, position_tracker)) - .expect("Failed to create custom test service") -} - -#[tokio::test] -async fn test_concentration_penalty_warmup_progression() { - let service = create_test_service(); - - // Test warmup progression (0-20 trades) - let test_cases = vec![ - (0, 0.7, "Zero samples"), - (5, 0.7, "Early warmup"), - (10, 0.7, "Mid warmup"), - (15, 0.7, "Late warmup"), - (20, 0.7, "Warmup complete"), - ]; - - let mut previous_penalty = 0.0; - - for (sample_size, confidence, description) in test_cases { - let penalty = service - .apply_concentration_limits( - 0.1, // fraction - 0.05, // current_allocation - 0.6, // portfolio_concentration (> 0.5 to trigger penalty) - confidence, // kelly_confidence - sample_size, // kelly_sample_size - ) - .expect(&format!("Failed to calculate penalty for {}", description)); - - // Verify penalty is within expected warmup range - if sample_size < 20 { - assert!( - penalty >= 0.5, - "{}: Penalty {:.3} should be >= 0.5 (warmup_min_penalty)", - description, - penalty - ); - assert!( - penalty <= 1.0, - "{}: Penalty {:.3} should be <= 1.0", - description, - penalty - ); - - // Verify monotonic increase during warmup - if sample_size > 0 { - assert!( - penalty >= previous_penalty, - "{}: Penalty should increase with sample size (current: {:.3}, previous: {:.3})", - description, - penalty, - previous_penalty - ); - } - } - - previous_penalty = penalty; - } -} - -#[tokio::test] -async fn test_concentration_penalty_confidence_scaling() { - let service = create_test_service(); - - // Test post-warmup confidence scaling - let confidence_levels = vec![ - (0.5, "Low confidence"), - (0.7, "Medium confidence"), - (0.85, "High confidence"), - (0.95, "Very high confidence"), - ]; - - for (confidence, description) in confidence_levels { - let penalty = service - .apply_concentration_limits( - 0.1, // fraction - 0.05, // current_allocation - 0.6, // portfolio_concentration (> 0.5) - confidence, - 25, // Post-warmup (> 20) - ) - .expect(&format!( - "Failed to calculate penalty for {}", - description - )); - - // Calculate expected penalty: base_penalty + (confidence * range) - // With defaults: 0.8 + (confidence * 0.20) - let expected = 0.8 + (confidence * 0.20); - - assert!( - (penalty - expected).abs() < 0.01, - "{}: Expected penalty {:.3}, got {:.3}", - description, - expected, - penalty - ); - - // Verify penalty scales with confidence - assert!( - penalty >= 0.8, - "{}: Penalty {:.3} should be >= base_penalty (0.8)", - description, - penalty - ); - assert!( - penalty <= 1.0, - "{}: Penalty {:.3} should be <= 1.0", - description, - penalty - ); - } -} - -#[tokio::test] -async fn test_no_hardcoded_thresholds() { - // Verify no magic numbers in concentration penalty logic - let service = create_test_service(); - - let mut penalties = Vec::new(); - - for sample_size in 0..30 { - let confidence = (sample_size as f64 / 30.0).min(1.0); - let penalty = service - .apply_concentration_limits( - 0.1, // fraction - 0.05, // current_allocation - 0.6, // portfolio_concentration - confidence, - sample_size, - ) - .expect("Failed to calculate penalty"); - - penalties.push((penalty * 1000.0) as i64); - } - - // Convert to set to count unique values - let unique_penalties: std::collections::HashSet<_> = penalties.iter().cloned().collect(); - - // Should have significant variation, not constant values - // With 30 samples, we should have at least 15 unique penalties - assert!( - unique_penalties.len() >= 15, - "Expected at least 15 unique penalties during 30-sample progression, got {}. \ - This suggests hardcoded thresholds.", - unique_penalties.len() - ); - - // Verify penalties are not clustered around the old hardcoded 0.8 value - let near_old_threshold = penalties - .iter() - .filter(|&&p| (p - 800).abs() < 10) // Within 1% of 0.8 - .count(); - - assert!( - near_old_threshold < penalties.len() / 3, - "Too many penalties ({}/{}) clustered near old hardcoded 0.8 threshold. \ - Expected dynamic scaling.", - near_old_threshold, - penalties.len() - ); -} - -#[tokio::test] -async fn test_kelly_warmup_prevents_signal_leakage() { - let service = create_test_service(); - - // Simulate training scenario with gradual data accumulation - let mut recommendations = Vec::new(); - - for epoch in 0..50 { - let sample_size = epoch / 2; // Gradual data accumulation - let confidence = (sample_size as f64 / 20.0).min(0.95); - - let penalty = service - .apply_concentration_limits( - 0.1, // fraction - 0.05, // current_allocation - 0.6, // portfolio_concentration - confidence, - sample_size, - ) - .expect("Failed to calculate penalty"); - - recommendations.push((sample_size, (penalty * 10000.0) as i64)); - } - - // Verify: No repeated fractions during warmup period - let warmup_penalties: Vec = recommendations - .iter() - .filter(|(size, _)| *size < 20) - .map(|(_, penalty)| *penalty) - .collect(); - - let unique_warmup: std::collections::HashSet<_> = - warmup_penalties.iter().cloned().collect(); - - // During warmup, should have variety not constant values - // With ~20 warmup epochs, should have at least 10 unique penalties - assert!( - unique_warmup.len() >= 10, - "Expected at least 10 unique penalties during warmup (20 samples), got {}. \ - This suggests signal leakage from hardcoded values.", - unique_warmup.len() - ); - - // Verify penalties increase monotonically during warmup - let mut warmup_monotonic = true; - for i in 1..warmup_penalties.len() { - if warmup_penalties[i] < warmup_penalties[i - 1] { - warmup_monotonic = false; - break; - } - } - - assert!( - warmup_monotonic, - "Penalties should increase monotonically during warmup to prevent memorization" - ); -} - -#[tokio::test] -async fn test_concentration_penalty_temporal_safety() { - let service = create_test_service(); - - // Verify penalty at time T doesn't depend on data from T+1 - let penalty_t0 = service - .apply_concentration_limits( - 0.1, // fraction - 0.05, // current_allocation - 0.6, // portfolio_concentration - 0.7, // confidence - 10, // sample_size - ) - .expect("Failed to calculate penalty at T0"); - - // Simulate "future" data accumulation (should not affect past penalty) - // In real scenario, more trades would accumulate - // But re-computing with same inputs should give same result - - let penalty_t0_recomputed = service - .apply_concentration_limits( - 0.1, // Same inputs - 0.05, 0.6, 0.7, 10, - ) - .expect("Failed to recompute penalty at T0"); - - // Penalty should be identical (no look-ahead bias) - assert_eq!( - (penalty_t0 * 10000.0) as i64, - (penalty_t0_recomputed * 10000.0) as i64, - "Penalty changed when recomputed with same inputs - suggests temporal leakage" - ); -} - -#[tokio::test] -async fn test_low_concentration_no_penalty() { - let service = create_test_service(); - - // Test that low concentration (< 0.5) results in no penalty - let penalty = service - .apply_concentration_limits( - 0.1, // fraction - 0.05, // current_allocation - 0.3, // portfolio_concentration (< 0.5, should trigger no penalty) - 0.7, // confidence - 10, // sample_size - ) - .expect("Failed to calculate penalty for low concentration"); - - // Should return the input fraction without penalty adjustment - // (after applying max allocation limits) - assert!( - (penalty - 0.1).abs() < 0.001, - "Expected no penalty for low concentration, got penalty factor {:.3}", - penalty - ); -} - -#[tokio::test] -async fn test_warmup_configuration_customization() { - // Test that custom warmup configuration is respected - let service = create_custom_test_service( - 30, // warmup_sample_size - 0.4, // warmup_min_penalty - 0.25, // confidence_penalty_range - ); - - // Test warmup period extends to 30 samples - let penalty_at_25 = service - .apply_concentration_limits(0.1, 0.05, 0.6, 0.7, 25) - .expect("Failed to calculate penalty"); - - // Should still be in warmup mode (< 30) - let expected_warmup_progress = 25.0 / 30.0; - let expected_penalty = 0.4 + (0.6 * expected_warmup_progress); - - assert!( - (penalty_at_25 - expected_penalty).abs() < 0.01, - "Custom warmup configuration not respected: expected {:.3}, got {:.3}", - expected_penalty, - penalty_at_25 - ); - - // Test post-warmup uses custom confidence range - let penalty_at_35 = service - .apply_concentration_limits(0.1, 0.05, 0.6, 0.8, 35) - .expect("Failed to calculate penalty"); - - // Should be post-warmup (>= 30) - let expected_post_warmup = 0.75 + (0.8 * 0.25); // base + (confidence * range) - - assert!( - (penalty_at_35 - expected_post_warmup).abs() < 0.01, - "Custom confidence range not respected: expected {:.3}, got {:.3}", - expected_post_warmup, - penalty_at_35 - ); -} - -#[tokio::test] -async fn test_zero_sample_size_handling() { - let service = create_test_service(); - - // Verify safe handling of zero sample size (start of training) - let penalty = service - .apply_concentration_limits( - 0.1, // fraction - 0.05, // current_allocation - 0.6, // portfolio_concentration - 0.0, // confidence (should be 0 with no samples) - 0, // sample_size (zero) - ) - .expect("Failed to handle zero sample size"); - - // Should apply most conservative warmup penalty - assert!( - (penalty - 0.5).abs() < 0.01, - "Expected minimum warmup penalty (0.5) for zero samples, got {:.3}", - penalty - ); -} diff --git a/crates/ml/src/validation/adapters.rs b/crates/ml/src/validation/adapters.rs index cd840597f..c5f2b381a 100644 --- a/crates/ml/src/validation/adapters.rs +++ b/crates/ml/src/validation/adapters.rs @@ -7,7 +7,7 @@ use std::cell::RefCell; use std::cmp::min; use crate::dqn::{DQNConfig, Experience, DQN}; -use crate::validation::types::{TimeSeriesData, ValidatableStrategy}; +use ml_validation::{TimeSeriesData, ValidatableStrategy}; use crate::MLError; /// Adapter that wraps a [`DQN`] agent behind the [`ValidatableStrategy`] trait. diff --git a/crates/ml/src/validation/harness.rs b/crates/ml/src/validation/harness.rs index 67ffbbfed..17e685c27 100644 --- a/crates/ml/src/validation/harness.rs +++ b/crates/ml/src/validation/harness.rs @@ -12,12 +12,13 @@ use serde::{Deserialize, Serialize}; use crate::dqn::RegimeType; use crate::MLError; -use super::{ - deflated_sharpe_ratio, excess_kurtosis, per_regime_breakdown, permutation_test, +use ml_validation::{ + deflated_sharpe_ratio, excess_kurtosis, permutation_test, probability_of_backtest_overfitting, sharpe_ratio, skewness, walk_forward_split, - DegradationReport, DsrResult, PboResult, PermutationResult, RegimeMetrics, TimeSeriesData, + DegradationReport, DsrResult, PboResult, PermutationResult, TimeSeriesData, ValidatableStrategy, WalkForwardConfig, }; +use super::{per_regime_breakdown, RegimeMetrics}; // --------------------------------------------------------------------------- // ValidationVerdict diff --git a/crates/ml/src/validation/mod.rs b/crates/ml/src/validation/mod.rs index 55939c89a..08de9b454 100644 --- a/crates/ml/src/validation/mod.rs +++ b/crates/ml/src/validation/mod.rs @@ -1,26 +1,17 @@ //! Statistical and Financial Validation Module //! -//! Provides two categories of validation: -//! - Financial validation (price/volume/quantity sanity checks) -//! - Statistical validation (walk-forward, DSR, PBO, permutation tests) +//! Facade re-exporting core algorithms from `ml-validation` plus bridge +//! modules that depend on DQN/PPO types internal to this crate. +// Re-export everything from the ml-validation sub-crate. +pub use ml_validation::*; + +// Bridge modules that stay in ml (DQN/PPO coupling) pub mod adapters; -pub mod degradation; pub mod financial; pub mod harness; -pub mod noise; pub mod ppo_adapter; pub mod regime_analysis; -pub mod statistical; -pub mod temporal_guard; -pub mod types; -pub mod walk_forward; - -// Re-export financial validation for backward compatibility -pub use financial::{ - validate_model_basic, validate_model_comprehensive, validate_type_conversions, - FinancialValidationResult, ValidationResult, -}; // Re-export DQN strategy adapter pub use adapters::DqnStrategy; @@ -28,30 +19,14 @@ pub use adapters::DqnStrategy; // Re-export PPO strategy adapters pub use ppo_adapter::{PpoLstmStrategy, PpoStrategy}; -// Re-export statistical validation types -pub use types::{TimeSeriesData, ValidatableStrategy}; - -// Re-export walk-forward cross-validation splitter -pub use walk_forward::{walk_forward_split, Fold, WalkForwardConfig}; - // Re-export per-regime performance analysis pub use regime_analysis::{per_regime_breakdown, RegimeMetrics}; // Re-export validation harness pub use harness::{ValidationHarness, ValidationHarnessConfig, ValidationReport, ValidationVerdict}; -// Re-export statistical validation tests -pub use statistical::{ - deflated_sharpe_ratio, excess_kurtosis, normal_cdf, normal_ppf, permutation_test, - probability_of_backtest_overfitting, sharpe_ratio, skewness, DsrResult, PboResult, - PermutationResult, +// Re-export financial validation for backward compatibility +pub use financial::{ + validate_model_basic, validate_model_comprehensive, validate_type_conversions, + FinancialValidationResult, ValidationResult, }; - -// Re-export temporal guard types -pub use temporal_guard::{LeakageAuditReport, NormalizationStats, TemporalGuard}; - -// Re-export degradation tracking types -pub use degradation::{DegradationReport, DegradationTracker, FoldMetrics}; - -// Re-export noise injection for robustness testing -pub use noise::{compute_robustness_score, NoiseConfig, NoiseInjector}; diff --git a/crates/ml/src/validation/ppo_adapter.rs b/crates/ml/src/validation/ppo_adapter.rs index 4b15bc761..e8886f04f 100644 --- a/crates/ml/src/validation/ppo_adapter.rs +++ b/crates/ml/src/validation/ppo_adapter.rs @@ -20,7 +20,7 @@ use crate::ppo::gae::GAEConfig; use crate::ppo::hidden_state_manager::HiddenStateManager; use crate::ppo::ppo::{ActorNetwork, CriticNetwork, PPOConfig, PPO}; use crate::ppo::trajectories::{Trajectory, TrajectoryBatch, TrajectoryStep}; -use crate::validation::types::{TimeSeriesData, ValidatableStrategy}; +use ml_validation::{TimeSeriesData, ValidatableStrategy}; use crate::MLError; // --------------------------------------------------------------------------- diff --git a/crates/ml/src/validation/regime_analysis.rs b/crates/ml/src/validation/regime_analysis.rs index 5cf4d412c..189708e9c 100644 --- a/crates/ml/src/validation/regime_analysis.rs +++ b/crates/ml/src/validation/regime_analysis.rs @@ -8,7 +8,7 @@ use std::collections::HashMap; use serde::{Deserialize, Serialize}; use crate::dqn::RegimeType; -use super::statistical::sharpe_ratio; +use ml_validation::statistical::sharpe_ratio; /// Per-regime performance metrics. #[derive(Debug, Clone, Serialize, Deserialize)]