foxhunt/crates/ml/src/lib.rs

#![deny(clippy::unwrap_used, clippy::expect_used)]
#![cfg_attr(test, allow(
    clippy::unwrap_used,
    clippy::expect_used,
    clippy::assertions_on_constants,
    clippy::assertions_on_result_states,
    clippy::double_comparisons,
    clippy::get_unwrap,
    clippy::inconsistent_digit_grouping,
    clippy::let_underscore_must_use,
    clippy::modulo_arithmetic,
    clippy::tests_outside_test_module,
    clippy::unseparated_literal_suffix,
    clippy::use_debug,
    clippy::wildcard_enum_match_arm,
))]
#![allow(dead_code)] // 10 ML model implementations with internal architecture not yet fully wired
#![allow(missing_docs)] // Internal implementation details don't require documentation
#![allow(missing_debug_implementations)] // Not all types need Debug
#![allow(unused_crate_dependencies)] // Dev dependencies not used in lib.rs
#![allow(clippy::float_arithmetic)] // ML operations require float arithmetic
// ML-specific lint overrides: these are intentional domain patterns, not safety issues
#![allow(clippy::non_ascii_literal)] // Box-drawing chars for tables, Greek letters for math
#![allow(clippy::str_to_string)] // Pervasive in ML config/display code, not a safety concern
#![allow(clippy::partial_pub_fields)] // ML model structs mix pub config with private state
#![allow(clippy::multiple_inherent_impl)] // Split impls for readability (core vs Display vs Builder)
#![allow(clippy::same_name_method)] // Trait methods intentionally shadow inherent methods
#![allow(clippy::shadow_reuse)] // Tensor code naturally shadows: let x = relu(x)
#![allow(clippy::shadow_unrelated)] // ML pipeline variable reuse across phases
#![allow(clippy::shadow_same)] // Rebinding after narrowing in match/if-let
// ML domain pedantic lints: these are noise in numerical/ML code, not safety issues
// Individual pedantic lints are managed at workspace level in Cargo.toml
#![allow(clippy::doc_markdown)] // Technical terms (AVX2, SIMD, HFT, etc.) in doc comments
#![allow(clippy::indexing_slicing)] // Tensor/array indexing is pervasive and bounds-checked in context
#![allow(clippy::missing_const_for_fn)] // Const fn not critical for ML model code
#![allow(clippy::module_name_repetitions)] // Module-prefixed types provide clarity (DQNAgent, TFTTrainer)
#![allow(clippy::integer_division)] // Integer division is intentional in batch/epoch calculations
#![allow(clippy::cognitive_complexity)] // ML training loops and model architectures are inherently complex
#![allow(clippy::similar_names)] // ML variables often have similar names (x, xs, x_hat, x_norm)
#![allow(clippy::clone_on_ref_ptr)] // Arc::clone is intentional for shared model state
#![allow(clippy::too_many_lines)] // Complex model implementations need many lines
#![allow(clippy::as_conversions)] // Necessary for tensor dimension/type conversions
#![allow(clippy::cast_precision_loss)] // Acceptable in ML with f32/f64 conversions
#![allow(clippy::cast_possible_truncation)] // Type conversions validated in context
#![allow(clippy::default_numeric_fallback)] // Float/int literals are contextually typed in ML code
#![allow(clippy::arithmetic_side_effects)] // ML math uses checked/saturating where needed
#![allow(clippy::needless_range_loop)] // Index-based loops often clearer for tensor operations
#![allow(clippy::into_iter_on_ref)] // .iter() vs .into_iter() on refs is stylistic
#![allow(clippy::new_without_default)] // Many ML types need config params, Default is inappropriate
#![allow(clippy::manual_let_else)] // if-let pattern preferred in many ML error paths
#![allow(clippy::unnecessary_wraps)] // Result/Option wrapping needed for trait consistency
#![allow(clippy::too_many_arguments)] // ML functions often need many hyperparameters
#![allow(clippy::must_use_candidate)] // Not all ML functions need must_use
#![allow(clippy::missing_errors_doc)] // Internal ML APIs don't need full error documentation
#![allow(clippy::cast_sign_loss)] // Sign loss validated in context
#![allow(clippy::cast_possible_wrap)] // Wrap validated in context
#![allow(clippy::cast_lossless)] // as casts are intentional for ML numeric conversions
#![allow(clippy::unused_async)] // Async needed for trait implementations
#![allow(clippy::match_same_arms)] // Explicit match arms preferred for clarity in ML code
#![allow(clippy::unused_self)] // Self parameter needed for trait consistency
#![allow(clippy::map_err_ignore)] // Error conversion doesn't need original context in ML pipelines
#![allow(clippy::single_match_else)] // Explicit match preferred for clarity in ML code
#![allow(clippy::wildcard_imports)] // Prelude-style imports common in ML modules
#![allow(clippy::unnecessary_cast)] // Explicit casts for tensor dimension/type clarity
#![allow(clippy::undocumented_unsafe_blocks)] // Unsafe blocks documented at usage site
#![allow(clippy::redundant_clone)] // Clones needed for ownership in async/parallel ML pipelines
#![allow(clippy::redundant_closure)] // Explicit closures preferred for readability
#![allow(clippy::type_complexity)] // Complex types unavoidable in ML generics
#![allow(clippy::manual_clamp)] // Explicit min/max preferred in numerical code
#![allow(clippy::clone_on_copy)] // Explicit clone for clarity on Copy types
#![allow(clippy::should_implement_trait)] // Custom builder patterns don't need std traits
#![allow(clippy::derivable_impls)] // Custom Default impls with domain-specific values
#![allow(clippy::useless_conversion)] // Explicit conversions for type clarity
#![allow(clippy::get_first)] // .get(0) preferred for consistency with .get(n)
#![allow(clippy::len_zero)] // .len() == 0 preferred for readability in some contexts
#![allow(clippy::assign_op_pattern)] // Explicit assignment preferred in numerical code
#![allow(clippy::if_same_then_else)] // Intentional identical branches for documentation
#![allow(clippy::unused_enumerate_index)] // Index used in debug/logging contexts
#![allow(clippy::doc_lazy_continuation)] // Doc formatting acceptable
#![allow(clippy::doc_overindented_list_items)] // Doc formatting acceptable
#![allow(clippy::single_char_add_str)] // String building patterns
#![allow(clippy::let_and_return)] // Named return values for clarity
#![allow(clippy::useless_format)] // Explicit format for consistency
#![allow(clippy::manual_div_ceil)] // Explicit ceiling division for clarity
#![allow(clippy::io_other_error)] // IO error construction patterns
#![allow(clippy::manual_range_contains)] // Explicit range checks for clarity
#![allow(clippy::unwrap_or_default)] // Explicit unwrap_or preferred in some contexts
#![allow(clippy::used_underscore_binding)] // Underscore-prefixed bindings used intentionally
#![allow(clippy::trivially_copy_pass_by_ref)] // Pass-by-ref for trait consistency
#![allow(clippy::needless_borrows_for_generic_args)] // Explicit borrows for clarity
#![allow(clippy::needless_borrow)] // Explicit borrows for clarity
#![allow(clippy::missing_safety_doc)] // Safety documented at usage site
#![allow(clippy::module_inception)] // Module name matches parent for re-export patterns
#![allow(clippy::if_not_else)] // Negated conditions preferred in some ML logic
#![allow(clippy::empty_line_after_doc_comments)] // Doc comment formatting acceptable
#![allow(clippy::unnecessary_lazy_evaluations)] // Explicit lazy evaluation for side effects
#![allow(clippy::collapsible_if)] // Separate ifs preferred for readability
#![allow(clippy::question_mark)] // Explicit error handling preferred
#![allow(clippy::op_ref)] // Explicit ref operations for numeric types
#![allow(clippy::iter_kv_map)] // Map iteration patterns
#![allow(clippy::ptr_arg)] // Ptr args for trait consistency
#![allow(clippy::needless_question_mark)] // Explicit ? for error propagation clarity
#![allow(clippy::explicit_auto_deref)] // Explicit derefs for clarity
#![allow(clippy::bind_instead_of_map)] // Bind preferred in some combinator chains
#![allow(clippy::upper_case_acronyms)] // ML acronyms (DQN, PPO, TFT, etc.)
#![allow(clippy::large_types_passed_by_value)] // Large types passed by value for ownership
#![allow(clippy::large_enum_variant)] // Large variants unavoidable in ML model enums
#![allow(clippy::collapsible_else_if)] // Separate else-if preferred for readability
#![allow(clippy::vec_init_then_push)] // Vec init then push for conditional building
#![allow(clippy::implicit_saturating_sub)] // Explicit subtraction preferred
#![allow(clippy::missing_fields_in_debug)] // Custom Debug impls for large ML structs
#![allow(clippy::field_reassign_with_default)] // Field reassignment after Default::default()
#![allow(clippy::len_without_is_empty)] // len() without is_empty() for ML containers
#![allow(clippy::iter_nth)] // .nth() for indexed iteration
#![allow(clippy::items_after_statements)] // Items after statements for local helpers
#![allow(clippy::uninlined_format_args)] // Non-inlined format args for readability
#![allow(clippy::manual_memcpy)] // Explicit loop copy for tensor operations
#![allow(clippy::single_char_lifetime_names)] // Standard Rust lifetime conventions in ODE solvers
#![allow(clippy::same_item_push)] // Intentional repeated pushes for sequence padding
#![allow(clippy::multiple_unsafe_ops_per_block)] // SIMD/hardware ops grouped for clarity
#![allow(clippy::arc_with_non_send_sync)] // Arc used for local-thread inference contexts
#![allow(clippy::format_in_format_args)] // Nested format for dynamic message building
#![allow(clippy::inherent_to_string)] // Custom to_string for display types
#![allow(clippy::redundant_locals)] // Rebinding for clarity in model pipelines
#![allow(clippy::to_string_trait_impl)] // Direct ToString impls for versioning types
#![recursion_limit = "256"] // Required for complex TFT quantile operations
//! Machine Learning Models for Foxhunt
//!
//! This crate provides comprehensive machine learning models and algorithms
//! for the Foxhunt high-frequency trading system. All ML operations use
//! enterprise-grade safety controls to prevent system failures.
//!
//! ## Safety Features
//!
//! - **Comprehensive mathematical safety**: All operations handle NaN/Infinity gracefully
//! - **Tensor bounds checking**: Prevents buffer overflows and memory issues
//! - **Model drift detection**: Automatic monitoring of model performance degradation
//! - **Financial validation**: Ensures all predictions use unified financial types
//! - **Memory management**: Prevents OOM conditions and memory leaks
//! - **Timeout handling**: Prevents hanging operations
//!
//! ## Usage
//!
//! ```no_run
//! // ML safety manager usage example
//! // Note: This is a conceptual example - actual implementation may vary
//! use ml::safety::MLSafetyConfig;
//!
//! #[tokio::main]
//! async fn main() -> Result<(), Box<dyn std::error::Error>> {
//!     // Initialize safety with custom configuration
//!     let _config = MLSafetyConfig::default();
//!
//!     // ML operations would use the safety manager
//!     // (actual implementation details depend on the safety module structure)
//!     Ok(())
//! }
//! ```

#![warn(missing_debug_implementations)]
#![warn(rust_2018_idioms)]
// NOTE: clippy::unwrap_used, clippy::expect_used, clippy::indexing_slicing
// are governed by workspace lints at "warn" level. The remaining lints below
// are already "deny" at workspace level -- kept here for explicitness.
#![deny(clippy::panic, clippy::unimplemented, clippy::unreachable)]

// Re-export all core types from ml-core
pub use ml_core::*;

// Explicit module re-exports from ml-core for backward compatibility.
// `pub use ml_core::*` re-exports items (types, functions) but NOT modules.
// These re-exports ensure paths like `ml::common::X`, `ml::config::X` etc. still work.
pub use ml_core::error;
pub use ml_core::types;
pub use ml_core::common;
pub use ml_core::config;
pub use ml_core::model;
pub use ml_core::traits;

// Compute primitives re-exported from ml-core (task 5b)
pub use ml_core::tensor_ops;
pub use ml_core::state_layout;
pub use ml_core::gpu;
pub use ml_core::safety;
pub use ml_core::memory_optimization;
pub use ml_core::batch_size_resolver;

// Native type replacements for candle (Candle elimination)
pub use ml_core::native_types;
pub use ml_core::cuda_autograd;
pub use ml_core::device;

// Shared infrastructure re-exported from ml-core (task 5e)
pub use ml_core::trading_action;
pub use ml_core::action_space;
pub use ml_core::xavier_init;
pub use ml_core::order_router;
pub use ml_core::portfolio_tracker;

// Metrics & performance re-exported from ml-core (task 5g)
pub use ml_core::metrics;
pub use ml_core::performance;

// Silence unused crate warnings for dependencies used in tests or feature-gated code
use approx as _;
use bincode as _;
use memmap2 as _;
use num as _;
use num_traits as _;
use semver as _;
use tempfile as _;

// Adam optimizer: use ml_core::cuda_autograd::GpuAdamW
// Use ml_core::cuda_autograd::GpuAdamW directly.

// Shared infrastructure modules re-exported from ml-core (see explicit re-exports above)

// ========== CORE ML MODULES ==========
// Core ML modules
pub mod backtesting; // Backtesting framework for barrier optimization
pub mod checkpoint;
// config module re-exported from ml-core (see below)
// cuda_compat re-exported from ml-core (see below)
pub mod cuda_pipeline; // GPU data pre-upload pipeline for DQN/PPO trainers
pub mod data_loaders; // Data loaders for ML training
pub mod deployment; // Model deployment, A/B testing, versioning, hot-swap
pub mod dqn;
// gradient_accumulation re-exported from ml-core (see below)
// gradient_utils re-exported from ml-core (see below)
// gpu re-exported from ml-core (see below)
pub mod diffusion;
pub mod ensemble;
pub mod evaluation; // DQN evaluation engine (backtest metrics, Sharpe ratio)
pub mod flash_attention;
pub mod hyperopt; // Bayesian hyperparameter optimization (egobox)
pub mod integration;
pub mod kan;
pub mod labeling;
pub mod liquid;
pub mod mamba;
// memory_optimization re-exported from ml-core (see explicit re-exports below)
pub mod microstructure;
// optimizers re-exported from ml-core (see below)
pub mod paper_trading;
pub mod ppo;
pub mod preprocessing; // Data preprocessing (log returns, normalization, outlier clipping)
pub mod risk;
// safety re-exported from ml-core (see explicit re-exports below)
pub mod security; // ML security (prediction validation, anomaly detection)
pub mod tft;
pub mod tgnn;
pub mod tlob;
pub mod xlstm;

pub mod trainers; // ML model trainers with gRPC integration
// types module re-exported from ml-core (see below)
pub mod transformers;
pub mod universe;

// ========== INFRASTRUCTURE MODULES ==========
// Infrastructure
pub mod benchmark;
pub mod benchmarks;
// common module re-exported from ml-core (see below)
// metrics re-exported from ml-core (see explicit re-exports above)
pub mod training;

// ========== MODEL DEPLOYMENT AND FACTORY ==========
pub mod model_factory;

// ========== CORE EXPORTS ==========
// error module re-exported from ml-core (see below)
pub mod features; // Feature cache and extraction (Parquet + MinIO)
pub mod feature_cache; // MBP-10 OFI feature caching for hyperopt speedup
pub mod fxcache; // Flat binary feature cache for zero-overhead GPU loading

pub mod inference;
// model module re-exported from ml-core (see below)
// performance re-exported from ml-core (see explicit re-exports above)
pub mod validation;

// ========== ADDITIONAL MODULES ==========
// Additional ML processing modules
pub mod batch_processing; // Batch processing for ML operations
pub mod bridge; // Type system bridge for ML-Financial integration
pub mod portfolio_transformer; // Portfolio-specific transformer
pub mod regime; // Wave D: Structural breaks and regime classification
pub mod regime_detection; // Market regime detection
// tensor_ops re-exported from ml-core (see below)
pub mod observability;
pub mod stress_testing; // Stress testing framework
pub mod training_pipeline; // Complete training pipeline system
// traits module re-exported from ml-core (see below)

// batch_size_resolver re-exported from ml-core
pub mod data_loader;
pub mod training_profile;
pub mod walk_forward;
pub mod data_validation;
pub mod explainability;
pub mod model_registry;
pub mod data_pipeline;
pub mod asset_selection;
pub mod registry; // Operational maturity: model lifecycle (Candidate -> Staging -> Production -> Archived)

// ========== FROM IMPLS FOR MODULE-LOCAL ERROR TYPES ==========
// These reference modules that remain in ml (not moved to ml-core)

// LabelingError → MLError impl moved to ml-labeling crate (orphan rule)

impl From<inference::InferenceError> for MLError {
    fn from(err: inference::InferenceError) -> Self {
        match err {
            inference::InferenceError::GpuRequired { reason } => {
                MLError::ModelError(format!("GPU required: {}", reason))
            },
            inference::InferenceError::ComputationFailed { reason } => {
                MLError::InferenceError(reason)
            },
            inference::InferenceError::FeatureMismatch { expected, actual } => {
                MLError::DimensionMismatch { expected, actual }
            },
            inference::InferenceError::PredictionValidation { reason } => {
                MLError::ValidationError { message: reason }
            },
            inference::InferenceError::HardwareError { reason } => {
                MLError::ModelError(format!("Hardware error: {}", reason))
            },
            other @ inference::InferenceError::ModelNotLoaded { .. }
            | other @ inference::InferenceError::ArchitectureError { .. }
            | other @ inference::InferenceError::TimeoutExceeded { .. }
            | other @ inference::InferenceError::ModelDrift { .. } => {
                MLError::InferenceError(other.to_string())
            },
        }
    }
}

// Implement From<ProductionTrainingError> for MLError
impl From<training_pipeline::ProductionTrainingError> for MLError {
    fn from(err: training_pipeline::ProductionTrainingError) -> Self {
        match err {
            training_pipeline::ProductionTrainingError::ConfigError { reason } => {
                MLError::ConfigError(reason)
            },
            training_pipeline::ProductionTrainingError::ArchitectureError { reason } => {
                MLError::ModelError(format!("Architecture error: {}", reason))
            },
            training_pipeline::ProductionTrainingError::DataError { reason } => {
                MLError::ValidationError {
                    message: format!("Data error: {}", reason),
                }
            },
            training_pipeline::ProductionTrainingError::OptimizationError { reason } => {
                MLError::TrainingError(format!("Optimization error: {}", reason))
            },
            training_pipeline::ProductionTrainingError::FinancialError { reason } => {
                MLError::ValidationError {
                    message: format!("Financial error: {}", reason),
                }
            },
            training_pipeline::ProductionTrainingError::SafetyViolation { reason } => {
                MLError::ValidationError {
                    message: format!("Safety violation: {}", reason),
                }
            },
            training_pipeline::ProductionTrainingError::ConvergenceError { reason } => {
                MLError::TrainingError(format!("Convergence error: {}", reason))
            },
            training_pipeline::ProductionTrainingError::ResourceError { reason } => {
                MLError::ModelError(format!("Resource error: {}", reason))
            },
            training_pipeline::ProductionTrainingError::GpuRequired { reason } => {
                MLError::ModelError(format!("GPU required: {}", reason))
            },
        }
    }
}

// Note: From trait for liquid::LiquidError is implemented in the liquid module to avoid conflicts

// From<ProductionTrainingError> for MLSafetyError lives here because both types
// are accessible: safety is in ml-core, training_pipeline is in ml.
impl From<training_pipeline::ProductionTrainingError> for safety::MLSafetyError {
    fn from(err: training_pipeline::ProductionTrainingError) -> Self {
        match err {
            training_pipeline::ProductionTrainingError::ConfigError { reason } => {
                Self::ValidationError { message: format!("Config error: {}", reason) }
            },
            training_pipeline::ProductionTrainingError::ArchitectureError { reason } => {
                Self::ValidationError { message: format!("Architecture error: {}", reason) }
            },
            training_pipeline::ProductionTrainingError::DataError { reason } => {
                Self::ValidationError { message: format!("Data error: {}", reason) }
            },
            training_pipeline::ProductionTrainingError::OptimizationError { reason } => {
                Self::ValidationError { message: format!("Optimization error: {}", reason) }
            },
            training_pipeline::ProductionTrainingError::FinancialError { reason } => {
                Self::FinancialValidation { reason }
            },
            training_pipeline::ProductionTrainingError::SafetyViolation { reason } => {
                Self::ValidationError { message: format!("Safety violation: {}", reason) }
            },
            training_pipeline::ProductionTrainingError::ConvergenceError { reason } => {
                Self::ValidationError { message: format!("Convergence error: {}", reason) }
            },
            training_pipeline::ProductionTrainingError::ResourceError { reason } => {
                Self::ResourceUnavailable { resource: reason }
            },
            training_pipeline::ProductionTrainingError::GpuRequired { reason } => {
                Self::ResourceUnavailable { resource: format!("GPU: {}", reason) }
            },
        }
    }
}

// ========== LIQUID CONVERSION BRIDGE ==========
// These functions depend on liquid::FixedPoint which lives in ml, not ml-core.

/// Liquid-specific conversion utilities (extends ml-core common::conversions)
pub mod liquid_conversions {
    use ::common::types::Price;

    /// Convert canonical Price to liquid submodule FixedPoint (8-decimal to 6-decimal precision)
    pub fn price_to_liquid_fixed_point(
        price: Price,
    ) -> Result<crate::liquid::FixedPoint, Box<dyn std::error::Error>> {
        let liquid_precision = 1_000_000_i64; // 6 decimal places

        // Scale down from 8-decimal to 6-decimal precision with proper error handling
        let price_f64 = price.to_f64();
        let scaled_value = (price_f64 * liquid_precision as f64) as i64;
        Ok(crate::liquid::FixedPoint(scaled_value))
    }

    /// Convert liquid submodule FixedPoint to canonical Price (6-decimal to 8-decimal precision)
    pub fn liquid_fixed_point_to_price(
        fixed_point: crate::liquid::FixedPoint,
    ) -> Result<Price, Box<dyn std::error::Error>> {
        let liquid_precision = 1_000_000_i64; // 6 decimal places

        // Scale up from 6-decimal to 8-decimal precision with proper error handling
        let value_f64 = fixed_point.0 as f64 / liquid_precision as f64;
        Price::from_f64(value_f64)
            .map_err(|e| format!("Failed to convert f64 to Price: {}", e).into())
    }
}

// ========== ML PRELUDE (extends ml-core prelude with ml-specific items) ==========
/// Prelude module for convenient imports of commonly used ML types
///
/// This module re-exports the most commonly used types and traits from the ML crate
/// to allow users to import everything they need with a single `use ml::prelude::*;`
pub mod prelude {
    // Re-export everything from ml-core's prelude
    pub use ml_core::prelude::*;

    // GPU device management (ml-specific)
    pub use crate::gpu::{capabilities::GpuCapabilities, DeviceConfig};

    // Data pipeline (ml-specific)
    pub use crate::data_pipeline::{DatasetManager, DatasetMode, DatasetSpec, PreparedDataset};

    // Asset selection (ml-specific)
    pub use crate::asset_selection::{ActiveSetSelector, AssetUniverse, PredictabilityScorer};
}

// Tests for FactoredAction <-> TradingAction now live in ml-core::common::action