From afd85b2f8f3af611bc98cee562c5cf6720edda1b Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Fri, 27 Feb 2026 01:33:18 +0100 Subject: [PATCH] chore: clean up examples, update ML binaries and risk tests - Delete 14 unused example files (-3,543 lines): config, adaptive-strategy, data, storage, trading_engine, api_gateway, backtesting, trading_service, chaos - Update ML training/eval binaries: improved CLI args, completion tracking, CUDA test cleanup, hyperopt enhancements - Fix KAN network and TFT module adjustments - Update risk test assertions for consistency - Fix backtesting repositories and promotion manager - Update .serena project config and Cargo dependencies Co-Authored-By: Claude Opus 4.6 --- .serena/project.yml | 13 +- Cargo.lock | 1 - config/examples/runtime_config_example.rs | 144 ------ .../examples/basic_strategy.rs | 67 --- .../examples/ppo_position_sizing_demo.rs | 47 -- crates/data/Cargo.toml | 7 - crates/data/examples/broker_connection.rs | 182 ------- .../data/examples/convert_dbn_to_parquet.rs | 137 ----- .../examples/convert_es_fut_to_parquet.rs | 88 ---- .../examples/convert_nq_fut_to_parquet.rs | 86 ---- crates/data/examples/download_mbp10_data.rs | 289 ----------- crates/ml/data/examples/basic_connection.rs | 76 --- .../ml/examples/baseline_common/completion.rs | 2 + crates/ml/examples/baseline_common/mod.rs | 10 +- crates/ml/examples/cuda_test.rs | 55 +- crates/ml/examples/download_baseline.rs | 11 +- crates/ml/examples/evaluate_baseline.rs | 43 +- crates/ml/examples/evaluate_supervised.rs | 24 +- crates/ml/examples/hyperopt_baseline_rl.rs | 13 +- .../examples/hyperopt_baseline_supervised.rs | 9 + crates/ml/examples/train_baseline_rl.rs | 70 ++- .../ml/examples/train_baseline_supervised.rs | 26 +- crates/ml/src/kan/network.rs | 2 +- crates/ml/src/tft/mod.rs | 94 ++-- .../compliance_breach_detection_tests.rs | 42 +- .../tests/compliance_comprehensive_tests.rs | 56 +- .../risk/tests/compliance_edge_cases_tests.rs | 62 +-- .../emergency_response_comprehensive_tests.rs | 32 +- .../tests/portfolio_optimization_tests.rs | 42 +- .../tests/position_limit_enforcement_tests.rs | 6 +- crates/risk/tests/risk_comprehensive_tests.rs | 24 +- .../tests/var_calculator_edge_cases_tests.rs | 70 +-- .../risk/tests/var_extreme_scenarios_tests.rs | 32 +- crates/risk/tests/var_zero_position_tests.rs | 26 +- .../storage/examples/checkpoint_uploader.rs | 319 ------------ .../examples/event_processing_demo.rs | 348 ------------- .../api_gateway/examples/metrics_example.rs | 161 ------ .../examples/rate_limiter_usage.rs | 208 -------- services/backtesting_service/Cargo.toml | 1 + .../examples/wave_comparison.rs | 57 --- .../backtesting_service/src/repositories.rs | 2 +- .../src/promotion_manager.rs | 6 + .../trading_service/examples/latency_demo.rs | 292 ----------- .../examples/test_ensemble_metrics.rs | 176 ------- testing/integration/chaos/examples/mod.rs | 5 - .../chaos/examples/usage_examples.rs | 484 ------------------ 46 files changed, 403 insertions(+), 3544 deletions(-) delete mode 100644 config/examples/runtime_config_example.rs delete mode 100644 crates/adaptive-strategy/examples/basic_strategy.rs delete mode 100644 crates/adaptive-strategy/examples/ppo_position_sizing_demo.rs delete mode 100644 crates/data/examples/broker_connection.rs delete mode 100644 crates/data/examples/convert_dbn_to_parquet.rs delete mode 100644 crates/data/examples/convert_es_fut_to_parquet.rs delete mode 100644 crates/data/examples/convert_nq_fut_to_parquet.rs delete mode 100644 crates/data/examples/download_mbp10_data.rs delete mode 100644 crates/ml/data/examples/basic_connection.rs delete mode 100644 crates/storage/examples/checkpoint_uploader.rs delete mode 100644 crates/trading_engine/examples/event_processing_demo.rs delete mode 100644 services/api_gateway/examples/metrics_example.rs delete mode 100644 services/api_gateway/examples/rate_limiter_usage.rs delete mode 100644 services/backtesting_service/examples/wave_comparison.rs delete mode 100644 services/trading_service/examples/latency_demo.rs delete mode 100644 services/trading_service/examples/test_ensemble_metrics.rs delete mode 100644 testing/integration/chaos/examples/mod.rs delete mode 100644 testing/integration/chaos/examples/usage_examples.rs diff --git a/.serena/project.yml b/.serena/project.yml index 8a0ba9075..03d7c38c8 100644 --- a/.serena/project.yml +++ b/.serena/project.yml @@ -113,6 +113,15 @@ encoding: utf-8 languages: - rust -# override of the corresponding setting in serena_config.yml, see the documentation there. -# If null or missing, the value from the global config is used. +# time budget (seconds) per tool call for the retrieval of additional symbol information +# such as docstrings or parameter information. +# This overrides the corresponding setting in the global configuration; see the documentation there. +# If null or missing, use the setting from the global configuration. symbol_info_budget: + +# The language backend to use for this project. +# If not set, the global setting from serena_config.yml is used. +# Valid values: LSP, JetBrains +# Note: the backend is fixed at startup. If a project with a different backend +# is activated post-init, an error will be returned. +language_backend: diff --git a/Cargo.lock b/Cargo.lock index cbc40d2ba..2bc445900 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2925,7 +2925,6 @@ dependencies = [ "bincode", "bytes", "chrono", - "clap", "common", "config", "criterion", diff --git a/config/examples/runtime_config_example.rs b/config/examples/runtime_config_example.rs deleted file mode 100644 index 9082be6d7..000000000 --- a/config/examples/runtime_config_example.rs +++ /dev/null @@ -1,144 +0,0 @@ -//! Runtime Configuration Example -//! -//! Demonstrates how to use the RuntimeConfig layer with environment-aware defaults. - -use config::runtime::{Environment, RuntimeConfig}; - -fn main() -> Result<(), Box> { - println!("=== Foxhunt Runtime Configuration Example ===\n"); - - // Example 1: Auto-detect environment from ENVIRONMENT variable - println!("1. Auto-detecting environment:"); - let config = RuntimeConfig::from_env()?; - println!(" Environment: {:?}", config.environment); - println!( - " Database query timeout: {:?}", - config.database.query_timeout - ); - println!(" Position cache TTL: {:?}", config.cache.position_ttl); - println!( - " gRPC request timeout: {:?}", - config.timeouts.grpc_request_timeout - ); - println!(" ML max batch size: {}", config.limits.ml_max_batch_size); - println!(); - - // Example 2: Development environment defaults - println!("2. Development environment defaults:"); - let dev_config = RuntimeConfig::with_defaults(Environment::Development); - println!( - " Database query timeout: {:?} (relaxed for debugging)", - dev_config.database.query_timeout - ); - println!( - " Position cache TTL: {:?} (longer for debugging)", - dev_config.cache.position_ttl - ); - println!( - " Safety check timeout: {:?} (relaxed)", - dev_config.limits.safety_check_timeout - ); - println!(); - - // Example 3: Production environment defaults - println!("3. Production environment defaults:"); - let prod_config = RuntimeConfig::with_defaults(Environment::Production); - println!( - " Database query timeout: {:?} (tight for HFT)", - prod_config.database.query_timeout - ); - println!( - " Position cache TTL: {:?} (short for HFT)", - prod_config.cache.position_ttl - ); - println!( - " Safety check timeout: {:?} (aggressive)", - prod_config.limits.safety_check_timeout - ); - println!(); - - // Example 4: Staging environment (middle ground) - println!("4. Staging environment defaults:"); - let staging_config = RuntimeConfig::with_defaults(Environment::Staging); - println!( - " Database query timeout: {:?}", - staging_config.database.query_timeout - ); - println!( - " Position cache TTL: {:?}", - staging_config.cache.position_ttl - ); - println!( - " Safety check timeout: {:?}", - staging_config.limits.safety_check_timeout - ); - println!(); - - // Example 5: Validation - println!("5. Configuration validation:"); - match config.validate() { - Ok(_) => println!(" โœ“ Configuration is valid"), - Err(e) => println!(" โœ— Configuration error: {}", e), - } - println!(); - - // Example 6: Environment variable override (simulated) - println!("6. Environment variable overrides:"); - println!(" Set DATABASE_QUERY_TIMEOUT_MS=500 to override query timeout"); - println!(" Set CACHE_POSITION_TTL_SECS=30 to override position cache TTL"); - println!(" Set ML_MAX_BATCH_SIZE=16384 to override ML batch size"); - println!(" Set RISK_VAR_CONFIDENCE=0.99 to override VaR confidence"); - println!(); - - // Example 7: Comparing environments - println!("7. Environment comparison (timeouts in ms):"); - println!(" Configuration | Development | Staging | Production"); - println!(" ----------------------- | ----------- | ------- | ----------"); - println!( - " DB Query Timeout | {:>11} | {:>7} | {:>10}", - dev_config.database.query_timeout.as_millis(), - staging_config.database.query_timeout.as_millis(), - prod_config.database.query_timeout.as_millis() - ); - println!( - " Safety Check Timeout | {:>11} | {:>7} | {:>10}", - dev_config.limits.safety_check_timeout.as_millis(), - staging_config.limits.safety_check_timeout.as_millis(), - prod_config.limits.safety_check_timeout.as_millis() - ); - println!( - " ML Inference Timeout | {:>11} | {:>7} | {:>10}", - dev_config.limits.ml_inference_timeout.as_millis(), - staging_config.limits.ml_inference_timeout.as_millis(), - prod_config.limits.ml_inference_timeout.as_millis() - ); - println!(); - - // Example 8: Cache TTLs (in seconds) - println!("8. Cache TTL comparison (seconds):"); - println!(" Cache Type | Development | Staging | Production"); - println!(" ------------------ | ----------- | ------- | ----------"); - println!( - " Position Cache | {:>11} | {:>7} | {:>10}", - dev_config.cache.position_ttl.as_secs(), - staging_config.cache.position_ttl.as_secs(), - prod_config.cache.position_ttl.as_secs() - ); - println!( - " VaR Cache | {:>11} | {:>7} | {:>10}", - dev_config.cache.var_ttl.as_secs(), - staging_config.cache.var_ttl.as_secs(), - prod_config.cache.var_ttl.as_secs() - ); - println!( - " Market Data Cache | {:>11} | {:>7} | {:>10}", - dev_config.cache.market_data_ttl.as_secs(), - staging_config.cache.market_data_ttl.as_secs(), - prod_config.cache.market_data_ttl.as_secs() - ); - println!(); - - println!("=== Example Complete ==="); - - Ok(()) -} diff --git a/crates/adaptive-strategy/examples/basic_strategy.rs b/crates/adaptive-strategy/examples/basic_strategy.rs deleted file mode 100644 index 7489c0258..000000000 --- a/crates/adaptive-strategy/examples/basic_strategy.rs +++ /dev/null @@ -1,67 +0,0 @@ -#![allow(unused_crate_dependencies)] -//! Basic adaptive strategy example -//! -//! This example demonstrates how to set up and run a basic adaptive trading strategy -//! with ensemble models, risk management, and execution algorithms. - -use adaptive_strategy::config::AdaptiveStrategyConfig; -use adaptive_strategy::AdaptiveStrategy; -use std::time::Duration; -use tokio::time::sleep; -use tracing::info; - -#[tokio::main] -async fn main() -> Result<(), Box> { - // Initialize logging - // DISABLED: tracing_subscriber not in dependencies - uncomment when added - // tracing_subscriber::fmt().with_max_level(Level::INFO).init(); - - info!("Starting basic adaptive strategy example"); - - // Create configuration - let config = create_strategy_config(); - - // Initialize the adaptive strategy - let strategy = AdaptiveStrategy::new(config).await?; - - info!("Strategy initialized successfully"); - - // Get initial state - let initial_state = strategy.get_state().await; - info!( - "Initial strategy state: active={}, regime={}", - initial_state.active, initial_state.current_regime - ); - - // Simulate running for a short period (in production, this would run continuously) - info!("Running strategy simulation for 10 seconds..."); - - // Start the strategy (this would run indefinitely in production) - // For demo purposes, we'll use a timeout - let strategy_task = tokio::spawn(async move { - if let Err(e) = strategy.start().await { - eprintln!("Strategy error: {}", e); - } - }); - - // Let it run for 10 seconds - sleep(Duration::from_secs(10)).await; - - info!("Stopping strategy simulation"); - strategy_task.abort(); - - info!("Example completed successfully"); - - Ok(()) -} - -/// Create a comprehensive strategy configuration -fn create_strategy_config() -> AdaptiveStrategyConfig { - let mut config = AdaptiveStrategyConfig::default(); - - // Customize the execution interval for demo - config.general.execution_interval = Duration::from_millis(500); - config.general.error_backoff_duration = Duration::from_secs(2); - - config -} diff --git a/crates/adaptive-strategy/examples/ppo_position_sizing_demo.rs b/crates/adaptive-strategy/examples/ppo_position_sizing_demo.rs deleted file mode 100644 index 5df1a9ad7..000000000 --- a/crates/adaptive-strategy/examples/ppo_position_sizing_demo.rs +++ /dev/null @@ -1,47 +0,0 @@ -#![allow(unused_crate_dependencies)] -//! PPO Position Sizing Integration Demo -//! -//! This example demonstrates how to use the PPO (Proximal Policy Optimization) -//! position sizer integrated into the adaptive-strategy crate for continuous, -//! risk-aware position optimization. - -use adaptive_strategy::config::RiskConfig; -use adaptive_strategy::risk::{PPOPositionSizerConfig, RiskManager}; - -#[tokio::main] -async fn main() -> Result<(), Box> { - println!("๐Ÿš€ PPO Position Sizing Integration Demo"); - println!("========================================"); - - // 1. Configure PPO Position Sizer - let _ppo_config = PPOPositionSizerConfig::default(); - - // 2. Configure Risk Management with PPO - let mut risk_config = RiskConfig::default(); - risk_config.max_portfolio_var = 0.02; - risk_config.max_drawdown_threshold = 0.05; - risk_config.kelly_fraction = 0.25; - risk_config.max_leverage = 2.0; - - // Save config values for display before moving - let max_var = risk_config.max_portfolio_var; - let max_drawdown = risk_config.max_drawdown_threshold; - let kelly = risk_config.kelly_fraction; - let leverage = risk_config.max_leverage; - - // 3. Initialize Risk Manager with PPO - let _risk_manager = RiskManager::new(risk_config)?; - - println!("โœ… PPO Position Sizer initialized with configuration:"); - println!(" - Max Portfolio VaR: {:.2}%", max_var * 100.0); - println!(" - Max Drawdown: {:.2}%", max_drawdown * 100.0); - println!(" - Kelly Fraction: {:.2}", kelly); - println!(" - Max Leverage: {:.1}x", leverage); - - println!("\n๐Ÿง  PPO Position Sizing Demo Complete!"); - println!(" - PPO configuration loaded successfully"); - println!(" - Risk manager initialized with PPO settings"); - println!(" - Ready for real-time position optimization"); - - Ok(()) -} diff --git a/crates/data/Cargo.toml b/crates/data/Cargo.toml index e1f9427ab..421ef7d5d 100644 --- a/crates/data/Cargo.toml +++ b/crates/data/Cargo.toml @@ -108,9 +108,6 @@ common = { path = "../common" } storage = { workspace = true } dbn = "0.42.0" -# CLI argument parsing (needed for examples) -clap = { version = "4", features = ["derive"] } - [dev-dependencies] tokio-test = { workspace = true } # proptest = { workspace = true } # REMOVED from workspace @@ -130,9 +127,5 @@ icmarkets = [] ib = [] mock = [] -[[example]] -name = "broker_connection" -path = "examples/broker_connection.rs" - [lints] workspace = true diff --git a/crates/data/examples/broker_connection.rs b/crates/data/examples/broker_connection.rs deleted file mode 100644 index 28ce8d0e1..000000000 --- a/crates/data/examples/broker_connection.rs +++ /dev/null @@ -1,182 +0,0 @@ -//! # Broker Connection Example -//! -//! Demonstrates how to connect to different brokers using the data module. -//! This example shows connection setup for Interactive Brokers. -#![allow(unused_crate_dependencies)] - -use common::{Order, OrderSide, OrderType, Price, Quantity, Symbol, TimeInForce}; -use data::brokers::interactive_brokers::{IBConfig, InteractiveBrokersAdapter}; -use tokio::time::{timeout, Duration}; -use tracing::{info, warn}; -// use trading_engine::prelude::*; // REMOVED - prelude does not exist - -#[tokio::main] -async fn main() -> anyhow::Result<()> { - // Initialize logging - tracing_subscriber::fmt::init(); - - info!("Starting broker connection example"); - - // Example 1: Interactive Brokers Connection - if let Err(e) = interactive_brokers_example().await { - warn!("Interactive Brokers example failed: {}", e); - } - - // Example 2: Data Manager with multiple providers - // Note: DataManager and DataConfig not yet implemented - // if let Err(e) = data_manager_example().await { - // warn!("Data manager example failed: {}", e); - // } - - info!("Broker connection examples completed"); - Ok(()) -} - -/// Example of connecting to Interactive Brokers TWS -async fn interactive_brokers_example() -> anyhow::Result<()> { - info!("=== Interactive Brokers Connection Example ==="); - - // Create IB configuration - let config = IBConfig { - host: "127.0.0.1".to_string(), - port: 7497, // Paper trading port - client_id: 1, - account_id: "DU123456".to_string(), // Demo account - connection_timeout: 30, - max_reconnect_attempts: 3, - heartbeat_interval: 60, - request_timeout: 10, - }; - - // Create adapter - let mut adapter = InteractiveBrokersAdapter::new(config); - info!("Created Interactive Brokers adapter"); - - // Attempt connection with timeout - match timeout(Duration::from_secs(10), adapter.connect()).await { - Ok(Ok(())) => { - info!("โœ“ Successfully connected to Interactive Brokers TWS"); - - // Test basic functionality - info!("Connection status: {}", adapter.is_connected()); - // info!("Adapter name: {}", adapter.name()); // DISABLED: name() method removed - - // Subscribe to market data for a test symbol - // DISABLED: Methods removed from adapter - /* - let symbol = Symbol::from("AAPL"); - match adapter.subscribe_market_data(symbol.clone()).await { - Ok(()) => { - info!("โœ“ Successfully subscribed to market data for {}", symbol); - - // Wait for some data - tokio::time::sleep(Duration::from_secs(5)).await; - - // Unsubscribe - if let Err(e) = adapter.unsubscribe_market_data(symbol).await { - warn!("Failed to unsubscribe from market data: {}", e); - } - }, - Err(e) => warn!("Failed to subscribe to market data: {}", e), - } - - */ - // Disconnect - info!("Disconnecting from Interactive Brokers..."); - if let Err(e) = adapter.disconnect().await { - warn!("Error during disconnect: {}", e); - } else { - info!("โœ“ Successfully disconnected"); - } - }, - Ok(Err(e)) => { - warn!("Failed to connect to Interactive Brokers: {}", e); - warn!("Make sure TWS or IB Gateway is running on port 7497"); - return Err(e.into()); - }, - Err(_) => { - warn!("Connection to Interactive Brokers timed out"); - warn!("Make sure TWS or IB Gateway is running and accepting connections"); - return Err(anyhow::anyhow!("Connection timeout")); - }, - } - - Ok(()) -} - -/// Example of using the DataManager for coordinated broker and provider access -async fn data_manager_example() -> anyhow::Result<()> { - // NOTE: DataManager and DataConfig not implemented yet - // This example is commented out until those types are available - println!("DataManager example not available - component not implemented"); - Ok(()) -} - -/// Example of order submission (commented out for safety) -async fn order_submission_example(_adapter: &mut InteractiveBrokersAdapter) -> anyhow::Result<()> { - info!("=== Order Submission Example (DEMO ONLY) ==="); - warn!("This example is for demonstration only - no actual orders will be submitted"); - - // Create a demo order (will not be submitted) - use rust_decimal_macros::dec; - let mut order = Order::new( - Symbol::from("AAPL"), - OrderSide::Buy, - Quantity::try_from(1.0)?, - Some(Price::from_decimal(dec!(100.0))), // Low price to avoid accidental fills - OrderType::Limit, - ); - order.time_in_force = TimeInForce::Day; - - info!("Demo order created:"); - info!(" Symbol: {}", order.symbol); - info!(" Side: {:?}", order.side); - info!(" Quantity: {}", order.quantity); - info!(" Type: {:?}", order.order_type); - info!(" Price: {:?}", order.price); - - // In a real application, you would submit the order like this: - // let result = adapter.submit_order(order).await; - // But for safety, we're just demonstrating the order structure - - info!("Order submission example completed (no actual order submitted)"); - Ok(()) -} - -/// Helper function to check if TWS is running -async fn check_tws_status() -> bool { - match tokio::net::TcpStream::connect("127.0.0.1:7497").await { - Ok(_) => { - info!("โœ“ TWS/IB Gateway appears to be running on port 7497"); - true - }, - Err(_) => { - warn!("โœ— Cannot connect to port 7497 - TWS/IB Gateway may not be running"); - warn!("To run this example successfully:"); - warn!("1. Install and start TWS or IB Gateway"); - warn!("2. Enable API connections in the configuration"); - warn!("3. Set the socket port to 7497 (paper trading)"); - false - }, - } -} - -/// Configuration helper -fn print_setup_instructions() { - info!("=== Setup Instructions ==="); - info!("To run broker connection examples:"); - info!(""); - info!("Interactive Brokers:"); - info!("1. Download and install TWS or IB Gateway"); - info!("2. Start the application and log in"); - info!("3. Go to Configure -> API -> Settings"); - info!("4. Enable 'Enable ActiveX and Socket Clients'"); - info!("5. Set Socket port to 7497 for paper trading"); - info!("6. Add 127.0.0.1 to trusted IP addresses"); - info!(""); - info!("Environment Variables (optional):"); - info!("- POLYGON_API_KEY: Your Polygon.io API key"); - info!("- IB_TWS_HOST: TWS host (default: 127.0.0.1)"); - info!("- IB_TWS_PORT: TWS port (default: 7497)"); - info!(""); -} diff --git a/crates/data/examples/convert_dbn_to_parquet.rs b/crates/data/examples/convert_dbn_to_parquet.rs deleted file mode 100644 index cad24e78b..000000000 --- a/crates/data/examples/convert_dbn_to_parquet.rs +++ /dev/null @@ -1,137 +0,0 @@ -//! CLI Tool: DBN to Parquet Converter -//! -//! Command-line tool for converting Databento binary format (DBN) files to Parquet format. -//! -//! ## Usage -//! -//! ```bash -//! # Convert a single DBN file -//! cargo run --example convert_dbn_to_parquet -- \ -//! --input test_data/real/databento/ES.FUT_ohlcv-1m_2024-01-02.dbn \ -//! --output market_data/converted -//! -//! # With custom configuration -//! cargo run --example convert_dbn_to_parquet -- \ -//! --input data.dbn \ -//! --output ./parquet \ -//! --batch-size 5000 \ -//! --compression snappy -//! ``` - -use anyhow::{Context, Result}; -use clap::Parser; -use data::parquet_persistence::ParquetConfig; -use data::providers::databento::{ConversionReport, DbnToParquetConverter}; -use std::path::PathBuf; -use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; - -#[derive(Parser, Debug)] -#[clap(name = "dbn-to-parquet")] -#[clap(about = "Convert Databento DBN files to Parquet format", long_about = None)] -struct Args { - /// Input DBN file path - #[clap(short, long)] - input: PathBuf, - - /// Output directory for Parquet files - #[clap(short, long, default_value = "./market_data")] - output: PathBuf, - - /// Batch size for processing (events per batch) - #[clap(short, long, default_value_t = 10000)] - batch_size: usize, - - /// Compression algorithm (snappy, gzip, lz4, zstd, none) - #[clap(short, long, default_value = "snappy")] - compression: String, - - /// Enable verbose logging - #[clap(short, long)] - verbose: bool, -} - -#[tokio::main] -async fn main() -> Result<()> { - let args = Args::parse(); - - // Initialize tracing - let log_level = if args.verbose { - tracing::Level::DEBUG - } else { - tracing::Level::INFO - }; - - tracing_subscriber::registry() - .with( - tracing_subscriber::fmt::layer() - .with_target(false) - .with_level(true), - ) - .with(tracing_subscriber::filter::LevelFilter::from_level( - log_level, - )) - .init(); - - // Validate input file exists - if !args.input.exists() { - anyhow::bail!("Input file does not exist: {:?}", args.input); - } - - // Create output directory if needed - if !args.output.exists() { - std::fs::create_dir_all(&args.output) - .with_context(|| format!("Failed to create output directory: {:?}", args.output))?; - } - - // Parse compression algorithm - let compression = match args.compression.to_lowercase().as_str() { - "snappy" => parquet::basic::Compression::SNAPPY, - "gzip" => parquet::basic::Compression::GZIP(parquet::basic::GzipLevel::default()), - "lz4" => parquet::basic::Compression::LZ4, - "zstd" => parquet::basic::Compression::ZSTD(parquet::basic::ZstdLevel::default()), - "none" => parquet::basic::Compression::UNCOMPRESSED, - other => anyhow::bail!("Unknown compression algorithm: {}", other), - }; - - // Configure Parquet writer - let config = ParquetConfig { - base_path: args.output.to_string_lossy().to_string(), - batch_size: args.batch_size, - compression, - ..Default::default() - }; - - // Create converter - tracing::info!("Initializing DBN to Parquet converter..."); - let mut converter = DbnToParquetConverter::new(config).await?; - - // Convert file - tracing::info!("Converting {:?}...", args.input); - let report = converter.convert_file(&args.input).await?; - - // Print results - print_report(&report); - - if !report.is_success() { - anyhow::bail!("Conversion completed with {} errors", report.events_failed); - } - - tracing::info!("โœ“ Conversion successful!"); - Ok(()) -} - -fn print_report(report: &ConversionReport) { - println!("\n{}", "=".repeat(60)); - println!("Conversion Report"); - println!("{}", "=".repeat(60)); - println!("Events processed: {}", report.events_processed); - println!("Events skipped: {}", report.events_skipped); - println!("Events failed: {}", report.events_failed); - println!("Duration: {:?}", report.duration); - println!( - "Throughput: {} events/sec", - report.throughput_events_per_sec - ); - println!("Success rate: {:.2}%", report.success_rate()); - println!("{}", "=".repeat(60)); -} diff --git a/crates/data/examples/convert_es_fut_to_parquet.rs b/crates/data/examples/convert_es_fut_to_parquet.rs deleted file mode 100644 index 2aab7246e..000000000 --- a/crates/data/examples/convert_es_fut_to_parquet.rs +++ /dev/null @@ -1,88 +0,0 @@ -//! Convert ES.FUT DBN file to Parquet format -//! -//! This example demonstrates converting a real Databento DBN file containing -//! ES.FUT (E-mini S&P 500 Futures) OHLCV data to Parquet format for backtesting. -//! -//! Input: test_data/real/databento/ES.FUT_ohlcv-1m_2024-01-02.dbn (96 KB, ~390 bars) -//! Output: test_data/real/parquet/ES.FUT_ohlcv-1m_2024-01-02.parquet -//! -//! Schema: 11 columns (timestamp_ns, symbol, venue, event_type, price, quantity, -//! sequence, latency_ns, open, high, low) -//! -//! Example usage: -//! ```bash -//! cargo run --example convert_es_fut_to_parquet -//! ``` - -use anyhow::Result; -use data::parquet_persistence::ParquetConfig; -use data::providers::databento::DbnToParquetConverter; - -#[tokio::main] -async fn main() -> Result<()> { - // Initialize tracing for logging - tracing_subscriber::fmt() - .with_env_filter( - tracing_subscriber::EnvFilter::from_default_env() - .add_directive(tracing::Level::INFO.into()), - ) - .init(); - - println!("โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”"); - println!(" DBN to Parquet Converter - ES.FUT OHLCV Example"); - println!("โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”"); - println!(); - - // Configure Parquet output - let config = ParquetConfig { - base_path: "test_data/real/parquet".to_string(), - batch_size: 10000, - ..Default::default() - }; - - println!("๐Ÿ“ Input: test_data/real/databento/ES.FUT_ohlcv-1m_2024-01-02.dbn"); - println!("๐Ÿ“‚ Output: test_data/real/parquet/"); - println!(); - - // Create converter - let mut converter = DbnToParquetConverter::new(config).await?; - - // Convert ES.FUT file - println!("โš™๏ธ Converting DBN to Parquet..."); - let report = converter - .convert_file("test_data/real/databento/ES.FUT_ohlcv-1m_2024-01-02.dbn") - .await?; - - println!(); - println!("โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”"); - println!(" Conversion Results"); - println!("โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”"); - println!(); - println!("โœ… Events processed: {}", report.events_processed); - println!("โญ๏ธ Events skipped: {}", report.events_skipped); - println!("โŒ Events failed: {}", report.events_failed); - println!("๐Ÿ“ˆ Success rate: {:.2}%", report.success_rate()); - println!("โฑ๏ธ Duration: {:?}", report.duration); - println!( - "๐Ÿš€ Throughput: {} events/sec", - report.throughput_events_per_sec - ); - println!(); - - if report.is_success() { - println!("โœ… SUCCESS! All events converted without errors."); - println!(); - println!("๐Ÿ“ฆ Output file ready for backtesting:"); - println!(" test_data/real/parquet/ES.FUT_ohlcv-1m_2024-01-02.parquet"); - } else { - println!( - "โš ๏ธ WARNING: Some events failed to convert ({} failures)", - report.events_failed - ); - } - - println!(); - println!("โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”"); - - Ok(()) -} diff --git a/crates/data/examples/convert_nq_fut_to_parquet.rs b/crates/data/examples/convert_nq_fut_to_parquet.rs deleted file mode 100644 index 4599a249f..000000000 --- a/crates/data/examples/convert_nq_fut_to_parquet.rs +++ /dev/null @@ -1,86 +0,0 @@ -//! Convert NQ.FUT DBN file to Parquet format -//! -//! This example converts the Databento DBN file containing -//! NQ.FUT (E-mini NASDAQ 100 Futures) OHLCV data to Parquet format for ML training. -//! -//! Input: test_data/NQ_FUT_180d.dbn (4.10 MB, ~262,442 bars) -//! Output: test_data/NQ_FUT_180d.parquet -//! -//! Example usage: -//! ```bash -//! cargo run -p data --example convert_nq_fut_to_parquet --release -//! ``` - -use anyhow::Result; -use data::parquet_persistence::ParquetConfig; -use data::providers::databento::DbnToParquetConverter; - -#[tokio::main] -async fn main() -> Result<()> { - // Initialize tracing for logging - tracing_subscriber::fmt() - .with_env_filter( - tracing_subscriber::EnvFilter::from_default_env() - .add_directive(tracing::Level::INFO.into()), - ) - .init(); - - println!("โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”"); - println!(" DBN to Parquet Converter - NQ.FUT 180d OHLCV"); - println!("โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”"); - println!(); - - // Configure Parquet output - let config = ParquetConfig { - base_path: "test_data".to_string(), - batch_size: 10000, - compression: parquet::basic::Compression::SNAPPY, - ..Default::default() - }; - - println!("๐Ÿ“ Input: test_data/NQ_FUT_180d_uncompressed.dbn"); - println!("๐Ÿ“‚ Output: test_data/NQ_FUT_180d.parquet"); - println!(); - - // Create converter - let mut converter = DbnToParquetConverter::new(config).await?; - - // Convert NQ.FUT file - println!("โš™๏ธ Converting DBN to Parquet..."); - let report = converter - .convert_file("test_data/NQ_FUT_180d_uncompressed.dbn") - .await?; - - println!(); - println!("โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”"); - println!(" Conversion Results"); - println!("โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”"); - println!(); - println!("โœ… Events processed: {}", report.events_processed); - println!("โญ๏ธ Events skipped: {}", report.events_skipped); - println!("โŒ Events failed: {}", report.events_failed); - println!("๐Ÿ“ˆ Success rate: {:.2}%", report.success_rate()); - println!("โฑ๏ธ Duration: {:?}", report.duration); - println!( - "๐Ÿš€ Throughput: {} events/sec", - report.throughput_events_per_sec - ); - println!(); - - if report.is_success() { - println!("โœ… SUCCESS! All events converted without errors."); - println!(); - println!("๐Ÿ“ฆ Output file ready for ML training:"); - println!(" test_data/NQ_FUT_180d.parquet"); - } else { - println!( - "โš ๏ธ WARNING: Some events failed to convert ({} failures)", - report.events_failed - ); - } - - println!(); - println!("โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”"); - - Ok(()) -} diff --git a/crates/data/examples/download_mbp10_data.rs b/crates/data/examples/download_mbp10_data.rs deleted file mode 100644 index 7d33bf470..000000000 --- a/crates/data/examples/download_mbp10_data.rs +++ /dev/null @@ -1,289 +0,0 @@ -//! Download MBP-10 (Market By Price, 10 levels) data from Databento -//! -//! This example downloads Level-2 order book data (MBP-10) for ES.FUT -//! to support TLOB neural network training. -//! -//! ## Usage -//! -//! ```bash -//! export DATABENTO_API_KEY="your_api_key_here" -//! cargo run --example download_mbp10_data --release -//! ``` -//! -//! ## Data Specifications -//! -//! - **Dataset**: GLBX.MDP3 (CME Globex MDP 3.0) -//! - **Schema**: mbp-10 (Market By Price, 10 levels) -//! - **Symbols**: ES.c.0 (E-mini S&P 500 continuous front month) -//! - **Date Range**: 2024-01-02 to 2024-01-10 (7 trading days) -//! - **Encoding**: dbn (Databento Binary Encoding v2) -//! - **Compression**: zstd (Zstandard) -//! - **Expected Size**: ~50-70 GB compressed - -use anyhow::{Context, Result}; -use reqwest::Client; -use serde::{Deserialize, Serialize}; -use std::env; -use std::fs::File; -use std::io::Write; -use std::path::PathBuf; -use std::time::Duration; -use tokio::time::sleep; - -#[derive(Debug, Deserialize)] -struct BatchSubmitResponse { - id: String, -} - -#[derive(Debug, Deserialize)] -struct BatchStatusResponse { - id: String, - state: String, - record_count: Option, - actual_size: Option, -} - -#[derive(Debug, Deserialize)] -struct BatchFileUrls { - https: String, - ftp: String, -} - -#[derive(Debug, Deserialize)] -struct BatchFileInfo { - filename: String, - size: u64, - hash: String, - urls: BatchFileUrls, -} - -const DATABENTO_API_BASE: &str = "https://hist.databento.com/v0"; - -#[tokio::main] -async fn main() -> Result<()> { - tracing_subscriber::fmt() - .with_max_level(tracing::Level::INFO) - .init(); - - // Get API key from environment - let api_key = - env::var("DATABENTO_API_KEY").context("DATABENTO_API_KEY environment variable not set")?; - - // Create output directory - let output_dir = PathBuf::from("test_data/mbp10"); - std::fs::create_dir_all(&output_dir).context("Failed to create output directory")?; - - println!("=== Databento MBP-10 Data Download ===\n"); - println!("Dataset: GLBX.MDP3"); - println!("Schema: mbp-10 (Market By Price, 10 levels)"); - println!("Symbol: ES.c.0"); - println!("Date Range: 2024-01-02 to 2024-01-10 (7 trading days)"); - println!("Output: {}", output_dir.display()); - println!("\n{}", "=".repeat(50)); - - // Step 1: Submit batch download job - println!("\n[1/4] Submitting batch download job..."); - let job_id = submit_batch_job(&api_key).await?; - println!("โœ“ Job submitted successfully: {}", job_id); - - // Step 2: Poll for job completion - println!("\n[2/4] Waiting for job to complete..."); - poll_job_status(&api_key, &job_id).await?; - println!("โœ“ Job completed successfully"); - - // Step 3: Get file list and download all files - println!("\n[3/4] Getting file list..."); - let files = list_batch_files(&api_key, &job_id).await?; - let data_files: Vec<_> = files - .iter() - .filter(|f| f.filename.ends_with(".dbn.zst")) - .collect(); - println!("โœ“ Found {} data files", data_files.len()); - - println!("\n[4/4] Downloading data files..."); - for (i, file) in data_files.iter().enumerate() { - println!("\n [{}/{}] {}", i + 1, data_files.len(), file.filename); - let output_file = output_dir.join(&file.filename); - download_file(&api_key, &file.urls.https, &output_file).await?; - } - println!("โœ“ Download completed successfully"); - - println!("\n{}", "=".repeat(50)); - println!("SUCCESS: MBP-10 data downloaded to {}", output_dir.display()); - println!("\nNext Steps:"); - println!("1. Decompress files: for f in {}/*.dbn.zst; do zstd -d $f; done", output_dir.display()); - println!("2. Validate schema: cargo run --example validate_dbn_schema"); - println!("3. Implement MBP-10 parser in data/src/providers/databento/"); - - Ok(()) -} - -async fn submit_batch_job(api_key: &str) -> Result { - let client = Client::new(); - - // Build form data (API expects application/x-www-form-urlencoded, not JSON) - let form_data = [ - ("dataset", "GLBX.MDP3"), - ("schema", "mbp-10"), - ("symbols", "ES.c.0"), - ("stype_in", "continuous"), - ("start", "2024-01-02"), - ("end", "2024-01-10"), - ("encoding", "dbn"), - ("compression", "zstd"), - ]; - - let url = format!("{}/batch.submit_job", DATABENTO_API_BASE); - let response = client - .post(&url) - .basic_auth(api_key, Some("")) - .form(&form_data) - .send() - .await - .context("Failed to submit batch job")?; - - if !response.status().is_success() { - let status = response.status(); - let body = response.text().await.unwrap_or_default(); - anyhow::bail!("API request failed: {} - {}", status, body); - } - - let result: BatchSubmitResponse = response - .json() - .await - .context("Failed to parse batch submit response")?; - - Ok(result.id) -} - -async fn poll_job_status(api_key: &str, job_id: &str) -> Result<()> { - let client = Client::new(); - let url = format!("{}/batch.list_jobs", DATABENTO_API_BASE); - - loop { - let response = client - .get(&url) - .basic_auth(api_key, Some("")) - .send() - .await - .context("Failed to check job status")?; - - if !response.status().is_success() { - let status = response.status(); - let body = response.text().await.unwrap_or_default(); - anyhow::bail!("API request failed: {} - {}", status, body); - } - - let jobs: Vec = response - .json() - .await - .context("Failed to parse batch list response")?; - - // Find our job in the list - let job = jobs - .iter() - .find(|j| j.id == job_id) - .context("Job not found in batch list")?; - - match job.state.as_str() { - "done" => { - if let Some(size) = job.actual_size { - let size_mb = size as f64 / 1_048_576.0; - println!(" โ€ข Size: {:.2} MB", size_mb); - } - if let Some(records) = job.record_count { - println!(" โ€ข Records: {}", records); - } - return Ok(()); - }, - "error" => { - anyhow::bail!("Job failed with error state"); - }, - state => { - print!("\r โ€ข Status: {} ... ", state); - std::io::stdout().flush()?; - sleep(Duration::from_secs(5)).await; - }, - } - } -} - -async fn list_batch_files(api_key: &str, job_id: &str) -> Result> { - let client = Client::new(); - let url = format!("{}/batch.list_files?job_id={}", DATABENTO_API_BASE, job_id); - - let response = client - .get(&url) - .basic_auth(api_key, Some("")) - .send() - .await - .context("Failed to list batch files")?; - - if !response.status().is_success() { - let status = response.status(); - let body = response.text().await.unwrap_or_default(); - anyhow::bail!("API request failed: {} - {}", status, body); - } - - let files: Vec = response.json().await.context("Failed to parse file list")?; - Ok(files) -} - -async fn download_file(api_key: &str, download_url: &str, output_path: &PathBuf) -> Result<()> { - let client = Client::new(); - - let response = client - .get(download_url) - .basic_auth(api_key, Some("")) - .send() - .await - .context("Failed to start download")?; - - if !response.status().is_success() { - let status = response.status(); - let body = response.text().await.unwrap_or_default(); - anyhow::bail!("Download failed: {} - {}", status, body); - } - - let total_size = response.content_length().unwrap_or(0); - let mut file = File::create(output_path).context("Failed to create output file")?; - - let mut downloaded: u64 = 0; - - // Use chunk() method for reqwest 0.12 (replaces bytes_stream()) - let mut response = response; - while let Some(chunk) = response.chunk().await.context("Failed to read chunk")? { - file.write_all(&chunk) - .context("Failed to write chunk to file")?; - - downloaded += chunk.len() as u64; - - if total_size > 0 { - let percent = (downloaded as f64 / total_size as f64) * 100.0; - print!( - "\r โ€ข Progress: {:.2}% ({:.2} MB / {:.2} MB)", - percent, - downloaded as f64 / 1_048_576.0, - total_size as f64 / 1_048_576.0 - ); - std::io::stdout().flush()?; - } - } - - println!(); - Ok(()) -} - -fn validate_file(path: &PathBuf) -> Result<()> { - let metadata = std::fs::metadata(path).context("Failed to read file metadata")?; - - let size_mb = metadata.len() as f64 / 1_048_576.0; - println!(" โ€ข File size: {:.2} MB", size_mb); - - if size_mb < 1.0 { - anyhow::bail!("File is suspiciously small (< 1 MB), may be corrupted"); - } - - println!(" โ€ข File appears valid"); - Ok(()) -} diff --git a/crates/ml/data/examples/basic_connection.rs b/crates/ml/data/examples/basic_connection.rs deleted file mode 100644 index 36c732b61..000000000 --- a/crates/ml/data/examples/basic_connection.rs +++ /dev/null @@ -1,76 +0,0 @@ -//! Basic TWS Connection Example -//! -//! This example demonstrates how to establish a basic connection to -//! Interactive Brokers TWS or Gateway. -//! -//! Usage: -//! cargo run --example basic_connection -//! -//! Prerequisites: -//! - TWS or IB Gateway running on localhost -//! - API connections enabled in TWS settings -//! - Socket port configured (default: 7497 for paper trading) - -use data::{init, paper_trading_config, validate_config, InteractiveBrokersAdapter}; -use std::time::Duration; -use tokio::time::sleep; -use tracing::{error, info}; - -#[tokio::main] -async fn main() -> Result<(), Box> { - // Initialize logging - init()?; - - info!("=== Interactive Brokers Basic Connection Example ==="); - - // Get paper trading configuration - let config = paper_trading_config(); - - // Validate configuration - if let Err(e) = validate_config(&config) { - error!("Invalid configuration: {}", e); - return Err(e.into()); - } - - info!("Configuration:"); - info!(" Host: {}", config.host); - info!(" Port: {}", config.port); - info!(" Client ID: {}", config.client_id); - info!(" Account ID: {}", config.account_id); - - // Create adapter - let mut adapter = InteractiveBrokersAdapter::new(config); - - // Attempt connection - info!("Connecting to TWS..."); - match adapter.connect().await { - Ok(()) => { - info!("โœ… Successfully connected to TWS!"); - - // Check connection state - let state = adapter.get_connection_state().await; - info!("Connection state: {:?}", state); - - // Keep connection alive for a few seconds - info!("Maintaining connection for 10 seconds..."); - sleep(Duration::from_secs(10)).await; - - // Disconnect cleanly - info!("Disconnecting from TWS..."); - adapter.disconnect().await?; - info!("โœ… Successfully disconnected from TWS"); - } - Err(e) => { - error!("โŒ Failed to connect to TWS: {}", e); - error!("Please ensure:"); - error!(" 1. TWS or IB Gateway is running"); - error!(" 2. API connections are enabled in settings"); - error!(" 3. Socket port is configured correctly"); - error!(" 4. Client ID is not already in use"); - return Err(e); - } - } - - info!("=== Example completed successfully ==="); - Ok(()) -} \ No newline at end of file diff --git a/crates/ml/examples/baseline_common/completion.rs b/crates/ml/examples/baseline_common/completion.rs index e034f6c34..e181c0484 100644 --- a/crates/ml/examples/baseline_common/completion.rs +++ b/crates/ml/examples/baseline_common/completion.rs @@ -11,6 +11,7 @@ use tracing::{error, info}; /// Summary metrics written alongside the `DONE` marker on success. #[derive(serde::Serialize)] +#[allow(clippy::module_name_repetitions)] pub struct CompletionMetrics { /// Model name (e.g. "kan", "dqn", "ppo") pub model: String, @@ -30,6 +31,7 @@ pub struct CompletionMetrics { /// /// Errors are logged but never cause a panic -- the training result is /// already persisted in checkpoints; the marker is best-effort. +#[allow(clippy::cognitive_complexity)] pub fn write_success_marker(output_dir: &Path, metrics: &CompletionMetrics) { // Write metrics.json match serde_json::to_string_pretty(metrics) { diff --git a/crates/ml/examples/baseline_common/mod.rs b/crates/ml/examples/baseline_common/mod.rs index 475d30d66..5b60d647c 100644 --- a/crates/ml/examples/baseline_common/mod.rs +++ b/crates/ml/examples/baseline_common/mod.rs @@ -51,7 +51,7 @@ fn collect_dbn_files_recursive(dir: &Path, out: &mut Vec) -> Result<()> /// Load OHLCV bars from a single .dbn or .dbn.zst file using the `dbn` crate decoder. /// -/// Reads OhlcvMsg records and converts them to [`OHLCVBar`]. +/// Reads `OhlcvMsg` records and converts them to [`OHLCVBar`]. fn load_bars_from_dbn(path: &Path) -> Result> { if path.to_string_lossy().ends_with(".dbn.zst") { let mut decoder = dbn::decode::dbn::Decoder::from_zstd_file(path) @@ -122,8 +122,9 @@ fn decode_ohlcv_records( Ok(bars) } -/// Convert nanosecond UNIX timestamp to chrono DateTime. +/// Convert nanosecond UNIX timestamp to chrono `DateTime`. fn nanos_to_datetime(nanos: u64) -> chrono::DateTime { + #[allow(clippy::integer_division)] let secs = (nanos / 1_000_000_000) as i64; let subsec_nanos = (nanos % 1_000_000_000) as u32; Utc.timestamp_opt(secs, subsec_nanos) @@ -135,6 +136,7 @@ fn nanos_to_datetime(nanos: u64) -> chrono::DateTime { /// /// Only loads files from `data_dir/symbol/` to avoid mixing different futures /// contracts (e.g. ES, NQ, 6E, ZN) into a single price series. +#[allow(clippy::cognitive_complexity)] pub fn load_all_bars(data_dir: &Path, symbol: &str) -> Result> { let symbol_dir = data_dir.join(symbol); if !symbol_dir.exists() { @@ -189,8 +191,8 @@ pub fn list_subdirs(dir: &Path) -> Vec { /// Compute round-trip spread slippage in basis points for a given price. /// /// Uses the bid-ask spread model from `backtesting/src/slippage.rs`: -/// half-spread = tick_size * spread_ticks / 2. Round-trip pays the full spread. -/// Converted to bps: spread_price / price * 10_000. +/// half-spread = `tick_size` * `spread_ticks` / 2. Round-trip pays the full spread. +/// Converted to bps: `spread_price` / price * `10_000`. pub fn spread_cost_bps(price: f64, tick_size: f64, spread_ticks: f64) -> f64 { if price.abs() < 1e-10 { return 0.0; diff --git a/crates/ml/examples/cuda_test.rs b/crates/ml/examples/cuda_test.rs index 39dd63c76..de68e24fe 100644 --- a/crates/ml/examples/cuda_test.rs +++ b/crates/ml/examples/cuda_test.rs @@ -7,6 +7,12 @@ use candle_core::{Device, Tensor}; use candle_nn::{linear, Module, VarBuilder, VarMap}; +/// Format a tensor's dimensions as a display string without using Debug formatting. +fn fmt_dims(dims: &[usize]) -> String { + let parts: Vec = dims.iter().map(|d| d.to_string()).collect(); + format!("[{}]", parts.join(", ")) +} + /// Test basic CUDA tensor operations pub fn test_cuda_basic() -> Result<(), Box> { println!("Testing CUDA compatibility..."); @@ -14,20 +20,23 @@ pub fn test_cuda_basic() -> Result<(), Box> { // Try to get CUDA device match Device::new_cuda(0) { Ok(device) => { - println!("โœ… CUDA device 0 available"); + println!("[OK] CUDA device 0 available"); // Create a simple tensor - let tensor = Tensor::randn(0f32, 1.0, (4, 4), &device)?; - println!("โœ… Created CUDA tensor: {:?}", tensor.shape()); + let tensor = Tensor::randn(0_f32, 1.0, (4, 4), &device)?; + println!("[OK] Created CUDA tensor: {}", fmt_dims(tensor.dims())); // Perform basic operations let result = tensor.matmul(&tensor.t()?)?; - println!("โœ… Matrix multiplication successful: {:?}", result.shape()); + println!( + "[OK] Matrix multiplication successful: {}", + fmt_dims(result.dims()) + ); Ok(()) }, Err(e) => { - println!("โš ๏ธ CUDA device not available: {}", e); + println!("[WARN] CUDA device not available: {}", e); println!("This is expected if no GPU is present, but CUDA compilation succeeded"); Ok(()) }, @@ -38,28 +47,24 @@ pub fn test_cuda_basic() -> Result<(), Box> { pub fn test_cuda_neural_network() -> Result<(), Box> { println!("Testing CUDA neural network components..."); - match Device::new_cuda(0) { - Ok(device) => { - let varmap = VarMap::new(); - let vs = VarBuilder::from_varmap(&varmap, candle_core::DType::F32, &device); + if let Ok(device) = Device::new_cuda(0) { + let varmap = VarMap::new(); + let vs = VarBuilder::from_varmap(&varmap, candle_core::DType::F32, &device); - // Create a simple linear layer - let linear_layer = linear(10, 5, vs.pp("linear"))?; - let input = Tensor::randn(0f32, 1.0, (1, 10), &device)?; + // Create a simple linear layer + let linear_layer = linear(10, 5, vs.pp("linear"))?; + let input = Tensor::randn(0_f32, 1.0, (1, 10), &device)?; - let output = linear_layer.forward(&input)?; - println!( - "โœ… Neural network forward pass successful: {:?}", - output.shape() - ); - - Ok(()) - }, - Err(_) => { - println!("โš ๏ธ CUDA neural network test skipped (no GPU)"); - Ok(()) - }, + let output = linear_layer.forward(&input)?; + println!( + "[OK] Neural network forward pass successful: {}", + fmt_dims(output.dims()) + ); + } else { + println!("[WARN] CUDA neural network test skipped (no GPU)"); } + + Ok(()) } #[cfg(test)] @@ -81,6 +86,6 @@ mod tests { fn main() -> Result<(), Box> { test_cuda_basic()?; test_cuda_neural_network()?; - println!("๐ŸŽ‰ CUDA compatibility verification complete!"); + println!("[DONE] CUDA compatibility verification complete!"); Ok(()) } diff --git a/crates/ml/examples/download_baseline.rs b/crates/ml/examples/download_baseline.rs index 9c0b51f63..a2db16dc4 100644 --- a/crates/ml/examples/download_baseline.rs +++ b/crates/ml/examples/download_baseline.rs @@ -8,7 +8,7 @@ //! //! Usage: //! # Dry run (preview config and cost estimate) -//! cargo run -p ml --example download_baseline --release -- --dry-run +//! cargo run -p ml --example `download_baseline` --release -- --dry-run //! //! # Download with confirmation prompt //! cargo run -p ml --example download_baseline --release @@ -99,6 +99,7 @@ fn generate_quarters(start: NaiveDate, end: NaiveDate) -> Vec { let mut cursor = start; while cursor < end { + #[allow(clippy::integer_division)] let q = (cursor.month() - 1) / 3 + 1; // 1..4 let year = cursor.year(); let label = format!("{}-Q{}", year, q); @@ -160,7 +161,7 @@ struct DownloadStats { } impl DownloadStats { - fn new() -> Self { + const fn new() -> Self { Self { successful: 0, failed: 0, @@ -175,7 +176,7 @@ async fn download_quarter( symbol: &str, quarter: &Quarter, dataset: &str, - output_dir: &PathBuf, + output_dir: &std::path::Path, ) -> Result { let filename = format!("{}_{}.dbn.zst", symbol, quarter.label); let symbol_dir = output_dir.join(symbol); @@ -193,8 +194,8 @@ async fn download_quarter( let dt_range = date_range(quarter.start, quarter.end)?; let params = GetRangeToFileParams::builder() - .dataset(dataset.to_string()) - .symbols(vec![symbol.to_string()]) + .dataset(dataset.to_owned()) + .symbols(vec![symbol.to_owned()]) .schema(Schema::Ohlcv1M) .stype_in(SType::Parent) .date_time_range(dt_range) diff --git a/crates/ml/examples/evaluate_baseline.rs b/crates/ml/examples/evaluate_baseline.rs index 36884160f..d6f841811 100644 --- a/crates/ml/examples/evaluate_baseline.rs +++ b/crates/ml/examples/evaluate_baseline.rs @@ -65,7 +65,7 @@ struct Args { #[arg(long, default_value = "both")] model: String, - /// Feature dimension (must match extract_ml_features output) + /// Feature dimension (must match `extract_ml_features` output) #[arg(long, default_value_t = 51)] feature_dim: usize, @@ -108,7 +108,7 @@ struct Args { tick_size: f64, /// Typical bid-ask spread in ticks (ES=1.0, ZN=1.0, 6E=2.0) - /// Full spread slippage is added to tx_cost_bps per round-trip trade. + /// Full spread slippage is added to `tx_cost_bps` per round-trip trade. #[arg(long, default_value_t = 1.0)] spread_ticks: f64, } @@ -181,7 +181,7 @@ struct ComputedMetrics { /// - Sharpe: annualized (mean / std * sqrt(252)) /// - Max drawdown: largest peak-to-trough drop on cumulative equity curve (%) /// - Win rate: percentage of returns > 0 -/// - Profit factor: gross_profit / gross_loss (inf if no losses) +/// - Profit factor: `gross_profit` / `gross_loss` (inf if no losses) /// - Total return: sum of returns * 100 (as percentage) /// - Num trades: count of non-zero returns (BUY or SELL actions) fn compute_metrics(returns: &[f64]) -> ComputedMetrics { @@ -338,9 +338,8 @@ fn evaluate_dqn_fold( let mut action_counts = [0_usize; 3]; // [buy, sell, hold] for i in 0..n.saturating_sub(1) { - let feat = match test_features.get(i) { - Some(f) => f, - None => continue, + let Some(feat) = test_features.get(i) else { + continue; }; let state: Vec = feat.iter().map(|&v| v as f32).collect(); @@ -462,9 +461,8 @@ fn evaluate_ppo_fold( let mut action_counts = [0_usize; 3]; // [buy, sell, hold] for i in 0..n.saturating_sub(1) { - let feat = match test_features.get(i) { - Some(f) => f, - None => continue, + let Some(feat) = test_features.get(i) else { + continue; }; let state: Vec = feat.iter().map(|&v| v as f32).collect(); @@ -567,6 +565,7 @@ fn run_sanity_checks( // Main // --------------------------------------------------------------------------- +#[allow(clippy::cognitive_complexity, clippy::too_many_lines)] fn main() -> Result<()> { // Initialize tracing tracing_subscriber::fmt() @@ -668,7 +667,7 @@ fn main() -> Result<()> { Ok(f) => f, Err(e) => { warn!( - " Fold {} โ€” test feature extraction failed: {}", + " Fold {} - test feature extraction failed: {}", window.fold, e ); continue; @@ -676,7 +675,7 @@ fn main() -> Result<()> { }; if test_features.is_empty() { - warn!(" Fold {} โ€” empty test features, skipping", window.fold); + warn!(" Fold {} - empty test features, skipping", window.fold); continue; } @@ -685,11 +684,7 @@ fn main() -> Result<()> { // Align bars to features (features skip warmup period) let warmup_offset = window.test.len().saturating_sub(test_norm.len()); - let test_bars_aligned = if warmup_offset < window.test.len() { - &window.test[warmup_offset..] - } else { - &window.test - }; + let test_bars_aligned = window.test.get(warmup_offset..).unwrap_or(&window.test); // Test period date range for the report let test_start = test_bars_aligned @@ -713,7 +708,7 @@ fn main() -> Result<()> { Ok((returns, action_counts)) => { let metrics = compute_metrics(&returns); info!( - " [DQN] Fold {} โ€” Sharpe={:.4} MaxDD={:.2}% WinRate={:.1}% PF={:.2} Return={:.4}% Trades={}", + " [DQN] Fold {} - Sharpe={:.4} MaxDD={:.2}% WinRate={:.1}% PF={:.2} Return={:.4}% Trades={}", window.fold, metrics.sharpe_ratio, metrics.max_drawdown_pct, @@ -723,14 +718,14 @@ fn main() -> Result<()> { metrics.num_trades, ); info!( - " [DQN] Actions โ€” BUY={} SELL={} HOLD={}", + " [DQN] Actions - BUY={} SELL={} HOLD={}", action_counts.first().copied().unwrap_or(0), action_counts.get(1).copied().unwrap_or(0), action_counts.get(2).copied().unwrap_or(0), ); all_fold_metrics.push(FoldMetrics { fold: window.fold, - model: "dqn".to_string(), + model: "dqn".to_owned(), sharpe_ratio: metrics.sharpe_ratio, max_drawdown_pct: metrics.max_drawdown_pct, win_rate_pct: metrics.win_rate_pct, @@ -760,7 +755,7 @@ fn main() -> Result<()> { Ok((returns, action_counts)) => { let metrics = compute_metrics(&returns); info!( - " [PPO] Fold {} โ€” Sharpe={:.4} MaxDD={:.2}% WinRate={:.1}% PF={:.2} Return={:.4}% Trades={}", + " [PPO] Fold {} - Sharpe={:.4} MaxDD={:.2}% WinRate={:.1}% PF={:.2} Return={:.4}% Trades={}", window.fold, metrics.sharpe_ratio, metrics.max_drawdown_pct, @@ -770,14 +765,14 @@ fn main() -> Result<()> { metrics.num_trades, ); info!( - " [PPO] Actions โ€” BUY={} SELL={} HOLD={}", + " [PPO] Actions - BUY={} SELL={} HOLD={}", action_counts.first().copied().unwrap_or(0), action_counts.get(1).copied().unwrap_or(0), action_counts.get(2).copied().unwrap_or(0), ); all_fold_metrics.push(FoldMetrics { fold: window.fold, - model: "ppo".to_string(), + model: "ppo".to_owned(), sharpe_ratio: metrics.sharpe_ratio, max_drawdown_pct: metrics.max_drawdown_pct, win_rate_pct: metrics.win_rate_pct, @@ -812,9 +807,9 @@ fn main() -> Result<()> { ppo_avg_win_rate, }; - info!(" DQN โ€” avg Sharpe={:.4} avg MaxDD={:.2}% avg WinRate={:.1}%", + info!(" DQN - avg Sharpe={:.4} avg MaxDD={:.2}% avg WinRate={:.1}%", dqn_avg_sharpe, dqn_avg_drawdown, dqn_avg_win_rate); - info!(" PPO โ€” avg Sharpe={:.4} avg MaxDD={:.2}% avg WinRate={:.1}%", + info!(" PPO - avg Sharpe={:.4} avg MaxDD={:.2}% avg WinRate={:.1}%", ppo_avg_sharpe, ppo_avg_drawdown, ppo_avg_win_rate); // 5. Sanity checks & report diff --git a/crates/ml/examples/evaluate_supervised.rs b/crates/ml/examples/evaluate_supervised.rs index b0b760922..8ce0c4d18 100644 --- a/crates/ml/examples/evaluate_supervised.rs +++ b/crates/ml/examples/evaluate_supervised.rs @@ -77,7 +77,7 @@ struct Args { #[arg(long)] model: String, - /// Feature dimension (must match extract_ml_features output) + /// Feature dimension (must match `extract_ml_features` output) #[arg(long, default_value_t = 51)] feature_dim: usize, @@ -392,6 +392,7 @@ fn create_model( /// Run supervised inference on test features and return per-bar trade returns, /// action counts, and directional accuracy. +#[allow(clippy::cognitive_complexity)] fn evaluate_fold( fold: usize, model_name: &str, @@ -529,6 +530,7 @@ fn evaluate_fold( // Main // --------------------------------------------------------------------------- +#[allow(clippy::cognitive_complexity, clippy::too_many_lines)] fn main() -> Result<()> { tracing_subscriber::fmt() .with_env_filter( @@ -552,7 +554,13 @@ fn main() -> Result<()> { args.tx_cost_bps, args.spread_ticks, args.tick_size ); - let device = Device::Cpu; + let device = if let Ok(d) = Device::cuda_if_available(0) { + info!("Using CUDA device for evaluation"); + d + } else { + info!("CUDA unavailable, using CPU"); + Device::Cpu + }; // 1. Load all OHLCV bars info!("Step 1/4: Loading OHLCV bars from DBN files..."); @@ -629,14 +637,14 @@ fn main() -> Result<()> { Ok(f) => f, Err(e) => { warn!( - " Fold {} โ€” test feature extraction failed: {}", + " Fold {} -- test feature extraction failed: {}", window.fold, e ); continue; } }; if test_features.is_empty() { - warn!(" Fold {} โ€” empty test features, skipping", window.fold); + warn!(" Fold {} -- empty test features, skipping", window.fold); continue; } let test_norm = norm_stats.normalize_batch(&test_features); @@ -644,7 +652,7 @@ fn main() -> Result<()> { // Align bars to features let warmup_offset = window.test.len().saturating_sub(test_norm.len()); let test_bars_aligned = if warmup_offset < window.test.len() { - &window.test[warmup_offset..] + window.test.get(warmup_offset..).unwrap_or(&window.test) } else { &window.test }; @@ -669,7 +677,7 @@ fn main() -> Result<()> { Ok((returns, action_counts, directional_accuracy)) => { let metrics = compute_metrics(&returns); info!( - " [{}] Fold {} โ€” Sharpe={:.4} MaxDD={:.2}% WinRate={:.1}% PF={:.2} Return={:.4}% Trades={} DirAcc={:.1}%", + " [{}] Fold {} -- Sharpe={:.4} MaxDD={:.2}% WinRate={:.1}% PF={:.2} Return={:.4}% Trades={} DirAcc={:.1}%", args.model.to_uppercase(), window.fold, metrics.sharpe_ratio, @@ -681,7 +689,7 @@ fn main() -> Result<()> { directional_accuracy, ); info!( - " [{}] Actions โ€” BUY={} SELL={} HOLD={}", + " [{}] Actions -- BUY={} SELL={} HOLD={}", args.model.to_uppercase(), action_counts.first().copied().unwrap_or(0), action_counts.get(1).copied().unwrap_or(0), @@ -748,7 +756,7 @@ fn main() -> Result<()> { }; info!( - " {} โ€” avg Sharpe={:.4} avg MaxDD={:.2}% avg WinRate={:.1}% avg DirAcc={:.1}%", + " {} -- avg Sharpe={:.4} avg MaxDD={:.2}% avg WinRate={:.1}% avg DirAcc={:.1}%", args.model.to_uppercase(), aggregate.avg_sharpe, aggregate.avg_drawdown, diff --git a/crates/ml/examples/hyperopt_baseline_rl.rs b/crates/ml/examples/hyperopt_baseline_rl.rs index d26e8a3fe..41d3bb5af 100644 --- a/crates/ml/examples/hyperopt_baseline_rl.rs +++ b/crates/ml/examples/hyperopt_baseline_rl.rs @@ -83,7 +83,7 @@ struct Args { base_dir: String, /// Symbol subdirectory to load (e.g. "ES.FUT", "NQ.FUT"). - /// Restricts data loading to data_dir/symbol/ to avoid mixing instruments. + /// Restricts data loading to `data_dir`/`symbol`/ to avoid mixing instruments. #[arg(long, default_value = "ES.FUT")] symbol: String, @@ -115,6 +115,7 @@ fn build_model_result( }) } +#[allow(clippy::cognitive_complexity)] fn run_dqn_hyperopt(args: &Args) -> Result { info!("========================================"); info!(" DQN Hyperparameter Optimization"); @@ -162,6 +163,7 @@ fn run_dqn_hyperopt(args: &Args) -> Result { )) } +#[allow(clippy::cognitive_complexity)] fn run_ppo_hyperopt(args: &Args) -> Result { info!("========================================"); info!(" PPO Hyperparameter Optimization"); @@ -210,6 +212,7 @@ fn run_ppo_hyperopt(args: &Args) -> Result { )) } +#[allow(clippy::cognitive_complexity)] fn main() -> Result<()> { // Initialize tracing tracing_subscriber::fmt() @@ -284,13 +287,13 @@ fn main() -> Result<()> { if run_dqn { match run_dqn_hyperopt(&args) { Ok(dqn_result) => { - results.insert("dqn".to_string(), dqn_result); + results.insert("dqn".to_owned(), dqn_result); }, Err(e) => { error!("DQN hyperopt failed: {:#}", e); warn!("Continuing with remaining models..."); results.insert( - "dqn".to_string(), + "dqn".to_owned(), serde_json::json!({ "error": format!("{:#}", e) }), ); }, @@ -301,13 +304,13 @@ fn main() -> Result<()> { if run_ppo { match run_ppo_hyperopt(&args) { Ok(ppo_result) => { - results.insert("ppo".to_string(), ppo_result); + results.insert("ppo".to_owned(), ppo_result); }, Err(e) => { error!("PPO hyperopt failed: {:#}", e); warn!("Continuing..."); results.insert( - "ppo".to_string(), + "ppo".to_owned(), serde_json::json!({ "error": format!("{:#}", e) }), ); }, diff --git a/crates/ml/examples/hyperopt_baseline_supervised.rs b/crates/ml/examples/hyperopt_baseline_supervised.rs index 301f0a410..b60b1b230 100644 --- a/crates/ml/examples/hyperopt_baseline_supervised.rs +++ b/crates/ml/examples/hyperopt_baseline_supervised.rs @@ -105,6 +105,7 @@ fn build_model_result( }) } +#[allow(clippy::cognitive_complexity)] fn run_tft_hyperopt(args: &Args) -> Result { info!("========================================"); info!(" TFT Hyperparameter Optimization"); @@ -152,6 +153,7 @@ fn run_tft_hyperopt(args: &Args) -> Result { )) } +#[allow(clippy::cognitive_complexity)] fn run_mamba2_hyperopt(args: &Args) -> Result { info!("========================================"); info!(" Mamba2 Hyperparameter Optimization"); @@ -198,6 +200,7 @@ fn run_mamba2_hyperopt(args: &Args) -> Result { )) } +#[allow(clippy::cognitive_complexity)] fn run_liquid_hyperopt(args: &Args) -> Result { info!("========================================"); info!(" Liquid Hyperparameter Optimization"); @@ -245,6 +248,7 @@ fn run_liquid_hyperopt(args: &Args) -> Result { )) } +#[allow(clippy::cognitive_complexity)] fn run_tggn_hyperopt(args: &Args) -> Result { info!("========================================"); info!(" TGGN Hyperparameter Optimization"); @@ -292,6 +296,7 @@ fn run_tggn_hyperopt(args: &Args) -> Result { )) } +#[allow(clippy::cognitive_complexity)] fn run_tlob_hyperopt(args: &Args) -> Result { info!("========================================"); info!(" TLOB Hyperparameter Optimization"); @@ -339,6 +344,7 @@ fn run_tlob_hyperopt(args: &Args) -> Result { )) } +#[allow(clippy::cognitive_complexity)] fn run_kan_hyperopt(args: &Args) -> Result { info!("========================================"); info!(" KAN Hyperparameter Optimization"); @@ -386,6 +392,7 @@ fn run_kan_hyperopt(args: &Args) -> Result { )) } +#[allow(clippy::cognitive_complexity)] fn run_xlstm_hyperopt(args: &Args) -> Result { info!("========================================"); info!(" xLSTM Hyperparameter Optimization"); @@ -433,6 +440,7 @@ fn run_xlstm_hyperopt(args: &Args) -> Result { )) } +#[allow(clippy::cognitive_complexity)] fn run_diffusion_hyperopt(args: &Args) -> Result { info!("========================================"); info!(" Diffusion Hyperparameter Optimization"); @@ -484,6 +492,7 @@ const VALID_MODELS: &[&str] = &[ "tft", "mamba2", "liquid", "tggn", "tlob", "kan", "xlstm", "diffusion", ]; +#[allow(clippy::cognitive_complexity)] fn main() -> Result<()> { tracing_subscriber::fmt() .with_max_level(Level::INFO) diff --git a/crates/ml/examples/train_baseline_rl.rs b/crates/ml/examples/train_baseline_rl.rs index 727e895f1..5cff268a5 100644 --- a/crates/ml/examples/train_baseline_rl.rs +++ b/crates/ml/examples/train_baseline_rl.rs @@ -74,7 +74,7 @@ struct Args { #[arg(long)] hyperopt_params: Option, - /// Feature dimension (must match extract_ml_features output) + /// Feature dimension (must match `extract_ml_features` output) #[arg(long, default_value_t = 51)] feature_dim: usize, @@ -130,7 +130,7 @@ struct Args { tick_size: f64, /// Typical bid-ask spread in ticks (ES=1.0, ZN=1.0, 6E=2.0) - /// Half-spread slippage is added to tx_cost_bps per trade. + /// Half-spread slippage is added to `tx_cost_bps` per trade. #[arg(long, default_value_t = 1.0)] spread_ticks: f64, } @@ -139,10 +139,11 @@ struct Args { // Hyperopt parameter loading // --------------------------------------------------------------------------- -/// Load best_params from a hyperopt results JSON file. +/// Load `best_params` from a hyperopt results JSON file. /// /// Expected format: `{ "model_key": { "best_params": { ... }, ... } }` /// Returns `None` if the file doesn't exist or can't be parsed. +#[allow(clippy::cognitive_complexity)] fn load_hyperopt_params(hp_path: &Option, model_key: &str) -> Option { let file_path = hp_path.as_ref()?; if !file_path.exists() { @@ -192,8 +193,8 @@ fn hp_usize(params: &Option, key: &str) -> Option { /// Q-value magnitudes. Total cost (commission + spread slippage) is subtracted /// from BUY/SELL rewards to teach the model that trading isn't free. /// -/// - action 0 (BUY): reward = return_bps - total_cost_bps -/// - action 1 (SELL): reward = -return_bps - total_cost_bps +/// - action 0 (BUY): reward = `return_bps` - `total_cost_bps` +/// - action 1 (SELL): reward = -`return_bps` - `total_cost_bps` /// - action 2 (HOLD): reward = 0 (no cost, no return) fn compute_reward(close_current: f64, close_next: f64, action_idx: u8, total_cost_bps: f64) -> f32 { // Convert to basis points: (next - cur) / cur * 10_000, clipped to ยฑ10 bps @@ -219,6 +220,7 @@ fn compute_reward(close_current: f64, close_next: f64, action_idx: u8, total_cos /// Train a DQN model on a single walk-forward fold. /// /// Returns the best validation loss achieved. +#[allow(clippy::cognitive_complexity)] fn train_dqn_fold( fold: usize, train_features: &[[f64; 51]], @@ -302,13 +304,11 @@ fn train_dqn_fold( // reward computation to avoid misalignment. let warmup_offset = train_bars.len().saturating_sub(n_train); for i in 0..step_limit { - let state_f64 = match train_features.get(i) { - Some(f) => f, - None => continue, + let Some(state_f64) = train_features.get(i) else { + continue; }; - let next_f64 = match train_features.get(i + 1) { - Some(f) => f, - None => continue, + let Some(next_f64) = train_features.get(i + 1) else { + continue; }; let state: Vec = state_f64.iter().map(|&v| v as f32).collect(); @@ -337,14 +337,9 @@ fn train_dqn_fold( } // Train step (returns (loss, grad_norm)) - match dqn.train_step(None) { - Ok((loss, _grad_norm)) => { - epoch_loss += loss as f64; - epoch_steps += 1; - } - Err(_) => { - // Training not ready yet (buffer too small) - } + if let Ok((loss, _grad_norm)) = dqn.train_step(None) { + epoch_loss += loss as f64; + epoch_steps += 1; } } @@ -402,7 +397,7 @@ fn train_dqn_fold( /// Evaluate DQN on validation features and return average loss proxy. /// -/// Since DQN.train_step uses replay buffer internally, we estimate validation +/// Since `DQN.train_step` uses replay buffer internally, we estimate validation /// performance via average absolute reward (lower is closer to zero = better). fn evaluate_dqn_validation( dqn: &mut DQN, @@ -475,6 +470,7 @@ fn evaluate_dqn_validation( /// Train a PPO model on a single walk-forward fold. /// /// Returns the best validation loss achieved. +#[allow(clippy::cognitive_complexity)] fn train_ppo_fold( fold: usize, train_features: &[[f64; 51]], @@ -610,6 +606,7 @@ fn train_ppo_fold( /// Respects `--max-steps-per-epoch` to cap trajectory length and prevent OOM. /// Uses `act_with_log_prob()` to get real policy log-probabilities for /// importance sampling (instead of a uniform prior). +#[allow(clippy::unnecessary_wraps)] fn collect_ppo_trajectory( ppo: &PPO, features: &[[f64; 51]], @@ -642,24 +639,22 @@ fn collect_ppo_trajectory( let warmup_offset = bars.len().saturating_sub(n); for i in 0..step_limit { - let state_f64 = match features.get(i) { - Some(f) => f, - None => break, + let Some(state_f64) = features.get(i) else { + break; }; let state: Vec = state_f64.iter().map(|&v| v as f32).collect(); // Get action, log_prob, and value from PPO policy - let (action, log_prob, value) = match ppo.act_with_log_prob(&state) { - Ok((a, lp, v)) => (a, lp, v), - Err(_) => { - // Fallback: random action with uniform log_prob and zero value - let a = match rng.gen_range(0..args.num_actions) { - 0 => TradingAction::Buy, - 1 => TradingAction::Sell, - _ => TradingAction::Hold, - }; - (a, -(args.num_actions as f32).ln(), 0.0_f32) - } + let (action, log_prob, value) = if let Ok((a, lp, v)) = ppo.act_with_log_prob(&state) { + (a, lp, v) + } else { + // Fallback: random action with uniform log_prob and zero value + let a = match rng.gen_range(0..args.num_actions) { + 0 => TradingAction::Buy, + 1 => TradingAction::Sell, + _ => TradingAction::Hold, + }; + (a, -(args.num_actions as f32).ln(), 0.0_f32) }; // Compute reward (offset by warmup to align features with bars) @@ -751,6 +746,7 @@ struct RlTrainingResult { /// Run the full walk-forward RL training pipeline. /// /// Returns per-model results so `main()` can write completion markers. +#[allow(clippy::cognitive_complexity, clippy::too_many_lines)] fn run_training(args: &Args) -> Result> { let train_dqn = args.model == "dqn" || args.model == "both"; let train_ppo = args.model == "ppo" || args.model == "both"; @@ -1001,11 +997,7 @@ fn main() -> Result<()> { let metrics = CompletionMetrics { model: result.model_name.clone(), symbol: args.symbol.clone(), - best_val_loss: if best_val < f64::MAX { - Some(best_val) - } else { - None - }, + best_val_loss: (best_val < f64::MAX).then_some(best_val), sharpe_ratio: None, epochs_completed: result.total_epochs, folds_completed: result.fold_results.len(), diff --git a/crates/ml/examples/train_baseline_supervised.rs b/crates/ml/examples/train_baseline_supervised.rs index 3a973ee4a..a4254eef8 100644 --- a/crates/ml/examples/train_baseline_supervised.rs +++ b/crates/ml/examples/train_baseline_supervised.rs @@ -87,7 +87,7 @@ struct Args { #[arg(long, default_value_t = 1e-3)] learning_rate: f64, - /// Feature dimension (must match extract_ml_features output) + /// Feature dimension (must match `extract_ml_features` output) #[arg(long, default_value_t = 51)] feature_dim: usize, @@ -164,7 +164,7 @@ fn validate_model(name: &str) -> Result<()> { // Hyperopt parameter loading // --------------------------------------------------------------------------- -/// Load best_params from a hyperopt results JSON file. +/// Load `best_params` from a hyperopt results JSON file. /// /// Expected format: `{ "model_key": { "best_params": { ... }, ... } }` /// Returns `None` if the file doesn't exist or can't be parsed. @@ -359,6 +359,7 @@ fn create_model( /// /// Features are extracted via `extract_ml_features` (51-dim OHLCV market features), /// z-score normalized, and the target is the next-bar return in basis points, clipped. +#[allow(clippy::type_complexity)] fn prepare_fold_data( train_bars: &[OHLCVBar], val_bars: &[OHLCVBar], @@ -622,15 +623,12 @@ fn run_training(args: &Args) -> Result> { }; // Device selection - let device = match Device::cuda_if_available(0) { - Ok(d) => { - info!("Using CUDA device"); - d - } - Err(_) => { - info!("CUDA unavailable, using CPU"); - Device::Cpu - } + let device = if let Ok(d) = Device::cuda_if_available(0) { + info!("Using CUDA device"); + d + } else { + info!("CUDA unavailable, using CPU"); + Device::Cpu }; // Load OHLCV bars @@ -790,11 +788,7 @@ fn main() -> Result<()> { let metrics = CompletionMetrics { model: result.model_name.clone(), symbol: args.symbol.clone(), - best_val_loss: if best_val < f64::MAX { - Some(best_val) - } else { - None - }, + best_val_loss: (best_val < f64::MAX).then_some(best_val), sharpe_ratio: None, epochs_completed: result.total_epochs, folds_completed: result.fold_results.len(), diff --git a/crates/ml/src/kan/network.rs b/crates/ml/src/kan/network.rs index fd0d21d0d..58d6ab552 100644 --- a/crates/ml/src/kan/network.rs +++ b/crates/ml/src/kan/network.rs @@ -29,7 +29,7 @@ impl KANNetwork { reason: "layer_widths must have at least 2 entries".to_owned(), }); } - if config.layer_widths.iter().any(|&w| w == 0) { + if config.layer_widths.contains(&0) { return Err(MLError::ConfigError { reason: "KAN requires all layer_widths > 0".to_owned(), }); diff --git a/crates/ml/src/tft/mod.rs b/crates/ml/src/tft/mod.rs index 2d28c176d..b3de84e32 100644 --- a/crates/ml/src/tft/mod.rs +++ b/crates/ml/src/tft/mod.rs @@ -340,15 +340,15 @@ impl TemporalFusionTransformer { // Create variable selection networks (skip when feature count is 0 โ€” CUDA // cannot handle zero-dim tensors in linear layers) - let static_variable_selection = if config.num_static_features > 0 { - Some(VariableSelectionNetwork::new( - config.num_static_features, - config.hidden_dim, - vs.pp("static_vsn"), - )?) - } else { - None - }; + let static_variable_selection = (config.num_static_features > 0) + .then(|| { + VariableSelectionNetwork::new( + config.num_static_features, + config.hidden_dim, + vs.pp("static_vsn"), + ) + }) + .transpose()?; let historical_variable_selection = VariableSelectionNetwork::new( config.num_unknown_features, @@ -356,28 +356,28 @@ impl TemporalFusionTransformer { vs.pp("historical_vsn"), )?; - let future_variable_selection = if config.num_known_features > 0 { - Some(VariableSelectionNetwork::new( - config.num_known_features, - config.hidden_dim, - vs.pp("future_vsn"), - )?) - } else { - None - }; + let future_variable_selection = (config.num_known_features > 0) + .then(|| { + VariableSelectionNetwork::new( + config.num_known_features, + config.hidden_dim, + vs.pp("future_vsn"), + ) + }) + .transpose()?; // Create encoding stacks (skip when corresponding VSN is absent) - let static_encoder = if config.num_static_features > 0 { - Some(GRNStack::new( - config.hidden_dim, - config.hidden_dim, - config.hidden_dim, - config.num_layers, - vs.pp("static_encoder"), - )?) - } else { - None - }; + let static_encoder = (config.num_static_features > 0) + .then(|| { + GRNStack::new( + config.hidden_dim, + config.hidden_dim, + config.hidden_dim, + config.num_layers, + vs.pp("static_encoder"), + ) + }) + .transpose()?; let historical_encoder = GRNStack::new( config.hidden_dim, @@ -387,17 +387,17 @@ impl TemporalFusionTransformer { vs.pp("historical_encoder"), )?; - let future_encoder = if config.num_known_features > 0 { - Some(GRNStack::new( - config.hidden_dim, - config.hidden_dim, - config.hidden_dim, - config.num_layers, - vs.pp("future_encoder"), - )?) - } else { - None - }; + let future_encoder = (config.num_known_features > 0) + .then(|| { + GRNStack::new( + config.hidden_dim, + config.hidden_dim, + config.hidden_dim, + config.num_layers, + vs.pp("future_encoder"), + ) + }) + .transpose()?; // Simplified LSTM layers (in practice, would use proper LSTM) let lstm_encoder = linear(config.hidden_dim, config.hidden_dim, vs.pp("lstm_encoder"))?; @@ -579,8 +579,11 @@ impl TemporalFusionTransformer { // 1. Variable Selection Networks (skip absent feature paths) let static_encoded = if let Some(ref mut static_vsn) = self.static_variable_selection { let static_selected = static_vsn.forward(static_features, None)?; - let encoder = self.static_encoder.as_mut() - .expect("static_encoder must exist when static_variable_selection exists"); + let encoder = self.static_encoder.as_mut().ok_or_else(|| { + MLError::ModelError( + "static_encoder must exist when static_variable_selection exists".to_owned(), + ) + })?; if use_checkpointing { Some(encoder.forward(&static_selected.detach(), None)?) } else { @@ -604,8 +607,11 @@ impl TemporalFusionTransformer { let future_encoded = if let Some(ref mut future_vsn) = self.future_variable_selection { let future_selected = future_vsn.forward(future_features, None)?; - let encoder = self.future_encoder.as_mut() - .expect("future_encoder must exist when future_variable_selection exists"); + let encoder = self.future_encoder.as_mut().ok_or_else(|| { + MLError::ModelError( + "future_encoder must exist when future_variable_selection exists".to_owned(), + ) + })?; if use_checkpointing { Some(encoder.forward(&future_selected.detach(), None)?) } else { diff --git a/crates/risk/tests/compliance_breach_detection_tests.rs b/crates/risk/tests/compliance_breach_detection_tests.rs index c46ae0109..c6af24f5e 100644 --- a/crates/risk/tests/compliance_breach_detection_tests.rs +++ b/crates/risk/tests/compliance_breach_detection_tests.rs @@ -52,7 +52,7 @@ mod threshold_boundary_tests { #[tokio::test] async fn test_position_exactly_at_limit() { let limit = PositionLimit { - instrument_id: "AAPL".to_string(), + instrument_id: "AAPL".to_owned(), max_position_size: Price::new(100000.0).unwrap(), max_daily_turnover: Price::new(500000.0).unwrap(), concentration_limit: dec!(0.10), @@ -73,7 +73,7 @@ mod threshold_boundary_tests { #[tokio::test] async fn test_position_one_cent_below_limit() { let limit = PositionLimit { - instrument_id: "MSFT".to_string(), + instrument_id: "MSFT".to_owned(), max_position_size: Price::new(100000.0).unwrap(), max_daily_turnover: Price::new(500000.0).unwrap(), concentration_limit: dec!(0.10), @@ -93,7 +93,7 @@ mod threshold_boundary_tests { #[tokio::test] async fn test_position_one_cent_over_limit() { let limit = PositionLimit { - instrument_id: "GOOGL".to_string(), + instrument_id: "GOOGL".to_owned(), max_position_size: Price::new(100000.0).unwrap(), max_daily_turnover: Price::new(500000.0).unwrap(), concentration_limit: dec!(0.10), @@ -112,7 +112,7 @@ mod threshold_boundary_tests { #[tokio::test] async fn test_concentration_at_exact_limit() { let limit = PositionLimit { - instrument_id: "TSLA".to_string(), + instrument_id: "TSLA".to_owned(), max_position_size: Price::new(100000.0).unwrap(), max_daily_turnover: Price::new(500000.0).unwrap(), concentration_limit: dec!(0.10), // 10% max @@ -132,7 +132,7 @@ mod threshold_boundary_tests { async fn test_fractional_position_limits() { // Test with crypto fractional positions let limit = PositionLimit { - instrument_id: "BTC-USD".to_string(), + instrument_id: "BTC-USD".to_owned(), max_position_size: Price::new(450000.0).unwrap(), max_daily_turnover: Price::new(2000000.0).unwrap(), concentration_limit: dec!(0.15), @@ -156,7 +156,7 @@ mod simultaneous_violation_tests { #[tokio::test] async fn test_multiple_limit_breaches_single_order() { let limit = PositionLimit { - instrument_id: "AMZN".to_string(), + instrument_id: "AMZN".to_owned(), max_position_size: Price::new(100000.0).unwrap(), max_daily_turnover: Price::new(500000.0).unwrap(), concentration_limit: dec!(0.10), @@ -170,8 +170,8 @@ mod simultaneous_violation_tests { // Check position limit if limit.current_position > limit.max_position_size { violations.push(ComplianceViolation { - violation_type: "Position Limit".to_string(), - severity: "High".to_string(), + violation_type: "Position Limit".to_owned(), + severity: "High".to_owned(), timestamp: Utc::now(), instrument_id: limit.instrument_id.clone(), exceeded_value: limit.current_position, @@ -182,8 +182,8 @@ mod simultaneous_violation_tests { // Check daily turnover limit if limit.daily_turnover > limit.max_daily_turnover { violations.push(ComplianceViolation { - violation_type: "Daily Turnover".to_string(), - severity: "Medium".to_string(), + violation_type: "Daily Turnover".to_owned(), + severity: "Medium".to_owned(), timestamp: Utc::now(), instrument_id: limit.instrument_id.clone(), exceeded_value: limit.daily_turnover, @@ -196,8 +196,8 @@ mod simultaneous_violation_tests { / limit.portfolio_value.to_decimal().unwrap(); if concentration > limit.concentration_limit { violations.push(ComplianceViolation { - violation_type: "Concentration Risk".to_string(), - severity: "Medium".to_string(), + violation_type: "Concentration Risk".to_owned(), + severity: "Medium".to_owned(), timestamp: Utc::now(), instrument_id: limit.instrument_id.clone(), exceeded_value: Price::new(concentration.to_f64().unwrap_or(0.0)).unwrap(), @@ -220,30 +220,30 @@ mod simultaneous_violation_tests { if current_loss < max_daily_loss { violations.push(ComplianceViolation { - violation_type: "Daily Loss Limit".to_string(), - severity: "Critical".to_string(), + violation_type: "Daily Loss Limit".to_owned(), + severity: "Critical".to_owned(), timestamp: Utc::now(), - instrument_id: "PORTFOLIO".to_string(), + instrument_id: "PORTFOLIO".to_owned(), exceeded_value: current_loss, limit_value: max_daily_loss, }); // 2. This triggers risk budget check violations.push(ComplianceViolation { - violation_type: "Risk Budget Exceeded".to_string(), - severity: "High".to_string(), + violation_type: "Risk Budget Exceeded".to_owned(), + severity: "High".to_owned(), timestamp: Utc::now() + Duration::milliseconds(10), - instrument_id: "PORTFOLIO".to_string(), + instrument_id: "PORTFOLIO".to_owned(), exceeded_value: current_loss, limit_value: max_daily_loss, }); // 3. Which triggers VaR limit check violations.push(ComplianceViolation { - violation_type: "Portfolio VaR Breach".to_string(), - severity: "High".to_string(), + violation_type: "Portfolio VaR Breach".to_owned(), + severity: "High".to_owned(), timestamp: Utc::now() + Duration::milliseconds(20), - instrument_id: "PORTFOLIO".to_string(), + instrument_id: "PORTFOLIO".to_owned(), exceeded_value: Price::new(30000.0).unwrap(), limit_value: Price::new(25000.0).unwrap(), }); diff --git a/crates/risk/tests/compliance_comprehensive_tests.rs b/crates/risk/tests/compliance_comprehensive_tests.rs index 3f6f30275..0c14b6447 100644 --- a/crates/risk/tests/compliance_comprehensive_tests.rs +++ b/crates/risk/tests/compliance_comprehensive_tests.rs @@ -36,11 +36,11 @@ mod mifid_ii_compliance_tests { // MiFID II transaction reporting fields let mut transaction_report: HashMap = HashMap::new(); - transaction_report.insert("instrument_id".to_string(), "ISIN:US0378331005".to_string()); - transaction_report.insert("trading_venue".to_string(), "XNYS".to_string()); - transaction_report.insert("buyer_id".to_string(), "LEI:XXXXXX".to_string()); - transaction_report.insert("seller_id".to_string(), "LEI:YYYYYY".to_string()); - transaction_report.insert("timestamp".to_string(), Utc::now().to_rfc3339()); + transaction_report.insert("instrument_id".to_owned(), "ISIN:US0378331005".to_owned()); + transaction_report.insert("trading_venue".to_owned(), "XNYS".to_owned()); + transaction_report.insert("buyer_id".to_owned(), "LEI:XXXXXX".to_owned()); + transaction_report.insert("seller_id".to_owned(), "LEI:YYYYYY".to_owned()); + transaction_report.insert("timestamp".to_owned(), Utc::now().to_rfc3339()); assert!(transaction_report.contains_key("instrument_id")); assert!(transaction_report.contains_key("trading_venue")); @@ -117,9 +117,9 @@ mod position_limit_compliance_tests { #[test] fn test_multiple_position_limits() { let mut limits: HashMap = HashMap::new(); - limits.insert("daily_limit".to_string(), 100_000.0); - limits.insert("monthly_limit".to_string(), 500_000.0); - limits.insert("annual_limit".to_string(), 2_000_000.0); + limits.insert("daily_limit".to_owned(), 100_000.0); + limits.insert("monthly_limit".to_owned(), 500_000.0); + limits.insert("annual_limit".to_owned(), 2_000_000.0); let current_position = 75_000.0; @@ -139,9 +139,9 @@ mod audit_trail_tests { fn test_audit_entry_creation() { let audit_entry = HashMap::from([ ("timestamp", Utc::now().to_rfc3339()), - ("user_id", "trader_123".to_string()), - ("action", "PLACE_ORDER".to_string()), - ("details", "Buy 100 AAPL @ 150.00".to_string()), + ("user_id", "trader_123".to_owned()), + ("action", "PLACE_ORDER".to_owned()), + ("details", "Buy 100 AAPL @ 150.00".to_owned()), ]); assert_eq!(audit_entry.get("action").unwrap(), "PLACE_ORDER"); @@ -152,8 +152,8 @@ mod audit_trail_tests { let mut audit_log: Vec> = Vec::new(); let entry1 = HashMap::from([ - ("id".to_string(), "1".to_string()), - ("action".to_string(), "ORDER_PLACED".to_string()), + ("id".to_owned(), "1".to_owned()), + ("action".to_owned(), "ORDER_PLACED".to_owned()), ]); audit_log.push(entry1); @@ -177,12 +177,12 @@ mod audit_trail_tests { ]; let audit_entry = HashMap::from([ - ("timestamp", "2025-10-03T12:00:00Z".to_string()), - ("user_id", "trader_123".to_string()), - ("action", "BUY".to_string()), - ("instrument", "AAPL".to_string()), - ("quantity", "100".to_string()), - ("price", "150.00".to_string()), + ("timestamp", "2025-10-03T12:00:00Z".to_owned()), + ("user_id", "trader_123".to_owned()), + ("action", "BUY".to_owned()), + ("instrument", "AAPL".to_owned()), + ("quantity", "100".to_owned()), + ("price", "150.00".to_owned()), ]); for field in &required_fields { @@ -194,9 +194,9 @@ mod audit_trail_tests { fn test_audit_trail_ordering() { let mut audit_log: Vec<(i64, String)> = Vec::new(); - audit_log.push((1, "First action".to_string())); - audit_log.push((2, "Second action".to_string())); - audit_log.push((3, "Third action".to_string())); + audit_log.push((1, "First action".to_owned())); + audit_log.push((2, "Second action".to_owned())); + audit_log.push((3, "Third action".to_owned())); // Should maintain chronological order assert_eq!(audit_log[0].0, 1); @@ -254,12 +254,12 @@ mod violation_detection_tests { // Check position limit if 150_000.0 > 100_000.0 { - violations.push("Position limit exceeded".to_string()); + violations.push("Position limit exceeded".to_owned()); } // Check loss limit if -25_000.0 < -20_000.0 { - violations.push("Loss limit exceeded".to_string()); + violations.push("Loss limit exceeded".to_owned()); } assert_eq!(violations.len(), 2); @@ -340,19 +340,19 @@ mod regulatory_flag_tests { if true { // Is algorithmic - flags.push("ALGO".to_string()); + flags.push("ALGO".to_owned()); } if true { // Large in scale - flags.push("LIS".to_string()); + flags.push("LIS".to_owned()); } if false { // Not a short sale // No flag } assert_eq!(flags.len(), 2); - assert!(flags.contains(&"ALGO".to_string())); - assert!(flags.contains(&"LIS".to_string())); + assert!(flags.contains(&"ALGO".to_owned())); + assert!(flags.contains(&"LIS".to_owned())); } } diff --git a/crates/risk/tests/compliance_edge_cases_tests.rs b/crates/risk/tests/compliance_edge_cases_tests.rs index 8da4f06a4..23663c5d7 100644 --- a/crates/risk/tests/compliance_edge_cases_tests.rs +++ b/crates/risk/tests/compliance_edge_cases_tests.rs @@ -27,7 +27,7 @@ mod threshold_boundary_tests { #[test] fn test_position_exactly_at_limit() { let limit = PositionLimit { - symbol: "AAPL".to_string(), + symbol: "AAPL".to_owned(), max_quantity: 10000.0, current_quantity: 10000.0, }; @@ -39,7 +39,7 @@ mod threshold_boundary_tests { #[test] fn test_position_one_unit_over_limit() { let limit = PositionLimit { - symbol: "AAPL".to_string(), + symbol: "AAPL".to_owned(), max_quantity: 10000.0, current_quantity: 10001.0, }; @@ -50,7 +50,7 @@ mod threshold_boundary_tests { #[test] fn test_position_one_unit_under_limit() { let limit = PositionLimit { - symbol: "AAPL".to_string(), + symbol: "AAPL".to_owned(), max_quantity: 10000.0, current_quantity: 9999.0, }; @@ -61,7 +61,7 @@ mod threshold_boundary_tests { #[test] fn test_fractional_position_at_limit() { let limit = PositionLimit { - symbol: "BTC".to_string(), + symbol: "BTC".to_owned(), max_quantity: 1.5, current_quantity: 1.5, }; @@ -72,7 +72,7 @@ mod threshold_boundary_tests { #[test] fn test_zero_position_limit() { let limit = PositionLimit { - symbol: "RESTRICTED".to_string(), + symbol: "RESTRICTED".to_owned(), max_quantity: 0.0, current_quantity: 0.0, }; @@ -90,18 +90,18 @@ mod simultaneous_violation_tests { fn test_multiple_simultaneous_violations() { let violations = vec![ ComplianceViolation { - violation_type: "Position Limit".to_string(), - severity: "High".to_string(), + violation_type: "Position Limit".to_owned(), + severity: "High".to_owned(), timestamp: Utc::now(), }, ComplianceViolation { - violation_type: "Concentration Risk".to_string(), - severity: "Medium".to_string(), + violation_type: "Concentration Risk".to_owned(), + severity: "Medium".to_owned(), timestamp: Utc::now(), }, ComplianceViolation { - violation_type: "Sector Exposure".to_string(), - severity: "Low".to_string(), + violation_type: "Sector Exposure".to_owned(), + severity: "Low".to_owned(), timestamp: Utc::now(), }, ]; @@ -121,21 +121,21 @@ mod simultaneous_violation_tests { // Primary violation violations.push(ComplianceViolation { - violation_type: "Daily Loss Limit".to_string(), - severity: "Critical".to_string(), + violation_type: "Daily Loss Limit".to_owned(), + severity: "Critical".to_owned(), timestamp: Utc::now(), }); // Cascading violations triggered by primary violations.push(ComplianceViolation { - violation_type: "Risk Budget Exceeded".to_string(), - severity: "High".to_string(), + violation_type: "Risk Budget Exceeded".to_owned(), + severity: "High".to_owned(), timestamp: Utc::now() + Duration::milliseconds(10), }); violations.push(ComplianceViolation { - violation_type: "Portfolio VaR Breach".to_string(), - severity: "High".to_string(), + violation_type: "Portfolio VaR Breach".to_owned(), + severity: "High".to_owned(), timestamp: Utc::now() + Duration::milliseconds(20), }); @@ -168,8 +168,8 @@ mod violation_during_market_close_tests { .and_utc(); let violation = ComplianceViolation { - violation_type: "EOD Position Check".to_string(), - severity: "High".to_string(), + violation_type: "EOD Position Check".to_owned(), + severity: "High".to_owned(), timestamp: market_close, }; @@ -187,8 +187,8 @@ mod violation_during_market_close_tests { .and_utc(); let violation = ComplianceViolation { - violation_type: "After Hours Trading".to_string(), - severity: "Medium".to_string(), + violation_type: "After Hours Trading".to_owned(), + severity: "Medium".to_owned(), timestamp: after_close, }; @@ -311,8 +311,8 @@ mod emergency_override_tests { #[test] fn test_emergency_override_audit_trail() { let override_event = ComplianceViolation { - violation_type: "Emergency Override Used".to_string(), - severity: "Critical".to_string(), + violation_type: "Emergency Override Used".to_owned(), + severity: "Critical".to_owned(), timestamp: Utc::now(), }; @@ -437,8 +437,8 @@ mod regulatory_reporting_edge_cases { fn test_violation_count_threshold_reporting() { let violations = vec![ ComplianceViolation { - violation_type: "Minor Breach".to_string(), - severity: "Low".to_string(), + violation_type: "Minor Breach".to_owned(), + severity: "Low".to_owned(), timestamp: Utc::now(), }; 10 @@ -453,8 +453,8 @@ mod regulatory_reporting_edge_cases { #[test] fn test_material_violation_immediate_reporting() { let violation = ComplianceViolation { - violation_type: "Market Manipulation".to_string(), - severity: "Critical".to_string(), + violation_type: "Market Manipulation".to_owned(), + severity: "Critical".to_owned(), timestamp: Utc::now(), }; @@ -468,8 +468,8 @@ mod regulatory_reporting_edge_cases { fn test_cross_day_violation_aggregation() { let day1_violations = vec![ ComplianceViolation { - violation_type: "Position Limit".to_string(), - severity: "Medium".to_string(), + violation_type: "Position Limit".to_owned(), + severity: "Medium".to_owned(), timestamp: Utc::now() - Duration::days(1), }; 3 @@ -477,8 +477,8 @@ mod regulatory_reporting_edge_cases { let day2_violations = vec![ ComplianceViolation { - violation_type: "Position Limit".to_string(), - severity: "Medium".to_string(), + violation_type: "Position Limit".to_owned(), + severity: "Medium".to_owned(), timestamp: Utc::now(), }; 2 diff --git a/crates/risk/tests/emergency_response_comprehensive_tests.rs b/crates/risk/tests/emergency_response_comprehensive_tests.rs index 4700a93e8..15a419e25 100644 --- a/crates/risk/tests/emergency_response_comprehensive_tests.rs +++ b/crates/risk/tests/emergency_response_comprehensive_tests.rs @@ -316,9 +316,9 @@ mod incident_response_tests { let mut incident_log: Vec> = Vec::new(); let incident = HashMap::from([ - ("timestamp".to_string(), Utc::now().to_rfc3339()), - ("type".to_string(), "POSITION_LIMIT_BREACH".to_string()), - ("severity".to_string(), "high".to_string()), + ("timestamp".to_owned(), Utc::now().to_rfc3339()), + ("type".to_owned(), "POSITION_LIMIT_BREACH".to_owned()), + ("severity".to_owned(), "high".to_owned()), ]); incident_log.push(incident); @@ -420,9 +420,9 @@ mod health_check_tests { fn test_system_health_indicators() { let mut health_status: HashMap = HashMap::new(); - health_status.insert("market_data".to_string(), true); - health_status.insert("risk_engine".to_string(), true); - health_status.insert("execution".to_string(), true); + health_status.insert("market_data".to_owned(), true); + health_status.insert("risk_engine".to_owned(), true); + health_status.insert("execution".to_owned(), true); let all_healthy = health_status.values().all(|&v| v); assert!(all_healthy); @@ -432,9 +432,9 @@ mod health_check_tests { fn test_degraded_mode_detection() { let mut health_status: HashMap = HashMap::new(); - health_status.insert("market_data".to_string(), true); - health_status.insert("risk_engine".to_string(), false); // Degraded - health_status.insert("execution".to_string(), true); + health_status.insert("market_data".to_owned(), true); + health_status.insert("risk_engine".to_owned(), false); // Degraded + health_status.insert("execution".to_owned(), true); let is_degraded = health_status.values().any(|&v| !v); assert!(is_degraded); @@ -488,10 +488,10 @@ mod emergency_shutdown_tests { fn test_orderly_shutdown_sequence() { let mut shutdown_steps: Vec = Vec::new(); - shutdown_steps.push("STOP_NEW_ORDERS".to_string()); - shutdown_steps.push("CLOSE_POSITIONS".to_string()); - shutdown_steps.push("DISCONNECT_FEEDS".to_string()); - shutdown_steps.push("HALT_TRADING".to_string()); + shutdown_steps.push("STOP_NEW_ORDERS".to_owned()); + shutdown_steps.push("CLOSE_POSITIONS".to_owned()); + shutdown_steps.push("DISCONNECT_FEEDS".to_owned()); + shutdown_steps.push("HALT_TRADING".to_owned()); assert_eq!(shutdown_steps.len(), 4); assert_eq!(shutdown_steps[0], "STOP_NEW_ORDERS"); @@ -552,9 +552,9 @@ mod circuit_breaker_coordination_tests { fn test_multiple_circuit_breakers() { let mut breakers: HashMap = HashMap::new(); - breakers.insert("position_limit".to_string(), false); - breakers.insert("loss_limit".to_string(), false); - breakers.insert("volatility".to_string(), true); // Triggered + breakers.insert("position_limit".to_owned(), false); + breakers.insert("loss_limit".to_owned(), false); + breakers.insert("volatility".to_owned(), true); // Triggered let any_triggered = breakers.values().any(|&v| v); assert!(any_triggered); diff --git a/crates/risk/tests/portfolio_optimization_tests.rs b/crates/risk/tests/portfolio_optimization_tests.rs index e6a36bd00..8dc782c0e 100644 --- a/crates/risk/tests/portfolio_optimization_tests.rs +++ b/crates/risk/tests/portfolio_optimization_tests.rs @@ -20,7 +20,7 @@ use risk::portfolio_optimization::{OptimizationMethod, PortfolioConstraints, Por /// Create a simple 3-asset portfolio for testing fn create_simple_portfolio() -> PortfolioOptimizer { - let assets = vec!["AAPL".to_string(), "GOOGL".to_string(), "MSFT".to_string()]; + let assets = vec!["AAPL".to_owned(), "GOOGL".to_owned(), "MSFT".to_owned()]; let returns = vec![0.10, 0.12, 0.08]; // 10%, 12%, 8% expected returns let covariance = vec![ vec![0.04, 0.01, 0.02], // AAPL variance = 0.04 (20% vol) @@ -40,7 +40,7 @@ fn create_simple_portfolio() -> PortfolioOptimizer { /// Create a 2-asset portfolio with no correlation fn create_uncorrelated_portfolio() -> PortfolioOptimizer { - let assets = vec!["A".to_string(), "B".to_string()]; + let assets = vec!["A".to_owned(), "B".to_owned()]; let returns = vec![0.10, 0.15]; let covariance = vec![ vec![0.04, 0.00], // No correlation @@ -59,7 +59,7 @@ fn create_uncorrelated_portfolio() -> PortfolioOptimizer { /// Create a portfolio with highly correlated assets (multicollinearity) fn create_correlated_portfolio() -> PortfolioOptimizer { - let assets = vec!["X".to_string(), "Y".to_string(), "Z".to_string()]; + let assets = vec!["X".to_owned(), "Y".to_owned(), "Z".to_owned()]; let returns = vec![0.10, 0.11, 0.12]; let covariance = vec![ vec![0.04, 0.038, 0.037], // Very high correlation @@ -79,7 +79,7 @@ fn create_correlated_portfolio() -> PortfolioOptimizer { /// Create a portfolio with singular covariance matrix fn create_singular_portfolio() -> PortfolioOptimizer { - let assets = vec!["P".to_string(), "Q".to_string()]; + let assets = vec!["P".to_owned(), "Q".to_owned()]; let returns = vec![0.10, 0.10]; let covariance = vec![ vec![0.04, 0.04], // Perfectly correlated (singular) @@ -111,7 +111,7 @@ fn test_portfolio_optimizer_creation_valid() { #[test] fn test_portfolio_optimizer_creation_mismatched_returns() { - let assets = vec!["A".to_string(), "B".to_string()]; + let assets = vec!["A".to_owned(), "B".to_owned()]; let returns = vec![0.10]; // Wrong size let covariance = vec![vec![0.04, 0.00], vec![0.00, 0.09]]; @@ -128,7 +128,7 @@ fn test_portfolio_optimizer_creation_mismatched_returns() { #[test] fn test_portfolio_optimizer_creation_non_square_covariance() { - let assets = vec!["A".to_string(), "B".to_string()]; + let assets = vec!["A".to_owned(), "B".to_owned()]; let returns = vec![0.10, 0.15]; let covariance = vec![ vec![0.04, 0.00, 0.01], // Extra column @@ -148,7 +148,7 @@ fn test_portfolio_optimizer_creation_non_square_covariance() { #[test] fn test_portfolio_optimizer_creation_mismatched_covariance_size() { - let assets = vec!["A".to_string(), "B".to_string()]; + let assets = vec!["A".to_owned(), "B".to_owned()]; let returns = vec![0.10, 0.15]; let covariance = vec![ vec![0.04, 0.00, 0.00], // Wrong dimensions @@ -216,7 +216,7 @@ fn test_sharpe_ratio_calculation() { #[test] fn test_sharpe_ratio_zero_volatility() { // Portfolio with zero volatility (all weight in risk-free asset) - let assets = vec!["RF".to_string()]; + let assets = vec!["RF".to_owned()]; let returns = vec![0.02]; // Risk-free return let covariance = vec![vec![0.0]]; // Zero variance @@ -274,7 +274,7 @@ fn test_mean_variance_optimization_uncorrelated() { #[test] fn test_mean_variance_optimization_negative_returns() { // Portfolio with negative expected returns (short-only scenario) - let assets = vec!["DOWN1".to_string(), "DOWN2".to_string()]; + let assets = vec!["DOWN1".to_owned(), "DOWN2".to_owned()]; let returns = vec![-0.10, -0.05]; // Negative returns let covariance = vec![vec![0.04, 0.01], vec![0.01, 0.09]]; @@ -320,7 +320,7 @@ fn test_minimum_variance_optimization() { #[test] fn test_minimum_variance_single_asset() { // Single asset portfolio - let assets = vec!["SOLO".to_string()]; + let assets = vec!["SOLO".to_owned()]; let returns = vec![0.10]; let covariance = vec![vec![0.04]]; @@ -405,7 +405,7 @@ fn test_kelly_criterion_optimization() { #[test] fn test_kelly_criterion_growth_optimal() { // Kelly criterion should maximize geometric growth - let assets = vec!["GROWTH".to_string(), "VALUE".to_string()]; + let assets = vec!["GROWTH".to_owned(), "VALUE".to_owned()]; let returns = vec![0.15, 0.08]; // Growth has higher return let covariance = vec![vec![0.09, 0.01], vec![0.01, 0.04]]; // Growth has higher vol @@ -427,7 +427,7 @@ fn test_kelly_criterion_growth_optimal() { #[test] fn test_kelly_criterion_with_zero_returns() { - let assets = vec!["A".to_string(), "B".to_string()]; + let assets = vec!["A".to_owned(), "B".to_owned()]; let returns = vec![0.0, 0.0]; // Zero expected returns let covariance = vec![vec![0.04, 0.00], vec![0.00, 0.09]]; @@ -469,7 +469,7 @@ fn test_risk_parity_optimization() { #[test] fn test_risk_parity_equal_risk_contribution() { // Two assets with different volatilities - let assets = vec!["LOW_VOL".to_string(), "HIGH_VOL".to_string()]; + let assets = vec!["LOW_VOL".to_owned(), "HIGH_VOL".to_owned()]; let returns = vec![0.08, 0.12]; let covariance = vec![ vec![0.01, 0.00], // 10% vol @@ -546,7 +546,7 @@ fn test_long_only_constraint() { #[test] fn test_max_position_constraint() { - let assets = vec!["A".to_string(), "B".to_string(), "C".to_string()]; + let assets = vec!["A".to_owned(), "B".to_owned(), "C".to_owned()]; let returns = vec![0.20, 0.10, 0.05]; // A has much higher return let covariance = vec![ vec![0.04, 0.00, 0.00], @@ -572,7 +572,7 @@ fn test_max_position_constraint() { #[test] fn test_minimum_position_constraint() { - let assets = vec!["A".to_string(), "B".to_string()]; + let assets = vec!["A".to_owned(), "B".to_owned()]; let returns = vec![0.10, 0.15]; let covariance = vec![vec![0.04, 0.00], vec![0.00, 0.09]]; @@ -627,7 +627,7 @@ fn test_empty_portfolio() { #[test] fn test_single_asset_portfolio() { - let assets = vec!["ONLY".to_string()]; + let assets = vec!["ONLY".to_owned()]; let returns = vec![0.10]; let covariance = vec![vec![0.04]]; @@ -675,7 +675,7 @@ fn test_singular_covariance_matrix() { #[test] fn test_zero_variance_asset() { // Asset with zero variance (risk-free) - let assets = vec!["RF".to_string(), "RISKY".to_string()]; + let assets = vec!["RF".to_owned(), "RISKY".to_owned()]; let returns = vec![0.02, 0.12]; let covariance = vec![ vec![0.0, 0.0], // Risk-free has zero variance @@ -703,7 +703,7 @@ fn test_zero_variance_asset() { #[test] fn test_negative_risk_free_rate() { // Negative risk-free rate (like Japan, Europe in 2020s) - let assets = vec!["A".to_string(), "B".to_string()]; + let assets = vec!["A".to_owned(), "B".to_owned()]; let returns = vec![0.05, 0.08]; let covariance = vec![vec![0.04, 0.01], vec![0.01, 0.09]]; @@ -835,7 +835,7 @@ fn test_efficient_frontier_monotonicity() { #[test] fn test_numerical_stability_large_numbers() { // Portfolio with large covariance values - let assets = vec!["BIG1".to_string(), "BIG2".to_string()]; + let assets = vec!["BIG1".to_owned(), "BIG2".to_owned()]; let returns = vec![0.10, 0.15]; let covariance = vec![ vec![1000.0, 100.0], // Large variances @@ -863,7 +863,7 @@ fn test_numerical_stability_large_numbers() { #[test] fn test_numerical_stability_small_numbers() { // Portfolio with very small covariance values - let assets = vec!["SMALL1".to_string(), "SMALL2".to_string()]; + let assets = vec!["SMALL1".to_owned(), "SMALL2".to_owned()]; let returns = vec![0.001, 0.002]; let covariance = vec![ vec![0.0001, 0.00001], // Small variances @@ -891,7 +891,7 @@ fn test_numerical_stability_small_numbers() { #[test] fn test_numerical_stability_ill_conditioned_matrix() { // Ill-conditioned covariance matrix (high condition number) - let assets = vec!["ILL1".to_string(), "ILL2".to_string(), "ILL3".to_string()]; + let assets = vec!["ILL1".to_owned(), "ILL2".to_owned(), "ILL3".to_owned()]; let returns = vec![0.10, 0.11, 0.12]; let covariance = vec![ vec![1.0, 0.99, 0.98], diff --git a/crates/risk/tests/position_limit_enforcement_tests.rs b/crates/risk/tests/position_limit_enforcement_tests.rs index 1657a2022..154ec65f6 100644 --- a/crates/risk/tests/position_limit_enforcement_tests.rs +++ b/crates/risk/tests/position_limit_enforcement_tests.rs @@ -30,17 +30,17 @@ mod concurrent_order_tests { let orders = vec![ Order { - symbol: "AAPL".to_string(), + symbol: "AAPL".to_owned(), quantity: 3000.0, timestamp: Utc::now(), }, Order { - symbol: "AAPL".to_string(), + symbol: "AAPL".to_owned(), quantity: 3000.0, timestamp: Utc::now(), }, Order { - symbol: "AAPL".to_string(), + symbol: "AAPL".to_owned(), quantity: 3000.0, timestamp: Utc::now(), }, diff --git a/crates/risk/tests/risk_comprehensive_tests.rs b/crates/risk/tests/risk_comprehensive_tests.rs index c5423738c..ef8361b95 100644 --- a/crates/risk/tests/risk_comprehensive_tests.rs +++ b/crates/risk/tests/risk_comprehensive_tests.rs @@ -132,7 +132,7 @@ async fn test_var_multi_asset_crisis_portfolio() { confidence_level: 0.95, time_horizon_days: 1, lookback_period_days: 252, - calculation_method: "parametric".to_string(), + calculation_method: "parametric".to_owned(), max_var_limit: 10.0, }; @@ -178,7 +178,7 @@ async fn test_var_leveraged_portfolio() { confidence_level: 0.99, time_horizon_days: 1, lookback_period_days: 252, - calculation_method: "parametric".to_string(), + calculation_method: "parametric".to_owned(), max_var_limit: 20.0, }; @@ -237,7 +237,7 @@ async fn test_var_hedged_portfolio() { confidence_level: 0.95, time_horizon_days: 1, lookback_period_days: 252, - calculation_method: "parametric".to_string(), + calculation_method: "parametric".to_owned(), max_var_limit: 5.0, }; @@ -384,7 +384,7 @@ async fn test_var_intraday_recalculation() { confidence_level: 0.95, time_horizon_days: 1, lookback_period_days: 252, - calculation_method: "parametric".to_string(), + calculation_method: "parametric".to_owned(), max_var_limit: 5.0, }; @@ -1123,10 +1123,10 @@ async fn test_risk_violation_audit_trail() { let violation = RiskViolation { timestamp: Utc::now(), - violation_type: "POSITION_LIMIT_BREACH".to_string(), - severity: "HIGH".to_string(), - account_id: "ACCT123".to_string(), - details: "Position size 6% exceeds 5% limit".to_string(), + violation_type: "POSITION_LIMIT_BREACH".to_owned(), + severity: "HIGH".to_owned(), + account_id: "ACCT123".to_owned(), + details: "Position size 6% exceeds 5% limit".to_owned(), }; // Verify violation is properly structured for audit @@ -1157,13 +1157,13 @@ async fn test_compliance_event_generation() { } let event = ComplianceEvent { - event_id: "CE-2024-001".to_string(), + event_id: "CE-2024-001".to_owned(), timestamp: Utc::now(), - event_type: "VAR_LIMIT_BREACH".to_string(), - risk_metric: "PORTFOLIO_VAR".to_string(), + event_type: "VAR_LIMIT_BREACH".to_owned(), + risk_metric: "PORTFOLIO_VAR".to_owned(), threshold: 0.05, // 5% VaR limit actual_value: 0.062, // 6.2% actual VaR - action_taken: "TRADING_HALTED".to_string(), + action_taken: "TRADING_HALTED".to_owned(), }; assert_eq!(event.event_type, "VAR_LIMIT_BREACH"); diff --git a/crates/risk/tests/var_calculator_edge_cases_tests.rs b/crates/risk/tests/var_calculator_edge_cases_tests.rs index b24067a4a..28b7d1395 100644 --- a/crates/risk/tests/var_calculator_edge_cases_tests.rs +++ b/crates/risk/tests/var_calculator_edge_cases_tests.rs @@ -22,7 +22,7 @@ fn create_position(symbol: &str, quantity: f64, market_price: f64) -> PositionIn average_cost: Price::from_f64(market_price * 0.95).unwrap_or(Price::ZERO), unrealized_pnl: Price::from_f64(quantity * market_price * 0.05).unwrap_or(Price::ZERO), realized_pnl: Price::ZERO, - currency: "USD".to_string(), + currency: "USD".to_owned(), timestamp: Utc::now(), } } @@ -36,7 +36,7 @@ fn create_prices( ) -> Vec { let mut prices = Vec::new(); let mut current_price = base_price; - let mut rng = 12345u64; + let mut rng = 12_345_u64; for i in 0..days { rng = rng.wrapping_mul(1664525).wrapping_add(1013904223); @@ -69,7 +69,7 @@ mod zero_position_tests { let position = create_position("AAPL", 0.0, 150.0); // Zero quantity let result = calculator.calculate_position_var( - &Symbol::from("AAPL".to_string()), + &Symbol::from("AAPL".to_owned()), &position, &prices, ); @@ -87,7 +87,7 @@ mod zero_position_tests { let position = create_position("AAPL", 100.0, 0.0); // Zero price let result = calculator.calculate_position_var( - &Symbol::from("AAPL".to_string()), + &Symbol::from("AAPL".to_owned()), &position, &prices, ); @@ -103,13 +103,13 @@ mod zero_position_tests { let mut positions = HashMap::new(); positions.insert( - Symbol::from("AAPL".to_string()), + Symbol::from("AAPL".to_owned()), create_position("AAPL", 0.0, 150.0), ); let mut historical_prices = HashMap::new(); historical_prices.insert( - Symbol::from("AAPL".to_string()), + Symbol::from("AAPL".to_owned()), create_prices("AAPL", 100, 150.0, 0.02), ); @@ -134,7 +134,7 @@ mod insufficient_data_tests { let position = create_position("AAPL", 100.0, 150.0); let result = calculator.calculate_position_var( - &Symbol::from("AAPL".to_string()), + &Symbol::from("AAPL".to_owned()), &position, &prices, ); @@ -148,13 +148,13 @@ mod insufficient_data_tests { let mut positions = HashMap::new(); positions.insert( - Symbol::from("AAPL".to_string()), + Symbol::from("AAPL".to_owned()), create_position("AAPL", 100.0, 150.0), ); let mut historical_prices = HashMap::new(); historical_prices.insert( - Symbol::from("AAPL".to_string()), + Symbol::from("AAPL".to_owned()), create_prices("AAPL", 20, 150.0, 0.02), // Less than 30 required ); @@ -170,7 +170,7 @@ mod insufficient_data_tests { let position = create_position("AAPL", 100.0, 150.0); let result = calculator.calculate_position_var( - &Symbol::from("AAPL".to_string()), + &Symbol::from("AAPL".to_owned()), &position, &prices, ); @@ -190,7 +190,7 @@ mod extreme_volatility_tests { let position = create_position("VOLATILE", 100.0, 100.0); let result = calculator.calculate_position_var( - &Symbol::from("VOLATILE".to_string()), + &Symbol::from("VOLATILE".to_owned()), &position, &prices, ); @@ -208,7 +208,7 @@ mod extreme_volatility_tests { let mut prices = Vec::new(); for i in 0..300 { prices.push(HistoricalPrice { - symbol: "FLAT".to_string(), + symbol: "FLAT".to_owned(), date: Utc::now() - Duration::days(300 - i as i64), open: Price::from_f64(100.0).unwrap(), high: Price::from_f64(100.0).unwrap(), @@ -221,7 +221,7 @@ mod extreme_volatility_tests { let position = create_position("FLAT", 100.0, 100.0); let result = calculator.calculate_position_var( - &Symbol::from("FLAT".to_string()), + &Symbol::from("FLAT".to_owned()), &position, &prices, ); @@ -239,22 +239,22 @@ mod extreme_volatility_tests { // Create two assets with different volatilities to avoid singular correlation matrix let mut positions = HashMap::new(); positions.insert( - Symbol::from("TECH".to_string()), + Symbol::from("TECH".to_owned()), create_position("TECH", 100.0, 100.0), ); positions.insert( - Symbol::from("ENERGY".to_string()), + Symbol::from("ENERGY".to_owned()), create_position("ENERGY", 100.0, 100.0), ); let mut historical_prices = HashMap::new(); // Use different volatilities to ensure non-singular correlation matrix historical_prices.insert( - Symbol::from("TECH".to_string()), + Symbol::from("TECH".to_owned()), create_prices("TECH", 100, 100.0, 0.03), // 3% volatility ); historical_prices.insert( - Symbol::from("ENERGY".to_string()), + Symbol::from("ENERGY".to_owned()), create_prices("ENERGY", 100, 100.0, 0.04), // 4% volatility ); @@ -285,7 +285,7 @@ mod negative_price_tests { for i in 0..300 { current_price *= 0.98; // 2% daily decline prices.push(HistoricalPrice { - symbol: "DECLINING".to_string(), + symbol: "DECLINING".to_owned(), date: Utc::now() - Duration::days(300 - i as i64), open: Price::from_f64(current_price * 1.01).unwrap(), high: Price::from_f64(current_price * 1.02).unwrap(), @@ -298,7 +298,7 @@ mod negative_price_tests { let position = create_position("DECLINING", 100.0, current_price); let result = calculator.calculate_position_var( - &Symbol::from("DECLINING".to_string()), + &Symbol::from("DECLINING".to_owned()), &position, &prices, ); @@ -322,11 +322,11 @@ mod confidence_level_tests { let calc_99 = HistoricalSimulationVaR::new(0.99, 252); let result_95 = calc_95 - .calculate_position_var(&Symbol::from("AAPL".to_string()), &position, &prices) + .calculate_position_var(&Symbol::from("AAPL".to_owned()), &position, &prices) .unwrap(); let result_99 = calc_99 - .calculate_position_var(&Symbol::from("AAPL".to_string()), &position, &prices) + .calculate_position_var(&Symbol::from("AAPL".to_owned()), &position, &prices) .unwrap(); // 99% VaR should be higher than 95% VaR @@ -337,13 +337,13 @@ mod confidence_level_tests { fn test_monte_carlo_confidence_levels() { let mut positions = HashMap::new(); positions.insert( - Symbol::from("AAPL".to_string()), + Symbol::from("AAPL".to_owned()), create_position("AAPL", 100.0, 150.0), ); let mut historical_prices = HashMap::new(); historical_prices.insert( - Symbol::from("AAPL".to_string()), + Symbol::from("AAPL".to_owned()), create_prices("AAPL", 100, 150.0, 0.02), ); @@ -374,7 +374,7 @@ mod time_scaling_tests { let position = create_position("AAPL", 100.0, 150.0); let result = calculator - .calculate_position_var(&Symbol::from("AAPL".to_string()), &position, &prices) + .calculate_position_var(&Symbol::from("AAPL".to_owned()), &position, &prices) .unwrap(); // 10-day VaR should be approximately sqrt(10) * 1-day VaR @@ -390,13 +390,13 @@ mod time_scaling_tests { fn test_monte_carlo_time_horizon_scaling() { let mut positions = HashMap::new(); positions.insert( - Symbol::from("AAPL".to_string()), + Symbol::from("AAPL".to_owned()), create_position("AAPL", 100.0, 150.0), ); let mut historical_prices = HashMap::new(); historical_prices.insert( - Symbol::from("AAPL".to_string()), + Symbol::from("AAPL".to_owned()), create_prices("AAPL", 100, 150.0, 0.02), ); @@ -427,7 +427,7 @@ mod expected_shortfall_tests { let position = create_position("AAPL", 100.0, 150.0); let result = calculator - .calculate_position_var(&Symbol::from("AAPL".to_string()), &position, &prices) + .calculate_position_var(&Symbol::from("AAPL".to_owned()), &position, &prices) .unwrap(); // Expected Shortfall should always be >= VaR @@ -438,13 +438,13 @@ mod expected_shortfall_tests { fn test_monte_carlo_expected_shortfall_relationship() { let mut positions = HashMap::new(); positions.insert( - Symbol::from("AAPL".to_string()), + Symbol::from("AAPL".to_owned()), create_position("AAPL", 100.0, 150.0), ); let mut historical_prices = HashMap::new(); historical_prices.insert( - Symbol::from("AAPL".to_string()), + Symbol::from("AAPL".to_owned()), create_prices("AAPL", 100, 150.0, 0.02), ); @@ -471,21 +471,21 @@ mod portfolio_diversification_tests { let mut positions = HashMap::new(); positions.insert( - Symbol::from("TECH".to_string()), + Symbol::from("TECH".to_owned()), create_position("TECH", 100.0, 100.0), ); positions.insert( - Symbol::from("ENERGY".to_string()), + Symbol::from("ENERGY".to_owned()), create_position("ENERGY", 100.0, 50.0), ); let mut historical_prices = HashMap::new(); historical_prices.insert( - Symbol::from("TECH".to_string()), + Symbol::from("TECH".to_owned()), create_prices("TECH", 300, 100.0, 0.03), ); historical_prices.insert( - Symbol::from("ENERGY".to_string()), + Symbol::from("ENERGY".to_owned()), create_prices("ENERGY", 300, 50.0, 0.04), ); @@ -510,7 +510,7 @@ mod rolling_var_tests { let position = create_position("AAPL", 100.0, 150.0); let result = calculator.calculate_rolling_var( - &Symbol::from("AAPL".to_string()), + &Symbol::from("AAPL".to_owned()), &position, &prices, 60, @@ -533,7 +533,7 @@ mod rolling_var_tests { let position = create_position("AAPL", 100.0, 150.0); let result = calculator.calculate_rolling_var( - &Symbol::from("AAPL".to_string()), + &Symbol::from("AAPL".to_owned()), &position, &prices, 60, diff --git a/crates/risk/tests/var_extreme_scenarios_tests.rs b/crates/risk/tests/var_extreme_scenarios_tests.rs index bb4951fda..cc0374dec 100644 --- a/crates/risk/tests/var_extreme_scenarios_tests.rs +++ b/crates/risk/tests/var_extreme_scenarios_tests.rs @@ -87,8 +87,8 @@ mod perfect_correlation_tests { let returns_a = vec![0.02, -0.01, 0.03, -0.02, 0.015]; let returns_b = vec![0.02, -0.01, 0.03, -0.02, 0.015]; - portfolio.insert("ASSET_A".to_string(), returns_a); - portfolio.insert("ASSET_B".to_string(), returns_b); + portfolio.insert("ASSET_A".to_owned(), returns_a); + portfolio.insert("ASSET_B".to_owned(), returns_b); let var = calculate_portfolio_var(&portfolio, 0.95); @@ -104,8 +104,8 @@ mod perfect_correlation_tests { let returns_a = vec![0.02, -0.01, 0.03, -0.02, 0.015]; let returns_b = vec![-0.02, 0.01, -0.03, 0.02, -0.015]; - portfolio.insert("ASSET_A".to_string(), returns_a); - portfolio.insert("ASSET_B".to_string(), returns_b); + portfolio.insert("ASSET_A".to_owned(), returns_a); + portfolio.insert("ASSET_B".to_owned(), returns_b); let var = calculate_portfolio_var(&portfolio, 0.95); @@ -122,8 +122,8 @@ mod perfect_correlation_tests { let returns_a = vec![0.01, 0.02, 0.015, -0.30, -0.25]; let returns_b = vec![-0.01, 0.01, -0.02, -0.28, -0.27]; - portfolio.insert("STOCK_A".to_string(), returns_a); - portfolio.insert("STOCK_B".to_string(), returns_b); + portfolio.insert("STOCK_A".to_owned(), returns_a); + portfolio.insert("STOCK_B".to_owned(), returns_b); let var = calculate_portfolio_var(&portfolio, 0.95); @@ -160,8 +160,8 @@ mod zero_variance_tests { #[test] fn test_var_zero_variance_multiple_assets() { let mut portfolio = HashMap::new(); - portfolio.insert("STABLE_A".to_string(), vec![0.01; 50]); - portfolio.insert("STABLE_B".to_string(), vec![0.015; 50]); + portfolio.insert("STABLE_A".to_owned(), vec![0.01; 50]); + portfolio.insert("STABLE_B".to_owned(), vec![0.015; 50]); let var = calculate_portfolio_var(&portfolio, 0.95); @@ -306,7 +306,7 @@ mod single_asset_no_diversification_tests { #[test] fn test_var_single_asset_concentrated() { let mut portfolio = HashMap::new(); - portfolio.insert("ONLY_ASSET".to_string(), vec![0.02, -0.05, 0.03, -0.02]); + portfolio.insert("ONLY_ASSET".to_owned(), vec![0.02, -0.05, 0.03, -0.02]); let var = calculate_portfolio_var(&portfolio, 0.95); @@ -318,7 +318,7 @@ mod single_asset_no_diversification_tests { fn test_var_single_volatile_asset() { let mut portfolio = HashMap::new(); let volatile_returns = vec![0.10, -0.08, 0.12, -0.15, 0.09]; - portfolio.insert("VOLATILE".to_string(), volatile_returns); + portfolio.insert("VOLATILE".to_owned(), volatile_returns); let var = calculate_portfolio_var(&portfolio, 0.95); @@ -329,13 +329,13 @@ mod single_asset_no_diversification_tests { #[test] fn test_var_concentration_risk() { let mut concentrated = HashMap::new(); - concentrated.insert("MAIN".to_string(), vec![0.05, -0.03, 0.04]); + concentrated.insert("MAIN".to_owned(), vec![0.05, -0.03, 0.04]); let var_concentrated = calculate_portfolio_var(&concentrated, 0.95); let mut diversified = HashMap::new(); - diversified.insert("ASSET1".to_string(), vec![0.01, -0.01, 0.015]); - diversified.insert("ASSET2".to_string(), vec![0.02, 0.01, -0.01]); - diversified.insert("ASSET3".to_string(), vec![-0.01, 0.02, 0.01]); + diversified.insert("ASSET1".to_owned(), vec![0.01, -0.01, 0.015]); + diversified.insert("ASSET2".to_owned(), vec![0.02, 0.01, -0.01]); + diversified.insert("ASSET3".to_owned(), vec![-0.01, 0.02, 0.01]); let var_diversified = calculate_portfolio_var(&diversified, 0.95); // Concentrated should have relatively higher VaR @@ -411,10 +411,10 @@ fn calculate_portfolio_var(portfolio: &HashMap>, confidence: f6 fn validate_returns(returns: &[f64]) -> Result { for &r in returns { if r.is_nan() { - return Err("NaN detected".to_string()); + return Err("NaN detected".to_owned()); } if r.is_infinite() { - return Err("Infinity detected".to_string()); + return Err("Infinity detected".to_owned()); } } Ok(true) diff --git a/crates/risk/tests/var_zero_position_tests.rs b/crates/risk/tests/var_zero_position_tests.rs index c405786c6..10e4d43b5 100644 --- a/crates/risk/tests/var_zero_position_tests.rs +++ b/crates/risk/tests/var_zero_position_tests.rs @@ -49,7 +49,7 @@ mod zero_position_tests { average_cost: Price::new(150.0).unwrap(), unrealized_pnl: Price::ZERO, realized_pnl: Price::ZERO, - currency: "USD".to_string(), + currency: "USD".to_owned(), timestamp: Utc::now(), }; @@ -67,7 +67,7 @@ mod zero_position_tests { average_cost: Price::new(150.0).unwrap(), unrealized_pnl: Price::ZERO, realized_pnl: Price::ZERO, - currency: "USD".to_string(), + currency: "USD".to_owned(), timestamp: Utc::now(), }, PositionInfo { @@ -77,7 +77,7 @@ mod zero_position_tests { average_cost: Price::new(300.0).unwrap(), unrealized_pnl: Price::ZERO, realized_pnl: Price::ZERO, - currency: "USD".to_string(), + currency: "USD".to_owned(), timestamp: Utc::now(), }, ]; @@ -100,7 +100,7 @@ mod zero_position_tests { average_cost: Price::new(50000.0).unwrap(), unrealized_pnl: Price::ZERO, realized_pnl: Price::ZERO, - currency: "USD".to_string(), + currency: "USD".to_owned(), timestamp: Utc::now(), }; @@ -122,7 +122,7 @@ mod single_position_var_tests { average_cost: Price::new(40000.0).unwrap(), unrealized_pnl: Price::new(5000.0).unwrap(), realized_pnl: Price::ZERO, - currency: "USD".to_string(), + currency: "USD".to_owned(), timestamp: Utc::now(), }; @@ -139,7 +139,7 @@ mod single_position_var_tests { average_cost: Price::new(100000.0).unwrap(), unrealized_pnl: Price::ZERO, realized_pnl: Price::ZERO, - currency: "USD".to_string(), + currency: "USD".to_owned(), timestamp: Utc::now(), }; @@ -156,7 +156,7 @@ mod single_position_var_tests { average_cost: Price::new(450.0).unwrap(), unrealized_pnl: Price::new(-1000.0).unwrap(), realized_pnl: Price::ZERO, - currency: "USD".to_string(), + currency: "USD".to_owned(), timestamp: Utc::now(), }; @@ -286,7 +286,7 @@ mod insufficient_data_tests { let mut price_history = BoundedVec::::new(1); price_history.push(HistoricalPrice { - symbol: "AAPL".to_string(), + symbol: "AAPL".to_owned(), date: Utc::now(), open: Price::new(150.0).unwrap(), high: Price::new(152.0).unwrap(), @@ -306,7 +306,7 @@ mod insufficient_data_tests { let mut price_history = BoundedVec::::new(2); price_history.push(HistoricalPrice { - symbol: "AAPL".to_string(), + symbol: "AAPL".to_owned(), date: Utc::now() - Duration::days(1), open: Price::new(150.0).unwrap(), high: Price::new(152.0).unwrap(), @@ -316,7 +316,7 @@ mod insufficient_data_tests { }); price_history.push(HistoricalPrice { - symbol: "AAPL".to_string(), + symbol: "AAPL".to_owned(), date: Utc::now(), open: Price::new(151.0).unwrap(), high: Price::new(153.0).unwrap(), @@ -337,7 +337,7 @@ mod insufficient_data_tests { for i in 0..5 { price_history.push(HistoricalPrice { - symbol: "AAPL".to_string(), + symbol: "AAPL".to_owned(), date: Utc::now() - Duration::days(i), open: Price::new(150.0 + i as f64).unwrap(), high: Price::new(152.0 + i as f64).unwrap(), @@ -360,7 +360,7 @@ mod insufficient_data_tests { // Add old data (30-40 days ago) for i in 30..40 { price_history.push(HistoricalPrice { - symbol: "AAPL".to_string(), + symbol: "AAPL".to_owned(), date: Utc::now() - Duration::days(i), open: Price::new(150.0).unwrap(), high: Price::new(152.0).unwrap(), @@ -493,7 +493,7 @@ mod bounded_vec_tests { #[test] fn test_bounded_vec_zero_capacity() { let mut bounded = BoundedVec::::new(0); - bounded.push("test".to_string()); + bounded.push("test".to_owned()); assert_eq!(bounded.len(), 0); assert!(bounded.is_empty()); diff --git a/crates/storage/examples/checkpoint_uploader.rs b/crates/storage/examples/checkpoint_uploader.rs deleted file mode 100644 index 0db773414..000000000 --- a/crates/storage/examples/checkpoint_uploader.rs +++ /dev/null @@ -1,319 +0,0 @@ -//! Checkpoint Uploader - Upload trained model checkpoints to S3 -//! -//! This utility uploads all trained model checkpoints from local storage -//! to S3-compatible storage (MinIO in development) for archival and -//! production deployment. -//! -//! Usage: -//! cargo run --example checkpoint_uploader -- --source-dir ml/trained_models/production - -use clap::Parser; -use config::schemas::S3Config; -use std::path::{Path, PathBuf}; -use std::time::Instant; -use storage::{ObjectStoreBackend, Storage}; -use tracing::{error, info, warn}; - -#[derive(Parser, Debug)] -#[clap(name = "checkpoint_uploader")] -#[clap(about = "Upload trained model checkpoints to S3")] -struct Args { - /// Source directory containing checkpoints - #[clap(short, long, default_value = "ml/trained_models/production")] - source_dir: PathBuf, - - /// S3 bucket name - #[clap(short, long, default_value = "foxhunt-ml-models")] - bucket: String, - - /// Dry run - don't actually upload - #[clap(short, long)] - dry_run: bool, -} - -#[derive(Debug)] -struct UploadStats { - total_files: usize, - uploaded_files: usize, - failed_files: usize, - total_bytes: u64, - duration_secs: f64, -} - -impl UploadStats { - fn new() -> Self { - Self { - total_files: 0, - uploaded_files: 0, - failed_files: 0, - total_bytes: 0, - duration_secs: 0.0, - } - } - - fn throughput_mbps(&self) -> f64 { - if self.duration_secs > 0.0 { - (self.total_bytes as f64) / (1024.0 * 1024.0 * self.duration_secs) - } else { - 0.0 - } - } -} - -/// Parse checkpoint filename to extract model name and version -fn parse_checkpoint_filename(filename: &str) -> Option<(String, String, String)> { - // Expected formats: - // - dqn_epoch_100.safetensors -> (dqn, epoch_100, .safetensors) - // - ppo_checkpoint_epoch_200.safetensors -> (ppo, epoch_200, .safetensors) - // - dqn_final_epoch500.safetensors -> (dqn, epoch500, .safetensors) - - if !filename.ends_with(".safetensors") { - return None; - } - - let name_without_ext = filename.trim_end_matches(".safetensors"); - - // Try to extract model name and epoch - if let Some(pos) = name_without_ext.find("_epoch") { - let model_name = &name_without_ext[..pos]; - let model_clean = model_name - .trim_end_matches("_checkpoint") - .trim_end_matches("_final"); - let epoch_part = &name_without_ext[pos..]; - - return Some(( - model_clean.to_string(), - epoch_part.to_string(), - ".safetensors".to_string(), - )); - } - - None -} - -/// Generate S3 path for checkpoint -fn get_s3_path(model_name: &str, version: &str, filename: &str) -> String { - format!("{}/{}/checkpoints/{}", model_name, version, filename) -} - -async fn upload_checkpoint( - backend: &ObjectStoreBackend, - source_path: &Path, - s3_path: &str, - dry_run: bool, -) -> Result> { - let file_size = tokio::fs::metadata(source_path).await?.len(); - - if dry_run { - info!( - "DRY RUN: Would upload {} ({} bytes) -> {}", - source_path.display(), - file_size, - s3_path - ); - return Ok(file_size); - } - - info!( - "Uploading {} ({} bytes) -> {}", - source_path.display(), - file_size, - s3_path - ); - - // Read file contents - let data = tokio::fs::read(source_path).await?; - - // Upload to S3 - backend.store(s3_path, &data).await?; - - info!("Successfully uploaded: {}", s3_path); - Ok(file_size) -} - -#[tokio::main] -async fn main() -> Result<(), Box> { - // Initialize logging - tracing_subscriber::fmt() - .with_env_filter( - tracing_subscriber::EnvFilter::from_default_env() - .add_directive("checkpoint_uploader=info".parse()?) - .add_directive("storage=info".parse()?), - ) - .init(); - - let args = Args::parse(); - - info!("Checkpoint Uploader"); - info!(" Source directory: {}", args.source_dir.display()); - info!(" S3 bucket: {}", args.bucket); - info!(" Dry run: {}", args.dry_run); - - // Configure S3 backend for MinIO - let s3_config = S3Config { - bucket_name: args.bucket.clone(), - region: "us-east-1".to_string(), - access_key_id: Some("foxhunt".to_string()), - secret_access_key: Some("foxhunt_dev_password".to_string()), - session_token: None, - endpoint_url: Some("http://localhost:9000".to_string()), - force_path_style: true, - timeout: std::time::Duration::from_secs(30), - max_retry_attempts: 3, - use_ssl: false, - }; - - info!("Initializing S3 backend..."); - let backend = ObjectStoreBackend::new(s3_config, None).await?; - info!("S3 backend initialized successfully"); - - // Scan source directory for checkpoints - info!("Scanning directory: {}", args.source_dir.display()); - - let mut entries = tokio::fs::read_dir(&args.source_dir).await?; - let mut checkpoints = Vec::new(); - - while let Some(entry) = entries.next_entry().await? { - let path = entry.path(); - if path.is_file() { - if let Some(filename) = path.file_name().and_then(|n| n.to_str()) { - if filename.ends_with(".safetensors") { - checkpoints.push(path); - } - } - } - } - - info!("Found {} checkpoint files", checkpoints.len()); - - // Upload checkpoints - let start = Instant::now(); - let mut stats = UploadStats::new(); - stats.total_files = checkpoints.len(); - - for checkpoint_path in checkpoints { - let filename = checkpoint_path - .file_name() - .and_then(|n| n.to_str()) - .unwrap_or("unknown"); - - // Parse filename to determine model and version - let (model_name, version) = - if let Some((model, ver, _)) = parse_checkpoint_filename(filename) { - (model, ver) - } else { - warn!( - "Could not parse checkpoint filename: {}, using defaults", - filename - ); - ("unknown".to_string(), "v1.0".to_string()) - }; - - // Generate S3 path - let s3_path = get_s3_path(&model_name, &version, filename); - - // Upload checkpoint - match upload_checkpoint(&backend, &checkpoint_path, &s3_path, args.dry_run).await { - Ok(size) => { - stats.uploaded_files += 1; - stats.total_bytes += size; - }, - Err(e) => { - error!("Failed to upload {}: {}", filename, e); - stats.failed_files += 1; - }, - } - } - - stats.duration_secs = start.elapsed().as_secs_f64(); - - // Print summary - println!("\nโ•”โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•—"); - println!("โ•‘ Checkpoint Upload Summary โ•‘"); - println!("โ• โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•ฃ"); - println!( - "โ•‘ Total files: {:>6} โ•‘", - stats.total_files - ); - println!( - "โ•‘ Uploaded: {:>6} โ•‘", - stats.uploaded_files - ); - println!( - "โ•‘ Failed: {:>6} โ•‘", - stats.failed_files - ); - println!( - "โ•‘ Total size: {:>6} MB โ•‘", - stats.total_bytes / (1024 * 1024) - ); - println!( - "โ•‘ Duration: {:>6.2} seconds โ•‘", - stats.duration_secs - ); - println!( - "โ•‘ Throughput: {:>6.2} MB/s โ•‘", - stats.throughput_mbps() - ); - println!("โ•šโ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•"); - - if args.dry_run { - println!("\nDRY RUN COMPLETE - No files were actually uploaded"); - } else { - println!("\nUpload complete!"); - - // Verify uploads by listing S3 bucket - info!("Verifying uploads..."); - let uploaded_objects = backend.list("").await?; - println!("S3 bucket now contains {} objects", uploaded_objects.len()); - } - - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_parse_checkpoint_filename() { - let test_cases = vec![ - ( - "dqn_epoch_100.safetensors", - Some(( - "dqn".to_string(), - "_epoch_100".to_string(), - ".safetensors".to_string(), - )), - ), - ( - "ppo_checkpoint_epoch_200.safetensors", - Some(( - "ppo".to_string(), - "_epoch_200".to_string(), - ".safetensors".to_string(), - )), - ), - ( - "dqn_final_epoch500.safetensors", - Some(( - "dqn".to_string(), - "_epoch500".to_string(), - ".safetensors".to_string(), - )), - ), - ("invalid.txt", None), - ]; - - for (input, expected) in test_cases { - let result = parse_checkpoint_filename(input); - assert_eq!(result, expected, "Failed for input: {}", input); - } - } - - #[test] - fn test_get_s3_path() { - let path = get_s3_path("dqn", "epoch_100", "dqn_epoch_100.safetensors"); - assert_eq!(path, "dqn/epoch_100/checkpoints/dqn_epoch_100.safetensors"); - } -} diff --git a/crates/trading_engine/examples/event_processing_demo.rs b/crates/trading_engine/examples/event_processing_demo.rs deleted file mode 100644 index 59b46bef4..000000000 --- a/crates/trading_engine/examples/event_processing_demo.rs +++ /dev/null @@ -1,348 +0,0 @@ -#![allow(unused_crate_dependencies)] -//! High-Performance Event Processing Demo -//! -//! This example demonstrates the event processing pipeline with: -//! - Sub-microsecond event capture -//! - Batched PostgreSQL persistence -//! - Real-time monitoring and metrics -//! - Error recovery and guaranteed delivery - -use anyhow::Result; -use rust_decimal::prelude::*; -use std::time::Duration; -use tokio::time::sleep; - -use trading_engine::events::event_types::{ - AlertSeverity, EventLevel, RiskAlertType, SystemEventType, TradingEvent, -}; -use trading_engine::events::{EventProcessor, EventProcessorConfig}; -use trading_engine::timing::HardwareTimestamp; - -#[tokio::main] -async fn main() -> Result<()> { - // Initialize tracing (simplified for demo) - // tracing_subscriber::fmt::init(); - println!("๐Ÿ“ Logging initialized (simplified)"); - - println!("๐Ÿš€ Starting High-Performance Event Processing Demo"); - - // Configure event processor for demo - let config = EventProcessorConfig { - // Use a test database or in-memory database for demo - database_url: std::env::var("DATABASE_URL").unwrap_or_else(|_| { - "postgresql://foxhunt:foxhunt@localhost/trading_events_demo".to_string() - }), - buffer_count: 4, - buffer_size: 1024, - batch_size: 100, - batch_timeout_ms: 50, - writer_threads: 2, - max_db_connections: 10, - db_timeout_seconds: 10, - enable_compression: true, - max_memory_usage: 50 * 1024 * 1024, // 50MB for demo - enable_monitoring: true, - max_retry_attempts: 3, - retry_delay_ms: 100, - }; - - // Initialize event processor - println!("๐Ÿ“Š Initializing event processor..."); - let processor = match EventProcessor::new(config).await { - Ok(p) => p, - Err(e) => { - eprintln!("โŒ Failed to initialize event processor: {}", e); - eprintln!("๐Ÿ’ก Make sure PostgreSQL is running and accessible"); - eprintln!("๐Ÿ’ก Create database: CREATE DATABASE trading_events_demo;"); - return Err(e); - }, - }; - - println!("โœ… Event processor initialized successfully"); - - // Demo 1: High-frequency order events - println!("\n๐Ÿ“ˆ Demo 1: High-frequency order events"); - demo_order_events(&processor).await?; - - // Demo 2: Risk monitoring events - println!("\nโš ๏ธ Demo 2: Risk monitoring events"); - demo_risk_events(&processor).await?; - - // Demo 3: System events - println!("\n๐Ÿ”ง Demo 3: System events"); - demo_system_events(&processor).await?; - - // Demo 4: Performance stress test - println!("\nโšก Demo 4: Performance stress test"); - demo_performance_test(&processor).await?; - - // Demo 5: Monitoring and metrics - println!("\n๐Ÿ“Š Demo 5: Monitoring and metrics"); - demo_monitoring(&processor).await?; - - // Graceful shutdown - println!("\n๐Ÿ›‘ Shutting down event processor..."); - processor.shutdown().await?; - println!("โœ… Event processor shutdown complete"); - - Ok(()) -} - -/// Demonstrate high-frequency order processing events -async fn demo_order_events(processor: &EventProcessor) -> Result<()> { - let symbols = ["EURUSD", "GBPUSD", "USDJPY", "AUDUSD"]; - let mut order_counter = 1; - - println!(" Capturing 100 order events..."); - - for i in 0..100 { - let symbol = symbols[i % symbols.len()]; - let order_id = format!("ORD-{:06}", order_counter); - order_counter += 1; - - // Create order submission event - let event = TradingEvent::OrderSubmitted { - order_id: order_id.clone(), - symbol: symbol.to_string(), - quantity: Decimal::from(100000) + Decimal::from(i * 1000), - price: Decimal::from_str("1.0850").unwrap() + Decimal::from(i) / Decimal::from(10000), - timestamp: HardwareTimestamp::now(), - sequence_number: None, // Will be set by processor - metadata: Some(serde_json::json!({ - "strategy": "mean_reversion", - "session": "london", - "demo_source": "order_events" - })), - }; - - // Capture event (sub-microsecond performance) - match processor.capture_event(event).await { - Ok(sequence) => { - if i % 20 == 0 { - println!( - " ๐Ÿ“ Order {} captured (seq: {})", - order_id, - sequence.number() - ); - } - }, - Err(e) => { - eprintln!(" โŒ Failed to capture order {}: {}", order_id, e); - }, - } - - // Small delay to prevent overwhelming the system in demo - if i % 10 == 0 { - sleep(Duration::from_millis(1)).await; - } - } - - println!(" โœ… Order events captured successfully"); - Ok(()) -} - -/// Demonstrate risk monitoring events -async fn demo_risk_events(processor: &EventProcessor) -> Result<()> { - println!(" Generating risk alerts..."); - - let risk_scenarios = [ - ( - RiskAlertType::PositionSizeLimit, - AlertSeverity::High, - "Position size exceeded 80% of limit for EURUSD", - ), - ( - RiskAlertType::DailyLossLimit, - AlertSeverity::Critical, - "Daily loss approaching 90% of limit", - ), - ( - RiskAlertType::VolatilitySpike, - AlertSeverity::Medium, - "Volatility spike detected in GBPUSD", - ), - ( - RiskAlertType::LiquidityConstraint, - AlertSeverity::Low, - "Low liquidity detected in overnight session", - ), - ]; - - for (alert_type, severity, message) in risk_scenarios { - let event = TradingEvent::RiskAlert { - alert_type, - symbol: Some("EURUSD".to_string()), - message: message.to_string(), - severity, - timestamp: HardwareTimestamp::now(), - sequence_number: None, - metadata: Some(serde_json::json!({ - "risk_engine": "var_calculator", - "threshold_breached": true, - "demo_source": "risk_events" - })), - }; - - match processor.capture_event(event).await { - Ok(sequence) => { - println!( - " ๐Ÿšจ Risk alert captured: {} (seq: {})", - message, - sequence.number() - ); - }, - Err(e) => { - eprintln!(" โŒ Failed to capture risk alert: {}", e); - }, - } - - sleep(Duration::from_millis(100)).await; - } - - println!(" โœ… Risk events captured successfully"); - Ok(()) -} - -/// Demonstrate system events -async fn demo_system_events(processor: &EventProcessor) -> Result<()> { - println!(" Generating system events..."); - - let system_scenarios = [ - ( - SystemEventType::ServiceConnected, - EventLevel::Info, - "Market data feed connected", - ), - ( - SystemEventType::ConfigurationChange, - EventLevel::Warning, - "Risk limits updated", - ), - ( - SystemEventType::PerformanceDegradation, - EventLevel::Error, - "Latency spike detected", - ), - ( - SystemEventType::Custom("maintenance".to_string()), - EventLevel::Info, - "Scheduled maintenance window started", - ), - ]; - - for (event_type, level, message) in system_scenarios { - let event = TradingEvent::SystemEvent { - event_type, - message: message.to_string(), - level, - timestamp: HardwareTimestamp::now(), - sequence_number: None, - metadata: Some(serde_json::json!({ - "service": "trading_engine", - "version": "1.0.0", - "demo_source": "system_events" - })), - }; - - match processor.capture_event(event).await { - Ok(sequence) => { - println!( - " ๐Ÿ”ง System event captured: {} (seq: {})", - message, - sequence.number() - ); - }, - Err(e) => { - eprintln!(" โŒ Failed to capture system event: {}", e); - }, - } - - sleep(Duration::from_millis(50)).await; - } - - println!(" โœ… System events captured successfully"); - Ok(()) -} - -/// Demonstrate high-performance stress test -async fn demo_performance_test(processor: &EventProcessor) -> Result<()> { - println!(" Running performance stress test (1000 events)..."); - - let start_time = std::time::Instant::now(); - let mut successful_captures = 0; - let mut failed_captures = 0; - - // Capture 1000 events as fast as possible - for i in 0..1000 { - let event = TradingEvent::OrderExecuted { - trade_id: format!("TRADE-{:06}", i), - symbol: "EURUSD".to_string(), - quantity: Decimal::from(50000), - price: Decimal::from_str("1.0851").unwrap() - + Decimal::from(i % 100) / Decimal::from(100000), - timestamp: HardwareTimestamp::now(), - sequence_number: None, - metadata: Some(serde_json::json!({ - "execution_venue": "prime_broker", - "demo_source": "performance_test" - })), - }; - - match processor.capture_event(event).await { - Ok(_) => successful_captures += 1, - Err(_) => failed_captures += 1, - } - } - - let elapsed = start_time.elapsed(); - let events_per_second = successful_captures as f64 / elapsed.as_secs_f64(); - let avg_latency_us = elapsed.as_micros() / successful_captures as u128; - - println!(" ๐Ÿ“Š Performance Results:"); - println!(" โšก Events/second: {:.0}", events_per_second); - println!(" ๐Ÿ• Avg latency: {} ฮผs", avg_latency_us); - println!(" โœ… Successful: {}", successful_captures); - println!(" โŒ Failed: {}", failed_captures); - - Ok(()) -} - -/// Demonstrate monitoring and metrics -async fn demo_monitoring(processor: &EventProcessor) -> Result<()> { - println!(" Collecting metrics and health status..."); - - // Get current metrics - let metrics = processor.get_metrics(); - println!(" ๐Ÿ“Š Current Metrics:"); - println!(" ๐Ÿ“ˆ Events captured: {}", metrics.events_captured); - println!(" ๐Ÿ“‰ Events dropped: {}", metrics.events_dropped); - println!(" ๐Ÿ’พ Events written: {}", metrics.events_written); - println!(" โšก Events/sec: {}", metrics.events_per_second); - println!( - " ๐Ÿ• Avg capture latency: {} ns", - metrics.avg_capture_latency_ns - ); - println!( - " ๐Ÿ’ฝ Avg write latency: {:.2} ms", - metrics.avg_write_latency_ms - ); - - // Get health status - let health = processor.get_health().await; - println!(" ๐Ÿฅ Health Status: {:?}", health); - - // Get buffer statistics - let buffer_stats = processor.get_buffer_stats().await; - println!(" ๐Ÿ”ง Buffer Statistics:"); - for stats in buffer_stats { - println!( - " Buffer {}: {:.1}% utilization, {} pushes, {} pops", - stats.buffer_id, - stats.current_utilization * 100.0, - stats.push_success_count, - stats.pop_success_count - ); - } - - Ok(()) -} diff --git a/services/api_gateway/examples/metrics_example.rs b/services/api_gateway/examples/metrics_example.rs deleted file mode 100644 index 8e82c2138..000000000 --- a/services/api_gateway/examples/metrics_example.rs +++ /dev/null @@ -1,161 +0,0 @@ -//! Metrics Integration Example -//! -//! Demonstrates how to integrate Prometheus metrics into the API Gateway - -use api_gateway::metrics::{metrics_router, GatewayMetrics}; -use std::net::SocketAddr; -use std::time::Instant; -use tokio::net::TcpListener; - -#[tokio::main] -async fn main() -> Result<(), Box> { - println!("๐Ÿš€ API Gateway Metrics Example"); - println!("=====================================\n"); - - // Initialize metrics - println!("1. Initializing metrics registry..."); - let metrics = GatewayMetrics::new()?; - println!(" โœ… Metrics registry created with {} subsystems\n", 3); - - // Simulate authentication events - println!("2. Recording authentication events..."); - - // Successful auth - for i in 0..100 { - let start = Instant::now(); - - // Simulate JWT extraction (0.5ฮผs) - std::thread::sleep(std::time::Duration::from_nanos(500)); - metrics.auth.jwt_extraction_duration_us.observe(0.5); - - // Simulate JWT validation (0.8ฮผs) - std::thread::sleep(std::time::Duration::from_nanos(800)); - metrics.auth.jwt_validation_duration_us.observe(0.8); - - // Simulate revocation check (0.3ฮผs) - std::thread::sleep(std::time::Duration::from_nanos(300)); - metrics.auth.revocation_check_duration_us.observe(0.3); - - // Simulate RBAC check (0.1ฮผs) - std::thread::sleep(std::time::Duration::from_nanos(100)); - metrics.auth.rbac_check_duration_us.observe(0.1); - - // Simulate rate limit check (0.05ฮผs) - std::thread::sleep(std::time::Duration::from_nanos(50)); - metrics.auth.rate_limit_check_duration_us.observe(0.05); - - let total_duration_us = start.elapsed().as_nanos() as f64 / 1000.0; - metrics.auth.record_success(total_duration_us); - metrics - .auth - .record_user_request(&format!("user_{}", i % 10)); - - if i % 10 == 0 { - println!(" โœ… Recorded {} successful auth requests", i + 1); - } - } - - // Simulate auth failures - println!("\n3. Recording authentication failures..."); - metrics.auth.record_failure("expired_jwt", Some("user_99")); - metrics.auth.record_failure("revoked_jwt", Some("user_88")); - metrics - .auth - .record_failure("permission_denied", Some("user_77")); - metrics.auth.record_rate_limit("user_66"); - println!(" โœ… Recorded 4 auth failures\n"); - - // Simulate backend proxy events - println!("4. Recording backend proxy events..."); - - // Trading service requests - for i in 0..50 { - metrics - .proxy - .record_backend_success("trading", "ExecuteTrade", 15.5); - if i % 10 == 0 { - println!(" โœ… Recorded {} trading service requests", i + 1); - } - } - - // Backtesting service requests - for i in 0..30 { - metrics - .proxy - .record_backend_success("backtesting", "RunBacktest", 250.0); - } - println!(" โœ… Recorded 30 backtesting service requests"); - - // ML Training service requests - metrics - .proxy - .record_backend_success("ml_training", "TrainModel", 5000.0); - println!(" โœ… Recorded ML training requests\n"); - - // Update health status - println!("5. Updating service health status..."); - metrics.proxy.update_health_status("trading", true); - metrics.proxy.update_health_status("backtesting", true); - metrics.proxy.update_health_status("ml_training", true); - println!(" โœ… All services healthy\n"); - - // Update connection pools - println!("6. Updating connection pool metrics..."); - metrics.proxy.update_connection_pool("trading", 10, 5, 50); - metrics - .proxy - .update_connection_pool("backtesting", 3, 7, 20); - metrics - .proxy - .update_connection_pool("ml_training", 2, 8, 10); - println!(" โœ… Connection pool stats updated\n"); - - // Simulate configuration events - println!("7. Recording configuration events..."); - metrics.config.record_notify_event(true); - metrics.config.record_config_reload("auth", 25.0); - metrics.config.record_config_reload("routing", 18.5); - metrics.config.update_listener_status(true); - println!(" โœ… Configuration hot-reload events recorded\n"); - - // Update cache metrics - println!("8. Recording cache performance..."); - metrics.auth.jwt_cache_hits.inc_by(950.0); - metrics.auth.jwt_cache_misses.inc_by(50.0); - metrics.auth.rbac_cache_hits.inc_by(980.0); - metrics.auth.rbac_cache_misses.inc_by(20.0); - metrics.config.config_cache_hits.inc_by(1500.0); - metrics.config.config_cache_misses.inc_by(100.0); - println!(" โœ… Cache hit/miss stats updated\n"); - - // Print metrics summary - println!("๐Ÿ“Š Metrics Summary"); - println!("====================================="); - println!("Auth Metrics:"); - println!(" - Requests: 100 success, 4 failures"); - println!(" - JWT Cache Hit Rate: 95%"); - println!(" - RBAC Cache Hit Rate: 98%"); - println!("\nProxy Metrics:"); - println!(" - Trading Service: 50 requests, 15.5ms avg"); - println!(" - Backtesting Service: 30 requests, 250ms avg"); - println!(" - ML Training Service: 1 request, 5000ms"); - println!("\nConfig Metrics:"); - println!(" - NOTIFY events: 1 processed"); - println!(" - Config reloads: 2 (auth, routing)"); - println!(" - Config Cache Hit Rate: 93.75%"); - println!("\n"); - - // Start Prometheus exporter - println!("9. Starting Prometheus metrics exporter..."); - let router = metrics_router(metrics.registry()); - let addr = SocketAddr::from(([127, 0, 0, 1], 9090)); - - println!(" โœ… Metrics available at http://{}/metrics\n", addr); - println!("๐Ÿ“ก Exporting metrics to Prometheus..."); - println!(" Press Ctrl+C to stop\n"); - - let listener = TcpListener::bind(&addr).await?; - axum::serve(listener, router).await?; - - Ok(()) -} diff --git a/services/api_gateway/examples/rate_limiter_usage.rs b/services/api_gateway/examples/rate_limiter_usage.rs deleted file mode 100644 index d9dbb7604..000000000 --- a/services/api_gateway/examples/rate_limiter_usage.rs +++ /dev/null @@ -1,208 +0,0 @@ -//! Rate Limiter Usage Examples -//! -//! Demonstrates how to use the RateLimiter in different scenarios - -use anyhow::Result; -use api_gateway::routing::RateLimiter; -use uuid::Uuid; - -// Note: This is a pseudo-code example showing integration patterns -// The actual types would come from the api_gateway crate - -/// Example 1: Basic rate limit check in authentication flow -async fn example_auth_flow( - rate_limiter: &RateLimiter, - user_id: &Uuid, - endpoint: &str, -) -> Result<()> { - // Check rate limit before processing request - let allowed = rate_limiter.check_limit(user_id, endpoint).await?; - - if !allowed { - return Err(anyhow::anyhow!("Rate limit exceeded")); - } - - // Process request... - Ok(()) -} - -/// Example 2: Rate limiting with different endpoint configurations -async fn example_endpoint_configs(rate_limiter: &RateLimiter) -> Result<()> { - use api_gateway::routing::RateLimitConfig; - - // Configure high-frequency trading endpoint - let trading_config = RateLimitConfig::trading_submit_order(); - rate_limiter.set_endpoint_config(trading_config).await; - - // Configure backtesting endpoint (low frequency) - let backtesting_config = RateLimitConfig::backtesting_run(); - rate_limiter.set_endpoint_config(backtesting_config).await; - - // Configure custom endpoint - let custom_config = RateLimitConfig { - endpoint: "custom.api".to_string(), - capacity: 50.0, - refill_rate: 50.0, - burst_size: 5, - }; - rate_limiter.set_endpoint_config(custom_config).await; - - Ok(()) -} - -/// Example 3: Monitoring cache statistics -async fn example_monitoring(rate_limiter: &RateLimiter) -> Result<()> { - // Get cache statistics - let stats = rate_limiter.get_cache_stats().await; - - println!("Rate Limiter Cache Statistics:"); - println!(" Current size: {}/{}", stats.size, stats.max_size); - println!(" Cache TTL: {} seconds", stats.ttl_seconds); - println!( - " Cache usage: {:.1}%", - (stats.size as f64 / stats.max_size as f64) * 100.0 - ); - - Ok(()) -} - -/// Example 4: Integration with gRPC interceptor -async fn example_grpc_integration( - rate_limiter: &RateLimiter, - user_id: &Uuid, - request_uri: &str, -) -> Result<()> { - // Extract endpoint from URI - let endpoint = request_uri.split('/').last().unwrap_or("unknown"); - - // Check rate limit - if !rate_limiter.check_limit(user_id, endpoint).await? { - // Log rate limit violation - tracing::warn!( - user_id = %user_id, - endpoint = %endpoint, - "Rate limit exceeded" - ); - - return Err(anyhow::anyhow!("Rate limit exceeded for {}", endpoint)); - } - - // Continue processing... - Ok(()) -} - -/// Example 5: Burst handling scenario -async fn example_burst_handling(rate_limiter: &RateLimiter) -> Result<()> { - let user_id = Uuid::new_v4(); - let endpoint = "trading.submit_order"; - - // Simulate burst of 100 requests - let mut allowed_count = 0; - let mut denied_count = 0; - - for _ in 0..100 { - if rate_limiter.check_limit(&user_id, endpoint).await? { - allowed_count += 1; - } else { - denied_count += 1; - } - } - - println!("Burst test results:"); - println!(" Allowed: {} requests", allowed_count); - println!(" Denied: {} requests", denied_count); - - // Should allow up to capacity (100 for trading endpoint) - assert_eq!(allowed_count, 100); - assert_eq!(denied_count, 0); - - Ok(()) -} - -/// Example 6: Cache management -async fn example_cache_management(rate_limiter: &RateLimiter) -> Result<()> { - // Clear cache (useful after configuration changes) - rate_limiter.clear_cache().await; - - println!("Cache cleared - all subsequent requests will hit Redis"); - - // First request will populate cache from Redis - let user_id = Uuid::new_v4(); - let _ = rate_limiter - .check_limit(&user_id, "trading.submit_order") - .await?; - - println!("Cache populated - subsequent requests will be <50ns"); - - Ok(()) -} - -/// Example 7: Performance testing -async fn example_performance_test(rate_limiter: &RateLimiter) -> Result<()> { - use std::time::Instant; - - let user_id = Uuid::new_v4(); - let endpoint = "trading.submit_order"; - - // Warm up cache - let _ = rate_limiter.check_limit(&user_id, endpoint).await?; - - // Measure cache hit performance - let iterations = 10_000; - let start = Instant::now(); - - for _ in 0..iterations { - let _ = rate_limiter.check_limit(&user_id, endpoint).await?; - } - - let elapsed = start.elapsed(); - let ns_per_check = elapsed.as_nanos() / iterations; - - println!("Performance test results:"); - println!(" Total time: {:?}", elapsed); - println!(" Iterations: {}", iterations); - println!(" Time per check: {} ns", ns_per_check); - println!(" Target: <50ns"); - - if ns_per_check < 50 { - println!(" โœ… Performance target met!"); - } else { - println!(" โš ๏ธ Performance target missed"); - } - - Ok(()) -} - -#[tokio::main] -async fn main() -> Result<()> { - // Initialize rate limiter - let rate_limiter = RateLimiter::new("redis://localhost:6379").await?; - - println!("Rate Limiter Usage Examples\n"); - - // Run examples - println!("Example 1: Basic auth flow"); - example_auth_flow(&rate_limiter, &Uuid::new_v4(), "trading.submit_order").await?; - - println!("\nExample 2: Endpoint configurations"); - example_endpoint_configs(&rate_limiter).await?; - - println!("\nExample 3: Monitoring"); - example_monitoring(&rate_limiter).await?; - - println!("\nExample 4: gRPC integration"); - example_grpc_integration(&rate_limiter, &Uuid::new_v4(), "/api/trading/submit_order").await?; - - println!("\nExample 5: Burst handling"); - example_burst_handling(&rate_limiter).await?; - - println!("\nExample 6: Cache management"); - example_cache_management(&rate_limiter).await?; - - println!("\nExample 7: Performance test"); - example_performance_test(&rate_limiter).await?; - - println!("\nโœ… All examples completed successfully!"); - - Ok(()) -} diff --git a/services/backtesting_service/Cargo.toml b/services/backtesting_service/Cargo.toml index 9e143961e..b59e0c6c4 100644 --- a/services/backtesting_service/Cargo.toml +++ b/services/backtesting_service/Cargo.toml @@ -110,3 +110,4 @@ default = ["postgres", "influxdb"] postgres = ["sqlx/postgres"] influxdb = [] standalone = [] +test-utils = [] diff --git a/services/backtesting_service/examples/wave_comparison.rs b/services/backtesting_service/examples/wave_comparison.rs deleted file mode 100644 index f4a1b0bd2..000000000 --- a/services/backtesting_service/examples/wave_comparison.rs +++ /dev/null @@ -1,57 +0,0 @@ -//! Wave Comparison Backtesting Example -//! -//! This example demonstrates how to run comprehensive backtesting to validate -//! performance improvements across Wave A, Wave B, and Wave C. -//! -//! Usage: -//! ```bash -//! cargo run -p backtesting_service --example wave_comparison -//! ``` -//! -//! Expected Output: -//! - Console summary with detailed metrics -//! - JSON export: results/wave_comparison_ES.FUT_YYYYMMDD_HHMMSS.json -//! - CSV export: results/wave_comparison_ES.FUT_YYYYMMDD_HHMMSS.csv - -use anyhow::Result; -use backtesting_service::repositories::{BacktestingRepositories, DefaultRepositories}; -use backtesting_service::wave_comparison::{DateRange, WaveComparisonBacktest}; -use chrono::{Duration, Utc}; -use std::sync::Arc; -use tracing::{info, Level}; -use tracing_subscriber; - -#[tokio::main] -async fn main() -> Result<()> { - // Initialize logging - tracing_subscriber::fmt().with_max_level(Level::INFO).init(); - - info!("๐Ÿš€ Starting Wave Comparison Backtest"); - - // Create repositories (mock for now, will integrate with DBN) - let repositories = Arc::new(DefaultRepositories::mock()); - - // Create backtest engine with $100,000 initial capital - let backtest = WaveComparisonBacktest::new(repositories, 100_000.0); - - // Define date range: last 30 days - let date_range = DateRange { - start: Utc::now() - Duration::days(30), - end: Utc::now(), - }; - - // Run comparison for ES.FUT (E-mini S&P 500) - info!("๐Ÿ“Š Running comparison for ES.FUT..."); - let results = backtest.run_comparison("ES.FUT", date_range).await?; - - // Print summary to console - backtest.print_summary(&results); - - // Export results to JSON and CSV - backtest.export_results(&results)?; - - info!("\nโœ… Wave Comparison Backtest Complete!"); - info!(" Check results/ directory for JSON and CSV exports"); - - Ok(()) -} diff --git a/services/backtesting_service/src/repositories.rs b/services/backtesting_service/src/repositories.rs index 0141dd778..5c84f72c4 100644 --- a/services/backtesting_service/src/repositories.rs +++ b/services/backtesting_service/src/repositories.rs @@ -120,7 +120,7 @@ impl BacktestingRepositories for DefaultRepositories { } } -#[cfg(test)] +#[cfg(any(test, feature = "test-utils"))] impl DefaultRepositories { /// Create a mock instance for testing /// diff --git a/services/ml_training_service/src/promotion_manager.rs b/services/ml_training_service/src/promotion_manager.rs index 8dab74ff2..e0dfd8b55 100644 --- a/services/ml_training_service/src/promotion_manager.rs +++ b/services/ml_training_service/src/promotion_manager.rs @@ -53,6 +53,12 @@ pub struct PromotionManager { active_models: Arc>>, } +impl Default for PromotionManager { + fn default() -> Self { + Self::new() + } +} + impl PromotionManager { /// Create a new promotion manager with empty state. pub fn new() -> Self { diff --git a/services/trading_service/examples/latency_demo.rs b/services/trading_service/examples/latency_demo.rs deleted file mode 100644 index ac57a9708..000000000 --- a/services/trading_service/examples/latency_demo.rs +++ /dev/null @@ -1,292 +0,0 @@ -//! Demonstration of sub-50ฮผs latency validation system -//! -//! This example shows how the hdrhistogram-based latency recorder works -//! and validates P50/P95/P99 measurements for trading operations. - -use hdrhistogram::Histogram; -use std::collections::HashMap; -use std::sync::{Arc, Mutex}; -use std::time::{Duration, Instant}; - -/// Simplified latency categories for demo -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -enum LatencyCategory { - OrderSubmission, - RiskValidation, - OrderProcessing, - EndToEndOrder, -} - -impl LatencyCategory { - fn name(&self) -> &'static str { - match self { - Self::OrderSubmission => "order_submission", - Self::RiskValidation => "risk_validation", - Self::OrderProcessing => "order_processing", - Self::EndToEndOrder => "end_to_end_order", - } - } -} - -/// Demo latency recorder -struct DemoLatencyRecorder { - histograms: Arc>>>, -} - -impl DemoLatencyRecorder { - fn new() -> Self { - Self { - histograms: Arc::new(Mutex::new(HashMap::new())), - } - } - - fn record(&self, category: LatencyCategory, latency_ns: u64) { - let mut histograms = self.histograms.lock().expect("INVARIANT: Lock should not be poisoned"); - let histogram = histograms.entry(category).or_insert_with(|| { - Histogram::new_with_bounds(1, 10_000_000, 3).expect("Failed to create histogram") - }); - - if let Err(e) = histogram.record(latency_ns) { - println!("Failed to record latency: {}", e); - } - } - - fn get_stats(&self, category: LatencyCategory) -> Option { - let histograms = self.histograms.lock().expect("INVARIANT: Lock should not be poisoned"); - histograms.get(&category).map(|histogram| LatencyStats { - count: histogram.len(), - p50_ns: histogram.value_at_quantile(0.50), - p95_ns: histogram.value_at_quantile(0.95), - p99_ns: histogram.value_at_quantile(0.99), - }) - } - - fn generate_report(&self) -> Vec<(LatencyCategory, LatencyStats)> { - let histograms = self.histograms.lock().expect("INVARIANT: Lock should not be poisoned"); - let mut results = Vec::new(); - - for (&category, histogram) in histograms.iter() { - if histogram.len() > 0 { - let stats = LatencyStats { - count: histogram.len(), - p50_ns: histogram.value_at_quantile(0.50), - p95_ns: histogram.value_at_quantile(0.95), - p99_ns: histogram.value_at_quantile(0.99), - }; - results.push((category, stats)); - } - } - - results - } -} - -#[derive(Debug)] -struct LatencyStats { - count: u64, - p50_ns: u64, - p95_ns: u64, - p99_ns: u64, -} - -impl LatencyStats { - fn p50_us(&self) -> f64 { - self.p50_ns as f64 / 1_000.0 - } - fn p95_us(&self) -> f64 { - self.p95_ns as f64 / 1_000.0 - } - fn p99_us(&self) -> f64 { - self.p99_ns as f64 / 1_000.0 - } - fn meets_target(&self, target_us: f64) -> bool { - self.p99_us() <= target_us - } -} - -fn simulate_cpu_work(duration: Duration) { - let start = Instant::now(); - let mut counter = 0u64; - - while start.elapsed() < duration { - counter = counter.wrapping_add(1); - } - - // Prevent optimization - if counter == u64::MAX { - println!("Unlikely: {}", counter); - } -} - -fn simulate_trading_operation(recorder: &DemoLatencyRecorder, _iteration: u64) { - // End-to-end timing - let end_to_end_start = Instant::now(); - - // Order submission (5ฮผs target) - let submission_start = Instant::now(); - simulate_cpu_work(Duration::from_nanos(5_000)); - recorder.record( - LatencyCategory::OrderSubmission, - submission_start.elapsed().as_nanos() as u64, - ); - - // Risk validation (8ฮผs target) - let risk_start = Instant::now(); - simulate_cpu_work(Duration::from_nanos(8_000)); - recorder.record( - LatencyCategory::RiskValidation, - risk_start.elapsed().as_nanos() as u64, - ); - - // Order processing (12ฮผs target) - let processing_start = Instant::now(); - simulate_cpu_work(Duration::from_nanos(12_000)); - recorder.record( - LatencyCategory::OrderProcessing, - processing_start.elapsed().as_nanos() as u64, - ); - - // Record end-to-end - recorder.record( - LatencyCategory::EndToEndOrder, - end_to_end_start.elapsed().as_nanos() as u64, - ); -} - -fn main() { - println!("๐Ÿš€ Foxhunt Trading Service - Sub-50ฮผs Latency Validation Demo"); - println!("=============================================================="); - - let recorder = DemoLatencyRecorder::new(); - let target_us = 50.0; - let iterations = 10_000; - - println!("Running {} trading operations...", iterations); - println!("Target P99 latency: {}ฮผs", target_us); - println!(); - - // Warm-up - println!("Warming up..."); - for i in 0..1000 { - simulate_trading_operation(&recorder, i); - } - - // Clear warm-up data and start fresh - let recorder = DemoLatencyRecorder::new(); - - // Main test - println!("Running main performance test..."); - let test_start = Instant::now(); - - for i in 0..iterations { - simulate_trading_operation(&recorder, i); - - if (i + 1) % 1000 == 0 { - println!(" Completed {} operations...", i + 1); - } - } - - let test_duration = test_start.elapsed(); - let ops_per_sec = iterations as f64 / test_duration.as_secs_f64(); - - println!(); - println!("๐Ÿ“Š PERFORMANCE RESULTS"); - println!("======================"); - println!("Test completed in {:.2}s", test_duration.as_secs_f64()); - println!("Throughput: {:.0} operations/second", ops_per_sec); - println!(); - - // Generate detailed report - let results = recorder.generate_report(); - let mut all_targets_met = true; - let mut passed_categories = 0; - - println!("๐Ÿ“ˆ LATENCY ANALYSIS"); - println!("==================="); - - for (category, stats) in &results { - let target_met = stats.meets_target(target_us); - let status = if target_met { "โœ… PASS" } else { "โŒ FAIL" }; - - if target_met { - passed_categories += 1; - } else { - all_targets_met = false; - } - - println!("{} {} ({} samples):", status, category.name(), stats.count); - println!( - " P50: {:.1}ฮผs | P95: {:.1}ฮผs | P99: {:.1}ฮผs", - stats.p50_us(), - stats.p95_us(), - stats.p99_us() - ); - } - - println!(); - println!("๐ŸŽฏ FINAL RESULTS"); - println!("================"); - - if all_targets_met { - println!( - "โœ… SUCCESS: All {} categories meet sub-{}ฮผs target!", - results.len(), - target_us - ); - println!("๐Ÿš€ Trading Service is ready for production deployment!"); - } else { - println!( - "โŒ PARTIAL: {}/{} categories meet sub-{}ฮผs target", - passed_categories, - results.len(), - target_us - ); - println!("๐Ÿ”ง Optimization needed for failed categories"); - } - - println!(); - println!("๐Ÿ“‹ SYSTEM VALIDATION COMPLETE"); - println!("=============================="); - println!("This demo validates that the hdrhistogram-based latency"); - println!("recording system can accurately measure and report P50/P95/P99"); - println!("latencies for critical trading operations at sub-50ฮผs precision."); - println!(); - println!("The actual Trading Service implementation includes:"); - println!(" โ€ข TimingGuard for automatic RAII-based measurement"); - println!(" โ€ข Comprehensive soak testing with configurable load"); - println!(" โ€ข Integration with all critical trading paths"); - println!(" โ€ข Production-ready latency validation tooling"); -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_latency_recording() { - let recorder = DemoLatencyRecorder::new(); - - // Record test latencies - recorder.record(LatencyCategory::OrderSubmission, 25_000); // 25ฮผs - recorder.record(LatencyCategory::OrderSubmission, 35_000); // 35ฮผs - recorder.record(LatencyCategory::OrderSubmission, 45_000); // 45ฮผs - - let stats = recorder - .get_stats(LatencyCategory::OrderSubmission) - .unwrap(); - assert_eq!(stats.count, 3); - assert!(stats.meets_target(50.0)); - assert!(stats.p99_us() < 50.0); - } - - #[test] - fn test_cpu_work_simulation() { - let start = Instant::now(); - simulate_cpu_work(Duration::from_micros(10)); - let elapsed = start.elapsed(); - - // Should take at least the requested time - assert!(elapsed >= Duration::from_micros(8)); - assert!(elapsed < Duration::from_micros(50)); - } -} diff --git a/services/trading_service/examples/test_ensemble_metrics.rs b/services/trading_service/examples/test_ensemble_metrics.rs deleted file mode 100644 index e10bd601d..000000000 --- a/services/trading_service/examples/test_ensemble_metrics.rs +++ /dev/null @@ -1,176 +0,0 @@ -//! Test Ensemble Metrics Collection -//! -//! This example tests the ensemble metrics system by: -//! 1. Running 1000 ensemble predictions -//! 2. Updating model weights every 100 predictions -//! 3. Recording P&L attribution -//! 4. Verifying all 10 metrics populate correctly -//! -//! Run with: -//! ```bash -//! cargo run -p trading_service --example test_ensemble_metrics -//! ``` - -use ml::{Features, MLResult}; -use std::time::Duration; -use tokio; -use tracing::{info, Level}; -use tracing_subscriber; - -use trading_service::ensemble_coordinator::EnsembleCoordinator; -use trading_service::ensemble_metrics::{ - ABTestAssignment, ABTestGroup, ABTestMetric, ABTestMetricDiff, CheckpointSwapEvent, - CheckpointSwapStatus, -}; - -#[tokio::main] -async fn main() -> MLResult<()> { - // Initialize logging - tracing_subscriber::fmt().with_max_level(Level::INFO).init(); - - info!("Starting Ensemble Metrics Test"); - info!("================================="); - - // Create ensemble coordinator - let coordinator = EnsembleCoordinator::new(); - - // Register models with weights - coordinator.register_model("DQN".to_string(), 0.35).await?; - coordinator.register_model("PPO".to_string(), 0.30).await?; - coordinator.register_model("TFT".to_string(), 0.35).await?; - - info!("Registered 3 models in ensemble"); - - // Run 1000 predictions with metric recording - info!("\nRunning 1000 ensemble predictions..."); - - let mut total_latency = 0.0; - let mut high_disagreement_count = 0; - - for i in 0..1000 { - // Create diverse features to generate varied predictions - let feature_values: Vec = (0..16) - .map(|j| ((i as f64 * 0.1) + (j as f64 * 0.05)).sin()) - .collect(); - - let features = Features::new( - feature_values, - (0..16).map(|j| format!("feature_{}", j)).collect(), - ); - - // Make prediction (metrics are recorded automatically) - let decision = coordinator.predict(&features).await?; - - // Track statistics - total_latency += 12.5; // Mock latency for demonstration - if decision.disagreement_rate > 0.5 { - high_disagreement_count += 1; - } - - // Update model weights every 100 predictions - if (i + 1) % 100 == 0 { - coordinator.update_model_weights().await?; - info!("Updated model weights at prediction {}", i + 1); - } - - // Simulate P&L attribution every 50 predictions - if (i + 1) % 50 == 0 { - // Mock P&L values for each model - coordinator.record_model_pnl("DQN", "ES.FUT", 125.0 + (i as f64 * 0.5)); - coordinator.record_model_pnl("PPO", "ES.FUT", 110.0 + (i as f64 * 0.3)); - coordinator.record_model_pnl("TFT", "ES.FUT", 95.0 + (i as f64 * 0.2)); - } - - // Progress indicator - if (i + 1) % 200 == 0 { - info!("Completed {} predictions", i + 1); - } - } - - info!("\nโœ… Completed 1000 predictions"); - info!(" Average latency: {:.2}ฮผs", total_latency / 1000.0); - info!(" High disagreement events: {}", high_disagreement_count); - - // Test checkpoint swap metrics - info!("\nTesting checkpoint swap metrics..."); - - let swap_success = CheckpointSwapEvent { - model_id: "DQN".to_string(), - status: CheckpointSwapStatus::Success, - }; - swap_success.record(); - - let swap_rollback = CheckpointSwapEvent { - model_id: "PPO".to_string(), - status: CheckpointSwapStatus::Rollback, - }; - swap_rollback.record(); - - info!("โœ… Recorded 2 checkpoint swap events"); - - // Test A/B testing metrics - info!("\nTesting A/B test metrics..."); - - for i in 0..100 { - let group = if i % 2 == 0 { - ABTestGroup::Control - } else { - ABTestGroup::Treatment - }; - - let assignment = ABTestAssignment { - test_id: "test-ensemble-001".to_string(), - group, - }; - assignment.record(); - } - - info!("โœ… Recorded 100 A/B test assignments (50/50 split)"); - - // Record A/B test metric differences - let sharpe_diff = ABTestMetricDiff { - test_id: "test-ensemble-001".to_string(), - metric: ABTestMetric::SharpeRatio, - difference: 0.32, // Treatment 32% better - }; - sharpe_diff.record(); - - let win_rate_diff = ABTestMetricDiff { - test_id: "test-ensemble-001".to_string(), - metric: ABTestMetric::WinRate, - difference: 0.07, // Treatment 7% better - }; - win_rate_diff.record(); - - info!("โœ… Recorded A/B test metric differences"); - - // Summary - info!("\n๐ŸŽฏ Metrics Collection Summary"); - info!("================================"); - info!("โœ… 1. ensemble_aggregation_latency_microseconds: 1000 samples"); - info!("โœ… 2. ensemble_confidence_score: 1000 updates"); - info!("โœ… 3. ensemble_disagreement_rate: 1000 updates"); - info!("โœ… 4. ensemble_predictions_total: 1000 increments"); - info!("โœ… 5. ensemble_model_weight: 30 updates (3 models ร— 10 batches)"); - info!( - "โœ… 6. ensemble_high_disagreement_total: {} events", - high_disagreement_count - ); - info!("โœ… 7. ensemble_model_pnl_contribution_dollars: 60 samples (3 models ร— 20 batches)"); - info!("โœ… 8. checkpoint_swaps_total: 2 events"); - info!("โœ… 9. ab_test_assignments_total: 100 assignments"); - info!("โœ… 10. ab_test_metric_difference: 2 metrics (Sharpe, WinRate)"); - - info!("\n๐Ÿ“Š Next Steps:"); - info!("1. Start Prometheus scraping: http://localhost:9092/metrics"); - info!("2. Import Grafana dashboard: monitoring/grafana/ensemble_ml_production.json"); - info!("3. View metrics in Grafana: http://localhost:3000"); - - // Keep metrics endpoint alive for scraping - info!("\nโณ Keeping process alive for 60 seconds to allow Prometheus scraping..."); - tokio::time::sleep(Duration::from_secs(60)).await; - - info!("โœ… Test complete!"); - - Ok(()) -} diff --git a/testing/integration/chaos/examples/mod.rs b/testing/integration/chaos/examples/mod.rs deleted file mode 100644 index 268c554ce..000000000 --- a/testing/integration/chaos/examples/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -//! Chaos Engineering Examples Module - -pub mod usage_examples; - -// DO NOT RE-EXPORT - Use explicit imports at usage sites diff --git a/testing/integration/chaos/examples/usage_examples.rs b/testing/integration/chaos/examples/usage_examples.rs deleted file mode 100644 index a75d6b984..000000000 --- a/testing/integration/chaos/examples/usage_examples.rs +++ /dev/null @@ -1,484 +0,0 @@ -//! Chaos Engineering Usage Examples -//! -//! This file demonstrates how to use the Foxhunt chaos engineering framework -//! for testing system resilience and recovery capabilities. - -use anyhow::Result; -use std::path::PathBuf; -use std::time::Duration; -use uuid::Uuid; - -// Import our chaos engineering modules -use super::super::chaos_framework::{ChaosExperiment, ChaosOrchestrator, FailureType, Signal}; -use super::super::ml_training_chaos::{MLChaosConfig, MLTrainingChaosTests, ModelType}; -use crate::chaos::nightly_chaos_runner::{NightlyChaosConfig, NightlyChaosRunner, ChaosJobEvent}; - -/// Example 1: Simple ML Training Service Kill/Restart Test -/// -/// This example demonstrates how to test the MLTrainingService's ability -/// to recover from process termination and resume training from checkpoints. -pub async fn example_ml_service_kill_test() -> Result<()> { - println!("๐Ÿงช Example 1: ML Training Service Kill/Restart Test"); - - // Create chaos orchestrator - let orchestrator = ChaosOrchestrator::new(1); - - // Define the experiment - let experiment = ChaosExperiment { - id: Uuid::new_v4(), - name: "MLTrainingService SIGTERM Recovery Test".to_string(), - description: "Test ML service recovery from graceful termination".to_string(), - target_service: "ml_training_service".to_string(), - failure_type: FailureType::ProcessKill { - signal: Signal::SIGTERM, - delay_before_restart_ms: 2000, // 2 second delay - }, - duration: Duration::from_secs(5), // Quick failure - recovery_timeout: Duration::from_secs(30), - max_recovery_time_ms: 100, // HFT requirement: sub-100ms - enabled: true, - }; - - // Register and execute the experiment - orchestrator.register_experiment(experiment.clone()).await?; - let result = orchestrator.execute_experiment(experiment.id).await?; - - println!("๐Ÿ“Š Result: {:?}", result.status); - if let Some(recovery_time) = result.recovery_time_ms { - println!("โฑ๏ธ Recovery time: {}ms", recovery_time); - if recovery_time <= 100 { - println!("โœ… Recovery time meets HFT requirements"); - } else { - println!("โš ๏ธ Recovery time exceeds HFT requirements"); - } - } - - println!( - "๐Ÿ”ง Checkpoint integrity: {}", - if result.checkpoint_integrity { - "โœ… Valid" - } else { - "โŒ Corrupted" - } - ); - - Ok(()) -} - -/// Example 2: Comprehensive ML Chaos Test Suite -/// -/// This example runs the full ML chaos test suite across all supported models -/// with different failure scenarios and generates a comprehensive report. -pub async fn example_comprehensive_ml_chaos_suite() -> Result<()> { - println!("๐Ÿงช Example 2: Comprehensive ML Chaos Test Suite"); - - // Configure ML chaos testing - let ml_config = MLChaosConfig { - ml_service_endpoint: "http://localhost:8080".to_string(), - checkpoint_base_path: PathBuf::from("/tmp/chaos_checkpoints"), - model_types: vec![ - ModelType::TLOB, // Ultra-low latency transformer - ModelType::DQN, // Deep Q-learning for strategy optimization - ModelType::MAMBA2, // State space model for sequence modeling - ], - training_timeout_secs: 300, - max_recovery_time_ms: 100, // HFT requirement - gpu_memory_threshold_mb: 4096, // 4GB for testing - }; - - // Create ML chaos test suite - let ml_chaos = MLTrainingChaosTests::new(ml_config); - - println!("๐Ÿš€ Starting ML chaos test suite..."); - let results = ml_chaos.run_ml_chaos_suite().await?; - - println!("๐Ÿ“ˆ Test Results:"); - println!(" Total tests: {}", results.len()); - - let successful = results - .iter() - .filter(|r| r.training_loss_continuity) - .count(); - let failed = results.len() - successful; - - println!( - " Successful: {} ({}%)", - successful, - (successful * 100) / results.len() - ); - println!( - " Failed: {} ({}%)", - failed, - (failed * 100) / results.len() - ); - - // Generate and display report - let report = ml_chaos.generate_chaos_report(&results).await?; - println!("\n๐Ÿ“‹ Detailed Report:"); - println!("{}", report); - - Ok(()) -} - -/// Example 3: Memory Pressure Test with Performance Monitoring -/// -/// This example demonstrates how to test system behavior under memory pressure -/// while monitoring performance metrics and recovery times. -pub async fn example_memory_pressure_test() -> Result<()> { - println!("๐Ÿงช Example 3: Memory Pressure Test with Performance Monitoring"); - - let orchestrator = ChaosOrchestrator::new(1); - - // Create memory pressure experiment - let experiment = ChaosExperiment { - id: Uuid::new_v4(), - name: "High Memory Pressure Test".to_string(), - description: "Test system behavior under 4GB memory pressure".to_string(), - target_service: "ml_training_service".to_string(), - failure_type: FailureType::MemoryPressure { - target_mb: 4096, // 4GB pressure - duration_ms: 30000, // 30 seconds - }, - duration: Duration::from_secs(35), - recovery_timeout: Duration::from_secs(60), - max_recovery_time_ms: 200, // Relaxed for memory pressure - enabled: true, - }; - - println!("๐Ÿ’พ Applying 4GB memory pressure for 30 seconds..."); - - orchestrator.register_experiment(experiment.clone()).await?; - let result = orchestrator.execute_experiment(experiment.id).await?; - - println!("๐Ÿ“Š Memory Pressure Test Results:"); - println!(" Status: {:?}", result.status); - - if let Some(recovery_time) = result.recovery_time_ms { - println!(" Recovery time: {}ms", recovery_time); - - // Check performance regression - if let Some(regression) = result.performance_regression { - println!(" Performance impact:"); - println!( - " Before: {}ns P99 latency", - regression.latency_p99_before_ns - ); - println!( - " After: {}ns P99 latency", - regression.latency_p99_after_ns - ); - println!(" Regression: {:.1}%", regression.regression_percent); - } - } - - Ok(()) -} - -/// Example 4: Network Partition Resilience Test -/// -/// This example tests the system's ability to handle network partitions -/// affecting critical services like PostgreSQL and Redis. -pub async fn example_network_partition_test() -> Result<()> { - println!("๐Ÿงช Example 4: Network Partition Resilience Test"); - - let orchestrator = ChaosOrchestrator::new(1); - - let experiment = ChaosExperiment { - id: Uuid::new_v4(), - name: "Database Network Partition Test".to_string(), - description: "Test resilience to database connection failures".to_string(), - target_service: "ml_training_service".to_string(), - failure_type: FailureType::NetworkPartition { - target_ports: vec![5432, 6379], // PostgreSQL, Redis - duration_ms: 15000, // 15 seconds - }, - duration: Duration::from_secs(20), - recovery_timeout: Duration::from_secs(45), - max_recovery_time_ms: 100, - enabled: true, - }; - - println!("๐ŸŒ Creating network partition affecting PostgreSQL and Redis..."); - - orchestrator.register_experiment(experiment.clone()).await?; - let result = orchestrator.execute_experiment(experiment.id).await?; - - println!("๐Ÿ“ก Network Partition Test Results:"); - println!(" Status: {:?}", result.status); - - match result.status { - super::super::chaos_framework::ChaosStatus::Succeeded => { - println!(" โœ… System successfully handled network partition"); - } - super::super::chaos_framework::ChaosStatus::RecoveryTimeout => { - println!(" โฐ Recovery took longer than expected"); - } - super::super::chaos_framework::ChaosStatus::Failed => { - println!(" โŒ System failed to recover from network partition"); - } - _ => { - println!(" โ“ Unexpected test status"); - } - } - - Ok(()) -} - -/// Example 5: GPU Resource Exhaustion Test -/// -/// This example tests ML model training behavior when GPU resources -/// are exhausted, simulating scenarios where multiple models compete -/// for limited GPU memory. -pub async fn example_gpu_exhaustion_test() -> Result<()> { - println!("๐Ÿงช Example 5: GPU Resource Exhaustion Test"); - - let orchestrator = ChaosOrchestrator::new(1); - - let experiment = ChaosExperiment { - id: Uuid::new_v4(), - name: "GPU Memory Exhaustion Test".to_string(), - description: "Test ML training behavior under GPU memory pressure".to_string(), - target_service: "ml_training_service".to_string(), - failure_type: FailureType::GpuResourceExhaustion { - memory_fill_percent: 95, // Fill 95% of GPU memory - duration_ms: 25000, // 25 seconds - }, - duration: Duration::from_secs(30), - recovery_timeout: Duration::from_secs(90), // GPU recovery can be slower - max_recovery_time_ms: 200, // Relaxed for GPU operations - enabled: true, - }; - - println!("๐ŸŽฎ Exhausting 95% of GPU memory for 25 seconds..."); - - orchestrator.register_experiment(experiment.clone()).await?; - let result = orchestrator.execute_experiment(experiment.id).await?; - - println!("๐ŸŽฏ GPU Exhaustion Test Results:"); - println!(" Status: {:?}", result.status); - println!( - " Checkpoint integrity: {}", - if result.checkpoint_integrity { - "โœ…" - } else { - "โŒ" - } - ); - - // GPU-specific analysis would check: - // - Model training continuation after GPU memory cleared - // - Checkpoint consistency during GPU pressure - // - Training loss continuity - // - GPU memory leak detection - - Ok(()) -} - -/// Example 6: Automated Nightly Chaos Testing -/// -/// This example sets up automated nightly chaos testing with proper -/// scheduling, notification, and reporting. -pub async fn example_nightly_chaos_automation() -> Result<()> { - println!("๐Ÿงช Example 6: Automated Nightly Chaos Testing Setup"); - - // Configure nightly chaos testing - let config = NightlyChaosConfig { - enabled: true, - schedule_time: chrono::NaiveTime::from_hms_opt(2, 0, 0).unwrap(), // 2 AM - timezone: "UTC".to_string(), - max_duration_hours: 3, - notification_webhook: Some("https://hooks.slack.com/services/YOUR/WEBHOOK".to_string()), - report_storage_path: PathBuf::from("./nightly_chaos_reports"), - ml_chaos_config: MLChaosConfig { - ml_service_endpoint: "http://localhost:8080".to_string(), - checkpoint_base_path: PathBuf::from("/production/ml_checkpoints"), - model_types: vec![ - ModelType::TLOB, // Critical for order book analysis - ModelType::DQN, // Risk management - ModelType::MAMBA2, // Market prediction - ], - training_timeout_secs: 600, // 10 minutes for production - max_recovery_time_ms: 100, - gpu_memory_threshold_mb: 16384, // 16GB production GPU - }, - exclude_weekends: true, // Skip weekends for production safety - retry_on_failure: true, - max_retries: 2, - }; - - println!("๐Ÿ“… Setting up nightly chaos testing at 2:00 AM UTC..."); - println!(" Excluding weekends: {}", config.exclude_weekends); - println!(" Max duration: {} hours", config.max_duration_hours); - println!(" Report storage: {:?}", config.report_storage_path); - - // Create and configure the runner - let runner = NightlyChaosRunner::new(config); - - // Subscribe to events for monitoring - let mut event_receiver = runner.subscribe_events(); - - // Start event monitoring in background - let _event_handler = tokio::spawn(async move { - while let Ok(event) = event_receiver.recv().await { - match event { - ChaosJobEvent::JobScheduled { id, scheduled_time } => { - println!("๐Ÿ“… Chaos job {} scheduled for {}", id, scheduled_time); - } - ChaosJobEvent::JobStarted { id } => { - println!("๐Ÿš€ Chaos job {} started", id); - } - ChaosJobEvent::JobCompleted { id, summary } => { - println!("โœ… Chaos job {} completed", id); - println!( - " Average recovery time: {:.1}ms", - summary.average_recovery_time_ms - ); - println!(" SLA violations: {}", summary.sla_violations); - println!(" Checkpoint failures: {}", summary.checkpoint_failures); - } - ChaosJobEvent::AlertTriggered { - message, - severity, - } => { - println!("๐Ÿšจ Alert ({:?}): {}", severity, message); - } - _ => {} - } - } - }); - - println!("๐Ÿ”„ Nightly chaos automation is configured and ready"); - println!(" Use `runner.start().await` to begin scheduled testing"); - - // In production, you would call runner.start().await here - // For this example, we just show the setup - - Ok(()) -} - -/// Example 7: CI/CD Integration - Quick Chaos Validation -/// -/// This example shows how to integrate chaos testing into CI/CD pipelines -/// with quick validation tests that can run in a few minutes. -pub async fn example_ci_cd_quick_validation() -> Result<()> { - println!("๐Ÿงช Example 7: CI/CD Quick Chaos Validation"); - - // Quick config for CI/CD - shorter timeouts, fewer models - let quick_ml_config = MLChaosConfig { - ml_service_endpoint: "http://localhost:8080".to_string(), - checkpoint_base_path: PathBuf::from("/tmp/ci_checkpoints"), - model_types: vec![ModelType::TLOB], // Just test TLOB for speed - training_timeout_secs: 60, // 1 minute max - max_recovery_time_ms: 100, - gpu_memory_threshold_mb: 2048, // Lower for CI environment - }; - - println!("โšก Running quick chaos validation for CI/CD..."); - println!(" Target: Sub-2 minute execution time"); - println!(" Models: TLOB only (fastest)"); - println!(" Recovery requirement: < 100ms"); - - let ml_chaos = MLTrainingChaosTests::new(quick_ml_config); - - // Run a single representative test - let orchestrator = ChaosOrchestrator::new(1); - - let quick_experiment = ChaosExperiment { - id: Uuid::new_v4(), - name: "CI Quick Recovery Test".to_string(), - description: "Fast recovery test for CI pipeline".to_string(), - target_service: "ml_training_service".to_string(), - failure_type: FailureType::ProcessKill { - signal: Signal::SIGTERM, - delay_before_restart_ms: 1000, // Quick restart - }, - duration: Duration::from_secs(2), // Very quick failure - recovery_timeout: Duration::from_secs(15), // Quick recovery - max_recovery_time_ms: 100, - enabled: true, - }; - - let start_time = std::time::Instant::now(); - - orchestrator - .register_experiment(quick_experiment.clone()) - .await?; - let result = orchestrator.execute_experiment(quick_experiment.id).await?; - - let total_time = start_time.elapsed(); - - println!("โฑ๏ธ Total execution time: {:.1}s", total_time.as_secs_f64()); - println!("๐Ÿ“Š Quick validation result: {:?}", result.status); - - match result.status { - super::super::chaos_framework::ChaosStatus::Succeeded => { - println!("โœ… CI/CD chaos validation PASSED"); - std::process::exit(0); - } - _ => { - println!("โŒ CI/CD chaos validation FAILED"); - if let Some(recovery_time) = result.recovery_time_ms { - println!(" Recovery time: {}ms", recovery_time); - } - println!(" Checkpoint integrity: {}", result.checkpoint_integrity); - std::process::exit(1); - } - } -} - -/// Run all examples -pub async fn run_all_examples() -> Result<()> { - println!("๐Ÿš€ Running Foxhunt Chaos Engineering Examples\n"); - - // Example 1: Basic ML service kill test - example_ml_service_kill_test().await?; - println!(); - - // Example 2: Comprehensive test suite - example_comprehensive_ml_chaos_suite().await?; - println!(); - - // Example 3: Memory pressure test - example_memory_pressure_test().await?; - println!(); - - // Example 4: Network partition test - example_network_partition_test().await?; - println!(); - - // Example 5: GPU exhaustion test - example_gpu_exhaustion_test().await?; - println!(); - - // Example 6: Nightly automation setup - example_nightly_chaos_automation().await?; - println!(); - - // Example 7: CI/CD integration - // example_ci_cd_quick_validation().await?; // Skip in demo to avoid exit - - println!("๐ŸŽ‰ All chaos engineering examples completed successfully!"); - - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[tokio::test] - async fn test_example_runs() { - // Test that examples can be set up without actual service calls - let ml_config = MLChaosConfig { - ml_service_endpoint: "http://localhost:8080".to_string(), - checkpoint_base_path: PathBuf::from("/tmp/test"), - model_types: vec![ModelType::TLOB], - training_timeout_secs: 60, - max_recovery_time_ms: 100, - gpu_memory_threshold_mb: 2048, - }; - - let _ml_chaos = MLTrainingChaosTests::new(ml_config); - assert!(true); // Basic creation test - } -}