feat(ml): WAVE 29 DQN Codebase Cleanup & Refactoring Campaign

BREAKING CHANGES: - Removed orphaned dqn.rs monolithic trainer (4,975 lines) - Removed orphaned dqn_ensemble.rs module (816 lines) - Removed orphaned tft.rs and tft_complete_int8_integration_test.rs - TFT trainer split into modular directory structure DQN Module Refactoring: - Split trainers/dqn.rs into modular structure (config.rs, statistics.rs, trainer.rs) - Fixed hyperopt 39D search space (continuous params only) - Boolean flags (use_dueling, use_double_dqn, use_per, use_noisy_nets) are now FIXED architectural decisions - use_distributional defaults to false (Candle BUG #36 - scatter_add gradient issues) Clean Module Structure: - ml/src/trainers/dqn/ directory with proper mod.rs exports - ml/src/trainers/tft/ directory with config.rs, types.rs, model.rs, trainer.rs, tests.rs - All P0 features validated: TD-error clamping, batch diversity, LR scheduler, priority staleness Documentation: - Added comprehensive docs in docs/codebase-cleanup/ - ADR-001 for DQN refactoring decisions - Rainbow DQN component matrix and quick reference guides Build Status: Compiles with zero errors 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
2025-11-27 23:46:13 +01:00
parent 2c1acda2f3
commit 2df1ea92e1
763 changed files with 247870 additions and 1714 deletions
--- a/trading_engine/tests/engine_execution_flow_tests.rs
+++ b/trading_engine/tests/engine_execution_flow_tests.rs
@@ -0,0 +1,588 @@
+//! Trading Engine Execution Flow Tests
+//!
+//! Tests the engine's execution processing - the critical path for fills
+//! Focuses on testable components without requiring broker configuration
+
+use chrono::Utc;
+use common::{MarketDataEvent, OrderId, OrderSide, OrderStatus, OrderType, TimeInForce};
+use rust_decimal::Decimal;
+use std::collections::HashMap;
+use std::sync::Arc;
+use tokio::sync::broadcast;
+use trading_engine::trading::data_interface::{DataProvider, Subscription};
+use trading_engine::trading::engine::TradingEngine;
+use trading_engine::trading_operations::{ExecutionResult, LiquidityFlag, TradingOrder};
+
+// ============================================================================
+// Mock Data Provider
+// ============================================================================
+
+#[derive(Debug, Clone)]
+struct MockDataProvider {
+    market_data_tx: broadcast::Sender<MarketDataEvent>,
+    order_update_tx: broadcast::Sender<MarketDataEvent>,
+}
+
+impl MockDataProvider {
+    fn new() -> Self {
+        let (market_data_tx, _) = broadcast::channel(1000);
+        let (order_update_tx, _) = broadcast::channel(1000);
+        Self {
+            market_data_tx,
+            order_update_tx,
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl DataProvider for MockDataProvider {
+    async fn subscribe_market_data(&self, _subscription: Subscription) -> Result<(), String> {
+        Ok(())
+    }
+
+    fn subscribe_market_data_events(&self) -> broadcast::Receiver<MarketDataEvent> {
+        self.market_data_tx.subscribe()
+    }
+
+    fn subscribe_order_update_events(&self) -> broadcast::Receiver<MarketDataEvent> {
+        self.order_update_tx.subscribe()
+    }
+}
+
+// ============================================================================
+// Helpers
+// ============================================================================
+
+fn create_engine() -> TradingEngine {
+    TradingEngine::new(Arc::new(MockDataProvider::new()))
+}
+
+fn create_test_order(id: &str, symbol: &str, side: OrderSide, qty: i64, price: i64) -> TradingOrder {
+    TradingOrder {
+        id: id.to_string().into(),
+        symbol: symbol.to_string(),
+        side,
+        order_type: OrderType::Limit,
+        quantity: Decimal::from(qty),
+        price: Decimal::from(price),
+        time_in_force: TimeInForce::Day,
+        account_id: None,
+        metadata: HashMap::new(),
+        created_at: Utc::now(),
+        submitted_at: None,
+        executed_at: None,
+        status: OrderStatus::Created,
+        fill_quantity: Decimal::ZERO,
+        average_fill_price: None,
+    }
+}
+
+fn create_execution(order_id: OrderId, symbol: &str, qty: i64, price: i64) -> ExecutionResult {
+    ExecutionResult {
+        order_id,
+        symbol: symbol.to_string(),
+        executed_quantity: Decimal::from(qty),
+        execution_price: Decimal::from(price),
+        execution_time: Utc::now(),
+        commission: Decimal::from(10),
+        liquidity_flag: LiquidityFlag::Maker,
+    }
+}
+
+// ============================================================================
+// Validation Tests (Tests that DON'T require broker)
+// ============================================================================
+
+#[tokio::test]
+async fn test_order_validation_zero_quantity() {
+    let engine = create_engine();
+
+    let result = engine
+        .submit_order(
+            "BTCUSD".to_string(),
+            OrderSide::Buy,
+            OrderType::Market,
+            Decimal::ZERO,
+            None,
+            None,
+        )
+        .await;
+
+    assert!(result.is_err());
+    assert!(result.unwrap_err().contains("positive"));
+}
+
+#[tokio::test]
+async fn test_order_validation_negative_quantity() {
+    let engine = create_engine();
+
+    let result = engine
+        .submit_order(
+            "BTCUSD".to_string(),
+            OrderSide::Buy,
+            OrderType::Market,
+            Decimal::from(-10),
+            None,
+            None,
+        )
+        .await;
+
+    assert!(result.is_err());
+    assert!(result.unwrap_err().contains("positive"));
+}
+
+#[tokio::test]
+async fn test_order_validation_empty_symbol() {
+    let engine = create_engine();
+
+    let result = engine
+        .submit_order(
+            "".to_string(),
+            OrderSide::Buy,
+            OrderType::Market,
+            Decimal::from(1),
+            None,
+            None,
+        )
+        .await;
+
+    assert!(result.is_err());
+    assert!(result.unwrap_err().contains("symbol"));
+}
+
+#[tokio::test]
+async fn test_order_validation_zero_limit_price() {
+    let engine = create_engine();
+
+    let result = engine
+        .submit_order(
+            "BTCUSD".to_string(),
+            OrderSide::Buy,
+            OrderType::Limit,
+            Decimal::from(1),
+            Some(Decimal::ZERO),
+            None,
+        )
+        .await;
+
+    assert!(result.is_err());
+    assert!(result.unwrap_err().contains("price"));
+}
+
+#[tokio::test]
+async fn test_order_validation_exceeds_buying_power() {
+    let engine = create_engine();
+
+    // Try to buy way more than account can afford
+    let result = engine
+        .submit_order(
+            "BTCUSD".to_string(),
+            OrderSide::Buy,
+            OrderType::Limit,
+            Decimal::from(100),
+            Some(Decimal::from(50000)),
+            None,
+        )
+        .await;
+
+    assert!(result.is_err());
+    assert!(result.unwrap_err().contains("buying power"));
+}
+
+// ============================================================================
+// Execution Processing Tests (Core critical path)
+// ============================================================================
+
+#[tokio::test]
+async fn test_process_execution_creates_position() {
+    let engine = create_engine();
+    let order_id: OrderId = "order-001".to_string().into();
+
+    let execution = create_execution(order_id, "BTCUSD", 100, 50000);
+    let result = engine.process_execution(execution).await;
+
+    assert!(result.is_ok());
+
+    // Verify position was created
+    let positions = engine.get_positions(Some("BTCUSD".to_string())).await;
+    assert!(positions.is_ok());
+
+    let pos_list = positions.unwrap();
+    assert_eq!(pos_list.len(), 1);
+    assert_eq!(pos_list[0].quantity, Decimal::from(100));
+    assert_eq!(pos_list[0].avg_cost, Decimal::from(50000));
+}
+
+#[tokio::test]
+async fn test_process_execution_updates_account() {
+    let engine = create_engine();
+    let order_id: OrderId = "order-002".to_string().into();
+
+    let initial_account = engine
+        .get_account_info("DEMO_ACCOUNT".to_string())
+        .await
+        .unwrap();
+
+    let initial_cash = initial_account.cash_balance;
+
+    let execution = ExecutionResult {
+        order_id,
+        symbol: "BTCUSD".to_string(),
+        executed_quantity: Decimal::from(1),
+        execution_price: Decimal::from(50000),
+        execution_time: Utc::now(),
+        commission: Decimal::from(25),
+        liquidity_flag: LiquidityFlag::Taker,
+    };
+
+    engine.process_execution(execution).await.unwrap();
+
+    let updated_account = engine
+        .get_account_info("DEMO_ACCOUNT".to_string())
+        .await
+        .unwrap();
+
+    // Commission should be deducted
+    assert_eq!(
+        updated_account.cash_balance,
+        initial_cash - Decimal::from(25)
+    );
+}
+
+#[tokio::test]
+async fn test_process_partial_fill() {
+    let engine = create_engine();
+    let order_id: OrderId = "order-003".to_string().into();
+
+    // Execute 30 out of 100
+    let execution = create_execution(order_id, "ETHUSD", 30, 3000);
+    engine.process_execution(execution).await.unwrap();
+
+    let positions = engine.get_positions(Some("ETHUSD".to_string())).await.unwrap();
+    assert_eq!(positions.len(), 1);
+    assert_eq!(positions[0].quantity, Decimal::from(30));
+}
+
+#[tokio::test]
+async fn test_process_multiple_executions_same_symbol() {
+    let engine = create_engine();
+
+    // First execution
+    let order1_id: OrderId = "order-004".to_string().into();
+    let exec1 = create_execution(order1_id, "BTCUSD", 50, 50000);
+    engine.process_execution(exec1).await.unwrap();
+
+    // Second execution at different price
+    let order2_id: OrderId = "order-005".to_string().into();
+    let exec2 = create_execution(order2_id, "BTCUSD", 50, 51000);
+    engine.process_execution(exec2).await.unwrap();
+
+    let positions = engine.get_positions(Some("BTCUSD".to_string())).await.unwrap();
+    assert_eq!(positions.len(), 1);
+    assert_eq!(positions[0].quantity, Decimal::from(100));
+
+    // Average cost should be (50*50000 + 50*51000)/100 = 50500
+    let expected_avg = (Decimal::from(50) * Decimal::from(50000)
+        + Decimal::from(50) * Decimal::from(51000)) / Decimal::from(100);
+    assert_eq!(positions[0].avg_cost, expected_avg);
+}
+
+#[tokio::test]
+async fn test_process_execution_with_commission() {
+    let engine = create_engine();
+    let order_id: OrderId = "order-006".to_string().into();
+
+    let execution = ExecutionResult {
+        order_id,
+        symbol: "SOLUSD".to_string(),
+        executed_quantity: Decimal::from(1000),
+        execution_price: Decimal::from(100),
+        execution_time: Utc::now(),
+        commission: Decimal::from(50),
+        liquidity_flag: LiquidityFlag::Taker,
+    };
+
+    engine.process_execution(execution).await.unwrap();
+
+    // Verify commission was processed
+    let account = engine.get_account_info("DEMO_ACCOUNT".to_string()).await.unwrap();
+
+    // Initial cash 50000 - commission 50 = 49950
+    assert_eq!(account.cash_balance, Decimal::from(50000) - Decimal::from(50));
+}
+
+#[tokio::test]
+async fn test_process_buy_then_sell_execution() {
+    let engine = create_engine();
+
+    // Buy 100
+    let buy_order_id: OrderId = "buy-001".to_string().into();
+    let buy_exec = create_execution(buy_order_id, "ETHUSD", 100, 3000);
+    engine.process_execution(buy_exec).await.unwrap();
+
+    // Sell 60 (reduce position)
+    let sell_order_id: OrderId = "sell-001".to_string().into();
+    let sell_exec = ExecutionResult {
+        order_id: sell_order_id,
+        symbol: "ETHUSD".to_string(),
+        executed_quantity: Decimal::from(-60), // Negative for sell
+        execution_price: Decimal::from(3100),
+        execution_time: Utc::now(),
+        commission: Decimal::from(10),
+        liquidity_flag: LiquidityFlag::Maker,
+    };
+    engine.process_execution(sell_exec).await.unwrap();
+
+    let positions = engine.get_positions(Some("ETHUSD".to_string())).await.unwrap();
+    assert_eq!(positions.len(), 1);
+    assert_eq!(positions[0].quantity, Decimal::from(40));
+}
+
+#[tokio::test]
+async fn test_process_execution_flatten_position() {
+    let engine = create_engine();
+
+    // Buy 50
+    let buy_order_id: OrderId = "buy-002".to_string().into();
+    let buy_exec = create_execution(buy_order_id, "SOLUSD", 50, 100);
+    engine.process_execution(buy_exec).await.unwrap();
+
+    // Sell 50 (flatten)
+    let sell_order_id: OrderId = "sell-002".to_string().into();
+    let sell_exec = ExecutionResult {
+        order_id: sell_order_id,
+        symbol: "SOLUSD".to_string(),
+        executed_quantity: Decimal::from(-50),
+        execution_price: Decimal::from(110),
+        execution_time: Utc::now(),
+        commission: Decimal::from(5),
+        liquidity_flag: LiquidityFlag::Taker,
+    };
+    engine.process_execution(sell_exec).await.unwrap();
+
+    let positions = engine.get_positions(Some("SOLUSD".to_string())).await.unwrap();
+    assert_eq!(positions.len(), 1);
+    assert_eq!(positions[0].quantity, Decimal::ZERO);
+
+    // Realized P&L should be 50 * (110 - 100) = 500
+    let expected_pnl = Decimal::from(50) * (Decimal::from(110) - Decimal::from(100));
+    assert_eq!(positions[0].realized_pnl, expected_pnl);
+}
+
+#[tokio::test]
+async fn test_process_execution_for_nonexistent_order() {
+    let engine = create_engine();
+    let fake_order_id: OrderId = "nonexistent".to_string().into();
+
+    let execution = create_execution(fake_order_id, "BTCUSD", 1, 50000);
+    let result = engine.process_execution(execution).await;
+
+    // Should still succeed - position is created even if order not tracked
+    // This is the engine's behavior: process execution regardless
+    assert!(result.is_ok());
+}
+
+// ============================================================================
+// Position Management Tests
+// ============================================================================
+
+#[tokio::test]
+async fn test_get_positions_empty() {
+    let engine = create_engine();
+    let positions = engine.get_positions(None).await.unwrap();
+    assert!(positions.is_empty());
+}
+
+#[tokio::test]
+async fn test_get_positions_multiple_symbols() {
+    let engine = create_engine();
+
+    // Create BTC position
+    let btc_order: OrderId = "btc-001".to_string().into();
+    let btc_exec = create_execution(btc_order, "BTCUSD", 1, 50000);
+    engine.process_execution(btc_exec).await.unwrap();
+
+    // Create ETH position
+    let eth_order: OrderId = "eth-001".to_string().into();
+    let eth_exec = create_execution(eth_order, "ETHUSD", 10, 3000);
+    engine.process_execution(eth_exec).await.unwrap();
+
+    let positions = engine.get_positions(None).await.unwrap();
+    assert_eq!(positions.len(), 2);
+
+    let symbols: Vec<String> = positions.iter().map(|p| p.symbol.to_string()).collect();
+    assert!(symbols.contains(&"BTCUSD".to_string()));
+    assert!(symbols.contains(&"ETHUSD".to_string()));
+}
+
+#[tokio::test]
+async fn test_get_positions_filtered_by_symbol() {
+    let engine = create_engine();
+
+    // Create multiple positions
+    let btc_order: OrderId = "btc-002".to_string().into();
+    let btc_exec = create_execution(btc_order, "BTCUSD", 1, 50000);
+    engine.process_execution(btc_exec).await.unwrap();
+
+    let eth_order: OrderId = "eth-002".to_string().into();
+    let eth_exec = create_execution(eth_order, "ETHUSD", 10, 3000);
+    engine.process_execution(eth_exec).await.unwrap();
+
+    // Filter for BTC only
+    let btc_positions = engine.get_positions(Some("BTCUSD".to_string())).await.unwrap();
+    assert_eq!(btc_positions.len(), 1);
+    assert_eq!(btc_positions[0].symbol.to_string(), "BTCUSD");
+}
+
+// ============================================================================
+// Account Management Tests
+// ============================================================================
+
+#[tokio::test]
+async fn test_get_account_info() {
+    let engine = create_engine();
+    let account = engine.get_account_info("DEMO_ACCOUNT".to_string()).await;
+
+    assert!(account.is_ok());
+    let account_info = account.unwrap();
+    assert_eq!(account_info.account_id, "DEMO_ACCOUNT");
+    assert_eq!(account_info.total_value, Decimal::from(100000));
+    assert_eq!(account_info.cash_balance, Decimal::from(50000));
+    assert_eq!(account_info.buying_power, Decimal::from(100000));
+}
+
+#[tokio::test]
+async fn test_get_account_info_nonexistent() {
+    let engine = create_engine();
+    let result = engine.get_account_info("NONEXISTENT".to_string()).await;
+
+    assert!(result.is_err());
+    assert!(result.unwrap_err().contains("not found"));
+}
+
+// ============================================================================
+// Market Data Subscription Tests
+// ============================================================================
+
+#[tokio::test]
+async fn test_subscribe_market_data() {
+    let engine = create_engine();
+    let result = engine.subscribe_market_data(vec!["BTCUSD".to_string()]).await;
+    assert!(result.is_ok());
+}
+
+#[tokio::test]
+async fn test_subscribe_order_updates() {
+    let engine = create_engine();
+    let result = engine.subscribe_order_updates(None).await;
+    assert!(result.is_ok());
+}
+
+// ============================================================================
+// Trading Stats Tests
+// ============================================================================
+
+#[tokio::test]
+async fn test_get_trading_stats_initial() {
+    let engine = create_engine();
+    let stats = engine.get_trading_stats().await;
+
+    // Initial stats
+    assert_eq!(stats.total_orders, 0);
+    assert_eq!(stats.filled_orders, 0);
+}
+
+// ============================================================================
+// Concurrent Execution Tests
+// ============================================================================
+
+#[tokio::test]
+async fn test_concurrent_executions() {
+    let engine = Arc::new(create_engine());
+
+    let mut handles = vec![];
+
+    for i in 0..10 {
+        let engine_clone = Arc::clone(&engine);
+        let handle = tokio::spawn(async move {
+            let order_id: OrderId = format!("concurrent-{}", i).into();
+            let execution = create_execution(order_id, "BTCUSD", 1, 50000);
+            engine_clone.process_execution(execution).await
+        });
+        handles.push(handle);
+    }
+
+    let results: Vec<_> = futures::future::join_all(handles).await;
+
+    // All should succeed
+    for result in results {
+        assert!(result.unwrap().is_ok());
+    }
+
+    // Verify final position
+    let positions = engine.get_positions(Some("BTCUSD".to_string())).await.unwrap();
+    assert_eq!(positions.len(), 1);
+    assert_eq!(positions[0].quantity, Decimal::from(10));
+}
+
+// ============================================================================
+// Edge Cases
+// ============================================================================
+
+#[tokio::test]
+async fn test_large_execution_quantity() {
+    let engine = create_engine();
+    let order_id: OrderId = "large-001".to_string().into();
+
+    let execution = create_execution(order_id, "BTCUSD", 1_000_000, 50000);
+    let result = engine.process_execution(execution).await;
+
+    assert!(result.is_ok());
+
+    let positions = engine.get_positions(Some("BTCUSD".to_string())).await.unwrap();
+    assert_eq!(positions[0].quantity, Decimal::from(1_000_000));
+}
+
+#[tokio::test]
+async fn test_fractional_execution_quantity() {
+    let engine = create_engine();
+    let order_id: OrderId = "frac-001".to_string().into();
+
+    let execution = ExecutionResult {
+        order_id,
+        symbol: "BTCUSD".to_string(),
+        executed_quantity: Decimal::new(15, 1), // 1.5
+        execution_price: Decimal::from(50000),
+        execution_time: Utc::now(),
+        commission: Decimal::from(10),
+        liquidity_flag: LiquidityFlag::Maker,
+    };
+
+    let result = engine.process_execution(execution).await;
+    assert!(result.is_ok());
+
+    let positions = engine.get_positions(Some("BTCUSD".to_string())).await.unwrap();
+    assert_eq!(positions[0].quantity, Decimal::new(15, 1)); // 1.5
+}
+
+#[tokio::test]
+async fn test_execution_with_high_commission() {
+    let engine = create_engine();
+    let order_id: OrderId = "highfee-001".to_string().into();
+
+    let execution = ExecutionResult {
+        order_id,
+        symbol: "BTCUSD".to_string(),
+        executed_quantity: Decimal::from(1),
+        execution_price: Decimal::from(50000),
+        execution_time: Utc::now(),
+        commission: Decimal::from(5000), // High commission
+        liquidity_flag: LiquidityFlag::Taker,
+    };
+
+    let result = engine.process_execution(execution).await;
+    assert!(result.is_ok());
+
+    let account = engine.get_account_info("DEMO_ACCOUNT".to_string()).await.unwrap();
+    assert_eq!(account.cash_balance, Decimal::from(50000) - Decimal::from(5000));
+}