From e2a0576ef531ec1ad99dcdbd16b207c0935c0866 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sun, 1 Mar 2026 22:50:38 +0100 Subject: [PATCH] docs: create/update README.md for all services, CLI, and testing crates Create 3 missing service READMEs (api_gateway, data_acquisition_service, trading_agent_service). Create bin/fxt/README.md. Create testing/service-integration/README.md. Update existing service and testing READMEs to standard template. Delete 4 subdirectory READMEs from testing/integration/ and testing/e2e/. Co-Authored-By: Claude Opus 4.6 --- bin/fxt/README.md | 18 + services/api_gateway/README.md | 21 + services/backtesting_service/README.md | 53 +- services/broker_gateway_service/README.md | 1263 +-------------- services/data_acquisition_service/README.md | 15 + services/ml_training_service/README.md | 43 +- services/trading_agent_service/README.md | 16 + services/trading_service/README.md | 88 +- testing/api-gateway-load/README.md | 320 +--- testing/e2e/README.md | 403 +---- testing/e2e/integration/README.md | 386 ----- testing/integration/README.md | 1524 +------------------ testing/integration/chaos/README.md | 292 ---- testing/integration/e2e_helpers/README.md | 257 ---- testing/integration/smoke_tests/README.md | 359 ----- testing/load/README.md | 471 +----- testing/service-integration/README.md | 9 + testing/service-load/README.md | 213 +-- testing/test-common/README.md | 288 +--- 19 files changed, 225 insertions(+), 5814 deletions(-) create mode 100644 bin/fxt/README.md create mode 100644 services/api_gateway/README.md create mode 100644 services/data_acquisition_service/README.md create mode 100644 services/trading_agent_service/README.md delete mode 100644 testing/e2e/integration/README.md delete mode 100644 testing/integration/chaos/README.md delete mode 100644 testing/integration/e2e_helpers/README.md delete mode 100644 testing/integration/smoke_tests/README.md create mode 100644 testing/service-integration/README.md diff --git a/bin/fxt/README.md b/bin/fxt/README.md new file mode 100644 index 000000000..d35132eed --- /dev/null +++ b/bin/fxt/README.md @@ -0,0 +1,18 @@ +# fxt + +Foxhunt CLI -- command-line interface for the Foxhunt HFT trading system. + +## Commands + +- `fxt auth` -- authentication and token management +- `fxt trade` -- order submission and management +- `fxt train` -- model training job management +- `fxt tune` -- hyperparameter optimization +- `fxt agent` -- trading agent control +- `fxt model` -- model listing and inspection +- `fxt backtest-ml` -- ML backtesting workflows + +## Configuration + +- `FXT_CONFIG` -- config file path (default: `~/.config/fxt/config.toml`) +- `TRADING_SERVICE_URL` -- gRPC endpoint for trading service diff --git a/services/api_gateway/README.md b/services/api_gateway/README.md new file mode 100644 index 000000000..8e6ad0398 --- /dev/null +++ b/services/api_gateway/README.md @@ -0,0 +1,21 @@ +# api_gateway + +API Gateway service with 6-layer authentication, RBAC, rate limiting, and gRPC proxy routing. + +## Key Types + +- `ApiGatewayService` -- main service implementation +- `RoleBasedAccessControl` -- RBAC authorization +- `RateLimiter` -- 3-tier rate limiting (auth/trading/compute) +- `CircuitBreaker` -- upstream service protection + +## Configuration + +- `JWT_SECRET` -- JWT signing secret (min 32 chars) +- `UPSTREAM_TRADING_URL` -- trading service gRPC endpoint +- `UPSTREAM_ML_URL` -- ML service gRPC endpoint + +## Features + +- `mfa` (default) -- multi-factor authentication support +- Build without: `cargo check -p api_gateway --no-default-features --features minimal` diff --git a/services/backtesting_service/README.md b/services/backtesting_service/README.md index b6ab60e1a..796e10810 100644 --- a/services/backtesting_service/README.md +++ b/services/backtesting_service/README.md @@ -1,49 +1,28 @@ -# Backtesting Service +# backtesting_service -## Overview +Strategy backtesting engine with historical data replay, performance reporting, and results persistence. -The `backtesting_service` offers an independent and isolated environment for rigorously testing and validating trading strategies against historical market data. It provides a robust platform for simulating trading performance, analyzing strategy efficacy, and generating comprehensive performance reports before live deployment. +## Key Types -## Features +- `BacktestingServiceImpl` -- main gRPC service +- `DataReplayEngine` -- historical market data replay +- `PerformanceReporter` -- Sharpe, drawdown, win rate metrics -* **Independent Backtesting Service**: Operates autonomously, allowing for parallel and isolated strategy evaluations. -* **gRPC API for Backtest Execution**: Exposes a clear API for submitting and managing backtesting jobs. -* **Strategy Testing and Validation**: Enables comprehensive testing of various trading strategies under different market conditions. -* **Performance Reporting**: Generates detailed reports including metrics like P&L, Sharpe ratio, drawdown, and win rate. -* **Data Replay Engine**: Accurately replays historical market data, simulating real-world order book dynamics and trade execution. -* **Results Persistence**: Stores backtesting results and reports for historical analysis and comparison. +## gRPC Endpoints -## gRPC API +- `RunBacktest` -- submit backtest configuration and strategy +- `GetBacktestResults` -- retrieve results for completed backtests +- `ListAvailableStrategies` -- list registered strategies +- `GetBacktestReport` -- detailed performance report -The `backtesting_service` exposes a gRPC API for initiating and retrieving backtest results. Key endpoints include: -- `RunBacktest` - Submit backtest configuration and strategy -- `GetBacktestResults` - Retrieve results for completed backtests -- `ListAvailableStrategies` - List registered strategies -- `GetBacktestReport` - Get detailed performance report +## Configuration -## Running the service - -To run the `backtesting_service` binary: - -```bash -cargo run --bin backtesting_service -``` - -## Data Requirements - -The service requires historical market data in Parquet format: -- Data should be stored in the configured data directory -- Supports tick data, order book snapshots, and OHLCV candles -- Data must include instrument, timestamp, and price/quantity fields +- `GRPC_PORT` -- gRPC listen port +- `DATABASE_URL` -- PostgreSQL connection string +- Historical data directory with Parquet tick/OHLCV files ## Testing -To run the tests for the `backtesting_service` crate: - ```bash -cargo test --package backtesting_service +SQLX_OFFLINE=true cargo test -p backtesting_service --lib ``` - -## Documentation - -Comprehensive API documentation is available at [docs.rs/backtesting_service](https://docs.rs/backtesting_service). diff --git a/services/broker_gateway_service/README.md b/services/broker_gateway_service/README.md index 564a2188c..d8a30dfb2 100644 --- a/services/broker_gateway_service/README.md +++ b/services/broker_gateway_service/README.md @@ -1,1253 +1,36 @@ -# Broker Gateway Service +# broker_gateway_service -**Version**: 0.1.0 -**Status**: MVP (Database persistence only, FIX protocol deferred to Phase 2) -**Port**: 50056 (gRPC), 8086 (Health), 9096 (Metrics) +Broker communication gateway with FIX 4.2/4.4 protocol, order routing, execution management, and position reconciliation via CQG. -## Table of Contents +**Status**: MVP (database persistence only, FIX protocol deferred to Phase 2) -- [Overview](#overview) -- [Architecture](#architecture) -- [FIX Protocol Flow](#fix-protocol-flow) -- [Order Lifecycle](#order-lifecycle) -- [Position Reconciliation](#position-reconciliation) -- [Error Handling](#error-handling) -- [Performance Characteristics](#performance-characteristics) -- [Configuration Reference](#configuration-reference) -- [Deployment Guide](#deployment-guide) -- [Development Guide](#development-guide) -- [Testing](#testing) -- [Monitoring](#monitoring) +## Key Types ---- +- `BrokerGatewayService` -- main gRPC service (7 methods) +- `SequenceManager` -- FIX MsgSeqNum tracking +- `OrderStateMachine` -- order lifecycle state transitions +- `PositionReconciler` -- 3-way sync (CQG, database, trading service) +- `CircuitBreaker` -- broker connection protection -## Overview +## gRPC Endpoints -The Broker Gateway Service is a gRPC microservice that handles all broker communication for the Foxhunt HFT Trading System. It provides order routing, execution management, and account state synchronization via the FIX 4.2/4.4 protocol to AMP Futures (CQG broker). +- `RouteOrder` -- submit order to broker +- `CancelOrder` -- cancel existing order +- `GetOrderStatus` -- query order state +- `GetPositions` -- current account positions +- `StreamExecutions` -- real-time execution reports -### Key Features +## Configuration -- **FIX Protocol Integration**: Complete FIX 4.2/4.4 implementation (Phase 2) -- **Order Routing**: Market, Limit, Stop, and Stop-Limit orders -- **Real-time Executions**: Streaming execution reports via gRPC -- **Position Reconciliation**: 3-way sync (CQG, Database, Trading Service) -- **Session Management**: Automatic reconnection, heartbeat monitoring -- **Audit Trail**: Complete order/execution history in PostgreSQL -- **High Performance**: Sub-millisecond order submission latency - -### Current MVP Status - -The MVP version (v0.1.0) implements: -- gRPC API endpoints (7 methods) -- Database persistence for orders -- Request validation and error handling -- Health checks and monitoring - -**Not yet implemented**: -- FIX protocol communication (returns placeholder responses) -- Actual broker connectivity -- Real-time execution streaming - ---- - -## Architecture - -### System Topology - -``` -┌─────────────────────┐ -│ Trading Agent │ -│ Service (50055) │ -└──────────┬──────────┘ - │ gRPC - ▼ -┌─────────────────────┐ -│ Broker Gateway │ -│ Service (50056) │ -│ ┌──────────────┐ │ -│ │ gRPC Server │ │ -│ ├──────────────┤ │ -│ │ FIX Session │ │ (Phase 2) -│ │ Manager │ │ -│ ├──────────────┤ │ -│ │ Order State │ │ -│ │ Machine │ │ -│ ├──────────────┤ │ -│ │ Sequence │ │ -│ │ Manager │ │ -│ └──────────────┘ │ -└──────────┬──────────┘ - │ - ├──────────► PostgreSQL - │ (broker_orders, - │ broker_executions) - │ - ├──────────► Redis - │ (session state, - │ sequence numbers) - │ - └──────────► CQG Broker - (FIX 4.2/4.4) - Phase 2 -``` - -### Component Responsibilities - -| Component | Responsibility | Phase | -|-----------|----------------|-------| -| **gRPC Server** | Handle incoming requests, validate inputs | MVP ✓ | -| **FIX Session Manager** | Maintain FIX session, handle LOGON/LOGOUT | Phase 2 | -| **Order Router** | Route orders to broker via FIX NewOrderSingle | Phase 2 | -| **Execution Handler** | Process ExecutionReport messages | Phase 2 | -| **Sequence Manager** | Track MsgSeqNum, detect gaps | Phase 2 | -| **Position Reconciler** | 3-way position sync | Phase 2 | -| **Health Monitor** | Check database, FIX session health | MVP ✓ | - -### Data Flow - -#### Order Submission Flow (MVP) - -``` -1. Trading Agent → RouteOrder(gRPC) -2. Validate request (symbol, quantity, price) -3. Generate client_order_id (UUID) -4. Save to broker_orders table (status: PENDING_SUBMIT) -5. Return success response -``` - -#### Order Submission Flow (Phase 2) - -``` -1. Trading Agent → RouteOrder(gRPC) -2. Validate request (symbol, quantity, price) -3. Generate client_order_id (UUID) -4. Save to broker_orders table (status: PENDING_SUBMIT) -5. Encode FIX NewOrderSingle message -6. Send to CQG via FIX session -7. Await ExecutionReport (MsgType=8) -8. Update broker_orders (status: SUBMITTED, broker_order_id) -9. Return success response -``` - -#### Execution Report Processing (Phase 2) - -``` -1. Receive FIX ExecutionReport from CQG -2. Validate sequence number, checksum -3. Parse ExecID, OrderID, ClOrdID, ExecType -4. Update broker_orders table -5. Insert into broker_executions table -6. Stream ExecutionEvent to subscribers -7. Notify Trading Service (position update) -``` - ---- - -## FIX Protocol Flow - -### Session Lifecycle - -``` -[DISCONNECTED] - │ - │ TCP Connect - ▼ -[CONNECTED] - │ - │ Send: Logon (MsgType=A, Tag 553=username, Tag 554=password) - ▼ -[LOGGING_IN] - │ - │ Receive: Logon (MsgType=A) - ▼ -[ACTIVE] - │ ◄─── Send/Receive: Heartbeat (MsgType=0) every 30s - │ ◄─── Send: NewOrderSingle (MsgType=D) - │ ◄─── Receive: ExecutionReport (MsgType=8) - │ ◄─── Send: OrderCancelRequest (MsgType=F) - │ - │ Send: Logout (MsgType=5) - ▼ -[LOGGING_OUT] - │ - │ Receive: Logout (MsgType=5) - ▼ -[DISCONNECTED] -``` - -### FIX Message Examples - -#### 1. Logon Message (Client → CQG) - -```rust -// Encode Logon (MsgType=A) -fn encode_logon( - sender_comp_id: &str, - target_comp_id: &str, - username: &str, - password: &str, - seq_num: u64, -) -> String { - format!( - "8=FIX.4.2|9=120|35=A|34={}|49={}|56={}|\ - 98=0|108=30|141=Y|553={}|554={}|10=123|", - seq_num, sender_comp_id, target_comp_id, username, password - ) -} - -// Example usage -let msg = encode_logon("FOXHUNT_CLIENT", "CQG", "myuser", "mypass", 1); -// Output: -// 8=FIX.4.2|9=120|35=A|34=1|49=FOXHUNT_CLIENT|56=CQG|98=0|108=30|141=Y|553=myuser|554=mypass|10=123| -``` - -**FIX Tags Explained**: -- **Tag 8**: BeginString (FIX.4.2) -- **Tag 9**: BodyLength (120 bytes) -- **Tag 35**: MsgType (A = Logon) -- **Tag 34**: MsgSeqNum (1) -- **Tag 49**: SenderCompID (FOXHUNT_CLIENT) -- **Tag 56**: TargetCompID (CQG) -- **Tag 98**: EncryptMethod (0 = None) -- **Tag 108**: HeartBtInt (30 seconds) -- **Tag 141**: ResetSeqNumFlag (Y = Reset to 1) -- **Tag 553**: Username -- **Tag 554**: Password -- **Tag 10**: CheckSum (123) - -#### 2. NewOrderSingle (Market Order) - -```rust -// Encode NewOrderSingle (MsgType=D) -fn encode_new_order_single_market( - client_order_id: &str, - account_id: &str, - symbol: &str, - side: u8, // 1=Buy, 2=Sell - quantity: f64, - seq_num: u64, -) -> String { - format!( - "8=FIX.4.2|9=180|35=D|34={}|49=FOXHUNT_CLIENT|56=CQG|\ - 11={}|1={}|55={}|54={}|38={}|40=1|59=0|21=1|10=234|", - seq_num, client_order_id, account_id, symbol, side, quantity - ) -} - -// Example: Buy 10 ES contracts at market -let msg = encode_new_order_single_market( - "ORDER_123456", - "ACCT_001", - "ES", - 1, // Buy - 10.0, - 2, // Sequence number 2 -); -``` - -**Additional FIX Tags**: -- **Tag 11**: ClOrdID (Client Order ID) -- **Tag 1**: Account -- **Tag 55**: Symbol (ES = E-mini S&P 500) -- **Tag 54**: Side (1=Buy, 2=Sell) -- **Tag 38**: OrderQty (10.0) -- **Tag 40**: OrdType (1=Market, 2=Limit, 3=Stop, 4=StopLimit) -- **Tag 59**: TimeInForce (0=Day, 1=GTC, 3=IOC) -- **Tag 21**: HandlInst (1=Automated) - -#### 3. NewOrderSingle (Limit Order) - -```rust -// Encode NewOrderSingle with Limit Price -fn encode_new_order_single_limit( - client_order_id: &str, - account_id: &str, - symbol: &str, - side: u8, - quantity: f64, - price: f64, - seq_num: u64, -) -> String { - format!( - "8=FIX.4.2|9=200|35=D|34={}|49=FOXHUNT_CLIENT|56=CQG|\ - 11={}|1={}|55={}|54={}|38={}|40=2|44={}|59=0|21=1|10=245|", - seq_num, client_order_id, account_id, symbol, side, quantity, price - ) -} - -// Example: Sell 5 NQ contracts at 18500.50 -let msg = encode_new_order_single_limit( - "ORDER_789012", - "ACCT_001", - "NQ", - 2, // Sell - 5.0, - 18500.50, - 3, -); -``` - -**Additional Tags**: -- **Tag 44**: Price (18500.50) - -#### 4. ExecutionReport (Fill) - -```rust -// Parse ExecutionReport (received from CQG) -fn parse_execution_report(msg: &str) -> ExecutionReport { - ExecutionReport { - execution_id: parse_tag(msg, 17), // ExecID - broker_order_id: parse_tag(msg, 37), // OrderID - client_order_id: parse_tag(msg, 11), // ClOrdID - exec_type: parse_tag(msg, 150), // ExecType (F=Fill) - order_status: parse_tag(msg, 39), // OrdStatus (2=Filled) - last_qty: parse_tag_f64(msg, 32), // LastQty (10.0) - last_price: parse_tag_f64(msg, 31), // LastPx (5800.25) - cum_qty: parse_tag_f64(msg, 14), // CumQty (10.0) - avg_price: parse_tag_f64(msg, 6), // AvgPx (5800.25) - transact_time: parse_tag_time(msg, 60), // TransactTime - } -} - -// Example ExecutionReport message: -// 8=FIX.4.2|9=250|35=8|34=10|49=CQG|56=FOXHUNT_CLIENT| -// 37=BROKER_123|11=ORDER_123456|17=EXEC_789|150=F|39=2| -// 55=ES|54=1|38=10|32=10|31=5800.25|14=10|6=5800.25| -// 60=20250109-14:30:00.000|10=234| -``` - -**Execution Report Tags**: -- **Tag 17**: ExecID (Execution ID) -- **Tag 37**: OrderID (Broker-assigned) -- **Tag 11**: ClOrdID (Our Order ID) -- **Tag 150**: ExecType (0=New, F=Fill, 4=Canceled, 8=Rejected) -- **Tag 39**: OrdStatus (0=New, 1=PartiallyFilled, 2=Filled, 4=Canceled, 8=Rejected) -- **Tag 32**: LastQty (Quantity filled in this report) -- **Tag 31**: LastPx (Fill price) -- **Tag 14**: CumQty (Total filled quantity) -- **Tag 6**: AvgPx (Average fill price) -- **Tag 60**: TransactTime (Execution timestamp) - -#### 5. OrderCancelRequest - -```rust -// Encode OrderCancelRequest (MsgType=F) -fn encode_order_cancel_request( - new_client_order_id: &str, - original_client_order_id: &str, - broker_order_id: &str, - symbol: &str, - side: u8, - seq_num: u64, -) -> String { - format!( - "8=FIX.4.2|9=150|35=F|34={}|49=FOXHUNT_CLIENT|56=CQG|\ - 11={}|37={}|41={}|55={}|54={}|60={}|10=089|", - seq_num, - new_client_order_id, - broker_order_id, - original_client_order_id, - symbol, - side, - chrono::Utc::now().format("%Y%m%d-%H:%M:%S.%3f") - ) -} - -// Example: Cancel ORDER_123456 -let msg = encode_order_cancel_request( - "CANCEL_123456", // New ClOrdID for cancel request - "ORDER_123456", // Original ClOrdID - "BROKER_123", // Broker's OrderID - "ES", - 1, // Buy - 4, -); -``` - -**Cancel Request Tags**: -- **Tag 11**: ClOrdID (New ID for cancel request) -- **Tag 37**: OrderID (Broker's order ID) -- **Tag 41**: OrigClOrdID (Original client order ID) -- **Tag 60**: TransactTime - -#### 6. Heartbeat - -```rust -// Encode Heartbeat (MsgType=0) -fn encode_heartbeat(seq_num: u64, test_req_id: Option<&str>) -> String { - let test_field = test_req_id - .map(|id| format!("|112={}", id)) - .unwrap_or_default(); - - format!( - "8=FIX.4.2|9=60|35=0|34={}|49=FOXHUNT_CLIENT|56=CQG{}|10=089|", - seq_num, test_field - ) -} - -// Example: Standard heartbeat -let msg = encode_heartbeat(10, None); - -// Example: Response to TestRequest -let msg = encode_heartbeat(11, Some("TEST_123")); -``` - -### Sequence Number Management - -FIX protocol requires strict sequence number tracking: - -```rust -use std::sync::atomic::{AtomicU64, Ordering}; - -struct SequenceManager { - sender_seq: Arc, // Our outgoing sequence - target_seq: Arc, // Expected incoming sequence -} - -impl SequenceManager { - fn new() -> Self { - Self { - sender_seq: Arc::new(AtomicU64::new(1)), - target_seq: Arc::new(AtomicU64::new(1)), - } - } - - // Get next outgoing sequence number (atomic) - fn next_sender_seq(&self) -> u64 { - self.sender_seq.fetch_add(1, Ordering::SeqCst) - } - - // Validate incoming sequence number - fn validate_target_seq(&self, received: u64) -> Result<(), String> { - let expected = self.target_seq.load(Ordering::SeqCst); - - if received == expected { - self.target_seq.fetch_add(1, Ordering::SeqCst); - Ok(()) - } else if received < expected { - Err(format!( - "Sequence too low: received {}, expected {}", - received, expected - )) - } else { - // Gap detected - send ResendRequest (MsgType=2) - Err(format!( - "Sequence gap: received {}, expected {}", - received, expected - )) - } - } - - // Reset sequences (Logon with ResetSeqNumFlag=Y) - fn reset(&self) { - self.sender_seq.store(1, Ordering::SeqCst); - self.target_seq.store(1, Ordering::SeqCst); - } - - // Persist to database (recovery on reconnect) - async fn persist(&self, db: &PgPool) -> Result<()> { - sqlx::query!( - "UPDATE fix_sessions SET sender_seq = $1, target_seq = $2 WHERE session_id = $3", - self.sender_seq.load(Ordering::SeqCst) as i64, - self.target_seq.load(Ordering::SeqCst) as i64, - "FOXHUNT-CQG" - ) - .execute(db) - .await?; - Ok(()) - } -} -``` - ---- - -## Order Lifecycle - -### Order States - -``` -PENDING_SUBMIT → SUBMITTED → PARTIALLY_FILLED → FILLED - ↓ ↓ - REJECTED CANCEL_PENDING → CANCELLED -``` - -### State Descriptions - -| State | Description | FIX Trigger | -|-------|-------------|-------------| -| **PENDING_SUBMIT** | Order created, awaiting FIX send | - | -| **SUBMITTED** | Order sent to broker, awaiting fill | ExecutionReport (ExecType=New, OrdStatus=New) | -| **PARTIALLY_FILLED** | Partial fill received | ExecutionReport (ExecType=Fill, OrdStatus=PartiallyFilled) | -| **FILLED** | Order fully filled | ExecutionReport (ExecType=Fill, OrdStatus=Filled) | -| **CANCEL_PENDING** | Cancel request sent | OrderCancelRequest sent | -| **CANCELLED** | Order cancelled | ExecutionReport (ExecType=Canceled, OrdStatus=Canceled) | -| **REJECTED** | Order rejected by broker | ExecutionReport (ExecType=Rejected, OrdStatus=Rejected) | - -### State Transitions - -```rust -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum OrderStatus { - PendingSubmit, - Submitted, - PartiallyFilled, - Filled, - CancelPending, - Cancelled, - Rejected, -} - -impl OrderStatus { - fn can_transition_to(&self, new_status: OrderStatus) -> bool { - use OrderStatus::*; - matches!( - (self, new_status), - (PendingSubmit, Submitted) - | (Submitted, PartiallyFilled) - | (Submitted, Filled) - | (Submitted, Cancelled) - | (Submitted, Rejected) - | (Submitted, CancelPending) - | (PartiallyFilled, Filled) - | (PartiallyFilled, Cancelled) - | (PartiallyFilled, CancelPending) - | (CancelPending, Cancelled) - ) - } -} - -// Example: Validate state transition -let current_status = OrderStatus::Submitted; -if current_status.can_transition_to(OrderStatus::PartiallyFilled) { - // Update order status - update_order_status(&order_id, OrderStatus::PartiallyFilled).await?; -} else { - return Err(anyhow!("Invalid state transition")); -} -``` - -### Order Flow Example - -```rust -// 1. Trading Agent submits order -let request = RouteOrderRequest { - symbol: "ES".to_string(), - side: OrderSide::Buy as i32, - quantity: 10.0, - order_type: OrderType::Market as i32, - price: None, - stop_price: None, - account_id: "ACCT_001".to_string(), - metadata: HashMap::new(), -}; - -let response = broker_client.route_order(request).await?; -println!("Order submitted: {}", response.client_order_id); -// Output: Order submitted: 550e8400-e29b-41d4-a716-446655440000 - -// 2. Database record created -// broker_orders table: -// | client_order_id | status | symbol | side | quantity | -// |-----------------|----------------|--------|------|----------| -// | 550e8400-... | PENDING_SUBMIT | ES | BUY | 10.0 | - -// 3. FIX NewOrderSingle sent (Phase 2) -// 8=FIX.4.2|35=D|11=550e8400-...|1=ACCT_001|55=ES|54=1|38=10|40=1|... - -// 4. ExecutionReport received (NEW) -// 8=FIX.4.2|35=8|37=BROKER_123|11=550e8400-...|150=0|39=0|... -// Update: status = SUBMITTED, broker_order_id = BROKER_123 - -// 5. ExecutionReport received (FILL) -// 8=FIX.4.2|35=8|37=BROKER_123|11=550e8400-...|150=F|39=2|32=10|31=5800.25|... -// Update: status = FILLED, filled_quantity = 10, avg_fill_price = 5800.25 - -// 6. Stream execution to Trading Agent -// ExecutionEvent { -// execution_id: "EXEC_789", -// broker_order_id: "BROKER_123", -// client_order_id: "550e8400-...", -// symbol: "ES", -// side: BUY, -// exec_type: TRADE, -// order_status: FILLED, -// last_qty: 10.0, -// last_price: 5800.25, -// cum_qty: 10.0, -// avg_price: 5800.25, -// } -``` - ---- - -## Position Reconciliation - -Position reconciliation ensures consistency across three systems: -1. **CQG Broker** (source of truth) -2. **Broker Gateway Database** (audit trail) -3. **Trading Service** (application state) - -### 3-Way Sync Logic - -```rust -struct PositionReconciler { - db_pool: PgPool, - redis_client: redis::Client, - broker_client: CqgClient, // Phase 2 -} - -impl PositionReconciler { - // Run reconciliation every 60 seconds - async fn reconcile_positions(&self, account_id: &str) -> Result { - // 1. Fetch positions from CQG broker (Phase 2) - let broker_positions = self.fetch_broker_positions(account_id).await?; - - // 2. Fetch positions from database (calculated from executions) - let db_positions = self.fetch_db_positions(account_id).await?; - - // 3. Fetch positions from Trading Service - let trading_positions = self.fetch_trading_positions(account_id).await?; - - // 4. Compare and identify discrepancies - let mut discrepancies = Vec::new(); - - for symbol in self.get_all_symbols(&broker_positions, &db_positions, &trading_positions) { - let broker_qty = broker_positions.get(&symbol).copied().unwrap_or(0.0); - let db_qty = db_positions.get(&symbol).copied().unwrap_or(0.0); - let trading_qty = trading_positions.get(&symbol).copied().unwrap_or(0.0); - - if (broker_qty - db_qty).abs() > 0.01 - || (broker_qty - trading_qty).abs() > 0.01 - { - discrepancies.push(PositionDiscrepancy { - symbol: symbol.clone(), - broker_qty, - db_qty, - trading_qty, - delta_broker_db: broker_qty - db_qty, - delta_broker_trading: broker_qty - trading_qty, - }); - } - } - - // 5. If discrepancies found, trigger reconciliation - if !discrepancies.is_empty() { - warn!( - "Position discrepancies detected for account {}: {} symbols", - account_id, - discrepancies.len() - ); - - for disc in &discrepancies { - // Update database to match broker (source of truth) - self.update_db_position(account_id, &disc.symbol, disc.broker_qty).await?; - - // Notify Trading Service - self.notify_trading_service(account_id, &disc.symbol, disc.broker_qty).await?; - - // Log discrepancy - self.log_reconciliation(account_id, disc).await?; - } - } - - Ok(ReconciliationReport { - account_id: account_id.to_string(), - timestamp: chrono::Utc::now(), - discrepancies, - total_symbols_checked: broker_positions.len() + db_positions.len() + trading_positions.len(), - }) - } - - // Fetch positions from broker via FIX RequestForPositions (Phase 2) - async fn fetch_broker_positions(&self, account_id: &str) -> Result> { - // Send FIX RequestForPositions (MsgType=AN) - // Receive PositionReport (MsgType=AP) - // Parse Tag 55 (Symbol) and Tag 702 (PosQty) - Ok(HashMap::new()) // Placeholder - } - - // Calculate positions from broker_executions table - async fn fetch_db_positions(&self, account_id: &str) -> Result> { - let rows = sqlx::query!( - r#" - SELECT - symbol, - SUM(CASE WHEN side = 'BUY' THEN last_qty ELSE -last_qty END) as net_qty - FROM broker_executions - WHERE account_id = $1 AND exec_type = 'TRADE' - GROUP BY symbol - "#, - account_id - ) - .fetch_all(&self.db_pool) - .await?; - - Ok(rows - .into_iter() - .map(|r| (r.symbol, r.net_qty.unwrap_or(0.0))) - .collect()) - } -} -``` - -### Reconciliation Scenarios - -| Scenario | Broker | DB | Trading | Action | -|----------|--------|----|---------| -------| -| **Normal** | 10 | 10 | 10 | None | -| **Missed Fill** | 10 | 0 | 0 | Insert phantom execution into DB, notify Trading Service | -| **Duplicate Execution** | 10 | 20 | 20 | Delete duplicate from DB, notify Trading Service | -| **Trading Service Desync** | 10 | 10 | 5 | Notify Trading Service to sync to 10 | - ---- - -## Error Handling - -### Error Categories - -| Category | Severity | Retry Strategy | Example | -|----------|----------|----------------|---------| -| **Validation** | LOW | No retry | Invalid symbol, negative quantity | -| **Network** | MEDIUM | Exponential backoff | TCP disconnect, timeout | -| **Broker Rejection** | MEDIUM | No retry | Insufficient margin, symbol not found | -| **Sequence Gap** | HIGH | Resend request | MsgSeqNum gap detected | -| **Session Failure** | CRITICAL | Reconnect + restore | Logon rejected, heartbeat timeout | - -### Retry Strategy - -```rust -async fn send_order_with_retry( - order: &NewOrderSingle, - max_retries: u32, -) -> Result { - let mut attempt = 0; - let mut delay = Duration::from_millis(100); - - loop { - match send_fix_message(order).await { - Ok(report) => return Ok(report), - Err(e) if is_retryable(&e) && attempt < max_retries => { - warn!( - "Order submission failed (attempt {}/{}): {}. Retrying in {:?}", - attempt + 1, - max_retries, - e, - delay - ); - - tokio::time::sleep(delay).await; - attempt += 1; - delay = Duration::min(delay * 2, Duration::from_secs(10)); // Cap at 10s - } - Err(e) => return Err(e), - } - } -} - -fn is_retryable(error: &anyhow::Error) -> bool { - // Retry on network errors, timeout, temporary broker unavailability - error.to_string().contains("timeout") - || error.to_string().contains("connection reset") - || error.to_string().contains("EAGAIN") -} -``` - -### Circuit Breaker - -```rust -struct CircuitBreaker { - failure_threshold: u32, - timeout: Duration, - state: Arc>, -} - -enum CircuitState { - Closed, - Open { until: Instant }, - HalfOpen, -} - -impl CircuitBreaker { - async fn call(&self, f: F) -> Result - where - F: Future>, - { - let state = self.state.read().await.clone(); - - match state { - CircuitState::Open { until } => { - if Instant::now() < until { - return Err(anyhow!("Circuit breaker OPEN - rejecting request")); - } - - // Transition to HalfOpen - *self.state.write().await = CircuitState::HalfOpen; - } - CircuitState::HalfOpen => { - // Allow one probe request - } - CircuitState::Closed => { - // Normal operation - } - } - - match f.await { - Ok(result) => { - // Success - close circuit - *self.state.write().await = CircuitState::Closed; - Ok(result) - } - Err(e) => { - // Failure - open circuit - *self.state.write().await = CircuitState::Open { - until: Instant::now() + self.timeout, - }; - Err(e) - } - } - } -} -``` - -### Fallback Strategies - -```rust -// Strategy 1: Graceful degradation (return cached data) -async fn get_account_state_with_fallback( - account_id: &str, -) -> Result { - match fetch_account_state_from_broker(account_id).await { - Ok(state) => { - // Cache for fallback - cache_account_state(account_id, &state).await?; - Ok(state) - } - Err(e) => { - warn!( - "Failed to fetch account state from broker: {}. Using cached data.", - e - ); - get_cached_account_state(account_id).await - } - } -} - -// Strategy 2: Dead letter queue (DLQ) -async fn handle_failed_order(order: &RouteOrderRequest) -> Result<()> { - // Persist to dead_letter_orders table - sqlx::query!( - "INSERT INTO dead_letter_orders (order_data, error, created_at) VALUES ($1, $2, NOW())", - serde_json::to_value(order)?, - "FIX session unavailable" - ) - .execute(&db_pool) - .await?; - - // Alert operations team - send_alert("Order failed - queued in DLQ").await?; - - Ok(()) -} -``` - ---- - -## Performance Characteristics - -### Latency Targets - -| Operation | Target | P50 | P99 | P99.9 | -|-----------|--------|-----|-----|-------| -| RouteOrder (MVP) | <1ms | 0.4ms | 0.8ms | 1.2ms | -| RouteOrder (FIX) | <5ms | 2.1ms | 4.5ms | 8.0ms | -| CancelOrder | <5ms | 1.8ms | 4.2ms | 7.5ms | -| GetPositions | <10ms | 3.5ms | 9.0ms | 15ms | -| StreamExecutions | <100ms | 45ms | 95ms | 150ms | -| FIX Heartbeat RTT | <50ms | 20ms | 45ms | 80ms | - -### Throughput - -- **Orders/sec**: 1,000-5,000 (limited by FIX session) -- **Executions/sec**: 500-2,000 (streaming) -- **Heartbeats**: 1 every 30s (FIX requirement) - -### Resource Usage - -```rust -// Memory footprint (approximate) -// - gRPC server: 50-100 MB -// - FIX session: 20-50 MB (message buffers, sequence tracking) -// - Database pool: 10-20 MB (10 connections) -// - Redis cache: 5-10 MB -// Total: ~100-200 MB -``` - -### Optimization Techniques - -```rust -// 1. Connection pooling (database) -let pool = PgPoolOptions::new() - .max_connections(10) - .acquire_timeout(Duration::from_secs(5)) - .idle_timeout(Some(Duration::from_secs(600))) - .connect(&database_url) - .await?; - -// 2. Message batching (FIX) -async fn send_batch_orders(orders: Vec) -> Result> { - let mut client_order_ids = Vec::new(); - - for order in orders { - let msg = encode_new_order_single(&order); - send_fix_message(&msg).await?; - client_order_ids.push(order.client_order_id.clone()); - } - - Ok(client_order_ids) -} - -// 3. Async execution report handling -tokio::spawn(async move { - while let Some(report) = execution_rx.recv().await { - process_execution_report(report).await; - } -}); -``` - ---- - -## Configuration Reference - -### Environment Variables - -```bash -# Service ports -GRPC_PORT=50056 # gRPC server port -HEALTH_PORT=8086 # HTTP health check port -METRICS_PORT=9096 # Prometheus metrics port - -# Database -DATABASE_URL=postgresql://foxhunt:foxhunt_dev_password@localhost:5432/foxhunt - -# Redis -REDIS_URL=redis://localhost:6379 - -# FIX session (Phase 2) -FIX_SENDER_COMP_ID=FOXHUNT_CLIENT -FIX_TARGET_COMP_ID=CQG -FIX_USERNAME=your_cqg_username -FIX_PASSWORD=your_cqg_password -FIX_HOST=fix.cqg.com -FIX_PORT=xxxx # Provided by CQG -FIX_HEARTBEAT_INTERVAL=30 # Seconds -FIX_RECONNECT_DELAY=5 # Seconds -FIX_MAX_RETRIES=3 - -# Logging -RUST_LOG=info,broker_gateway_service=debug -LOG_FORMAT=json # json or text -``` - -### Configuration File (config.toml) - -```toml -[service] -grpc_port = 50056 -health_port = 8086 -metrics_port = 9096 - -[database] -url = "postgresql://foxhunt:foxhunt_dev_password@localhost:5432/foxhunt" -max_connections = 10 -acquire_timeout_secs = 5 -idle_timeout_secs = 600 - -[redis] -url = "redis://localhost:6379" -pool_size = 10 - -[fix] -sender_comp_id = "FOXHUNT_CLIENT" -target_comp_id = "CQG" -host = "fix.cqg.com" -port = xxxx -heartbeat_interval_secs = 30 -reconnect_delay_secs = 5 -max_retries = 3 - -[performance] -order_queue_size = 1000 -execution_stream_buffer = 100 -``` - ---- - -## Deployment Guide - -See [DEPLOYMENT.md](docs/DEPLOYMENT.md) for detailed deployment instructions including: -- Docker deployment -- Kubernetes deployment -- Production readiness checklist -- Rollback procedures - -### Quick Start (Development) - -```bash -# 1. Start dependencies -docker-compose up -d postgres redis - -# 2. Run database migrations -cargo sqlx migrate run - -# 3. Build service -cargo build --release -p broker_gateway_service - -# 4. Run service -export DATABASE_URL=postgresql://foxhunt:foxhunt_dev_password@localhost:5432/foxhunt -export REDIS_URL=redis://localhost:6379 -./target/release/broker_gateway_service - -# 5. Verify health -curl http://localhost:8086/health -``` - ---- - -## Development Guide - -### Project Structure - -``` -broker_gateway_service/ -├── Cargo.toml -├── build.rs # Protobuf compilation -├── proto/ -│ └── broker_gateway.proto # gRPC service definition -├── src/ -│ ├── main.rs # Service entry point -│ ├── lib.rs # Library exports -│ └── service.rs # gRPC implementation -├── tests/ -│ ├── unit_tests.rs # Unit tests (47 tests) -│ ├── integration_tests.rs # Integration tests -│ └── mock_fix_server.rs # Mock FIX server -├── benches/ -│ └── benchmarks.rs # Performance benchmarks -└── docs/ - ├── API.md # API reference - ├── TROUBLESHOOTING.md # Troubleshooting guide - └── DEPLOYMENT.md # Deployment guide -``` - -### Adding a New Order Type - -```rust -// 1. Add enum variant to proto/broker_gateway.proto -enum OrderType { - ORDER_TYPE_ICEBERG = 5; // New type -} - -// 2. Update validation in src/service.rs -fn validate_order(&self, req: &RouteOrderRequest) -> Result<(), Status> { - match OrderType::try_from(req.order_type) { - Ok(OrderType::Iceberg) => { - if req.iceberg_qty.is_none() { - return Err(Status::invalid_argument( - "Iceberg quantity is required for ICEBERG orders" - )); - } - } - // ... - } -} - -// 3. Implement FIX encoding -fn encode_iceberg_order(order: &IcebergOrder) -> String { - // Add Tag 111 (MaxFloor) for iceberg orders - format!( - "8=FIX.4.2|35=D|11={}|55={}|54={}|38={}|40=2|44={}|111={}|...", - order.client_order_id, - order.symbol, - order.side, - order.total_qty, - order.price, - order.display_qty - ) -} - -// 4. Add tests -#[test] -fn test_iceberg_order_validation() { - let req = RouteOrderRequest { - order_type: OrderType::Iceberg as i32, - iceberg_qty: None, // Missing required field - // ... - }; - - let result = service.validate_order(&req); - assert!(result.is_err()); -} -``` - ---- +- `GRPC_PORT` (50056) -- gRPC listen port +- `HEALTH_PORT` (8086) -- HTTP health check +- `METRICS_PORT` (9096) -- Prometheus metrics +- `DATABASE_URL` -- PostgreSQL (broker_orders, broker_executions tables) +- `REDIS_URL` -- session state and sequence numbers +- `FIX_*` -- FIX session parameters (Phase 2) ## Testing -### Running Tests - ```bash -# Run all tests -cargo test -p broker_gateway_service - -# Run unit tests only -cargo test -p broker_gateway_service unit_tests - -# Run integration tests (requires database) -docker-compose up -d postgres redis -cargo test -p broker_gateway_service integration_tests - -# Run with coverage -cargo tarpaulin -p broker_gateway_service --out Html +SQLX_OFFLINE=true cargo test -p broker_gateway_service --lib ``` - -### Test Coverage - -| Module | Tests | Coverage | -|--------|-------|----------| -| FIX Encoder | 12 | 95% | -| FIX Decoder | 15 | 92% | -| Sequence Manager | 8 | 100% | -| Order State Machine | 10 | 100% | -| Session Manager | 12 | 88% | -| **Total** | **57** | **94%** | - -### Example Integration Test - -```rust -#[tokio::test] -async fn test_route_order_end_to_end() { - // 1. Setup test database - let db_pool = setup_test_db().await; - - // 2. Create service - let service = BrokerGatewayService::new(db_pool.clone(), "redis://localhost:6379")?; - - // 3. Submit order - let request = RouteOrderRequest { - symbol: "ES".to_string(), - side: OrderSide::Buy as i32, - quantity: 10.0, - order_type: OrderType::Market as i32, - price: None, - stop_price: None, - account_id: "TEST_ACCT".to_string(), - metadata: HashMap::new(), - }; - - let response = service.route_order(Request::new(request)).await?; - let response = response.into_inner(); - - // 4. Verify response - assert_eq!(response.status, OrderStatus::PendingSubmit as i32); - assert!(!response.client_order_id.is_empty()); - - // 5. Verify database record - let order = sqlx::query!( - "SELECT * FROM broker_orders WHERE client_order_id = $1", - response.client_order_id - ) - .fetch_one(&db_pool) - .await?; - - assert_eq!(order.symbol, "ES"); - assert_eq!(order.side, "BUY"); - assert_eq!(order.quantity.unwrap().to_string(), "10"); - assert_eq!(order.status, "PENDING_SUBMIT"); -} -``` - ---- - -## Monitoring - -### Prometheus Metrics - -```rust -// Counter: Total orders submitted -broker_orders_total{symbol="ES", side="BUY", status="SUBMITTED"} 1523 - -// Counter: Total executions received -broker_executions_total{symbol="ES", exec_type="FILL"} 1401 - -// Gauge: FIX session state (0=Disconnected, 3=Active) -fix_session_state{session_id="FOXHUNT-CQG"} 3 - -// Gauge: Current sequence numbers -fix_sender_seq_num{session_id="FOXHUNT-CQG"} 5432 -fix_target_seq_num{session_id="FOXHUNT-CQG"} 5398 - -// Histogram: Order submission latency (seconds) -broker_order_submit_latency_seconds_bucket{le="0.001"} 850 -broker_order_submit_latency_seconds_bucket{le="0.005"} 1450 -broker_order_submit_latency_seconds_bucket{le="0.010"} 1500 - -// Histogram: FIX heartbeat RTT (seconds) -fix_heartbeat_rtt_seconds_bucket{le="0.050"} 980 -fix_heartbeat_rtt_seconds_bucket{le="0.100"} 1000 -``` - -### Grafana Dashboard - -Key panels: -1. **Order Flow**: Orders/sec, Executions/sec -2. **FIX Session Health**: Session state, heartbeat RTT, sequence numbers -3. **Latency**: P50/P99/P99.9 for RouteOrder, CancelOrder -4. **Error Rate**: Rejections, timeouts, sequence gaps -5. **Position Discrepancies**: Reconciliation alerts - -### Alerts - -```yaml -# Alert: FIX session down -- alert: FixSessionDown - expr: fix_session_state{session_id="FOXHUNT-CQG"} != 3 - for: 1m - annotations: - summary: FIX session not active - -# Alert: High order rejection rate -- alert: HighOrderRejectionRate - expr: rate(broker_orders_total{status="REJECTED"}[5m]) > 0.1 - for: 5m - annotations: - summary: Order rejection rate > 10% - -# Alert: Position discrepancy detected -- alert: PositionDiscrepancy - expr: position_reconciliation_discrepancies > 0 - for: 1m - annotations: - summary: Position mismatch between broker and database -``` - ---- - -## API Reference - -See [API.md](docs/API.md) for detailed API documentation including: -- gRPC method signatures -- Request/response examples -- Error codes -- Rate limits - ---- - -## Troubleshooting - -See [TROUBLESHOOTING.md](docs/TROUBLESHOOTING.md) for common issues and solutions. - ---- - -## License - -Proprietary - Foxhunt HFT Trading System diff --git a/services/data_acquisition_service/README.md b/services/data_acquisition_service/README.md new file mode 100644 index 000000000..cd92dbaec --- /dev/null +++ b/services/data_acquisition_service/README.md @@ -0,0 +1,15 @@ +# data_acquisition_service + +Automated Databento data downloading, validation, and S3 upload for HFT trading. + +## Key Types + +- `DataAcquisitionServiceImpl` -- main gRPC service +- `Downloader` -- Databento API client +- `Uploader` -- S3 artifact upload +- `Validator` -- data quality checks + +## Configuration + +- `DATABENTO_API_KEY` -- Databento API credentials +- `S3_BUCKET` -- target storage bucket diff --git a/services/ml_training_service/README.md b/services/ml_training_service/README.md index 464647cfa..97a96c23d 100644 --- a/services/ml_training_service/README.md +++ b/services/ml_training_service/README.md @@ -1,48 +1,27 @@ # ml_training_service -Model training orchestration and lifecycle management for the Foxhunt HFT trading system. Manages training jobs for DQN, PPO, TFT, Mamba2, TLOB, and Liquid models with progress tracking, resource allocation, and model artifact storage. +Model training orchestration and lifecycle management for DQN, PPO, TFT, Mamba2, TLOB, and Liquid models with progress tracking and artifact storage. -## Building +## Key Types -```bash -# Default (minimal features) -cargo build --release -p ml_training_service - -# With GPU acceleration (requires CUDA) -cargo build --release -p ml_training_service --features gpu - -# With mock training data (testing only, bypasses database) -cargo build --release -p ml_training_service --features mock-data -``` +- `MlTrainingServiceImpl` -- main gRPC service +- `JobTracker` -- training job state machine +- `CheckpointManager` -- model artifact persistence ## Features -| Feature | Default | Description | -|-------------|---------|--------------------------------------------------| -| `minimal` | Yes | Minimal ML feature set for financial models | -| `gpu` | No | SIMD GPU acceleration (requires CUDA) | -| `debug` | No | Additional debug logging | -| `mock-data` | No | Use mock training data instead of PostgreSQL | +- `minimal` (default) -- minimal ML feature set for financial models +- `gpu` -- SIMD GPU acceleration (requires CUDA) +- `mock-data` -- mock training data (testing, bypasses database) ## Configuration -The gRPC listen port is set via the `GRPC_PORT` environment variable. Prometheus metrics are exposed on port 9094. - -PostgreSQL (via sqlx) is used for job metadata, training history, and state management. Set the connection string with `DATABASE_URL`. - -## Running - -```bash -GRPC_PORT=50053 DATABASE_URL="postgresql://user:pass@localhost:5432/foxhunt_training" \ - ./target/release/ml_training_service serve -``` +- `GRPC_PORT` -- gRPC listen port +- `DATABASE_URL` -- PostgreSQL for job metadata and training history +- Prometheus metrics on port 9094 ## Testing ```bash -# Unit tests (offline, no database required) SQLX_OFFLINE=true cargo test -p ml_training_service --lib - -# Integration tests (requires running PostgreSQL) -cargo test -p ml_training_service ``` diff --git a/services/trading_agent_service/README.md b/services/trading_agent_service/README.md new file mode 100644 index 000000000..26b26fb16 --- /dev/null +++ b/services/trading_agent_service/README.md @@ -0,0 +1,16 @@ +# trading_agent_service + +Portfolio management with autonomous universe selection, asset allocation, and order generation. + +## Key Types + +- `TradingAgentServiceImpl` -- main gRPC service +- `AutonomousUniverseManager` -- dynamic universe scaling +- `AssetSelector` -- instrument selection +- `Allocator` -- position sizing and allocation +- `OrderGenerator` -- order creation from signals + +## Configuration + +- `ML_SERVICE_URL` -- ensemble ML service endpoint +- `TRADING_SERVICE_URL` -- order execution endpoint diff --git a/services/trading_service/README.md b/services/trading_service/README.md index c53f76d17..7197ba5e4 100644 --- a/services/trading_service/README.md +++ b/services/trading_service/README.md @@ -1,82 +1,32 @@ -# Trading Service +# trading_service -## Overview +Core execution engine for order placement, position keeping, risk integration, and real-time P&L tracking. -The `trading_service` is the core execution engine for the Foxhunt HFT platform. It manages the entire lifecycle of trading operations, from order placement and execution to real-time position keeping and risk management. This service is critical for high-frequency, low-latency trading activities, ensuring compliance and optimal performance. +## Key Types -## Features +- `TradingServiceImpl` -- main gRPC service +- `OrderExecutor` -- high-throughput order handling +- `PositionManager` -- real-time position and P&L tracking +- `RiskIntegration` -- pre-trade and post-trade compliance -* **Order Execution**: Handles high-throughput order placement, modification, and cancellation across various exchanges. -* **Position Management**: Maintains real-time tracking of all open positions, including P&L calculations and exposure. -* **Risk Integration**: Integrates with upstream risk systems to enforce pre-trade and post-trade compliance checks. -* **Compliance Checks**: Automatically applies regulatory and internal compliance rules to all trading activities. -* **Market Data Subscriptions**: Subscribes to and processes real-time market data feeds for informed decision-making. -* **Real-time P&L Tracking**: Provides immediate profit and loss updates for active strategies and overall portfolio. -* **Health Checks and Metrics**: Exposes endpoints for monitoring service health and operational metrics. +## gRPC Endpoints -## gRPC API - -The `trading_service` exposes a gRPC API for interacting with its core functionalities. Key endpoints include: -- `PlaceOrder` - Submit new orders -- `CancelOrder` - Cancel existing orders -- `GetPosition` - Query current positions -- `SubscribeMarketData` - Subscribe to market data feeds -- `GetPnlUpdates` - Retrieve real-time P&L updates - -## Running the service - -To run the `trading_service` binary: - -```bash -cargo run --bin trading_service -``` +- `PlaceOrder` -- submit new orders +- `CancelOrder` -- cancel existing orders +- `GetPosition` -- query current positions +- `SubscribeMarketData` -- market data feeds +- `GetPnlUpdates` -- real-time P&L updates ## Configuration -The service is configured via the central `config` crate with PostgreSQL backend. Key configuration includes: -- Database connection strings -- Risk parameters and limits -- Broker connection settings -- gRPC server port and TLS settings - -### TLS Configuration (Agent S3) - -The Trading Service supports TLS 1.3 with optional mutual TLS (mTLS) for secure gRPC communications. - -**Environment Variables**: -```bash -TLS_ENABLED=false # Enable TLS (default: false) -TLS_CERT_PATH=/app/certs/trading_service/server.crt # Server certificate -TLS_KEY_PATH=/app/certs/trading_service/server.key # Server private key -TLS_CA_PATH=/app/certs/trading_service/ca.crt # CA certificate -TLS_REQUIRE_CLIENT_CERT=false # Require client certs (default: false) -``` - -**Certificate Directory Structure**: -``` -/app/certs/trading_service/ -├── server.crt # Server certificate -├── server.key # Server private key -└── ca.crt # CA certificate for client verification -``` - -**Features**: -- TLS 1.3 encryption for all gRPC traffic -- Mutual TLS (mTLS) support for client certificate authentication -- 6-layer certificate validation (expiration, purpose, constraints, extensions, SANs, revocation) -- Role-based access control (RBAC) via certificate Organizational Unit (OU) -- CRL (Certificate Revocation List) support - -**Security Note**: TLS is disabled by default for development. Enable `TLS_ENABLED=true` for production deployments. +- `GRPC_PORT` -- gRPC listen port +- `DATABASE_URL` -- PostgreSQL connection string +- `TLS_ENABLED` -- enable TLS 1.3 (default: false) +- `TLS_CERT_PATH`, `TLS_KEY_PATH`, `TLS_CA_PATH` -- TLS certificates +- `TLS_REQUIRE_CLIENT_CERT` -- mutual TLS (default: false) ## Testing -To run the tests for the `trading_service` crate: - ```bash -cargo test --package trading_service +SQLX_OFFLINE=true cargo test -p trading_service --lib ``` - -## Documentation - -Comprehensive API documentation is available at [docs.rs/trading_service](https://docs.rs/trading_service). diff --git a/testing/api-gateway-load/README.md b/testing/api-gateway-load/README.md index d27a8bfdc..2b3e2332e 100644 --- a/testing/api-gateway-load/README.md +++ b/testing/api-gateway-load/README.md @@ -1,316 +1,24 @@ -# API Gateway Load Testing Framework +# api-gateway-load -Comprehensive load testing infrastructure for validating API Gateway performance under high concurrency. +Load testing framework for API Gateway with 4 scenarios: normal (1K clients), spike (10K ramp), sustained (24h), and stress (incremental to failure). -## Overview +## Key Types -This framework provides four test scenarios with detailed metrics collection, time-series analysis, and HTML report generation: +- `TestOrchestrator` -- scenario coordination +- `VirtualClientPool` -- authenticated HTTP client generation +- `MetricsCollector` -- HDR histogram latency tracking +- `ReportGenerator` -- HTML + SVG chart output -1. **Normal Load**: 1K concurrent clients for 60 seconds -2. **Spike Load**: 0→10K clients in 10s, sustain 60s -3. **Sustained Load**: 100 clients for 24 hours (endurance test) -4. **Stress Test**: Incrementally increase load until failure +## Scenarios -## Architecture - -``` -┌─────────────────────┐ -│ Test Orchestrator │ -└──────────┬──────────┘ - │ - ├─────────────────┬─────────────────┬─────────────────┐ - │ │ │ │ - ┌──────▼──────┐ ┌──────▼──────┐ ┌──────▼──────┐ ┌──────▼──────┐ - │ Normal │ │ Spike │ │ Sustained │ │ Stress │ - │ Load │ │ Load │ │ Load │ │ Test │ - └──────┬──────┘ └──────┬──────┘ └──────┬──────┘ └──────┬──────┘ - │ │ │ │ - └─────────────────┴─────────────────┴─────────────────┘ - │ - ┌───────────▼───────────┐ - │ Virtual Client Pool │ - │ (Authenticated HTTP │ - │ + Mixed Workload) │ - └───────────┬───────────┘ - │ - ┌───────────▼───────────┐ - │ Metrics Collector │ - │ - HDR Histogram │ - │ - Time Series Data │ - │ - Per-Service Stats │ - └───────────┬───────────┘ - │ - ┌───────────▼───────────┐ - │ Report Generator │ - │ - HTML + SVG Charts │ - │ - Capacity Analysis │ - └───────────────────────┘ -``` - -## Installation - -```bash -cd /home/jgrusewski/Work/foxhunt/services/api_gateway/load_tests -cargo build --release -``` +- `normal` -- 1K concurrent clients, 60s, target <10ms P99 +- `spike` -- 0 to 10K in 10s, sustain 60s +- `sustained` -- 100 clients, 24h endurance +- `stress` -- incremental increase until failure ## Usage -### Run Individual Scenarios - ```bash -# Normal load test (1K clients, 60s) -cargo run --release -- normal \ - --gateway-url http://localhost:50050 \ - --num-clients 1000 \ - --duration-secs 60 - -# Spike load test (0→10K in 10s, sustain 60s) -cargo run --release -- spike \ - --gateway-url http://localhost:50050 \ - --target-clients 10000 \ - --ramp-up-secs 10 \ - --sustain-secs 60 - -# Sustained load test (100 clients, 24h) -cargo run --release -- sustained \ - --gateway-url http://localhost:50050 \ - --num-clients 100 \ - --duration-secs 86400 - -# Stress test (incrementally increase until failure) -cargo run --release -- stress \ - --gateway-url http://localhost:50050 \ - --initial-clients 100 \ - --increment 100 \ - --increment-interval-secs 60 \ - --max-p99-latency-ms 50.0 \ - --max-error-rate-pct 5.0 +cargo run --release -p api-gateway-load -- normal --gateway-url http://localhost:50050 +cargo run --release -p api-gateway-load -- all --gateway-url http://localhost:50050 ``` - -### Run All Scenarios - -```bash -cargo run --release -- all --gateway-url http://localhost:50050 -``` - -## Metrics Collected - -### Latency Statistics -- **Min/Max/Mean**: Full latency range -- **Percentiles**: P50, P90, P95, P99, P99.9 -- **Standard Deviation**: Latency consistency - -### Request Breakdown -- **Total Requests**: Aggregate count -- **Successful**: 2xx responses -- **Failed**: 4xx/5xx errors -- **Timeout**: Connection/request timeouts -- **Rate Limited**: 429 responses -- **Circuit Breaker**: 503 responses - -### Time Series Data (1-second intervals) -- Requests per second (RPS) -- P99 latency -- Error rate percentage -- Active client count - -### Per-Service Statistics -- **Trading Service**: Order submission, position queries -- **Backtesting Service**: Backtest execution -- **ML Training Service**: Model training requests - -## Workload Distribution - -Mixed workload simulates realistic usage: - -- **60%** - Order submissions -- **30%** - Position queries -- **8%** - Backtesting requests -- **2%** - ML training requests - -Each client has random think time (1-50ms) between requests to simulate human behavior. - -## Report Generation - -HTML reports are automatically generated with: - -1. **Summary Cards**: Total requests, RPS, error rate, P99 latency -2. **Latency Table**: All percentiles with statistics -3. **Request Breakdown**: Success/failure categorization -4. **Performance Charts** (SVG): - - Requests per second over time - - P99 latency over time - - Error rate over time -5. **Capacity Recommendations**: Based on observed performance - -## Success Criteria - -### Normal Load -- **Target**: <10ms P99 latency, 0% errors -- **Pass**: Error rate < 1%, P99 < 10ms -- **Fail**: Error rate ≥ 5%, P99 ≥ 50ms - -### Spike Load -- **Target**: Graceful handling, circuit breakers activate -- **Pass**: Error rate < 10%, circuit breakers respond correctly -- **Fail**: System crashes, uncontrolled cascading failures - -### Sustained Load -- **Target**: No memory leaks, stable latency -- **Pass**: Latency drift < 5%, error rate stddev < 2% -- **Fail**: Latency increases > 10%, memory exhaustion - -### Stress Test -- **Target**: Identify capacity limits -- **Pass**: Breaking point identified with clear bottleneck -- **Fail**: Undefined behavior, data corruption - -## Example Report Output - -``` -Load Test Report: Normal Load Test -Test Period: 2025-10-03 12:00:00 to 2025-10-03 12:01:00 -Duration: 60 seconds (0.02 hours) - -Summary: -┌──────────────────┬──────────┐ -│ Total Requests │ 120,000 │ -│ Requests/Second │ 2,000 │ -│ Error Rate │ 0.12% │ -│ P99 Latency │ 8.5ms │ -└──────────────────┴──────────┘ - -Latency Statistics: -┌──────────┬──────────┐ -│ P50 │ 3.2ms │ -│ P90 │ 5.1ms │ -│ P95 │ 6.8ms │ -│ P99 │ 8.5ms │ -│ P99.9 │ 12.3ms │ -└──────────┴──────────┘ - -Capacity Recommendation: -✓ System handled 1,000 clients with 0.12% error rate -✓ P99 latency well within 10ms target -✓ Safe for production at this load level -``` - -## Load Generator Resources - -### Requirements -- **CPU**: 4+ cores for 1K clients, 8+ cores for 10K clients -- **Memory**: 2GB for 1K clients, 8GB for 10K clients -- **Network**: Low-latency connection to API Gateway - -### Monitoring Load Generator - -The framework monitors its own resource usage to ensure the load generator doesn't become a bottleneck. If you see warnings about load generator CPU/memory, consider: - -1. Running on a larger machine -2. Distributing load across multiple generators -3. Reducing concurrent client count - -## Advanced Configuration - -### Custom JWT Token - -Set authentication parameters in the code: - -```rust -let auth_client = AuthenticatedClient::new( - gateway_url, - "your-jwt-secret", - "user-id", - "username" -).await?; -``` - -### Custom Test Duration - -All scenarios support custom durations: - -```bash -# Extended normal load test (5 minutes) -cargo run --release -- normal --duration-secs 300 - -# Long-running stress test -cargo run --release -- stress --increment-interval-secs 300 -``` - -## Troubleshooting - -### Connection Refused -``` -Error: Connection refused (os error 111) -``` -**Solution**: Ensure API Gateway is running at `http://localhost:50050` - -### Too Many Open Files -``` -Error: Too many open files (os error 24) -``` -**Solution**: Increase system file descriptor limit: -```bash -ulimit -n 10000 -``` - -### Memory Exhaustion -``` -Error: Cannot allocate memory -``` -**Solution**: Reduce `--num-clients` or run on a larger machine - -### High Latency from Generator -``` -Warning: Load generator CPU > 80%, results may be unreliable -``` -**Solution**: Use a more powerful machine or reduce client count - -## Integration with CI/CD - -### GitHub Actions Example - -```yaml -name: Load Testing - -on: - schedule: - - cron: '0 0 * * 0' # Weekly - -jobs: - load-test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Start API Gateway - run: | - docker-compose up -d api_gateway - sleep 10 - - name: Run Load Tests - run: | - cd services/api_gateway/load_tests - cargo run --release -- normal - - name: Upload Reports - uses: actions/upload-artifact@v3 - with: - name: load-test-reports - path: | - normal_load_report.html - *.svg -``` - -## Performance Baseline - -Expected results for reference hardware (AWS c5.4xlarge): - -| Scenario | Clients | RPS | P99 Latency | Error Rate | -|----------------|---------|-------|-------------|------------| -| Normal Load | 1,000 | 2,000 | 8ms | <0.1% | -| Spike Load | 10,000 | 8,000 | 25ms | <2% | -| Sustained Load | 100 | 200 | 5ms | <0.01% | -| Stress Test | 5,000 | 5,000 | 45ms | Breaking | - -## License - -Part of the Foxhunt HFT Trading System - MIT OR Apache-2.0 diff --git a/testing/e2e/README.md b/testing/e2e/README.md index 50135a104..08f003767 100644 --- a/testing/e2e/README.md +++ b/testing/e2e/README.md @@ -1,398 +1,23 @@ -# Foxhunt E2E Testing Framework +# foxhunt-e2e -A comprehensive End-to-End testing framework for the Foxhunt High-Frequency Trading system. This framework tests the complete integration between TLI client, all three services (Trading, Backtesting, ML Training), database interactions, ML model inference, and complete trading workflows. +End-to-end testing framework covering service orchestration, gRPC clients, database integration, ML pipeline, and complete trading workflows. -## 🎯 Overview +## Key Types -The E2E testing framework provides: +- `E2ETestFramework` -- core orchestrator +- `ServiceManager` -- automated service startup/shutdown +- `DatabaseTestHarness` -- transaction-isolated DB testing +- `MLPipelineTestHarness` -- mock ML model inference -- **Service Orchestration**: Automated startup/shutdown of all services -- **gRPC Client Testing**: Authentication, streaming, and error handling -- **Database Integration**: Transaction management and configuration hot-reload -- **ML Pipeline Testing**: Model inference, training, and ensemble predictions -- **Complete Workflow Testing**: End-to-end trading scenarios -- **Performance Benchmarking**: Load testing and performance metrics -- **Corrode-MCP Integration**: Advanced test execution and reporting +## Test Categories -## 🏗️ Architecture +- Service startup, shutdown, recovery +- gRPC client connections, streaming, error handling +- ML inference, training, ensemble predictions +- Trading workflows, order lifecycle, risk, emergency stop -``` -tests/e2e/ -├── Cargo.toml # Project configuration -├── build.rs # gRPC proto compilation -├── src/ -│ ├── lib.rs # Main library and test macros -│ ├── framework.rs # Core E2E testing framework -│ ├── services.rs # Service management and orchestration -│ ├── clients.rs # gRPC test clients -│ ├── database.rs # Database testing harness -│ ├── ml_pipeline.rs # ML model testing framework -│ ├── workflows.rs # Complete trading workflow tests -│ ├── utils.rs # Test utilities and data generation -│ ├── corrode.rs # Corrode-MCP integration -│ └── bin/ -│ ├── test_runner.rs # Test execution runner -│ └── service_orchestrator.rs # Service management tool -├── tests/ -│ └── integration_test.rs # Example integration tests -└── README.md # This file -``` - -## 🚀 Quick Start - -### Prerequisites - -1. **Rust Toolchain**: Ensure you have Rust 1.75+ installed -2. **PostgreSQL**: Running instance for database tests -3. **Corrode-MCP**: Install corrode for advanced test execution +## Usage ```bash -# Install corrode-mcp (if not already installed) -cargo install corrode-mcp - -# Set up environment -export DATABASE_URL="postgresql://localhost/foxhunt_test" -export RUST_LOG="info" +SQLX_OFFLINE=true cargo test -p foxhunt-e2e --lib ``` - -### Running Tests - -#### Option 1: Using Test Runner (Recommended) - -```bash -# Build the test runner -cargo build --bin test_runner --release - -# Run all E2E tests -./target/release/test_runner run --test all - -# Run specific test categories -./target/release/test_runner run --test trading --parallel 2 -./target/release/test_runner run --test ml --verbose -./target/release/test_runner run --test smoke --fail-fast - -# List available tests -./target/release/test_runner list - -# Generate test report -./target/release/test_runner report --results-dir ./test-results --format html -``` - -#### Option 2: Using Service Orchestrator - -```bash -# Build the service orchestrator -cargo build --bin service_orchestrator --release - -# Start all services for testing -./target/release/service_orchestrator start --services all --wait - -# Check service status -./target/release/service_orchestrator status - -# Run specific tests against running services -cargo test --package foxhunt-e2e - -# Stop services when done -./target/release/service_orchestrator stop --services all -``` - -#### Option 3: Direct Cargo Testing - -```bash -# Run all integration tests -cargo test --package foxhunt-e2e - -# Run specific test -cargo test --package foxhunt-e2e test_complete_trading_workflow - -# Run with output -cargo test --package foxhunt-e2e -- --nocapture -``` - -## 📋 Test Categories - -### 🔧 Service Tests -- **service_startup**: Verify all services start and respond to health checks -- **service_shutdown**: Test graceful service shutdown -- **service_recovery**: Test service recovery after failures - -### 🗄️ Database Tests -- **database_integration**: Test PostgreSQL integration and queries -- **database_migrations**: Test database schema migrations -- **database_performance**: Test database query performance - -### 📡 gRPC Tests -- **grpc_clients**: Test all gRPC client connections and authentication -- **grpc_streaming**: Test streaming gRPC calls (market data, order updates) -- **grpc_error_handling**: Test gRPC error scenarios and recovery - -### 🤖 ML Pipeline Tests -- **ml_inference**: Test ML model inference pipelines -- **ml_training**: Test ML model training workflows -- **ml_ensemble**: Test ensemble prediction workflows - -### 💼 Trading Tests -- **trading_workflows**: Complete trading workflow tests -- **order_lifecycle**: Order submission to execution lifecycle -- **risk_management**: Risk management and safety mechanisms -- **emergency_stop**: Emergency stop and kill switch tests - -### 🎯 Full Suite -- **all**: Run complete E2E test suite -- **smoke**: Run smoke tests for quick validation -- **performance**: Run performance and load tests - -## 🛠️ Framework Components - -### E2ETestFramework -The core framework that orchestrates all components: - -```rust -use foxhunt_e2e::{e2e_test, framework::E2ETestFramework}; - -e2e_test!(my_test, |framework: E2ETestFramework| async { - // Your test logic here - let tli_client = framework.get_tli_client().await?; - let health = framework.check_services_health().await?; - assert!(health.all_healthy); - Ok(()) -}); -``` - -### Service Management -Automated service lifecycle management: - -```rust -use foxhunt_e2e::services::ServiceManager; - -let mut manager = ServiceManager::new(); -manager.start_all_services().await?; -// Tests run here -manager.stop_all_services().await?; -``` - -### gRPC Clients -Type-safe gRPC client implementations: - -```rust -use foxhunt_e2e::clients::{TradingServiceClient, MLTrainingServiceClient}; - -let mut trading = TradingServiceClient::new("http://localhost:50051").await?; -let portfolio = trading.get_portfolio().await?; - -let mut ml = MLTrainingServiceClient::new("http://localhost:50053").await?; -let prediction = ml.predict(features).await?; -``` - -### Database Testing -Transaction-isolated database testing: - -```rust -use foxhunt_e2e::database::DatabaseTestHarness; - -let db = DatabaseTestHarness::new().await?; -let mut tx = db.begin_test_transaction().await?; -// Database operations here - will auto-rollback -``` - -### ML Pipeline Testing -Mock ML models for testing: - -```rust -use foxhunt_e2e::ml_pipeline::MLPipelineTestHarness; - -let ml = MLPipelineTestHarness::new().await?; -let result = ml.test_model_inference("mamba", features).await?; -let ensemble = ml.test_ensemble_prediction(features).await?; -``` - -## 🎛️ Configuration - -### Environment Variables - -- `DATABASE_URL`: PostgreSQL connection string for test database -- `RUST_LOG`: Log level (debug, info, warn, error) -- `FOXHUNT_TEST_MODE`: Set to "true" for test mode -- `CUDA_VISIBLE_DEVICES`: GPU configuration for ML tests -- `TORCH_DEVICE`: PyTorch device (cpu/cuda) for ML tests - -### Test Configuration - -```toml -# tests/e2e/Cargo.toml -[package.metadata.e2e] -default_timeout = 600 -max_parallel_sessions = 4 -service_startup_timeout = 120 -database_url = "postgresql://localhost/foxhunt_test" -``` - -## 📊 Performance Benchmarks - -The framework includes comprehensive performance testing: - -### Order Submission Performance -- Target: >10 orders/second -- Success rate: >90% -- Latency: <100ms average - -### ML Inference Performance -- Target: >20 inferences/second -- Latency: <50ms average -- GPU utilization monitoring - -### Database Performance -- Query execution time monitoring -- Connection pool performance -- Transaction throughput - -## 🔍 Debugging and Troubleshooting - -### Enable Debug Logging -```bash -export RUST_LOG=debug -cargo test --package foxhunt-e2e -- --nocapture -``` - -### Service Logs -```bash -# View service logs -./target/release/service_orchestrator logs trading --follow - -# Check service status -./target/release/service_orchestrator status -``` - -### Database Issues -```bash -# Check database connection -psql $DATABASE_URL -c "SELECT 1;" - -# Reset test database -dropdb foxhunt_test && createdb foxhunt_test -``` - -### Common Issues - -1. **Service startup timeouts**: Increase `startup_timeout` in service configs -2. **gRPC connection errors**: Verify services are running and ports are correct -3. **Database connection failures**: Check PostgreSQL is running and credentials -4. **ML model loading errors**: Ensure model files exist or use mock models - -## 🧪 Writing Custom Tests - -### Basic Test Structure - -```rust -use foxhunt_e2e::{e2e_test, framework::E2ETestFramework}; -use anyhow::Result; - -e2e_test!(test_my_feature, |framework: E2ETestFramework| async { - // Test setup - let client = framework.get_tli_client().await?; - - // Test execution - let result = client.my_operation().await?; - - // Assertions - assert!(result.success, "Operation failed"); - - // Cleanup (automatic) - Ok(()) -}); -``` - -### Advanced Test Features - -```rust -e2e_test!(test_complex_workflow, |framework: E2ETestFramework| async { - // Use test data generator - let mut generator = TestDataGenerator::new(); - let market_data = generator.generate_market_data()?; - - // Measure performance - let (result, duration) = TestUtils::measure_execution_time(|| async { - // Your operation here - Ok(42) - }).await?; - - // Database testing - let db = &framework.database_harness; - let mut tx = db.begin_test_transaction().await?; - // Database operations... - - // ML testing - let ml = &framework.ml_pipeline; - let prediction = ml.test_ensemble_prediction(features).await?; - - Ok(()) -}); -``` - -## 📈 Continuous Integration - -### GitHub Actions Example - -```yaml -name: E2E Tests -on: [push, pull_request] - -jobs: - e2e-tests: - runs-on: ubuntu-latest - services: - postgres: - image: postgres:15 - env: - POSTGRES_PASSWORD: postgres - POSTGRES_DB: foxhunt_test - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - - steps: - - uses: actions/checkout@v3 - - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - - - name: Install corrode-mcp - run: cargo install corrode-mcp - - - name: Run E2E tests - env: - DATABASE_URL: postgresql://postgres:postgres@localhost/foxhunt_test - RUST_LOG: info - run: | - cargo build --bin service_orchestrator --release - ./target/release/service_orchestrator start --services all --wait --background & - sleep 10 - cargo test --package foxhunt-e2e -``` - -## 🤝 Contributing - -1. **Add new tests**: Create new test functions using the `e2e_test!` macro -2. **Extend framework**: Add new components to the framework modules -3. **Improve performance**: Optimize test execution and resource usage -4. **Documentation**: Update this README and code documentation - -### Test Naming Convention - -- `test_[component]_[scenario]`: e.g., `test_trading_order_lifecycle` -- Use descriptive names that explain what is being tested -- Group related tests in the same file - -### Code Style - -- Follow Rust standard formatting (`cargo fmt`) -- Add comprehensive error handling -- Include informative log messages -- Write clear assertions with descriptive failure messages - -## 📝 License - -This E2E testing framework is part of the Foxhunt HFT Trading System and follows the same license terms as the main project. \ No newline at end of file diff --git a/testing/e2e/integration/README.md b/testing/e2e/integration/README.md deleted file mode 100644 index 18ef52b54..000000000 --- a/testing/e2e/integration/README.md +++ /dev/null @@ -1,386 +0,0 @@ -# Foxhunt E2E Integration Test Suite - -Comprehensive end-to-end integration tests for the Foxhunt HFT Trading System. - -## 🚀 Quick Start - -```bash -# 1. Start PostgreSQL -docker run -d --name foxhunt-postgres \ - -e POSTGRES_PASSWORD=postgres \ - -e POSTGRES_DB=foxhunt \ - -p 5433:5432 postgres:15 - -# 2. Run all tests -export DATABASE_URL="postgresql://postgres:postgres@localhost:5433/foxhunt" -./e2e_test_suite.sh -``` - -## 📋 Test Suite Overview - -| Test | Description | Duration | Status | -|------|-------------|----------|--------| -| **auth_flow_test.sh** | Full authentication flow (JWT + MFA + RBAC) | ~12s | ✅ Complete | -| **trading_flow_test.sh** | Complete trading lifecycle | ~8s | ✅ Complete | -| **hot_reload_test.sh** | Configuration hot-reload (NOTIFY) | ~6s | ✅ Complete | -| **backtesting_flow_test.sh** | Backtesting workflow | TBD | 🚧 Future | -| **ml_training_flow_test.sh** | ML training workflow | TBD | 🚧 Future | - -## 🔐 Test 1: Authentication Flow - -**Tests**: Login → JWT → MFA → RBAC → Authenticated Request - -### What It Validates - -- ✅ Trading service accessibility -- ✅ User creation with bcrypt password hashing -- ✅ JWT token generation (HS256 signature) -- ✅ TOTP/MFA infrastructure -- ✅ RBAC permission validation -- ✅ Token expiration and structure -- ✅ Security (invalid credential rejection) -- ✅ Authentication audit trail - -### Usage - -```bash -./auth_flow_test.sh -``` - -### Output - -Generates JWT token stored in: -- `/tmp/foxhunt_test_token_` -- `$FOXHUNT_JWT_TOKEN` environment variable - -## 💹 Test 2: Trading Flow - -**Tests**: Order Submission → Risk Checks → Execution → Position Update - -### What It Validates - -- ✅ Order submission validation -- ✅ Pre-trade risk limits (max order size) -- ✅ Order execution simulation -- ✅ Position calculation (quantity + avg price) -- ✅ Complete audit trail -- ✅ SOX compliance validation - -### Usage - -```bash -./trading_flow_test.sh -``` - -### Trade Lifecycle - -``` -Order Submit → Risk Check → Execution → Position Update → Audit -``` - -## 🔄 Test 3: Configuration Hot-Reload - -**Tests**: Config Update → PostgreSQL NOTIFY → Service Reload - -### What It Validates - -- ✅ PostgreSQL NOTIFY/LISTEN mechanism -- ✅ Configuration change detection -- ✅ Hot-reload without service restart -- ✅ Change history audit trail -- ✅ Connection stability -- ✅ Performance (< 100ms latency) - -### Usage - -```bash -./hot_reload_test.sh -``` - -### Hot-Reload Flow - -``` -UPDATE config → Trigger → NOTIFY → Service → Reload -``` - -## 🎯 Master Test Suite - -**Script**: `e2e_test_suite.sh` - -Orchestrates all tests with: -- Pre-flight checks (database, tools) -- Sequential test execution -- Result aggregation and reporting -- Colored output with progress tracking - -### Usage - -```bash -# Run all tests -./e2e_test_suite.sh - -# With custom configuration -export TRADING_SERVICE_HOST="localhost" -export TRADING_SERVICE_PORT="50051" -export DATABASE_URL="postgresql://postgres:postgres@localhost:5433/foxhunt" -./e2e_test_suite.sh -``` - -### Expected Output - -``` -╔════════════════════════════════════════════════════════════════╗ -║ Foxhunt HFT E2E Integration Test Suite ║ -╚════════════════════════════════════════════════════════════════╝ - -[Test 1] Full Authentication Flow -═══════════════════════════════════════════════════════ -✅ Full Authentication Flow PASSED (12s) - -[Test 2] Complete Trading Flow -═══════════════════════════════════════════════════════ -✅ Complete Trading Flow PASSED (8s) - -[Test 3] Configuration Hot-Reload -═══════════════════════════════════════════════════════ -✅ Configuration Hot-Reload PASSED (6s) - -╔════════════════════════════════════════════════════════════════╗ -║ Test Suite Summary ║ -╚════════════════════════════════════════════════════════════════╝ - -Total Tests: 3 -Passed: 3 -Failed: 0 -Skipped: 0 -Success Rate: 100.0% - -╔════════════════════════════════════════════════════════════════╗ -║ ALL E2E TESTS PASSED! ✅ ║ -╚════════════════════════════════════════════════════════════════╝ -``` - -## 🔧 Prerequisites - -### Required Services - -1. **PostgreSQL Database** (port 5433) - ```bash - docker run -d --name foxhunt-postgres \ - -e POSTGRES_PASSWORD=postgres \ - -e POSTGRES_DB=foxhunt \ - -p 5433:5432 postgres:15 - ``` - -2. **Trading Service** (optional for full tests) - ```bash - DATABASE_URL=postgresql://postgres:postgres@localhost:5433/foxhunt \ - cargo run --bin trading_service - ``` - -### Required Tools - -```bash -# Ubuntu/Debian -sudo apt-get install -y \ - postgresql-client \ - bc \ - jq \ - netcat-openbsd \ - oath-toolkit - -# Verify -psql --version -bc --version -jq --version -nc -h -oathtool --version -``` - -## 🌍 Environment Variables - -| Variable | Default | Description | -|----------|---------|-------------| -| `TRADING_SERVICE_HOST` | localhost | Trading service hostname | -| `TRADING_SERVICE_PORT` | 50051 | Trading service gRPC port | -| `DATABASE_URL` | postgresql://postgres:postgres@localhost:5433/foxhunt | PostgreSQL connection string | -| `REDIS_URL` | redis://localhost:6379 | Redis connection string (optional) | - -## 📊 Performance Benchmarks - -| Test | Duration | Target | -|------|----------|--------| -| Authentication Flow | ~12s | < 15s | -| Trading Flow | ~8s | < 10s | -| Hot-Reload | ~6s | < 10s | -| **Total Suite** | **~26s** | **< 60s** | - -### Hot-Reload Latency - -| Operation | Latency | -|-----------|---------| -| Config Update | < 50ms | -| NOTIFY Propagation | < 10ms | -| Service Reload | < 40ms | -| **End-to-End** | **< 100ms** | - -## 🛡️ Security & Compliance - -### Security Features - -- ✅ Bcrypt password hashing (cost factor 12) -- ✅ JWT with HMAC-SHA256 signing -- ✅ TOTP/MFA infrastructure -- ✅ SQL injection prevention (parameterized queries) -- ✅ Token expiration validation - -### Regulatory Compliance - -- ✅ **SOX**: Immutable audit trails with timestamps -- ✅ **MiFID II**: Order lifecycle tracking -- ✅ **Data Retention**: Configurable retention policies - -## 🐛 Troubleshooting - -### Database Connection Failed - -```bash -# Check PostgreSQL is running -docker ps | grep postgres - -# Start if needed -docker run -d --name foxhunt-postgres \ - -e POSTGRES_PASSWORD=postgres \ - -e POSTGRES_DB=foxhunt \ - -p 5433:5432 postgres:15 -``` - -### Trading Service Not Accessible - -```bash -# Check if running -ps aux | grep trading_service - -# Start if needed -DATABASE_URL=postgresql://postgres:postgres@localhost:5433/foxhunt \ -cargo run --bin trading_service -``` - -### Missing Tools - -```bash -# Install all required tools -sudo apt-get install -y postgresql-client bc jq netcat-openbsd oath-toolkit -``` - -### Permission Denied - -```bash -# Make scripts executable -chmod +x *.sh -``` - -## 📚 Documentation - -- **Full Documentation**: `/home/jgrusewski/Work/foxhunt/docs/WAVE75_AGENT11_E2E_TESTING.md` -- **Project Instructions**: `/home/jgrusewski/Work/foxhunt/CLAUDE.md` - -## 🔮 Future Enhancements - -### Planned Tests - -1. **Backtesting Flow** - - Strategy creation - - Backtest execution - - Results retrieval - -2. **ML Training Flow** - - Training job submission - - Model deployment - - Inference validation - -3. **WebSocket Streaming** - - Real-time market data - - Order events - - Position updates - -4. **Kill Switch** - - Emergency shutdown - - State synchronization - - Service recovery - -5. **Load Testing** - - Concurrent orders - - Rate limiting - - Performance degradation - -### CI/CD Integration - -```yaml -# GitHub Actions example -- name: Run E2E Tests - env: - DATABASE_URL: postgresql://postgres:postgres@localhost:5433/foxhunt - run: | - cd tests/e2e/integration - ./e2e_test_suite.sh -``` - -## ✅ Success Criteria - -- ✅ All core tests passing (3/5 implemented) -- ✅ Complete trading flow validated -- ✅ Inter-service communication working -- ✅ Hot-reload functional (< 100ms) -- ✅ Audit trails persisted correctly -- ✅ SOX compliance validated -- ✅ Security measures validated - -## 📝 Files - -``` -/home/jgrusewski/Work/foxhunt/tests/e2e/integration/ -├── e2e_test_suite.sh # Master orchestration script -├── auth_flow_test.sh # Authentication test -├── trading_flow_test.sh # Trading flow test -├── hot_reload_test.sh # Hot-reload test -└── README.md # This file -``` - -## 🎓 Usage Examples - -### Run Single Test - -```bash -./auth_flow_test.sh -``` - -### Run with Custom Database - -```bash -export DATABASE_URL="postgresql://user:pass@host:5432/dbname" -./e2e_test_suite.sh -``` - -### Debug Mode - -```bash -set -x # Enable debug output -./e2e_test_suite.sh -``` - -### Continuous Testing - -```bash -# Run tests every 5 minutes -while true; do - ./e2e_test_suite.sh - sleep 300 -done -``` - ---- - -**Wave 75 Agent 11 - End-to-End Integration Testing** -**Status**: ✅ Production Ready -**Date**: 2025-10-03 diff --git a/testing/integration/README.md b/testing/integration/README.md index 279cf6fbf..78edb1ab6 100644 --- a/testing/integration/README.md +++ b/testing/integration/README.md @@ -1,1523 +1,15 @@ -# Foxhunt HFT Trading System - Comprehensive Test Documentation +# integration -## 🎯 Overview +Comprehensive integration test suite covering cross-crate interactions, database queries, gRPC service contracts, and ML model pipelines. -This document provides comprehensive test documentation for the Foxhunt High-Frequency Trading (HFT) system, covering test architecture, execution guides, mock strategies, CI/CD pipeline integration, and coverage requirements. +## Subdirectories -## 📋 Table of Contents +- `chaos/` -- chaos engineering and fault injection tests +- `e2e_helpers/` -- shared E2E test utilities (JWT generation, service probes) +- `smoke_tests/` -- quick smoke tests for CI validation -1. [Test Architecture](#test-architecture) -2. [Test Running Guide](#test-running-guide) -3. [Mock Strategies Documentation](#mock-strategies-documentation) -4. [CI/CD Test Pipeline](#cicd-test-pipeline) -5. [Coverage Requirements](#coverage-requirements) -6. [Performance Requirements](#performance-requirements) -7. [Troubleshooting](#troubleshooting) - ---- - -## 🏗️ Test Architecture - -### System Overview - -The Foxhunt test architecture is designed for enterprise-grade HFT systems with zero-tolerance for failures in production. The architecture follows a layered approach with comprehensive coverage across all system components. - -``` -┌─────────────────────────────────────────────────────────────┐ -│ TEST ARCHITECTURE │ -├─────────────────────────────────────────────────────────────┤ -│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │ -│ │ E2E │ │Integration │ │Performance │ │ -│ │ Tests │ │ Tests │ │ Tests │ │ -│ └─────────────┘ └─────────────┘ └─────────────┘ │ -│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │ -│ │ Unit │ │ Chaos │ │ Compliance │ │ -│ │ Tests │ │Engineering │ │ Tests │ │ -│ └─────────────┘ └─────────────┘ └─────────────┘ │ -│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │ -│ │ Mock │ │ Security │ │ Load │ │ -│ │ Strategies │ │ Tests │ │ Tests │ │ -│ └─────────────┘ └─────────────┘ └─────────────┘ │ -└─────────────────────────────────────────────────────────────┘ -``` - -### Test Layer Structure - -#### 1. Unit Tests (`/tests/unit/`) -- **Purpose**: Test individual components in isolation -- **Coverage**: Core trading algorithms, ML models, risk calculations -- **Performance Target**: <1ms per test -- **Safety**: Zero panic operations, comprehensive error handling - -**Key Components:** -```rust -// Core financial calculations with precision validation -tests/unit/financial_calculation_precision.rs -- Price calculation accuracy (14+ decimal places) -- Volume calculation correctness -- P&L computation validation - -// ML model accuracy validation -tests/unit/ml_model_accuracy_validation.rs -- MAMBA-2 SSM model testing -- TLOB Transformer validation -- DQN/PPO reinforcement learning tests - -// Risk management validation -tests/unit/risk_management_tests.rs -- VaR calculations -- Kelly criterion sizing -- Position limit enforcement -``` - -#### 2. Integration Tests (`/tests/integration/`) -- **Purpose**: Test service-to-service communication -- **Coverage**: gRPC interfaces, database operations, message passing -- **Performance Target**: <100ms per integration test -- **Safety**: Circuit breaker validation, timeout handling - -**Key Components:** -```rust -// Service communication validation -tests/integration/service_communication_validation.rs -- Trading Service ↔ TLI communication -- ML Training Service ↔ Trading Service data flow -- Backtesting Service ↔ Data Service integration - -// Database integration tests -tests/integration/database_integration.rs -- PostgreSQL connection pooling -- Real-time configuration updates -- Transaction consistency validation -``` - -#### 3. End-to-End Tests (`/tests/e2e/`) -- **Purpose**: Complete trading workflow validation -- **Coverage**: Full order lifecycle from signal to execution -- **Performance Target**: <500ms complete workflow -- **Safety**: Production-like scenarios, error recovery - -#### 4. Performance Tests (`/tests/performance/`) -- **Purpose**: HFT latency and throughput validation -- **Coverage**: Sub-microsecond operations, high-frequency scenarios -- **Performance Target**: Meet HFT requirements (see Performance Requirements) - -#### 5. Chaos Engineering Tests (`/tests/chaos/`) -- **Purpose**: System resilience under failure conditions -- **Coverage**: Network partitions, service crashes, resource exhaustion -- **Performance Target**: <1s recovery time for critical services - -### Test Framework Components - -#### Safety Framework (`framework/production_test_safety.rs`) - -```rust -/// Production-safe test result type eliminating panic operations -pub type TestResult = Result; - -/// Comprehensive error handling for all test failure modes -#[derive(Debug, thiserror::Error)] -pub enum TestSafetyError { - #[error("Integration failure in {service}.{operation}: {details}")] - IntegrationFailure { - service: String, - operation: String, - details: String, - }, - #[error("Performance requirement not met: {operation} took {actual_ns}ns, max allowed {max_ns}ns")] - PerformanceViolation { - operation: String, - actual_ns: u64, - max_ns: u64, - }, - #[error("Timeout exceeded: {operation} timed out after {timeout_ms}ms")] - TimeoutExceeded { - operation: String, - timeout_ms: u64, - }, - // ... additional error types -} -``` - -#### Performance Validator - -```rust -/// HFT performance validation with hardware-level precision -pub struct HftPerformanceValidator; - -impl HftPerformanceValidator { - /// Validates operation meets HFT latency requirements - /// Uses RDTSC for sub-nanosecond timing precision - pub fn validate_latency(operation: &str, duration_ns: u64) -> TestResult<()> { - let requirement = match operation { - "order_validation" => 10_000, // 10μs max - "risk_check" => 50_000, // 50μs max - "market_data_processing" => 1_000, // 1μs max - "position_update" => 5_000, // 5μs max - "price_calculation" => 2_000, // 2μs max - _ => return Err(TestSafetyError::UnknownOperation { operation: operation.to_string() }) - }; - - if duration_ns > requirement { - return Err(TestSafetyError::PerformanceViolation { - operation: operation.to_string(), - actual_ns: duration_ns, - max_ns: requirement, - }); - } - - Ok(()) - } -} -``` - ---- - -## 🚀 Test Running Guide - -### Quick Start - -#### Prerequisites -```bash -# Set environment variables -export DATABASE_URL="postgresql://localhost/foxhunt_test" -export RUST_LOG=debug -export CUDA_VISIBLE_DEVICES=0 # For GPU tests - -# Install test database -psql -c "CREATE DATABASE foxhunt_test;" -psql foxhunt_test -f tests/fixtures/test_schema.sql -``` - -#### Basic Test Commands +## Usage ```bash -# Run all tests (comprehensive suite) -cargo test --workspace - -# Run tests with coverage -cargo test --workspace -- --nocapture -cargo tarpaulin --out Html --output-dir coverage/ - -# Run specific test suites -cargo test --package tests unit_tests -cargo test --package tests integration_tests -cargo test --package tests e2e_tests +SQLX_OFFLINE=true cargo test -p integration --lib ``` - -### Test Execution Modes - -#### 1. Development Mode (Fast Feedback) -```bash -# Quick unit tests only (30-60 seconds) -cargo test --package tests --lib unit - -# With file watching for continuous testing -cargo watch -x "test --package tests --lib unit" -``` - -#### 2. Integration Mode (Service Validation) -```bash -# Integration tests with service startup (2-5 minutes) -./scripts/start_test_services.sh -cargo test --package tests integration -./scripts/stop_test_services.sh -``` - -#### 3. Performance Mode (HFT Validation) -```bash -# Performance and benchmarking tests (5-10 minutes) -cargo test --package tests performance --release -cargo bench --package tests - -# GPU-accelerated ML model tests -cargo test --package tests --features cuda gpu_tests -``` - -#### 4. Chaos Engineering Mode (Resilience Testing) -```bash -# Chaos engineering tests (10-15 minutes) -cargo test --package tests chaos --release -- --test-threads=1 - -# With network simulation -sudo cargo test --package tests chaos::network_partition -``` - -#### 5. Production Validation Mode (Full Suite) -```bash -# Complete production readiness validation (30-45 minutes) -./scripts/run_production_tests.sh - -# Expected output for production readiness: -# ✅ ALL TESTS PASSED - System ready for production deployment! -``` - -### Service-Specific Test Commands - -#### Trading Service Tests -```bash -# Core trading logic -cargo test --package services --bin trading_service - -# Trading service integration -cargo test --package tests trading_service_integration - -# Performance validation -cargo bench trading_latency -``` - -#### ML Training Service Tests -```bash -# ML model accuracy tests -cargo test --package tests ml_model_accuracy_validation - -# GPU performance tests -cargo test --package tests gpu_performance --features cuda - -# Model inference benchmarks -cargo bench ml_inference -``` - -#### TLI (Terminal Line Interface) Tests -```bash -# TLI functionality tests -cargo test --package tli - -# TLI performance tests -cargo bench tli_performance_validation -``` - -### Environment-Specific Testing - -#### Docker Environment -```bash -# Start test environment -docker-compose -f docker-compose.test.yml up -d - -# Run containerized tests -docker-compose -f docker-compose.test.yml run tests - -# Cleanup -docker-compose -f docker-compose.test.yml down -``` - -#### Kubernetes Environment -```bash -# Deploy test cluster -kubectl apply -f tests/k8s/test-namespace.yaml -kubectl apply -f tests/k8s/ - -# Run distributed tests -kubectl run test-runner --image=foxhunt:test --command -- cargo test - -# Monitor test execution -kubectl logs -f test-runner -``` - ---- - -## 🎭 Mock Strategies Documentation - -### Overview - -The Foxhunt system uses sophisticated mocking strategies to simulate real trading environments while maintaining deterministic, repeatable tests. - -### Mock Architecture - -``` -┌─────────────────────────────────────────────────────────────┐ -│ MOCK ARCHITECTURE │ -├─────────────────────────────────────────────────────────────┤ -│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │ -│ │ Market │ │ Broker │ │ Data Feed │ │ -│ │ Mocks │ │ Mocks │ │ Mocks │ │ -│ └─────────────┘ └─────────────┘ └─────────────┘ │ -│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │ -│ │ External │ │ Time │ │ Network │ │ -│ │ API Mocks │ │ Mocks │ │ Mocks │ │ -│ └─────────────┘ └─────────────┘ └─────────────┘ │ -└─────────────────────────────────────────────────────────────┘ -``` - -### 1. Market Data Mocks (`/tests/mocks/market_data.rs`) - -#### Realistic Market Simulation -```rust -/// High-fidelity market data simulator for HFT testing -pub struct MarketDataMock { - /// Tick-by-tick price movements with microsecond precision - tick_generator: TickGenerator, - /// Order book depth simulation (L2 data) - order_book: OrderBookSimulator, - /// Market microstructure effects - microstructure: MicrostructureSimulator, -} - -impl MarketDataMock { - /// Creates realistic EUR/USD market conditions - pub fn eurusd_realistic() -> Self { - Self { - tick_generator: TickGenerator::new() - .with_spread(0.00001) // 0.1 pip spread - .with_volatility(0.0012) // 12 bps daily vol - .with_frequency(1000), // 1000 ticks/second - order_book: OrderBookSimulator::new() - .with_depth(10) // 10 levels deep - .with_liquidity(1_000_000), // $1M per level - microstructure: MicrostructureSimulator::new() - .with_latency_distribution(LatencyDist::Normal(50, 10)), // 50μs ± 10μs - } - } -} -``` - -#### Market Scenario Simulation -```rust -/// Predefined market scenarios for testing -pub enum MarketScenario { - /// Normal trading conditions - Normal, - /// High volatility period (e.g., NFP release) - HighVolatility, - /// Low liquidity conditions (e.g., holiday trading) - LowLiquidity, - /// Flash crash scenario - FlashCrash, - /// Market closure/opening - MarketTransition, -} - -impl MarketDataMock { - /// Simulates specific market scenarios with realistic parameters - pub fn with_scenario(scenario: MarketScenario) -> Self { - match scenario { - MarketScenario::HighVolatility => { - Self::eurusd_realistic() - .with_volatility(0.008) // 80 bps (5x normal) - .with_frequency(5000) // 5x tick rate - .with_spread_widening(3.0) // 3x wider spreads - }, - MarketScenario::FlashCrash => { - Self::eurusd_realistic() - .with_price_shock(-0.02) // 200 pip drop - .with_liquidity_drain(0.1) // 90% liquidity removal - .with_recovery_time(300) // 5-minute recovery - }, - // ... other scenarios - } - } -} -``` - -### 2. Broker API Mocks (`/tests/mocks/broker.rs`) - -#### Interactive Brokers Mock -```rust -/// Mock Interactive Brokers TWS API -pub struct IBApiMock { - /// Connection simulation with realistic latency - connection: MockConnection, - /// Account information simulation - account: AccountSimulator, - /// Order execution simulation - execution_engine: ExecutionSimulator, -} - -impl IBApiMock { - /// Simulates realistic order execution with market impact - pub async fn place_order(&mut self, order: Order) -> TestResult { - // Simulate network latency (realistic: 1-5ms) - self.simulate_latency(Duration::from_micros(2500)).await; - - // Simulate order validation (realistic: broker-side checks) - self.validate_order(&order)?; - - // Simulate execution with realistic slippage - let execution = self.execution_engine.execute_with_slippage(order).await?; - - Ok(OrderResponse { - order_id: execution.order_id, - status: OrderStatus::Filled, - fill_price: execution.fill_price, - fill_time: SystemTime::now(), - commission: self.calculate_commission(&execution), - }) - } -} -``` - -#### Order Execution Simulation -```rust -/// Realistic order execution simulation including market impact -pub struct ExecutionSimulator { - /// Market impact model - impact_model: MarketImpactModel, - /// Slippage simulation - slippage_model: SlippageModel, -} - -impl ExecutionSimulator { - /// Executes order with realistic market conditions - pub async fn execute_with_slippage(&self, order: Order) -> TestResult { - let market_price = self.get_current_price(order.symbol).await?; - - // Calculate market impact based on order size - let impact = self.impact_model.calculate_impact( - order.quantity, - self.get_average_daily_volume(order.symbol).await? - ); - - // Apply slippage based on market conditions - let slippage = self.slippage_model.calculate_slippage( - order.quantity, - self.get_current_spread(order.symbol).await? - ); - - let fill_price = match order.side { - OrderSide::Buy => market_price + impact + slippage, - OrderSide::Sell => market_price - impact - slippage, - }; - - Ok(Execution { - order_id: order.id, - fill_price, - fill_quantity: order.quantity, - fill_time: SystemTime::now(), - }) - } -} -``` - -### 3. Time Mocks (`/tests/mocks/time.rs`) - -#### Deterministic Time Control -```rust -/// Mock time provider for deterministic testing -pub struct MockTimeProvider { - /// Current mock time - current_time: Arc>, - /// Time advancement step size - step_size: Duration, -} - -impl MockTimeProvider { - /// Advances time by specified duration - pub fn advance_time(&self, duration: Duration) -> TestResult<()> { - let mut current = self.current_time.lock() - .map_err(|_| TestSafetyError::TimeProviderError)?; - *current += duration; - Ok(()) - } - - /// Fast-forwards through market session - pub fn fast_forward_market_session(&self) -> TestResult<()> { - // Simulate 8-hour trading session in 1 second - self.advance_time(Duration::from_hours(8)) - } -} -``` - -### 4. Network Mocks (`/tests/mocks/network.rs`) - -#### Network Condition Simulation -```rust -/// Simulates various network conditions for resilience testing -pub struct NetworkConditionMock { - /// Latency simulation - latency: LatencySimulator, - /// Packet loss simulation - packet_loss: PacketLossSimulator, - /// Bandwidth simulation - bandwidth: BandwidthSimulator, -} - -impl NetworkConditionMock { - /// Simulates poor network conditions - pub fn poor_connection() -> Self { - Self { - latency: LatencySimulator::new() - .with_base_latency(Duration::from_millis(50)) - .with_jitter(Duration::from_millis(20)) - .with_spikes(0.05), // 5% of requests have 500ms spike - packet_loss: PacketLossSimulator::new() - .with_loss_rate(0.01), // 1% packet loss - bandwidth: BandwidthSimulator::new() - .with_throughput(1_000_000), // 1Mbps - } - } -} -``` - -### Mock Testing Patterns - -#### 1. Dependency Injection Pattern -```rust -/// Service with mockable dependencies -pub struct TradingService { - time_provider: T, - market_data: M, - broker: B, -} - -impl TradingService { - /// Creates service with all mocks for testing - pub fn with_mocks() -> Self { - Self { - time_provider: MockTimeProvider::new(), - market_data: MarketDataMock::eurusd_realistic(), - broker: IBApiMock::new(), - } - } -} -``` - -#### 2. Scenario-Based Testing -```rust -#[cfg(test)] -mod scenario_tests { - use super::*; - - #[tokio::test] - async fn test_high_volatility_scenario() -> TestResult<()> { - let mut trading_service = TradingService::with_mocks(); - - // Configure high volatility market conditions - trading_service.market_data = MarketDataMock::with_scenario( - MarketScenario::HighVolatility - ); - - // Execute trading strategy - let result = trading_service.execute_strategy().await?; - - // Verify appropriate risk management response - assert!(result.position_size < normal_position_size * 0.5); - assert!(result.stop_loss_tighter_than_normal); - - Ok(()) - } -} -``` - ---- - -## 🔄 CI/CD Test Pipeline - -### Overview - -The CI/CD pipeline ensures comprehensive testing at every stage of development, from commit to production deployment. - -### Pipeline Architecture - -``` -┌─────────────────────────────────────────────────────────────┐ -│ CI/CD PIPELINE STAGES │ -├─────────────────────────────────────────────────────────────┤ -│ Commit → Fast Tests → Integration → Performance → Deploy │ -│ ↓ ↓ ↓ ↓ ↓ │ -│ Lint Unit Tests Service Benchmarks Production │ -│ Check (30-60s) Tests (5-10min) Validation │ -│ (5s) (2-5min) (30min) │ -└─────────────────────────────────────────────────────────────┘ -``` - -### GitHub Actions Configuration - -#### Main Workflow (`.github/workflows/ci.yml`) -```yaml -name: Foxhunt HFT CI/CD Pipeline - -on: - push: - branches: [main, develop] - pull_request: - branches: [main] - -env: - RUST_VERSION: 1.75 - DATABASE_URL: postgresql://postgres:postgres@localhost:5432/foxhunt_test - CARGO_TERM_COLOR: always - -jobs: - # Stage 1: Fast Feedback (30-60 seconds) - fast_feedback: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Cache Cargo Dependencies - uses: actions/cache@v3 - with: - path: | - ~/.cargo/registry - ~/.cargo/git - target/ - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - - - name: Install Rust Toolchain - uses: actions-rs/toolchain@v1 - with: - toolchain: ${{ env.RUST_VERSION }} - components: clippy, rustfmt - override: true - - - name: Code Formatting Check - run: cargo fmt --all -- --check - - - name: Clippy Analysis (Zero Panic Policy) - run: | - cargo clippy --workspace --all-targets --all-features -- \ - -D warnings \ - -D clippy::unwrap_used \ - -D clippy::expect_used \ - -D clippy::panic - - - name: Security Audit - uses: actions-rs/audit@v1 - - - name: Unit Tests (Fast) - run: cargo test --workspace --lib --bins --tests unit - env: - RUST_BACKTRACE: 1 - - # Stage 2: Integration Testing (2-5 minutes) - integration_tests: - runs-on: ubuntu-latest - needs: fast_feedback - services: - postgres: - image: postgres:15 - env: - POSTGRES_PASSWORD: postgres - POSTGRES_DB: foxhunt_test - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 5432:5432 - - redis: - image: redis:7 - options: >- - --health-cmd "redis-cli ping" - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 6379:6379 - - steps: - - uses: actions/checkout@v4 - - - name: Setup Test Database - run: | - psql $DATABASE_URL -c "CREATE EXTENSION IF NOT EXISTS \"uuid-ossp\";" - psql $DATABASE_URL -f tests/fixtures/test_schema.sql - - - name: Integration Tests - run: cargo test --workspace integration - env: - RUST_BACKTRACE: 1 - TEST_DATABASE_URL: ${{ env.DATABASE_URL }} - - - name: Service Communication Tests - run: cargo test --package tests service_communication_validation - - - name: Database Integration Tests - run: cargo test --package tests database_integration - - # Stage 3: Performance Testing (5-10 minutes) - performance_tests: - runs-on: ubuntu-latest - needs: integration_tests - steps: - - uses: actions/checkout@v4 - - - name: Install Performance Dependencies - run: | - sudo apt-get update - sudo apt-get install -y linux-tools-generic - - - name: HFT Performance Validation - run: | - cargo test --package tests performance --release -- --nocapture - cargo bench --workspace - env: - RUST_BACKTRACE: 1 - - - name: Latency Benchmarks - run: | - echo "=== Trading Latency Benchmarks ===" - cargo bench trading_latency - echo "=== Order Processing Benchmarks ===" - cargo bench order_processing - echo "=== Risk Calculation Benchmarks ===" - cargo bench risk_calculations - - - name: Performance Regression Check - run: | - # Compare with baseline performance metrics - python scripts/check_performance_regression.py - - # Stage 4: GPU Testing (CUDA-enabled runners) - gpu_tests: - runs-on: [self-hosted, gpu] # Requires GPU-enabled runner - needs: fast_feedback - if: contains(github.event.head_commit.message, '[gpu]') || github.ref == 'refs/heads/main' - - steps: - - uses: actions/checkout@v4 - - - name: GPU Environment Setup - run: | - nvidia-smi - export CUDA_VISIBLE_DEVICES=0 - - - name: GPU ML Model Tests - run: | - cargo test --package tests --features cuda gpu_tests - cargo test --package ml --features cuda - - - name: ML Performance Benchmarks - run: cargo bench ml_inference --features cuda - - # Stage 5: Chaos Engineering (Optional, on schedule) - chaos_tests: - runs-on: ubuntu-latest - needs: integration_tests - if: github.event_name == 'schedule' || contains(github.event.head_commit.message, '[chaos]') - - steps: - - uses: actions/checkout@v4 - - - name: Chaos Engineering Tests - run: | - cargo test --package tests chaos --release -- --test-threads=1 - timeout-minutes: 30 - - - name: Network Partition Tests - run: | - sudo cargo test --package tests chaos::network_partition - timeout-minutes: 15 - - # Stage 6: Production Readiness Validation - production_validation: - runs-on: ubuntu-latest - needs: [integration_tests, performance_tests] - if: github.ref == 'refs/heads/main' - - steps: - - uses: actions/checkout@v4 - - - name: Comprehensive Production Tests - run: | - ./scripts/run_production_tests.sh - timeout-minutes: 45 - - - name: Production Readiness Check - run: | - if ./scripts/run_production_tests.sh | grep -q "✅ ALL TESTS PASSED"; then - echo "✅ System ready for production deployment" - echo "production_ready=true" >> $GITHUB_OUTPUT - else - echo "❌ System not ready for production" - echo "production_ready=false" >> $GITHUB_OUTPUT - exit 1 - fi - id: readiness_check - - - name: Generate Test Report - run: | - ./scripts/generate_test_report.sh > test_report.md - - - name: Upload Test Report - uses: actions/upload-artifact@v3 - with: - name: test-report - path: test_report.md - - # Stage 7: Deployment Gate - deployment_gate: - runs-on: ubuntu-latest - needs: production_validation - if: github.ref == 'refs/heads/main' && needs.production_validation.outputs.production_ready == 'true' - - steps: - - name: Production Deployment Authorization - run: | - echo "🚀 Production deployment authorized" - echo "All tests passed - system ready for production" - - - name: Trigger Deployment - uses: peter-evans/repository-dispatch@v2 - with: - token: ${{ secrets.GITHUB_TOKEN }} - event-type: deploy-production - client-payload: | - { - "ref": "${{ github.ref }}", - "sha": "${{ github.sha }}", - "test_status": "passed" - } -``` - -#### Performance Regression Monitoring -```yaml -# .github/workflows/performance-monitoring.yml -name: Performance Regression Monitoring - -on: - schedule: - - cron: '0 2 * * *' # Daily at 2 AM - workflow_dispatch: - -jobs: - performance_baseline: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Benchmark Current Performance - run: | - cargo bench --workspace > benchmark_results.txt - - - name: Compare with Baseline - run: | - python scripts/performance_regression_analysis.py - - - name: Alert on Regression - if: failure() - uses: 8398a7/action-slack@v3 - with: - status: failure - text: "🚨 Performance regression detected in Foxhunt HFT system" - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }} -``` - -### Branch-Specific Testing - -#### Feature Branch Testing -```yaml -# Lightweight testing for feature branches -- name: Feature Branch Tests - if: github.ref != 'refs/heads/main' - run: | - cargo test --workspace --lib unit - cargo test --package tests integration::basic -``` - -#### Release Branch Testing -```yaml -# Comprehensive testing for release branches -- name: Release Validation - if: startsWith(github.ref, 'refs/heads/release/') - run: | - ./scripts/run_comprehensive_tests.sh - ./scripts/validate_production_readiness.sh -``` - -### Test Metrics Collection - -#### Test Execution Metrics -```yaml -- name: Collect Test Metrics - run: | - echo "test_duration=$(date +%s)" >> $GITHUB_ENV - cargo test --workspace -- --format json > test_results.json - - # Parse test results - python scripts/parse_test_results.py test_results.json -``` - -#### Coverage Collection -```yaml -- name: Code Coverage - run: | - cargo install cargo-tarpaulin - cargo tarpaulin --out Xml --output-dir coverage/ - -- name: Upload Coverage - uses: codecov/codecov-action@v3 - with: - file: coverage/tarpaulin-report.xml - fail_ci_if_error: true -``` - ---- - -## 📊 Coverage Requirements - -### Coverage Targets - -The Foxhunt system maintains strict coverage requirements to ensure production reliability: - -| Component | Line Coverage | Branch Coverage | Function Coverage | Requirements | -|-----------|---------------|-----------------|-------------------|--------------| -| **Core Trading** | ≥95% | ≥90% | 100% | Critical path | -| **Risk Management** | ≥98% | ≥95% | 100% | Zero tolerance | -| **ML Models** | ≥85% | ≥80% | ≥95% | Model validation | -| **Data Processing** | ≥90% | ≥85% | ≥95% | Data integrity | -| **Services** | ≥90% | ≥85% | ≥95% | Service reliability | -| **Utilities** | ≥80% | ≥75% | ≥90% | Support functions | - -### Coverage Analysis - -#### Core Components Coverage - -```bash -# Generate detailed coverage report -cargo tarpaulin --workspace --out Html --output-dir coverage/ \ - --exclude-files "tests/*" "benches/*" \ - --timeout 300 - -# Critical components detailed analysis -cargo tarpaulin --packages foxhunt-core,risk,ml \ - --out Xml --output-dir coverage/critical/ -``` - -#### Coverage Report Structure -``` -coverage/ -├── index.html # Main coverage dashboard -├── core/ # Core trading system coverage -│ ├── trading/ # Trading algorithms -│ ├── risk/ # Risk management -│ └── compliance/ # Compliance systems -├── ml/ # ML model coverage -│ ├── models/ # Individual model coverage -│ └── inference/ # Inference pipeline coverage -├── services/ # Service coverage -└── integration/ # Integration test coverage -``` - -#### Coverage Enforcement - -```rust -// Coverage enforcement in CI -#[cfg(test)] -mod coverage_requirements { - use super::*; - - #[test] - fn enforce_critical_path_coverage() { - // This test fails if critical paths are not fully covered - let coverage = get_line_coverage("core/src/trading/"); - assert!( - coverage >= 0.95, - "Critical trading path coverage {} below required 95%", - coverage - ); - } - - #[test] - fn enforce_risk_management_coverage() { - // Risk management must have near-perfect coverage - let coverage = get_line_coverage("risk/src/"); - assert!( - coverage >= 0.98, - "Risk management coverage {} below required 98%", - coverage - ); - } -} -``` - -### Coverage Exclusions - -#### Justified Exclusions -```rust -// Example of justified coverage exclusion -impl OrderManager { - pub fn process_order(&self, order: Order) -> Result { - // Normal processing logic (covered by tests) - match self.validate_order(&order) { - Ok(_) => self.execute_order(order), - Err(e) => { - // Emergency logging - exclude from coverage - #[cfg(not(tarpaulin_include))] - emergency_log!("Critical order validation failure: {}", e); - Err(e) - } - } - } -} -``` - -#### Coverage Configuration (`.cargo/config.toml`) -```toml -[env] -# Coverage exclusion patterns -TARPAULIN_EXCLUDE = [ - "tests/*", - "benches/*", - "examples/*", - "*/main.rs", - "*emergency_log*" -] -``` - -### Differential Coverage - -#### Pull Request Coverage -```yaml -- name: Differential Coverage Check - run: | - # Check coverage only on changed files - git diff --name-only origin/main...HEAD | \ - grep "\.rs$" | \ - xargs cargo tarpaulin --files - - # Ensure new code meets coverage standards - python scripts/check_differential_coverage.py -``` - -#### Coverage Regression Prevention -```bash -#!/bin/bash -# scripts/check_coverage_regression.sh - -BASELINE_COVERAGE=$(cat coverage/baseline.txt) -CURRENT_COVERAGE=$(cargo tarpaulin --workspace --output-dir /tmp | grep "Coverage:" | cut -d' ' -f2) - -if (( $(echo "$CURRENT_COVERAGE < $BASELINE_COVERAGE - 1.0" | bc -l) )); then - echo "❌ Coverage regression detected: $CURRENT_COVERAGE% < $BASELINE_COVERAGE%" - exit 1 -else - echo "✅ Coverage maintained: $CURRENT_COVERAGE%" -fi -``` - ---- - -## ⚡ Performance Requirements - -### HFT Performance Standards - -The Foxhunt system must meet strict latency requirements for high-frequency trading: - -| Operation | Latency Requirement | Throughput Requirement | Validation Method | -|-----------|-------------------|----------------------|------------------| -| **Order Validation** | <10μs (99.9% ile) | >100K orders/sec | Hardware timing | -| **Risk Check** | <50μs (99.9% ile) | >50K checks/sec | RDTSC validation | -| **Market Data Processing** | <1μs (99.9% ile) | >1M ticks/sec | Lock-free queues | -| **Position Update** | <5μs (99.9% ile) | >200K updates/sec | Atomic operations | -| **Price Calculation** | <2μs (99.9% ile) | >500K calcs/sec | SIMD validation | -| **ML Inference** | <100μs (99.9% ile) | >10K predictions/sec | GPU acceleration | - -### Performance Test Implementation - -#### Latency Testing with Hardware Precision -```rust -/// Hardware-level latency measurement using RDTSC -pub struct HardwareTimer { - cpu_frequency: u64, -} - -impl HardwareTimer { - pub fn new() -> Self { - Self { - cpu_frequency: Self::detect_cpu_frequency(), - } - } - - /// Measures operation latency with nanosecond precision - pub fn measure(&self, operation: F) -> (T, Duration) - where - F: FnOnce() -> T, - { - unsafe { - let start = core::arch::x86_64::_rdtsc(); - let result = operation(); - let end = core::arch::x86_64::_rdtsc(); - - let cycles = end - start; - let nanos = (cycles * 1_000_000_000) / self.cpu_frequency; - - (result, Duration::from_nanos(nanos)) - } - } -} - -#[cfg(test)] -mod performance_tests { - use super::*; - - #[test] - fn test_order_validation_latency() -> TestResult<()> { - let timer = HardwareTimer::new(); - let order_manager = OrderManager::new(); - let test_order = create_test_order(); - - // Warm up CPU caches - for _ in 0..1000 { - let _ = order_manager.validate_order(&test_order); - } - - // Measure actual latency (1000 iterations for statistical significance) - let mut latencies = Vec::with_capacity(1000); - for _ in 0..1000 { - let (_, latency) = timer.measure(|| { - order_manager.validate_order(&test_order) - }); - latencies.push(latency.as_nanos() as u64); - } - - // Statistical analysis - let p99_9 = percentile(&mut latencies, 99.9); - let mean = latencies.iter().sum::() / latencies.len() as u64; - - // Validate performance requirements - HftPerformanceValidator::validate_latency("order_validation", p99_9)?; - - println!("Order Validation Performance:"); - println!(" Mean: {}ns", mean); - println!(" 99.9%ile: {}ns (requirement: <10,000ns)", p99_9); - - Ok(()) - } -} -``` - -#### Throughput Testing -```rust -#[test] -fn test_order_processing_throughput() -> TestResult<()> { - let order_manager = OrderManager::new(); - let test_orders: Vec = (0..100_000) - .map(|_| create_test_order()) - .collect(); - - let start = Instant::now(); - - // Process orders in parallel to measure throughput - let results: Vec<_> = test_orders - .par_iter() - .map(|order| order_manager.process_order(order)) - .collect(); - - let duration = start.elapsed(); - let throughput = test_orders.len() as f64 / duration.as_secs_f64(); - - // Validate throughput requirement - assert!( - throughput >= 100_000.0, - "Order processing throughput {}orders/sec below required 100K/sec", - throughput as u64 - ); - - println!("Order Processing Throughput: {:.0} orders/sec", throughput); - Ok(()) -} -``` - -### Memory Performance Testing -```rust -#[test] -fn test_memory_allocation_performance() -> TestResult<()> { - use std::alloc::{GlobalAlloc, Layout, System}; - use std::time::Instant; - - // Test memory pool allocation performance - let memory_pool = MemoryPool::new(1024 * 1024); // 1MB pool - - let timer = HardwareTimer::new(); - let mut allocation_times = Vec::with_capacity(10000); - - for _ in 0..10000 { - let (_, duration) = timer.measure(|| { - let ptr = memory_pool.allocate(64); // Allocate 64-byte order structure - memory_pool.deallocate(ptr); - }); - allocation_times.push(duration.as_nanos() as u64); - } - - let p99 = percentile(&mut allocation_times, 99.0); - - // Memory allocation must be <100ns for HFT - assert!( - p99 < 100, - "Memory allocation P99 latency {}ns exceeds 100ns requirement", - p99 - ); - - Ok(()) -} -``` - ---- - -## 🔧 Troubleshooting - -### Common Test Issues - -#### 1. Database Connection Issues -```bash -# Problem: Tests fail with database connection errors -# Solution: -export DATABASE_URL="postgresql://localhost/foxhunt_test" -psql -c "CREATE DATABASE foxhunt_test;" -psql foxhunt_test -f tests/fixtures/test_schema.sql - -# For Docker environments: -docker-compose -f docker-compose.test.yml up postgres -d -``` - -#### 2. GPU Tests Failing -```bash -# Problem: CUDA tests fail on CI -# Solution: Check GPU availability -nvidia-smi -export CUDA_VISIBLE_DEVICES=0 - -# Skip GPU tests if no GPU available: -cargo test --workspace -- --skip gpu_tests -``` - -#### 3. Performance Tests Inconsistent -```bash -# Problem: Performance tests show inconsistent results -# Solution: Isolate CPU cores and disable frequency scaling -sudo cpufreq-set -g performance -sudo taskset -c 0-3 cargo test performance --release - -# Set CPU affinity in tests: -core_affinity::set_for_current(CoreId { id: 0 }); -``` - -#### 4. Integration Tests Timeout -```bash -# Problem: Integration tests timeout -# Solution: Increase timeout and check service health -export TEST_TIMEOUT=300 # 5 minutes -./scripts/check_service_health.sh - -# Debug service startup: -RUST_LOG=debug cargo test integration --nocapture -``` - -### Test Environment Setup - -#### Development Environment -```bash -#!/bin/bash -# scripts/setup_test_environment.sh - -echo "Setting up Foxhunt test environment..." - -# Database setup -createdb foxhunt_test -psql foxhunt_test -f tests/fixtures/test_schema.sql - -# Redis setup -redis-server --daemonize yes --port 6380 - -# Environment variables -export DATABASE_URL="postgresql://localhost/foxhunt_test" -export REDIS_URL="redis://localhost:6380" -export RUST_LOG=debug -export TEST_ENV=development - -# Performance optimizations -echo performance | sudo tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -echo 0 | sudo tee /proc/sys/kernel/randomize_va_space - -echo "✅ Test environment ready" -``` - -#### CI Environment Debug -```bash -# Debug CI failures -export RUST_BACKTRACE=full -export RUST_LOG=trace -cargo test --workspace --verbose -- --nocapture - -# Generate detailed test report -cargo test --workspace -- --format json > test_results.json -python scripts/analyze_test_failures.py test_results.json -``` - -### Performance Debugging - -#### Latency Spikes Investigation -```rust -#[cfg(test)] -mod performance_debug { - #[test] - fn debug_latency_spikes() -> TestResult<()> { - let timer = HardwareTimer::new(); - let mut latencies = Vec::new(); - - // Measure with detailed timing - for i in 0..10000 { - let (_, latency) = timer.measure(|| { - // Your operation here - expensive_operation() - }); - - let nanos = latency.as_nanos() as u64; - latencies.push((i, nanos)); - - // Log spikes for investigation - if nanos > 50_000 { // >50μs spike - println!("Latency spike at iteration {}: {}ns", i, nanos); - // Additional debugging info - print_cpu_state(); - print_memory_state(); - } - } - - Ok(()) - } -} -``` - -### Test Data Management - -#### Test Data Generation -```rust -/// Generates realistic test data for HFT scenarios -pub struct TestDataGenerator { - rng: ChaCha8Rng, -} - -impl TestDataGenerator { - /// Creates realistic EUR/USD order flow for testing - pub fn generate_eurusd_orders(&mut self, count: usize) -> Vec { - (0..count) - .map(|_| Order { - symbol: "EURUSD".to_string(), - side: if self.rng.gen_bool(0.5) { OrderSide::Buy } else { OrderSide::Sell }, - quantity: Decimal::from_f64( - self.rng.gen_range(1_000.0..1_000_000.0) - ).unwrap(), - price: Decimal::from_f64( - self.rng.gen_range(1.0500..1.1500) - ).unwrap(), - order_type: OrderType::Limit, - time_in_force: TimeInForce::GoodTillCancel, - timestamp: SystemTime::now(), - }) - .collect() - } -} -``` - -### Monitoring Test Health - -#### Test Metrics Dashboard -```rust -/// Collects test execution metrics -pub struct TestMetricsCollector { - execution_times: HashMap>, - failure_counts: HashMap, - memory_usage: Vec, -} - -impl TestMetricsCollector { - pub fn record_test_execution(&mut self, test_name: &str, duration: Duration) { - self.execution_times - .entry(test_name.to_string()) - .or_default() - .push(duration); - } - - pub fn generate_health_report(&self) -> TestHealthReport { - TestHealthReport { - total_tests: self.execution_times.len(), - average_execution_time: self.calculate_average_time(), - slowest_tests: self.get_slowest_tests(10), - failure_rate: self.calculate_failure_rate(), - memory_trend: self.analyze_memory_trend(), - } - } -} -``` - ---- - -## 📈 Advanced Testing Strategies - -### Property-Based Testing - -#### Financial Property Validation -```rust -use proptest::prelude::*; - -proptest! { - #[test] - fn test_position_value_conservation( - orders in prop::collection::vec(arbitrary_order(), 1..100) - ) { - let mut position_manager = PositionManager::new(); - let initial_value = position_manager.total_value(); - - // Apply all orders - for order in orders { - position_manager.apply_order(order)?; - } - - // Property: Total value should be conserved (minus commissions) - let final_value = position_manager.total_value(); - let commissions = position_manager.total_commissions(); - - prop_assert_eq!( - initial_value, - final_value + commissions, - "Position value not conserved" - ); - } -} -``` - -### Mutation Testing - -#### Code Robustness Validation -```bash -# Install cargo-mutagen for mutation testing -cargo install cargo-mutagen - -# Run mutation tests on critical components -cargo mutagen --package foxhunt-core --package risk - -# Analyze mutation test results -python scripts/analyze_mutation_results.py -``` - -### Stress Testing - -#### System Limits Exploration -```rust -#[test] -fn stress_test_order_processing() -> TestResult<()> { - let order_manager = OrderManager::new(); - let orders_per_second = [1_000, 10_000, 50_000, 100_000, 200_000]; - - for &rate in &orders_per_second { - println!("Testing {} orders/second...", rate); - - let start = Instant::now(); - let orders = generate_orders(rate); - - let results: Result, _> = orders - .into_iter() - .map(|order| order_manager.process_order(order)) - .collect(); - - match results { - Ok(_) => { - println!("✅ Successfully processed {} orders/second", rate); - }, - Err(e) => { - println!("❌ Failed at {} orders/second: {}", rate, e); - break; - } - } - } - - Ok(()) -} -``` - -This comprehensive test documentation provides a complete guide to testing the Foxhunt HFT Trading System, covering architecture, execution, mocking strategies, CI/CD integration, and coverage requirements. The documentation ensures production-ready testing practices with zero-tolerance for failures. \ No newline at end of file diff --git a/testing/integration/chaos/README.md b/testing/integration/chaos/README.md deleted file mode 100644 index d20a115b2..000000000 --- a/testing/integration/chaos/README.md +++ /dev/null @@ -1,292 +0,0 @@ -# Foxhunt Chaos Engineering Framework - -A comprehensive chaos engineering framework specifically designed for high-frequency trading (HFT) systems with sub-100ms recovery requirements. - -## 🎯 Overview - -This framework provides systematic failure injection and recovery validation for the Foxhunt HFT trading system, focusing on: - -- **MLTrainingService Resilience**: Kill/restart scenarios with checkpoint recovery -- **Performance Validation**: Sub-100ms recovery time requirements -- **Failure Injection**: Network, memory, GPU, disk, and database failures -- **Automated Testing**: Nightly chaos job scheduling with reporting -- **HFT-Specific Requirements**: Ultra-low latency validation and monitoring - -## 🏗️ Architecture - -``` -tests/chaos/ -├── chaos_framework.rs # Core chaos orchestration engine -├── ml_training_chaos.rs # ML-specific chaos tests -├── nightly_chaos_runner.rs # Automated scheduling and execution -├── chaos_cli.rs # Command-line interface -├── examples/ -│ ├── usage_examples.rs # Comprehensive usage examples -│ └── chaos_config.toml # Configuration template -└── README.md # This file -``` - -## 🚀 Quick Start - -### 1. Basic ML Service Kill Test - -```rust -use foxhunt_tests::chaos::*; - -#[tokio::main] -async fn main() -> Result<()> { - // Initialize chaos framework - let runner = initialize_foxhunt_chaos().await?; - - // Run quick ML service resilience test - let results = run_quick_chaos_test().await?; - - println!("Chaos test results: {:?}", results); - Ok(()) -} -``` - -### 2. Command Line Usage - -```bash -# Run a single process kill experiment -cargo run --bin foxhunt-chaos run --experiment-type process-kill --service ml_training_service - -# Run comprehensive ML test suite -cargo run --bin foxhunt-chaos ml-suite --endpoint http://localhost:8080 --generate-report - -# Start nightly scheduler -cargo run --bin foxhunt-chaos schedule --time 02:00 --exclude-weekends --webhook https://hooks.slack.com/... - -# Validate system readiness -cargo run --bin foxhunt-chaos validate --check-ml-service --check-database -``` - -### 3. Configuration File - -```toml -# chaos_config.toml -[general] -enabled = true -schedule_time = "02:00" -max_duration_hours = 3 - -[ml_chaos_config] -ml_service_endpoint = "http://localhost:8080" -max_recovery_time_ms = 100 # HFT requirement -model_types = ["tlob", "dqn", "mamba2"] -``` - -## 🧪 Supported Failure Types - -### Process Failures -- **SIGTERM/SIGKILL**: Graceful and forceful process termination -- **Service Restart**: Automatic restart with configurable delays - -### Resource Exhaustion -- **Memory Pressure**: Configurable memory consumption (2GB-8GB) -- **GPU Exhaustion**: GPU memory filling (80%-95% capacity) -- **CPU Throttling**: CPU limit enforcement (25%-75%) - -### Infrastructure Failures -- **Network Partitions**: Port-specific network isolation -- **Disk I/O Failures**: File system failure injection -- **Database Disconnections**: Connection pool exhaustion - -## 📊 ML Model Support - -The framework supports chaos testing across all Foxhunt ML models: - -| Model | Type | Recovery Target | Checkpoint Interval | -|-------|------|----------------|-------------------| -| **TLOB** | Transformer | 25ms | 30s | -| **MAMBA-2** | State Space | 40ms | 60s | -| **DQN** | Deep Q-Learning | 80ms | 120s | -| **PPO** | Policy Optimization | 60ms | 90s | -| **Liquid** | Neural Network | 35ms | 45s | -| **TFT** | Temporal Fusion | 95ms | 180s | - -## 🕒 Nightly Automation - -### Scheduling Features -- **Configurable Time**: Any time zone and schedule -- **Weekend Exclusion**: Skip weekends for production safety -- **Retry Logic**: Automatic retry on failure with exponential backoff -- **Notification Integration**: Slack/Teams webhooks for alerts - -### Alert Thresholds -- **Critical**: SLA violations (>100ms recovery) -- **Warning**: Checkpoint failures or performance regressions -- **Info**: Successful completion notifications - -## 📈 Performance Requirements - -### HFT Latency Targets -- **Order Processing**: <50μs end-to-end -- **Market Data**: <30μs ingestion latency -- **Risk Calculation**: <25μs computation -- **Recovery Time**: <100ms system restoration - -### Validation Metrics -- **P50/P95/P99 Latency**: Histogram tracking -- **Recovery Time Distribution**: Statistical analysis -- **Checkpoint Integrity**: Binary validation -- **Performance Regression**: Pre/post comparison - -## 🔧 Integration - -### Test Infrastructure Integration - -```rust -// In your tests/lib.rs -pub mod chaos; - -#[tokio::test] -async fn test_ml_service_resilience() { - use crate::chaos::*; - - let results = run_quick_chaos_test().await.unwrap(); - assert!(!results.is_empty()); -} -``` - -### CI/CD Integration - -```yaml -# .github/workflows/chaos.yml -name: Chaos Engineering -on: - schedule: - - cron: '0 2 * * *' # Run at 2 AM daily - -jobs: - chaos-tests: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Run Chaos Tests - run: cargo test --test chaos_integration -``` - -## 📋 Example Scenarios - -### 1. ML Training Kill/Restart -```rust -let experiment = ChaosExperiment { - name: "TLOB Training Kill Test".to_string(), - target_service: "ml_training_service".to_string(), - failure_type: FailureType::ProcessKill { - signal: Signal::SIGTERM, - delay_before_restart_ms: 2000, - }, - max_recovery_time_ms: 25, // TLOB target - // ... -}; -``` - -### 2. GPU Memory Exhaustion -```rust -let experiment = ChaosExperiment { - name: "GPU Memory Pressure Test".to_string(), - failure_type: FailureType::GpuResourceExhaustion { - memory_fill_percent: 95, - duration_ms: 20000, - }, - max_recovery_time_ms: 150, // Relaxed for GPU - // ... -}; -``` - -### 3. Network Partition -```rust -let experiment = ChaosExperiment { - name: "Database Partition Test".to_string(), - failure_type: FailureType::NetworkPartition { - target_ports: vec![5432, 6379], // PostgreSQL, Redis - duration_ms: 15000, - }, - // ... -}; -``` - -## 🛡️ Safety Features - -### Production Safeguards -- **Weekend Exclusion**: Automatic weekend skipping -- **Duration Limits**: Maximum 3-hour chaos windows -- **Recovery Timeouts**: Automatic experiment termination -- **Checkpoint Validation**: Pre/post integrity checks - -### Monitoring Integration -- **Real-time Alerting**: Immediate notification of failures -- **Performance Tracking**: Latency histogram recording -- **Report Generation**: Automated markdown reports -- **Event Streaming**: Live experiment status updates - -## 📊 Reporting - -### Automated Reports -```markdown -# ML Training Chaos Engineering Report - -**Generated:** 2025-01-21 02:30:00 UTC -**Total Tests:** 18 - -## Summary -- ✅ **Successful:** 16 (88.9%) -- ❌ **Failed:** 2 (11.1%) -- 📊 **Success Rate:** 88.9% - -## Results by Model Type -- **tlob:** 6/6 (100.0%) -- **dqn:** 5/6 (83.3%) -- **mamba2:** 5/6 (83.3%) - -## Performance Analysis -- ✅ **No Performance Regressions Detected** -- **Average Recovery Time:** 45.2ms -- **Max Recovery Time:** 78ms -``` - -## 🔍 Troubleshooting - -### Common Issues - -1. **Service Not Found** - ```bash - # Check service status - systemctl status ml_training_service - ``` - -2. **GPU Not Available** - ```bash - # Verify GPU access - nvidia-smi - ``` - -3. **Permission Errors** - ```bash - # Check chaos framework permissions - sudo usermod -a -G docker $USER - ``` - -### Debug Mode -```bash -# Enable verbose logging -cargo run --bin foxhunt-chaos --verbose run --experiment-type process-kill -``` - -## 🤝 Contributing - -1. **Add New Failure Types**: Extend `FailureType` enum -2. **ML Model Support**: Add new models to `ModelType` -3. **Monitoring Integration**: Extend metrics collection -4. **Custom Experiments**: Create domain-specific tests - -## 📝 License - -This chaos engineering framework is part of the Foxhunt HFT trading system and follows the same licensing terms. - ---- - -**⚠️ Important**: This framework is designed specifically for HFT systems with sub-100ms recovery requirements. Always test in non-production environments first and ensure proper safeguards are in place. \ No newline at end of file diff --git a/testing/integration/e2e_helpers/README.md b/testing/integration/e2e_helpers/README.md deleted file mode 100644 index 68d2e268a..000000000 --- a/testing/integration/e2e_helpers/README.md +++ /dev/null @@ -1,257 +0,0 @@ -# E2E Testing Helpers - -Helper scripts and utilities for end-to-end testing of the Foxhunt HFT Trading System. - -## JWT Token Generator - -**File**: `jwt_token_generator.sh` - -Generates valid JWT tokens for testing API Gateway authentication. The token structure matches the production API Gateway implementation. - -### Quick Start - -```bash -# Generate default trader token -./jwt_token_generator.sh - -# Generate admin token -./jwt_token_generator.sh admin_user admin "api.access,system.admin" - -# Generate token with 10-minute expiration -./jwt_token_generator.sh test_user trader "api.access" 600 - -# Use in curl request -TOKEN=$(./jwt_token_generator.sh) -curl -H "Authorization: Bearer $TOKEN" http://localhost:50051/api/v1/orders -``` - -### Arguments - -| Position | Name | Default | Description | -|----------|------|---------|-------------| -| 1 | user_id | `test_user_123` | User identifier | -| 2 | role | `trader` | User role (trader, admin, viewer) | -| 3 | permissions | `api.access` | Comma-separated permissions | -| 4 | ttl_seconds | `3600` | Token expiration in seconds | - -### Environment Variables - -- `JWT_SECRET` - JWT signing secret (default: test secret matching API Gateway) - -**Production**: Set `JWT_SECRET` environment variable to match your deployment: -```bash -export JWT_SECRET="your-production-secret-key" -./jwt_token_generator.sh -``` - -### Token Structure - -Generated tokens include the following claims (matching `services/api_gateway/tests/common/mod.rs`): - -**Standard JWT Claims**: -- `sub` - Subject (user ID) -- `iat` - Issued at (Unix timestamp) -- `exp` - Expiration (Unix timestamp) -- `nbf` - Not before (Unix timestamp) -- `iss` - Issuer (`foxhunt-api-gateway`) -- `aud` - Audience (`foxhunt-services`) -- `jti` - JWT ID (UUID, for revocation support) - -**Foxhunt-Specific Claims**: -- `roles` - User roles array (RBAC) -- `permissions` - Granular permissions array -- `token_type` - Token type (`access` or `refresh`) -- `session_id` - Session identifier (UUID) - -### Common Use Cases - -#### 1. Test API Gateway Authentication - -```bash -# Generate token -TOKEN=$(./jwt_token_generator.sh) - -# Test health endpoint (no auth required) -curl http://localhost:8080/health - -# Test authenticated endpoint -curl -H "Authorization: Bearer $TOKEN" \ - http://localhost:50051/api/v1/orders -``` - -#### 2. Test Role-Based Access Control (RBAC) - -```bash -# Trader role (limited permissions) -TRADER_TOKEN=$(./jwt_token_generator.sh trader_user trader "api.access") - -# Admin role (full permissions) -ADMIN_TOKEN=$(./jwt_token_generator.sh admin_user admin "api.access,system.admin") - -# Test trader permissions -curl -H "Authorization: Bearer $TRADER_TOKEN" \ - http://localhost:50051/api/v1/orders - -# Test admin permissions -curl -H "Authorization: Bearer $ADMIN_TOKEN" \ - http://localhost:50051/api/v1/config -``` - -#### 3. Test Token Expiration - -```bash -# Short-lived token (30 seconds) -SHORT_TOKEN=$(./jwt_token_generator.sh test_user trader "api.access" 30) - -# Use immediately (should succeed) -curl -H "Authorization: Bearer $SHORT_TOKEN" \ - http://localhost:50051/api/v1/orders - -# Wait 31 seconds -sleep 31 - -# Use again (should fail with 401 Unauthorized) -curl -H "Authorization: Bearer $SHORT_TOKEN" \ - http://localhost:50051/api/v1/orders -``` - -#### 4. Load Testing with Multiple Users - -```bash -# Generate 10 unique user tokens -for i in {1..10}; do - TOKEN=$(./jwt_token_generator.sh "user_$i" trader "api.access") - echo "$TOKEN" > "token_$i.txt" -done - -# Use in load test -TOKEN=$(cat token_1.txt) -curl -H "Authorization: Bearer $TOKEN" \ - http://localhost:50051/api/v1/orders -``` - -#### 5. Integration Test Scripts - -```bash -#!/bin/bash -# test_trading_flow.sh - -# Generate authentication token -TOKEN=$(./jwt_token_generator.sh) - -# Submit order -ORDER_RESPONSE=$(curl -s -X POST \ - -H "Authorization: Bearer $TOKEN" \ - -H "Content-Type: application/json" \ - -d '{"symbol": "BTC/USD", "quantity": 1.0, "price": 50000.0}' \ - http://localhost:50051/api/v1/orders) - -# Extract order ID -ORDER_ID=$(echo "$ORDER_RESPONSE" | jq -r '.order_id') - -# Check order status -curl -H "Authorization: Bearer $TOKEN" \ - "http://localhost:50051/api/v1/orders/$ORDER_ID" -``` - -### Validation - -The generated tokens are validated against the same structure used in Rust tests: - -**Source**: `services/api_gateway/tests/common/mod.rs` (lines 28-62) - -```rust -pub fn generate_test_token( - user_id: &str, - roles: Vec, - permissions: Vec, - ttl_seconds: u64, -) -> Result<(String, String)> { - // ... (matches this script's implementation) -} -``` - -### Dependencies - -**Required**: -- Python 3.x -- PyJWT library: `pip install pyjwt` - -**Installation**: -```bash -# Ubuntu/Debian -sudo apt-get install python3 python3-pip -pip3 install pyjwt - -# macOS -brew install python3 -pip3 install pyjwt - -# Verify installation -python3 -c "import jwt; print('PyJWT installed:', jwt.__version__)" -``` - -### Troubleshooting - -#### Error: "PyJWT library not found" - -```bash -# Install PyJWT -pip3 install pyjwt - -# Or use system package manager -sudo apt-get install python3-jwt # Debian/Ubuntu -``` - -#### Error: "python3 not found" - -```bash -# Install Python 3 -sudo apt-get install python3 # Debian/Ubuntu -brew install python3 # macOS -``` - -#### Token Validation Fails - -Ensure JWT_SECRET matches your API Gateway configuration: - -```bash -# Check API Gateway secret (default for development) -export JWT_SECRET="test-secret-must-be-at-least-64-characters-long-for-security-validation-ok-1234567890" - -# Generate token with correct secret -TOKEN=$(./jwt_token_generator.sh) -``` - -#### Token Expired - -Default expiration is 1 hour (3600 seconds). Generate fresh token: - -```bash -# Generate new token -TOKEN=$(./jwt_token_generator.sh) - -# Or increase TTL to 24 hours -TOKEN=$(./jwt_token_generator.sh test_user trader "api.access" 86400) -``` - -### Security Notes - -- **Development Only**: The default JWT secret is for testing only -- **Production**: Always use a strong, randomly-generated secret -- **Storage**: Never commit tokens or secrets to version control -- **Expiration**: Use short-lived tokens (15-60 minutes) in production -- **Revocation**: Tokens include `jti` claim for server-side revocation - -### Related Files - -- **API Gateway Auth**: `services/api_gateway/src/auth/interceptor.rs` -- **JWT Service**: `services/api_gateway/src/auth/jwt/service.rs` -- **Test Utilities**: `services/api_gateway/tests/common/mod.rs` -- **E2E Tests**: `services/api_gateway/tests/e2e_tests.rs` - -### References - -- JWT Standard: [RFC 7519](https://tools.ietf.org/html/rfc7519) -- PyJWT Documentation: https://pyjwt.readthedocs.io/ -- API Gateway RBAC: `services/api_gateway/src/auth/README.md` diff --git a/testing/integration/smoke_tests/README.md b/testing/integration/smoke_tests/README.md deleted file mode 100644 index 520915d11..000000000 --- a/testing/integration/smoke_tests/README.md +++ /dev/null @@ -1,359 +0,0 @@ -# Smoke Tests - End-to-End System Validation - -## Overview - -Automated smoke test suite for validating complete Foxhunt HFT system functionality after deployment. These tests provide quick validation that all critical components are operational. - -## Test Categories - -### 1. Infrastructure Health (`infrastructure_health.rs`) -Tests core infrastructure services: -- **PostgreSQL**: Connection, schema validation, TimescaleDB extension -- **Redis**: Connection, SET/GET operations, key expiration -- **Vault**: Connectivity and health status -- **InfluxDB**: Ping endpoint and availability -- **Prometheus**: Health endpoint and metrics availability -- **Grafana**: API health check - -### 2. Service Health (`service_health.rs`) -Tests gRPC microservices: -- **Trading Service**: HTTP health check, gRPC connection (port 50052) -- **API Gateway**: gRPC connection (port 50051) -- **Backtesting Service**: gRPC connection (port 50053) - BLOCKED -- **ML Training Service**: gRPC connection (port 50054) - BLOCKED -- **Service Ports**: Validates all services are listening -- **Response Times**: Measures connection latency -- **Metrics Endpoints**: Validates Prometheus exporters (ports 9091-9094) - -### 3. Authentication Flow (`authentication_flow.rs`) -Tests authentication and security: -- **JWT Generation**: Token creation with claims -- **JWT Validation**: Signature verification and claim extraction -- **JWT Expiration**: Expired token rejection -- **JWT Signature**: Invalid signature detection -- **Session Storage**: Redis-based session management -- **Token Revocation**: Revocation list checking -- **Rate Limiting**: Request throttling with Redis - -### 4. Basic Order Flow (`basic_order_flow.rs`) -Tests core trading operations: -- **Order Submission**: Insert orders into PostgreSQL -- **Order Query**: Retrieve order by ID -- **Order Cancellation**: Update order status to cancelled -- **Position Management**: Create and query positions -- **Order History**: Query user's order history -- **Complete Lifecycle**: Full order flow from submission to cleanup - -## Usage - -### Run All Smoke Tests - -```bash -# Using the automated script (recommended) -./run_smoke_tests.sh - -# Using Cargo directly -cargo test --test smoke_tests --features smoke-tests -``` - -### Run Specific Categories - -```bash -# Infrastructure only -./run_smoke_tests.sh --category infrastructure - -# Service health only -./run_smoke_tests.sh --category service - -# Authentication only -./run_smoke_tests.sh --category authentication - -# Order flow only -./run_smoke_tests.sh --category order_flow -``` - -### Fast Mode (Critical Tests Only) - -```bash -# Run only infrastructure and service health checks -./run_smoke_tests.sh --fast -``` - -### Verbose Mode - -```bash -# Run with debug logging -./run_smoke_tests.sh --verbose -``` - -## Environment Variables - -All tests use environment variables with sensible defaults: - -```bash -# Infrastructure -DATABASE_URL=postgresql://foxhunt:foxhunt_dev_password@localhost:5432/foxhunt -REDIS_URL=redis://localhost:6379 -VAULT_ADDR=http://localhost:8200 -INFLUXDB_URL=http://localhost:8086 - -# Services -API_GATEWAY_URL=http://localhost:50051 -TRADING_SERVICE_URL=http://localhost:50052 -BACKTESTING_SERVICE_URL=http://localhost:50053 -ML_TRAINING_SERVICE_URL=http://localhost:50054 - -# Monitoring -PROMETHEUS_URL=http://localhost:9090 -GRAFANA_URL=http://localhost:3000 - -# Authentication -JWT_SECRET=dev_secret_key_change_in_production - -# Logging -RUST_LOG=info # Set to 'debug' for verbose output -``` - -## Known Issues and Blockers - -### Blocked Tests (Agent 96 Findings) - -1. **Backtesting Service** - Configuration issues prevent deployment - - Test marked with `#[ignore]` attribute - - Skips gracefully when service unavailable - -2. **ML Training Service** - Configuration issues prevent deployment - - Test marked with `#[ignore]` attribute - - Skips gracefully when service unavailable - -### Working Tests - -- ✅ Infrastructure Health (PostgreSQL, Redis, Vault, InfluxDB, Prometheus, Grafana) -- ✅ Trading Service Health (confirmed working by Agent 96) -- ✅ API Gateway Health -- ✅ Authentication Flow (JWT, Sessions, Rate Limiting) -- ✅ Basic Order Flow (Database operations) - -## Test Architecture - -### Graceful Failure Handling - -Tests use the `skip_if_unavailable!` macro to handle service unavailability: - -```rust -skip_if_unavailable!("Service Name", { - // Test code here - result -}); -``` - -This allows tests to: -- Pass when services are available -- Skip gracefully when services are down (connection refused, timeout) -- Fail hard for actual test failures - -### Timeout Management - -All tests have configurable timeouts: -- **Standard smoke tests**: 10 seconds -- **Infrastructure tests**: 5 seconds - -Timeouts prevent hanging tests and provide quick feedback. - -### Parallel Execution - -Tests can run in parallel by default, but use `--test-threads=1` for sequential execution when debugging: - -```bash -cargo test --test smoke_tests -- --test-threads=1 --nocapture -``` - -## Integration with CI/CD - -### Docker Deployment Validation - -After deploying with Docker Compose: - -```bash -# 1. Start all services -docker-compose up -d - -# 2. Wait for health checks -docker-compose ps - -# 3. Run smoke tests -./run_smoke_tests.sh - -# 4. Check exit code -if [ $? -eq 0 ]; then - echo "Deployment validated - ready for production" -else - echo "Deployment issues detected - review logs" -fi -``` - -### Kubernetes Deployment Validation - -```bash -# 1. Deploy to cluster -kubectl apply -f k8s/ - -# 2. Wait for pods -kubectl wait --for=condition=ready pod -l app=foxhunt --timeout=300s - -# 3. Port forward services -kubectl port-forward svc/api-gateway 50051:50051 & -kubectl port-forward svc/trading-service 50052:50051 & - -# 4. Run smoke tests -./run_smoke_tests.sh - -# 5. Cleanup port forwards -pkill -f "kubectl port-forward" -``` - -## Metrics and Reporting - -### Test Pass Rate - -The smoke test runner reports: -- Total categories tested -- Passed/failed counts -- Pass percentage -- Exit code (0 = success, 1 = failure) - -### Sample Output - -``` -━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - Foxhunt HFT System - Smoke Test Suite -━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -Environment Configuration: - Database: postgresql://foxhunt:***@localhost:5432/foxhunt - Redis: redis://localhost:6379 - Vault: http://localhost:8200 - API Gateway: http://localhost:50051 - Trading Service: http://localhost:50052 - Log Level: info - -━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ -Testing: Infrastructure Health -━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - -✅ PostgreSQL connection successful (TimescaleDB: true) -✅ Redis connection and operations successful -✅ Vault connectivity successful (status: 200) - -✅ Infrastructure Health - PASSED - -━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - Smoke Test Summary -━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ - - Total Categories: 4 - Passed: 4 - Failed: 0 - - Pass Rate: 100% - -✅ All smoke tests passed! - System is ready for deployment. -``` - -## Future Enhancements - -### Planned Test Categories (Not Implemented) - -5. **Market Data Tests** (blocked by service availability) - - Subscribe to market data stream - - Receive quote updates - - Historical data queries - -6. **ML Inference Tests** (blocked by service availability) - - Model prediction requests - - Feature engineering pipeline - - Inference latency validation (<100ms) - -7. **Compliance Tests** (requires ML service) - - Audit log creation - - Best execution analysis - - Risk check validation - -8. **Advanced Monitoring** (requires full deployment) - - Alert manager connectivity - - Custom dashboard validation - - Log aggregation checks - -## Troubleshooting - -### Common Issues - -1. **Connection Refused** - ```bash - # Check if services are running - docker-compose ps - - # Restart failed services - docker-compose restart trading_service - ``` - -2. **Database Schema Missing** - ```bash - # Run migrations - cargo sqlx migrate run - ``` - -3. **Redis Connection Failed** - ```bash - # Check Redis is running - redis-cli ping - - # Restart Redis - docker-compose restart redis - ``` - -4. **JWT Secret Not Set** - ```bash - # Set JWT secret - export JWT_SECRET="your-secret-key-here" - ``` - -### Debug Mode - -Run tests with full debug output: - -```bash -RUST_LOG=debug ./run_smoke_tests.sh --verbose -``` - -This shows: -- Full test execution flow -- Connection attempts -- Query execution -- Error details - -## Contributing - -When adding new smoke tests: - -1. Create test file in `tests/smoke_tests/` -2. Add module to `tests/smoke_tests/mod.rs` -3. Use `skip_if_unavailable!` macro for graceful failures -4. Add timeout with `with_timeout()` wrapper -5. Update this README with test documentation -6. Update `run_smoke_tests.sh` with new category - -### Test Template - -```rust -#[tokio::test] -async fn test_new_feature() { - let result = with_timeout(async { - // Your test code here - Ok(()) - }).await; - - skip_if_unavailable!("Service Name", { result }); -} -``` diff --git a/testing/load/README.md b/testing/load/README.md index 245d59453..e908bba3a 100644 --- a/testing/load/README.md +++ b/testing/load/README.md @@ -1,464 +1,25 @@ -# Load Tests - Minimal Dependency Crate +# load -**Purpose**: Fast-compiling load tests for Foxhunt Trading Service - -**Compilation Time**: 20-30 seconds (vs 120-180s in original tests/ crate) - -**Dependency Reduction**: 86% (5 deps vs 36 deps) - ---- - -## Quick Start - -### Rust Load Tests (Direct Trading Service - Port 50052) - -```bash -cd tests/load_tests -cargo test --release -- --nocapture -``` - -### Authenticated ghz Load Tests (API Gateway - Port 50051) - -```bash -cd tests/load_tests - -# Quick authentication test (1 request) -./ghz_quick_auth_test.sh - -# Full authenticated load test suite -./ghz_authenticated.sh -``` - ---- +Minimal-dependency load tests for Trading Service throughput validation. Rust gRPC tests (direct port 50052) plus authenticated ghz shell scripts (API Gateway port 50051). ## Test Suites -### 1. Rust Load Tests (Minimal Dependencies) +### Rust Tests (direct trading service) -**Target**: Trading Service direct (port 50052) -**Auth**: Not required (direct backend access) +- `test_1_baseline_latency` -- 1K sequential orders +- `test_2_concurrent_connections` -- 100 clients x 100 orders +- `test_3_sustained_load` (ignored) -- 5 min, 10K orders/sec +- `test_4_database_performance` -- 5K order DB writes +- `test_5_resource_monitoring` -- health + metrics (feature `health-checks`) +- `test_6_production_readiness` -- 50 clients x 200 orders, success >= 99% -#### Run Specific Test +### ghz Scripts (API Gateway with JWT) + +- `ghz_quick_auth_test.sh` -- single authenticated request +- `ghz_authenticated.sh` -- baseline/medium/high/sustained scenarios + +## Usage ```bash -# Baseline latency (1000 sequential orders) -cargo test --release test_1_baseline_latency -- --nocapture - -# Concurrent connections (100 clients, 100 orders each) -cargo test --release test_2_concurrent_connections -- --nocapture - -# Database performance (5000 orders) -cargo test --release test_4_database_performance -- --nocapture - -# Resource monitoring (health + metrics) -cargo test --release test_5_resource_monitoring -- --nocapture - -# Production readiness assessment -cargo test --release test_6_production_readiness -- --nocapture +cargo test -p load --release -- --nocapture ``` - -#### Run Sustained Load Test (Ignored by Default) - -```bash -# 5-minute sustained load (50 clients, 200 orders/sec each = 10K total) -cargo test --release test_3_sustained_load -- --ignored --nocapture -``` - -### 2. Authenticated ghz Load Tests (Shell Scripts) - -**Target**: API Gateway (port 50051) -**Auth**: JWT tokens (auto-generated) -**Protocol**: gRPC with metadata - -#### Prerequisites - -1. **Install ghz** (if not already installed): -```bash -# Ubuntu/Debian -wget https://github.com/bojand/ghz/releases/download/v0.117.0/ghz-linux-x86_64.tar.gz -tar -xzf ghz-linux-x86_64.tar.gz -sudo mv ghz /usr/local/bin/ - -# MacOS -brew install ghz - -# Arch Linux -yay -S ghz -``` - -2. **Install jq** (optional, for result parsing): -```bash -sudo apt-get install jq # Ubuntu/Debian -brew install jq # MacOS -``` - -3. **Start API Gateway**: -```bash -docker-compose up -d api_gateway postgres trading_service -``` - -4. **Configure JWT Secret** (already in .env): -```bash -# Verify JWT_SECRET is set -grep JWT_SECRET .env -``` - -#### Available Scripts - -##### Quick Authentication Test -```bash -# Verify JWT auth works (1 request only) -./ghz_quick_auth_test.sh -``` - -**Output**: Single authenticated request to validate setup - -##### Full Authenticated Load Suite -```bash -# Run all 4 test scenarios (baseline, medium, high, sustained) -./ghz_authenticated.sh -``` - -**Test Scenarios**: -1. **Baseline**: 1,000 requests @ 100 RPS (10 concurrent) -2. **Medium**: 5,000 requests @ 500 RPS (50 concurrent) -3. **High**: 10,000 requests @ 1,000 RPS (100 concurrent) -4. **Sustained**: 2 minutes @ 500 RPS (60,000 total requests) - -**Output Files**: `results/baseline_authenticated_*.json`, etc. - -#### JWT Token Generation - -The scripts automatically generate JWT tokens using: -```bash -# Manual token generation (if needed) -./tests/e2e_helpers/jwt_token_generator.sh [username] [role] - -# Example -./tests/e2e_helpers/jwt_token_generator.sh "load_test_user" "trader" -``` - -**Token Features**: -- 1-hour expiration -- Includes trading permissions (submit_order, view_positions, cancel_order) -- Signed with JWT_SECRET from .env -- Includes jti, role, sub fields (required by API Gateway) - -#### Results Analysis - -**JSON Output** (with jq installed): -```bash -# View summary of latest test -jq '.' tests/load_tests/results/baseline_authenticated_*.json | tail -1 -``` - -**Metrics Collected**: -- Total requests -- Success rate (%) -- P50, P95, P99 latency (ms) -- Throughput (req/s) -- Error distribution - -**Monitoring Endpoints**: -- Prometheus: http://localhost:9091/metrics -- Grafana: http://localhost:3000 - ---- - -## Prerequisites - -### Infrastructure Running - -```bash -# For Rust tests (Trading Service direct) -docker-compose up -d postgres trading_service - -# For ghz tests (API Gateway) -docker-compose up -d postgres trading_service api_gateway - -# Verify services healthy -docker-compose ps -``` - -### Service Endpoints - -| Service | Protocol | Port | Auth | Used By | -|---------|----------|------|------|---------| -| Trading Service | gRPC | 50052 | No | Rust tests | -| API Gateway | gRPC | 50051 | JWT | ghz scripts | -| Health (Trading) | HTTP | 8081 | No | test_5 | -| Metrics (Trading) | HTTP | 9092 | No | test_5 | -| Metrics (Gateway) | HTTP | 9091 | No | Monitoring | - ---- - -## Test Details - -### Rust Test Suite - -#### Test 1: Baseline Latency -- **Orders**: 1,000 sequential -- **Purpose**: Single-client latency baseline -- **Metrics**: P50, P95, P99 latency + throughput - -#### Test 2: Concurrent Connections -- **Clients**: 100 concurrent -- **Orders per client**: 100 -- **Total orders**: 10,000 -- **Purpose**: Concurrency stress test -- **Metrics**: Latency distribution + success rate - -#### Test 3: Sustained Load (Ignored) -- **Duration**: 5 minutes -- **Clients**: 50 concurrent -- **Target rate**: 10,000 orders/sec total -- **Purpose**: Sustained load validation -- **Metrics**: Long-term stability - -#### Test 4: Database Performance -- **Orders**: 5,000 -- **Purpose**: Database write throughput -- **Target**: >2,000 writes/sec - -#### Test 5: Resource Monitoring -- **Purpose**: Health + metrics validation -- **Checks**: HTTP health endpoint, Prometheus metrics -- **Requires**: `health-checks` feature - -#### Test 6: Production Readiness -- **Clients**: 50 concurrent -- **Orders per client**: 200 -- **Total orders**: 10,000 -- **Criteria**: - - Success rate >= 99% - - Throughput >= 5,000 orders/sec - - P99 latency < 100ms - -### ghz Authenticated Test Suite - -#### Test 1: Baseline Authenticated Load -- **Requests**: 1,000 -- **RPS**: 100 -- **Concurrency**: 10 -- **Purpose**: Verify JWT auth + baseline latency -- **Expected**: 100% success, <50ms P99 - -#### Test 2: Medium Authenticated Load -- **Requests**: 5,000 -- **RPS**: 500 -- **Concurrency**: 50 -- **Purpose**: Medium load with authentication -- **Expected**: >99% success, <100ms P99 - -#### Test 3: High Authenticated Load -- **Requests**: 10,000 -- **RPS**: 1,000 -- **Concurrency**: 100 -- **Purpose**: High throughput with JWT overhead -- **Expected**: >95% success, <150ms P99 - -#### Test 4: Sustained Authenticated Load -- **Duration**: 2 minutes -- **RPS**: 500 -- **Concurrency**: 50 -- **Total**: ~60,000 requests -- **Purpose**: Long-term stability validation -- **Expected**: >99% success, stable latency - ---- - -## Features - -### Default (No Features) -- Core gRPC load testing (tests 1-4, 6) -- Dependencies: `tokio`, `tonic`, `uuid` - -### `health-checks` (Optional) -```bash -cargo test --release --features health-checks -``` -- Enables test_5 (resource monitoring) -- Adds `reqwest` dependency -- HTTP health + metrics checks - ---- - -## Performance Targets - -| Metric | Target | Typical (Direct) | Typical (Gateway) | -|--------|--------|------------------|-------------------| -| Success Rate | >= 99% | 99.5-100% | 99-100% | -| Throughput | >= 5K orders/sec | 7-10K | 5-7K | -| P50 Latency | < 20ms | 10-15ms | 15-25ms | -| P99 Latency | < 100ms | 30-50ms | 50-100ms | -| DB Writes/sec | >= 2K | 2.5-3K | 2-2.5K | - -**Note**: API Gateway adds ~5-10ms latency due to JWT validation and proxying. - ---- - -## Troubleshooting - -### "Connection refused" Error - -**For Rust tests (port 50052)**: -```bash -docker-compose up -d trading_service -docker-compose ps # Verify "Up" status -``` - -**For ghz tests (port 50051)**: -```bash -docker-compose up -d api_gateway -docker-compose ps # Verify "Up" status -``` - -### "Failed to generate JWT token" - -**Check JWT_SECRET**: -```bash -# Verify secret exists -grep JWT_SECRET .env - -# If missing, add to .env -echo 'JWT_SECRET=your-secret-key-here' >> .env -``` - -### "Too many open files" Error -```bash -ulimit -n 4096 # Increase file descriptor limit -``` - -### Authentication Failures (401 errors) - -**Check token format**: -```bash -# Generate test token -./tests/e2e_helpers/jwt_token_generator.sh test_user trader - -# Verify token has 3 parts (header.payload.signature) -``` - -**Check API Gateway logs**: -```bash -docker-compose logs api_gateway | grep -i "auth\|jwt\|401" -``` - -### High Latency - -**Check**: -1. PostgreSQL synchronous_commit setting -2. Network latency (localhost vs Docker) -3. System load (CPU, memory) -4. API Gateway JWT validation overhead - -**Optimize PostgreSQL**: -```sql --- In PostgreSQL -ALTER SYSTEM SET synchronous_commit = off; -SELECT pg_reload_conf(); -``` - ---- - -## Compilation Time Comparison - -| Crate | Dependencies | Compile Time | Speedup | -|-------|--------------|--------------|---------| -| `tests/` (original) | 36 | 120-180s | Baseline | -| `tests/load_tests` | 5 | 20-30s | **6x faster** | -| ghz scripts | N/A | 0s | **Instant** | - ---- - -## Architecture - -### Rust Tests (Minimal Dependencies) -```toml -[dependencies] -tokio = { workspace = true } # Async runtime -tonic = { workspace = true } # gRPC client -tonic-prost = { workspace = true } # Protobuf runtime -prost = { workspace = true } # Protobuf types -uuid = { workspace = true } # Order IDs -reqwest = { optional = true } # HTTP (feature-gated) -``` - -### ghz Scripts (Shell + OpenSSL) -```bash -# Dependencies -- bash -- ghz (gRPC load testing) -- openssl (JWT signing) -- jq (optional, result parsing) -- nc (netcat, connectivity check) -``` - -### Build Process -1. `build.rs` compiles `trading.proto` from Trading Service -2. Generated code included via `tonic::include_proto!("trading")` -3. No heavy dependencies (ML, database clients, test frameworks) - ---- - -## CI/CD Integration - -### GitHub Actions -```yaml -- name: Run Rust Load Tests - run: | - docker-compose up -d postgres trading_service - cd tests/load_tests - cargo test --release --features health-checks - -- name: Run Authenticated ghz Tests - run: | - docker-compose up -d api_gateway postgres trading_service - cd tests/load_tests - ./ghz_quick_auth_test.sh - ./ghz_authenticated.sh -``` - -### GitLab CI -```yaml -rust_load_tests: - script: - - docker-compose up -d postgres trading_service - - cd tests/load_tests - - cargo test --release --features health-checks - -ghz_load_tests: - script: - - docker-compose up -d api_gateway postgres trading_service - - cd tests/load_tests - - ./ghz_authenticated.sh -``` - ---- - -## Related Documentation - -- [LOAD_TEST_DEPENDENCY_OPTIMIZATION.md](../../LOAD_TEST_DEPENDENCY_OPTIMIZATION.md) - Detailed analysis -- [LOAD_TEST_OPTIMIZATION_SUMMARY.md](../../LOAD_TEST_OPTIMIZATION_SUMMARY.md) - Implementation summary -- [TESTING_PLAN.md](../../TESTING_PLAN.md) - Overall testing strategy -- [WAVE_132_AUTH_VALIDATION_SUMMARY.md](../../WAVE_132_AUTH_VALIDATION_SUMMARY.md) - JWT authentication validation - ---- - -## Summary - -| Test Type | Target | Auth | Compilation | Execution | Use Case | -|-----------|--------|------|-------------|-----------|----------| -| Rust Tests | Trading Service (50052) | No | 20-30s | Fast | Backend performance | -| ghz Scripts | API Gateway (50051) | JWT | 0s | Fast | End-to-end auth flow | - -**Recommendation**: Use **both** test types for comprehensive validation: -1. **Rust tests** for backend performance benchmarks -2. **ghz scripts** for authenticated API Gateway validation - ---- - -**Status**: ✅ Production Ready -**Rust Tests**: < 30 seconds compilation ✅ -**ghz Scripts**: Instant execution ✅ -**JWT Authentication**: Fully validated ✅ diff --git a/testing/service-integration/README.md b/testing/service-integration/README.md new file mode 100644 index 000000000..27bcb5849 --- /dev/null +++ b/testing/service-integration/README.md @@ -0,0 +1,9 @@ +# service-integration + +Cross-service integration tests for gRPC service interactions. + +## Usage + +```bash +SQLX_OFFLINE=true cargo test -p service-integration --lib +``` diff --git a/testing/service-load/README.md b/testing/service-load/README.md index 2f40cc3aa..d4855298b 100644 --- a/testing/service-load/README.md +++ b/testing/service-load/README.md @@ -1,216 +1,17 @@ -# Load Tests - Trading Service Throughput Validation +# service-load -## Overview +Trading service throughput load tests with 4 scenarios: sustained (10K/s for 60s), peak burst (50K/s for 10s), market data streaming (1M updates), and connection pool saturation (1K clients). -Comprehensive load testing suite for validating trading service throughput and performance under various load scenarios. +## Scenarios -## Test Scenarios - -### 1. Sustained Load (10,000 orders/sec for 60s) -- **Target**: 10,000 orders/second sustained throughput -- **Duration**: 60 seconds -- **Concurrent Clients**: 100 -- **Validates**: System stability under sustained load - -### 2. Peak Burst (50,000 orders/sec for 10s) -- **Target**: 50,000 orders/second peak burst -- **Duration**: 10 seconds -- **Concurrent Clients**: 500 -- **Validates**: System behavior under peak load spikes - -### 3. Market Data Streaming (1M updates) -- **Target**: 1,000,000 concurrent market data updates -- **Streams**: 1,000 concurrent streams -- **Duration**: 30 seconds -- **Validates**: Streaming infrastructure capacity - -### 4. Connection Pool Saturation (1,000 clients) -- **Target**: 1,000 concurrent clients -- **Requests per Client**: 100 -- **Validates**: Connection pool management and resource limits +- `sustained` -- 10K orders/sec, 60s, 100 clients +- `burst` -- 50K orders/sec, 10s, 500 clients +- `streaming` -- 1M market data updates, 1K streams +- `pool` -- 1K concurrent clients, 100 requests each ## Usage -### Run All Tests ```bash cargo run -p load_tests --release -- --scenario all -``` - -### Run Individual Scenarios -```bash -# Sustained load cargo run -p load_tests --release -- --scenario sustained - -# Peak burst -cargo run -p load_tests --release -- --scenario burst - -# Streaming -cargo run -p load_tests --release -- --scenario streaming - -# Connection pool -cargo run -p load_tests --release -- --scenario pool ``` - -### Custom Configuration -```bash -cargo run -p load_tests --release -- \ - --scenario sustained \ - --url http://trading-service:50052 \ - --output /path/to/report.md \ - --verbose -``` - -## Metrics Collected - -### Throughput Metrics -- Requests per second (sustained and peak) -- Total requests processed -- Success/failure rates - -### Latency Distribution -- P50 (median) latency -- P95 latency -- P99 latency -- Maximum latency - -### Resource Usage -- Memory consumption (average) -- Connection pool utilization -- Stream management overhead - -## Output Report - -Test results are saved as Markdown reports containing: -- Executive summary -- Detailed metrics breakdown -- Latency distribution charts -- Resource usage analysis -- Performance recommendations - -Default output: `/tmp/WAVE_120_AGENT_5_LOAD_TESTING.md` - -## Prerequisites - -1. **Trading Service Running**: - ```bash - docker-compose up -d trading_service - # OR - cargo run -p trading_service - ``` - -2. **Database Available**: - ```bash - docker-compose up -d postgres redis - ``` - -3. **Sufficient System Resources**: - - 8GB+ RAM recommended - - Multi-core CPU for parallel clients - - Network bandwidth for 50k+ req/sec - -## Architecture - -### Components - -- **Scenarios**: Test scenario implementations - - `sustained_load.rs`: 10k orders/sec for 60s - - `burst_load.rs`: 50k orders/sec for 10s - - `streaming_load.rs`: 1M market data updates - - `pool_saturation.rs`: 1000 concurrent clients - - `comprehensive.rs`: All scenarios sequentially - -- **Clients**: gRPC client implementations - - `trading_client.rs`: Trading service client wrapper - -- **Metrics**: Performance measurement - - `metrics.rs`: HDR histogram-based metrics collection - - `monitor.rs`: System resource monitoring - -### Load Generation Pattern - -```rust -// Concurrent client pattern -for client_id in 0..NUM_CLIENTS { - tokio::spawn(async move { - let client = TradingClient::connect(url).await?; - - // Submit orders with rate limiting - while duration_remaining { - client.submit_order(...).await?; - tokio::time::sleep(rate_limit).await; - } - }); -} -``` - -## Performance Targets - -### Sustained Load -- ✅ Throughput: ≥9,000 req/sec -- ✅ Error Rate: <1% -- ✅ P95 Latency: <10ms - -### Peak Burst -- ✅ Throughput: ≥40,000 req/sec -- ✅ Error Rate: <5% -- ✅ P99 Latency: <50ms - -### Streaming -- ✅ Updates: ≥900k received -- ✅ Concurrent Streams: 1000 -- ✅ Stream Stability: <1% failures - -### Connection Pool -- ✅ Concurrent Connections: 1000 -- ✅ Error Rate: <5% -- ✅ P99 Latency: <100ms - -## Troubleshooting - -### Connection Refused -```bash -# Verify trading service is running -grpc_health_probe -addr=localhost:50052 -``` - -### High Error Rates -- Check system resource limits (ulimit, file descriptors) -- Verify database connection pool size -- Review trading service logs for errors - -### Memory Issues -- Reduce concurrent clients -- Enable connection pooling -- Check for memory leaks in trading service - -## Integration with CI/CD - -```yaml -# .github/workflows/load-test.yml -- name: Run Load Tests - run: | - docker-compose up -d - cargo run -p load_tests --release -- --scenario all - -- name: Upload Report - uses: actions/upload-artifact@v3 - with: - name: load-test-report - path: /tmp/WAVE_120_AGENT_5_LOAD_TESTING.md -``` - -## Wave 120 Objectives - -**Agent 5 Tasks**: -- ✅ Create load_tests package -- ✅ Implement 4 throughput scenarios -- ✅ Measure latency, throughput, error rates -- ✅ Monitor memory usage -- ⏳ Run tests against live service -- ⏳ Generate performance report - -**Expected Outcomes**: -- Validate 10k orders/sec sustained capacity -- Confirm 50k orders/sec peak burst handling -- Verify 1M concurrent stream updates -- Validate 1000+ concurrent client support diff --git a/testing/test-common/README.md b/testing/test-common/README.md index e85a979a4..ed3115d69 100644 --- a/testing/test-common/README.md +++ b/testing/test-common/README.md @@ -1,282 +1,30 @@ -# Test Common Crate +# test-common -Shared test fixtures, builders, and utilities for the Foxhunt project. +Shared test fixtures, builders, and assertions for the Foxhunt workspace. Consolidates duplicate test code across 39+ files. -This crate consolidates **duplicate test code across 39+ test files**, saving approximately **3,500+ LOC** and significantly improving test maintainability. +## Key Types -## 📦 What's Included +- `TestDb` -- isolated test database with automatic cleanup +- `MockOrderBuilder` -- fluent order/trade builder +- `BarBuilder` -- OHLCV bar generation with realistic price movements +- `PositionBuilder` -- mock positions with P&L calculations -### Fixtures (`fixtures/`) +## Fixtures -#### Database (`database.rs`) -- **TestDb**: Isolated test database with automatic schema cleanup -- **setup_test_data()**: Pre-populate test data -- Pattern consolidates **39 duplicate `setup_test_db()` functions** +- `generate_ohlcv_bars()` -- realistic OHLCV data +- `generate_random_walk()` -- Monte Carlo price simulations +- `generate_crisis_returns()` -- 2008 crisis patterns +- `generate_order_book()` -- order book levels -```rust -use test_common::TestDb; +## Assertions -#[tokio::test] -async fn my_test() { - let db = TestDb::with_migrations().await; - let result = sqlx::query("SELECT * FROM users") - .fetch_all(db.pool()) - .await; - // Test continues... -} -``` +- `assert_approx_eq!` -- percentage tolerance comparison +- `assert_ohlc_valid` -- OHLC bar validation +- `assert_var_exceedances` -- VaR exceedance checks -#### Orders (`orders.rs`) -- **MockOrderBuilder**: Fluent API for creating test orders -- **MockTradeBuilder**: Fluent API for creating test trades -- Pattern consolidates **22 duplicate `create_mock_order()` functions** - -```rust -use test_common::MockOrderBuilder; - -let order = MockOrderBuilder::new() - .symbol("TSLA") - .sell() - .quantity(50.0) - .limit_price(200.0) - .filled() - .build(); -``` - -#### Market Data (`market_data.rs`) -- **generate_ohlcv_bars()**: Realistic OHLCV data with price movements -- **generate_random_walk()**: Monte Carlo price simulations -- **generate_crisis_returns()**: 2008 Financial Crisis patterns -- **generate_covid_crash_returns()**: March 2020 patterns -- **generate_order_book()**: Order book levels -- Pattern consolidates **15+ duplicate bar/tick generators** - -```rust -use test_common::generate_ohlcv_bars; - -let bars = generate_ohlcv_bars(100, 150.0); -assert_eq!(bars.len(), 100); -``` - -#### Config (`config.rs`) -- **TestConfig**: General test configuration -- **MLTestConfig**: ML training test config -- **RiskTestConfig**: Risk engine test config - -```rust -use test_common::fixtures::config::RiskTestConfig; - -let config = RiskTestConfig::conservative(); -assert_eq!(config.circuit_breaker_threshold, 0.05); -``` - -#### Network (`network.rs`) -- **MockHttpResponse**: HTTP response builder -- **mock_market_data_response()**: API response fixtures -- **mock_websocket_*_message()**: WebSocket message fixtures - -### Builders (`builders/`) - -#### BarBuilder (`bar_builder.rs`) -Fluent API for creating OHLCV bars with realistic data: - -```rust -use test_common::BarBuilder; - -let bars = BarBuilder::new() - .price(150.0) - .bullish() - .build_series(100, 5); // 100 bars, 5 minutes apart -``` - -#### OrderBuilder (`order_builder.rs`) -Re-export of `MockOrderBuilder` for convenience. - -#### PositionBuilder (`position_builder.rs`) -Create mock positions with P&L calculations: - -```rust -use test_common::PositionBuilder; - -let pos = PositionBuilder::new() - .symbol("NVDA") - .long(100.0) - .entry_price(450.0) - .profitable(10.0) // 10% profit - .build(); - -assert_eq!(pos.pnl(), 4500.0); -``` - -### Assertions (`assertions/`) - -Domain-specific assertions for financial testing: - -```rust -use test_common::assert_approx_eq; -use test_common::assertions::assert_ohlc_valid; - -// Percentage tolerance -assert_approx_eq!(100.0, 101.0, 0.02); // 2% tolerance - -// OHLC validation -assert_ohlc_valid(open, high, low, close); - -// VaR exceedances -assert_var_exceedances(&returns, var, 0.99, 0.1); - -// P&L validation -assert_pnl(entry_price, current_price, quantity, expected, 0.01); -``` - -## 🚀 Quick Start - -### Add to your test crate - -Update your crate's `Cargo.toml`: +## Usage ```toml [dev-dependencies] -test_common = { path = "../../tests/test_common" } +test_common = { path = "../../testing/test-common" } ``` - -### Import and use - -```rust -use test_common::{TestDb, MockOrderBuilder, generate_ohlcv_bars}; - -#[tokio::test] -async fn comprehensive_test() { - // Setup database - let db = TestDb::with_migrations().await; - - // Create test data - let bars = generate_ohlcv_bars(100, 150.0); - let order = MockOrderBuilder::new() - .symbol("AAPL") - .buy() - .build(); - - // Run your tests... -} -``` - -## 📊 Impact Analysis - -### LOC Savings by Pattern - -| Pattern | Files | Avg LOC/File | Total Saved | -|---------|-------|--------------|-------------| -| `setup_test_db()` | 39 | 15 | 585 | -| `create_mock_order()` | 22 | 25 | 550 | -| `create_mock_bars()` | 15 | 35 | 525 | -| Database setup helpers | 39 | 20 | 780 | -| Market data generators | 15 | 30 | 450 | -| Config builders | 10 | 15 | 150 | -| Mock responses | 8 | 20 | 160 | -| Custom assertions | 12 | 18 | 216 | -| **TOTAL** | **160+** | **~22** | **~3,416** | - -### Files Consolidated - -#### ML Training Service Tests (15 files) -- `checkpoint_manager_tests.rs` -- `job_tracker_test.rs` -- `integration_tests.rs` -- `model_lifecycle_edge_cases.rs` -- `training_error_recovery_tests.rs` -- And 10+ more integration tests - -#### Risk Tests (18 files) -- `risk_comprehensive_tests.rs` (1,197 LOC) -- `risk_var_calculations_tests.rs` (855 LOC) -- `portfolio_optimization_tests.rs` (1,002 LOC) -- And 15+ more risk/compliance tests - -#### ML Tests (10+ files) -- `feature_cache_tests.rs` -- `dqn_feature_cache_test.rs` -- `inference_optimization_tests.rs` -- And more - -## 🔧 Migration Guide - -### Before (Duplicate Code) - -```rust -// In every test file: -async fn setup_test_db() -> PgPool { - let database_url = std::env::var("DATABASE_URL") - .unwrap_or_else(|_| "postgresql://...".to_string()); - PgPool::connect(&database_url).await.unwrap() -} - -fn create_mock_bars(count: usize) -> Vec { - let mut bars = Vec::new(); - // 30+ lines of duplicate bar generation... - bars -} - -#[tokio::test] -async fn my_test() { - let pool = setup_test_db().await; - let bars = create_mock_bars(100); - // Test logic... -} -``` - -### After (Using test_common) - -```rust -use test_common::{TestDb, generate_ohlcv_bars}; - -#[tokio::test] -async fn my_test() { - let db = TestDb::with_migrations().await; - let bars = generate_ohlcv_bars(100, 150.0); - // Test logic... -} -``` - -**Result**: Reduced from ~60 LOC to ~8 LOC per test file! - -## 📈 Benefits - -1. **Maintainability**: Update fixtures once, benefit everywhere -2. **Consistency**: All tests use identical setup patterns -3. **Discoverability**: Single location for test utilities -4. **Type Safety**: Builder patterns prevent invalid test data -5. **Documentation**: Well-documented patterns and examples -6. **Speed**: Pre-compiled fixtures load faster than copies - -## 🧪 Test Coverage - -The `test_common` crate itself has **90%+ test coverage**: - -```bash -cd tests/test_common -cargo test -``` - -All fixtures and builders include their own unit tests to ensure correctness. - -## 📝 Contributing - -When adding new test patterns: - -1. Check if similar code exists in 3+ test files -2. Extract to appropriate module in `test_common` -3. Add builder pattern if applicable -4. Include unit tests -5. Document with examples -6. Update this README - -## 🔗 Related Documentation - -- [Risk Tests](../../risk/tests/README.md) -- [ML Tests](../../ml/tests/README.md) -- [Database Schema](../../migrations/README.md) - -## 📜 License - -Part of the Foxhunt trading system. Same license as parent project.