9c3d741a08
Move 17 library crates into crates/, CLI binary into bin/fxt, consolidate 10 test crates into testing/, split config crate from deployment config files. Root directory reduced from 38+ to ~17 directories. All Cargo.toml paths and build.rs proto refs updated. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
875 lines
32 KiB
Rust
875 lines
32 KiB
Rust
//! ML Trading Pipeline Integration Tests
|
|
//!
|
|
//! Tests comprehensive integration of ML models with trading pipeline.
|
|
//! Validates end-to-end flow from market data to ML inference to trading decisions.
|
|
//!
|
|
//! Coverage Areas:
|
|
//! - Market data preprocessing for ML features
|
|
//! - Real-time ML inference pipeline
|
|
//! - ML model predictions to trading signals
|
|
//! - Feature engineering and data pipeline
|
|
//! - Model performance under market stress
|
|
//! - Latency optimization for HFT requirements
|
|
|
|
use std::sync::Arc;
|
|
use std::time::Duration;
|
|
use tokio::time::timeout;
|
|
use std::collections::HashMap;
|
|
|
|
// Import core types and modules
|
|
use trading_engine::{
|
|
timing::HardwareTimestamp,
|
|
types::prelude::*,
|
|
simd::SimdPriceOps,
|
|
};
|
|
|
|
/// Test result type for safe error handling (no panics)
|
|
type TestResult<T> = Result<T, Box<dyn std::error::Error + Send + Sync>>;
|
|
|
|
/// ML pipeline configuration for testing
|
|
#[derive(Debug, Clone)]
|
|
pub struct MLPipelineConfig {
|
|
pub feature_window_size: usize,
|
|
pub prediction_horizon_ms: u64,
|
|
pub confidence_threshold: f64,
|
|
pub model_inference_timeout_ms: u64,
|
|
pub max_prediction_latency_ns: u64,
|
|
}
|
|
|
|
impl Default for MLPipelineConfig {
|
|
fn default() -> Self {
|
|
Self {
|
|
feature_window_size: 100,
|
|
prediction_horizon_ms: 1000, // 1 second ahead
|
|
confidence_threshold: 0.7,
|
|
model_inference_timeout_ms: 10, // 10ms max
|
|
max_prediction_latency_ns: 50_000, // 50μs for HFT
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Market data tick for ML processing
|
|
#[derive(Debug, Clone)]
|
|
pub struct MarketTick {
|
|
pub symbol: String,
|
|
pub price: Decimal,
|
|
pub volume: u64,
|
|
pub bid: Decimal,
|
|
pub ask: Decimal,
|
|
pub bid_size: u64,
|
|
pub ask_size: u64,
|
|
pub timestamp: HardwareTimestamp,
|
|
}
|
|
|
|
impl MarketTick {
|
|
pub fn new(symbol: String, price: Decimal, volume: u64) -> Self {
|
|
let spread = Decimal::new(5, 2); // $0.05 spread
|
|
Self {
|
|
symbol,
|
|
price,
|
|
volume,
|
|
bid: price - spread,
|
|
ask: price + spread,
|
|
bid_size: volume / 2,
|
|
ask_size: volume / 2,
|
|
timestamp: HardwareTimestamp::now(),
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Feature vector for ML models
|
|
#[derive(Debug, Clone)]
|
|
pub struct FeatureVector {
|
|
pub features: Vec<f32>,
|
|
pub feature_names: Vec<String>,
|
|
pub extraction_latency_ns: u64,
|
|
pub timestamp: HardwareTimestamp,
|
|
}
|
|
|
|
impl FeatureVector {
|
|
pub fn new(features: Vec<f32>, feature_names: Vec<String>, extraction_latency_ns: u64) -> Self {
|
|
Self {
|
|
features,
|
|
feature_names,
|
|
extraction_latency_ns,
|
|
timestamp: HardwareTimestamp::now(),
|
|
}
|
|
}
|
|
}
|
|
|
|
/// ML model prediction result
|
|
#[derive(Debug, Clone)]
|
|
pub struct MLPrediction {
|
|
pub signal: TradingSignal,
|
|
pub confidence: f64,
|
|
pub probability_distribution: Vec<f64>,
|
|
pub model_name: String,
|
|
pub inference_latency_ns: u64,
|
|
pub timestamp: HardwareTimestamp,
|
|
}
|
|
|
|
#[derive(Debug, Clone)]
|
|
pub enum TradingSignal {
|
|
StrongBuy(f64), // confidence score
|
|
Buy(f64),
|
|
Hold(f64),
|
|
Sell(f64),
|
|
StrongSell(f64),
|
|
}
|
|
|
|
impl TradingSignal {
|
|
pub fn confidence(&self) -> f64 {
|
|
match self {
|
|
TradingSignal::StrongBuy(conf) => *conf,
|
|
TradingSignal::Buy(conf) => *conf,
|
|
TradingSignal::Hold(conf) => *conf,
|
|
TradingSignal::Sell(conf) => *conf,
|
|
TradingSignal::StrongSell(conf) => *conf,
|
|
}
|
|
}
|
|
|
|
pub fn is_actionable(&self, threshold: f64) -> bool {
|
|
self.confidence() >= threshold
|
|
}
|
|
}
|
|
|
|
/// Feature engineering pipeline
|
|
#[derive(Debug)]
|
|
pub struct FeatureEngineer {
|
|
pub config: MLPipelineConfig,
|
|
pub price_history: Arc<std::sync::Mutex<Vec<Decimal>>>,
|
|
pub volume_history: Arc<std::sync::Mutex<Vec<u64>>>,
|
|
pub spread_history: Arc<std::sync::Mutex<Vec<Decimal>>>,
|
|
}
|
|
|
|
impl FeatureEngineer {
|
|
pub fn new(config: MLPipelineConfig) -> Self {
|
|
Self {
|
|
config,
|
|
price_history: Arc::new(std::sync::Mutex::new(Vec::new())),
|
|
volume_history: Arc::new(std::sync::Mutex::new(Vec::new())),
|
|
spread_history: Arc::new(std::sync::Mutex::new(Vec::new())),
|
|
}
|
|
}
|
|
|
|
/// Extract features from market tick with SIMD optimization
|
|
pub async fn extract_features(&self, tick: &MarketTick) -> TestResult<FeatureVector> {
|
|
let start_time = HardwareTimestamp::now();
|
|
|
|
// Update price history
|
|
if let Ok(mut price_hist) = self.price_history.lock() {
|
|
price_hist.push(tick.price);
|
|
if price_hist.len() > 1000 {
|
|
price_hist.remove(0);
|
|
}
|
|
}
|
|
|
|
// Update volume history
|
|
if let Ok(mut volume_hist) = self.volume_history.lock() {
|
|
volume_hist.push(tick.volume);
|
|
if volume_hist.len() > 1000 {
|
|
volume_hist.remove(0);
|
|
}
|
|
}
|
|
|
|
// Calculate spread
|
|
let spread = tick.ask - tick.bid;
|
|
if let Ok(mut spread_hist) = self.spread_history.lock() {
|
|
spread_hist.push(spread);
|
|
if spread_hist.len() > 1000 {
|
|
spread_hist.remove(0);
|
|
}
|
|
}
|
|
|
|
// Extract technical indicators using SIMD
|
|
let simd_start = HardwareTimestamp::now();
|
|
let features = self.calculate_technical_features(tick).await?;
|
|
let simd_latency = HardwareTimestamp::now().latency_ns(&simd_start);
|
|
|
|
// SIMD feature calculation should be sub-microsecond
|
|
if simd_latency > 1_000 {
|
|
eprintln!("WARNING: SIMD feature calculation took {}ns, expected <1000ns", simd_latency);
|
|
}
|
|
|
|
let total_latency = HardwareTimestamp::now().latency_ns(&start_time);
|
|
|
|
let feature_names = vec![
|
|
"price".to_string(),
|
|
"volume".to_string(),
|
|
"spread".to_string(),
|
|
"price_sma_10".to_string(),
|
|
"price_sma_20".to_string(),
|
|
"volume_sma_10".to_string(),
|
|
"price_momentum".to_string(),
|
|
"volume_momentum".to_string(),
|
|
"spread_normalized".to_string(),
|
|
"volatility_1min".to_string(),
|
|
];
|
|
|
|
Ok(FeatureVector::new(features, feature_names, total_latency))
|
|
}
|
|
|
|
/// Calculate technical features using SIMD operations
|
|
async fn calculate_technical_features(&self, tick: &MarketTick) -> TestResult<Vec<f32>> {
|
|
let mut features = Vec::with_capacity(10);
|
|
|
|
// Basic features
|
|
features.push(tick.price.to_f32().unwrap_or(0.0));
|
|
features.push(tick.volume as f32);
|
|
features.push((tick.ask - tick.bid).to_f32().unwrap_or(0.0));
|
|
|
|
// Moving averages using SIMD (simulated)
|
|
let recent_prices = self.get_recent_prices(20).await?;
|
|
if recent_prices.len() >= 10 {
|
|
let simd_ops = if std::arch::is_x86_feature_detected!("avx2") {
|
|
// SAFETY: AVX2 feature detection verified before SIMD operations
|
|
unsafe { Some(SimdPriceOps::new()) }
|
|
} else {
|
|
None
|
|
};
|
|
|
|
let sma_10 = if let Some(ref ops) = simd_ops {
|
|
ops.calculate_vwap(&recent_prices[..10], &vec![1.0; 10])
|
|
} else {
|
|
recent_prices[..10].iter().sum::<Decimal>() / Decimal::new(10, 0)
|
|
};
|
|
features.push(sma_10.to_f32().unwrap_or(0.0));
|
|
|
|
if recent_prices.len() >= 20 {
|
|
let sma_20 = if let Some(ref ops) = simd_ops {
|
|
ops.calculate_vwap(&recent_prices, &vec![1.0; recent_prices.len()])
|
|
} else {
|
|
recent_prices.iter().sum::<Decimal>() / Decimal::new(recent_prices.len() as i64, 0)
|
|
};
|
|
features.push(sma_20.to_f32().unwrap_or(0.0));
|
|
} else {
|
|
features.push(0.0);
|
|
}
|
|
} else {
|
|
features.push(0.0);
|
|
features.push(0.0);
|
|
}
|
|
|
|
// Volume SMA
|
|
let recent_volumes = self.get_recent_volumes(10).await?;
|
|
if !recent_volumes.is_empty() {
|
|
let volume_avg = recent_volumes.iter().sum::<u64>() as f32 / recent_volumes.len() as f32;
|
|
features.push(volume_avg);
|
|
} else {
|
|
features.push(0.0);
|
|
}
|
|
|
|
// Momentum indicators
|
|
if recent_prices.len() >= 5 {
|
|
let price_momentum = (recent_prices[0] - recent_prices[4]).to_f32().unwrap_or(0.0);
|
|
features.push(price_momentum);
|
|
} else {
|
|
features.push(0.0);
|
|
}
|
|
|
|
if recent_volumes.len() >= 5 {
|
|
let volume_momentum = recent_volumes[0] as f32 - recent_volumes[4] as f32;
|
|
features.push(volume_momentum);
|
|
} else {
|
|
features.push(0.0);
|
|
}
|
|
|
|
// Normalized spread
|
|
let spread_normalized = if tick.price > Decimal::ZERO {
|
|
((tick.ask - tick.bid) / tick.price).to_f32().unwrap_or(0.0)
|
|
} else {
|
|
0.0
|
|
};
|
|
features.push(spread_normalized);
|
|
|
|
// Volatility calculation using SIMD
|
|
if recent_prices.len() >= 10 {
|
|
let volatility = self.calculate_volatility(&recent_prices[..10])?;
|
|
features.push(volatility);
|
|
} else {
|
|
features.push(0.0);
|
|
}
|
|
|
|
Ok(features)
|
|
}
|
|
|
|
async fn get_recent_prices(&self, count: usize) -> TestResult<Vec<Decimal>> {
|
|
let mut prices = Vec::new();
|
|
// Simulate reading from ring buffer
|
|
// In real implementation, would use actual ring buffer data
|
|
for i in 0..count.min(20) {
|
|
let price = Decimal::new(150_00 + i as i64, 2); // Simulated price data
|
|
prices.push(price);
|
|
}
|
|
Ok(prices)
|
|
}
|
|
|
|
async fn get_recent_volumes(&self, count: usize) -> TestResult<Vec<u64>> {
|
|
let mut volumes = Vec::new();
|
|
// Simulate reading from ring buffer
|
|
for i in 0..count.min(10) {
|
|
volumes.push(1000 + (i * 100) as u64); // Simulated volume data
|
|
}
|
|
Ok(volumes)
|
|
}
|
|
|
|
fn calculate_volatility(&self, prices: &[Decimal]) -> TestResult<f32> {
|
|
if prices.is_empty() {
|
|
return Ok(0.0);
|
|
}
|
|
|
|
let mean = prices.iter().sum::<Decimal>() / Decimal::new(prices.len() as i64, 0);
|
|
let variance = prices.iter()
|
|
.map(|&p| {
|
|
let diff = p - mean;
|
|
(diff * diff).to_f32().unwrap_or(0.0)
|
|
})
|
|
.sum::<f32>() / prices.len() as f32;
|
|
|
|
Ok(variance.sqrt())
|
|
}
|
|
}
|
|
|
|
/// Mock ML model for testing
|
|
#[derive(Debug, Clone)]
|
|
pub struct MockMLModel {
|
|
pub model_name: String,
|
|
pub config: MLPipelineConfig,
|
|
pub inference_stats: Arc<std::sync::Mutex<Vec<u64>>>,
|
|
}
|
|
|
|
impl MockMLModel {
|
|
pub fn new(model_name: String, config: MLPipelineConfig) -> Self {
|
|
Self {
|
|
model_name,
|
|
config,
|
|
inference_stats: Arc::new(std::sync::Mutex::new(Vec::new())),
|
|
}
|
|
}
|
|
|
|
/// Run ML inference on feature vector
|
|
pub async fn predict(&self, features: &FeatureVector) -> TestResult<MLPrediction> {
|
|
let start_time = HardwareTimestamp::now();
|
|
|
|
// Validate feature vector
|
|
if features.features.len() < 5 {
|
|
return Err(format!(
|
|
"Insufficient features: got {}, expected at least 5",
|
|
features.features.len()
|
|
).into());
|
|
}
|
|
|
|
// Simulate model inference latency (should be <50μs for HFT)
|
|
tokio::time::sleep(Duration::from_nanos(30_000)).await; // 30μs
|
|
|
|
// Generate prediction based on features
|
|
let prediction = self.generate_prediction(&features.features)?;
|
|
|
|
let inference_latency = HardwareTimestamp::now().latency_ns(&start_time);
|
|
|
|
// Record latency statistics
|
|
if let Ok(mut stats) = self.inference_stats.lock() {
|
|
stats.push(inference_latency);
|
|
}
|
|
|
|
// Validate HFT latency requirement
|
|
if inference_latency > self.config.max_prediction_latency_ns {
|
|
eprintln!("WARNING: ML inference took {}ns, exceeds limit {}ns",
|
|
inference_latency, self.config.max_prediction_latency_ns);
|
|
}
|
|
|
|
Ok(MLPrediction {
|
|
signal: prediction.0,
|
|
confidence: prediction.1,
|
|
probability_distribution: prediction.2,
|
|
model_name: self.model_name.clone(),
|
|
inference_latency_ns: inference_latency,
|
|
timestamp: HardwareTimestamp::now(),
|
|
})
|
|
}
|
|
|
|
fn generate_prediction(&self, features: &[f32]) -> TestResult<(TradingSignal, f64, Vec<f64>)> {
|
|
// Simple heuristic model for testing
|
|
let price_feature = features[0];
|
|
let volume_feature = features[1];
|
|
let momentum_feature = features.get(6).copied().unwrap_or(0.0);
|
|
|
|
// Calculate signal strength
|
|
let signal_strength = (momentum_feature / price_feature) + (volume_feature / 10000.0);
|
|
let confidence = (signal_strength.abs() * 0.8).min(0.95).max(0.1);
|
|
|
|
let signal = if signal_strength > 0.05 {
|
|
TradingSignal::Buy(confidence as f64)
|
|
} else if signal_strength < -0.05 {
|
|
TradingSignal::Sell(confidence as f64)
|
|
} else {
|
|
TradingSignal::Hold(confidence as f64)
|
|
};
|
|
|
|
// Generate probability distribution
|
|
let prob_dist = match signal {
|
|
TradingSignal::Buy(_) => vec![0.1, 0.7, 0.2, 0.0, 0.0],
|
|
TradingSignal::Sell(_) => vec![0.0, 0.0, 0.2, 0.7, 0.1],
|
|
_ => vec![0.0, 0.2, 0.6, 0.2, 0.0],
|
|
};
|
|
|
|
Ok((signal, confidence as f64, prob_dist))
|
|
}
|
|
|
|
pub fn get_average_latency(&self) -> TestResult<u64> {
|
|
let stats = self.inference_stats.lock()
|
|
.map_err(|e| format!("Failed to acquire stats lock: {}", e))?;
|
|
|
|
if stats.is_empty() {
|
|
return Ok(0);
|
|
}
|
|
|
|
let sum: u64 = stats.iter().sum();
|
|
Ok(sum / stats.len() as u64)
|
|
}
|
|
}
|
|
|
|
/// Trading signal processor
|
|
#[derive(Debug)]
|
|
pub struct TradingSignalProcessor {
|
|
pub config: MLPipelineConfig,
|
|
pub signal_history: Arc<std::sync::Mutex<Vec<MLPrediction>>>,
|
|
}
|
|
|
|
impl TradingSignalProcessor {
|
|
pub fn new(config: MLPipelineConfig) -> Self {
|
|
Self {
|
|
config,
|
|
signal_history: Arc::new(std::sync::Mutex::new(Vec::new())),
|
|
}
|
|
}
|
|
|
|
/// Process ML prediction into trading decision
|
|
pub async fn process_signal(&self, prediction: MLPrediction) -> TestResult<TradingDecision> {
|
|
let start_time = HardwareTimestamp::now();
|
|
|
|
// Store prediction in history
|
|
if let Ok(mut history) = self.signal_history.lock() {
|
|
history.push(prediction.clone());
|
|
if history.len() > 1000 {
|
|
history.remove(0);
|
|
}
|
|
}
|
|
|
|
// Check if signal meets confidence threshold
|
|
if prediction.confidence < self.config.confidence_threshold {
|
|
return Ok(TradingDecision {
|
|
action: TradingAction::NoAction,
|
|
reason: format!("Confidence {} below threshold {}",
|
|
prediction.confidence, self.config.confidence_threshold),
|
|
quantity: Decimal::ZERO,
|
|
confidence: prediction.confidence,
|
|
processing_latency_ns: HardwareTimestamp::now().latency_ns(&start_time),
|
|
});
|
|
}
|
|
|
|
// Generate trading action based on signal
|
|
let (action, quantity) = match &prediction.signal {
|
|
TradingSignal::StrongBuy(conf) => {
|
|
let qty = Decimal::new((conf * 1000.0) as i64, 0); // Scale by confidence
|
|
(TradingAction::Buy, qty)
|
|
},
|
|
TradingSignal::Buy(conf) => {
|
|
let qty = Decimal::new((conf * 500.0) as i64, 0);
|
|
(TradingAction::Buy, qty)
|
|
},
|
|
TradingSignal::StrongSell(conf) => {
|
|
let qty = Decimal::new((conf * 1000.0) as i64, 0);
|
|
(TradingAction::Sell, qty)
|
|
},
|
|
TradingSignal::Sell(conf) => {
|
|
let qty = Decimal::new((conf * 500.0) as i64, 0);
|
|
(TradingAction::Sell, qty)
|
|
},
|
|
TradingSignal::Hold(_) => {
|
|
(TradingAction::NoAction, Decimal::ZERO)
|
|
},
|
|
};
|
|
|
|
let processing_latency = HardwareTimestamp::now().latency_ns(&start_time);
|
|
|
|
Ok(TradingDecision {
|
|
action,
|
|
reason: format!("ML signal: {} with confidence {}",
|
|
prediction.model_name, prediction.confidence),
|
|
quantity,
|
|
confidence: prediction.confidence,
|
|
processing_latency_ns: processing_latency,
|
|
})
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Clone)]
|
|
pub struct TradingDecision {
|
|
pub action: TradingAction,
|
|
pub reason: String,
|
|
pub quantity: Decimal,
|
|
pub confidence: f64,
|
|
pub processing_latency_ns: u64,
|
|
}
|
|
|
|
#[derive(Debug, Clone)]
|
|
pub enum TradingAction {
|
|
Buy,
|
|
Sell,
|
|
NoAction,
|
|
}
|
|
|
|
// =============================================================================
|
|
// INTEGRATION TESTS
|
|
// =============================================================================
|
|
|
|
#[tokio::test]
|
|
async fn test_market_data_to_features_pipeline() -> TestResult<()> {
|
|
let config = MLPipelineConfig::default();
|
|
let feature_engineer = FeatureEngineer::new(config.clone());
|
|
|
|
// Test 1: Single tick feature extraction
|
|
let tick = MarketTick::new(
|
|
"AAPL".to_string(),
|
|
Decimal::new(150_75, 2), // $150.75
|
|
2500
|
|
);
|
|
|
|
let features = feature_engineer.extract_features(&tick).await?;
|
|
|
|
assert_eq!(features.features.len(), 10, "Should extract 10 features");
|
|
assert!(features.extraction_latency_ns < 10_000,
|
|
"Feature extraction should be <10μs, got {}ns", features.extraction_latency_ns);
|
|
|
|
// Validate feature values
|
|
assert!((features.features[0] - 150.75).abs() < 0.01, "Price feature should match tick price");
|
|
assert!((features.features[1] - 2500.0).abs() < 1.0, "Volume feature should match tick volume");
|
|
|
|
// Test 2: Multiple ticks for moving averages
|
|
let ticks = vec![
|
|
MarketTick::new("AAPL".to_string(), Decimal::new(150_00, 2), 1000),
|
|
MarketTick::new("AAPL".to_string(), Decimal::new(151_00, 2), 1100),
|
|
MarketTick::new("AAPL".to_string(), Decimal::new(152_00, 2), 1200),
|
|
MarketTick::new("AAPL".to_string(), Decimal::new(151_50, 2), 1300),
|
|
MarketTick::new("AAPL".to_string(), Decimal::new(150_75, 2), 1400),
|
|
];
|
|
|
|
let mut total_latency = 0u64;
|
|
for tick in ticks {
|
|
let features = feature_engineer.extract_features(&tick).await?;
|
|
total_latency += features.extraction_latency_ns;
|
|
|
|
assert!(features.extraction_latency_ns < 10_000,
|
|
"Each feature extraction should be <10μs");
|
|
}
|
|
|
|
let avg_latency = total_latency / 5;
|
|
assert!(avg_latency < 10_000,
|
|
"Average feature extraction latency should be <10μs, got {}ns", avg_latency);
|
|
|
|
println!("✓ Market data to features pipeline test passed (avg latency: {}ns)", avg_latency);
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_ml_inference_pipeline() -> TestResult<()> {
|
|
let config = MLPipelineConfig::default();
|
|
let model = MockMLModel::new("TLOB_Transformer".to_string(), config.clone());
|
|
|
|
// Test 1: Basic inference
|
|
let features = FeatureVector::new(
|
|
vec![150.75, 2500.0, 0.05, 150.5, 150.2, 2400.0, 0.25, 100.0, 0.0003, 0.15],
|
|
vec!["price".to_string(), "volume".to_string()], // Simplified for test
|
|
5_000 // 5μs feature extraction
|
|
);
|
|
|
|
let prediction = model.predict(&features).await?;
|
|
|
|
assert!(prediction.inference_latency_ns < 50_000,
|
|
"ML inference should be <50μs, got {}ns", prediction.inference_latency_ns);
|
|
assert!(prediction.confidence >= 0.0 && prediction.confidence <= 1.0,
|
|
"Confidence should be between 0 and 1, got {}", prediction.confidence);
|
|
assert_eq!(prediction.probability_distribution.len(), 5,
|
|
"Should return 5 probability values");
|
|
|
|
// Test 2: High-frequency inference
|
|
let num_predictions = 100;
|
|
let mut latencies = Vec::new();
|
|
let start_time = HardwareTimestamp::now();
|
|
|
|
for i in 0..num_predictions {
|
|
let test_features = FeatureVector::new(
|
|
vec![150.0 + (i as f32 * 0.1), 2500.0, 0.05, 150.5, 150.2, 2400.0, 0.25, 100.0, 0.0003, 0.15],
|
|
vec!["price".to_string()],
|
|
1_000 // Fast feature extraction
|
|
);
|
|
|
|
let prediction = model.predict(&test_features).await?;
|
|
latencies.push(prediction.inference_latency_ns);
|
|
}
|
|
|
|
let total_time = HardwareTimestamp::now().latency_ns(&start_time);
|
|
let throughput = (num_predictions as f64 / (total_time as f64 / 1_000_000_000.0)) as u64;
|
|
|
|
latencies.sort_unstable();
|
|
let p95_latency = latencies[latencies.len() * 95 / 100];
|
|
let avg_latency = model.get_average_latency()?;
|
|
|
|
assert!(p95_latency < 50_000,
|
|
"P95 inference latency should be <50μs, got {}ns", p95_latency);
|
|
assert!(throughput > 1_000,
|
|
"ML inference throughput should be >1000/sec, got {}/sec", throughput);
|
|
|
|
println!("✓ ML inference pipeline test passed: {} predictions/sec, P95: {}ns, avg: {}ns",
|
|
throughput, p95_latency, avg_latency);
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_signal_processing_pipeline() -> TestResult<()> {
|
|
let config = MLPipelineConfig::default();
|
|
let signal_processor = TradingSignalProcessor::new(config.clone());
|
|
|
|
// Test 1: High confidence buy signal
|
|
let buy_prediction = MLPrediction {
|
|
signal: TradingSignal::Buy(0.85),
|
|
confidence: 0.85,
|
|
probability_distribution: vec![0.0, 0.8, 0.2, 0.0, 0.0],
|
|
model_name: "Test_Model".to_string(),
|
|
inference_latency_ns: 30_000,
|
|
timestamp: HardwareTimestamp::now(),
|
|
};
|
|
|
|
let decision = signal_processor.process_signal(buy_prediction).await?;
|
|
|
|
assert!(matches!(decision.action, TradingAction::Buy), "Should generate buy action");
|
|
assert!(decision.quantity > Decimal::ZERO, "Should have positive quantity");
|
|
assert!(decision.processing_latency_ns < 5_000,
|
|
"Signal processing should be <5μs, got {}ns", decision.processing_latency_ns);
|
|
|
|
// Test 2: Low confidence signal (should be filtered)
|
|
let low_conf_prediction = MLPrediction {
|
|
signal: TradingSignal::Buy(0.5),
|
|
confidence: 0.5, // Below 0.7 threshold
|
|
probability_distribution: vec![0.1, 0.5, 0.4, 0.0, 0.0],
|
|
model_name: "Test_Model".to_string(),
|
|
inference_latency_ns: 25_000,
|
|
timestamp: HardwareTimestamp::now(),
|
|
};
|
|
|
|
let decision = signal_processor.process_signal(low_conf_prediction).await?;
|
|
|
|
assert!(matches!(decision.action, TradingAction::NoAction),
|
|
"Low confidence signal should result in no action");
|
|
assert_eq!(decision.quantity, Decimal::ZERO, "Should have zero quantity");
|
|
|
|
// Test 3: Sell signal
|
|
let sell_prediction = MLPrediction {
|
|
signal: TradingSignal::Sell(0.9),
|
|
confidence: 0.9,
|
|
probability_distribution: vec![0.0, 0.0, 0.1, 0.9, 0.0],
|
|
model_name: "Test_Model".to_string(),
|
|
inference_latency_ns: 35_000,
|
|
timestamp: HardwareTimestamp::now(),
|
|
};
|
|
|
|
let decision = signal_processor.process_signal(sell_prediction).await?;
|
|
|
|
assert!(matches!(decision.action, TradingAction::Sell), "Should generate sell action");
|
|
assert!(decision.quantity > Decimal::ZERO, "Should have positive quantity");
|
|
|
|
println!("✓ Signal processing pipeline test passed");
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_end_to_end_ml_trading_flow() -> TestResult<()> {
|
|
let config = MLPipelineConfig::default();
|
|
let feature_engineer = FeatureEngineer::new(config.clone());
|
|
let ml_model = MockMLModel::new("End2End_Model".to_string(), config.clone());
|
|
let signal_processor = TradingSignalProcessor::new(config.clone());
|
|
|
|
// Simulate realistic market data stream
|
|
let market_ticks = vec![
|
|
MarketTick::new("AAPL".to_string(), Decimal::new(150_00, 2), 1000),
|
|
MarketTick::new("AAPL".to_string(), Decimal::new(150_25, 2), 1200),
|
|
MarketTick::new("AAPL".to_string(), Decimal::new(150_50, 2), 1400),
|
|
MarketTick::new("AAPL".to_string(), Decimal::new(150_75, 2), 1600),
|
|
MarketTick::new("AAPL".to_string(), Decimal::new(151_00, 2), 1800),
|
|
];
|
|
|
|
let mut end_to_end_latencies = Vec::new();
|
|
let mut trading_decisions = Vec::new();
|
|
|
|
for tick in market_ticks {
|
|
let pipeline_start = HardwareTimestamp::now();
|
|
|
|
// Step 1: Feature extraction
|
|
let features = feature_engineer.extract_features(&tick).await?;
|
|
|
|
// Step 2: ML inference
|
|
let prediction = ml_model.predict(&features).await?;
|
|
|
|
// Step 3: Signal processing
|
|
let decision = signal_processor.process_signal(prediction).await?;
|
|
|
|
let end_to_end_latency = HardwareTimestamp::now().latency_ns(&pipeline_start);
|
|
|
|
end_to_end_latencies.push(end_to_end_latency);
|
|
trading_decisions.push(decision);
|
|
|
|
// Validate end-to-end latency for HFT requirements
|
|
assert!(end_to_end_latency < 100_000,
|
|
"End-to-end ML pipeline should be <100μs, got {}ns", end_to_end_latency);
|
|
}
|
|
|
|
// Analyze results
|
|
let avg_latency = end_to_end_latencies.iter().sum::<u64>() / end_to_end_latencies.len() as u64;
|
|
let max_latency = *end_to_end_latencies.iter().max().unwrap_or(&0);
|
|
|
|
let actionable_decisions = trading_decisions.iter()
|
|
.filter(|d| !matches!(d.action, TradingAction::NoAction))
|
|
.count();
|
|
|
|
assert!(avg_latency < 80_000,
|
|
"Average end-to-end latency should be <80μs, got {}ns", avg_latency);
|
|
assert!(max_latency < 100_000,
|
|
"Max end-to-end latency should be <100μs, got {}ns", max_latency);
|
|
assert!(actionable_decisions > 0,
|
|
"Should generate at least one actionable trading decision");
|
|
|
|
println!("✓ End-to-end ML trading flow test passed: avg {}ns, max {}ns, {} actionable decisions",
|
|
avg_latency, max_latency, actionable_decisions);
|
|
Ok(())
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_ml_pipeline_under_stress() -> TestResult<()> {
|
|
let config = MLPipelineConfig::default();
|
|
let feature_engineer = Arc::new(FeatureEngineer::new(config.clone()));
|
|
let ml_model = Arc::new(MockMLModel::new("Stress_Test_Model".to_string(), config.clone()));
|
|
let signal_processor = Arc::new(TradingSignalProcessor::new(config.clone()));
|
|
|
|
// Generate high-frequency market data
|
|
let num_ticks = 1000;
|
|
let mut handles = Vec::new();
|
|
let start_time = HardwareTimestamp::now();
|
|
|
|
for i in 0..num_ticks {
|
|
let feature_engineer = feature_engineer.clone();
|
|
let ml_model = ml_model.clone();
|
|
let signal_processor = signal_processor.clone();
|
|
|
|
let handle = tokio::spawn(async move {
|
|
let tick = MarketTick::new(
|
|
format!("STOCK_{}", i % 10), // 10 different symbols
|
|
Decimal::new(150_00 + (i % 100) as i64, 2),
|
|
1000 + (i % 500) as u64
|
|
);
|
|
|
|
let pipeline_start = HardwareTimestamp::now();
|
|
|
|
// Full ML pipeline
|
|
let features = feature_engineer.extract_features(&tick).await?;
|
|
let prediction = ml_model.predict(&features).await?;
|
|
let decision = signal_processor.process_signal(prediction).await?;
|
|
|
|
let pipeline_latency = HardwareTimestamp::now().latency_ns(&pipeline_start);
|
|
|
|
Ok::<_, Box<dyn std::error::Error + Send + Sync>>((
|
|
pipeline_latency,
|
|
decision.action,
|
|
prediction.confidence
|
|
))
|
|
});
|
|
|
|
handles.push(handle);
|
|
}
|
|
|
|
// Process all concurrent ML pipelines
|
|
let mut results = Vec::new();
|
|
for handle in handles {
|
|
results.push(handle.await);
|
|
}
|
|
let total_time = HardwareTimestamp::now().latency_ns(&start_time);
|
|
|
|
let mut successful_pipelines = 0;
|
|
let mut pipeline_latencies = Vec::new();
|
|
let mut actionable_count = 0;
|
|
|
|
for result in results {
|
|
match result {
|
|
Ok(Ok((latency, action, confidence))) => {
|
|
successful_pipelines += 1;
|
|
pipeline_latencies.push(latency);
|
|
|
|
if !matches!(action, TradingAction::NoAction) {
|
|
actionable_count += 1;
|
|
}
|
|
|
|
if confidence < 0.0 || confidence > 1.0 {
|
|
eprintln!("WARNING: Invalid confidence value: {}", confidence);
|
|
}
|
|
}
|
|
Ok(Err(e)) => eprintln!("Pipeline failed: {}", e),
|
|
Err(e) => eprintln!("Task failed: {}", e),
|
|
}
|
|
}
|
|
|
|
// Calculate performance metrics
|
|
let throughput = (successful_pipelines as f64 / (total_time as f64 / 1_000_000_000.0)) as u64;
|
|
|
|
pipeline_latencies.sort_unstable();
|
|
let p95_latency = pipeline_latencies.get(pipeline_latencies.len() * 95 / 100).copied().unwrap_or(0);
|
|
let avg_latency = pipeline_latencies.iter().sum::<u64>() / pipeline_latencies.len().max(1) as u64;
|
|
|
|
// Validate HFT performance under stress
|
|
assert!(p95_latency < 100_000,
|
|
"P95 ML pipeline latency should be <100μs under stress, got {}ns", p95_latency);
|
|
assert!(throughput > 500,
|
|
"ML pipeline throughput should be >500/sec under stress, got {}/sec", throughput);
|
|
assert!(successful_pipelines >= num_ticks * 90 / 100,
|
|
"At least 90% of pipelines should succeed under stress, got {}%",
|
|
successful_pipelines * 100 / num_ticks);
|
|
|
|
let actionable_rate = actionable_count as f64 / successful_pipelines as f64;
|
|
assert!(actionable_rate > 0.1,
|
|
"At least 10% of signals should be actionable, got {:.1}%", actionable_rate * 100.0);
|
|
|
|
println!("✓ ML pipeline stress test passed: {} pipelines/sec, P95: {}ns, {:.1}% actionable",
|
|
throughput, p95_latency, actionable_rate * 100.0);
|
|
Ok(())
|
|
}
|
|
|
|
// =============================================================================
|
|
// INTEGRATION TEST RUNNER
|
|
// =============================================================================
|
|
|
|
#[tokio::test]
|
|
async fn run_all_ml_trading_pipeline_tests() -> TestResult<()> {
|
|
println!("=== ML TRADING PIPELINE INTEGRATION TEST SUITE ===");
|
|
|
|
let test_timeout = Duration::from_secs(120);
|
|
|
|
// Run all integration tests with timeout protection
|
|
timeout(test_timeout, async { test_market_data_to_features_pipeline().await }).await??;
|
|
timeout(test_timeout, async { test_ml_inference_pipeline().await }).await??;
|
|
timeout(test_timeout, async { test_signal_processing_pipeline().await }).await??;
|
|
timeout(test_timeout, async { test_end_to_end_ml_trading_flow().await }).await??;
|
|
timeout(test_timeout, async { test_ml_pipeline_under_stress().await }).await??;
|
|
|
|
println!("=== ALL ML TRADING PIPELINE INTEGRATION TESTS PASSED ===");
|
|
println!("✓ Market data to features pipeline with SIMD optimization");
|
|
println!("✓ ML inference pipeline with <50μs latency");
|
|
println!("✓ Signal processing and decision generation");
|
|
println!("✓ End-to-end ML trading flow <100μs");
|
|
println!("✓ High-frequency stress testing >500 pipelines/sec");
|
|
println!("✓ Feature extraction <10μs with SIMD");
|
|
println!("✓ ML model inference <50μs");
|
|
println!("✓ Signal processing <5μs");
|
|
println!("✓ 90%+ success rate under stress");
|
|
println!("✓ 10%+ actionable trading signals");
|
|
|
|
Ok(())
|
|
} |