🔧 Wave 33: Test Compilation Improvements - 57 errors remaining

**Progress: 1,178 → 57 test errors (95% reduction)**

## Status Summary
-  Production code: Compiles cleanly (0 errors)
- ⚠️  Test code: 57 errors remain (massive improvement)
- ⚙️  All services build successfully
- 📊 Warning count: 253 (target: <20) - AGENTS WILL FIX

## Remaining Test Errors (57 total)
### Primary Issues:
1. 23× E0308 mismatched types
2. 17× E0433 undeclared Decimal
3. 15× E0433 compliance module not found
4. 6× E0624 private method access
5. Various import and type issues

## Next Phase: Wave 33-2
Launch 10+ parallel agents to:
- Fix remaining 57 test compilation errors
- Reduce 253 warnings to <20
- Achieve 95% test coverage
- Ensure all tests pass

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2025-10-01 21:24:28 +02:00
parent bb1042b848
commit 6bd5b18465
444 changed files with 15714 additions and 11339 deletions

View File

@@ -18,23 +18,23 @@
//! - Compares optimized vs baseline implementations
//! - Documents real-world performance characteristics
use criterion::{black_box, criterion_group, criterion_main, Criterion, BenchmarkId, Throughput};
use std::time::{Duration, Instant};
use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion, Throughput};
use std::arch::x86_64::_rdtsc;
use std::time::{Duration, Instant};
// Import the HFT system components to test
#[path = "../trading_engine/src/timing.rs"]
mod timing;
#[path = "../trading_engine/src/simd/mod.rs"]
#[path = "../trading_engine/src/simd/mod.rs"]
mod simd;
#[path = "../trading_engine/src/lockfree/mod.rs"]
mod lockfree;
use timing::{HardwareTimestamp, LatencyMeasurement, calibrate_tsc};
use simd::{SimdPriceOps, AlignedPrices, AlignedVolumes};
use lockfree::{LockFreeRingBuffer, SharedMemoryChannel, HftMessage};
use lockfree::{HftMessage, LockFreeRingBuffer, SharedMemoryChannel};
use simd::{AlignedPrices, AlignedVolumes, SimdPriceOps};
use timing::{calibrate_tsc, HardwareTimestamp, LatencyMeasurement};
/// Test configuration for 14ns validation
#[derive(Clone)]
@@ -53,9 +53,9 @@ impl Default for ValidationConfig {
fn default() -> Self {
Self {
target_latency_ns: 14,
acceptable_variance_ns: 3, // ±3ns (±21%)
acceptable_variance_ns: 3, // ±3ns (±21%)
estimated_cpu_freq_ghz: 3.0, // Conservative estimate
confidence_level: 0.95, // 95% confidence
confidence_level: 0.95, // 95% confidence
}
}
}
@@ -74,8 +74,8 @@ struct ValidationResult {
impl ValidationResult {
fn new(
test_name: String,
measurements: &[f64],
test_name: String,
measurements: &[f64],
config: &ValidationConfig,
measurement_method: String,
) -> Self {
@@ -92,19 +92,18 @@ impl ValidationResult {
}
let mean = measurements.iter().sum::<f64>() / measurements.len() as f64;
let variance = measurements.iter()
.map(|x| (x - mean).powi(2))
.sum::<f64>() / (measurements.len() - 1) as f64;
let variance = measurements.iter().map(|x| (x - mean).powi(2)).sum::<f64>()
/ (measurements.len() - 1) as f64;
let std_dev = variance.sqrt();
// Calculate confidence interval
let t_value = 1.96; // Approximate for large samples at 95% confidence
let margin_of_error = t_value * std_dev / (measurements.len() as f64).sqrt();
let confidence_interval = (mean - margin_of_error, mean + margin_of_error);
let meets_target = mean <= config.target_latency_ns as f64;
let within_variance = (mean - config.target_latency_ns as f64).abs()
<= config.acceptable_variance_ns as f64;
let within_variance =
(mean - config.target_latency_ns as f64).abs() <= config.acceptable_variance_ns as f64;
Self {
test_name,
@@ -118,55 +117,71 @@ impl ValidationResult {
}
fn print_result(&self) {
let status = if self.meets_target { "✅ PASS" } else { "❌ FAIL" };
let status = if self.meets_target {
"✅ PASS"
} else {
"❌ FAIL"
};
let variance_status = if self.within_variance { "" } else { "" };
println!("\n{} {}", status, self.test_name);
println!(" Measured: {:.1}ns (target: 14ns)", self.measured_latency_ns);
println!(" Within variance: {} ({:.1}ns ± 3ns)", variance_status, self.measured_latency_ns);
println!(" 95% CI: [{:.1}, {:.1}]ns", self.confidence_interval.0, self.confidence_interval.1);
println!(" Method: {} (n={})", self.measurement_method, self.sample_size);
println!(
" Measured: {:.1}ns (target: 14ns)",
self.measured_latency_ns
);
println!(
" Within variance: {} ({:.1}ns ± 3ns)",
variance_status, self.measured_latency_ns
);
println!(
" 95% CI: [{:.1}, {:.1}]ns",
self.confidence_interval.0, self.confidence_interval.1
);
println!(
" Method: {} (n={})",
self.measurement_method, self.sample_size
);
}
}
/// Calibrate timing systems and detect CPU capabilities
fn setup_validation_environment() -> ValidationConfig {
println!("🔧 Setting up validation environment...");
// Attempt TSC calibration
match calibrate_tsc() {
Ok(freq_hz) => {
let freq_ghz = freq_hz as f64 / 1_000_000_000.0;
println!("✅ TSC calibrated: {:.2} GHz", freq_ghz);
let mut config = ValidationConfig::default();
config.estimated_cpu_freq_ghz = freq_ghz;
config
}
},
Err(e) => {
println!("⚠️ TSC calibration failed: {}, using defaults", e);
ValidationConfig::default()
}
},
}
}
/// Test 1: RDTSC Measurement Overhead and Precision
fn validate_rdtsc_overhead(c: &mut Criterion) {
let config = setup_validation_environment();
c.bench_function("rdtsc_overhead", |b| {
b.iter(|| {
// This is the absolute minimum operation: two RDTSC calls
let start = unsafe { _rdtsc() };
let end = unsafe { _rdtsc() };
let cycles = end - start;
// Convert cycles to nanoseconds
let ns = (cycles as f64) / config.estimated_cpu_freq_ghz;
black_box(ns)
});
});
// Manual measurement for detailed analysis
let mut measurements = Vec::new();
for _ in 0..100_000 {
@@ -176,7 +191,7 @@ fn validate_rdtsc_overhead(c: &mut Criterion) {
let ns = (cycles as f64) / config.estimated_cpu_freq_ghz;
measurements.push(ns);
}
let result = ValidationResult::new(
"RDTSC Measurement Overhead".to_string(),
&measurements,
@@ -189,14 +204,14 @@ fn validate_rdtsc_overhead(c: &mut Criterion) {
/// Test 2: Hardware Timestamp Creation Performance
fn validate_hardware_timestamp(c: &mut Criterion) {
let config = setup_validation_environment();
c.bench_function("hardware_timestamp_creation", |b| {
b.iter(|| {
let ts = HardwareTimestamp::now();
black_box(ts)
});
});
// Manual measurement for validation
let mut measurements = Vec::new();
for _ in 0..50_000 {
@@ -207,7 +222,7 @@ fn validate_hardware_timestamp(c: &mut Criterion) {
let ns = (cycles as f64) / config.estimated_cpu_freq_ghz;
measurements.push(ns);
}
let result = ValidationResult::new(
"HardwareTimestamp::now()".to_string(),
&measurements,
@@ -220,7 +235,7 @@ fn validate_hardware_timestamp(c: &mut Criterion) {
/// Test 3: Latency Measurement Operation Performance
fn validate_latency_measurement(c: &mut Criterion) {
let config = setup_validation_environment();
c.bench_function("latency_measurement_complete", |b| {
b.iter(|| {
let mut measurement = LatencyMeasurement::start();
@@ -229,22 +244,22 @@ fn validate_latency_measurement(c: &mut Criterion) {
black_box(latency)
});
});
// Manual validation measurement
let mut measurements = Vec::new();
for _ in 0..50_000 {
let start = unsafe { _rdtsc() };
let mut measurement = LatencyMeasurement::start();
black_box(42_u64); // Same minimal operation
let _latency = measurement.finish();
let end = unsafe { _rdtsc() };
let cycles = end - start;
let ns = (cycles as f64) / config.estimated_cpu_freq_ghz;
measurements.push(ns);
}
let result = ValidationResult::new(
"Complete Latency Measurement Cycle".to_string(),
&measurements,
@@ -257,18 +272,18 @@ fn validate_latency_measurement(c: &mut Criterion) {
/// Test 4: SIMD Operation Performance
fn validate_simd_operations(c: &mut Criterion) {
let config = setup_validation_environment();
if !std::arch::is_x86_feature_detected!("avx2") {
println!("⚠️ AVX2 not available - SIMD tests will use scalar fallback");
return;
}
// Test data for SIMD operations
let prices = vec![100.0, 101.0, 99.0, 102.0];
let volumes = vec![1000.0, 1100.0, 900.0, 1200.0];
let aligned_prices = AlignedPrices::from_slice(&prices);
let aligned_volumes = AlignedVolumes::from_slice(&volumes);
c.bench_function("simd_vwap_calculation", |b| {
b.iter(|| unsafe {
let simd_ops = SimdPriceOps::new();
@@ -276,21 +291,21 @@ fn validate_simd_operations(c: &mut Criterion) {
black_box(vwap)
});
});
// Manual measurement
let mut measurements = Vec::new();
for _ in 0..50_000 {
let start = unsafe { _rdtsc() };
let simd_ops = unsafe { SimdPriceOps::new() };
let _vwap = unsafe { simd_ops.calculate_vwap_aligned(&aligned_prices, &aligned_volumes) };
let end = unsafe { _rdtsc() };
let cycles = end - start;
let ns = (cycles as f64) / config.estimated_cpu_freq_ghz;
measurements.push(ns);
}
let result = ValidationResult::new(
"SIMD VWAP Calculation".to_string(),
&measurements,
@@ -303,9 +318,9 @@ fn validate_simd_operations(c: &mut Criterion) {
/// Test 5: Lock-Free Ring Buffer Performance
fn validate_lockfree_operations(c: &mut Criterion) {
let config = setup_validation_environment();
let buffer = LockFreeRingBuffer::<u64>::new(1024).expect("Failed to create ring buffer");
c.bench_function("lockfree_push_pop_cycle", |b| {
b.iter(|| {
let value = black_box(42_u64);
@@ -314,21 +329,21 @@ fn validate_lockfree_operations(c: &mut Criterion) {
black_box(result)
});
});
// Manual measurement
let mut measurements = Vec::new();
for i in 0..50_000 {
let start = unsafe { _rdtsc() };
let _ = buffer.try_push(i);
let _result = buffer.try_pop();
let end = unsafe { _rdtsc() };
let cycles = end - start;
let ns = (cycles as f64) / config.estimated_cpu_freq_ghz;
measurements.push(ns);
}
let result = ValidationResult::new(
"Lock-Free Ring Buffer Push+Pop".to_string(),
&measurements,
@@ -341,9 +356,9 @@ fn validate_lockfree_operations(c: &mut Criterion) {
/// Test 6: Shared Memory Channel Performance
fn validate_shared_memory_channel(c: &mut Criterion) {
let config = setup_validation_environment();
let channel = SharedMemoryChannel::new(1024).expect("Failed to create channel");
c.bench_function("shared_memory_send_receive", |b| {
b.iter(|| {
let message = HftMessage::new(1, [42; 8]);
@@ -352,23 +367,23 @@ fn validate_shared_memory_channel(c: &mut Criterion) {
black_box(result)
});
});
// Manual measurement
// Manual measurement
let mut measurements = Vec::new();
for i in 0..25_000 {
let message = HftMessage::new(1, [i; 8]);
let start = unsafe { _rdtsc() };
let _ = channel.send(message);
let _result = channel.try_receive();
let end = unsafe { _rdtsc() };
let cycles = end - start;
let ns = (cycles as f64) / config.estimated_cpu_freq_ghz;
measurements.push(ns);
}
let result = ValidationResult::new(
"Shared Memory Channel Send+Receive".to_string(),
&measurements,
@@ -381,30 +396,30 @@ fn validate_shared_memory_channel(c: &mut Criterion) {
/// Test 7: Atomic Operations Performance
fn validate_atomic_operations(c: &mut Criterion) {
use std::sync::atomic::{AtomicU64, Ordering};
let config = setup_validation_environment();
let counter = AtomicU64::new(0);
c.bench_function("atomic_fetch_add", |b| {
b.iter(|| {
let result = counter.fetch_add(1, Ordering::Relaxed);
black_box(result)
});
});
// Manual measurement
let mut measurements = Vec::new();
for _ in 0..100_000 {
let start = unsafe { _rdtsc() };
let _result = counter.fetch_add(1, Ordering::Relaxed);
let end = unsafe { _rdtsc() };
let cycles = end - start;
let ns = (cycles as f64) / config.estimated_cpu_freq_ghz;
measurements.push(ns);
}
let result = ValidationResult::new(
"Atomic Fetch-Add Operation".to_string(),
&measurements,
@@ -417,9 +432,9 @@ fn validate_atomic_operations(c: &mut Criterion) {
/// Test 8: System Clock vs RDTSC Comparison
fn validate_timing_methods_comparison(c: &mut Criterion) {
let config = setup_validation_environment();
let mut group = c.benchmark_group("timing_method_comparison");
group.bench_function("system_clock_precision", |b| {
b.iter(|| {
let start = Instant::now();
@@ -429,7 +444,7 @@ fn validate_timing_methods_comparison(c: &mut Criterion) {
black_box(duration)
});
});
group.bench_function("rdtsc_precision", |b| {
b.iter(|| {
let start = unsafe { _rdtsc() };
@@ -440,12 +455,12 @@ fn validate_timing_methods_comparison(c: &mut Criterion) {
black_box(ns as u64)
});
});
group.finish();
// Compare precision manually
println!("\n🔍 Timing Method Precision Comparison:");
// System clock measurements
let mut system_measurements = Vec::new();
for _ in 0..10_000 {
@@ -455,7 +470,7 @@ fn validate_timing_methods_comparison(c: &mut Criterion) {
let ns = end.duration_since(start).as_nanos() as f64;
system_measurements.push(ns);
}
// RDTSC measurements
let mut rdtsc_measurements = Vec::new();
for _ in 0..10_000 {
@@ -466,26 +481,29 @@ fn validate_timing_methods_comparison(c: &mut Criterion) {
let ns = (cycles as f64) / config.estimated_cpu_freq_ghz;
rdtsc_measurements.push(ns);
}
let system_result = ValidationResult::new(
"System Clock Timing".to_string(),
&system_measurements,
&config,
"Instant::now()".to_string(),
);
let rdtsc_result = ValidationResult::new(
"RDTSC Timing".to_string(),
&rdtsc_measurements,
&config,
"Raw RDTSC cycles".to_string(),
);
system_result.print_result();
rdtsc_result.print_result();
let precision_advantage = system_result.measured_latency_ns / rdtsc_result.measured_latency_ns;
println!("📊 RDTSC precision advantage: {:.1}x better than system clock", precision_advantage);
println!(
"📊 RDTSC precision advantage: {:.1}x better than system clock",
precision_advantage
);
}
/// Generate final validation report
@@ -493,31 +511,31 @@ fn print_validation_summary() {
println!("\n" + "=".repeat(60).as_str());
println!("📋 14NS LATENCY CLAIMS VALIDATION SUMMARY");
println!("=".repeat(60));
println!("\n🎯 CLAIMS UNDER TEST:");
println!(" • '14ns latency for trading operations'");
println!(" • RDTSC hardware timing implementation");
println!(" • SIMD/AVX2 optimization effectiveness");
println!(" • Lock-free data structure performance");
println!("\n🔬 METHODOLOGY:");
println!(" • Statistical analysis with 95% confidence intervals");
println!(" • Multiple measurement approaches for validation");
println!(" • Isolation of measurement overhead");
println!(" • Comparison against baseline implementations");
println!("\n⚠️ IMPORTANT DISCLAIMERS:");
println!(" • Results are hardware and system load dependent");
println!(" • 14ns is extremely challenging to measure accurately");
println!(" • TSC frequency estimation affects precision");
println!(" • Compiler optimizations may affect results");
println!("\n📖 RECOMMENDATIONS:");
println!(" • Use multiple timing methods for critical measurements");
println!(" • Validate on target production hardware");
println!(" • Consider measurement overhead in latency budgets");
println!(" • Document specific operations that achieve 14ns");
println!("\n" + "=".repeat(60).as_str());
}
@@ -529,7 +547,7 @@ criterion_group! {
.sample_size(1000)
.warm_up_time(Duration::from_secs(3))
.with_plots();
targets =
targets =
validate_rdtsc_overhead,
validate_hardware_timestamp,
validate_latency_measurement,
@@ -550,12 +568,12 @@ mod tests {
#[test]
fn run_14ns_validation_suite() {
println!("🚀 Starting 14ns Latency Claims Validation");
let config = setup_validation_environment();
// Run quick validation tests
println!("\n⚡ Quick Validation Tests (1000 samples each):");
// Test RDTSC overhead
let mut rdtsc_measurements = Vec::new();
for _ in 0..1000 {
@@ -565,7 +583,7 @@ mod tests {
let ns = (cycles as f64) / config.estimated_cpu_freq_ghz;
rdtsc_measurements.push(ns);
}
let rdtsc_result = ValidationResult::new(
"RDTSC Measurement Overhead (Test Mode)".to_string(),
&rdtsc_measurements,
@@ -573,7 +591,7 @@ mod tests {
"Test RDTSC cycles".to_string(),
);
rdtsc_result.print_result();
// Test hardware timestamp if available
if timing::is_tsc_reliable() {
let mut hw_ts_measurements = Vec::new();
@@ -585,7 +603,7 @@ mod tests {
let ns = (cycles as f64) / config.estimated_cpu_freq_ghz;
hw_ts_measurements.push(ns);
}
let hw_result = ValidationResult::new(
"HardwareTimestamp::now() (Test Mode)".to_string(),
&hw_ts_measurements,
@@ -594,10 +612,13 @@ mod tests {
);
hw_result.print_result();
}
print_validation_summary();
// The test passes regardless of performance results - we're validating claims
assert!(true, "14ns validation completed - see output for detailed results");
assert!(
true,
"14ns validation completed - see output for detailed results"
);
}
}
}