e166a4fc02
- Updated 73 test files across 10 categories - Total 557 replacements (225 → 54) - DQN tests: 252/262 passing (9 failures - slice index blocker) - TFT tests: 98/98 passing - MAMBA-2 tests: 11/11 passing - Hyperopt tests: 98/98 passing Critical findings: - Blocker: ml/src/trainers/dqn.rs:3444 hardcoded slice indices - Architecture mismatch: extract_current_features() vs extract_current_features_v2() Wave 3 Agent breakdown: - Agent 1: DQN test files (12 files) - Agent 2: PPO test files (2 files) - Agent 3: TFT test files (6 files) - Agent 4: MAMBA-2 test files (2 files) - Agent 5: Feature extraction tests (3 files) - Agent 6: Integration test files (9 files) - Agent 7: Data loader test files (3 files) - Agent 8: Hyperopt test files (1 file) - Agent 9: Benchmark test files (9 files) - Agent 10: Utility & misc test files (73 files) Next: Fix slice index blocker, then Wave 4 (OFI integration 46→54)
398 lines
13 KiB
Rust
398 lines
13 KiB
Rust
//! Feature Normalization Tests
|
|
//!
|
|
//! Tests for percentile-based feature clipping to prevent outliers
|
|
//! from crushing the feature distribution during min-max normalization.
|
|
//!
|
|
//! ## Problem
|
|
//!
|
|
//! OBV (On-Balance Volume) features accumulate signed volume over time,
|
|
//! leading to extreme outliers (e.g., -863K to +863K). When using
|
|
//! min-max normalization, these outliers compress 51/54 other features
|
|
//! into a narrow range [0.48, 0.52], making them indistinguishable.
|
|
//!
|
|
//! ## Solution
|
|
//!
|
|
//! Apply percentile clipping (1st to 99th percentile) BEFORE min-max
|
|
//! normalization. This preserves 98% of data while preventing outliers
|
|
//! from dominating the normalization scale.
|
|
|
|
use std::f64;
|
|
|
|
/// Compute percentile value from sorted data
|
|
fn percentile(sorted_data: &[f64], p: f64) -> f64 {
|
|
assert!(
|
|
!sorted_data.is_empty(),
|
|
"Cannot compute percentile of empty data"
|
|
);
|
|
assert!(p >= 0.0 && p <= 1.0, "Percentile must be in [0, 1]");
|
|
|
|
let idx = (sorted_data.len() as f64 * p).round() as usize;
|
|
let idx = idx.min(sorted_data.len() - 1);
|
|
sorted_data[idx]
|
|
}
|
|
|
|
/// Apply percentile clipping to features
|
|
fn clip_features_by_percentile(features: &[f64], p_low: f64, p_high: f64) -> Vec<f64> {
|
|
let mut sorted = features.to_vec();
|
|
sorted.sort_by(|a, b| a.partial_cmp(b).unwrap());
|
|
|
|
let p1 = percentile(&sorted, p_low);
|
|
let p99 = percentile(&sorted, p_high);
|
|
|
|
println!("Percentile p1 ({:.2}): {:.2}", p_low, p1);
|
|
println!("Percentile p99 ({:.2}): {:.2}", p_high, p99);
|
|
|
|
features.iter().map(|&x| x.clamp(p1, p99)).collect()
|
|
}
|
|
|
|
/// Normalize features to [0, 1] range
|
|
fn normalize_min_max(features: &[f64]) -> Vec<f64> {
|
|
let min = features.iter().copied().fold(f64::INFINITY, f64::min);
|
|
let max = features.iter().copied().fold(f64::NEG_INFINITY, f64::max);
|
|
let range = max - min;
|
|
|
|
if range.abs() < 1e-10 {
|
|
// All values are the same, return 0.5
|
|
return vec![0.5; features.len()];
|
|
}
|
|
|
|
features.iter().map(|&x| (x - min) / range).collect()
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn test_percentile_computation() {
|
|
let data = vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0];
|
|
|
|
// Test extremes
|
|
assert!((percentile(&data, 0.0) - 1.0).abs() < 1e-10);
|
|
assert!((percentile(&data, 1.0) - 10.0).abs() < 1e-10);
|
|
|
|
// Test median (50th percentile)
|
|
let p50 = percentile(&data, 0.5);
|
|
assert!(
|
|
p50 >= 5.0 && p50 <= 6.0,
|
|
"Median should be ~5.5, got {}",
|
|
p50
|
|
);
|
|
|
|
// Test 99th percentile
|
|
let p99 = percentile(&data, 0.99);
|
|
assert!(
|
|
p99 >= 9.0 && p99 <= 10.0,
|
|
"99th percentile should be ~10, got {}",
|
|
p99
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn test_clip_features_without_outliers() {
|
|
// Data without outliers - clipping should have minimal effect
|
|
let features = vec![10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0, 80.0, 90.0, 100.0];
|
|
let clipped = clip_features_by_percentile(&features, 0.01, 0.99);
|
|
|
|
// Most values should be unchanged
|
|
for i in 1..9 {
|
|
assert!(
|
|
(clipped[i] - features[i]).abs() < 1.0,
|
|
"Value {} should be mostly unchanged",
|
|
i
|
|
);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn test_clip_features_with_extreme_outliers() {
|
|
// Test that clipping works when percentiles exclude outliers
|
|
// Key insight: outliers must be OUTSIDE the 1st-99th percentile range
|
|
let mut features = Vec::new();
|
|
|
|
// Add 100 normal values in range [-100, 100]
|
|
for i in -50..50 {
|
|
features.push(i as f64 * 2.0);
|
|
}
|
|
|
|
// Add extreme outliers at beginning and end
|
|
// These will be at the 0.5% and 99.5% positions
|
|
features.insert(0, -863_000.0);
|
|
features.push(863_000.0);
|
|
|
|
println!("Total features: {}", features.len());
|
|
|
|
let clipped = clip_features_by_percentile(&features, 0.02, 0.98);
|
|
|
|
// Extreme outliers should be clipped to 2nd and 98th percentile values
|
|
let min_clipped = clipped.iter().copied().fold(f64::INFINITY, f64::min);
|
|
let max_clipped = clipped.iter().copied().fold(f64::NEG_INFINITY, f64::max);
|
|
|
|
println!("Min clipped: {}, Max clipped: {}", min_clipped, max_clipped);
|
|
|
|
// After clipping, outliers should be replaced with percentile boundary values
|
|
// which are within the normal range
|
|
assert!(
|
|
max_clipped < 200.0,
|
|
"Max should be clipped to reasonable range, got {}",
|
|
max_clipped
|
|
);
|
|
assert!(
|
|
min_clipped > -200.0,
|
|
"Min should be clipped to reasonable range, got {}",
|
|
min_clipped
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn test_normalize_min_max_basic() {
|
|
let features = vec![0.0, 25.0, 50.0, 75.0, 100.0];
|
|
let normalized = normalize_min_max(&features);
|
|
|
|
// Check bounds
|
|
assert!((normalized[0] - 0.0).abs() < 1e-10, "Min should map to 0");
|
|
assert!((normalized[4] - 1.0).abs() < 1e-10, "Max should map to 1");
|
|
|
|
// Check midpoint
|
|
assert!(
|
|
(normalized[2] - 0.5).abs() < 1e-10,
|
|
"Midpoint should map to 0.5"
|
|
);
|
|
|
|
// Check all values in [0, 1]
|
|
for val in &normalized {
|
|
assert!(
|
|
*val >= 0.0 && *val <= 1.0,
|
|
"Normalized value {} out of range",
|
|
val
|
|
);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn test_normalize_constant_features() {
|
|
// All values the same - should return 0.5
|
|
let features = vec![42.0; 10];
|
|
let normalized = normalize_min_max(&features);
|
|
|
|
for val in &normalized {
|
|
assert!(
|
|
(val - 0.5).abs() < 1e-10,
|
|
"Constant features should normalize to 0.5"
|
|
);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn test_full_pipeline_with_outliers() {
|
|
// Simulate realistic scenario: 54 features with OBV outliers
|
|
let mut features = Vec::new();
|
|
|
|
// 51 normal features (range: 0-100)
|
|
for _ in 0..51 {
|
|
for i in 0..10 {
|
|
features.push(i as f64 * 10.0);
|
|
}
|
|
}
|
|
|
|
// 3 OBV features with extreme outliers
|
|
for _ in 0..3 {
|
|
features.push(-863_000.0);
|
|
features.push(863_000.0);
|
|
for i in -5..5 {
|
|
features.push(i as f64 * 100.0);
|
|
}
|
|
}
|
|
|
|
println!("\n=== Feature Normalization Test ===");
|
|
println!("Total features: {}", features.len());
|
|
|
|
// BEFORE: Direct normalization (broken)
|
|
let normalized_before = normalize_min_max(&features);
|
|
let min_before = normalized_before
|
|
.iter()
|
|
.copied()
|
|
.fold(f64::INFINITY, f64::min);
|
|
let max_before = normalized_before
|
|
.iter()
|
|
.copied()
|
|
.fold(f64::NEG_INFINITY, f64::max);
|
|
|
|
println!("\nBEFORE percentile clipping:");
|
|
println!(
|
|
" Feature range: {:.2} to {:.2}",
|
|
features.iter().copied().fold(f64::INFINITY, f64::min),
|
|
features.iter().copied().fold(f64::NEG_INFINITY, f64::max)
|
|
);
|
|
println!(" Normalized range: [{:.6}, {:.6}]", min_before, max_before);
|
|
|
|
// Count how many values are in narrow range [0.48, 0.52]
|
|
let crushed_before = normalized_before
|
|
.iter()
|
|
.filter(|&&x| x >= 0.48 && x <= 0.52)
|
|
.count();
|
|
println!(
|
|
" Values crushed to [0.48, 0.52]: {} ({:.1}%)",
|
|
crushed_before,
|
|
100.0 * crushed_before as f64 / normalized_before.len() as f64
|
|
);
|
|
|
|
// AFTER: Percentile clipping + normalization (fixed)
|
|
let clipped = clip_features_by_percentile(&features, 0.01, 0.99);
|
|
let normalized_after = normalize_min_max(&clipped);
|
|
let min_after = normalized_after
|
|
.iter()
|
|
.copied()
|
|
.fold(f64::INFINITY, f64::min);
|
|
let max_after = normalized_after
|
|
.iter()
|
|
.copied()
|
|
.fold(f64::NEG_INFINITY, f64::max);
|
|
|
|
println!("\nAFTER percentile clipping:");
|
|
println!(
|
|
" Clipped range: {:.2} to {:.2}",
|
|
clipped.iter().copied().fold(f64::INFINITY, f64::min),
|
|
clipped.iter().copied().fold(f64::NEG_INFINITY, f64::max)
|
|
);
|
|
println!(" Normalized range: [{:.6}, {:.6}]", min_after, max_after);
|
|
|
|
// Count distribution after fix
|
|
let crushed_after = normalized_after
|
|
.iter()
|
|
.filter(|&&x| x >= 0.48 && x <= 0.52)
|
|
.count();
|
|
println!(
|
|
" Values crushed to [0.48, 0.52]: {} ({:.1}%)",
|
|
crushed_after,
|
|
100.0 * crushed_after as f64 / normalized_after.len() as f64
|
|
);
|
|
|
|
// Assert fix works
|
|
assert!(
|
|
crushed_after < crushed_before / 2,
|
|
"Percentile clipping should reduce feature crushing significantly"
|
|
);
|
|
|
|
// Verify full utilization of [0, 1] range
|
|
assert!(
|
|
(min_after - 0.0).abs() < 0.1,
|
|
"Min should be close to 0 after fix"
|
|
);
|
|
assert!(
|
|
(max_after - 1.0).abs() < 0.1,
|
|
"Max should be close to 1 after fix"
|
|
);
|
|
|
|
println!("\n=== Fix Validated ===");
|
|
println!("Percentile clipping prevents outliers from crushing feature distribution!");
|
|
}
|
|
|
|
#[test]
|
|
fn test_obv_realistic_scenario() {
|
|
// Realistic OBV outlier scenario from ES_FUT_180d.parquet
|
|
let mut features = Vec::new();
|
|
|
|
// Generate OBV-like data: accumulates over time
|
|
let mut obv = 0.0;
|
|
for i in 0..1000 {
|
|
let volume = 100.0 + (i as f64 % 50.0);
|
|
let direction = if i % 3 == 0 { 1.0 } else { -1.0 };
|
|
obv += volume * direction;
|
|
features.push(obv);
|
|
}
|
|
|
|
// Add other normal features (RSI, MACD, etc.)
|
|
for _ in 0..53 {
|
|
for i in 0..1000 {
|
|
features.push((i % 100) as f64);
|
|
}
|
|
}
|
|
|
|
println!("\n=== OBV Realistic Scenario ===");
|
|
let orig_min = features.iter().copied().fold(f64::INFINITY, f64::min);
|
|
let orig_max = features.iter().copied().fold(f64::NEG_INFINITY, f64::max);
|
|
println!("Original feature range: [{:.0}, {:.0}]", orig_min, orig_max);
|
|
|
|
// Apply percentile clipping
|
|
let clipped = clip_features_by_percentile(&features, 0.01, 0.99);
|
|
let clipped_min = clipped.iter().copied().fold(f64::INFINITY, f64::min);
|
|
let clipped_max = clipped.iter().copied().fold(f64::NEG_INFINITY, f64::max);
|
|
println!(
|
|
"Clipped feature range: [{:.0}, {:.0}]",
|
|
clipped_min, clipped_max
|
|
);
|
|
|
|
// Range should be much smaller after clipping
|
|
let orig_range = orig_max - orig_min;
|
|
let clipped_range = clipped_max - clipped_min;
|
|
println!(
|
|
"Range reduction: {:.0} → {:.0} ({:.1}% reduction)",
|
|
orig_range,
|
|
clipped_range,
|
|
100.0 * (1.0 - clipped_range / orig_range)
|
|
);
|
|
|
|
assert!(
|
|
clipped_range < orig_range * 0.5,
|
|
"Clipping should reduce range by at least 50%"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn test_edge_case_all_same_value() {
|
|
let features = vec![42.0; 100];
|
|
let clipped = clip_features_by_percentile(&features, 0.01, 0.99);
|
|
let normalized = normalize_min_max(&clipped);
|
|
|
|
// All values should normalize to 0.5
|
|
for val in &normalized {
|
|
assert!((val - 0.5).abs() < 1e-10);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn test_edge_case_two_values() {
|
|
let features = vec![0.0, 100.0];
|
|
let clipped = clip_features_by_percentile(&features, 0.01, 0.99);
|
|
let normalized = normalize_min_max(&clipped);
|
|
|
|
assert!((normalized[0] - 0.0).abs() < 1e-10);
|
|
assert!((normalized[1] - 1.0).abs() < 1e-10);
|
|
}
|
|
|
|
#[test]
|
|
fn test_preserves_98_percent_of_data() {
|
|
// Generate 10000 normal values + 200 outliers
|
|
let mut features = Vec::new();
|
|
|
|
// 98% normal (0-100)
|
|
for i in 0..9800 {
|
|
features.push((i % 100) as f64);
|
|
}
|
|
|
|
// 2% outliers (-100000, +100000)
|
|
for _ in 0..100 {
|
|
features.push(-100_000.0);
|
|
features.push(100_000.0);
|
|
}
|
|
|
|
let clipped = clip_features_by_percentile(&features, 0.01, 0.99);
|
|
|
|
// Count how many normal values are preserved exactly
|
|
let preserved = features
|
|
.iter()
|
|
.filter(|&&x| x >= 0.0 && x <= 100.0)
|
|
.filter(|&&x| clipped.contains(&x))
|
|
.count();
|
|
|
|
let preservation_rate = preserved as f64 / 9800.0;
|
|
println!("\nPreservation rate: {:.1}%", preservation_rate * 100.0);
|
|
|
|
assert!(
|
|
preservation_rate > 0.95,
|
|
"At least 95% of normal data should be preserved"
|
|
);
|
|
}
|
|
}
|