ca4c38d921
- Reduce CI GPU test datasets 16x for walltime reduction - Reduce early-stop epochs 50→10, add --test-threads=1 - Serialize all GPU lib tests to prevent cuBLAS init race - Align state_dim to 16 for BF16 tensor core HMMA dispatch - BF16 precision tolerance in ml-dqn tests - Enable branching DQN + tracing subscriber in smoke tests - Prevent min_replay_size > buffer_size deadlock in early-stop tests - Prevent AutoReplaySizer from breaking gradient collapse warmup - Replace racy tokio::spawn checkpoint counter with AtomicUsize - Set warmup_steps=0 and max_training_steps_per_epoch=300 in early-stop tests - RealDataLoader respects TEST_DATA_DIR for CI PVC layout - Add collapse_warmup_capacity to gpu_smoketest DQNConfig - Drain CUDA context between test binaries - Detached HEAD checkout prevents local branch corruption - GPU pipeline tests: fix BF16 dtype and rank-1 squeeze assertions - OOD input handling tests use use_gpu: true Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
470 lines
15 KiB
Rust
470 lines
15 KiB
Rust
#![allow(
|
|
clippy::assertions_on_constants,
|
|
clippy::assertions_on_result_states,
|
|
clippy::clone_on_copy,
|
|
clippy::decimal_literal_representation,
|
|
clippy::doc_markdown,
|
|
clippy::empty_line_after_doc_comments,
|
|
clippy::field_reassign_with_default,
|
|
clippy::get_unwrap,
|
|
clippy::identity_op,
|
|
clippy::inconsistent_digit_grouping,
|
|
clippy::indexing_slicing,
|
|
clippy::integer_division,
|
|
clippy::len_zero,
|
|
clippy::let_underscore_must_use,
|
|
clippy::manual_div_ceil,
|
|
clippy::manual_let_else,
|
|
clippy::manual_range_contains,
|
|
clippy::modulo_arithmetic,
|
|
clippy::needless_range_loop,
|
|
clippy::non_ascii_literal,
|
|
clippy::redundant_clone,
|
|
clippy::shadow_reuse,
|
|
clippy::shadow_same,
|
|
clippy::shadow_unrelated,
|
|
clippy::single_match_else,
|
|
clippy::str_to_string,
|
|
clippy::string_slice,
|
|
clippy::tests_outside_test_module,
|
|
clippy::too_many_lines,
|
|
clippy::unnecessary_wraps,
|
|
clippy::unseparated_literal_suffix,
|
|
clippy::use_debug,
|
|
clippy::useless_vec,
|
|
clippy::wildcard_enum_match_arm,
|
|
clippy::else_if_without_else,
|
|
clippy::expect_used,
|
|
clippy::missing_const_for_fn,
|
|
clippy::similar_names,
|
|
clippy::type_complexity,
|
|
clippy::collapsible_else_if,
|
|
clippy::doc_lazy_continuation,
|
|
clippy::items_after_test_module,
|
|
clippy::map_clone,
|
|
clippy::multiple_unsafe_ops_per_block,
|
|
clippy::unwrap_or_default,
|
|
clippy::assign_op_pattern,
|
|
clippy::needless_borrow,
|
|
clippy::println_empty_string,
|
|
clippy::unnecessary_cast,
|
|
clippy::used_underscore_binding,
|
|
clippy::create_dir,
|
|
clippy::implicit_saturating_sub,
|
|
clippy::exit,
|
|
clippy::expect_fun_call,
|
|
clippy::too_many_arguments,
|
|
clippy::unnecessary_map_or,
|
|
clippy::unwrap_used,
|
|
dead_code,
|
|
unused_imports,
|
|
unused_variables,
|
|
clippy::cloned_ref_to_slice_refs,
|
|
clippy::neg_multiply,
|
|
clippy::while_let_loop,
|
|
clippy::bool_assert_comparison,
|
|
clippy::excessive_precision,
|
|
clippy::trivially_copy_pass_by_ref,
|
|
clippy::op_ref,
|
|
clippy::redundant_closure,
|
|
clippy::unnecessary_lazy_evaluations,
|
|
clippy::if_then_some_else_none,
|
|
clippy::unnecessary_to_owned,
|
|
clippy::single_component_path_imports,
|
|
)]
|
|
//! Feature Normalization Tests
|
|
//!
|
|
//! Tests for percentile-based feature clipping to prevent outliers
|
|
//! from crushing the feature distribution during min-max normalization.
|
|
//!
|
|
//! ## Problem
|
|
//!
|
|
//! OBV (On-Balance Volume) features accumulate signed volume over time,
|
|
//! leading to extreme outliers (e.g., -863K to +863K). When using
|
|
//! min-max normalization, these outliers compress 51/54 other features
|
|
//! into a narrow range [0.48, 0.52], making them indistinguishable.
|
|
//!
|
|
//! ## Solution
|
|
//!
|
|
//! Apply percentile clipping (1st to 99th percentile) BEFORE min-max
|
|
//! normalization. This preserves 98% of data while preventing outliers
|
|
//! from dominating the normalization scale.
|
|
|
|
use std::f64;
|
|
use tracing::info;
|
|
|
|
/// Compute percentile value from sorted data
|
|
fn percentile(sorted_data: &[f64], p: f64) -> f64 {
|
|
assert!(
|
|
!sorted_data.is_empty(),
|
|
"Cannot compute percentile of empty data"
|
|
);
|
|
assert!(p >= 0.0 && p <= 1.0, "Percentile must be in [0, 1]");
|
|
|
|
let idx = (sorted_data.len() as f64 * p).round() as usize;
|
|
let idx = idx.min(sorted_data.len() - 1);
|
|
sorted_data[idx]
|
|
}
|
|
|
|
/// Apply percentile clipping to features
|
|
fn clip_features_by_percentile(features: &[f64], p_low: f64, p_high: f64) -> Vec<f64> {
|
|
let mut sorted = features.to_vec();
|
|
sorted.sort_by(|a, b| a.partial_cmp(b).unwrap());
|
|
|
|
let p1 = percentile(&sorted, p_low);
|
|
let p99 = percentile(&sorted, p_high);
|
|
|
|
info!(p_low, p1, "Percentile lower bound");
|
|
info!(p_high, p99, "Percentile upper bound");
|
|
|
|
features.iter().map(|&x| x.clamp(p1, p99)).collect()
|
|
}
|
|
|
|
/// Normalize features to [0, 1] range
|
|
fn normalize_min_max(features: &[f64]) -> Vec<f64> {
|
|
let min = features.iter().copied().fold(f64::INFINITY, f64::min);
|
|
let max = features.iter().copied().fold(f64::NEG_INFINITY, f64::max);
|
|
let range = max - min;
|
|
|
|
if range.abs() < 1e-10 {
|
|
// All values are the same, return 0.5
|
|
return vec![0.5; features.len()];
|
|
}
|
|
|
|
features.iter().map(|&x| (x - min) / range).collect()
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn test_percentile_computation() {
|
|
let data = vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0];
|
|
|
|
// Test extremes
|
|
assert!((percentile(&data, 0.0) - 1.0).abs() < 1e-10);
|
|
assert!((percentile(&data, 1.0) - 10.0).abs() < 1e-10);
|
|
|
|
// Test median (50th percentile)
|
|
let p50 = percentile(&data, 0.5);
|
|
assert!(
|
|
p50 >= 5.0 && p50 <= 6.0,
|
|
"Median should be ~5.5, got {}",
|
|
p50
|
|
);
|
|
|
|
// Test 99th percentile
|
|
let p99 = percentile(&data, 0.99);
|
|
assert!(
|
|
p99 >= 9.0 && p99 <= 10.0,
|
|
"99th percentile should be ~10, got {}",
|
|
p99
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn test_clip_features_without_outliers() {
|
|
// Data without outliers - clipping should have minimal effect
|
|
let features = vec![10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0, 80.0, 90.0, 100.0];
|
|
let clipped = clip_features_by_percentile(&features, 0.01, 0.99);
|
|
|
|
// Most values should be unchanged
|
|
for i in 1..9 {
|
|
assert!(
|
|
(clipped[i] - features[i]).abs() < 1.0,
|
|
"Value {} should be mostly unchanged",
|
|
i
|
|
);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn test_clip_features_with_extreme_outliers() {
|
|
// Test that clipping works when percentiles exclude outliers
|
|
// Key insight: outliers must be OUTSIDE the 1st-99th percentile range
|
|
let mut features = Vec::new();
|
|
|
|
// Add 100 normal values in range [-100, 100]
|
|
for i in -50..50 {
|
|
features.push(i as f64 * 2.0);
|
|
}
|
|
|
|
// Add extreme outliers at beginning and end
|
|
// These will be at the 0.5% and 99.5% positions
|
|
features.insert(0, -863_000.0);
|
|
features.push(863_000.0);
|
|
|
|
info!(total_features = features.len(), "Total features");
|
|
|
|
let clipped = clip_features_by_percentile(&features, 0.02, 0.98);
|
|
|
|
// Extreme outliers should be clipped to 2nd and 98th percentile values
|
|
let min_clipped = clipped.iter().copied().fold(f64::INFINITY, f64::min);
|
|
let max_clipped = clipped.iter().copied().fold(f64::NEG_INFINITY, f64::max);
|
|
|
|
info!(min_clipped, max_clipped, "Clipped range");
|
|
|
|
// After clipping, outliers should be replaced with percentile boundary values
|
|
// which are within the normal range
|
|
assert!(
|
|
max_clipped < 200.0,
|
|
"Max should be clipped to reasonable range, got {}",
|
|
max_clipped
|
|
);
|
|
assert!(
|
|
min_clipped > -200.0,
|
|
"Min should be clipped to reasonable range, got {}",
|
|
min_clipped
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn test_normalize_min_max_basic() {
|
|
let features = vec![0.0, 25.0, 50.0, 75.0, 100.0];
|
|
let normalized = normalize_min_max(&features);
|
|
|
|
// Check bounds
|
|
assert!((normalized[0] - 0.0).abs() < 1e-10, "Min should map to 0");
|
|
assert!((normalized[4] - 1.0).abs() < 1e-10, "Max should map to 1");
|
|
|
|
// Check midpoint
|
|
assert!(
|
|
(normalized[2] - 0.5).abs() < 1e-10,
|
|
"Midpoint should map to 0.5"
|
|
);
|
|
|
|
// Check all values in [0, 1]
|
|
for val in &normalized {
|
|
assert!(
|
|
*val >= 0.0 && *val <= 1.0,
|
|
"Normalized value {} out of range",
|
|
val
|
|
);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn test_normalize_constant_features() {
|
|
// All values the same - should return 0.5
|
|
let features = vec![42.0; 10];
|
|
let normalized = normalize_min_max(&features);
|
|
|
|
for val in &normalized {
|
|
assert!(
|
|
(val - 0.5).abs() < 1e-10,
|
|
"Constant features should normalize to 0.5"
|
|
);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn test_full_pipeline_with_outliers() {
|
|
// Simulate realistic scenario: 54 features with OBV outliers
|
|
let mut features = Vec::new();
|
|
|
|
// 51 normal features (range: 0-100)
|
|
for _ in 0..51 {
|
|
for i in 0..10 {
|
|
features.push(i as f64 * 10.0);
|
|
}
|
|
}
|
|
|
|
// 3 OBV features with extreme outliers
|
|
for _ in 0..3 {
|
|
features.push(-863_000.0);
|
|
features.push(863_000.0);
|
|
for i in -5..5 {
|
|
features.push(i as f64 * 100.0);
|
|
}
|
|
}
|
|
|
|
info!("Feature Normalization Test");
|
|
info!(total_features = features.len(), "Total features");
|
|
|
|
// BEFORE: Direct normalization (broken)
|
|
let normalized_before = normalize_min_max(&features);
|
|
let min_before = normalized_before
|
|
.iter()
|
|
.copied()
|
|
.fold(f64::INFINITY, f64::min);
|
|
let max_before = normalized_before
|
|
.iter()
|
|
.copied()
|
|
.fold(f64::NEG_INFINITY, f64::max);
|
|
|
|
info!("BEFORE percentile clipping");
|
|
info!(
|
|
feature_min = features.iter().copied().fold(f64::INFINITY, f64::min),
|
|
feature_max = features.iter().copied().fold(f64::NEG_INFINITY, f64::max),
|
|
"Feature range before clipping"
|
|
);
|
|
info!(min_before, max_before, "Normalized range before clipping");
|
|
|
|
// Count how many values are in narrow range [0.48, 0.52]
|
|
let crushed_before = normalized_before
|
|
.iter()
|
|
.filter(|&&x| x >= 0.48 && x <= 0.52)
|
|
.count();
|
|
info!(
|
|
crushed_before,
|
|
crushed_before_pct = 100.0 * crushed_before as f64 / normalized_before.len() as f64,
|
|
"Values crushed to [0.48, 0.52] before clipping"
|
|
);
|
|
|
|
// AFTER: Percentile clipping + normalization (fixed)
|
|
let clipped = clip_features_by_percentile(&features, 0.01, 0.99);
|
|
let normalized_after = normalize_min_max(&clipped);
|
|
let min_after = normalized_after
|
|
.iter()
|
|
.copied()
|
|
.fold(f64::INFINITY, f64::min);
|
|
let max_after = normalized_after
|
|
.iter()
|
|
.copied()
|
|
.fold(f64::NEG_INFINITY, f64::max);
|
|
|
|
info!("AFTER percentile clipping");
|
|
info!(
|
|
clipped_min = clipped.iter().copied().fold(f64::INFINITY, f64::min),
|
|
clipped_max = clipped.iter().copied().fold(f64::NEG_INFINITY, f64::max),
|
|
"Clipped feature range"
|
|
);
|
|
info!(min_after, max_after, "Normalized range after clipping");
|
|
|
|
// Count distribution after fix
|
|
let crushed_after = normalized_after
|
|
.iter()
|
|
.filter(|&&x| x >= 0.48 && x <= 0.52)
|
|
.count();
|
|
info!(
|
|
crushed_after,
|
|
crushed_after_pct = 100.0 * crushed_after as f64 / normalized_after.len() as f64,
|
|
"Values crushed to [0.48, 0.52] after clipping"
|
|
);
|
|
|
|
// Assert fix works
|
|
assert!(
|
|
crushed_after < crushed_before / 2,
|
|
"Percentile clipping should reduce feature crushing significantly"
|
|
);
|
|
|
|
// Verify full utilization of [0, 1] range
|
|
assert!(
|
|
(min_after - 0.0).abs() < 0.1,
|
|
"Min should be close to 0 after fix"
|
|
);
|
|
assert!(
|
|
(max_after - 1.0).abs() < 0.1,
|
|
"Max should be close to 1 after fix"
|
|
);
|
|
|
|
info!("Fix Validated");
|
|
info!("Percentile clipping prevents outliers from crushing feature distribution");
|
|
}
|
|
|
|
#[test]
|
|
fn test_obv_realistic_scenario() {
|
|
// Realistic OBV outlier scenario from ES_FUT_180d.parquet
|
|
let mut features = Vec::new();
|
|
|
|
// Generate OBV-like data: accumulates over time
|
|
let mut obv = 0.0;
|
|
for i in 0..1000 {
|
|
let volume = 100.0 + (i as f64 % 50.0);
|
|
let direction = if i % 3 == 0 { 1.0 } else { -1.0 };
|
|
obv += volume * direction;
|
|
features.push(obv);
|
|
}
|
|
|
|
// Add other normal features (RSI, MACD, etc.)
|
|
for _ in 0..53 {
|
|
for i in 0..1000 {
|
|
features.push((i % 100) as f64);
|
|
}
|
|
}
|
|
|
|
info!("OBV Realistic Scenario");
|
|
let orig_min = features.iter().copied().fold(f64::INFINITY, f64::min);
|
|
let orig_max = features.iter().copied().fold(f64::NEG_INFINITY, f64::max);
|
|
info!(orig_min, orig_max, "Original feature range");
|
|
|
|
// Apply percentile clipping
|
|
let clipped = clip_features_by_percentile(&features, 0.01, 0.99);
|
|
let clipped_min = clipped.iter().copied().fold(f64::INFINITY, f64::min);
|
|
let clipped_max = clipped.iter().copied().fold(f64::NEG_INFINITY, f64::max);
|
|
info!(clipped_min, clipped_max, "Clipped feature range");
|
|
|
|
// Range should be much smaller after clipping
|
|
let orig_range = orig_max - orig_min;
|
|
let clipped_range = clipped_max - clipped_min;
|
|
info!(
|
|
orig_range,
|
|
clipped_range,
|
|
reduction_pct = 100.0 * (1.0 - clipped_range / orig_range),
|
|
"Range reduction after clipping"
|
|
);
|
|
|
|
assert!(
|
|
clipped_range < orig_range * 0.5,
|
|
"Clipping should reduce range by at least 50%"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn test_edge_case_all_same_value() {
|
|
let features = vec![42.0; 100];
|
|
let clipped = clip_features_by_percentile(&features, 0.01, 0.99);
|
|
let normalized = normalize_min_max(&clipped);
|
|
|
|
// All values should normalize to 0.5
|
|
for val in &normalized {
|
|
assert!((val - 0.5).abs() < 1e-10);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn test_edge_case_two_values() {
|
|
let features = vec![0.0, 100.0];
|
|
let clipped = clip_features_by_percentile(&features, 0.01, 0.99);
|
|
let normalized = normalize_min_max(&clipped);
|
|
|
|
assert!((normalized[0] - 0.0).abs() < 1e-10);
|
|
assert!((normalized[1] - 1.0).abs() < 1e-10);
|
|
}
|
|
|
|
#[test]
|
|
fn test_preserves_98_percent_of_data() {
|
|
// Generate 10000 normal values + 200 outliers
|
|
let mut features = Vec::new();
|
|
|
|
// 98% normal (0-100)
|
|
for i in 0..9800 {
|
|
features.push((i % 100) as f64);
|
|
}
|
|
|
|
// 2% outliers (-100000, +100000)
|
|
for _ in 0..100 {
|
|
features.push(-100_000.0);
|
|
features.push(100_000.0);
|
|
}
|
|
|
|
let clipped = clip_features_by_percentile(&features, 0.01, 0.99);
|
|
|
|
// Count how many normal values are preserved exactly
|
|
let preserved = features
|
|
.iter()
|
|
.filter(|&&x| x >= 0.0 && x <= 100.0)
|
|
.filter(|&&x| clipped.contains(&x))
|
|
.count();
|
|
|
|
let preservation_rate = preserved as f64 / 9800.0;
|
|
info!(preservation_rate_pct = preservation_rate * 100.0, "Preservation rate");
|
|
|
|
assert!(
|
|
preservation_rate > 0.95,
|
|
"At least 95% of normal data should be preserved"
|
|
);
|
|
}
|
|
}
|