b4178952d4
- Cast input to weight dtype in DQN residual, rmsnorm, noisy_layers - Set use_gpu=true in QNetworkConfig defaults and all config sites - Resolve BF16 boundary mismatches in attention, curiosity, branching, distributional_dueling across ml-dqn - GPU-resident regime ops with BF16 boundary casts, eliminate .expect() in CUDA paths - Eliminate all Device::Cpu fallbacks — GPU-only across 10 ML crates - PPO: cast logits to F32 before softmax, cast batch tensors to training dtype - Gradient collapse detection for RegimeConditionalDQN - Wire halt_grad_collapse from CUDA guard kernel to halt training - Dead neuron detection uses active network VarMap + squeeze factored readback - Increment gradient_logging_step in GPU PER path - Gradient collapse warmup guards use original buffer_size - Cap training steps per epoch + tracing migration - Replace Tensor::all() with sum_all() for pinned Candle compatibility Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1160 lines
36 KiB
Rust
1160 lines
36 KiB
Rust
//! Feature Normalization Pipeline (Wave C + Wave D)
|
||
//!
|
||
//! This module implements online/incremental normalization for 225-dimension ML features.
|
||
//! Uses category-specific strategies for optimal ML model convergence.
|
||
//!
|
||
//! ## Normalization Categories
|
||
//! 1. **Price Features** (60 features): Z-score normalization (mean=0, std=1)
|
||
//! 2. **Volume Features** (40 features): Percentile rank normalization (0-1)
|
||
//! 3. **Microstructure Features** (50 features): Log transform + z-score
|
||
//! 4. **Technical Indicators** (10 features): Already normalized (0-1 or -1 to +1)
|
||
//! 5. **Time/Statistical Features** (96 features): Already normalized
|
||
//! 6. **CUSUM Features** (10 features, Wave D): Z-score normalization (indices 201-210)
|
||
//! 7. **ADX Features** (5 features, Wave D): Min-max scaling `[0,1]` (indices 211-215)
|
||
//! 8. **Transition Features** (5 features, Wave D): Z-score normalization (indices 216-220)
|
||
//! 9. **Adaptive Features** (4 features, Wave D): Min-max scaling `[0,2]` (indices 221-224)
|
||
//!
|
||
//! ## Performance
|
||
//! - Target: <200μs for normalizing all 225 features per bar
|
||
//! - Memory: <20KB per symbol (rolling statistics, with lazy allocation)
|
||
//! - Online: No batch recomputation required
|
||
//!
|
||
//! ## Memory Optimization (Wave G)
|
||
//! - Fixed-size ring buffers replace `VecDeque` (zero heap allocations)
|
||
//! - Lazy buffer initialization (only allocate when first value arrives)
|
||
//! - Projected savings: 49.4 KB → ~10 KB per symbol (80% reduction)
|
||
//!
|
||
//! ## Usage
|
||
//! ```rust
|
||
//! use ml::features::normalization::FeatureNormalizer;
|
||
//!
|
||
//! let mut normalizer = FeatureNormalizer::new();
|
||
//! let mut features = [0.0; 225]; // Raw features from extraction
|
||
//! normalizer.normalize(&mut features)?; // In-place normalization
|
||
//! ```
|
||
|
||
use anyhow::Result;
|
||
use tracing::warn;
|
||
|
||
const EPSILON: f64 = 1e-8; // Prevent division by zero
|
||
|
||
/// Fixed-size ring buffer with zero heap allocations
|
||
///
|
||
/// Uses compile-time const generics for inline storage on the stack.
|
||
/// Provides O(1) push/pop operations with automatic overwriting of oldest values.
|
||
///
|
||
/// ## Memory Layout (Wave G Optimization)
|
||
/// - `data`: `[T; N]` stored inline (no heap allocation, no Option overhead)
|
||
/// - `head`: Current write position (wraps around at N)
|
||
/// - `len`: Number of valid elements (0..=N)
|
||
///
|
||
/// ## Memory Optimization
|
||
/// - OLD: [Option<T>; N] → 16 bytes per f64 element (8 value + 8 discriminant)
|
||
/// - NEW: [T; N] → 8 bytes per f64 element (50% reduction)
|
||
/// - Savings: 100 elements × 8 bytes = 800 bytes per buffer
|
||
///
|
||
/// ## Performance
|
||
/// - Push: O(1)
|
||
/// - Iteration: O(N)
|
||
/// - Mean/StdDev: O(N)
|
||
/// - Memory: `N × sizeof(T)` bytes (stack-allocated)
|
||
///
|
||
/// ## Example
|
||
/// ```rust
|
||
/// let mut buffer: RingBuffer<f64, 100> = RingBuffer::new();
|
||
/// buffer.push(42.0);
|
||
/// assert_eq!(buffer.len(), 1);
|
||
/// assert_eq!(buffer.mean(), 42.0);
|
||
/// ```
|
||
#[derive(Clone, Debug)]
|
||
pub struct RingBuffer<T: Copy + Default, const N: usize> {
|
||
data: [T; N],
|
||
head: usize,
|
||
len: usize,
|
||
}
|
||
|
||
impl<T: Copy + Default, const N: usize> Default for RingBuffer<T, N> {
|
||
fn default() -> Self {
|
||
Self::new()
|
||
}
|
||
}
|
||
|
||
impl<T: Copy + Default, const N: usize> RingBuffer<T, N> {
|
||
/// Create new empty ring buffer
|
||
pub fn new() -> Self {
|
||
Self {
|
||
data: [T::default(); N],
|
||
head: 0,
|
||
len: 0,
|
||
}
|
||
}
|
||
|
||
/// Push value into ring buffer (overwrites oldest if full)
|
||
pub const fn push(&mut self, value: T) {
|
||
if N == 0 {
|
||
return; // Zero-capacity buffer: cannot store any values
|
||
}
|
||
self.data[self.head] = value;
|
||
self.head = (self.head + 1) % N;
|
||
if self.len < N {
|
||
self.len += 1;
|
||
}
|
||
}
|
||
|
||
/// Get current number of elements
|
||
pub const fn len(&self) -> usize {
|
||
self.len
|
||
}
|
||
|
||
/// Check if buffer is empty
|
||
pub const fn is_empty(&self) -> bool {
|
||
self.len == 0
|
||
}
|
||
|
||
/// Iterate over valid elements in insertion order
|
||
pub fn iter(&self) -> impl Iterator<Item = &T> + '_ {
|
||
let start_idx = if self.len < N { 0 } else { self.head };
|
||
|
||
(0..self.len).map(move |i| {
|
||
let idx = (start_idx + i) % N;
|
||
&self.data[idx]
|
||
})
|
||
}
|
||
|
||
/// Clear all elements (resets to default values)
|
||
pub fn clear(&mut self) {
|
||
self.data = [T::default(); N];
|
||
self.head = 0;
|
||
self.len = 0;
|
||
}
|
||
}
|
||
|
||
// Extension trait for f64 statistics
|
||
impl<const N: usize> RingBuffer<f64, N> {
|
||
/// Compute mean of all values
|
||
pub fn mean(&self) -> f64 {
|
||
if self.len == 0 {
|
||
return 0.0;
|
||
}
|
||
let sum: f64 = self.iter().copied().sum();
|
||
sum / self.len as f64
|
||
}
|
||
|
||
/// Compute standard deviation
|
||
pub fn std_dev(&self) -> f64 {
|
||
if self.len < 2 {
|
||
return 0.0;
|
||
}
|
||
let mean = self.mean();
|
||
let variance: f64 =
|
||
self.iter().map(|&x| (x - mean).powi(2)).sum::<f64>() / (self.len - 1) as f64;
|
||
variance.max(0.0).sqrt()
|
||
}
|
||
|
||
/// Get minimum value
|
||
pub fn min(&self) -> f64 {
|
||
self.iter().copied().fold(f64::MAX, f64::min)
|
||
}
|
||
|
||
/// Get maximum value
|
||
pub fn max(&self) -> f64 {
|
||
self.iter().copied().fold(f64::MIN, f64::max)
|
||
}
|
||
}
|
||
|
||
/// Main feature normalizer coordinating all normalization strategies
|
||
#[derive(Debug)]
|
||
pub struct FeatureNormalizer {
|
||
/// Price feature normalizers (indices 15-74, 60 features)
|
||
price_normalizers: Vec<RollingZScore>,
|
||
|
||
/// Volume feature normalizers (indices 75-114, 40 features)
|
||
volume_normalizers: Vec<RollingPercentileRank>,
|
||
|
||
/// Microstructure feature normalizers (indices 115-164, 50 features)
|
||
microstructure_normalizers: Vec<LogZScoreNormalizer>,
|
||
|
||
/// CUSUM feature normalizers (indices 201-210, 10 features, Wave D)
|
||
cusum_normalizers: Vec<RollingZScore>,
|
||
|
||
/// ADX feature normalizers (indices 211-215, 5 features, Wave D)
|
||
adx_normalizers: Vec<RollingPercentileRank>,
|
||
|
||
/// Transition feature normalizers (indices 216-220, 5 features, Wave D)
|
||
transition_normalizers: Vec<RollingZScore>,
|
||
|
||
/// Adaptive feature normalizers (indices 221-224, 4 features, Wave D)
|
||
adaptive_normalizers: Vec<RollingPercentileRank>,
|
||
|
||
/// NaN handler for input validation
|
||
nan_handler: NaNHandler,
|
||
}
|
||
|
||
impl FeatureNormalizer {
|
||
/// Create new feature normalizer with default window sizes
|
||
pub fn new() -> Self {
|
||
Self::with_config(50, 50, 20, 30)
|
||
}
|
||
|
||
/// Create feature normalizer with custom window sizes
|
||
///
|
||
/// # Arguments
|
||
/// - `price_window`: Rolling window for price features (recommended: 50)
|
||
/// - `volume_window`: Rolling window for volume features (recommended: 50)
|
||
/// - `microstructure_window`: Rolling window for microstructure features (recommended: 20)
|
||
/// - `regime_window`: Rolling window for Wave D regime features (recommended: 30)
|
||
pub fn with_config(
|
||
price_window: usize,
|
||
volume_window: usize,
|
||
microstructure_window: usize,
|
||
regime_window: usize,
|
||
) -> Self {
|
||
Self {
|
||
// 60 price features (indices 15-74)
|
||
price_normalizers: (0..60).map(|_| RollingZScore::new(price_window)).collect(),
|
||
|
||
// 40 volume features (indices 75-114)
|
||
volume_normalizers: (0..40)
|
||
.map(|_| RollingPercentileRank::new(volume_window))
|
||
.collect(),
|
||
|
||
// 50 microstructure features (indices 115-164)
|
||
// Scale factors: roll=1.0, amihud=1e8, corwin=100.0, others=1.0
|
||
microstructure_normalizers: vec![
|
||
// Roll spread (index 115)
|
||
LogZScoreNormalizer::new(1.0, microstructure_window),
|
||
// Amihud illiquidity (index 116)
|
||
LogZScoreNormalizer::new(1e8, microstructure_window),
|
||
// Corwin-Schultz spread (index 117)
|
||
LogZScoreNormalizer::new(100.0, microstructure_window),
|
||
// Remaining 47 microstructure features (indices 118-164)
|
||
]
|
||
.into_iter()
|
||
.chain((0..47).map(|_| LogZScoreNormalizer::new(1.0, microstructure_window)))
|
||
.collect(),
|
||
|
||
// Wave D: 10 CUSUM features (indices 201-210)
|
||
cusum_normalizers: (0..10).map(|_| RollingZScore::new(regime_window)).collect(),
|
||
|
||
// Wave D: 5 ADX features (indices 211-215)
|
||
adx_normalizers: (0..5)
|
||
.map(|_| RollingPercentileRank::new(regime_window))
|
||
.collect(),
|
||
|
||
// Wave D: 5 transition features (indices 216-220)
|
||
transition_normalizers: (0..5).map(|_| RollingZScore::new(regime_window)).collect(),
|
||
|
||
// Wave D: 4 adaptive features (indices 221-224)
|
||
adaptive_normalizers: (0..4)
|
||
.map(|_| RollingPercentileRank::new(regime_window))
|
||
.collect(),
|
||
|
||
nan_handler: NaNHandler::new(),
|
||
}
|
||
}
|
||
|
||
/// Normalize 225-dimensional feature vector in-place
|
||
///
|
||
/// # Arguments
|
||
/// - `features`: Mutable reference to feature vector (modified in-place)
|
||
///
|
||
/// # Returns
|
||
/// - `Ok(())` if normalization successful
|
||
/// - `Err(anyhow::Error)` if any feature is non-finite after normalization
|
||
///
|
||
/// # Feature Ranges
|
||
/// - Indices 0-4: OHLCV (already normalized, no change)
|
||
/// - Indices 5-14: Technical indicators (already normalized, no change)
|
||
/// - Indices 15-74: Price features (z-score normalization)
|
||
/// - Indices 75-114: Volume features (percentile rank normalization)
|
||
/// - Indices 115-164: Microstructure features (log + z-score normalization)
|
||
/// - Indices 165-224: Time/statistical features (already normalized, no change)
|
||
pub fn normalize(&mut self, features: &mut [f64; 225]) -> Result<()> {
|
||
// 1. Handle NaN/Inf in input (impute with last valid value)
|
||
self.nan_handler.handle_input(features);
|
||
|
||
// 2. Validate input (all features must be finite after imputation)
|
||
for (i, &val) in features.iter().enumerate() {
|
||
if !val.is_finite() {
|
||
anyhow::bail!("Feature {} is non-finite after imputation: {}", i, val);
|
||
}
|
||
}
|
||
|
||
// 3. Normalize OHLCV (indices 0-4) - ALREADY NORMALIZED, skip
|
||
|
||
// 4. Normalize Technical Indicators (indices 5-14) - ALREADY NORMALIZED, skip
|
||
|
||
// 5. Normalize Price Patterns (indices 15-74)
|
||
for (idx, feature) in features[15..75].iter_mut().enumerate() {
|
||
*feature = self.price_normalizers[idx].update(*feature);
|
||
}
|
||
|
||
// 6. Normalize Volume Patterns (indices 75-114)
|
||
for (idx, feature) in features[75..115].iter_mut().enumerate() {
|
||
*feature = self.volume_normalizers[idx].update(*feature);
|
||
}
|
||
|
||
// 7. Normalize Microstructure (indices 115-164)
|
||
for (idx, feature) in features[115..165].iter_mut().enumerate() {
|
||
*feature = self.microstructure_normalizers[idx].update(*feature);
|
||
}
|
||
|
||
// 8. Time features (165-174) - ALREADY NORMALIZED, skip
|
||
|
||
// 9. Statistical features (175-200) - ALREADY NORMALIZED, skip
|
||
|
||
// 10. Normalize CUSUM Features (indices 201-210, Wave D)
|
||
for (idx, feature) in features[201..211].iter_mut().enumerate() {
|
||
*feature = self.cusum_normalizers[idx].update(*feature);
|
||
}
|
||
|
||
// 11. Normalize ADX Features (indices 211-215, Wave D)
|
||
// ADX features are already in [0, 100] range, scale to [0, 1]
|
||
for (idx, feature) in features[211..216].iter_mut().enumerate() {
|
||
let scaled = *feature / 100.0; // Scale from [0, 100] to [0, 1]
|
||
*feature = self.adx_normalizers[idx].update(scaled);
|
||
}
|
||
|
||
// 12. Normalize Transition Features (indices 216-220, Wave D)
|
||
for (idx, feature) in features[216..221].iter_mut().enumerate() {
|
||
*feature = self.transition_normalizers[idx].update(*feature);
|
||
}
|
||
|
||
// 13. Normalize Adaptive Features (indices 221-224, Wave D)
|
||
for (idx, feature) in features[221..225].iter_mut().enumerate() {
|
||
*feature = self.adaptive_normalizers[idx].update(*feature);
|
||
}
|
||
|
||
// 14. Final validation (all features must be finite)
|
||
for (i, &val) in features.iter().enumerate() {
|
||
if !val.is_finite() {
|
||
anyhow::bail!("Normalized feature {} is non-finite: {}", i, val);
|
||
}
|
||
}
|
||
|
||
Ok(())
|
||
}
|
||
|
||
/// Reset all normalizers (useful for backtesting)
|
||
pub fn reset(&mut self) {
|
||
for norm in &mut self.price_normalizers {
|
||
norm.reset();
|
||
}
|
||
for norm in &mut self.volume_normalizers {
|
||
norm.reset();
|
||
}
|
||
for norm in &mut self.microstructure_normalizers {
|
||
norm.reset();
|
||
}
|
||
// Wave D normalizers
|
||
for norm in &mut self.cusum_normalizers {
|
||
norm.reset();
|
||
}
|
||
for norm in &mut self.adx_normalizers {
|
||
norm.reset();
|
||
}
|
||
for norm in &mut self.transition_normalizers {
|
||
norm.reset();
|
||
}
|
||
for norm in &mut self.adaptive_normalizers {
|
||
norm.reset();
|
||
}
|
||
self.nan_handler.reset();
|
||
}
|
||
|
||
/// Get normalization statistics for debugging
|
||
pub fn get_stats(&self) -> NormalizationStats {
|
||
NormalizationStats {
|
||
price_mean: self
|
||
.price_normalizers
|
||
.first()
|
||
.map(|n| n.mean)
|
||
.unwrap_or(0.0),
|
||
price_std: self
|
||
.price_normalizers
|
||
.first()
|
||
.map(|n| n.std())
|
||
.unwrap_or(0.0),
|
||
volume_percentile: self
|
||
.volume_normalizers
|
||
.first()
|
||
.and_then(|n| {
|
||
n.buffer
|
||
.as_ref()
|
||
.map(|b| b.len() as f64 / n.window_size as f64)
|
||
})
|
||
.unwrap_or(0.0),
|
||
nan_count: self.nan_handler.total_nan_count(),
|
||
}
|
||
}
|
||
}
|
||
|
||
impl Default for FeatureNormalizer {
|
||
fn default() -> Self {
|
||
Self::new()
|
||
}
|
||
}
|
||
|
||
/// Normalization statistics for debugging
|
||
#[derive(Debug, Clone)]
|
||
pub struct NormalizationStats {
|
||
pub price_mean: f64,
|
||
pub price_std: f64,
|
||
pub volume_percentile: f64,
|
||
pub nan_count: u32,
|
||
}
|
||
|
||
//
|
||
// Category 1: Z-Score Normalization for Price Features
|
||
//
|
||
|
||
/// Rolling z-score normalization with lazy buffer allocation
|
||
///
|
||
/// Uses fixed-size ring buffer (const generic) for zero heap allocations.
|
||
/// Buffer is lazily initialized only when first value arrives.
|
||
///
|
||
/// ## Memory Optimization
|
||
/// - OLD: `VecDeque` with `capacity=window_size` → heap allocation
|
||
/// - NEW: Option<`RingBuffer`<f64, 100>> → None until first value
|
||
/// - Savings: 800 bytes per normalizer (100 × 8 bytes)
|
||
#[derive(Debug)]
|
||
pub struct RollingZScore {
|
||
window_size: usize,
|
||
/// Lazy ring buffer (None until first value)
|
||
buffer: Option<RingBuffer<f64, 100>>,
|
||
mean: f64,
|
||
m2: f64, // Sum of squared deviations (for std)
|
||
count: usize,
|
||
}
|
||
|
||
impl RollingZScore {
|
||
/// Create new z-score normalizer with specified window size
|
||
///
|
||
/// Note: Buffer is NOT allocated until first value arrives (lazy init)
|
||
pub fn new(window_size: usize) -> Self {
|
||
Self {
|
||
window_size: window_size.min(100), // Cap at 100 (ring buffer size)
|
||
buffer: None, // Lazy allocation
|
||
mean: 0.0,
|
||
m2: 0.0,
|
||
count: 0,
|
||
}
|
||
}
|
||
|
||
/// Update normalizer with new value and return normalized value
|
||
///
|
||
/// # Returns
|
||
/// - Normalized value clipped to ±3σ
|
||
/// - Returns 0.0 during warmup period (first 10 values)
|
||
pub fn update(&mut self, value: f64) -> f64 {
|
||
// Lazy buffer initialization
|
||
let buffer = self.buffer.get_or_insert_with(RingBuffer::new);
|
||
|
||
// Track old value for rolling window update
|
||
let old_val = (buffer.len() >= self.window_size).then(|| {
|
||
// Extract oldest value before push (will be overwritten)
|
||
let start_idx = buffer.head;
|
||
buffer.data[start_idx]
|
||
});
|
||
|
||
// Add new value
|
||
buffer.push(value);
|
||
|
||
if let Some(old_val) = old_val {
|
||
// Rolling window: remove old value, add new value
|
||
let delta = value - old_val;
|
||
self.mean += delta / self.count as f64;
|
||
self.m2 += delta * (value - self.mean + old_val - self.mean);
|
||
} else {
|
||
// Warmup phase: incremental update
|
||
self.count = buffer.len();
|
||
let delta = value - self.mean;
|
||
self.mean += delta / self.count as f64;
|
||
let delta2 = value - self.mean;
|
||
self.m2 += delta * delta2;
|
||
}
|
||
|
||
// Warmup period: return 0.0 for first 10 values (insufficient data)
|
||
if self.count < 10 {
|
||
return 0.0;
|
||
}
|
||
|
||
// Compute normalized value
|
||
let std = self.std();
|
||
let normalized = (value - self.mean) / (std + EPSILON);
|
||
|
||
// Clip to ±3σ (99.7% of Gaussian distribution)
|
||
normalized.clamp(-3.0, 3.0)
|
||
}
|
||
|
||
/// Get current standard deviation
|
||
pub fn std(&self) -> f64 {
|
||
if self.count < 2 {
|
||
return 0.0;
|
||
}
|
||
// Ensure m2 is non-negative (prevent NaN from floating-point errors)
|
||
let variance = self.m2.max(0.0) / (self.count - 1) as f64;
|
||
variance.sqrt()
|
||
}
|
||
|
||
/// Reset normalizer state
|
||
pub const fn reset(&mut self) {
|
||
self.buffer = None; // Drop buffer (lazy reallocation on next update)
|
||
self.mean = 0.0;
|
||
self.m2 = 0.0;
|
||
self.count = 0;
|
||
}
|
||
}
|
||
|
||
//
|
||
// Category 2: Percentile Rank Normalization for Volume Features
|
||
//
|
||
|
||
/// Rolling percentile rank normalization with lazy buffer allocation
|
||
///
|
||
/// Maps values to [0, 1] based on their rank within rolling window.
|
||
/// Handles skewed distributions (e.g., volume) better than z-score.
|
||
///
|
||
/// ## Memory Optimization
|
||
/// - OLD: `VecDeque` with `capacity=window_size` → heap allocation
|
||
/// - NEW: Option<`RingBuffer`<f64, 100>> → None until first value
|
||
/// - Savings: 800 bytes per normalizer
|
||
#[derive(Debug)]
|
||
pub struct RollingPercentileRank {
|
||
window_size: usize,
|
||
/// Lazy ring buffer (None until first value)
|
||
buffer: Option<RingBuffer<f64, 100>>,
|
||
}
|
||
|
||
impl RollingPercentileRank {
|
||
/// Create new percentile rank normalizer
|
||
///
|
||
/// Note: Buffer is NOT allocated until first value arrives (lazy init)
|
||
pub fn new(window_size: usize) -> Self {
|
||
Self {
|
||
window_size: window_size.min(100), // Cap at 100 (ring buffer size)
|
||
buffer: None, // Lazy allocation
|
||
}
|
||
}
|
||
|
||
/// Update normalizer with new value and return percentile rank [0, 1]
|
||
///
|
||
/// # Returns
|
||
/// - Percentile rank in [0, 1] range
|
||
/// - Returns 0.5 during warmup period (first 10 values)
|
||
pub fn update(&mut self, value: f64) -> f64 {
|
||
// Lazy buffer initialization
|
||
let buffer = self.buffer.get_or_insert_with(RingBuffer::new);
|
||
|
||
// Add new value
|
||
buffer.push(value);
|
||
|
||
// Warmup period: return 0.5 (median) for first 10 values
|
||
if buffer.len() < 10 {
|
||
return 0.5;
|
||
}
|
||
|
||
// Compute percentile rank (count of values < current value)
|
||
let rank = buffer.iter().filter(|&&v| v < value).count();
|
||
|
||
// Normalize to [0, 1]
|
||
let normalized = rank as f64 / buffer.len() as f64;
|
||
normalized.clamp(0.0, 1.0)
|
||
}
|
||
|
||
/// Reset normalizer state
|
||
pub const fn reset(&mut self) {
|
||
self.buffer = None; // Drop buffer (lazy reallocation on next update)
|
||
}
|
||
}
|
||
|
||
//
|
||
// Category 3: Log Transform + Z-Score for Microstructure Features
|
||
//
|
||
|
||
/// Log transform followed by z-score normalization
|
||
///
|
||
/// Handles highly skewed microstructure features (e.g., Amihud illiquidity: 1e-9 to 1e-5).
|
||
/// Log transform stabilizes variance and makes distribution more Gaussian.
|
||
#[derive(Debug)]
|
||
pub struct LogZScoreNormalizer {
|
||
scale_factor: f64,
|
||
zscore: RollingZScore,
|
||
}
|
||
|
||
impl LogZScoreNormalizer {
|
||
/// Create new log+z-score normalizer
|
||
///
|
||
/// # Arguments
|
||
/// - `scale_factor`: Multiplier before log transform (maps to reasonable log range)
|
||
/// - `window_size`: Rolling window size for z-score computation
|
||
///
|
||
/// # Scale Factors
|
||
/// - Roll spread: 1.0 (already in price units, ~0.01-10.0)
|
||
/// - Amihud illiquidity: 1e8 (map 1e-8 → 1.0 for ln)
|
||
/// - Corwin-Schultz spread: 100.0 (map 0.01 → 1.0 for ln)
|
||
pub fn new(scale_factor: f64, window_size: usize) -> Self {
|
||
Self {
|
||
scale_factor,
|
||
zscore: RollingZScore::new(window_size),
|
||
}
|
||
}
|
||
|
||
/// Update normalizer with new value and return normalized value
|
||
///
|
||
/// # Returns
|
||
/// - Normalized value clipped to ±3σ after log transform
|
||
/// - Returns 0.0 during warmup period
|
||
pub fn update(&mut self, value: f64) -> f64 {
|
||
// Step 1: Log transform
|
||
let log_val = if value > 0.0 {
|
||
(value * self.scale_factor).ln()
|
||
} else {
|
||
// Zero or negative values: map to -10.0 (extreme negative, clipped to -3σ)
|
||
-10.0
|
||
};
|
||
|
||
// Step 2: Z-score normalization
|
||
self.zscore.update(log_val)
|
||
}
|
||
|
||
/// Reset normalizer state
|
||
pub const fn reset(&mut self) {
|
||
self.zscore.reset();
|
||
}
|
||
}
|
||
|
||
//
|
||
// NaN/Inf Handling
|
||
//
|
||
|
||
/// NaN/Inf handler using last-valid-value imputation
|
||
///
|
||
/// Prevents NaN propagation through feature pipeline while preserving
|
||
/// temporal continuity (better than zero imputation which biases toward zero).
|
||
#[derive(Debug)]
|
||
struct NaNHandler {
|
||
last_valid: [f64; 225],
|
||
nan_count: [u32; 225],
|
||
}
|
||
|
||
impl NaNHandler {
|
||
const fn new() -> Self {
|
||
Self {
|
||
last_valid: [0.0; 225],
|
||
nan_count: [0; 225],
|
||
}
|
||
}
|
||
|
||
/// Handle NaN/Inf in input features (impute with last valid value)
|
||
fn handle_input(&mut self, features: &mut [f64; 225]) {
|
||
for (i, val) in features.iter_mut().enumerate() {
|
||
if !val.is_finite() {
|
||
// Impute with last valid value
|
||
*val = self.last_valid[i];
|
||
self.nan_count[i] += 1;
|
||
|
||
// Log warning if excessive NaNs (every 100 occurrences)
|
||
if self.nan_count[i] % 100 == 0 {
|
||
warn!(
|
||
feature_index = i,
|
||
nan_count = self.nan_count[i],
|
||
"Excessive NaN occurrences for feature"
|
||
);
|
||
}
|
||
} else {
|
||
// Update last valid value
|
||
self.last_valid[i] = *val;
|
||
self.nan_count[i] = 0; // Reset counter
|
||
}
|
||
}
|
||
}
|
||
|
||
/// Get total NaN count across all features
|
||
fn total_nan_count(&self) -> u32 {
|
||
self.nan_count.iter().sum()
|
||
}
|
||
|
||
/// Reset handler state
|
||
const fn reset(&mut self) {
|
||
self.last_valid = [0.0; 225];
|
||
self.nan_count = [0; 225];
|
||
}
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
|
||
//
|
||
// RollingZScore Tests (5 tests)
|
||
//
|
||
|
||
#[test]
|
||
fn test_rolling_zscore_basic() {
|
||
let mut zscore = RollingZScore::new(10);
|
||
|
||
// Add 10 values: 1, 2, 3, ..., 10 (mean=5.5, std≈2.87)
|
||
for i in 1..=10 {
|
||
zscore.update(i as f64);
|
||
}
|
||
|
||
// Next value: 11 (z-score ≈ (11 - 5.5) / 2.87 ≈ 1.92)
|
||
let normalized = zscore.update(11.0);
|
||
assert!(
|
||
normalized > 1.5 && normalized < 2.5,
|
||
"Z-score for 11 should be ~1.92, got {}",
|
||
normalized
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn test_rolling_zscore_warmup() {
|
||
let mut zscore = RollingZScore::new(50);
|
||
|
||
// First 10 values should return 0.0 (warmup period)
|
||
for i in 1..=9 {
|
||
let normalized = zscore.update(i as f64);
|
||
assert_eq!(
|
||
normalized, 0.0,
|
||
"Warmup period should return 0.0, got {}",
|
||
normalized
|
||
);
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn test_rolling_zscore_clipping() {
|
||
let mut zscore = RollingZScore::new(20);
|
||
|
||
// Add 20 normal values (mean=50, std≈1)
|
||
for _ in 0..20 {
|
||
zscore.update(50.0);
|
||
}
|
||
|
||
// Add extreme outlier (>3σ)
|
||
let normalized = zscore.update(100.0);
|
||
assert!(
|
||
normalized <= 3.0,
|
||
"Z-score should be clipped to +3σ, got {}",
|
||
normalized
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn test_rolling_zscore_mean_std() {
|
||
let mut zscore = RollingZScore::new(50);
|
||
|
||
// Add 50 values: 1, 2, 3, ..., 50 (mean=25.5)
|
||
for i in 1..=50 {
|
||
zscore.update(i as f64);
|
||
}
|
||
|
||
assert!(
|
||
(zscore.mean - 25.5).abs() < 0.1,
|
||
"Mean should be ~25.5, got {}",
|
||
zscore.mean
|
||
);
|
||
assert!(
|
||
zscore.std() > 14.0 && zscore.std() < 15.0,
|
||
"Std should be ~14.4, got {}",
|
||
zscore.std()
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn test_rolling_zscore_reset() {
|
||
let mut zscore = RollingZScore::new(20);
|
||
|
||
// Add 20 values
|
||
for i in 1..=20 {
|
||
zscore.update(i as f64);
|
||
}
|
||
|
||
// Reset
|
||
zscore.reset();
|
||
assert_eq!(zscore.count, 0, "Count should be 0 after reset");
|
||
assert_eq!(zscore.mean, 0.0, "Mean should be 0.0 after reset");
|
||
assert!(zscore.buffer.is_none(), "Buffer should be None after reset");
|
||
}
|
||
|
||
//
|
||
// RollingPercentileRank Tests (5 tests)
|
||
//
|
||
|
||
#[test]
|
||
fn test_percentile_rank_basic() {
|
||
let mut percentile = RollingPercentileRank::new(10);
|
||
|
||
// Add 10 values: 1, 2, 3, ..., 10
|
||
for i in 1..=10 {
|
||
percentile.update(i as f64);
|
||
}
|
||
|
||
// Next value: 5.5 (rank should be ~5/10 = 0.5)
|
||
let normalized = percentile.update(5.5);
|
||
assert!(
|
||
(normalized - 0.5).abs() < 0.2,
|
||
"Percentile rank for 5.5 should be ~0.5, got {}",
|
||
normalized
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn test_percentile_rank_warmup() {
|
||
let mut percentile = RollingPercentileRank::new(50);
|
||
|
||
// First 10 values should return 0.5 (warmup period)
|
||
for i in 1..=9 {
|
||
let normalized = percentile.update(i as f64);
|
||
assert_eq!(
|
||
normalized, 0.5,
|
||
"Warmup period should return 0.5, got {}",
|
||
normalized
|
||
);
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn test_percentile_rank_bounds() {
|
||
let mut percentile = RollingPercentileRank::new(20);
|
||
|
||
// Add 20 values: 1, 2, 3, ..., 20
|
||
for i in 1..=20 {
|
||
percentile.update(i as f64);
|
||
}
|
||
|
||
// Minimum value: 0 (rank = 0)
|
||
let min_normalized = percentile.update(0.0);
|
||
assert_eq!(min_normalized, 0.0, "Minimum should be 0.0");
|
||
|
||
// Maximum value: 100 (rank = 1.0)
|
||
let max_normalized = percentile.update(100.0);
|
||
assert!(
|
||
max_normalized >= 0.9,
|
||
"Maximum should be close to 1.0, got {}",
|
||
max_normalized
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn test_percentile_rank_skewed_distribution() {
|
||
let mut percentile = RollingPercentileRank::new(20);
|
||
|
||
// Add skewed distribution: 1, 1, 1, ..., 1 (18 times), 100, 100
|
||
for _ in 0..18 {
|
||
percentile.update(1.0);
|
||
}
|
||
percentile.update(100.0);
|
||
percentile.update(100.0);
|
||
|
||
// New value: 50 (rank should be ~18/20 = 0.9)
|
||
let normalized = percentile.update(50.0);
|
||
assert!(
|
||
normalized > 0.8,
|
||
"Percentile rank for 50 in skewed distribution should be high, got {}",
|
||
normalized
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn test_percentile_rank_reset() {
|
||
let mut percentile = RollingPercentileRank::new(20);
|
||
|
||
// Add 20 values
|
||
for i in 1..=20 {
|
||
percentile.update(i as f64);
|
||
}
|
||
|
||
// Reset
|
||
percentile.reset();
|
||
assert!(
|
||
percentile.buffer.is_none(),
|
||
"Buffer should be None after reset"
|
||
);
|
||
}
|
||
|
||
//
|
||
// LogZScoreNormalizer Tests (5 tests)
|
||
//
|
||
|
||
#[test]
|
||
fn test_log_zscore_basic() {
|
||
let mut log_zscore = LogZScoreNormalizer::new(1.0, 20);
|
||
|
||
// Add 20 values: 1, 2, 3, ..., 20 (log transforms to more Gaussian)
|
||
for i in 1..=20 {
|
||
log_zscore.update(i as f64);
|
||
}
|
||
|
||
// Next value: 10 (log(10) ≈ 2.3, should be near mean after normalization)
|
||
let normalized = log_zscore.update(10.0);
|
||
assert!(
|
||
normalized.abs() < 2.0,
|
||
"Normalized log(10) should be near 0, got {}",
|
||
normalized
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn test_log_zscore_scale_factor() {
|
||
let mut log_zscore = LogZScoreNormalizer::new(1e8, 20);
|
||
|
||
// Add 20 small values: 1e-8, 2e-8, ..., 20e-8 (Amihud illiquidity range)
|
||
for i in 1..=20 {
|
||
log_zscore.update(i as f64 * 1e-8);
|
||
}
|
||
|
||
// Next value: 10e-8 (log(10e-8 * 1e8) = log(1) = 0)
|
||
let normalized = log_zscore.update(10e-8);
|
||
assert!(
|
||
normalized.abs() < 2.0,
|
||
"Normalized log(10e-8) with scale 1e8 should be near 0, got {}",
|
||
normalized
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn test_log_zscore_zero_handling() {
|
||
let mut log_zscore = LogZScoreNormalizer::new(1.0, 20);
|
||
|
||
// Add 20 positive values
|
||
for i in 1..=20 {
|
||
log_zscore.update(i as f64);
|
||
}
|
||
|
||
// Zero value should map to -10.0 before z-score (extreme negative)
|
||
let normalized = log_zscore.update(0.0);
|
||
assert!(
|
||
normalized <= -3.0,
|
||
"Zero should be clipped to -3σ or lower, got {}",
|
||
normalized
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn test_log_zscore_negative_handling() {
|
||
let mut log_zscore = LogZScoreNormalizer::new(1.0, 20);
|
||
|
||
// Add 20 positive values
|
||
for i in 1..=20 {
|
||
log_zscore.update(i as f64);
|
||
}
|
||
|
||
// Negative value should map to -10.0 (same as zero)
|
||
let normalized = log_zscore.update(-5.0);
|
||
assert!(
|
||
normalized <= -3.0,
|
||
"Negative should be clipped to -3σ or lower, got {}",
|
||
normalized
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn test_log_zscore_reset() {
|
||
let mut log_zscore = LogZScoreNormalizer::new(1.0, 20);
|
||
|
||
// Add 20 values
|
||
for i in 1..=20 {
|
||
log_zscore.update(i as f64);
|
||
}
|
||
|
||
// Reset
|
||
log_zscore.reset();
|
||
assert_eq!(
|
||
log_zscore.zscore.count, 0,
|
||
"Z-score count should be 0 after reset"
|
||
);
|
||
}
|
||
|
||
//
|
||
// NaNHandler Tests (5 tests)
|
||
//
|
||
|
||
#[test]
|
||
fn test_nan_handler_basic() {
|
||
let mut handler = NaNHandler::new();
|
||
let mut features = [1.0; 225];
|
||
|
||
// Set feature 10 to NaN
|
||
features[10] = f64::NAN;
|
||
|
||
// Handle input (should impute with last valid value = 0.0 initially)
|
||
handler.handle_input(&mut features);
|
||
assert_eq!(
|
||
features[10], 0.0,
|
||
"NaN should be imputed with last valid value (0.0)"
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn test_nan_handler_last_valid_value() {
|
||
let mut handler = NaNHandler::new();
|
||
|
||
// First update: set feature 10 to 42.0
|
||
let mut features = [0.0; 225];
|
||
features[10] = 42.0;
|
||
handler.handle_input(&mut features);
|
||
|
||
// Second update: set feature 10 to NaN (should impute with 42.0)
|
||
features[10] = f64::NAN;
|
||
handler.handle_input(&mut features);
|
||
assert_eq!(
|
||
features[10], 42.0,
|
||
"NaN should be imputed with last valid value (42.0)"
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn test_nan_handler_inf() {
|
||
let mut handler = NaNHandler::new();
|
||
let mut features = [1.0; 225];
|
||
|
||
// Set feature 20 to Inf
|
||
features[20] = f64::INFINITY;
|
||
|
||
// Handle input (should impute with last valid value = 0.0)
|
||
handler.handle_input(&mut features);
|
||
assert_eq!(
|
||
features[20], 0.0,
|
||
"Inf should be imputed with last valid value (0.0)"
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn test_nan_handler_count() {
|
||
let mut handler = NaNHandler::new();
|
||
|
||
// First update: NaN at feature 10
|
||
let mut features = [0.0; 225];
|
||
features[10] = f64::NAN;
|
||
handler.handle_input(&mut features);
|
||
assert_eq!(handler.nan_count[10], 1, "NaN count should be 1");
|
||
|
||
// Second update: valid value at feature 10 (count resets)
|
||
features[10] = 42.0;
|
||
handler.handle_input(&mut features);
|
||
assert_eq!(
|
||
handler.nan_count[10], 0,
|
||
"NaN count should reset to 0 after valid value"
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn test_nan_handler_reset() {
|
||
let mut handler = NaNHandler::new();
|
||
|
||
// Add some NaNs
|
||
let mut features = [0.0; 225];
|
||
features[10] = f64::NAN;
|
||
handler.handle_input(&mut features);
|
||
|
||
// Reset
|
||
handler.reset();
|
||
assert_eq!(
|
||
handler.total_nan_count(),
|
||
0,
|
||
"Total NaN count should be 0 after reset"
|
||
);
|
||
}
|
||
|
||
//
|
||
// FeatureNormalizer Integration Tests (5 tests)
|
||
//
|
||
|
||
#[test]
|
||
fn test_feature_normalizer_basic() {
|
||
let mut normalizer = FeatureNormalizer::new();
|
||
let mut features = [1.0; 225];
|
||
|
||
// Normalize (should succeed)
|
||
let result = normalizer.normalize(&mut features);
|
||
assert!(result.is_ok(), "Normalization should succeed");
|
||
}
|
||
|
||
#[test]
|
||
fn test_feature_normalizer_price_features() {
|
||
let mut normalizer = FeatureNormalizer::new();
|
||
|
||
// Feed 50 bars to warmup
|
||
for i in 0..50 {
|
||
let mut features = [0.0; 225];
|
||
features[15] = i as f64; // Price feature at index 15
|
||
normalizer.normalize(&mut features).unwrap();
|
||
}
|
||
|
||
// Next bar: should have normalized price feature
|
||
let mut features = [0.0; 225];
|
||
features[15] = 25.0; // Mean value
|
||
normalizer.normalize(&mut features).unwrap();
|
||
|
||
// Price feature should be near 0 (mean)
|
||
assert!(
|
||
features[15].abs() < 1.0,
|
||
"Normalized price feature should be near 0, got {}",
|
||
features[15]
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn test_feature_normalizer_volume_features() {
|
||
let mut normalizer = FeatureNormalizer::new();
|
||
|
||
// Feed 50 bars to warmup
|
||
for i in 0..50 {
|
||
let mut features = [0.0; 225];
|
||
features[75] = i as f64; // Volume feature at index 75
|
||
normalizer.normalize(&mut features).unwrap();
|
||
}
|
||
|
||
// Next bar: minimum volume
|
||
let mut features = [0.0; 225];
|
||
features[75] = 0.0;
|
||
normalizer.normalize(&mut features).unwrap();
|
||
|
||
// Volume feature should be 0.0 (minimum percentile)
|
||
assert!(
|
||
features[75] < 0.2,
|
||
"Normalized volume feature should be near 0.0, got {}",
|
||
features[75]
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn test_feature_normalizer_nan_handling() {
|
||
let mut normalizer = FeatureNormalizer::new();
|
||
|
||
// First bar: valid features
|
||
let mut features = [42.0; 225];
|
||
normalizer.normalize(&mut features).unwrap();
|
||
|
||
// Second bar: NaN at feature 15 (should impute with last valid value)
|
||
let mut features2 = [42.0; 225];
|
||
features2[15] = f64::NAN;
|
||
let result = normalizer.normalize(&mut features2);
|
||
assert!(
|
||
result.is_ok(),
|
||
"Normalization should succeed after NaN imputation"
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn test_feature_normalizer_reset() {
|
||
let mut normalizer = FeatureNormalizer::new();
|
||
|
||
// Feed 50 bars
|
||
for i in 0..50 {
|
||
let mut features = [i as f64; 225];
|
||
normalizer.normalize(&mut features).unwrap();
|
||
}
|
||
|
||
// Reset
|
||
normalizer.reset();
|
||
|
||
// Stats should be reset
|
||
let stats = normalizer.get_stats();
|
||
assert_eq!(stats.price_mean, 0.0, "Mean should be 0.0 after reset");
|
||
assert_eq!(stats.nan_count, 0, "NaN count should be 0 after reset");
|
||
}
|
||
}
|