9c3d741a08
Move 17 library crates into crates/, CLI binary into bin/fxt, consolidate 10 test crates into testing/, split config crate from deployment config files. Root directory reduced from 38+ to ~17 directories. All Cargo.toml paths and build.rs proto refs updated. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
486 lines
16 KiB
Rust
486 lines
16 KiB
Rust
//! Walk-forward evaluation framework for time-series ML models.
|
|
//!
|
|
//! Implements expanding-window walk-forward cross-validation with configurable
|
|
//! train/val/test splits and feature normalization. This prevents lookahead bias
|
|
//! by ensuring models are always evaluated on unseen future data.
|
|
//!
|
|
//! # Walk-Forward Split Diagram
|
|
//!
|
|
//! ```text
|
|
//! Fold 0: |------- Train (12mo) -------|-- Val (3mo) --|-- Test (3mo) --|
|
|
//! Fold 1: |---------- Train (15mo) ----------|-- Val --|-- Test --|
|
|
//! Fold 2: |------------- Train (18mo) --------------|-- Val --|-- Test --|
|
|
//! ```
|
|
//!
|
|
//! # Usage
|
|
//!
|
|
//! ```rust,no_run
|
|
//! use ml::walk_forward::{WalkForwardConfig, generate_walk_forward_windows, NormStats};
|
|
//!
|
|
//! let config = WalkForwardConfig::default();
|
|
//! let windows = generate_walk_forward_windows(&bars, &config);
|
|
//!
|
|
//! for window in &windows {
|
|
//! let features = extract_ml_features(&window.train)?;
|
|
//! let stats = NormStats::from_features(&features);
|
|
//! let normalized = stats.normalize_batch(&features);
|
|
//! // Train model on normalized features...
|
|
//! }
|
|
//! ```
|
|
|
|
use crate::types::OHLCVBar;
|
|
use chrono::{Months, NaiveDate};
|
|
|
|
/// Configuration for walk-forward evaluation window generation.
|
|
#[derive(Debug, Clone)]
|
|
pub struct WalkForwardConfig {
|
|
/// Number of months in the initial training window (default: 12).
|
|
pub initial_train_months: u32,
|
|
/// Number of months in the validation window (default: 3).
|
|
pub val_months: u32,
|
|
/// Number of months in the test window (default: 3).
|
|
pub test_months: u32,
|
|
/// Number of months to step forward between folds (default: 3).
|
|
pub step_months: u32,
|
|
}
|
|
|
|
impl Default for WalkForwardConfig {
|
|
fn default() -> Self {
|
|
Self {
|
|
initial_train_months: 12,
|
|
val_months: 3,
|
|
test_months: 3,
|
|
step_months: 3,
|
|
}
|
|
}
|
|
}
|
|
|
|
/// A single walk-forward evaluation window containing train/val/test splits.
|
|
#[derive(Debug, Clone)]
|
|
pub struct WalkForwardWindow {
|
|
/// Zero-based fold index.
|
|
pub fold: usize,
|
|
/// Training bars (expanding window).
|
|
pub train: Vec<OHLCVBar>,
|
|
/// Validation bars.
|
|
pub val: Vec<OHLCVBar>,
|
|
/// Test bars.
|
|
pub test: Vec<OHLCVBar>,
|
|
/// End date of the training period (exclusive boundary).
|
|
pub train_end: NaiveDate,
|
|
/// End date of the validation period (exclusive boundary).
|
|
pub val_end: NaiveDate,
|
|
/// End date of the test period (exclusive boundary).
|
|
pub test_end: NaiveDate,
|
|
}
|
|
|
|
/// Minimum number of bars required in each split to form a valid window.
|
|
const MIN_BARS_PER_SPLIT: usize = 50;
|
|
|
|
/// Generate expanding walk-forward windows from sorted OHLCV bars.
|
|
///
|
|
/// Each successive fold extends the training window while keeping
|
|
/// val/test windows the same size, stepping forward by `step_months`.
|
|
///
|
|
/// Returns an empty `Vec` if the data is insufficient for even one fold.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `bars` - Chronologically sorted OHLCV bars
|
|
/// * `config` - Walk-forward configuration
|
|
pub fn generate_walk_forward_windows(
|
|
bars: &[OHLCVBar],
|
|
config: &WalkForwardConfig,
|
|
) -> Vec<WalkForwardWindow> {
|
|
if bars.is_empty() {
|
|
return Vec::new();
|
|
}
|
|
|
|
// Determine date range from the data
|
|
let first_bar = match bars.first() {
|
|
Some(b) => b,
|
|
None => return Vec::new(),
|
|
};
|
|
let last_bar = match bars.last() {
|
|
Some(b) => b,
|
|
None => return Vec::new(),
|
|
};
|
|
|
|
let data_start = first_bar.timestamp.date_naive();
|
|
let data_end = last_bar.timestamp.date_naive();
|
|
|
|
let mut windows = Vec::new();
|
|
let mut fold = 0_usize;
|
|
|
|
loop {
|
|
// Train end: initial_train_months + fold * step_months from data_start
|
|
let total_train_months = config
|
|
.initial_train_months
|
|
.saturating_add(fold as u32 * config.step_months);
|
|
let train_end = match data_start.checked_add_months(Months::new(total_train_months)) {
|
|
Some(d) => d,
|
|
None => break,
|
|
};
|
|
|
|
// Val end: train_end + val_months
|
|
let val_end = match train_end.checked_add_months(Months::new(config.val_months)) {
|
|
Some(d) => d,
|
|
None => break,
|
|
};
|
|
|
|
// Test end: val_end + test_months
|
|
let test_end = match val_end.checked_add_months(Months::new(config.test_months)) {
|
|
Some(d) => d,
|
|
None => break,
|
|
};
|
|
|
|
// If test_end exceeds data range, we cannot form this fold
|
|
if test_end > data_end {
|
|
break;
|
|
}
|
|
|
|
// Partition bars into train/val/test using date boundaries
|
|
let train_bars: Vec<OHLCVBar> = bars
|
|
.iter()
|
|
.filter(|b| b.timestamp.date_naive() < train_end)
|
|
.copied()
|
|
.collect();
|
|
|
|
let val_bars: Vec<OHLCVBar> = bars
|
|
.iter()
|
|
.filter(|b| {
|
|
let d = b.timestamp.date_naive();
|
|
d >= train_end && d < val_end
|
|
})
|
|
.copied()
|
|
.collect();
|
|
|
|
let test_bars: Vec<OHLCVBar> = bars
|
|
.iter()
|
|
.filter(|b| {
|
|
let d = b.timestamp.date_naive();
|
|
d >= val_end && d < test_end
|
|
})
|
|
.copied()
|
|
.collect();
|
|
|
|
// Skip folds where any split has fewer than MIN_BARS_PER_SPLIT bars
|
|
if train_bars.len() < MIN_BARS_PER_SPLIT
|
|
|| val_bars.len() < MIN_BARS_PER_SPLIT
|
|
|| test_bars.len() < MIN_BARS_PER_SPLIT
|
|
{
|
|
// If training set is too small, later folds might still work
|
|
// But if val/test are too small at the end, no point continuing
|
|
if val_bars.len() < MIN_BARS_PER_SPLIT || test_bars.len() < MIN_BARS_PER_SPLIT {
|
|
// Check if we have already generated at least one fold or if
|
|
// the data simply does not support this fold size
|
|
if fold > 0 {
|
|
break;
|
|
}
|
|
// For fold 0 with insufficient data, skip but try next fold
|
|
fold = fold.saturating_add(1);
|
|
continue;
|
|
}
|
|
fold = fold.saturating_add(1);
|
|
continue;
|
|
}
|
|
|
|
windows.push(WalkForwardWindow {
|
|
fold,
|
|
train: train_bars,
|
|
val: val_bars,
|
|
test: test_bars,
|
|
train_end,
|
|
val_end,
|
|
test_end,
|
|
});
|
|
|
|
fold = fold.saturating_add(1);
|
|
}
|
|
|
|
windows
|
|
}
|
|
|
|
/// Per-feature normalization statistics (mean and standard deviation).
|
|
///
|
|
/// Computed from training data and applied to val/test data to prevent
|
|
/// information leakage. Uses z-score normalization: `(x - mean) / std`.
|
|
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
|
pub struct NormStats {
|
|
/// Per-feature mean values (length 51).
|
|
pub mean: Vec<f64>,
|
|
/// Per-feature standard deviation values (length 51, clamped >= 1e-8).
|
|
pub std: Vec<f64>,
|
|
}
|
|
|
|
/// Number of features in a standard feature vector.
|
|
const FEATURE_DIM: usize = 51;
|
|
|
|
/// Minimum standard deviation to prevent division by zero.
|
|
const MIN_STD: f64 = 1e-8;
|
|
|
|
impl NormStats {
|
|
/// Compute normalization statistics from a batch of 51-dimensional feature vectors.
|
|
///
|
|
/// If the input is empty, returns mean = 0.0 and std = 1.0 for all 51 features.
|
|
pub fn from_features(features: &[[f64; FEATURE_DIM]]) -> Self {
|
|
if features.is_empty() {
|
|
return Self {
|
|
mean: vec![0.0; FEATURE_DIM],
|
|
std: vec![1.0; FEATURE_DIM],
|
|
};
|
|
}
|
|
|
|
let n = features.len() as f64;
|
|
let mut mean = vec![0.0_f64; FEATURE_DIM];
|
|
let mut variance = vec![0.0_f64; FEATURE_DIM];
|
|
|
|
// Accumulate sums for mean
|
|
for feature_vec in features {
|
|
for (m, val) in mean.iter_mut().zip(feature_vec.iter()) {
|
|
*m += val;
|
|
}
|
|
}
|
|
|
|
// Compute mean
|
|
for m in &mut mean {
|
|
*m /= n;
|
|
}
|
|
|
|
// Accumulate variance
|
|
for feature_vec in features {
|
|
for (v, (val, m)) in variance
|
|
.iter_mut()
|
|
.zip(feature_vec.iter().zip(mean.iter()))
|
|
{
|
|
let diff = val - m;
|
|
*v += diff * diff;
|
|
}
|
|
}
|
|
|
|
// Compute std with minimum clamp
|
|
let std_vec: Vec<f64> = variance
|
|
.iter()
|
|
.map(|v| (v / n).sqrt().max(MIN_STD))
|
|
.collect();
|
|
|
|
Self {
|
|
mean,
|
|
std: std_vec,
|
|
}
|
|
}
|
|
|
|
/// Z-score normalize a single 51-dimensional feature vector.
|
|
///
|
|
/// Returns `(feature - mean) / std` element-wise.
|
|
pub fn normalize(&self, features: &[f64; FEATURE_DIM]) -> [f64; FEATURE_DIM] {
|
|
let mut result = [0.0_f64; FEATURE_DIM];
|
|
for ((r, val), (m, s)) in result
|
|
.iter_mut()
|
|
.zip(features.iter())
|
|
.zip(self.mean.iter().zip(self.std.iter()))
|
|
{
|
|
*r = (val - m) / s;
|
|
}
|
|
result
|
|
}
|
|
|
|
/// Z-score normalize a batch of 51-dimensional feature vectors.
|
|
pub fn normalize_batch(&self, features: &[[f64; FEATURE_DIM]]) -> Vec<[f64; FEATURE_DIM]> {
|
|
features.iter().map(|f| self.normalize(f)).collect()
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use chrono::{Datelike, NaiveDate, NaiveTime, TimeZone, Utc, Weekday};
|
|
|
|
/// Create a single OHLCV bar at the given date.
|
|
fn make_bar(year: i32, month: u32, day: u32) -> OHLCVBar {
|
|
let date = NaiveDate::from_ymd_opt(year, month, day).unwrap_or_default();
|
|
let time = NaiveTime::from_hms_opt(10, 0, 0).unwrap_or_default();
|
|
let dt = date.and_time(time);
|
|
let timestamp = Utc.from_utc_datetime(&dt);
|
|
OHLCVBar {
|
|
timestamp,
|
|
open: 100.0,
|
|
high: 101.0,
|
|
low: 99.0,
|
|
close: 100.5,
|
|
volume: 1000.0,
|
|
}
|
|
}
|
|
|
|
/// Create bars for every weekday in [start, end) with one bar per day.
|
|
/// For testing purposes, this creates enough density to exceed the minimum
|
|
/// bars threshold when covering multi-month ranges.
|
|
fn make_bars_range(start: NaiveDate, end: NaiveDate) -> Vec<OHLCVBar> {
|
|
let mut bars = Vec::new();
|
|
let mut current = start;
|
|
let time = NaiveTime::from_hms_opt(10, 0, 0).unwrap_or_default();
|
|
while current < end {
|
|
let weekday = current.weekday();
|
|
if weekday != Weekday::Sat && weekday != Weekday::Sun {
|
|
// Generate multiple bars per trading day to simulate 1-min bars
|
|
// We need enough density: ~390 bars per day for 6.5h of trading
|
|
// For test efficiency, generate 20 bars per day (enough for density)
|
|
for minute_offset in 0..20 {
|
|
let bar_time =
|
|
NaiveTime::from_hms_opt(9, 30_u32.saturating_add(minute_offset), 0)
|
|
.unwrap_or(time);
|
|
let dt = current.and_time(bar_time);
|
|
let timestamp = Utc.from_utc_datetime(&dt);
|
|
bars.push(OHLCVBar {
|
|
timestamp,
|
|
open: 100.0 + (bars.len() as f64 * 0.001),
|
|
high: 101.0 + (bars.len() as f64 * 0.001),
|
|
low: 99.0 + (bars.len() as f64 * 0.001),
|
|
close: 100.5 + (bars.len() as f64 * 0.001),
|
|
volume: 1000.0,
|
|
});
|
|
}
|
|
}
|
|
current = current
|
|
.succ_opt()
|
|
.unwrap_or(current);
|
|
}
|
|
bars
|
|
}
|
|
|
|
#[test]
|
|
fn test_generate_walk_forward_empty_bars() {
|
|
let config = WalkForwardConfig::default();
|
|
let windows = generate_walk_forward_windows(&[], &config);
|
|
assert!(windows.is_empty());
|
|
}
|
|
|
|
#[test]
|
|
fn test_walk_forward_windows_24_months() {
|
|
// Generate 24 months of synthetic bars
|
|
let start = NaiveDate::from_ymd_opt(2023, 1, 1).unwrap_or_default();
|
|
let end = NaiveDate::from_ymd_opt(2025, 1, 1).unwrap_or_default();
|
|
let bars = make_bars_range(start, end);
|
|
|
|
assert!(
|
|
!bars.is_empty(),
|
|
"Should have generated bars for 24-month range"
|
|
);
|
|
|
|
let config = WalkForwardConfig::default(); // 12/3/3/3
|
|
let windows = generate_walk_forward_windows(&bars, &config);
|
|
|
|
// With 24 months of data and 12+3+3=18 months for first fold, stepping by 3:
|
|
// Fold 0: train 12mo, val 3mo, test 3mo = 18 months (fits in 24)
|
|
// Fold 1: train 15mo, val 3mo, test 3mo = 21 months (fits in 24)
|
|
// Fold 2: train 18mo, val 3mo, test 3mo = 24 months (might barely fit)
|
|
assert!(
|
|
windows.len() >= 2,
|
|
"Expected at least 2 windows, got {}",
|
|
windows.len()
|
|
);
|
|
|
|
// Check fold 0 has non-empty splits
|
|
if let Some(w0) = windows.first() {
|
|
assert!(!w0.train.is_empty(), "Fold 0 train should be non-empty");
|
|
assert!(!w0.val.is_empty(), "Fold 0 val should be non-empty");
|
|
assert!(!w0.test.is_empty(), "Fold 0 test should be non-empty");
|
|
|
|
// Train must end before val
|
|
assert!(
|
|
w0.train_end <= w0.val_end,
|
|
"Train end ({}) should be <= val end ({})",
|
|
w0.train_end,
|
|
w0.val_end
|
|
);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn test_walk_forward_expanding_windows() {
|
|
// Generate 30 months to get multiple folds
|
|
let start = NaiveDate::from_ymd_opt(2023, 1, 1).unwrap_or_default();
|
|
let end = NaiveDate::from_ymd_opt(2025, 7, 1).unwrap_or_default();
|
|
let bars = make_bars_range(start, end);
|
|
|
|
let config = WalkForwardConfig::default();
|
|
let windows = generate_walk_forward_windows(&bars, &config);
|
|
|
|
assert!(
|
|
windows.len() >= 2,
|
|
"Need at least 2 windows to test expanding, got {}",
|
|
windows.len()
|
|
);
|
|
|
|
// Each successive fold should have strictly more training data
|
|
for pair in windows.windows(2) {
|
|
let (prev, next) = match (pair.first(), pair.get(1)) {
|
|
(Some(p), Some(n)) => (p, n),
|
|
_ => continue,
|
|
};
|
|
assert!(
|
|
next.train.len() > prev.train.len(),
|
|
"Fold {} train ({} bars) should have more data than fold {} train ({} bars)",
|
|
next.fold,
|
|
next.train.len(),
|
|
prev.fold,
|
|
prev.train.len()
|
|
);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn test_norm_stats_roundtrip() {
|
|
// Features: [[1,1,...], [2,2,...], [3,3,...]]
|
|
let f1 = [1.0_f64; FEATURE_DIM];
|
|
let f2 = [2.0_f64; FEATURE_DIM];
|
|
let f3 = [3.0_f64; FEATURE_DIM];
|
|
let features = [f1, f2, f3];
|
|
|
|
let stats = NormStats::from_features(&features);
|
|
|
|
// Mean should be 2.0 for all features
|
|
for m in &stats.mean {
|
|
assert!(
|
|
(*m - 2.0).abs() < 1e-10,
|
|
"Expected mean ~2.0, got {}",
|
|
m
|
|
);
|
|
}
|
|
|
|
// Normalizing the mean vector should give ~0 for all features
|
|
let normalized = stats.normalize(&f2);
|
|
for val in &normalized {
|
|
assert!(
|
|
val.abs() < 1e-10,
|
|
"Expected normalized mean ~0.0, got {}",
|
|
val
|
|
);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn test_norm_stats_empty() {
|
|
let features: &[[f64; FEATURE_DIM]] = &[];
|
|
let stats = NormStats::from_features(features);
|
|
|
|
// Should return safe defaults
|
|
assert_eq!(stats.mean.len(), FEATURE_DIM);
|
|
assert_eq!(stats.std.len(), FEATURE_DIM);
|
|
|
|
// Mean should be 0.0
|
|
for m in &stats.mean {
|
|
assert!((*m).abs() < 1e-10, "Expected mean 0.0, got {}", m);
|
|
}
|
|
|
|
// Std should be 1.0
|
|
for s in &stats.std {
|
|
assert!(
|
|
(*s - 1.0).abs() < 1e-10,
|
|
"Expected std 1.0, got {}",
|
|
s
|
|
);
|
|
}
|
|
}
|
|
}
|