test(ml): add real-data validation integration test for 6E.FUT
Runs full ValidationHarness on actual Databento 6E.FUT 1-minute OHLCV bars (~30k bars). Extracts 15-dim features (5 OHLCV + 10 technical indicators), configures DQN with walk-forward validation, and prints a detailed report including DSR, PBO, permutation test, and per-regime breakdown. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
252
ml/tests/validation_real_data_test.rs
Normal file
252
ml/tests/validation_real_data_test.rs
Normal file
@@ -0,0 +1,252 @@
|
||||
//! Real-data validation test: runs the full ValidationHarness on actual Databento 6E.FUT data.
|
||||
//!
|
||||
//! This test loads 1-minute OHLCV bars from a DBN file, extracts 15-dimensional
|
||||
//! features (5 OHLCV + 10 technical indicators), builds a TimeSeriesData,
|
||||
//! wraps a DQN agent via DqnStrategy, and runs walk-forward validation with
|
||||
//! DSR, PBO, permutation tests, and per-regime breakdown.
|
||||
//!
|
||||
//! Requires: `test_data/real/databento/6E.FUT_ohlcv-1m_*.dbn` to exist.
|
||||
//! Run with: `SQLX_OFFLINE=true cargo test --manifest-path ml/Cargo.toml --test validation_real_data_test -- --nocapture`
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use ml::dqn::DQNConfig;
|
||||
use ml::real_data_loader::RealDataLoader;
|
||||
use ml::validation::{
|
||||
DqnStrategy, TimeSeriesData, ValidationHarness, ValidationHarnessConfig, WalkForwardConfig,
|
||||
};
|
||||
|
||||
/// Build a 15-dimensional feature vector per bar from RealDataLoader output.
|
||||
///
|
||||
/// Features:
|
||||
/// 0-4: Normalized OHLCV (open, high, low, close, volume)
|
||||
/// 5: RSI(14)
|
||||
/// 6-7: EMA fast(12), EMA slow(26)
|
||||
/// 8-10: MACD (line, signal, histogram = line - signal)
|
||||
/// 11-13: Bollinger Bands (upper, middle, lower)
|
||||
/// 14: ATR(14)
|
||||
fn build_features_from_loader(loader: &RealDataLoader, bars: &[ml::real_data_loader::OHLCVBar]) -> Vec<Vec<f32>> {
|
||||
let feat_matrix = loader
|
||||
.extract_features(bars)
|
||||
.expect("extract_features failed");
|
||||
|
||||
let indicators = loader
|
||||
.calculate_indicators(bars)
|
||||
.expect("calculate_indicators failed");
|
||||
|
||||
let n = bars.len();
|
||||
let mut features = Vec::with_capacity(n);
|
||||
|
||||
for i in 0..n {
|
||||
let mut row = Vec::with_capacity(15);
|
||||
|
||||
// 0-4: normalized OHLCV
|
||||
if let Some(price_row) = feat_matrix.prices.get(i) {
|
||||
row.extend_from_slice(price_row);
|
||||
} else {
|
||||
row.extend_from_slice(&[0.0_f32; 5]);
|
||||
}
|
||||
|
||||
// 5: RSI
|
||||
row.push(indicators.rsi.get(i).copied().unwrap_or(50.0) / 100.0); // normalize to 0-1
|
||||
|
||||
// 6-7: EMA fast, slow (normalize relative to close)
|
||||
let close = bars.get(i).map(|b| b.close as f32).unwrap_or(1.0);
|
||||
let denom = if close.abs() > 1e-10 { close } else { 1.0 };
|
||||
row.push(indicators.ema_fast.get(i).copied().unwrap_or(0.0) / denom);
|
||||
row.push(indicators.ema_slow.get(i).copied().unwrap_or(0.0) / denom);
|
||||
|
||||
// 8-10: MACD line, signal, histogram (already small values)
|
||||
let macd_line = indicators.macd.get(i).copied().unwrap_or(0.0);
|
||||
let macd_signal = indicators.macd_signal.get(i).copied().unwrap_or(0.0);
|
||||
row.push(macd_line);
|
||||
row.push(macd_signal);
|
||||
row.push(macd_line - macd_signal); // histogram
|
||||
|
||||
// 11-13: Bollinger Bands (normalized relative to close)
|
||||
row.push(indicators.bb_upper.get(i).copied().unwrap_or(0.0) / denom);
|
||||
row.push(indicators.bb_middle.get(i).copied().unwrap_or(0.0) / denom);
|
||||
row.push(indicators.bb_lower.get(i).copied().unwrap_or(0.0) / denom);
|
||||
|
||||
// 14: ATR (as fraction of close)
|
||||
row.push(indicators.atr.get(i).copied().unwrap_or(0.0) / denom);
|
||||
|
||||
features.push(row);
|
||||
}
|
||||
|
||||
features
|
||||
}
|
||||
|
||||
fn make_dqn_config() -> DQNConfig {
|
||||
let mut config = DQNConfig::default();
|
||||
config.state_dim = 15; // 5 OHLCV + 10 indicators
|
||||
config.num_actions = 3; // short / flat / long
|
||||
config.hidden_dims = vec![64, 32];
|
||||
config.batch_size = 16;
|
||||
config.min_replay_size = 16;
|
||||
config.warmup_steps = 0;
|
||||
config.use_noisy_nets = false;
|
||||
config.use_iqn = false;
|
||||
config.use_distributional = false;
|
||||
config.use_dueling = true;
|
||||
config.use_per = false;
|
||||
config.epsilon_start = 0.3;
|
||||
config.epsilon_end = 0.01;
|
||||
config
|
||||
}
|
||||
|
||||
/// Full walk-forward validation on real 6E.FUT minute-bar data.
|
||||
///
|
||||
/// Loads ~30 days of 1-minute OHLCV, extracts 15-dim features,
|
||||
/// runs walk-forward with embargo, and prints the complete
|
||||
/// ValidationReport including DSR, PBO, permutation test, and per-regime metrics.
|
||||
#[tokio::test]
|
||||
async fn test_validation_on_real_6e_data() {
|
||||
// 1. Load real data (auto-detect workspace root)
|
||||
let mut loader = RealDataLoader::new_from_workspace()
|
||||
.expect("Failed to find workspace root — run from foxhunt repo");
|
||||
let bars = loader
|
||||
.load_symbol_data("6E.FUT")
|
||||
.await
|
||||
.expect("Failed to load 6E.FUT data — check test_data/real/databento/ exists");
|
||||
|
||||
println!("Loaded {} bars for 6E.FUT", bars.len());
|
||||
assert!(
|
||||
bars.len() > 500,
|
||||
"Expected at least 500 bars from 6E.FUT DBN, got {}",
|
||||
bars.len()
|
||||
);
|
||||
|
||||
// Print data range
|
||||
if let (Some(first), Some(last)) = (bars.first(), bars.last()) {
|
||||
println!(
|
||||
"Data range: {} to {}",
|
||||
first.timestamp, last.timestamp
|
||||
);
|
||||
println!(
|
||||
"First bar: O={:.5} H={:.5} L={:.5} C={:.5} V={:.0}",
|
||||
first.open, first.high, first.low, first.close, first.volume
|
||||
);
|
||||
}
|
||||
|
||||
// 2. Build features
|
||||
let features = build_features_from_loader(&loader, &bars);
|
||||
assert_eq!(features.len(), bars.len());
|
||||
|
||||
// Verify features are finite
|
||||
for (i, row) in features.iter().enumerate() {
|
||||
assert_eq!(row.len(), 15, "Bar {} has {} features, expected 15", i, row.len());
|
||||
for (j, v) in row.iter().enumerate() {
|
||||
assert!(
|
||||
v.is_finite(),
|
||||
"Feature [{},{}] is not finite: {}",
|
||||
i,
|
||||
j,
|
||||
v
|
||||
);
|
||||
}
|
||||
}
|
||||
println!("Features: {} bars × {} dims, all finite", features.len(), 15);
|
||||
|
||||
// 3. Build TimeSeriesData
|
||||
let timestamps: Vec<DateTime<Utc>> = bars.iter().map(|b| b.timestamp).collect();
|
||||
let prices: Vec<f64> = bars.iter().map(|b| b.close).collect();
|
||||
|
||||
let data = TimeSeriesData::new(timestamps, features, prices)
|
||||
.expect("Failed to create TimeSeriesData");
|
||||
|
||||
println!(
|
||||
"TimeSeriesData: {} bars, {} returns",
|
||||
data.len(),
|
||||
data.returns.len()
|
||||
);
|
||||
|
||||
// 4. Create DQN strategy
|
||||
let config = make_dqn_config();
|
||||
let mut strategy =
|
||||
DqnStrategy::new(config).expect("Failed to create DqnStrategy");
|
||||
|
||||
// 5. Configure walk-forward harness
|
||||
// For 1-min bars: 500 bars ≈ 8 hours of training data
|
||||
// test = 100 bars ≈ 1.5 hours
|
||||
// embargo = 20 bars ≈ 20 minutes
|
||||
let num_bars = data.len();
|
||||
let train_bars = (num_bars / 5).max(200);
|
||||
let test_bars = (num_bars / 20).max(50);
|
||||
|
||||
let harness_config = ValidationHarnessConfig {
|
||||
wf_config: WalkForwardConfig {
|
||||
train_bars,
|
||||
test_bars,
|
||||
embargo_bars: 20,
|
||||
step_bars: test_bars,
|
||||
min_train_samples: 100,
|
||||
},
|
||||
num_permutations: 500, // Reduced for speed; 10k for final
|
||||
num_trials: 1,
|
||||
seed: 42,
|
||||
};
|
||||
|
||||
println!("\n=== Walk-Forward Config ===");
|
||||
println!(" train_bars: {}", train_bars);
|
||||
println!(" test_bars: {}", test_bars);
|
||||
println!(" embargo: 20");
|
||||
println!(" step: {}", test_bars);
|
||||
println!(" total bars: {}", num_bars);
|
||||
|
||||
let harness = ValidationHarness::new(harness_config);
|
||||
|
||||
// 6. Run validation
|
||||
println!("\n=== Running validation... ===");
|
||||
let report = harness
|
||||
.validate(&mut strategy, &data)
|
||||
.expect("Validation harness failed");
|
||||
|
||||
// 7. Print full report
|
||||
println!("\n╔══════════════════════════════════════════╗");
|
||||
println!("║ VALIDATION REPORT: {} ║", report.strategy_name);
|
||||
println!("╠══════════════════════════════════════════╣");
|
||||
println!("║ Folds: {:>20} ║", report.num_folds);
|
||||
println!("║ Aggregate Sharpe: {:>20.4} ║", report.aggregate_sharpe);
|
||||
println!("╠══════════════════════════════════════════╣");
|
||||
println!("║ DEFLATED SHARPE RATIO ║");
|
||||
println!("║ Observed SR: {:>20.4} ║", report.dsr.observed_sharpe);
|
||||
println!("║ Expected max: {:>20.4} ║", report.dsr.expected_max_sharpe);
|
||||
println!("║ SE: {:>20.4} ║", report.dsr.sharpe_std_error);
|
||||
println!("║ DSR statistic: {:>20.4} ║", report.dsr.deflated_sharpe);
|
||||
println!("║ p-value: {:>20.4} ║", report.dsr.pvalue);
|
||||
println!("╠══════════════════════════════════════════╣");
|
||||
println!("║ PBO (CSCV) ║");
|
||||
println!("║ PBO: {:>20.4} ║", report.pbo.pbo);
|
||||
println!("║ Combinations: {:>20} ║", report.pbo.num_combinations);
|
||||
println!("╠══════════════════════════════════════════╣");
|
||||
println!("║ PERMUTATION TEST ║");
|
||||
println!("║ Observed SR: {:>20.4} ║", report.permutation.observed_sharpe);
|
||||
println!("║ Null mean: {:>20.4} ║", report.permutation.null_mean);
|
||||
println!("║ Null std: {:>20.4} ║", report.permutation.null_std);
|
||||
println!("║ p-value: {:>20.4} ║", report.permutation.pvalue);
|
||||
println!("║ Permutations: {:>20} ║", report.permutation.num_permutations);
|
||||
println!("╠══════════════════════════════════════════╣");
|
||||
println!("║ PER-FOLD SHARPES ║");
|
||||
for (i, sr) in report.per_fold_sharpes.iter().enumerate() {
|
||||
println!("║ Fold {:>2}: {:>20.4} ║", i, sr);
|
||||
}
|
||||
println!("╠══════════════════════════════════════════╣");
|
||||
println!("║ PER-REGIME BREAKDOWN ║");
|
||||
for (regime, m) in &report.per_regime_metrics {
|
||||
println!(
|
||||
"║ {:?}: SR={:.4}, bars={}, WR={:.2}%, avg_ret={:.6}",
|
||||
regime, m.sharpe, m.num_bars, m.win_rate * 100.0, m.avg_return
|
||||
);
|
||||
}
|
||||
println!("╠══════════════════════════════════════════╣");
|
||||
println!("║ VERDICT: {:>32} ║", format!("{}", report.verdict));
|
||||
println!("╚══════════════════════════════════════════╝");
|
||||
|
||||
// 8. Structural assertions (not outcome-dependent)
|
||||
assert!(report.num_folds >= 2, "Need at least 2 folds");
|
||||
assert!(report.aggregate_sharpe.is_finite());
|
||||
assert!((0.0..=1.0).contains(&report.dsr.pvalue));
|
||||
assert!((0.0..=1.0).contains(&report.pbo.pbo));
|
||||
assert!((0.0..=1.0).contains(&report.permutation.pvalue));
|
||||
assert!(!report.per_regime_metrics.is_empty());
|
||||
}
|
||||
Reference in New Issue
Block a user