285 lines
8.8 KiB
Rust
285 lines
8.8 KiB
Rust
//! End-to-end integration tests for the validation harness pipeline.
|
|
//!
|
|
//! Tests the full flow: DQN strategy creation -> walk-forward splitting ->
|
|
//! train/evaluate per fold -> DSR/PBO/permutation -> report with verdict.
|
|
|
|
use chrono::{Duration, TimeZone, Utc};
|
|
use ml::dqn::DQNConfig;
|
|
use ml::validation::{
|
|
deflated_sharpe_ratio, probability_of_backtest_overfitting, walk_forward_split,
|
|
DqnStrategy, TimeSeriesData, ValidationHarness,
|
|
ValidationHarnessConfig, WalkForwardConfig,
|
|
};
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Helpers
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/// Create `n` UTC timestamps starting from 2024-01-01, one day apart.
|
|
fn make_timestamps(n: usize) -> Vec<chrono::DateTime<Utc>> {
|
|
(0..n)
|
|
.map(|i| {
|
|
Utc.with_ymd_and_hms(2024, 1, 1, 0, 0, 0)
|
|
.single()
|
|
.unwrap_or_else(Utc::now)
|
|
+ Duration::days(i as i64)
|
|
})
|
|
.collect()
|
|
}
|
|
|
|
/// Create `n` feature rows of dimension `dim` with deterministic non-zero values.
|
|
fn make_features(n: usize, dim: usize) -> Vec<Vec<f32>> {
|
|
(0..n)
|
|
.map(|i| {
|
|
(0..dim)
|
|
.map(|j| ((i * 7 + j * 3) % 100) as f32 * 0.01)
|
|
.collect()
|
|
})
|
|
.collect()
|
|
}
|
|
|
|
/// Build a minimal DQNConfig suitable for fast integration testing.
|
|
fn make_small_dqn_config() -> DQNConfig {
|
|
let mut config = DQNConfig::default();
|
|
config.state_dim = 8;
|
|
config.num_actions = 3;
|
|
config.hidden_dims = vec![16, 8];
|
|
config.batch_size = 4;
|
|
config.min_replay_size = 4;
|
|
config.warmup_steps = 0;
|
|
config.use_noisy_nets = false;
|
|
config.use_iqn = false;
|
|
config.use_distributional = false;
|
|
config.use_dueling = false;
|
|
config.use_per = false;
|
|
config.epsilon_start = 0.3;
|
|
config
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Test 1: Full validation pipeline with DQN
|
|
// ---------------------------------------------------------------------------
|
|
|
|
#[test]
|
|
fn test_full_validation_pipeline_with_dqn() {
|
|
// 1. Create DQN strategy
|
|
let config = make_small_dqn_config();
|
|
let mut strategy =
|
|
DqnStrategy::new(config).unwrap_or_else(|e| panic!("DqnStrategy::new failed: {}", e));
|
|
|
|
// 2. Create synthetic TimeSeriesData (300 bars, feature_dim=8)
|
|
// Prices follow sine + trend: 100.0 + sin(i*0.1)*5.0 + i*0.01
|
|
let n = 300_usize;
|
|
let prices: Vec<f64> = (0..n)
|
|
.map(|i| 100.0 + (i as f64 * 0.1).sin() * 5.0 + i as f64 * 0.01)
|
|
.collect();
|
|
let timestamps = make_timestamps(n);
|
|
let features = make_features(n, 8);
|
|
|
|
let data = TimeSeriesData::new(timestamps, features, prices)
|
|
.unwrap_or_else(|e| panic!("TimeSeriesData::new failed: {}", e));
|
|
|
|
// 3. Configure harness
|
|
let harness_config = ValidationHarnessConfig {
|
|
wf_config: WalkForwardConfig {
|
|
train_bars: 50,
|
|
test_bars: 30,
|
|
embargo_bars: 5,
|
|
step_bars: 30,
|
|
min_train_samples: 20,
|
|
},
|
|
num_permutations: 100,
|
|
num_trials: 1,
|
|
seed: 42,
|
|
};
|
|
|
|
let harness = ValidationHarness::new(harness_config);
|
|
|
|
// 4. Run validation
|
|
let report = harness
|
|
.validate(&mut strategy, &data)
|
|
.unwrap_or_else(|e| panic!("validate failed: {}", e));
|
|
|
|
// 5. Assertions
|
|
assert_eq!(
|
|
report.strategy_name, "DQN",
|
|
"Strategy name should be 'DQN', got '{}'",
|
|
report.strategy_name
|
|
);
|
|
|
|
assert!(
|
|
report.num_folds >= 2,
|
|
"Expected at least 2 folds, got {}",
|
|
report.num_folds
|
|
);
|
|
|
|
assert!(
|
|
report.aggregate_sharpe.is_finite(),
|
|
"Aggregate Sharpe should be finite, got {}",
|
|
report.aggregate_sharpe
|
|
);
|
|
|
|
// DSR p-value in [0, 1]
|
|
assert!(
|
|
(0.0..=1.0).contains(&report.dsr.pvalue),
|
|
"DSR p-value out of range [0,1]: {}",
|
|
report.dsr.pvalue
|
|
);
|
|
|
|
// PBO in [0, 1]
|
|
assert!(
|
|
(0.0..=1.0).contains(&report.pbo.pbo),
|
|
"PBO out of range [0,1]: {}",
|
|
report.pbo.pbo
|
|
);
|
|
|
|
// Permutation p-value in [0, 1]
|
|
assert!(
|
|
(0.0..=1.0).contains(&report.permutation.pvalue),
|
|
"Permutation p-value out of range [0,1]: {}",
|
|
report.permutation.pvalue
|
|
);
|
|
|
|
// Per-regime metrics should not be empty
|
|
assert!(
|
|
!report.per_regime_metrics.is_empty(),
|
|
"per_regime_metrics should not be empty"
|
|
);
|
|
|
|
// Print summary
|
|
println!("=== Validation Report Summary ===");
|
|
println!("Strategy: {}", report.strategy_name);
|
|
println!("Folds: {}", report.num_folds);
|
|
println!("Aggregate SR: {:.4}", report.aggregate_sharpe);
|
|
println!("DSR p-value: {:.4}", report.dsr.pvalue);
|
|
println!("PBO: {:.4}", report.pbo.pbo);
|
|
println!("MC perm p: {:.4}", report.permutation.pvalue);
|
|
println!("Verdict: {}", report.verdict);
|
|
println!("Regimes: {}", report.per_regime_metrics.len());
|
|
for (regime, metrics) in &report.per_regime_metrics {
|
|
println!(
|
|
" {:?}: sharpe={:.4}, bars={}, win_rate={:.2}, avg_ret={:.6}",
|
|
regime, metrics.sharpe, metrics.num_bars, metrics.win_rate, metrics.avg_return
|
|
);
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Test 2: Walk-forward split standalone
|
|
// ---------------------------------------------------------------------------
|
|
|
|
#[test]
|
|
fn test_walk_forward_split_standalone() {
|
|
let config = WalkForwardConfig {
|
|
train_bars: 50,
|
|
test_bars: 20,
|
|
embargo_bars: 5,
|
|
step_bars: 20,
|
|
min_train_samples: 20,
|
|
};
|
|
|
|
let folds = walk_forward_split(200, &config);
|
|
|
|
assert!(
|
|
!folds.is_empty(),
|
|
"Expected at least one fold from 200 bars"
|
|
);
|
|
|
|
for fold in &folds {
|
|
// test_range.end must not exceed total bars
|
|
assert!(
|
|
fold.test_range.end <= 200,
|
|
"Fold {} test_range.end ({}) exceeds 200",
|
|
fold.fold_index,
|
|
fold.test_range.end
|
|
);
|
|
|
|
// train.end <= embargo.start (they should be equal by construction)
|
|
assert!(
|
|
fold.train_range.end <= fold.embargo_range.start,
|
|
"Fold {} train.end ({}) > embargo.start ({})",
|
|
fold.fold_index,
|
|
fold.train_range.end,
|
|
fold.embargo_range.start
|
|
);
|
|
|
|
// embargo.end <= test.start (they should be equal by construction)
|
|
assert!(
|
|
fold.embargo_range.end <= fold.test_range.start,
|
|
"Fold {} embargo.end ({}) > test.start ({})",
|
|
fold.fold_index,
|
|
fold.embargo_range.end,
|
|
fold.test_range.start
|
|
);
|
|
}
|
|
|
|
println!(
|
|
"Walk-forward split: {} folds from 200 bars",
|
|
folds.len()
|
|
);
|
|
for fold in &folds {
|
|
println!(
|
|
" Fold {}: train={:?}, embargo={:?}, test={:?}",
|
|
fold.fold_index, fold.train_range, fold.embargo_range, fold.test_range
|
|
);
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Test 3: DSR and PBO standalone
|
|
// ---------------------------------------------------------------------------
|
|
|
|
#[test]
|
|
fn test_dsr_and_pbo_standalone() {
|
|
// --- DSR ---
|
|
// High Sharpe (2.5) with few trials (5) should be significant -> p < 0.1
|
|
let dsr = deflated_sharpe_ratio(2.5, 5, 0.5, 0.1, 3.5, 500);
|
|
assert!(
|
|
dsr.pvalue < 0.1,
|
|
"DSR: SR=2.5 with 5 trials should have p < 0.1, got {:.4}",
|
|
dsr.pvalue
|
|
);
|
|
assert!(
|
|
dsr.observed_sharpe.is_finite(),
|
|
"DSR observed_sharpe should be finite"
|
|
);
|
|
assert!(
|
|
dsr.deflated_sharpe.is_finite(),
|
|
"DSR deflated_sharpe should be finite"
|
|
);
|
|
assert!(
|
|
dsr.sharpe_std_error.is_finite() && dsr.sharpe_std_error >= 0.0,
|
|
"DSR sharpe_std_error should be non-negative and finite, got {}",
|
|
dsr.sharpe_std_error
|
|
);
|
|
|
|
println!("DSR result: observed={:.4}, deflated={:.4}, p={:.4}",
|
|
dsr.observed_sharpe, dsr.deflated_sharpe, dsr.pvalue);
|
|
|
|
// --- PBO ---
|
|
// Consistent positive Sharpes across 8 folds -> should have num_combinations > 0
|
|
let sharpes = vec![1.0, 1.2, 0.8, 1.1, 0.9, 1.3, 1.0, 0.95];
|
|
let pbo = probability_of_backtest_overfitting(&sharpes);
|
|
assert!(
|
|
pbo.num_combinations > 0,
|
|
"PBO should have evaluated combinations, got {}",
|
|
pbo.num_combinations
|
|
);
|
|
assert!(
|
|
(0.0..=1.0).contains(&pbo.pbo),
|
|
"PBO value should be in [0,1], got {}",
|
|
pbo.pbo
|
|
);
|
|
assert!(
|
|
!pbo.logit_distribution.is_empty(),
|
|
"PBO logit_distribution should not be empty"
|
|
);
|
|
|
|
println!(
|
|
"PBO result: pbo={:.4}, num_combinations={}, logit_entries={}",
|
|
pbo.pbo,
|
|
pbo.num_combinations,
|
|
pbo.logit_distribution.len()
|
|
);
|
|
}
|