e3f32742fa
Fix zstd decoder in train_baseline.rs and evaluate_baseline.rs (same pattern as hyperopt adapters — branch on .dbn.zst extension). Add CLI flags for walk-forward config (train/val/test/step months), learning rate, and max-steps-per-epoch to make pipeline validation feasible. Pipeline validated end-to-end: hyperopt (5 trials, best Sharpe 2.37) → walk-forward training (4 folds, 6/1/1 month windows on ES.FUT) → evaluation (4 fold test sets, checkpoints + norm stats saved). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
935 lines
31 KiB
Rust
935 lines
31 KiB
Rust
//! Walk-forward evaluation binary for DQN and PPO baseline models.
|
||
//!
|
||
//! Loads trained model checkpoints, runs inference on walk-forward test data,
|
||
//! computes financial metrics (Sharpe, drawdown, win rate, profit factor),
|
||
//! and generates a JSON report.
|
||
//!
|
||
//! # Usage
|
||
//!
|
||
//! ```bash
|
||
//! SQLX_OFFLINE=true cargo run -p ml --example evaluate_baseline -- \
|
||
//! --model both --models-dir ml/trained_models \
|
||
//! --data-dir data/cache/futures-baseline \
|
||
//! --output ml/trained_models/evaluation_report.json
|
||
//! ```
|
||
|
||
#![allow(unused_crate_dependencies)]
|
||
#![deny(
|
||
clippy::unwrap_used,
|
||
clippy::expect_used,
|
||
clippy::panic,
|
||
clippy::indexing_slicing
|
||
)]
|
||
|
||
use std::path::{Path, PathBuf};
|
||
|
||
use anyhow::{Context, Result};
|
||
use chrono::{DateTime, TimeZone, Utc};
|
||
use clap::Parser;
|
||
use serde::Serialize;
|
||
use tracing::{error, info, warn};
|
||
|
||
use dbn::decode::DecodeRecord;
|
||
use ml::dqn::{DQNConfig, DQN};
|
||
use ml::features::extraction::extract_ml_features;
|
||
use ml::ppo::ppo::{PPOConfig, PPO};
|
||
use ml::types::OHLCVBar;
|
||
use ml::walk_forward::{generate_walk_forward_windows, NormStats, WalkForwardConfig};
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// CLI Arguments
|
||
// ---------------------------------------------------------------------------
|
||
|
||
/// Walk-forward evaluation binary for DQN/PPO baseline models.
|
||
#[derive(Parser, Debug)]
|
||
#[command(
|
||
name = "evaluate_baseline",
|
||
about = "Evaluate trained DQN/PPO checkpoints with walk-forward test data"
|
||
)]
|
||
struct Args {
|
||
/// Directory containing trained model checkpoints
|
||
#[arg(long, default_value = "ml/trained_models")]
|
||
models_dir: PathBuf,
|
||
|
||
/// Path to directory containing .dbn.zst files
|
||
#[arg(long, default_value = "data/cache/futures-baseline")]
|
||
data_dir: PathBuf,
|
||
|
||
/// Output path for evaluation report JSON
|
||
#[arg(long, default_value = "ml/trained_models/evaluation_report.json")]
|
||
output: PathBuf,
|
||
|
||
/// Which model(s) to evaluate: "dqn", "ppo", or "both"
|
||
#[arg(long, default_value = "both")]
|
||
model: String,
|
||
|
||
/// Feature dimension (must match extract_ml_features output)
|
||
#[arg(long, default_value_t = 51)]
|
||
feature_dim: usize,
|
||
|
||
/// Number of actions for the DQN/PPO action space
|
||
#[arg(long, default_value_t = 3)]
|
||
num_actions: usize,
|
||
|
||
/// Walk-forward: initial training window in months
|
||
#[arg(long, default_value_t = 12)]
|
||
train_months: u32,
|
||
|
||
/// Walk-forward: validation window in months
|
||
#[arg(long, default_value_t = 3)]
|
||
val_months: u32,
|
||
|
||
/// Walk-forward: test window in months
|
||
#[arg(long, default_value_t = 3)]
|
||
test_months: u32,
|
||
|
||
/// Walk-forward: step size in months between folds
|
||
#[arg(long, default_value_t = 3)]
|
||
step_months: u32,
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Report Data Types
|
||
// ---------------------------------------------------------------------------
|
||
|
||
/// Metrics for a single fold/model combination.
|
||
#[derive(Debug, Serialize)]
|
||
struct FoldMetrics {
|
||
fold: usize,
|
||
model: String,
|
||
sharpe_ratio: f64,
|
||
max_drawdown_pct: f64,
|
||
win_rate_pct: f64,
|
||
profit_factor: f64,
|
||
total_return_pct: f64,
|
||
num_trades: usize,
|
||
test_start: String,
|
||
test_end: String,
|
||
}
|
||
|
||
/// Aggregate metrics across all folds for both models.
|
||
#[derive(Debug, Serialize)]
|
||
struct AggregateMetrics {
|
||
dqn_avg_sharpe: f64,
|
||
dqn_avg_drawdown: f64,
|
||
dqn_avg_win_rate: f64,
|
||
ppo_avg_sharpe: f64,
|
||
ppo_avg_drawdown: f64,
|
||
ppo_avg_win_rate: f64,
|
||
}
|
||
|
||
/// Sanity checks to flag obviously broken models.
|
||
#[derive(Debug, Serialize)]
|
||
struct SanityChecks {
|
||
/// True if any model has Sharpe > 0
|
||
beats_random: bool,
|
||
/// True if all 3 actions (buy/sell/hold) were used
|
||
action_diversity: bool,
|
||
/// True if Sharpe std < 2x |mean Sharpe|
|
||
fold_consistency: bool,
|
||
}
|
||
|
||
/// Full evaluation report written to JSON.
|
||
#[derive(Debug, Serialize)]
|
||
struct EvaluationReport {
|
||
folds: Vec<FoldMetrics>,
|
||
aggregate: AggregateMetrics,
|
||
sanity_checks: SanityChecks,
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// DBN Loading (reused from train_baseline)
|
||
// ---------------------------------------------------------------------------
|
||
|
||
/// Recursively discover .dbn.zst files under `dir`.
|
||
fn find_dbn_files(dir: &Path) -> Result<Vec<PathBuf>> {
|
||
let mut files = Vec::new();
|
||
if !dir.exists() {
|
||
anyhow::bail!("Data directory does not exist: {}", dir.display());
|
||
}
|
||
collect_dbn_files_recursive(dir, &mut files)?;
|
||
files.sort();
|
||
Ok(files)
|
||
}
|
||
|
||
/// Recursive helper that walks the directory tree without `walkdir`.
|
||
fn collect_dbn_files_recursive(dir: &Path, out: &mut Vec<PathBuf>) -> Result<()> {
|
||
let entries = std::fs::read_dir(dir)
|
||
.with_context(|| format!("Cannot read directory: {}", dir.display()))?;
|
||
for entry in entries {
|
||
let entry = entry.with_context(|| "Failed to read dir entry")?;
|
||
let path = entry.path();
|
||
if path.is_dir() {
|
||
collect_dbn_files_recursive(&path, out)?;
|
||
} else if path
|
||
.file_name()
|
||
.and_then(|n| n.to_str())
|
||
.map(|n| n.ends_with(".dbn.zst"))
|
||
.unwrap_or(false)
|
||
{
|
||
out.push(path);
|
||
}
|
||
}
|
||
Ok(())
|
||
}
|
||
|
||
/// Load OHLCV bars from a single .dbn or .dbn.zst file.
|
||
fn load_bars_from_dbn(path: &Path) -> Result<Vec<OHLCVBar>> {
|
||
if path.to_string_lossy().ends_with(".dbn.zst") {
|
||
let mut decoder = dbn::decode::dbn::Decoder::from_zstd_file(path)
|
||
.with_context(|| format!("Failed to create zstd DBN decoder for {}", path.display()))?;
|
||
decode_ohlcv_records(&mut decoder, path)
|
||
} else {
|
||
let file = std::fs::File::open(path)
|
||
.with_context(|| format!("Cannot open DBN file: {}", path.display()))?;
|
||
let buf = std::io::BufReader::new(file);
|
||
let mut decoder = dbn::decode::dbn::Decoder::new(buf)
|
||
.with_context(|| format!("Failed to create DBN decoder for {}", path.display()))?;
|
||
decode_ohlcv_records(&mut decoder, path)
|
||
}
|
||
}
|
||
|
||
/// Decode OHLCV records from an already-opened DBN decoder.
|
||
fn decode_ohlcv_records<R: std::io::Read>(
|
||
decoder: &mut dbn::decode::dbn::Decoder<R>,
|
||
path: &Path,
|
||
) -> Result<Vec<OHLCVBar>> {
|
||
use dbn::OhlcvMsg;
|
||
|
||
let mut bars = Vec::new();
|
||
while let Some(record) = decoder
|
||
.decode_record::<OhlcvMsg>()
|
||
.with_context(|| format!("Error decoding records from {}", path.display()))?
|
||
{
|
||
let ts_nanos = record.hd.ts_event;
|
||
let timestamp = nanos_to_datetime(ts_nanos);
|
||
let price_scale = 1e-9_f64;
|
||
bars.push(OHLCVBar {
|
||
timestamp,
|
||
open: record.open as f64 * price_scale,
|
||
high: record.high as f64 * price_scale,
|
||
low: record.low as f64 * price_scale,
|
||
close: record.close as f64 * price_scale,
|
||
volume: record.volume as f64,
|
||
});
|
||
}
|
||
Ok(bars)
|
||
}
|
||
|
||
/// Convert nanosecond UNIX timestamp to chrono DateTime<Utc>.
|
||
fn nanos_to_datetime(nanos: u64) -> DateTime<Utc> {
|
||
let secs = (nanos / 1_000_000_000) as i64;
|
||
let subsec_nanos = (nanos % 1_000_000_000) as u32;
|
||
Utc.timestamp_opt(secs, subsec_nanos)
|
||
.single()
|
||
.unwrap_or_else(Utc::now)
|
||
}
|
||
|
||
/// Load all OHLCV bars from a directory of .dbn.zst files, sorted chronologically.
|
||
fn load_all_bars(data_dir: &Path) -> Result<Vec<OHLCVBar>> {
|
||
let dbn_files = find_dbn_files(data_dir)?;
|
||
if dbn_files.is_empty() {
|
||
anyhow::bail!(
|
||
"No .dbn.zst files found in {}. Run download_baseline first.",
|
||
data_dir.display()
|
||
);
|
||
}
|
||
info!(
|
||
"Found {} .dbn.zst files in {}",
|
||
dbn_files.len(),
|
||
data_dir.display()
|
||
);
|
||
|
||
let mut all_bars = Vec::new();
|
||
for path in &dbn_files {
|
||
match load_bars_from_dbn(path) {
|
||
Ok(bars) => {
|
||
info!(" {} -> {} bars", path.display(), bars.len());
|
||
all_bars.extend(bars);
|
||
}
|
||
Err(e) => {
|
||
warn!(" Skipping {} — {}", path.display(), e);
|
||
}
|
||
}
|
||
}
|
||
|
||
all_bars.sort_by_key(|b| b.timestamp);
|
||
info!("Total bars loaded: {}", all_bars.len());
|
||
Ok(all_bars)
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Financial Metrics
|
||
// ---------------------------------------------------------------------------
|
||
|
||
/// Container for computed financial metrics from a sequence of trade returns.
|
||
struct ComputedMetrics {
|
||
sharpe_ratio: f64,
|
||
max_drawdown_pct: f64,
|
||
win_rate_pct: f64,
|
||
profit_factor: f64,
|
||
total_return_pct: f64,
|
||
num_trades: usize,
|
||
}
|
||
|
||
/// Compute financial metrics from a sequence of per-bar trade returns.
|
||
///
|
||
/// - Sharpe: annualized (mean / std * sqrt(252))
|
||
/// - Max drawdown: largest peak-to-trough drop on cumulative equity curve (%)
|
||
/// - Win rate: percentage of returns > 0
|
||
/// - Profit factor: gross_profit / gross_loss (inf if no losses)
|
||
/// - Total return: sum of returns * 100 (as percentage)
|
||
/// - Num trades: count of non-zero returns (BUY or SELL actions)
|
||
fn compute_metrics(returns: &[f64]) -> ComputedMetrics {
|
||
let n = returns.len();
|
||
if n == 0 {
|
||
return ComputedMetrics {
|
||
sharpe_ratio: 0.0,
|
||
max_drawdown_pct: 0.0,
|
||
win_rate_pct: 0.0,
|
||
profit_factor: 0.0,
|
||
total_return_pct: 0.0,
|
||
num_trades: 0,
|
||
};
|
||
}
|
||
|
||
// Count actual trades (non-zero returns, i.e. BUY or SELL actions)
|
||
let num_trades = returns.iter().filter(|&&r| r.abs() > 1e-12).count();
|
||
|
||
// Mean and std of returns
|
||
let sum: f64 = returns.iter().sum();
|
||
let mean = sum / n as f64;
|
||
|
||
let variance: f64 = returns.iter().map(|&r| (r - mean).powi(2)).sum::<f64>() / n as f64;
|
||
let std = variance.sqrt();
|
||
|
||
// Annualized Sharpe ratio (1-minute bars: 390 bars/day × 252 days/year)
|
||
let bars_per_year: f64 = 252.0 * 390.0; // ~98,280 bars/year
|
||
let sharpe_ratio = if std > 1e-12 {
|
||
(mean / std) * bars_per_year.sqrt()
|
||
} else {
|
||
0.0
|
||
};
|
||
|
||
// Max drawdown on cumulative equity curve
|
||
let mut equity = 1.0_f64;
|
||
let mut peak = 1.0_f64;
|
||
let mut max_drawdown = 0.0_f64;
|
||
|
||
for &ret in returns {
|
||
equity += ret;
|
||
if equity > peak {
|
||
peak = equity;
|
||
}
|
||
let drawdown = if peak > 1e-12 {
|
||
(peak - equity) / peak
|
||
} else {
|
||
0.0
|
||
};
|
||
if drawdown > max_drawdown {
|
||
max_drawdown = drawdown;
|
||
}
|
||
}
|
||
let max_drawdown_pct = max_drawdown * 100.0;
|
||
|
||
// Win rate
|
||
let wins = returns.iter().filter(|&&r| r > 0.0).count();
|
||
let win_rate_pct = if num_trades > 0 {
|
||
(wins as f64 / num_trades as f64) * 100.0
|
||
} else {
|
||
0.0
|
||
};
|
||
|
||
// Profit factor
|
||
let gross_profit: f64 = returns.iter().filter(|&&r| r > 0.0).sum();
|
||
let gross_loss: f64 = returns.iter().filter(|&&r| r < 0.0).map(|&r| r.abs()).sum();
|
||
let profit_factor = if gross_loss > 1e-12 {
|
||
gross_profit / gross_loss
|
||
} else if gross_profit > 0.0 {
|
||
f64::INFINITY
|
||
} else {
|
||
0.0
|
||
};
|
||
|
||
// Total return
|
||
let total_return_pct = sum * 100.0;
|
||
|
||
ComputedMetrics {
|
||
sharpe_ratio,
|
||
max_drawdown_pct,
|
||
win_rate_pct,
|
||
profit_factor,
|
||
total_return_pct,
|
||
num_trades,
|
||
}
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// DQN Evaluation
|
||
// ---------------------------------------------------------------------------
|
||
|
||
/// Run DQN inference on test features and return per-bar trade returns and
|
||
/// action counts (buy, sell, hold).
|
||
fn evaluate_dqn_fold(
|
||
fold: usize,
|
||
test_features: &[[f64; 51]],
|
||
test_bars: &[OHLCVBar],
|
||
models_dir: &Path,
|
||
args: &Args,
|
||
) -> Result<(Vec<f64>, [usize; 3])> {
|
||
let ckpt_path = models_dir.join(format!("dqn_fold{}_best.safetensors", fold));
|
||
if !ckpt_path.exists() {
|
||
anyhow::bail!(
|
||
"DQN checkpoint not found: {}",
|
||
ckpt_path.display()
|
||
);
|
||
}
|
||
|
||
// Create DQN with same config as training
|
||
let config = DQNConfig {
|
||
state_dim: args.feature_dim,
|
||
num_actions: args.num_actions,
|
||
hidden_dims: vec![128, 64],
|
||
learning_rate: 1e-4,
|
||
gamma: 0.99,
|
||
epsilon_start: 0.0, // No exploration during evaluation
|
||
epsilon_end: 0.0,
|
||
epsilon_decay: 1.0,
|
||
replay_buffer_capacity: 100, // Minimal buffer, not used for eval
|
||
batch_size: 64,
|
||
min_replay_size: 64,
|
||
target_update_freq: 500,
|
||
warmup_steps: 0,
|
||
use_double_dqn: true,
|
||
use_huber_loss: true,
|
||
use_per: false,
|
||
use_dueling: false,
|
||
use_distributional: false,
|
||
use_noisy_nets: false,
|
||
use_cql: false,
|
||
use_iqn: false,
|
||
use_cvar_action_selection: false,
|
||
..DQNConfig::default()
|
||
};
|
||
|
||
let mut dqn = DQN::new(config).context("Failed to create DQN model")?;
|
||
|
||
// Set epsilon to zero for pure greedy evaluation
|
||
dqn.set_epsilon(0.0);
|
||
|
||
// Load trained weights
|
||
dqn.load_from_safetensors(&ckpt_path.to_string_lossy())
|
||
.with_context(|| format!("Failed to load DQN checkpoint: {}", ckpt_path.display()))?;
|
||
|
||
info!(
|
||
" [DQN] Loaded checkpoint: {} (eps={})",
|
||
ckpt_path.display(),
|
||
dqn.get_epsilon()
|
||
);
|
||
|
||
// Run inference
|
||
let n = test_features.len();
|
||
let mut returns = Vec::with_capacity(n);
|
||
let mut action_counts = [0_usize; 3]; // [buy, sell, hold]
|
||
|
||
for i in 0..n.saturating_sub(1) {
|
||
let feat = match test_features.get(i) {
|
||
Some(f) => f,
|
||
None => continue,
|
||
};
|
||
|
||
let state: Vec<f32> = feat.iter().map(|&v| v as f32).collect();
|
||
|
||
// Select greedy action
|
||
let action_idx = match dqn.select_action(&state) {
|
||
Ok(fa) => {
|
||
let idx = fa.to_index().min(args.num_actions.saturating_sub(1));
|
||
idx as u8
|
||
}
|
||
Err(e) => {
|
||
warn!(" [DQN] select_action error at step {}: {}", i, e);
|
||
2 // Default to HOLD on error
|
||
}
|
||
};
|
||
|
||
// Map action index to trade direction
|
||
let mapped_action = action_idx.min(2);
|
||
|
||
// Track action diversity
|
||
if let Some(count) = action_counts.get_mut(mapped_action as usize) {
|
||
*count += 1;
|
||
}
|
||
|
||
// Compute percentage return (not absolute dollar change)
|
||
let close_cur = test_bars.get(i).map(|b| b.close).unwrap_or(0.0);
|
||
let close_next = test_bars.get(i + 1).map(|b| b.close).unwrap_or(close_cur);
|
||
let pct_change = if close_cur.abs() > 1e-12 {
|
||
(close_next - close_cur) / close_cur
|
||
} else {
|
||
0.0
|
||
};
|
||
|
||
let ret = match mapped_action {
|
||
0 => pct_change, // BUY
|
||
1 => -pct_change, // SELL
|
||
_ => 0.0, // HOLD
|
||
};
|
||
returns.push(ret);
|
||
}
|
||
|
||
Ok((returns, action_counts))
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// PPO Evaluation
|
||
// ---------------------------------------------------------------------------
|
||
|
||
/// Run PPO inference on test features and return per-bar trade returns and
|
||
/// action counts (buy, sell, hold).
|
||
fn evaluate_ppo_fold(
|
||
fold: usize,
|
||
test_features: &[[f64; 51]],
|
||
test_bars: &[OHLCVBar],
|
||
models_dir: &Path,
|
||
args: &Args,
|
||
) -> Result<(Vec<f64>, [usize; 3])> {
|
||
let actor_path = models_dir.join(format!("ppo_fold{}_actor.safetensors", fold));
|
||
let critic_path = models_dir.join(format!("ppo_fold{}_critic.safetensors", fold));
|
||
|
||
if !actor_path.exists() {
|
||
anyhow::bail!(
|
||
"PPO actor checkpoint not found: {}",
|
||
actor_path.display()
|
||
);
|
||
}
|
||
if !critic_path.exists() {
|
||
anyhow::bail!(
|
||
"PPO critic checkpoint not found: {}",
|
||
critic_path.display()
|
||
);
|
||
}
|
||
|
||
// Create PPO config matching training
|
||
let config = PPOConfig {
|
||
state_dim: args.feature_dim,
|
||
num_actions: args.num_actions,
|
||
policy_hidden_dims: vec![128, 64],
|
||
value_hidden_dims: vec![128, 64],
|
||
policy_learning_rate: 3e-4,
|
||
value_learning_rate: 1e-3,
|
||
clip_epsilon: 0.2,
|
||
value_loss_coeff: 0.5,
|
||
entropy_coeff: 0.01,
|
||
batch_size: 64,
|
||
mini_batch_size: 64,
|
||
num_epochs: 4,
|
||
max_grad_norm: 0.5,
|
||
use_lstm: false,
|
||
..PPOConfig::default()
|
||
};
|
||
|
||
// Load PPO from checkpoint
|
||
let ppo = PPO::load_checkpoint(
|
||
&actor_path.to_string_lossy(),
|
||
&critic_path.to_string_lossy(),
|
||
config,
|
||
candle_core::Device::Cpu,
|
||
)
|
||
.with_context(|| {
|
||
format!(
|
||
"Failed to load PPO checkpoint: actor={}, critic={}",
|
||
actor_path.display(),
|
||
critic_path.display()
|
||
)
|
||
})?;
|
||
|
||
info!(
|
||
" [PPO] Loaded checkpoint: {}",
|
||
actor_path.display()
|
||
);
|
||
|
||
// Run inference
|
||
let n = test_features.len();
|
||
let mut returns = Vec::with_capacity(n);
|
||
let mut action_counts = [0_usize; 3]; // [buy, sell, hold]
|
||
|
||
for i in 0..n.saturating_sub(1) {
|
||
let feat = match test_features.get(i) {
|
||
Some(f) => f,
|
||
None => continue,
|
||
};
|
||
|
||
let state: Vec<f32> = feat.iter().map(|&v| v as f32).collect();
|
||
|
||
// Get action from policy
|
||
let action_idx = match ppo.act(&state) {
|
||
Ok((action, _value)) => action.to_int().min(2),
|
||
Err(e) => {
|
||
warn!(" [PPO] act error at step {}: {}", i, e);
|
||
2 // Default to HOLD on error
|
||
}
|
||
};
|
||
|
||
// Track action diversity
|
||
if let Some(count) = action_counts.get_mut(action_idx as usize) {
|
||
*count += 1;
|
||
}
|
||
|
||
// Compute percentage return (not absolute dollar change)
|
||
let close_cur = test_bars.get(i).map(|b| b.close).unwrap_or(0.0);
|
||
let close_next = test_bars.get(i + 1).map(|b| b.close).unwrap_or(close_cur);
|
||
let pct_change = if close_cur.abs() > 1e-12 {
|
||
(close_next - close_cur) / close_cur
|
||
} else {
|
||
0.0
|
||
};
|
||
|
||
let ret = match action_idx {
|
||
0 => pct_change, // BUY
|
||
1 => -pct_change, // SELL
|
||
_ => 0.0, // HOLD
|
||
};
|
||
returns.push(ret);
|
||
}
|
||
|
||
Ok((returns, action_counts))
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Aggregate & Sanity Checks
|
||
// ---------------------------------------------------------------------------
|
||
|
||
/// Compute average metrics for a specific model across all folds.
|
||
fn compute_aggregate(folds: &[FoldMetrics], model_name: &str) -> (f64, f64, f64) {
|
||
let model_folds: Vec<&FoldMetrics> = folds.iter().filter(|f| f.model == model_name).collect();
|
||
if model_folds.is_empty() {
|
||
return (0.0, 0.0, 0.0);
|
||
}
|
||
let n = model_folds.len() as f64;
|
||
let avg_sharpe = model_folds.iter().map(|f| f.sharpe_ratio).sum::<f64>() / n;
|
||
let avg_dd = model_folds.iter().map(|f| f.max_drawdown_pct).sum::<f64>() / n;
|
||
let avg_wr = model_folds.iter().map(|f| f.win_rate_pct).sum::<f64>() / n;
|
||
(avg_sharpe, avg_dd, avg_wr)
|
||
}
|
||
|
||
/// Run sanity checks across all fold metrics.
|
||
fn run_sanity_checks(
|
||
folds: &[FoldMetrics],
|
||
all_action_counts: &[[usize; 3]],
|
||
) -> SanityChecks {
|
||
// beats_random: any model Sharpe > 0?
|
||
let beats_random = folds.iter().any(|f| f.sharpe_ratio > 0.0);
|
||
|
||
// action_diversity: all 3 actions used across all evaluations?
|
||
let mut total_actions = [0_usize; 3];
|
||
for counts in all_action_counts {
|
||
for (total, &count) in total_actions.iter_mut().zip(counts.iter()) {
|
||
*total += count;
|
||
}
|
||
}
|
||
let action_diversity = total_actions.iter().all(|&c| c > 0);
|
||
|
||
// fold_consistency: std(Sharpe) < 2 * |mean(Sharpe)| across all folds
|
||
let sharpe_values: Vec<f64> = folds.iter().map(|f| f.sharpe_ratio).collect();
|
||
let fold_consistency = if sharpe_values.is_empty() {
|
||
false
|
||
} else {
|
||
let n = sharpe_values.len() as f64;
|
||
let mean_sharpe = sharpe_values.iter().sum::<f64>() / n;
|
||
let var = sharpe_values
|
||
.iter()
|
||
.map(|&s| (s - mean_sharpe).powi(2))
|
||
.sum::<f64>()
|
||
/ n;
|
||
let std_sharpe = var.sqrt();
|
||
std_sharpe < 2.0 * mean_sharpe.abs()
|
||
};
|
||
|
||
SanityChecks {
|
||
beats_random,
|
||
action_diversity,
|
||
fold_consistency,
|
||
}
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Main
|
||
// ---------------------------------------------------------------------------
|
||
|
||
fn main() -> Result<()> {
|
||
// Initialize tracing
|
||
tracing_subscriber::fmt()
|
||
.with_env_filter(
|
||
tracing_subscriber::EnvFilter::try_from_default_env()
|
||
.unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")),
|
||
)
|
||
.init();
|
||
|
||
let args = Args::parse();
|
||
|
||
let eval_dqn = args.model == "dqn" || args.model == "both";
|
||
let eval_ppo = args.model == "ppo" || args.model == "both";
|
||
|
||
info!("=== Walk-Forward Baseline Evaluation ===");
|
||
info!(" Model(s): {}", args.model);
|
||
info!(" Models dir: {}", args.models_dir.display());
|
||
info!(" Data dir: {}", args.data_dir.display());
|
||
info!(" Output: {}", args.output.display());
|
||
info!(" Feature dim: {}", args.feature_dim);
|
||
info!(" Num actions: {}", args.num_actions);
|
||
|
||
// 1. Load all OHLCV bars from DBN files
|
||
info!("Step 1/5: Loading OHLCV bars from DBN files...");
|
||
let bars = load_all_bars(&args.data_dir)?;
|
||
if bars.is_empty() {
|
||
anyhow::bail!("No bars loaded from {}", args.data_dir.display());
|
||
}
|
||
info!(
|
||
" Loaded {} bars ({} to {})",
|
||
bars.len(),
|
||
bars.first().map(|b| b.timestamp.to_string()).unwrap_or_default(),
|
||
bars.last().map(|b| b.timestamp.to_string()).unwrap_or_default(),
|
||
);
|
||
|
||
// 2. Generate walk-forward windows (same config as training)
|
||
info!("Step 2/5: Generating walk-forward windows...");
|
||
let wf_config = WalkForwardConfig {
|
||
initial_train_months: args.train_months,
|
||
val_months: args.val_months,
|
||
test_months: args.test_months,
|
||
step_months: args.step_months,
|
||
};
|
||
let windows = generate_walk_forward_windows(&bars, &wf_config);
|
||
if windows.is_empty() {
|
||
anyhow::bail!(
|
||
"No walk-forward windows generated. Need at least {} months of data.",
|
||
wf_config.initial_train_months + wf_config.val_months + wf_config.test_months
|
||
);
|
||
}
|
||
info!(" Generated {} walk-forward folds", windows.len());
|
||
|
||
// 3. Evaluate each fold
|
||
info!("Step 3/5: Evaluating models on test data...");
|
||
let mut all_fold_metrics: Vec<FoldMetrics> = Vec::new();
|
||
let mut all_action_counts: Vec<[usize; 3]> = Vec::new();
|
||
|
||
for window in &windows {
|
||
info!(
|
||
"--- Fold {} --- Test: {} bars ({} to {})",
|
||
window.fold,
|
||
window.test.len(),
|
||
window.test
|
||
.first()
|
||
.map(|b| b.timestamp.to_string())
|
||
.unwrap_or_default(),
|
||
window.test
|
||
.last()
|
||
.map(|b| b.timestamp.to_string())
|
||
.unwrap_or_default(),
|
||
);
|
||
|
||
// Load NormStats from training
|
||
let norm_path = args
|
||
.models_dir
|
||
.join(format!("norm_stats_fold{}.json", window.fold));
|
||
let norm_stats: NormStats = if norm_path.exists() {
|
||
let norm_json = std::fs::read_to_string(&norm_path)
|
||
.with_context(|| format!("Failed to read {}", norm_path.display()))?;
|
||
serde_json::from_str(&norm_json)
|
||
.with_context(|| format!("Failed to parse {}", norm_path.display()))?
|
||
} else {
|
||
warn!(
|
||
" NormStats not found at {}, computing from test data (degraded)",
|
||
norm_path.display()
|
||
);
|
||
// Fallback: compute from test data (not ideal, but allows evaluation)
|
||
let test_feat = extract_ml_features(&window.test)
|
||
.context("Feature extraction failed for test bars")?;
|
||
NormStats::from_features(&test_feat)
|
||
};
|
||
|
||
// Extract features from test bars
|
||
let test_features = match extract_ml_features(&window.test) {
|
||
Ok(f) => f,
|
||
Err(e) => {
|
||
warn!(
|
||
" Fold {} — test feature extraction failed: {}",
|
||
window.fold, e
|
||
);
|
||
continue;
|
||
}
|
||
};
|
||
|
||
if test_features.is_empty() {
|
||
warn!(" Fold {} — empty test features, skipping", window.fold);
|
||
continue;
|
||
}
|
||
|
||
// Normalize test features
|
||
let test_norm = norm_stats.normalize_batch(&test_features);
|
||
|
||
// Align bars to features (features skip warmup period)
|
||
let warmup_offset = window.test.len().saturating_sub(test_norm.len());
|
||
let test_bars_aligned = if warmup_offset < window.test.len() {
|
||
&window.test[warmup_offset..]
|
||
} else {
|
||
&window.test
|
||
};
|
||
|
||
// Test period date range for the report
|
||
let test_start = test_bars_aligned
|
||
.first()
|
||
.map(|b| b.timestamp.format("%Y-%m-%d").to_string())
|
||
.unwrap_or_default();
|
||
let test_end = test_bars_aligned
|
||
.last()
|
||
.map(|b| b.timestamp.format("%Y-%m-%d").to_string())
|
||
.unwrap_or_default();
|
||
|
||
// Evaluate DQN
|
||
if eval_dqn {
|
||
match evaluate_dqn_fold(
|
||
window.fold,
|
||
&test_norm,
|
||
test_bars_aligned,
|
||
&args.models_dir,
|
||
&args,
|
||
) {
|
||
Ok((returns, action_counts)) => {
|
||
let metrics = compute_metrics(&returns);
|
||
info!(
|
||
" [DQN] Fold {} — Sharpe={:.4} MaxDD={:.2}% WinRate={:.1}% PF={:.2} Return={:.4}% Trades={}",
|
||
window.fold,
|
||
metrics.sharpe_ratio,
|
||
metrics.max_drawdown_pct,
|
||
metrics.win_rate_pct,
|
||
metrics.profit_factor,
|
||
metrics.total_return_pct,
|
||
metrics.num_trades,
|
||
);
|
||
info!(
|
||
" [DQN] Actions — BUY={} SELL={} HOLD={}",
|
||
action_counts.first().copied().unwrap_or(0),
|
||
action_counts.get(1).copied().unwrap_or(0),
|
||
action_counts.get(2).copied().unwrap_or(0),
|
||
);
|
||
all_fold_metrics.push(FoldMetrics {
|
||
fold: window.fold,
|
||
model: "dqn".to_string(),
|
||
sharpe_ratio: metrics.sharpe_ratio,
|
||
max_drawdown_pct: metrics.max_drawdown_pct,
|
||
win_rate_pct: metrics.win_rate_pct,
|
||
profit_factor: metrics.profit_factor,
|
||
total_return_pct: metrics.total_return_pct,
|
||
num_trades: metrics.num_trades,
|
||
test_start: test_start.clone(),
|
||
test_end: test_end.clone(),
|
||
});
|
||
all_action_counts.push(action_counts);
|
||
}
|
||
Err(e) => {
|
||
error!(" [DQN] Fold {} evaluation failed: {}", window.fold, e);
|
||
}
|
||
}
|
||
}
|
||
|
||
// Evaluate PPO
|
||
if eval_ppo {
|
||
match evaluate_ppo_fold(
|
||
window.fold,
|
||
&test_norm,
|
||
test_bars_aligned,
|
||
&args.models_dir,
|
||
&args,
|
||
) {
|
||
Ok((returns, action_counts)) => {
|
||
let metrics = compute_metrics(&returns);
|
||
info!(
|
||
" [PPO] Fold {} — Sharpe={:.4} MaxDD={:.2}% WinRate={:.1}% PF={:.2} Return={:.4}% Trades={}",
|
||
window.fold,
|
||
metrics.sharpe_ratio,
|
||
metrics.max_drawdown_pct,
|
||
metrics.win_rate_pct,
|
||
metrics.profit_factor,
|
||
metrics.total_return_pct,
|
||
metrics.num_trades,
|
||
);
|
||
info!(
|
||
" [PPO] Actions — BUY={} SELL={} HOLD={}",
|
||
action_counts.first().copied().unwrap_or(0),
|
||
action_counts.get(1).copied().unwrap_or(0),
|
||
action_counts.get(2).copied().unwrap_or(0),
|
||
);
|
||
all_fold_metrics.push(FoldMetrics {
|
||
fold: window.fold,
|
||
model: "ppo".to_string(),
|
||
sharpe_ratio: metrics.sharpe_ratio,
|
||
max_drawdown_pct: metrics.max_drawdown_pct,
|
||
win_rate_pct: metrics.win_rate_pct,
|
||
profit_factor: metrics.profit_factor,
|
||
total_return_pct: metrics.total_return_pct,
|
||
num_trades: metrics.num_trades,
|
||
test_start: test_start.clone(),
|
||
test_end: test_end.clone(),
|
||
});
|
||
all_action_counts.push(action_counts);
|
||
}
|
||
Err(e) => {
|
||
error!(" [PPO] Fold {} evaluation failed: {}", window.fold, e);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// 4. Compute aggregate metrics
|
||
info!("Step 4/5: Computing aggregate metrics...");
|
||
let (dqn_avg_sharpe, dqn_avg_drawdown, dqn_avg_win_rate) =
|
||
compute_aggregate(&all_fold_metrics, "dqn");
|
||
let (ppo_avg_sharpe, ppo_avg_drawdown, ppo_avg_win_rate) =
|
||
compute_aggregate(&all_fold_metrics, "ppo");
|
||
|
||
let aggregate = AggregateMetrics {
|
||
dqn_avg_sharpe,
|
||
dqn_avg_drawdown,
|
||
dqn_avg_win_rate,
|
||
ppo_avg_sharpe,
|
||
ppo_avg_drawdown,
|
||
ppo_avg_win_rate,
|
||
};
|
||
|
||
info!(" DQN — avg Sharpe={:.4} avg MaxDD={:.2}% avg WinRate={:.1}%",
|
||
dqn_avg_sharpe, dqn_avg_drawdown, dqn_avg_win_rate);
|
||
info!(" PPO — avg Sharpe={:.4} avg MaxDD={:.2}% avg WinRate={:.1}%",
|
||
ppo_avg_sharpe, ppo_avg_drawdown, ppo_avg_win_rate);
|
||
|
||
// 5. Sanity checks & report
|
||
info!("Step 5/5: Running sanity checks and saving report...");
|
||
let sanity_checks = run_sanity_checks(&all_fold_metrics, &all_action_counts);
|
||
|
||
info!(" Beats random: {}", sanity_checks.beats_random);
|
||
info!(" Action diversity: {}", sanity_checks.action_diversity);
|
||
info!(" Fold consistency: {}", sanity_checks.fold_consistency);
|
||
|
||
let report = EvaluationReport {
|
||
folds: all_fold_metrics,
|
||
aggregate,
|
||
sanity_checks,
|
||
};
|
||
|
||
// Save report
|
||
if let Some(parent) = args.output.parent() {
|
||
std::fs::create_dir_all(parent)
|
||
.with_context(|| format!("Failed to create output dir: {}", parent.display()))?;
|
||
}
|
||
let report_json = serde_json::to_string_pretty(&report)
|
||
.context("Failed to serialize evaluation report")?;
|
||
std::fs::write(&args.output, &report_json)
|
||
.with_context(|| format!("Failed to write report to {}", args.output.display()))?;
|
||
|
||
info!("=== Evaluation Complete ===");
|
||
info!(" Report saved to: {}", args.output.display());
|
||
info!(" Total fold evaluations: {}", report.folds.len());
|
||
|
||
Ok(())
|
||
}
|