feat(ml): add walk-forward evaluation binary with financial metrics
Loads trained DQN/PPO checkpoints, runs greedy inference on walk-forward test windows, computes Sharpe ratio, max drawdown, win rate, profit factor, and total return per fold. Outputs a JSON evaluation report with aggregate metrics and sanity checks (beats-random, action diversity, fold consistency). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
894
ml/examples/evaluate_baseline.rs
Normal file
894
ml/examples/evaluate_baseline.rs
Normal file
@@ -0,0 +1,894 @@
|
||||
//! Walk-forward evaluation binary for DQN and PPO baseline models.
|
||||
//!
|
||||
//! Loads trained model checkpoints, runs inference on walk-forward test data,
|
||||
//! computes financial metrics (Sharpe, drawdown, win rate, profit factor),
|
||||
//! and generates a JSON report.
|
||||
//!
|
||||
//! # Usage
|
||||
//!
|
||||
//! ```bash
|
||||
//! SQLX_OFFLINE=true cargo run -p ml --example evaluate_baseline -- \
|
||||
//! --model both --models-dir ml/trained_models \
|
||||
//! --data-dir data/cache/futures-baseline \
|
||||
//! --output ml/trained_models/evaluation_report.json
|
||||
//! ```
|
||||
|
||||
#![allow(unused_crate_dependencies)]
|
||||
#![deny(
|
||||
clippy::unwrap_used,
|
||||
clippy::expect_used,
|
||||
clippy::panic,
|
||||
clippy::indexing_slicing
|
||||
)]
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use chrono::{DateTime, TimeZone, Utc};
|
||||
use clap::Parser;
|
||||
use serde::Serialize;
|
||||
use tracing::{error, info, warn};
|
||||
|
||||
use dbn::decode::DecodeRecord;
|
||||
use ml::dqn::{DQNConfig, DQN};
|
||||
use ml::features::extraction::extract_ml_features;
|
||||
use ml::ppo::ppo::{PPOConfig, PPO};
|
||||
use ml::types::OHLCVBar;
|
||||
use ml::walk_forward::{generate_walk_forward_windows, NormStats, WalkForwardConfig};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// CLI Arguments
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Walk-forward evaluation binary for DQN/PPO baseline models.
|
||||
#[derive(Parser, Debug)]
|
||||
#[command(
|
||||
name = "evaluate_baseline",
|
||||
about = "Evaluate trained DQN/PPO checkpoints with walk-forward test data"
|
||||
)]
|
||||
struct Args {
|
||||
/// Directory containing trained model checkpoints
|
||||
#[arg(long, default_value = "ml/trained_models")]
|
||||
models_dir: PathBuf,
|
||||
|
||||
/// Path to directory containing .dbn.zst files
|
||||
#[arg(long, default_value = "data/cache/futures-baseline")]
|
||||
data_dir: PathBuf,
|
||||
|
||||
/// Output path for evaluation report JSON
|
||||
#[arg(long, default_value = "ml/trained_models/evaluation_report.json")]
|
||||
output: PathBuf,
|
||||
|
||||
/// Which model(s) to evaluate: "dqn", "ppo", or "both"
|
||||
#[arg(long, default_value = "both")]
|
||||
model: String,
|
||||
|
||||
/// Feature dimension (must match extract_ml_features output)
|
||||
#[arg(long, default_value_t = 51)]
|
||||
feature_dim: usize,
|
||||
|
||||
/// Number of actions for the DQN/PPO action space
|
||||
#[arg(long, default_value_t = 3)]
|
||||
num_actions: usize,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Report Data Types
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Metrics for a single fold/model combination.
|
||||
#[derive(Debug, Serialize)]
|
||||
struct FoldMetrics {
|
||||
fold: usize,
|
||||
model: String,
|
||||
sharpe_ratio: f64,
|
||||
max_drawdown_pct: f64,
|
||||
win_rate_pct: f64,
|
||||
profit_factor: f64,
|
||||
total_return_pct: f64,
|
||||
num_trades: usize,
|
||||
test_start: String,
|
||||
test_end: String,
|
||||
}
|
||||
|
||||
/// Aggregate metrics across all folds for both models.
|
||||
#[derive(Debug, Serialize)]
|
||||
struct AggregateMetrics {
|
||||
dqn_avg_sharpe: f64,
|
||||
dqn_avg_drawdown: f64,
|
||||
dqn_avg_win_rate: f64,
|
||||
ppo_avg_sharpe: f64,
|
||||
ppo_avg_drawdown: f64,
|
||||
ppo_avg_win_rate: f64,
|
||||
}
|
||||
|
||||
/// Sanity checks to flag obviously broken models.
|
||||
#[derive(Debug, Serialize)]
|
||||
struct SanityChecks {
|
||||
/// True if any model has Sharpe > 0
|
||||
beats_random: bool,
|
||||
/// True if all 3 actions (buy/sell/hold) were used
|
||||
action_diversity: bool,
|
||||
/// True if Sharpe std < 2x |mean Sharpe|
|
||||
fold_consistency: bool,
|
||||
}
|
||||
|
||||
/// Full evaluation report written to JSON.
|
||||
#[derive(Debug, Serialize)]
|
||||
struct EvaluationReport {
|
||||
folds: Vec<FoldMetrics>,
|
||||
aggregate: AggregateMetrics,
|
||||
sanity_checks: SanityChecks,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// DBN Loading (reused from train_baseline)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Recursively discover .dbn.zst files under `dir`.
|
||||
fn find_dbn_files(dir: &Path) -> Result<Vec<PathBuf>> {
|
||||
let mut files = Vec::new();
|
||||
if !dir.exists() {
|
||||
anyhow::bail!("Data directory does not exist: {}", dir.display());
|
||||
}
|
||||
collect_dbn_files_recursive(dir, &mut files)?;
|
||||
files.sort();
|
||||
Ok(files)
|
||||
}
|
||||
|
||||
/// Recursive helper that walks the directory tree without `walkdir`.
|
||||
fn collect_dbn_files_recursive(dir: &Path, out: &mut Vec<PathBuf>) -> Result<()> {
|
||||
let entries = std::fs::read_dir(dir)
|
||||
.with_context(|| format!("Cannot read directory: {}", dir.display()))?;
|
||||
for entry in entries {
|
||||
let entry = entry.with_context(|| "Failed to read dir entry")?;
|
||||
let path = entry.path();
|
||||
if path.is_dir() {
|
||||
collect_dbn_files_recursive(&path, out)?;
|
||||
} else if path
|
||||
.file_name()
|
||||
.and_then(|n| n.to_str())
|
||||
.map(|n| n.ends_with(".dbn.zst"))
|
||||
.unwrap_or(false)
|
||||
{
|
||||
out.push(path);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Load OHLCV bars from a single .dbn.zst file.
|
||||
fn load_bars_from_dbn(path: &Path) -> Result<Vec<OHLCVBar>> {
|
||||
use dbn::decode::dbn::Decoder;
|
||||
use dbn::OhlcvMsg;
|
||||
|
||||
let file = std::fs::File::open(path)
|
||||
.with_context(|| format!("Cannot open DBN file: {}", path.display()))?;
|
||||
let buf = std::io::BufReader::new(file);
|
||||
let mut decoder = Decoder::new(buf)
|
||||
.with_context(|| format!("Failed to create DBN decoder for {}", path.display()))?;
|
||||
|
||||
let mut bars = Vec::new();
|
||||
|
||||
while let Some(record) = decoder
|
||||
.decode_record::<OhlcvMsg>()
|
||||
.with_context(|| format!("Error decoding records from {}", path.display()))?
|
||||
{
|
||||
let ts_nanos = record.hd.ts_event;
|
||||
let timestamp = nanos_to_datetime(ts_nanos);
|
||||
let price_scale = 1e-9_f64;
|
||||
bars.push(OHLCVBar {
|
||||
timestamp,
|
||||
open: record.open as f64 * price_scale,
|
||||
high: record.high as f64 * price_scale,
|
||||
low: record.low as f64 * price_scale,
|
||||
close: record.close as f64 * price_scale,
|
||||
volume: record.volume as f64,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(bars)
|
||||
}
|
||||
|
||||
/// Convert nanosecond UNIX timestamp to chrono DateTime<Utc>.
|
||||
fn nanos_to_datetime(nanos: u64) -> DateTime<Utc> {
|
||||
let secs = (nanos / 1_000_000_000) as i64;
|
||||
let subsec_nanos = (nanos % 1_000_000_000) as u32;
|
||||
Utc.timestamp_opt(secs, subsec_nanos)
|
||||
.single()
|
||||
.unwrap_or_else(Utc::now)
|
||||
}
|
||||
|
||||
/// Load all OHLCV bars from a directory of .dbn.zst files, sorted chronologically.
|
||||
fn load_all_bars(data_dir: &Path) -> Result<Vec<OHLCVBar>> {
|
||||
let dbn_files = find_dbn_files(data_dir)?;
|
||||
if dbn_files.is_empty() {
|
||||
anyhow::bail!(
|
||||
"No .dbn.zst files found in {}. Run download_baseline first.",
|
||||
data_dir.display()
|
||||
);
|
||||
}
|
||||
info!(
|
||||
"Found {} .dbn.zst files in {}",
|
||||
dbn_files.len(),
|
||||
data_dir.display()
|
||||
);
|
||||
|
||||
let mut all_bars = Vec::new();
|
||||
for path in &dbn_files {
|
||||
match load_bars_from_dbn(path) {
|
||||
Ok(bars) => {
|
||||
info!(" {} -> {} bars", path.display(), bars.len());
|
||||
all_bars.extend(bars);
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(" Skipping {} — {}", path.display(), e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
all_bars.sort_by_key(|b| b.timestamp);
|
||||
info!("Total bars loaded: {}", all_bars.len());
|
||||
Ok(all_bars)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Financial Metrics
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Container for computed financial metrics from a sequence of trade returns.
|
||||
struct ComputedMetrics {
|
||||
sharpe_ratio: f64,
|
||||
max_drawdown_pct: f64,
|
||||
win_rate_pct: f64,
|
||||
profit_factor: f64,
|
||||
total_return_pct: f64,
|
||||
num_trades: usize,
|
||||
}
|
||||
|
||||
/// Compute financial metrics from a sequence of per-bar trade returns.
|
||||
///
|
||||
/// - Sharpe: annualized (mean / std * sqrt(252))
|
||||
/// - Max drawdown: largest peak-to-trough drop on cumulative equity curve (%)
|
||||
/// - Win rate: percentage of returns > 0
|
||||
/// - Profit factor: gross_profit / gross_loss (inf if no losses)
|
||||
/// - Total return: sum of returns * 100 (as percentage)
|
||||
/// - Num trades: count of non-zero returns (BUY or SELL actions)
|
||||
fn compute_metrics(returns: &[f64]) -> ComputedMetrics {
|
||||
let n = returns.len();
|
||||
if n == 0 {
|
||||
return ComputedMetrics {
|
||||
sharpe_ratio: 0.0,
|
||||
max_drawdown_pct: 0.0,
|
||||
win_rate_pct: 0.0,
|
||||
profit_factor: 0.0,
|
||||
total_return_pct: 0.0,
|
||||
num_trades: 0,
|
||||
};
|
||||
}
|
||||
|
||||
// Count actual trades (non-zero returns, i.e. BUY or SELL actions)
|
||||
let num_trades = returns.iter().filter(|&&r| r.abs() > 1e-12).count();
|
||||
|
||||
// Mean and std of returns
|
||||
let sum: f64 = returns.iter().sum();
|
||||
let mean = sum / n as f64;
|
||||
|
||||
let variance: f64 = returns.iter().map(|&r| (r - mean).powi(2)).sum::<f64>() / n as f64;
|
||||
let std = variance.sqrt();
|
||||
|
||||
// Annualized Sharpe ratio
|
||||
let sharpe_ratio = if std > 1e-12 {
|
||||
(mean / std) * 252.0_f64.sqrt()
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
|
||||
// Max drawdown on cumulative equity curve
|
||||
let mut equity = 1.0_f64;
|
||||
let mut peak = 1.0_f64;
|
||||
let mut max_drawdown = 0.0_f64;
|
||||
|
||||
for &ret in returns {
|
||||
equity += ret;
|
||||
if equity > peak {
|
||||
peak = equity;
|
||||
}
|
||||
let drawdown = if peak > 1e-12 {
|
||||
(peak - equity) / peak
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
if drawdown > max_drawdown {
|
||||
max_drawdown = drawdown;
|
||||
}
|
||||
}
|
||||
let max_drawdown_pct = max_drawdown * 100.0;
|
||||
|
||||
// Win rate
|
||||
let wins = returns.iter().filter(|&&r| r > 0.0).count();
|
||||
let win_rate_pct = if num_trades > 0 {
|
||||
(wins as f64 / num_trades as f64) * 100.0
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
|
||||
// Profit factor
|
||||
let gross_profit: f64 = returns.iter().filter(|&&r| r > 0.0).sum();
|
||||
let gross_loss: f64 = returns.iter().filter(|&&r| r < 0.0).map(|&r| r.abs()).sum();
|
||||
let profit_factor = if gross_loss > 1e-12 {
|
||||
gross_profit / gross_loss
|
||||
} else if gross_profit > 0.0 {
|
||||
f64::INFINITY
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
|
||||
// Total return
|
||||
let total_return_pct = sum * 100.0;
|
||||
|
||||
ComputedMetrics {
|
||||
sharpe_ratio,
|
||||
max_drawdown_pct,
|
||||
win_rate_pct,
|
||||
profit_factor,
|
||||
total_return_pct,
|
||||
num_trades,
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// DQN Evaluation
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Run DQN inference on test features and return per-bar trade returns and
|
||||
/// action counts (buy, sell, hold).
|
||||
fn evaluate_dqn_fold(
|
||||
fold: usize,
|
||||
test_features: &[[f64; 51]],
|
||||
test_bars: &[OHLCVBar],
|
||||
models_dir: &Path,
|
||||
args: &Args,
|
||||
) -> Result<(Vec<f64>, [usize; 3])> {
|
||||
let ckpt_path = models_dir.join(format!("dqn_fold{}_best.safetensors", fold));
|
||||
if !ckpt_path.exists() {
|
||||
anyhow::bail!(
|
||||
"DQN checkpoint not found: {}",
|
||||
ckpt_path.display()
|
||||
);
|
||||
}
|
||||
|
||||
// Create DQN with same config as training
|
||||
let config = DQNConfig {
|
||||
state_dim: args.feature_dim,
|
||||
num_actions: args.num_actions,
|
||||
hidden_dims: vec![128, 64],
|
||||
learning_rate: 1e-4,
|
||||
gamma: 0.99,
|
||||
epsilon_start: 0.0, // No exploration during evaluation
|
||||
epsilon_end: 0.0,
|
||||
epsilon_decay: 1.0,
|
||||
replay_buffer_capacity: 100, // Minimal buffer, not used for eval
|
||||
batch_size: 64,
|
||||
min_replay_size: 64,
|
||||
target_update_freq: 500,
|
||||
warmup_steps: 0,
|
||||
use_double_dqn: true,
|
||||
use_huber_loss: true,
|
||||
use_per: false,
|
||||
use_dueling: false,
|
||||
use_distributional: false,
|
||||
use_noisy_nets: false,
|
||||
use_cql: false,
|
||||
use_iqn: false,
|
||||
use_cvar_action_selection: false,
|
||||
..DQNConfig::default()
|
||||
};
|
||||
|
||||
let mut dqn = DQN::new(config).context("Failed to create DQN model")?;
|
||||
|
||||
// Set epsilon to zero for pure greedy evaluation
|
||||
dqn.set_epsilon(0.0);
|
||||
|
||||
// Load trained weights
|
||||
dqn.load_from_safetensors(&ckpt_path.to_string_lossy())
|
||||
.with_context(|| format!("Failed to load DQN checkpoint: {}", ckpt_path.display()))?;
|
||||
|
||||
info!(
|
||||
" [DQN] Loaded checkpoint: {} (eps={})",
|
||||
ckpt_path.display(),
|
||||
dqn.get_epsilon()
|
||||
);
|
||||
|
||||
// Run inference
|
||||
let n = test_features.len();
|
||||
let mut returns = Vec::with_capacity(n);
|
||||
let mut action_counts = [0_usize; 3]; // [buy, sell, hold]
|
||||
|
||||
for i in 0..n.saturating_sub(1) {
|
||||
let feat = match test_features.get(i) {
|
||||
Some(f) => f,
|
||||
None => continue,
|
||||
};
|
||||
|
||||
let state: Vec<f32> = feat.iter().map(|&v| v as f32).collect();
|
||||
|
||||
// Select greedy action
|
||||
let action_idx = match dqn.select_action(&state) {
|
||||
Ok(fa) => {
|
||||
let idx = fa.to_index().min(args.num_actions.saturating_sub(1));
|
||||
idx as u8
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(" [DQN] select_action error at step {}: {}", i, e);
|
||||
2 // Default to HOLD on error
|
||||
}
|
||||
};
|
||||
|
||||
// Map action index to trade direction
|
||||
let mapped_action = action_idx.min(2);
|
||||
|
||||
// Track action diversity
|
||||
if let Some(count) = action_counts.get_mut(mapped_action as usize) {
|
||||
*count += 1;
|
||||
}
|
||||
|
||||
// Compute return
|
||||
let close_cur = test_bars.get(i).map(|b| b.close).unwrap_or(0.0);
|
||||
let close_next = test_bars.get(i + 1).map(|b| b.close).unwrap_or(close_cur);
|
||||
let price_change = close_next - close_cur;
|
||||
|
||||
let ret = match mapped_action {
|
||||
0 => price_change, // BUY
|
||||
1 => -price_change, // SELL
|
||||
_ => 0.0, // HOLD
|
||||
};
|
||||
returns.push(ret);
|
||||
}
|
||||
|
||||
Ok((returns, action_counts))
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// PPO Evaluation
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Run PPO inference on test features and return per-bar trade returns and
|
||||
/// action counts (buy, sell, hold).
|
||||
fn evaluate_ppo_fold(
|
||||
fold: usize,
|
||||
test_features: &[[f64; 51]],
|
||||
test_bars: &[OHLCVBar],
|
||||
models_dir: &Path,
|
||||
args: &Args,
|
||||
) -> Result<(Vec<f64>, [usize; 3])> {
|
||||
let actor_path = models_dir.join(format!("ppo_fold{}_actor.safetensors", fold));
|
||||
let critic_path = models_dir.join(format!("ppo_fold{}_critic.safetensors", fold));
|
||||
|
||||
if !actor_path.exists() {
|
||||
anyhow::bail!(
|
||||
"PPO actor checkpoint not found: {}",
|
||||
actor_path.display()
|
||||
);
|
||||
}
|
||||
if !critic_path.exists() {
|
||||
anyhow::bail!(
|
||||
"PPO critic checkpoint not found: {}",
|
||||
critic_path.display()
|
||||
);
|
||||
}
|
||||
|
||||
// Create PPO config matching training
|
||||
let config = PPOConfig {
|
||||
state_dim: args.feature_dim,
|
||||
num_actions: args.num_actions,
|
||||
policy_hidden_dims: vec![128, 64],
|
||||
value_hidden_dims: vec![128, 64],
|
||||
policy_learning_rate: 3e-4,
|
||||
value_learning_rate: 1e-3,
|
||||
clip_epsilon: 0.2,
|
||||
value_loss_coeff: 0.5,
|
||||
entropy_coeff: 0.01,
|
||||
batch_size: 64,
|
||||
mini_batch_size: 64,
|
||||
num_epochs: 4,
|
||||
max_grad_norm: 0.5,
|
||||
use_lstm: false,
|
||||
..PPOConfig::default()
|
||||
};
|
||||
|
||||
// Load PPO from checkpoint
|
||||
let ppo = PPO::load_checkpoint(
|
||||
&actor_path.to_string_lossy(),
|
||||
&critic_path.to_string_lossy(),
|
||||
config,
|
||||
candle_core::Device::Cpu,
|
||||
)
|
||||
.with_context(|| {
|
||||
format!(
|
||||
"Failed to load PPO checkpoint: actor={}, critic={}",
|
||||
actor_path.display(),
|
||||
critic_path.display()
|
||||
)
|
||||
})?;
|
||||
|
||||
info!(
|
||||
" [PPO] Loaded checkpoint: {}",
|
||||
actor_path.display()
|
||||
);
|
||||
|
||||
// Run inference
|
||||
let n = test_features.len();
|
||||
let mut returns = Vec::with_capacity(n);
|
||||
let mut action_counts = [0_usize; 3]; // [buy, sell, hold]
|
||||
|
||||
for i in 0..n.saturating_sub(1) {
|
||||
let feat = match test_features.get(i) {
|
||||
Some(f) => f,
|
||||
None => continue,
|
||||
};
|
||||
|
||||
let state: Vec<f32> = feat.iter().map(|&v| v as f32).collect();
|
||||
|
||||
// Get action from policy
|
||||
let action_idx = match ppo.act(&state) {
|
||||
Ok((action, _value)) => action.to_int().min(2),
|
||||
Err(e) => {
|
||||
warn!(" [PPO] act error at step {}: {}", i, e);
|
||||
2 // Default to HOLD on error
|
||||
}
|
||||
};
|
||||
|
||||
// Track action diversity
|
||||
if let Some(count) = action_counts.get_mut(action_idx as usize) {
|
||||
*count += 1;
|
||||
}
|
||||
|
||||
// Compute return
|
||||
let close_cur = test_bars.get(i).map(|b| b.close).unwrap_or(0.0);
|
||||
let close_next = test_bars.get(i + 1).map(|b| b.close).unwrap_or(close_cur);
|
||||
let price_change = close_next - close_cur;
|
||||
|
||||
let ret = match action_idx {
|
||||
0 => price_change, // BUY
|
||||
1 => -price_change, // SELL
|
||||
_ => 0.0, // HOLD
|
||||
};
|
||||
returns.push(ret);
|
||||
}
|
||||
|
||||
Ok((returns, action_counts))
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Aggregate & Sanity Checks
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Compute average metrics for a specific model across all folds.
|
||||
fn compute_aggregate(folds: &[FoldMetrics], model_name: &str) -> (f64, f64, f64) {
|
||||
let model_folds: Vec<&FoldMetrics> = folds.iter().filter(|f| f.model == model_name).collect();
|
||||
if model_folds.is_empty() {
|
||||
return (0.0, 0.0, 0.0);
|
||||
}
|
||||
let n = model_folds.len() as f64;
|
||||
let avg_sharpe = model_folds.iter().map(|f| f.sharpe_ratio).sum::<f64>() / n;
|
||||
let avg_dd = model_folds.iter().map(|f| f.max_drawdown_pct).sum::<f64>() / n;
|
||||
let avg_wr = model_folds.iter().map(|f| f.win_rate_pct).sum::<f64>() / n;
|
||||
(avg_sharpe, avg_dd, avg_wr)
|
||||
}
|
||||
|
||||
/// Run sanity checks across all fold metrics.
|
||||
fn run_sanity_checks(
|
||||
folds: &[FoldMetrics],
|
||||
all_action_counts: &[[usize; 3]],
|
||||
) -> SanityChecks {
|
||||
// beats_random: any model Sharpe > 0?
|
||||
let beats_random = folds.iter().any(|f| f.sharpe_ratio > 0.0);
|
||||
|
||||
// action_diversity: all 3 actions used across all evaluations?
|
||||
let mut total_actions = [0_usize; 3];
|
||||
for counts in all_action_counts {
|
||||
for (total, &count) in total_actions.iter_mut().zip(counts.iter()) {
|
||||
*total += count;
|
||||
}
|
||||
}
|
||||
let action_diversity = total_actions.iter().all(|&c| c > 0);
|
||||
|
||||
// fold_consistency: std(Sharpe) < 2 * |mean(Sharpe)| across all folds
|
||||
let sharpe_values: Vec<f64> = folds.iter().map(|f| f.sharpe_ratio).collect();
|
||||
let fold_consistency = if sharpe_values.is_empty() {
|
||||
false
|
||||
} else {
|
||||
let n = sharpe_values.len() as f64;
|
||||
let mean_sharpe = sharpe_values.iter().sum::<f64>() / n;
|
||||
let var = sharpe_values
|
||||
.iter()
|
||||
.map(|&s| (s - mean_sharpe).powi(2))
|
||||
.sum::<f64>()
|
||||
/ n;
|
||||
let std_sharpe = var.sqrt();
|
||||
std_sharpe < 2.0 * mean_sharpe.abs()
|
||||
};
|
||||
|
||||
SanityChecks {
|
||||
beats_random,
|
||||
action_diversity,
|
||||
fold_consistency,
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Main
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn main() -> Result<()> {
|
||||
// Initialize tracing
|
||||
tracing_subscriber::fmt()
|
||||
.with_env_filter(
|
||||
tracing_subscriber::EnvFilter::try_from_default_env()
|
||||
.unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")),
|
||||
)
|
||||
.init();
|
||||
|
||||
let args = Args::parse();
|
||||
|
||||
let eval_dqn = args.model == "dqn" || args.model == "both";
|
||||
let eval_ppo = args.model == "ppo" || args.model == "both";
|
||||
|
||||
info!("=== Walk-Forward Baseline Evaluation ===");
|
||||
info!(" Model(s): {}", args.model);
|
||||
info!(" Models dir: {}", args.models_dir.display());
|
||||
info!(" Data dir: {}", args.data_dir.display());
|
||||
info!(" Output: {}", args.output.display());
|
||||
info!(" Feature dim: {}", args.feature_dim);
|
||||
info!(" Num actions: {}", args.num_actions);
|
||||
|
||||
// 1. Load all OHLCV bars from DBN files
|
||||
info!("Step 1/5: Loading OHLCV bars from DBN files...");
|
||||
let bars = load_all_bars(&args.data_dir)?;
|
||||
if bars.is_empty() {
|
||||
anyhow::bail!("No bars loaded from {}", args.data_dir.display());
|
||||
}
|
||||
info!(
|
||||
" Loaded {} bars ({} to {})",
|
||||
bars.len(),
|
||||
bars.first().map(|b| b.timestamp.to_string()).unwrap_or_default(),
|
||||
bars.last().map(|b| b.timestamp.to_string()).unwrap_or_default(),
|
||||
);
|
||||
|
||||
// 2. Generate walk-forward windows (same config as training)
|
||||
info!("Step 2/5: Generating walk-forward windows...");
|
||||
let wf_config = WalkForwardConfig::default();
|
||||
let windows = generate_walk_forward_windows(&bars, &wf_config);
|
||||
if windows.is_empty() {
|
||||
anyhow::bail!(
|
||||
"No walk-forward windows generated. Need at least {} months of data.",
|
||||
wf_config.initial_train_months + wf_config.val_months + wf_config.test_months
|
||||
);
|
||||
}
|
||||
info!(" Generated {} walk-forward folds", windows.len());
|
||||
|
||||
// 3. Evaluate each fold
|
||||
info!("Step 3/5: Evaluating models on test data...");
|
||||
let mut all_fold_metrics: Vec<FoldMetrics> = Vec::new();
|
||||
let mut all_action_counts: Vec<[usize; 3]> = Vec::new();
|
||||
|
||||
for window in &windows {
|
||||
info!(
|
||||
"--- Fold {} --- Test: {} bars ({} to {})",
|
||||
window.fold,
|
||||
window.test.len(),
|
||||
window.test
|
||||
.first()
|
||||
.map(|b| b.timestamp.to_string())
|
||||
.unwrap_or_default(),
|
||||
window.test
|
||||
.last()
|
||||
.map(|b| b.timestamp.to_string())
|
||||
.unwrap_or_default(),
|
||||
);
|
||||
|
||||
// Load NormStats from training
|
||||
let norm_path = args
|
||||
.models_dir
|
||||
.join(format!("norm_stats_fold{}.json", window.fold));
|
||||
let norm_stats: NormStats = if norm_path.exists() {
|
||||
let norm_json = std::fs::read_to_string(&norm_path)
|
||||
.with_context(|| format!("Failed to read {}", norm_path.display()))?;
|
||||
serde_json::from_str(&norm_json)
|
||||
.with_context(|| format!("Failed to parse {}", norm_path.display()))?
|
||||
} else {
|
||||
warn!(
|
||||
" NormStats not found at {}, computing from test data (degraded)",
|
||||
norm_path.display()
|
||||
);
|
||||
// Fallback: compute from test data (not ideal, but allows evaluation)
|
||||
let test_feat = extract_ml_features(&window.test)
|
||||
.context("Feature extraction failed for test bars")?;
|
||||
NormStats::from_features(&test_feat)
|
||||
};
|
||||
|
||||
// Extract features from test bars
|
||||
let test_features = match extract_ml_features(&window.test) {
|
||||
Ok(f) => f,
|
||||
Err(e) => {
|
||||
warn!(
|
||||
" Fold {} — test feature extraction failed: {}",
|
||||
window.fold, e
|
||||
);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
if test_features.is_empty() {
|
||||
warn!(" Fold {} — empty test features, skipping", window.fold);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Normalize test features
|
||||
let test_norm = norm_stats.normalize_batch(&test_features);
|
||||
|
||||
// Align bars to features (features skip warmup period)
|
||||
let warmup_offset = window.test.len().saturating_sub(test_norm.len());
|
||||
let test_bars_aligned = if warmup_offset < window.test.len() {
|
||||
&window.test[warmup_offset..]
|
||||
} else {
|
||||
&window.test
|
||||
};
|
||||
|
||||
// Test period date range for the report
|
||||
let test_start = test_bars_aligned
|
||||
.first()
|
||||
.map(|b| b.timestamp.format("%Y-%m-%d").to_string())
|
||||
.unwrap_or_default();
|
||||
let test_end = test_bars_aligned
|
||||
.last()
|
||||
.map(|b| b.timestamp.format("%Y-%m-%d").to_string())
|
||||
.unwrap_or_default();
|
||||
|
||||
// Evaluate DQN
|
||||
if eval_dqn {
|
||||
match evaluate_dqn_fold(
|
||||
window.fold,
|
||||
&test_norm,
|
||||
test_bars_aligned,
|
||||
&args.models_dir,
|
||||
&args,
|
||||
) {
|
||||
Ok((returns, action_counts)) => {
|
||||
let metrics = compute_metrics(&returns);
|
||||
info!(
|
||||
" [DQN] Fold {} — Sharpe={:.4} MaxDD={:.2}% WinRate={:.1}% PF={:.2} Return={:.4}% Trades={}",
|
||||
window.fold,
|
||||
metrics.sharpe_ratio,
|
||||
metrics.max_drawdown_pct,
|
||||
metrics.win_rate_pct,
|
||||
metrics.profit_factor,
|
||||
metrics.total_return_pct,
|
||||
metrics.num_trades,
|
||||
);
|
||||
info!(
|
||||
" [DQN] Actions — BUY={} SELL={} HOLD={}",
|
||||
action_counts.first().copied().unwrap_or(0),
|
||||
action_counts.get(1).copied().unwrap_or(0),
|
||||
action_counts.get(2).copied().unwrap_or(0),
|
||||
);
|
||||
all_fold_metrics.push(FoldMetrics {
|
||||
fold: window.fold,
|
||||
model: "dqn".to_string(),
|
||||
sharpe_ratio: metrics.sharpe_ratio,
|
||||
max_drawdown_pct: metrics.max_drawdown_pct,
|
||||
win_rate_pct: metrics.win_rate_pct,
|
||||
profit_factor: metrics.profit_factor,
|
||||
total_return_pct: metrics.total_return_pct,
|
||||
num_trades: metrics.num_trades,
|
||||
test_start: test_start.clone(),
|
||||
test_end: test_end.clone(),
|
||||
});
|
||||
all_action_counts.push(action_counts);
|
||||
}
|
||||
Err(e) => {
|
||||
error!(" [DQN] Fold {} evaluation failed: {}", window.fold, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Evaluate PPO
|
||||
if eval_ppo {
|
||||
match evaluate_ppo_fold(
|
||||
window.fold,
|
||||
&test_norm,
|
||||
test_bars_aligned,
|
||||
&args.models_dir,
|
||||
&args,
|
||||
) {
|
||||
Ok((returns, action_counts)) => {
|
||||
let metrics = compute_metrics(&returns);
|
||||
info!(
|
||||
" [PPO] Fold {} — Sharpe={:.4} MaxDD={:.2}% WinRate={:.1}% PF={:.2} Return={:.4}% Trades={}",
|
||||
window.fold,
|
||||
metrics.sharpe_ratio,
|
||||
metrics.max_drawdown_pct,
|
||||
metrics.win_rate_pct,
|
||||
metrics.profit_factor,
|
||||
metrics.total_return_pct,
|
||||
metrics.num_trades,
|
||||
);
|
||||
info!(
|
||||
" [PPO] Actions — BUY={} SELL={} HOLD={}",
|
||||
action_counts.first().copied().unwrap_or(0),
|
||||
action_counts.get(1).copied().unwrap_or(0),
|
||||
action_counts.get(2).copied().unwrap_or(0),
|
||||
);
|
||||
all_fold_metrics.push(FoldMetrics {
|
||||
fold: window.fold,
|
||||
model: "ppo".to_string(),
|
||||
sharpe_ratio: metrics.sharpe_ratio,
|
||||
max_drawdown_pct: metrics.max_drawdown_pct,
|
||||
win_rate_pct: metrics.win_rate_pct,
|
||||
profit_factor: metrics.profit_factor,
|
||||
total_return_pct: metrics.total_return_pct,
|
||||
num_trades: metrics.num_trades,
|
||||
test_start: test_start.clone(),
|
||||
test_end: test_end.clone(),
|
||||
});
|
||||
all_action_counts.push(action_counts);
|
||||
}
|
||||
Err(e) => {
|
||||
error!(" [PPO] Fold {} evaluation failed: {}", window.fold, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Compute aggregate metrics
|
||||
info!("Step 4/5: Computing aggregate metrics...");
|
||||
let (dqn_avg_sharpe, dqn_avg_drawdown, dqn_avg_win_rate) =
|
||||
compute_aggregate(&all_fold_metrics, "dqn");
|
||||
let (ppo_avg_sharpe, ppo_avg_drawdown, ppo_avg_win_rate) =
|
||||
compute_aggregate(&all_fold_metrics, "ppo");
|
||||
|
||||
let aggregate = AggregateMetrics {
|
||||
dqn_avg_sharpe,
|
||||
dqn_avg_drawdown,
|
||||
dqn_avg_win_rate,
|
||||
ppo_avg_sharpe,
|
||||
ppo_avg_drawdown,
|
||||
ppo_avg_win_rate,
|
||||
};
|
||||
|
||||
info!(" DQN — avg Sharpe={:.4} avg MaxDD={:.2}% avg WinRate={:.1}%",
|
||||
dqn_avg_sharpe, dqn_avg_drawdown, dqn_avg_win_rate);
|
||||
info!(" PPO — avg Sharpe={:.4} avg MaxDD={:.2}% avg WinRate={:.1}%",
|
||||
ppo_avg_sharpe, ppo_avg_drawdown, ppo_avg_win_rate);
|
||||
|
||||
// 5. Sanity checks & report
|
||||
info!("Step 5/5: Running sanity checks and saving report...");
|
||||
let sanity_checks = run_sanity_checks(&all_fold_metrics, &all_action_counts);
|
||||
|
||||
info!(" Beats random: {}", sanity_checks.beats_random);
|
||||
info!(" Action diversity: {}", sanity_checks.action_diversity);
|
||||
info!(" Fold consistency: {}", sanity_checks.fold_consistency);
|
||||
|
||||
let report = EvaluationReport {
|
||||
folds: all_fold_metrics,
|
||||
aggregate,
|
||||
sanity_checks,
|
||||
};
|
||||
|
||||
// Save report
|
||||
if let Some(parent) = args.output.parent() {
|
||||
std::fs::create_dir_all(parent)
|
||||
.with_context(|| format!("Failed to create output dir: {}", parent.display()))?;
|
||||
}
|
||||
let report_json = serde_json::to_string_pretty(&report)
|
||||
.context("Failed to serialize evaluation report")?;
|
||||
std::fs::write(&args.output, &report_json)
|
||||
.with_context(|| format!("Failed to write report to {}", args.output.display()))?;
|
||||
|
||||
info!("=== Evaluation Complete ===");
|
||||
info!(" Report saved to: {}", args.output.display());
|
||||
info!(" Total fold evaluations: {}", report.folds.len());
|
||||
|
||||
Ok(())
|
||||
}
|
||||
Reference in New Issue
Block a user