86fda69bfd
Loads trained DQN/PPO checkpoints, runs greedy inference on walk-forward test windows, computes Sharpe ratio, max drawdown, win rate, profit factor, and total return per fold. Outputs a JSON evaluation report with aggregate metrics and sanity checks (beats-random, action diversity, fold consistency). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
895 lines
30 KiB
Rust
895 lines
30 KiB
Rust
//! Walk-forward evaluation binary for DQN and PPO baseline models.
|
|
//!
|
|
//! Loads trained model checkpoints, runs inference on walk-forward test data,
|
|
//! computes financial metrics (Sharpe, drawdown, win rate, profit factor),
|
|
//! and generates a JSON report.
|
|
//!
|
|
//! # Usage
|
|
//!
|
|
//! ```bash
|
|
//! SQLX_OFFLINE=true cargo run -p ml --example evaluate_baseline -- \
|
|
//! --model both --models-dir ml/trained_models \
|
|
//! --data-dir data/cache/futures-baseline \
|
|
//! --output ml/trained_models/evaluation_report.json
|
|
//! ```
|
|
|
|
#![allow(unused_crate_dependencies)]
|
|
#![deny(
|
|
clippy::unwrap_used,
|
|
clippy::expect_used,
|
|
clippy::panic,
|
|
clippy::indexing_slicing
|
|
)]
|
|
|
|
use std::path::{Path, PathBuf};
|
|
|
|
use anyhow::{Context, Result};
|
|
use chrono::{DateTime, TimeZone, Utc};
|
|
use clap::Parser;
|
|
use serde::Serialize;
|
|
use tracing::{error, info, warn};
|
|
|
|
use dbn::decode::DecodeRecord;
|
|
use ml::dqn::{DQNConfig, DQN};
|
|
use ml::features::extraction::extract_ml_features;
|
|
use ml::ppo::ppo::{PPOConfig, PPO};
|
|
use ml::types::OHLCVBar;
|
|
use ml::walk_forward::{generate_walk_forward_windows, NormStats, WalkForwardConfig};
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// CLI Arguments
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/// Walk-forward evaluation binary for DQN/PPO baseline models.
|
|
#[derive(Parser, Debug)]
|
|
#[command(
|
|
name = "evaluate_baseline",
|
|
about = "Evaluate trained DQN/PPO checkpoints with walk-forward test data"
|
|
)]
|
|
struct Args {
|
|
/// Directory containing trained model checkpoints
|
|
#[arg(long, default_value = "ml/trained_models")]
|
|
models_dir: PathBuf,
|
|
|
|
/// Path to directory containing .dbn.zst files
|
|
#[arg(long, default_value = "data/cache/futures-baseline")]
|
|
data_dir: PathBuf,
|
|
|
|
/// Output path for evaluation report JSON
|
|
#[arg(long, default_value = "ml/trained_models/evaluation_report.json")]
|
|
output: PathBuf,
|
|
|
|
/// Which model(s) to evaluate: "dqn", "ppo", or "both"
|
|
#[arg(long, default_value = "both")]
|
|
model: String,
|
|
|
|
/// Feature dimension (must match extract_ml_features output)
|
|
#[arg(long, default_value_t = 51)]
|
|
feature_dim: usize,
|
|
|
|
/// Number of actions for the DQN/PPO action space
|
|
#[arg(long, default_value_t = 3)]
|
|
num_actions: usize,
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Report Data Types
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/// Metrics for a single fold/model combination.
|
|
#[derive(Debug, Serialize)]
|
|
struct FoldMetrics {
|
|
fold: usize,
|
|
model: String,
|
|
sharpe_ratio: f64,
|
|
max_drawdown_pct: f64,
|
|
win_rate_pct: f64,
|
|
profit_factor: f64,
|
|
total_return_pct: f64,
|
|
num_trades: usize,
|
|
test_start: String,
|
|
test_end: String,
|
|
}
|
|
|
|
/// Aggregate metrics across all folds for both models.
|
|
#[derive(Debug, Serialize)]
|
|
struct AggregateMetrics {
|
|
dqn_avg_sharpe: f64,
|
|
dqn_avg_drawdown: f64,
|
|
dqn_avg_win_rate: f64,
|
|
ppo_avg_sharpe: f64,
|
|
ppo_avg_drawdown: f64,
|
|
ppo_avg_win_rate: f64,
|
|
}
|
|
|
|
/// Sanity checks to flag obviously broken models.
|
|
#[derive(Debug, Serialize)]
|
|
struct SanityChecks {
|
|
/// True if any model has Sharpe > 0
|
|
beats_random: bool,
|
|
/// True if all 3 actions (buy/sell/hold) were used
|
|
action_diversity: bool,
|
|
/// True if Sharpe std < 2x |mean Sharpe|
|
|
fold_consistency: bool,
|
|
}
|
|
|
|
/// Full evaluation report written to JSON.
|
|
#[derive(Debug, Serialize)]
|
|
struct EvaluationReport {
|
|
folds: Vec<FoldMetrics>,
|
|
aggregate: AggregateMetrics,
|
|
sanity_checks: SanityChecks,
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// DBN Loading (reused from train_baseline)
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/// Recursively discover .dbn.zst files under `dir`.
|
|
fn find_dbn_files(dir: &Path) -> Result<Vec<PathBuf>> {
|
|
let mut files = Vec::new();
|
|
if !dir.exists() {
|
|
anyhow::bail!("Data directory does not exist: {}", dir.display());
|
|
}
|
|
collect_dbn_files_recursive(dir, &mut files)?;
|
|
files.sort();
|
|
Ok(files)
|
|
}
|
|
|
|
/// Recursive helper that walks the directory tree without `walkdir`.
|
|
fn collect_dbn_files_recursive(dir: &Path, out: &mut Vec<PathBuf>) -> Result<()> {
|
|
let entries = std::fs::read_dir(dir)
|
|
.with_context(|| format!("Cannot read directory: {}", dir.display()))?;
|
|
for entry in entries {
|
|
let entry = entry.with_context(|| "Failed to read dir entry")?;
|
|
let path = entry.path();
|
|
if path.is_dir() {
|
|
collect_dbn_files_recursive(&path, out)?;
|
|
} else if path
|
|
.file_name()
|
|
.and_then(|n| n.to_str())
|
|
.map(|n| n.ends_with(".dbn.zst"))
|
|
.unwrap_or(false)
|
|
{
|
|
out.push(path);
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
/// Load OHLCV bars from a single .dbn.zst file.
|
|
fn load_bars_from_dbn(path: &Path) -> Result<Vec<OHLCVBar>> {
|
|
use dbn::decode::dbn::Decoder;
|
|
use dbn::OhlcvMsg;
|
|
|
|
let file = std::fs::File::open(path)
|
|
.with_context(|| format!("Cannot open DBN file: {}", path.display()))?;
|
|
let buf = std::io::BufReader::new(file);
|
|
let mut decoder = Decoder::new(buf)
|
|
.with_context(|| format!("Failed to create DBN decoder for {}", path.display()))?;
|
|
|
|
let mut bars = Vec::new();
|
|
|
|
while let Some(record) = decoder
|
|
.decode_record::<OhlcvMsg>()
|
|
.with_context(|| format!("Error decoding records from {}", path.display()))?
|
|
{
|
|
let ts_nanos = record.hd.ts_event;
|
|
let timestamp = nanos_to_datetime(ts_nanos);
|
|
let price_scale = 1e-9_f64;
|
|
bars.push(OHLCVBar {
|
|
timestamp,
|
|
open: record.open as f64 * price_scale,
|
|
high: record.high as f64 * price_scale,
|
|
low: record.low as f64 * price_scale,
|
|
close: record.close as f64 * price_scale,
|
|
volume: record.volume as f64,
|
|
});
|
|
}
|
|
|
|
Ok(bars)
|
|
}
|
|
|
|
/// Convert nanosecond UNIX timestamp to chrono DateTime<Utc>.
|
|
fn nanos_to_datetime(nanos: u64) -> DateTime<Utc> {
|
|
let secs = (nanos / 1_000_000_000) as i64;
|
|
let subsec_nanos = (nanos % 1_000_000_000) as u32;
|
|
Utc.timestamp_opt(secs, subsec_nanos)
|
|
.single()
|
|
.unwrap_or_else(Utc::now)
|
|
}
|
|
|
|
/// Load all OHLCV bars from a directory of .dbn.zst files, sorted chronologically.
|
|
fn load_all_bars(data_dir: &Path) -> Result<Vec<OHLCVBar>> {
|
|
let dbn_files = find_dbn_files(data_dir)?;
|
|
if dbn_files.is_empty() {
|
|
anyhow::bail!(
|
|
"No .dbn.zst files found in {}. Run download_baseline first.",
|
|
data_dir.display()
|
|
);
|
|
}
|
|
info!(
|
|
"Found {} .dbn.zst files in {}",
|
|
dbn_files.len(),
|
|
data_dir.display()
|
|
);
|
|
|
|
let mut all_bars = Vec::new();
|
|
for path in &dbn_files {
|
|
match load_bars_from_dbn(path) {
|
|
Ok(bars) => {
|
|
info!(" {} -> {} bars", path.display(), bars.len());
|
|
all_bars.extend(bars);
|
|
}
|
|
Err(e) => {
|
|
warn!(" Skipping {} — {}", path.display(), e);
|
|
}
|
|
}
|
|
}
|
|
|
|
all_bars.sort_by_key(|b| b.timestamp);
|
|
info!("Total bars loaded: {}", all_bars.len());
|
|
Ok(all_bars)
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Financial Metrics
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/// Container for computed financial metrics from a sequence of trade returns.
|
|
struct ComputedMetrics {
|
|
sharpe_ratio: f64,
|
|
max_drawdown_pct: f64,
|
|
win_rate_pct: f64,
|
|
profit_factor: f64,
|
|
total_return_pct: f64,
|
|
num_trades: usize,
|
|
}
|
|
|
|
/// Compute financial metrics from a sequence of per-bar trade returns.
|
|
///
|
|
/// - Sharpe: annualized (mean / std * sqrt(252))
|
|
/// - Max drawdown: largest peak-to-trough drop on cumulative equity curve (%)
|
|
/// - Win rate: percentage of returns > 0
|
|
/// - Profit factor: gross_profit / gross_loss (inf if no losses)
|
|
/// - Total return: sum of returns * 100 (as percentage)
|
|
/// - Num trades: count of non-zero returns (BUY or SELL actions)
|
|
fn compute_metrics(returns: &[f64]) -> ComputedMetrics {
|
|
let n = returns.len();
|
|
if n == 0 {
|
|
return ComputedMetrics {
|
|
sharpe_ratio: 0.0,
|
|
max_drawdown_pct: 0.0,
|
|
win_rate_pct: 0.0,
|
|
profit_factor: 0.0,
|
|
total_return_pct: 0.0,
|
|
num_trades: 0,
|
|
};
|
|
}
|
|
|
|
// Count actual trades (non-zero returns, i.e. BUY or SELL actions)
|
|
let num_trades = returns.iter().filter(|&&r| r.abs() > 1e-12).count();
|
|
|
|
// Mean and std of returns
|
|
let sum: f64 = returns.iter().sum();
|
|
let mean = sum / n as f64;
|
|
|
|
let variance: f64 = returns.iter().map(|&r| (r - mean).powi(2)).sum::<f64>() / n as f64;
|
|
let std = variance.sqrt();
|
|
|
|
// Annualized Sharpe ratio
|
|
let sharpe_ratio = if std > 1e-12 {
|
|
(mean / std) * 252.0_f64.sqrt()
|
|
} else {
|
|
0.0
|
|
};
|
|
|
|
// Max drawdown on cumulative equity curve
|
|
let mut equity = 1.0_f64;
|
|
let mut peak = 1.0_f64;
|
|
let mut max_drawdown = 0.0_f64;
|
|
|
|
for &ret in returns {
|
|
equity += ret;
|
|
if equity > peak {
|
|
peak = equity;
|
|
}
|
|
let drawdown = if peak > 1e-12 {
|
|
(peak - equity) / peak
|
|
} else {
|
|
0.0
|
|
};
|
|
if drawdown > max_drawdown {
|
|
max_drawdown = drawdown;
|
|
}
|
|
}
|
|
let max_drawdown_pct = max_drawdown * 100.0;
|
|
|
|
// Win rate
|
|
let wins = returns.iter().filter(|&&r| r > 0.0).count();
|
|
let win_rate_pct = if num_trades > 0 {
|
|
(wins as f64 / num_trades as f64) * 100.0
|
|
} else {
|
|
0.0
|
|
};
|
|
|
|
// Profit factor
|
|
let gross_profit: f64 = returns.iter().filter(|&&r| r > 0.0).sum();
|
|
let gross_loss: f64 = returns.iter().filter(|&&r| r < 0.0).map(|&r| r.abs()).sum();
|
|
let profit_factor = if gross_loss > 1e-12 {
|
|
gross_profit / gross_loss
|
|
} else if gross_profit > 0.0 {
|
|
f64::INFINITY
|
|
} else {
|
|
0.0
|
|
};
|
|
|
|
// Total return
|
|
let total_return_pct = sum * 100.0;
|
|
|
|
ComputedMetrics {
|
|
sharpe_ratio,
|
|
max_drawdown_pct,
|
|
win_rate_pct,
|
|
profit_factor,
|
|
total_return_pct,
|
|
num_trades,
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// DQN Evaluation
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/// Run DQN inference on test features and return per-bar trade returns and
|
|
/// action counts (buy, sell, hold).
|
|
fn evaluate_dqn_fold(
|
|
fold: usize,
|
|
test_features: &[[f64; 51]],
|
|
test_bars: &[OHLCVBar],
|
|
models_dir: &Path,
|
|
args: &Args,
|
|
) -> Result<(Vec<f64>, [usize; 3])> {
|
|
let ckpt_path = models_dir.join(format!("dqn_fold{}_best.safetensors", fold));
|
|
if !ckpt_path.exists() {
|
|
anyhow::bail!(
|
|
"DQN checkpoint not found: {}",
|
|
ckpt_path.display()
|
|
);
|
|
}
|
|
|
|
// Create DQN with same config as training
|
|
let config = DQNConfig {
|
|
state_dim: args.feature_dim,
|
|
num_actions: args.num_actions,
|
|
hidden_dims: vec![128, 64],
|
|
learning_rate: 1e-4,
|
|
gamma: 0.99,
|
|
epsilon_start: 0.0, // No exploration during evaluation
|
|
epsilon_end: 0.0,
|
|
epsilon_decay: 1.0,
|
|
replay_buffer_capacity: 100, // Minimal buffer, not used for eval
|
|
batch_size: 64,
|
|
min_replay_size: 64,
|
|
target_update_freq: 500,
|
|
warmup_steps: 0,
|
|
use_double_dqn: true,
|
|
use_huber_loss: true,
|
|
use_per: false,
|
|
use_dueling: false,
|
|
use_distributional: false,
|
|
use_noisy_nets: false,
|
|
use_cql: false,
|
|
use_iqn: false,
|
|
use_cvar_action_selection: false,
|
|
..DQNConfig::default()
|
|
};
|
|
|
|
let mut dqn = DQN::new(config).context("Failed to create DQN model")?;
|
|
|
|
// Set epsilon to zero for pure greedy evaluation
|
|
dqn.set_epsilon(0.0);
|
|
|
|
// Load trained weights
|
|
dqn.load_from_safetensors(&ckpt_path.to_string_lossy())
|
|
.with_context(|| format!("Failed to load DQN checkpoint: {}", ckpt_path.display()))?;
|
|
|
|
info!(
|
|
" [DQN] Loaded checkpoint: {} (eps={})",
|
|
ckpt_path.display(),
|
|
dqn.get_epsilon()
|
|
);
|
|
|
|
// Run inference
|
|
let n = test_features.len();
|
|
let mut returns = Vec::with_capacity(n);
|
|
let mut action_counts = [0_usize; 3]; // [buy, sell, hold]
|
|
|
|
for i in 0..n.saturating_sub(1) {
|
|
let feat = match test_features.get(i) {
|
|
Some(f) => f,
|
|
None => continue,
|
|
};
|
|
|
|
let state: Vec<f32> = feat.iter().map(|&v| v as f32).collect();
|
|
|
|
// Select greedy action
|
|
let action_idx = match dqn.select_action(&state) {
|
|
Ok(fa) => {
|
|
let idx = fa.to_index().min(args.num_actions.saturating_sub(1));
|
|
idx as u8
|
|
}
|
|
Err(e) => {
|
|
warn!(" [DQN] select_action error at step {}: {}", i, e);
|
|
2 // Default to HOLD on error
|
|
}
|
|
};
|
|
|
|
// Map action index to trade direction
|
|
let mapped_action = action_idx.min(2);
|
|
|
|
// Track action diversity
|
|
if let Some(count) = action_counts.get_mut(mapped_action as usize) {
|
|
*count += 1;
|
|
}
|
|
|
|
// Compute return
|
|
let close_cur = test_bars.get(i).map(|b| b.close).unwrap_or(0.0);
|
|
let close_next = test_bars.get(i + 1).map(|b| b.close).unwrap_or(close_cur);
|
|
let price_change = close_next - close_cur;
|
|
|
|
let ret = match mapped_action {
|
|
0 => price_change, // BUY
|
|
1 => -price_change, // SELL
|
|
_ => 0.0, // HOLD
|
|
};
|
|
returns.push(ret);
|
|
}
|
|
|
|
Ok((returns, action_counts))
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// PPO Evaluation
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/// Run PPO inference on test features and return per-bar trade returns and
|
|
/// action counts (buy, sell, hold).
|
|
fn evaluate_ppo_fold(
|
|
fold: usize,
|
|
test_features: &[[f64; 51]],
|
|
test_bars: &[OHLCVBar],
|
|
models_dir: &Path,
|
|
args: &Args,
|
|
) -> Result<(Vec<f64>, [usize; 3])> {
|
|
let actor_path = models_dir.join(format!("ppo_fold{}_actor.safetensors", fold));
|
|
let critic_path = models_dir.join(format!("ppo_fold{}_critic.safetensors", fold));
|
|
|
|
if !actor_path.exists() {
|
|
anyhow::bail!(
|
|
"PPO actor checkpoint not found: {}",
|
|
actor_path.display()
|
|
);
|
|
}
|
|
if !critic_path.exists() {
|
|
anyhow::bail!(
|
|
"PPO critic checkpoint not found: {}",
|
|
critic_path.display()
|
|
);
|
|
}
|
|
|
|
// Create PPO config matching training
|
|
let config = PPOConfig {
|
|
state_dim: args.feature_dim,
|
|
num_actions: args.num_actions,
|
|
policy_hidden_dims: vec![128, 64],
|
|
value_hidden_dims: vec![128, 64],
|
|
policy_learning_rate: 3e-4,
|
|
value_learning_rate: 1e-3,
|
|
clip_epsilon: 0.2,
|
|
value_loss_coeff: 0.5,
|
|
entropy_coeff: 0.01,
|
|
batch_size: 64,
|
|
mini_batch_size: 64,
|
|
num_epochs: 4,
|
|
max_grad_norm: 0.5,
|
|
use_lstm: false,
|
|
..PPOConfig::default()
|
|
};
|
|
|
|
// Load PPO from checkpoint
|
|
let ppo = PPO::load_checkpoint(
|
|
&actor_path.to_string_lossy(),
|
|
&critic_path.to_string_lossy(),
|
|
config,
|
|
candle_core::Device::Cpu,
|
|
)
|
|
.with_context(|| {
|
|
format!(
|
|
"Failed to load PPO checkpoint: actor={}, critic={}",
|
|
actor_path.display(),
|
|
critic_path.display()
|
|
)
|
|
})?;
|
|
|
|
info!(
|
|
" [PPO] Loaded checkpoint: {}",
|
|
actor_path.display()
|
|
);
|
|
|
|
// Run inference
|
|
let n = test_features.len();
|
|
let mut returns = Vec::with_capacity(n);
|
|
let mut action_counts = [0_usize; 3]; // [buy, sell, hold]
|
|
|
|
for i in 0..n.saturating_sub(1) {
|
|
let feat = match test_features.get(i) {
|
|
Some(f) => f,
|
|
None => continue,
|
|
};
|
|
|
|
let state: Vec<f32> = feat.iter().map(|&v| v as f32).collect();
|
|
|
|
// Get action from policy
|
|
let action_idx = match ppo.act(&state) {
|
|
Ok((action, _value)) => action.to_int().min(2),
|
|
Err(e) => {
|
|
warn!(" [PPO] act error at step {}: {}", i, e);
|
|
2 // Default to HOLD on error
|
|
}
|
|
};
|
|
|
|
// Track action diversity
|
|
if let Some(count) = action_counts.get_mut(action_idx as usize) {
|
|
*count += 1;
|
|
}
|
|
|
|
// Compute return
|
|
let close_cur = test_bars.get(i).map(|b| b.close).unwrap_or(0.0);
|
|
let close_next = test_bars.get(i + 1).map(|b| b.close).unwrap_or(close_cur);
|
|
let price_change = close_next - close_cur;
|
|
|
|
let ret = match action_idx {
|
|
0 => price_change, // BUY
|
|
1 => -price_change, // SELL
|
|
_ => 0.0, // HOLD
|
|
};
|
|
returns.push(ret);
|
|
}
|
|
|
|
Ok((returns, action_counts))
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Aggregate & Sanity Checks
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/// Compute average metrics for a specific model across all folds.
|
|
fn compute_aggregate(folds: &[FoldMetrics], model_name: &str) -> (f64, f64, f64) {
|
|
let model_folds: Vec<&FoldMetrics> = folds.iter().filter(|f| f.model == model_name).collect();
|
|
if model_folds.is_empty() {
|
|
return (0.0, 0.0, 0.0);
|
|
}
|
|
let n = model_folds.len() as f64;
|
|
let avg_sharpe = model_folds.iter().map(|f| f.sharpe_ratio).sum::<f64>() / n;
|
|
let avg_dd = model_folds.iter().map(|f| f.max_drawdown_pct).sum::<f64>() / n;
|
|
let avg_wr = model_folds.iter().map(|f| f.win_rate_pct).sum::<f64>() / n;
|
|
(avg_sharpe, avg_dd, avg_wr)
|
|
}
|
|
|
|
/// Run sanity checks across all fold metrics.
|
|
fn run_sanity_checks(
|
|
folds: &[FoldMetrics],
|
|
all_action_counts: &[[usize; 3]],
|
|
) -> SanityChecks {
|
|
// beats_random: any model Sharpe > 0?
|
|
let beats_random = folds.iter().any(|f| f.sharpe_ratio > 0.0);
|
|
|
|
// action_diversity: all 3 actions used across all evaluations?
|
|
let mut total_actions = [0_usize; 3];
|
|
for counts in all_action_counts {
|
|
for (total, &count) in total_actions.iter_mut().zip(counts.iter()) {
|
|
*total += count;
|
|
}
|
|
}
|
|
let action_diversity = total_actions.iter().all(|&c| c > 0);
|
|
|
|
// fold_consistency: std(Sharpe) < 2 * |mean(Sharpe)| across all folds
|
|
let sharpe_values: Vec<f64> = folds.iter().map(|f| f.sharpe_ratio).collect();
|
|
let fold_consistency = if sharpe_values.is_empty() {
|
|
false
|
|
} else {
|
|
let n = sharpe_values.len() as f64;
|
|
let mean_sharpe = sharpe_values.iter().sum::<f64>() / n;
|
|
let var = sharpe_values
|
|
.iter()
|
|
.map(|&s| (s - mean_sharpe).powi(2))
|
|
.sum::<f64>()
|
|
/ n;
|
|
let std_sharpe = var.sqrt();
|
|
std_sharpe < 2.0 * mean_sharpe.abs()
|
|
};
|
|
|
|
SanityChecks {
|
|
beats_random,
|
|
action_diversity,
|
|
fold_consistency,
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Main
|
|
// ---------------------------------------------------------------------------
|
|
|
|
fn main() -> Result<()> {
|
|
// Initialize tracing
|
|
tracing_subscriber::fmt()
|
|
.with_env_filter(
|
|
tracing_subscriber::EnvFilter::try_from_default_env()
|
|
.unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")),
|
|
)
|
|
.init();
|
|
|
|
let args = Args::parse();
|
|
|
|
let eval_dqn = args.model == "dqn" || args.model == "both";
|
|
let eval_ppo = args.model == "ppo" || args.model == "both";
|
|
|
|
info!("=== Walk-Forward Baseline Evaluation ===");
|
|
info!(" Model(s): {}", args.model);
|
|
info!(" Models dir: {}", args.models_dir.display());
|
|
info!(" Data dir: {}", args.data_dir.display());
|
|
info!(" Output: {}", args.output.display());
|
|
info!(" Feature dim: {}", args.feature_dim);
|
|
info!(" Num actions: {}", args.num_actions);
|
|
|
|
// 1. Load all OHLCV bars from DBN files
|
|
info!("Step 1/5: Loading OHLCV bars from DBN files...");
|
|
let bars = load_all_bars(&args.data_dir)?;
|
|
if bars.is_empty() {
|
|
anyhow::bail!("No bars loaded from {}", args.data_dir.display());
|
|
}
|
|
info!(
|
|
" Loaded {} bars ({} to {})",
|
|
bars.len(),
|
|
bars.first().map(|b| b.timestamp.to_string()).unwrap_or_default(),
|
|
bars.last().map(|b| b.timestamp.to_string()).unwrap_or_default(),
|
|
);
|
|
|
|
// 2. Generate walk-forward windows (same config as training)
|
|
info!("Step 2/5: Generating walk-forward windows...");
|
|
let wf_config = WalkForwardConfig::default();
|
|
let windows = generate_walk_forward_windows(&bars, &wf_config);
|
|
if windows.is_empty() {
|
|
anyhow::bail!(
|
|
"No walk-forward windows generated. Need at least {} months of data.",
|
|
wf_config.initial_train_months + wf_config.val_months + wf_config.test_months
|
|
);
|
|
}
|
|
info!(" Generated {} walk-forward folds", windows.len());
|
|
|
|
// 3. Evaluate each fold
|
|
info!("Step 3/5: Evaluating models on test data...");
|
|
let mut all_fold_metrics: Vec<FoldMetrics> = Vec::new();
|
|
let mut all_action_counts: Vec<[usize; 3]> = Vec::new();
|
|
|
|
for window in &windows {
|
|
info!(
|
|
"--- Fold {} --- Test: {} bars ({} to {})",
|
|
window.fold,
|
|
window.test.len(),
|
|
window.test
|
|
.first()
|
|
.map(|b| b.timestamp.to_string())
|
|
.unwrap_or_default(),
|
|
window.test
|
|
.last()
|
|
.map(|b| b.timestamp.to_string())
|
|
.unwrap_or_default(),
|
|
);
|
|
|
|
// Load NormStats from training
|
|
let norm_path = args
|
|
.models_dir
|
|
.join(format!("norm_stats_fold{}.json", window.fold));
|
|
let norm_stats: NormStats = if norm_path.exists() {
|
|
let norm_json = std::fs::read_to_string(&norm_path)
|
|
.with_context(|| format!("Failed to read {}", norm_path.display()))?;
|
|
serde_json::from_str(&norm_json)
|
|
.with_context(|| format!("Failed to parse {}", norm_path.display()))?
|
|
} else {
|
|
warn!(
|
|
" NormStats not found at {}, computing from test data (degraded)",
|
|
norm_path.display()
|
|
);
|
|
// Fallback: compute from test data (not ideal, but allows evaluation)
|
|
let test_feat = extract_ml_features(&window.test)
|
|
.context("Feature extraction failed for test bars")?;
|
|
NormStats::from_features(&test_feat)
|
|
};
|
|
|
|
// Extract features from test bars
|
|
let test_features = match extract_ml_features(&window.test) {
|
|
Ok(f) => f,
|
|
Err(e) => {
|
|
warn!(
|
|
" Fold {} — test feature extraction failed: {}",
|
|
window.fold, e
|
|
);
|
|
continue;
|
|
}
|
|
};
|
|
|
|
if test_features.is_empty() {
|
|
warn!(" Fold {} — empty test features, skipping", window.fold);
|
|
continue;
|
|
}
|
|
|
|
// Normalize test features
|
|
let test_norm = norm_stats.normalize_batch(&test_features);
|
|
|
|
// Align bars to features (features skip warmup period)
|
|
let warmup_offset = window.test.len().saturating_sub(test_norm.len());
|
|
let test_bars_aligned = if warmup_offset < window.test.len() {
|
|
&window.test[warmup_offset..]
|
|
} else {
|
|
&window.test
|
|
};
|
|
|
|
// Test period date range for the report
|
|
let test_start = test_bars_aligned
|
|
.first()
|
|
.map(|b| b.timestamp.format("%Y-%m-%d").to_string())
|
|
.unwrap_or_default();
|
|
let test_end = test_bars_aligned
|
|
.last()
|
|
.map(|b| b.timestamp.format("%Y-%m-%d").to_string())
|
|
.unwrap_or_default();
|
|
|
|
// Evaluate DQN
|
|
if eval_dqn {
|
|
match evaluate_dqn_fold(
|
|
window.fold,
|
|
&test_norm,
|
|
test_bars_aligned,
|
|
&args.models_dir,
|
|
&args,
|
|
) {
|
|
Ok((returns, action_counts)) => {
|
|
let metrics = compute_metrics(&returns);
|
|
info!(
|
|
" [DQN] Fold {} — Sharpe={:.4} MaxDD={:.2}% WinRate={:.1}% PF={:.2} Return={:.4}% Trades={}",
|
|
window.fold,
|
|
metrics.sharpe_ratio,
|
|
metrics.max_drawdown_pct,
|
|
metrics.win_rate_pct,
|
|
metrics.profit_factor,
|
|
metrics.total_return_pct,
|
|
metrics.num_trades,
|
|
);
|
|
info!(
|
|
" [DQN] Actions — BUY={} SELL={} HOLD={}",
|
|
action_counts.first().copied().unwrap_or(0),
|
|
action_counts.get(1).copied().unwrap_or(0),
|
|
action_counts.get(2).copied().unwrap_or(0),
|
|
);
|
|
all_fold_metrics.push(FoldMetrics {
|
|
fold: window.fold,
|
|
model: "dqn".to_string(),
|
|
sharpe_ratio: metrics.sharpe_ratio,
|
|
max_drawdown_pct: metrics.max_drawdown_pct,
|
|
win_rate_pct: metrics.win_rate_pct,
|
|
profit_factor: metrics.profit_factor,
|
|
total_return_pct: metrics.total_return_pct,
|
|
num_trades: metrics.num_trades,
|
|
test_start: test_start.clone(),
|
|
test_end: test_end.clone(),
|
|
});
|
|
all_action_counts.push(action_counts);
|
|
}
|
|
Err(e) => {
|
|
error!(" [DQN] Fold {} evaluation failed: {}", window.fold, e);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Evaluate PPO
|
|
if eval_ppo {
|
|
match evaluate_ppo_fold(
|
|
window.fold,
|
|
&test_norm,
|
|
test_bars_aligned,
|
|
&args.models_dir,
|
|
&args,
|
|
) {
|
|
Ok((returns, action_counts)) => {
|
|
let metrics = compute_metrics(&returns);
|
|
info!(
|
|
" [PPO] Fold {} — Sharpe={:.4} MaxDD={:.2}% WinRate={:.1}% PF={:.2} Return={:.4}% Trades={}",
|
|
window.fold,
|
|
metrics.sharpe_ratio,
|
|
metrics.max_drawdown_pct,
|
|
metrics.win_rate_pct,
|
|
metrics.profit_factor,
|
|
metrics.total_return_pct,
|
|
metrics.num_trades,
|
|
);
|
|
info!(
|
|
" [PPO] Actions — BUY={} SELL={} HOLD={}",
|
|
action_counts.first().copied().unwrap_or(0),
|
|
action_counts.get(1).copied().unwrap_or(0),
|
|
action_counts.get(2).copied().unwrap_or(0),
|
|
);
|
|
all_fold_metrics.push(FoldMetrics {
|
|
fold: window.fold,
|
|
model: "ppo".to_string(),
|
|
sharpe_ratio: metrics.sharpe_ratio,
|
|
max_drawdown_pct: metrics.max_drawdown_pct,
|
|
win_rate_pct: metrics.win_rate_pct,
|
|
profit_factor: metrics.profit_factor,
|
|
total_return_pct: metrics.total_return_pct,
|
|
num_trades: metrics.num_trades,
|
|
test_start: test_start.clone(),
|
|
test_end: test_end.clone(),
|
|
});
|
|
all_action_counts.push(action_counts);
|
|
}
|
|
Err(e) => {
|
|
error!(" [PPO] Fold {} evaluation failed: {}", window.fold, e);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// 4. Compute aggregate metrics
|
|
info!("Step 4/5: Computing aggregate metrics...");
|
|
let (dqn_avg_sharpe, dqn_avg_drawdown, dqn_avg_win_rate) =
|
|
compute_aggregate(&all_fold_metrics, "dqn");
|
|
let (ppo_avg_sharpe, ppo_avg_drawdown, ppo_avg_win_rate) =
|
|
compute_aggregate(&all_fold_metrics, "ppo");
|
|
|
|
let aggregate = AggregateMetrics {
|
|
dqn_avg_sharpe,
|
|
dqn_avg_drawdown,
|
|
dqn_avg_win_rate,
|
|
ppo_avg_sharpe,
|
|
ppo_avg_drawdown,
|
|
ppo_avg_win_rate,
|
|
};
|
|
|
|
info!(" DQN — avg Sharpe={:.4} avg MaxDD={:.2}% avg WinRate={:.1}%",
|
|
dqn_avg_sharpe, dqn_avg_drawdown, dqn_avg_win_rate);
|
|
info!(" PPO — avg Sharpe={:.4} avg MaxDD={:.2}% avg WinRate={:.1}%",
|
|
ppo_avg_sharpe, ppo_avg_drawdown, ppo_avg_win_rate);
|
|
|
|
// 5. Sanity checks & report
|
|
info!("Step 5/5: Running sanity checks and saving report...");
|
|
let sanity_checks = run_sanity_checks(&all_fold_metrics, &all_action_counts);
|
|
|
|
info!(" Beats random: {}", sanity_checks.beats_random);
|
|
info!(" Action diversity: {}", sanity_checks.action_diversity);
|
|
info!(" Fold consistency: {}", sanity_checks.fold_consistency);
|
|
|
|
let report = EvaluationReport {
|
|
folds: all_fold_metrics,
|
|
aggregate,
|
|
sanity_checks,
|
|
};
|
|
|
|
// Save report
|
|
if let Some(parent) = args.output.parent() {
|
|
std::fs::create_dir_all(parent)
|
|
.with_context(|| format!("Failed to create output dir: {}", parent.display()))?;
|
|
}
|
|
let report_json = serde_json::to_string_pretty(&report)
|
|
.context("Failed to serialize evaluation report")?;
|
|
std::fs::write(&args.output, &report_json)
|
|
.with_context(|| format!("Failed to write report to {}", args.output.display()))?;
|
|
|
|
info!("=== Evaluation Complete ===");
|
|
info!(" Report saved to: {}", args.output.display());
|
|
info!(" Total fold evaluations: {}", report.folds.len());
|
|
|
|
Ok(())
|
|
}
|