3ef80b9efe
- Atomic checkpoint writes: write to .tmp then fs::rename() (POSIX atomic) - Remove redundant thread::sleep(50ms) after CUDA tensor readback sync - NormStats: bail on missing file instead of computing from test data (lookahead bias) - q_value_std: warn when missing from training metrics instead of silent 0.0 fallback Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
958 lines
34 KiB
Rust
958 lines
34 KiB
Rust
//! Walk-forward evaluation binary for DQN and PPO baseline models.
|
||
//!
|
||
//! Loads trained model checkpoints, runs inference on walk-forward test data,
|
||
//! computes financial metrics (Sharpe, drawdown, win rate, profit factor),
|
||
//! and generates a JSON report.
|
||
//!
|
||
//! # Usage
|
||
//!
|
||
//! ```bash
|
||
//! SQLX_OFFLINE=true cargo run -p ml --example evaluate_baseline -- \
|
||
//! --model both --models-dir ml/trained_models \
|
||
//! --data-dir test_data/futures-baseline \
|
||
//! --output ml/trained_models/evaluation_report.json
|
||
//! ```
|
||
|
||
#![allow(unused_crate_dependencies)]
|
||
#![deny(
|
||
clippy::unwrap_used,
|
||
clippy::expect_used,
|
||
clippy::panic,
|
||
clippy::indexing_slicing
|
||
)]
|
||
|
||
use std::path::{Path, PathBuf};
|
||
|
||
use anyhow::{Context, Result};
|
||
use clap::Parser;
|
||
use serde::Serialize;
|
||
use serde_json::Value;
|
||
use tracing::{error, info, warn};
|
||
|
||
use common::metrics::{server as metrics_server, training_metrics as tm};
|
||
use ml::dqn::{DQNConfig, DQN};
|
||
|
||
#[allow(unreachable_pub)]
|
||
mod baseline_common;
|
||
use baseline_common::{load_all_bars, spread_cost_bps};
|
||
use ml::features::extraction::extract_ml_features;
|
||
use ml::ppo::ppo::{PPOConfig, PPO};
|
||
use ml::types::OHLCVBar;
|
||
use ml::walk_forward::{generate_walk_forward_windows, NormStats, WalkForwardConfig};
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// CLI Arguments
|
||
// ---------------------------------------------------------------------------
|
||
|
||
/// Walk-forward evaluation binary for DQN/PPO baseline models.
|
||
#[derive(Parser, Debug)]
|
||
#[command(
|
||
name = "evaluate_baseline",
|
||
about = "Evaluate trained DQN/PPO checkpoints with walk-forward test data"
|
||
)]
|
||
struct Args {
|
||
/// Directory containing trained model checkpoints
|
||
#[arg(long, default_value = "ml/trained_models")]
|
||
models_dir: PathBuf,
|
||
|
||
/// Path to directory containing .dbn.zst files
|
||
#[arg(long, default_value = "test_data/futures-baseline")]
|
||
data_dir: PathBuf,
|
||
|
||
/// Output path for evaluation report JSON
|
||
#[arg(long, default_value = "ml/trained_models/evaluation_report.json")]
|
||
output: PathBuf,
|
||
|
||
/// Which model(s) to evaluate: "dqn", "ppo", or "both"
|
||
#[arg(long, default_value = "both")]
|
||
model: String,
|
||
|
||
/// Feature dimension (must match `extract_ml_features` output)
|
||
#[arg(long, default_value_t = 51)]
|
||
feature_dim: usize,
|
||
|
||
/// Number of actions for the DQN/PPO action space
|
||
#[arg(long, default_value_t = 3)]
|
||
num_actions: usize,
|
||
|
||
/// Walk-forward: initial training window in months
|
||
#[arg(long, default_value_t = 12)]
|
||
train_months: u32,
|
||
|
||
/// Walk-forward: validation window in months
|
||
#[arg(long, default_value_t = 3)]
|
||
val_months: u32,
|
||
|
||
/// Walk-forward: test window in months
|
||
#[arg(long, default_value_t = 3)]
|
||
test_months: u32,
|
||
|
||
/// Walk-forward: step size in months between folds
|
||
#[arg(long, default_value_t = 3)]
|
||
step_months: u32,
|
||
|
||
/// Symbol subdirectory to load (e.g. "ES.FUT", "NQ.FUT")
|
||
#[arg(long, default_value = "ES.FUT")]
|
||
symbol: String,
|
||
|
||
/// Maximum absolute per-bar return; larger moves are clamped (contract roll filter)
|
||
#[arg(long, default_value_t = 0.01)]
|
||
max_bar_return: f64,
|
||
|
||
/// Round-trip commission cost in basis points (1 bps = 0.01%)
|
||
/// Applied to BUY/SELL returns; HOLD is free.
|
||
/// Default 1.0 bps covers ~$4 exchange+broker for ES e-mini.
|
||
#[arg(long, default_value_t = 1.0)]
|
||
tx_cost_bps: f64,
|
||
|
||
/// Instrument tick size in price units (ES=0.25, NQ=0.25, ZN=1/64)
|
||
#[arg(long, default_value_t = 0.25)]
|
||
tick_size: f64,
|
||
|
||
/// Typical bid-ask spread in ticks (ES=1.0, ZN=1.0, 6E=2.0)
|
||
/// Full spread slippage is added to `tx_cost_bps` per round-trip trade.
|
||
#[arg(long, default_value_t = 1.0)]
|
||
spread_ticks: f64,
|
||
|
||
/// Optional path to hyperopt results JSON -- overrides matching config fields
|
||
/// (must match the file used during training so network architecture is identical)
|
||
#[arg(long)]
|
||
hyperopt_params: Option<PathBuf>,
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Hyperopt parameter loading
|
||
// ---------------------------------------------------------------------------
|
||
|
||
/// Load `best_params` from a hyperopt results JSON file.
|
||
///
|
||
/// Expected format: `{ "model_key": { "best_params": { ... }, ... } }`
|
||
/// Returns `None` if the file doesn't exist or can't be parsed.
|
||
#[allow(clippy::cognitive_complexity)]
|
||
fn load_hyperopt_params(hp_path: &Option<PathBuf>, model_key: &str) -> Option<Value> {
|
||
let file_path = hp_path.as_ref()?;
|
||
if !file_path.exists() {
|
||
info!("Hyperopt params file not found: {}, using defaults", file_path.display());
|
||
return None;
|
||
}
|
||
let contents = match std::fs::read_to_string(file_path) {
|
||
Ok(c) => c,
|
||
Err(e) => {
|
||
warn!("Failed to read hyperopt params {}: {}", file_path.display(), e);
|
||
return None;
|
||
}
|
||
};
|
||
let json: Value = match serde_json::from_str(&contents) {
|
||
Ok(v) => v,
|
||
Err(e) => {
|
||
warn!("Failed to parse hyperopt params JSON: {}", e);
|
||
return None;
|
||
}
|
||
};
|
||
let params = json.get(model_key)
|
||
.and_then(|m| m.get("best_params"))
|
||
.cloned();
|
||
if params.is_some() {
|
||
info!("Loaded hyperopt params for '{}' from {}", model_key, file_path.display());
|
||
} else {
|
||
warn!("No best_params found for '{}' in {}", model_key, file_path.display());
|
||
}
|
||
params
|
||
}
|
||
|
||
#[allow(dead_code)]
|
||
fn hp_f64(params: &Option<Value>, key: &str) -> Option<f64> {
|
||
params.as_ref()?.get(key)?.as_f64()
|
||
}
|
||
|
||
fn hp_usize(params: &Option<Value>, key: &str) -> Option<usize> {
|
||
params.as_ref()?.get(key)?.as_u64().map(|v| v as usize)
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Report Data Types
|
||
// ---------------------------------------------------------------------------
|
||
|
||
/// Metrics for a single fold/model combination.
|
||
#[derive(Debug, Serialize)]
|
||
struct FoldMetrics {
|
||
fold: usize,
|
||
model: String,
|
||
sharpe_ratio: f64,
|
||
max_drawdown_pct: f64,
|
||
win_rate_pct: f64,
|
||
profit_factor: f64,
|
||
total_return_pct: f64,
|
||
num_trades: usize,
|
||
test_start: String,
|
||
test_end: String,
|
||
}
|
||
|
||
/// Aggregate metrics across all folds for both models.
|
||
#[derive(Debug, Serialize)]
|
||
struct AggregateMetrics {
|
||
dqn_avg_sharpe: f64,
|
||
dqn_avg_drawdown: f64,
|
||
dqn_avg_win_rate: f64,
|
||
ppo_avg_sharpe: f64,
|
||
ppo_avg_drawdown: f64,
|
||
ppo_avg_win_rate: f64,
|
||
}
|
||
|
||
/// Sanity checks to flag obviously broken models.
|
||
#[derive(Debug, Serialize)]
|
||
struct SanityChecks {
|
||
/// True if any model has Sharpe > 0
|
||
beats_random: bool,
|
||
/// True if all 3 actions (buy/sell/hold) were used
|
||
action_diversity: bool,
|
||
/// True if Sharpe std < 2x |mean Sharpe|
|
||
fold_consistency: bool,
|
||
}
|
||
|
||
/// Full evaluation report written to JSON.
|
||
#[derive(Debug, Serialize)]
|
||
struct EvaluationReport {
|
||
folds: Vec<FoldMetrics>,
|
||
aggregate: AggregateMetrics,
|
||
sanity_checks: SanityChecks,
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Financial Metrics
|
||
// ---------------------------------------------------------------------------
|
||
|
||
/// Container for computed financial metrics from a sequence of trade returns.
|
||
struct ComputedMetrics {
|
||
sharpe_ratio: f64,
|
||
max_drawdown_pct: f64,
|
||
win_rate_pct: f64,
|
||
profit_factor: f64,
|
||
total_return_pct: f64,
|
||
num_trades: usize,
|
||
}
|
||
|
||
/// Compute financial metrics from a sequence of per-bar trade returns.
|
||
///
|
||
/// - Sharpe: annualized (mean / std * sqrt(252))
|
||
/// - Max drawdown: largest peak-to-trough drop on cumulative equity curve (%)
|
||
/// - Win rate: percentage of returns > 0
|
||
/// - Profit factor: `gross_profit` / `gross_loss` (inf if no losses)
|
||
/// - Total return: sum of returns * 100 (as percentage)
|
||
/// - Num trades: count of non-zero returns (BUY or SELL actions)
|
||
fn compute_metrics(returns: &[f64]) -> ComputedMetrics {
|
||
let n = returns.len();
|
||
if n == 0 {
|
||
return ComputedMetrics {
|
||
sharpe_ratio: 0.0,
|
||
max_drawdown_pct: 0.0,
|
||
win_rate_pct: 0.0,
|
||
profit_factor: 0.0,
|
||
total_return_pct: 0.0,
|
||
num_trades: 0,
|
||
};
|
||
}
|
||
|
||
// Count actual trades (non-zero returns, i.e. BUY or SELL actions)
|
||
let num_trades = returns.iter().filter(|&&r| r.abs() > 1e-12).count();
|
||
|
||
// Mean and std of returns
|
||
let sum: f64 = returns.iter().sum();
|
||
let mean = sum / n as f64;
|
||
|
||
let variance: f64 = returns.iter().map(|&r| (r - mean).powi(2)).sum::<f64>() / n as f64;
|
||
let std = variance.sqrt();
|
||
|
||
// Annualized Sharpe ratio (1-minute bars for ES futures: ~23h/day × 60 = 1380 bars/day)
|
||
// CME E-mini futures trade Sun 5pm–Fri 4pm CT with a 1h daily halt.
|
||
let bars_per_year: f64 = 252.0 * 1380.0; // ~347,760 bars/year
|
||
let sharpe_ratio = if std > 1e-12 {
|
||
(mean / std) * bars_per_year.sqrt()
|
||
} else {
|
||
0.0
|
||
};
|
||
|
||
// Max drawdown on cumulative equity curve
|
||
let mut equity = 1.0_f64;
|
||
let mut peak = 1.0_f64;
|
||
let mut max_drawdown = 0.0_f64;
|
||
|
||
for &ret in returns {
|
||
equity += ret;
|
||
if equity > peak {
|
||
peak = equity;
|
||
}
|
||
let drawdown = if peak > 1e-12 {
|
||
(peak - equity) / peak
|
||
} else {
|
||
0.0
|
||
};
|
||
if drawdown > max_drawdown {
|
||
max_drawdown = drawdown;
|
||
}
|
||
}
|
||
let max_drawdown_pct = max_drawdown * 100.0;
|
||
|
||
// Win rate
|
||
let wins = returns.iter().filter(|&&r| r > 0.0).count();
|
||
let win_rate_pct = if num_trades > 0 {
|
||
(wins as f64 / num_trades as f64) * 100.0
|
||
} else {
|
||
0.0
|
||
};
|
||
|
||
// Profit factor
|
||
let gross_profit: f64 = returns.iter().filter(|&&r| r > 0.0).sum();
|
||
let gross_loss: f64 = returns.iter().filter(|&&r| r < 0.0).map(|&r| r.abs()).sum();
|
||
let profit_factor = if gross_loss > 1e-12 {
|
||
gross_profit / gross_loss
|
||
} else if gross_profit > 0.0 {
|
||
f64::INFINITY
|
||
} else {
|
||
0.0
|
||
};
|
||
|
||
// Total return
|
||
let total_return_pct = sum * 100.0;
|
||
|
||
ComputedMetrics {
|
||
sharpe_ratio,
|
||
max_drawdown_pct,
|
||
win_rate_pct,
|
||
profit_factor,
|
||
total_return_pct,
|
||
num_trades,
|
||
}
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// DQN Evaluation
|
||
// ---------------------------------------------------------------------------
|
||
|
||
/// Run DQN inference on test features and return per-bar trade returns and
|
||
/// action counts (buy, sell, hold).
|
||
fn evaluate_dqn_fold(
|
||
fold: usize,
|
||
test_features: &[[f64; 51]],
|
||
test_bars: &[OHLCVBar],
|
||
models_dir: &Path,
|
||
args: &Args,
|
||
hp: &Option<Value>,
|
||
) -> Result<(Vec<f64>, [usize; 3])> {
|
||
let ckpt_path = models_dir.join(format!("dqn_fold{}_best.safetensors", fold));
|
||
if !ckpt_path.exists() {
|
||
anyhow::bail!(
|
||
"DQN checkpoint not found: {}",
|
||
ckpt_path.display()
|
||
);
|
||
}
|
||
|
||
// Create DQN with same config as training (gamma must match train_baseline)
|
||
#[allow(clippy::integer_division)]
|
||
let config = DQNConfig {
|
||
state_dim: args.feature_dim,
|
||
num_actions: args.num_actions,
|
||
hidden_dims: {
|
||
let base = hp_usize(hp, "hidden_dim_base").unwrap_or(128);
|
||
if base > 128 {
|
||
vec![base, base / 2, base / 4] // GPU-scaled: 3-layer network
|
||
} else {
|
||
vec![128, 64] // Default small network
|
||
}
|
||
},
|
||
learning_rate: 1e-4,
|
||
gamma: 0.95, // must match train_baseline (shorter horizon, 20 bars)
|
||
epsilon_start: 0.0, // No exploration during evaluation
|
||
epsilon_end: 0.0,
|
||
epsilon_decay: 1.0,
|
||
replay_buffer_capacity: 100, // Minimal buffer, not used for eval
|
||
batch_size: 64,
|
||
min_replay_size: 64,
|
||
target_update_freq: 500,
|
||
warmup_steps: 0,
|
||
use_double_dqn: true,
|
||
use_huber_loss: true,
|
||
use_per: false,
|
||
use_dueling: false,
|
||
use_distributional: false,
|
||
use_noisy_nets: false,
|
||
use_cql: false,
|
||
use_iqn: false,
|
||
use_cvar_action_selection: false,
|
||
..DQNConfig::default()
|
||
};
|
||
|
||
let mut dqn = DQN::new(config).context("Failed to create DQN model")?;
|
||
|
||
// Set epsilon to zero for pure greedy evaluation
|
||
dqn.set_epsilon(0.0);
|
||
|
||
// Load trained weights
|
||
dqn.load_from_safetensors(&ckpt_path.to_string_lossy())
|
||
.with_context(|| format!("Failed to load DQN checkpoint: {}", ckpt_path.display()))?;
|
||
|
||
info!(
|
||
" [DQN] Loaded checkpoint: {} (eps={})",
|
||
ckpt_path.display(),
|
||
dqn.get_epsilon()
|
||
);
|
||
|
||
// Run inference
|
||
let n = test_features.len();
|
||
let mut returns = Vec::with_capacity(n);
|
||
let mut action_counts = [0_usize; 3]; // [buy, sell, hold]
|
||
|
||
for i in 0..n.saturating_sub(1) {
|
||
let Some(feat) = test_features.get(i) else {
|
||
continue;
|
||
};
|
||
|
||
let state: Vec<f32> = feat.iter().map(|&v| v as f32).collect();
|
||
|
||
// Select greedy action (returns FactoredAction)
|
||
let action = match dqn.select_action(&state) {
|
||
Ok(fa) => fa,
|
||
Err(e) => {
|
||
warn!(" [DQN] select_action error at step {}: {}", i, e);
|
||
// Default to Flat/Market/Normal (Hold equivalent)
|
||
ml::common::action::FactoredAction::new(
|
||
ml::common::action::ExposureLevel::Flat,
|
||
ml::common::action::OrderType::Market,
|
||
ml::common::action::Urgency::Normal,
|
||
)
|
||
}
|
||
};
|
||
|
||
// Track action diversity by legacy category (buy/sell/hold)
|
||
let legacy_idx = if action.is_buy() { 0 } else if action.is_sell() { 1 } else { 2 };
|
||
if let Some(count) = action_counts.get_mut(legacy_idx) {
|
||
*count += 1;
|
||
}
|
||
|
||
// Compute percentage return, clamped to filter contract roll boundaries
|
||
let close_cur = test_bars.get(i).map(|b| b.close).unwrap_or(0.0);
|
||
let close_next = test_bars.get(i + 1).map(|b| b.close).unwrap_or(close_cur);
|
||
let pct_change = if close_cur.abs() > 1e-12 {
|
||
((close_next - close_cur) / close_cur).clamp(-args.max_bar_return, args.max_bar_return)
|
||
} else {
|
||
0.0
|
||
};
|
||
|
||
// Exposure-weighted return minus order-type-specific transaction cost
|
||
let exposure = action.target_exposure() as f64;
|
||
let tx_cost_bps = args.tx_cost_bps + spread_cost_bps(close_cur, args.tick_size, args.spread_ticks);
|
||
let tx_cost = if action.is_hold() { 0.0 } else { tx_cost_bps * 0.0001 };
|
||
let ret = exposure * pct_change - tx_cost;
|
||
returns.push(ret);
|
||
}
|
||
|
||
Ok((returns, action_counts))
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// PPO Evaluation
|
||
// ---------------------------------------------------------------------------
|
||
|
||
/// Run PPO inference on test features and return per-bar trade returns and
|
||
/// action counts (buy, sell, hold).
|
||
fn evaluate_ppo_fold(
|
||
fold: usize,
|
||
test_features: &[[f64; 51]],
|
||
test_bars: &[OHLCVBar],
|
||
models_dir: &Path,
|
||
args: &Args,
|
||
hp: &Option<Value>,
|
||
) -> Result<(Vec<f64>, [usize; 3])> {
|
||
let actor_path = models_dir.join(format!("ppo_fold{}_actor.safetensors", fold));
|
||
let critic_path = models_dir.join(format!("ppo_fold{}_critic.safetensors", fold));
|
||
|
||
if !actor_path.exists() {
|
||
anyhow::bail!(
|
||
"PPO actor checkpoint not found: {}",
|
||
actor_path.display()
|
||
);
|
||
}
|
||
if !critic_path.exists() {
|
||
anyhow::bail!(
|
||
"PPO critic checkpoint not found: {}",
|
||
critic_path.display()
|
||
);
|
||
}
|
||
|
||
// Create PPO config matching training
|
||
#[allow(clippy::integer_division)]
|
||
let config = PPOConfig {
|
||
state_dim: args.feature_dim,
|
||
num_actions: args.num_actions,
|
||
policy_hidden_dims: {
|
||
let base = hp_usize(hp, "hidden_dim_base").unwrap_or(128);
|
||
vec![base, base / 2]
|
||
},
|
||
value_hidden_dims: {
|
||
let base = hp_usize(hp, "hidden_dim_base").unwrap_or(128);
|
||
vec![base * 2, base, base / 2]
|
||
},
|
||
policy_learning_rate: 3e-4,
|
||
value_learning_rate: 1e-3,
|
||
clip_epsilon: 0.2,
|
||
value_loss_coeff: 0.5,
|
||
entropy_coeff: 0.01,
|
||
batch_size: 64,
|
||
mini_batch_size: 64,
|
||
num_epochs: 4,
|
||
max_grad_norm: 0.5,
|
||
use_lstm: false,
|
||
..PPOConfig::default()
|
||
};
|
||
|
||
// Load PPO from checkpoint
|
||
let ppo = PPO::load_checkpoint(
|
||
&actor_path.to_string_lossy(),
|
||
&critic_path.to_string_lossy(),
|
||
config,
|
||
candle_core::Device::Cpu,
|
||
)
|
||
.with_context(|| {
|
||
format!(
|
||
"Failed to load PPO checkpoint: actor={}, critic={}",
|
||
actor_path.display(),
|
||
critic_path.display()
|
||
)
|
||
})?;
|
||
|
||
info!(
|
||
" [PPO] Loaded checkpoint: {}",
|
||
actor_path.display()
|
||
);
|
||
|
||
// Run inference
|
||
let n = test_features.len();
|
||
let mut returns = Vec::with_capacity(n);
|
||
let mut action_counts = [0_usize; 3]; // [buy, sell, hold]
|
||
|
||
for i in 0..n.saturating_sub(1) {
|
||
let Some(feat) = test_features.get(i) else {
|
||
continue;
|
||
};
|
||
|
||
let state: Vec<f32> = feat.iter().map(|&v| v as f32).collect();
|
||
|
||
// Get greedy (deterministic) action from policy for reproducible evaluation
|
||
let action = match ppo.greedy_action(&state) {
|
||
Ok(fa) => fa,
|
||
Err(e) => {
|
||
warn!(" [PPO] greedy_action error at step {}: {}", i, e);
|
||
// Default to Flat/Market/Normal (Hold equivalent)
|
||
ml::common::action::FactoredAction::new(
|
||
ml::common::action::ExposureLevel::Flat,
|
||
ml::common::action::OrderType::Market,
|
||
ml::common::action::Urgency::Normal,
|
||
)
|
||
}
|
||
};
|
||
|
||
// Track action diversity by legacy category (buy/sell/hold)
|
||
let legacy_idx = if action.is_buy() { 0 } else if action.is_sell() { 1 } else { 2 };
|
||
if let Some(count) = action_counts.get_mut(legacy_idx) {
|
||
*count += 1;
|
||
}
|
||
|
||
// Compute percentage return, clamped to filter contract roll boundaries
|
||
let close_cur = test_bars.get(i).map(|b| b.close).unwrap_or(0.0);
|
||
let close_next = test_bars.get(i + 1).map(|b| b.close).unwrap_or(close_cur);
|
||
let pct_change = if close_cur.abs() > 1e-12 {
|
||
((close_next - close_cur) / close_cur).clamp(-args.max_bar_return, args.max_bar_return)
|
||
} else {
|
||
0.0
|
||
};
|
||
|
||
// Exposure-weighted return minus order-type-specific transaction cost
|
||
let exposure = action.target_exposure() as f64;
|
||
let tx_cost_bps = args.tx_cost_bps + spread_cost_bps(close_cur, args.tick_size, args.spread_ticks);
|
||
let tx_cost = if action.is_hold() { 0.0 } else { tx_cost_bps * 0.0001 };
|
||
let ret = exposure * pct_change - tx_cost;
|
||
returns.push(ret);
|
||
}
|
||
|
||
Ok((returns, action_counts))
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Aggregate & Sanity Checks
|
||
// ---------------------------------------------------------------------------
|
||
|
||
/// Compute average metrics for a specific model across all folds.
|
||
fn compute_aggregate(folds: &[FoldMetrics], model_name: &str) -> (f64, f64, f64) {
|
||
let model_folds: Vec<&FoldMetrics> = folds.iter().filter(|f| f.model == model_name).collect();
|
||
if model_folds.is_empty() {
|
||
return (0.0, 0.0, 0.0);
|
||
}
|
||
let n = model_folds.len() as f64;
|
||
let avg_sharpe = model_folds.iter().map(|f| f.sharpe_ratio).sum::<f64>() / n;
|
||
let avg_dd = model_folds.iter().map(|f| f.max_drawdown_pct).sum::<f64>() / n;
|
||
let avg_wr = model_folds.iter().map(|f| f.win_rate_pct).sum::<f64>() / n;
|
||
(avg_sharpe, avg_dd, avg_wr)
|
||
}
|
||
|
||
/// Run sanity checks across all fold metrics.
|
||
fn run_sanity_checks(
|
||
folds: &[FoldMetrics],
|
||
all_action_counts: &[[usize; 3]],
|
||
) -> SanityChecks {
|
||
// beats_random: any model Sharpe > 0?
|
||
let beats_random = folds.iter().any(|f| f.sharpe_ratio > 0.0);
|
||
|
||
// action_diversity: all 3 actions used across all evaluations?
|
||
let mut total_actions = [0_usize; 3];
|
||
for counts in all_action_counts {
|
||
for (total, &count) in total_actions.iter_mut().zip(counts.iter()) {
|
||
*total += count;
|
||
}
|
||
}
|
||
let action_diversity = total_actions.iter().all(|&c| c > 0);
|
||
|
||
// fold_consistency: std(Sharpe) < 2 * |mean(Sharpe)| across all folds
|
||
let sharpe_values: Vec<f64> = folds.iter().map(|f| f.sharpe_ratio).collect();
|
||
let fold_consistency = if sharpe_values.is_empty() {
|
||
false
|
||
} else {
|
||
let n = sharpe_values.len() as f64;
|
||
let mean_sharpe = sharpe_values.iter().sum::<f64>() / n;
|
||
let var = sharpe_values
|
||
.iter()
|
||
.map(|&s| (s - mean_sharpe).powi(2))
|
||
.sum::<f64>()
|
||
/ n;
|
||
let std_sharpe = var.sqrt();
|
||
std_sharpe < 2.0 * mean_sharpe.abs()
|
||
};
|
||
|
||
SanityChecks {
|
||
beats_random,
|
||
action_diversity,
|
||
fold_consistency,
|
||
}
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Main
|
||
// ---------------------------------------------------------------------------
|
||
|
||
#[allow(clippy::cognitive_complexity, clippy::too_many_lines)]
|
||
fn main() -> Result<()> {
|
||
// Initialize tracing with optional OTLP export to Tempo
|
||
let otlp_endpoint = std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT").ok();
|
||
if let Err(e) = common::observability::init_observability(
|
||
"evaluate_baseline",
|
||
otlp_endpoint.as_deref(),
|
||
) {
|
||
eprintln!("Observability init failed (non-fatal): {e}");
|
||
}
|
||
|
||
tm::init();
|
||
metrics_server::start_metrics_server(9094);
|
||
tm::set_active_workers(1.0);
|
||
|
||
let args = Args::parse();
|
||
|
||
let eval_dqn = args.model == "dqn" || args.model == "both";
|
||
let eval_ppo = args.model == "ppo" || args.model == "both";
|
||
|
||
info!("=== Walk-Forward Baseline Evaluation ===");
|
||
info!(" Model(s): {}", args.model);
|
||
info!(" Symbol: {}", args.symbol);
|
||
info!(" Models dir: {}", args.models_dir.display());
|
||
info!(" Data dir: {}", args.data_dir.display());
|
||
info!(" Output: {}", args.output.display());
|
||
info!(" Feature dim: {}", args.feature_dim);
|
||
info!(" Num actions: {}", args.num_actions);
|
||
info!(" Max bar return: {:.2}%", args.max_bar_return * 100.0);
|
||
info!(" Tx cost: {:.1} bps commission + {:.1} tick spread (tick_size={:.4})",
|
||
args.tx_cost_bps, args.spread_ticks, args.tick_size);
|
||
if let Some(ref hp_path) = args.hyperopt_params {
|
||
info!(" Hyperopt params: {}", hp_path.display());
|
||
}
|
||
|
||
// 1. Load all OHLCV bars from DBN files
|
||
info!("Step 1/5: Loading OHLCV bars from DBN files...");
|
||
let data_load_start = std::time::Instant::now();
|
||
let bars = load_all_bars(&args.data_dir, &args.symbol)?;
|
||
let data_load_secs = data_load_start.elapsed().as_secs_f64();
|
||
if eval_dqn {
|
||
tm::record_data_load("dqn", data_load_secs);
|
||
}
|
||
if eval_ppo {
|
||
tm::record_data_load("ppo", data_load_secs);
|
||
}
|
||
if bars.is_empty() {
|
||
anyhow::bail!("No bars loaded from {}", args.data_dir.display());
|
||
}
|
||
info!(
|
||
" Loaded {} bars ({} to {})",
|
||
bars.len(),
|
||
bars.first().map(|b| b.timestamp.to_string()).unwrap_or_default(),
|
||
bars.last().map(|b| b.timestamp.to_string()).unwrap_or_default(),
|
||
);
|
||
|
||
// 2. Generate walk-forward windows (same config as training)
|
||
info!("Step 2/5: Generating walk-forward windows...");
|
||
let wf_config = WalkForwardConfig {
|
||
initial_train_months: args.train_months,
|
||
val_months: args.val_months,
|
||
test_months: args.test_months,
|
||
step_months: args.step_months,
|
||
};
|
||
let windows = generate_walk_forward_windows(&bars, &wf_config);
|
||
if windows.is_empty() {
|
||
anyhow::bail!(
|
||
"No walk-forward windows generated. Need at least {} months of data.",
|
||
wf_config.initial_train_months + wf_config.val_months + wf_config.test_months
|
||
);
|
||
}
|
||
info!(" Generated {} walk-forward folds", windows.len());
|
||
|
||
// 3. Evaluate each fold
|
||
info!("Step 3/5: Evaluating models on test data...");
|
||
let mut all_fold_metrics: Vec<FoldMetrics> = Vec::new();
|
||
let mut all_action_counts: Vec<[usize; 3]> = Vec::new();
|
||
|
||
for window in &windows {
|
||
info!(
|
||
"--- Fold {} --- Test: {} bars ({} to {})",
|
||
window.fold,
|
||
window.test.len(),
|
||
window.test
|
||
.first()
|
||
.map(|b| b.timestamp.to_string())
|
||
.unwrap_or_default(),
|
||
window.test
|
||
.last()
|
||
.map(|b| b.timestamp.to_string())
|
||
.unwrap_or_default(),
|
||
);
|
||
|
||
// Load NormStats from training
|
||
let norm_path = args
|
||
.models_dir
|
||
.join(format!("norm_stats_fold{}.json", window.fold));
|
||
let norm_stats: NormStats = if norm_path.exists() {
|
||
let norm_json = std::fs::read_to_string(&norm_path)
|
||
.with_context(|| format!("Failed to read {}", norm_path.display()))?;
|
||
serde_json::from_str(&norm_json)
|
||
.with_context(|| format!("Failed to parse {}", norm_path.display()))?
|
||
} else {
|
||
anyhow::bail!(
|
||
"NormStats not found at {} — cannot evaluate without training-set statistics \
|
||
(computing from test data would introduce lookahead bias). \
|
||
Run training first to generate this file.",
|
||
norm_path.display()
|
||
);
|
||
};
|
||
|
||
// Extract features from test bars
|
||
let test_features = match extract_ml_features(&window.test) {
|
||
Ok(f) => f,
|
||
Err(e) => {
|
||
warn!(
|
||
" Fold {} - test feature extraction failed: {}",
|
||
window.fold, e
|
||
);
|
||
continue;
|
||
}
|
||
};
|
||
|
||
if test_features.is_empty() {
|
||
warn!(" Fold {} - empty test features, skipping", window.fold);
|
||
continue;
|
||
}
|
||
|
||
// Normalize test features
|
||
let test_norm = norm_stats.normalize_batch(&test_features);
|
||
|
||
// Align bars to features (features skip warmup period)
|
||
let warmup_offset = window.test.len().saturating_sub(test_norm.len());
|
||
let test_bars_aligned = window.test.get(warmup_offset..).unwrap_or(&window.test);
|
||
|
||
// Test period date range for the report
|
||
let test_start = test_bars_aligned
|
||
.first()
|
||
.map(|b| b.timestamp.format("%Y-%m-%d").to_string())
|
||
.unwrap_or_default();
|
||
let test_end = test_bars_aligned
|
||
.last()
|
||
.map(|b| b.timestamp.format("%Y-%m-%d").to_string())
|
||
.unwrap_or_default();
|
||
|
||
// Evaluate DQN
|
||
if eval_dqn {
|
||
let hp = load_hyperopt_params(&args.hyperopt_params, "dqn");
|
||
match evaluate_dqn_fold(
|
||
window.fold,
|
||
&test_norm,
|
||
test_bars_aligned,
|
||
&args.models_dir,
|
||
&args,
|
||
&hp,
|
||
) {
|
||
Ok((returns, action_counts)) => {
|
||
let fold_metrics = compute_metrics(&returns);
|
||
let fold_str = window.fold.to_string();
|
||
tm::set_epoch("dqn", &fold_str, window.fold as f64);
|
||
tm::set_eval_metrics(
|
||
"dqn",
|
||
&fold_str,
|
||
fold_metrics.win_rate_pct / 100.0,
|
||
fold_metrics.sharpe_ratio,
|
||
fold_metrics.profit_factor,
|
||
fold_metrics.total_return_pct / 100.0,
|
||
);
|
||
info!(
|
||
" [DQN] Fold {} - Sharpe={:.4} MaxDD={:.2}% WinRate={:.1}% PF={:.2} Return={:.4}% Trades={}",
|
||
window.fold,
|
||
fold_metrics.sharpe_ratio,
|
||
fold_metrics.max_drawdown_pct,
|
||
fold_metrics.win_rate_pct,
|
||
fold_metrics.profit_factor,
|
||
fold_metrics.total_return_pct,
|
||
fold_metrics.num_trades,
|
||
);
|
||
info!(
|
||
" [DQN] Actions - BUY={} SELL={} HOLD={}",
|
||
action_counts.first().copied().unwrap_or(0),
|
||
action_counts.get(1).copied().unwrap_or(0),
|
||
action_counts.get(2).copied().unwrap_or(0),
|
||
);
|
||
all_fold_metrics.push(FoldMetrics {
|
||
fold: window.fold,
|
||
model: "dqn".to_owned(),
|
||
sharpe_ratio: fold_metrics.sharpe_ratio,
|
||
max_drawdown_pct: fold_metrics.max_drawdown_pct,
|
||
win_rate_pct: fold_metrics.win_rate_pct,
|
||
profit_factor: fold_metrics.profit_factor,
|
||
total_return_pct: fold_metrics.total_return_pct,
|
||
num_trades: fold_metrics.num_trades,
|
||
test_start: test_start.clone(),
|
||
test_end: test_end.clone(),
|
||
});
|
||
all_action_counts.push(action_counts);
|
||
}
|
||
Err(e) => {
|
||
error!(" [DQN] Fold {} evaluation failed: {}", window.fold, e);
|
||
}
|
||
}
|
||
}
|
||
|
||
// Evaluate PPO
|
||
if eval_ppo {
|
||
let hp = load_hyperopt_params(&args.hyperopt_params, "ppo");
|
||
match evaluate_ppo_fold(
|
||
window.fold,
|
||
&test_norm,
|
||
test_bars_aligned,
|
||
&args.models_dir,
|
||
&args,
|
||
&hp,
|
||
) {
|
||
Ok((returns, action_counts)) => {
|
||
let fold_metrics = compute_metrics(&returns);
|
||
let fold_str = window.fold.to_string();
|
||
tm::set_epoch("ppo", &fold_str, window.fold as f64);
|
||
tm::set_eval_metrics(
|
||
"ppo",
|
||
&fold_str,
|
||
fold_metrics.win_rate_pct / 100.0,
|
||
fold_metrics.sharpe_ratio,
|
||
fold_metrics.profit_factor,
|
||
fold_metrics.total_return_pct / 100.0,
|
||
);
|
||
info!(
|
||
" [PPO] Fold {} - Sharpe={:.4} MaxDD={:.2}% WinRate={:.1}% PF={:.2} Return={:.4}% Trades={}",
|
||
window.fold,
|
||
fold_metrics.sharpe_ratio,
|
||
fold_metrics.max_drawdown_pct,
|
||
fold_metrics.win_rate_pct,
|
||
fold_metrics.profit_factor,
|
||
fold_metrics.total_return_pct,
|
||
fold_metrics.num_trades,
|
||
);
|
||
info!(
|
||
" [PPO] Actions - BUY={} SELL={} HOLD={}",
|
||
action_counts.first().copied().unwrap_or(0),
|
||
action_counts.get(1).copied().unwrap_or(0),
|
||
action_counts.get(2).copied().unwrap_or(0),
|
||
);
|
||
all_fold_metrics.push(FoldMetrics {
|
||
fold: window.fold,
|
||
model: "ppo".to_owned(),
|
||
sharpe_ratio: fold_metrics.sharpe_ratio,
|
||
max_drawdown_pct: fold_metrics.max_drawdown_pct,
|
||
win_rate_pct: fold_metrics.win_rate_pct,
|
||
profit_factor: fold_metrics.profit_factor,
|
||
total_return_pct: fold_metrics.total_return_pct,
|
||
num_trades: fold_metrics.num_trades,
|
||
test_start: test_start.clone(),
|
||
test_end: test_end.clone(),
|
||
});
|
||
all_action_counts.push(action_counts);
|
||
}
|
||
Err(e) => {
|
||
error!(" [PPO] Fold {} evaluation failed: {}", window.fold, e);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// 4. Compute aggregate metrics
|
||
info!("Step 4/5: Computing aggregate metrics...");
|
||
let (dqn_avg_sharpe, dqn_avg_drawdown, dqn_avg_win_rate) =
|
||
compute_aggregate(&all_fold_metrics, "dqn");
|
||
let (ppo_avg_sharpe, ppo_avg_drawdown, ppo_avg_win_rate) =
|
||
compute_aggregate(&all_fold_metrics, "ppo");
|
||
|
||
let aggregate = AggregateMetrics {
|
||
dqn_avg_sharpe,
|
||
dqn_avg_drawdown,
|
||
dqn_avg_win_rate,
|
||
ppo_avg_sharpe,
|
||
ppo_avg_drawdown,
|
||
ppo_avg_win_rate,
|
||
};
|
||
|
||
info!(" DQN - avg Sharpe={:.4} avg MaxDD={:.2}% avg WinRate={:.1}%",
|
||
dqn_avg_sharpe, dqn_avg_drawdown, dqn_avg_win_rate);
|
||
info!(" PPO - avg Sharpe={:.4} avg MaxDD={:.2}% avg WinRate={:.1}%",
|
||
ppo_avg_sharpe, ppo_avg_drawdown, ppo_avg_win_rate);
|
||
|
||
// 5. Sanity checks & report
|
||
info!("Step 5/5: Running sanity checks and saving report...");
|
||
let sanity_checks = run_sanity_checks(&all_fold_metrics, &all_action_counts);
|
||
|
||
info!(" Beats random: {}", sanity_checks.beats_random);
|
||
info!(" Action diversity: {}", sanity_checks.action_diversity);
|
||
info!(" Fold consistency: {}", sanity_checks.fold_consistency);
|
||
|
||
let report = EvaluationReport {
|
||
folds: all_fold_metrics,
|
||
aggregate,
|
||
sanity_checks,
|
||
};
|
||
|
||
// Save report
|
||
if let Some(parent) = args.output.parent() {
|
||
std::fs::create_dir_all(parent)
|
||
.with_context(|| format!("Failed to create output dir: {}", parent.display()))?;
|
||
}
|
||
let report_json = serde_json::to_string_pretty(&report)
|
||
.context("Failed to serialize evaluation report")?;
|
||
std::fs::write(&args.output, &report_json)
|
||
.with_context(|| format!("Failed to write report to {}", args.output.display()))?;
|
||
|
||
info!("=== Evaluation Complete ===");
|
||
info!(" Report saved to: {}", args.output.display());
|
||
info!(" Total fold evaluations: {}", report.folds.len());
|
||
|
||
tm::set_active_workers(0.0);
|
||
|
||
Ok(())
|
||
}
|