f39219be9e
Root cause: catastrophic OOS eval (Sharpe -280) traced to 11 issues including hardcoded training dynamics and per-bar CPU inference. Search space (dqn.rs): - Add warmup_ratio [0.0, 0.15] — was hardcoded to 0 - Add lr_decay_type [Constant/Linear/Cosine] — was hardcoded Constant - Add min_epochs_before_stopping [2, 6] — was 1000 (disabled) - Add minimum_profit_factor [1.1, 2.0] — was hardcoded 1.5 - Widen entropy_coefficient [0.01, 0.2] for 45-action factored space GPU-batched eval (evaluate_baseline.rs): - 1024-bar chunked inference for both DQN and PPO - DQN: batch_greedy_actions per chunk (was per-bar select_action) - PPO: action_probabilities + GPU argmax per chunk - ~1000x fewer GPU kernel launches - Add trade_sharpe_ratio for hyperopt-comparable metric Preprocessing (preprocessing.rs): - Add compute_clip_bounds/clip_outliers_with_bounds for leakage-free clipping across train/val/test splits Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1224 lines
48 KiB
Rust
1224 lines
48 KiB
Rust
//! Walk-forward evaluation binary for DQN and PPO baseline models.
|
||
//!
|
||
//! Loads trained model checkpoints, runs inference on walk-forward test data,
|
||
//! computes financial metrics (Sharpe, drawdown, win rate, profit factor),
|
||
//! and generates a JSON report.
|
||
//!
|
||
//! # GPU-Batched Inference
|
||
//!
|
||
//! Both DQN and PPO evaluation use chunked GPU-batched inference (chunk size 1024).
|
||
//! Instead of one GPU forward pass per bar (N kernel launches), bars are grouped into
|
||
//! chunks with shared portfolio state, and each chunk does a single batched GPU forward
|
||
//! pass. This reduces GPU kernel launches by ~1000x vs the old per-bar loop.
|
||
//!
|
||
//! Within a chunk, all bars share the portfolio state (equity, exposure, spread) from
|
||
//! the start of the chunk. Between chunks, the portfolio state is updated based on the
|
||
//! sequential trade simulation results. This matches the hyperopt adapter approach.
|
||
//!
|
||
//! # Usage
|
||
//!
|
||
//! ```bash
|
||
//! SQLX_OFFLINE=true cargo run -p ml --example evaluate_baseline -- \
|
||
//! --model both --models-dir ml/trained_models \
|
||
//! --data-dir test_data/futures-baseline \
|
||
//! --output ml/trained_models/evaluation_report.json
|
||
//! ```
|
||
|
||
#![allow(unused_crate_dependencies)]
|
||
#![deny(
|
||
clippy::unwrap_used,
|
||
clippy::expect_used,
|
||
clippy::panic,
|
||
clippy::indexing_slicing
|
||
)]
|
||
|
||
use std::path::{Path, PathBuf};
|
||
|
||
use anyhow::{Context, Result};
|
||
use clap::Parser;
|
||
use serde::Serialize;
|
||
use serde_json::Value;
|
||
use tracing::{error, info, warn};
|
||
|
||
use common::metrics::{server as metrics_server, training_metrics as tm};
|
||
use ml::common::action::{ExposureLevel, FactoredAction, OrderType as ActionOrderType, Urgency};
|
||
use ml::dqn::{DQNConfig, DQN};
|
||
|
||
#[allow(unreachable_pub)]
|
||
mod baseline_common;
|
||
use baseline_common::{load_all_bars, spread_cost_bps};
|
||
use candle_core::Tensor;
|
||
use ml::features::extraction::extract_ml_features;
|
||
use ml::ppo::ppo::{PPOConfig, PPO};
|
||
use ml::types::OHLCVBar;
|
||
use ml::walk_forward::{generate_walk_forward_windows, NormStats, WalkForwardConfig};
|
||
|
||
/// Number of bars processed per GPU forward pass.
|
||
///
|
||
/// All bars in a chunk share the portfolio state (equity, exposure, spread) from
|
||
/// the start of the chunk. The portfolio is updated sequentially on CPU between
|
||
/// chunks. With 1024 bars, the stale-portfolio approximation is negligible
|
||
/// (~0.7 trading days for ES) while reducing GPU kernel launches by ~1000x.
|
||
const EVAL_CHUNK_SIZE: usize = 1024;
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// CLI Arguments
|
||
// ---------------------------------------------------------------------------
|
||
|
||
/// Walk-forward evaluation binary for DQN/PPO baseline models.
|
||
#[derive(Parser, Debug)]
|
||
#[command(
|
||
name = "evaluate_baseline",
|
||
about = "Evaluate trained DQN/PPO checkpoints with walk-forward test data"
|
||
)]
|
||
struct Args {
|
||
/// Directory containing trained model checkpoints
|
||
#[arg(long, default_value = "ml/trained_models")]
|
||
models_dir: PathBuf,
|
||
|
||
/// Path to directory containing .dbn.zst files
|
||
#[arg(long, default_value = "test_data/futures-baseline")]
|
||
data_dir: PathBuf,
|
||
|
||
/// Output path for evaluation report JSON
|
||
#[arg(long, default_value = "ml/trained_models/evaluation_report.json")]
|
||
output: PathBuf,
|
||
|
||
/// Which model(s) to evaluate: "dqn", "ppo", or "both"
|
||
#[arg(long, default_value = "both")]
|
||
model: String,
|
||
|
||
/// Feature dimension (51 market + 3 portfolio = 54, must match training config)
|
||
#[arg(long, default_value_t = 54)]
|
||
feature_dim: usize,
|
||
|
||
/// Number of actions for the DQN/PPO action space (45 = 5 exposure x 3 order x 3 urgency)
|
||
#[arg(long, default_value_t = 45)]
|
||
num_actions: usize,
|
||
|
||
/// Walk-forward: initial training window in months
|
||
#[arg(long, default_value_t = 12)]
|
||
train_months: u32,
|
||
|
||
/// Walk-forward: validation window in months
|
||
#[arg(long, default_value_t = 3)]
|
||
val_months: u32,
|
||
|
||
/// Walk-forward: test window in months
|
||
#[arg(long, default_value_t = 3)]
|
||
test_months: u32,
|
||
|
||
/// Walk-forward: step size in months between folds
|
||
#[arg(long, default_value_t = 3)]
|
||
step_months: u32,
|
||
|
||
/// Symbol subdirectory to load (e.g. "ES.FUT", "NQ.FUT")
|
||
#[arg(long, default_value = "ES.FUT")]
|
||
symbol: String,
|
||
|
||
/// Maximum absolute per-bar return; larger moves are clamped (contract roll filter)
|
||
#[arg(long, default_value_t = 0.01)]
|
||
max_bar_return: f64,
|
||
|
||
/// Round-trip commission cost in basis points (1 bps = 0.01%)
|
||
/// Applied to BUY/SELL returns; HOLD is free.
|
||
/// Default 1.0 bps covers ~$4 exchange+broker for ES e-mini.
|
||
#[arg(long, default_value_t = 1.0)]
|
||
tx_cost_bps: f64,
|
||
|
||
/// Instrument tick size in price units (ES=0.25, NQ=0.25, ZN=1/64)
|
||
#[arg(long, default_value_t = 0.25)]
|
||
tick_size: f64,
|
||
|
||
/// Typical bid-ask spread in ticks (ES=1.0, ZN=1.0, 6E=2.0)
|
||
/// Full spread slippage is added to `tx_cost_bps` per round-trip trade.
|
||
#[arg(long, default_value_t = 1.0)]
|
||
spread_ticks: f64,
|
||
|
||
/// Bars per year for Sharpe annualization (ES/NQ=347760, 6E=345000, ZN=105840).
|
||
/// Default 347760 = 252 trading days × 1380 bars/day (ES 23h session).
|
||
#[arg(long, default_value_t = 347_760.0)]
|
||
bars_per_year: f64,
|
||
|
||
/// Optional path to hyperopt results JSON -- overrides matching config fields
|
||
/// (must match the file used during training so network architecture is identical)
|
||
#[arg(long)]
|
||
hyperopt_params: Option<PathBuf>,
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Hyperopt parameter loading
|
||
// ---------------------------------------------------------------------------
|
||
|
||
/// Load `best_params` from a hyperopt results JSON file.
|
||
///
|
||
/// Expected format: `{ "model_key": { "best_params": { ... }, ... } }`
|
||
/// Returns `None` if the file doesn't exist or can't be parsed.
|
||
#[allow(clippy::cognitive_complexity)]
|
||
fn load_hyperopt_params(hp_path: &Option<PathBuf>, model_key: &str) -> Option<Value> {
|
||
let file_path = hp_path.as_ref()?;
|
||
if !file_path.exists() {
|
||
info!("Hyperopt params file not found: {}, using defaults", file_path.display());
|
||
return None;
|
||
}
|
||
let contents = match std::fs::read_to_string(file_path) {
|
||
Ok(c) => c,
|
||
Err(e) => {
|
||
warn!("Failed to read hyperopt params {}: {}", file_path.display(), e);
|
||
return None;
|
||
}
|
||
};
|
||
let json: Value = match serde_json::from_str(&contents) {
|
||
Ok(v) => v,
|
||
Err(e) => {
|
||
warn!("Failed to parse hyperopt params JSON: {}", e);
|
||
return None;
|
||
}
|
||
};
|
||
let params = json.get(model_key)
|
||
.and_then(|m| m.get("best_params"))
|
||
.cloned();
|
||
if params.is_some() {
|
||
info!("Loaded hyperopt params for '{}' from {}", model_key, file_path.display());
|
||
} else {
|
||
warn!("No best_params found for '{}' in {}", model_key, file_path.display());
|
||
}
|
||
params
|
||
}
|
||
|
||
#[allow(dead_code)]
|
||
fn hp_f64(params: &Option<Value>, key: &str) -> Option<f64> {
|
||
params.as_ref()?.get(key)?.as_f64()
|
||
}
|
||
|
||
fn hp_usize(params: &Option<Value>, key: &str) -> Option<usize> {
|
||
params.as_ref()?.get(key)?.as_u64().map(|v| v as usize)
|
||
}
|
||
|
||
fn hp_bool(params: &Option<Value>, key: &str) -> Option<bool> {
|
||
params.as_ref()?.get(key)?.as_bool()
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Report Data Types
|
||
// ---------------------------------------------------------------------------
|
||
|
||
/// Metrics for a single fold/model combination.
|
||
#[derive(Debug, Serialize)]
|
||
struct FoldMetrics {
|
||
fold: usize,
|
||
model: String,
|
||
sharpe_ratio: f64,
|
||
/// Trade-level Sharpe (non-zero returns only, sqrt(252) annualization).
|
||
/// Comparable to hyperopt's `PerformanceMetrics::from_trades` Sharpe.
|
||
trade_sharpe_ratio: f64,
|
||
max_drawdown_pct: f64,
|
||
win_rate_pct: f64,
|
||
profit_factor: f64,
|
||
total_return_pct: f64,
|
||
num_trades: usize,
|
||
test_start: String,
|
||
test_end: String,
|
||
}
|
||
|
||
/// Aggregate metrics across all folds for both models.
|
||
#[derive(Debug, Serialize)]
|
||
struct AggregateMetrics {
|
||
dqn_avg_sharpe: f64,
|
||
dqn_avg_trade_sharpe: f64,
|
||
dqn_avg_drawdown: f64,
|
||
dqn_avg_win_rate: f64,
|
||
ppo_avg_sharpe: f64,
|
||
ppo_avg_trade_sharpe: f64,
|
||
ppo_avg_drawdown: f64,
|
||
ppo_avg_win_rate: f64,
|
||
}
|
||
|
||
/// Sanity checks to flag obviously broken models.
|
||
#[derive(Debug, Serialize)]
|
||
struct SanityChecks {
|
||
/// True if any model has Sharpe > 0
|
||
beats_random: bool,
|
||
/// True if all 3 actions (buy/sell/hold) were used
|
||
action_diversity: bool,
|
||
/// True if Sharpe std < 2x |mean Sharpe|
|
||
fold_consistency: bool,
|
||
}
|
||
|
||
/// Full evaluation report written to JSON.
|
||
#[derive(Debug, Serialize)]
|
||
struct EvaluationReport {
|
||
folds: Vec<FoldMetrics>,
|
||
aggregate: AggregateMetrics,
|
||
sanity_checks: SanityChecks,
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Financial Metrics
|
||
// ---------------------------------------------------------------------------
|
||
|
||
/// Container for computed financial metrics from a sequence of trade returns.
|
||
struct ComputedMetrics {
|
||
sharpe_ratio: f64,
|
||
/// Trade-level Sharpe ratio (matches hyperopt's `PerformanceMetrics::from_trades`).
|
||
/// Computed from non-zero returns only, annualized with `sqrt(252)`.
|
||
trade_sharpe_ratio: f64,
|
||
max_drawdown_pct: f64,
|
||
win_rate_pct: f64,
|
||
profit_factor: f64,
|
||
total_return_pct: f64,
|
||
num_trades: usize,
|
||
}
|
||
|
||
/// Compute financial metrics from a sequence of per-bar trade returns.
|
||
///
|
||
/// - Sharpe (bar): annualized (mean / std * `sqrt(bars_per_year)`), includes HOLD bars
|
||
/// - Sharpe (trade): annualized (mean / std * `sqrt(252)`), non-zero returns only
|
||
/// - Max drawdown: largest peak-to-trough drop on cumulative equity curve (%)
|
||
/// - Win rate: percentage of returns > 0
|
||
/// - Profit factor: `gross_profit` / `gross_loss` (inf if no losses)
|
||
/// - Total return: sum of returns * 100 (as percentage)
|
||
/// - Num trades: count of non-zero returns (BUY or SELL actions)
|
||
fn compute_metrics(returns: &[f64], bars_per_year: f64) -> ComputedMetrics {
|
||
let n = returns.len();
|
||
if n == 0 {
|
||
return ComputedMetrics {
|
||
sharpe_ratio: 0.0,
|
||
trade_sharpe_ratio: 0.0,
|
||
max_drawdown_pct: 0.0,
|
||
win_rate_pct: 0.0,
|
||
profit_factor: 0.0,
|
||
total_return_pct: 0.0,
|
||
num_trades: 0,
|
||
};
|
||
}
|
||
|
||
// Count actual trades (non-zero returns, i.e. BUY or SELL actions)
|
||
let num_trades = returns.iter().filter(|&&r| r.abs() > 1e-12).count();
|
||
|
||
// Per-bar Sharpe: mean and std over ALL bars (including HOLD=0), annualized
|
||
// with sqrt(bars_per_year). This produces larger absolute values because the
|
||
// annualization factor (~590) is much larger than sqrt(252) (~15.9).
|
||
let sum: f64 = returns.iter().sum();
|
||
let mean = sum / n as f64;
|
||
|
||
let variance: f64 = returns.iter().map(|&r| (r - mean).powi(2)).sum::<f64>() / n as f64;
|
||
let std = variance.sqrt();
|
||
let sharpe_ratio = if std > 1e-12 {
|
||
(mean / std) * bars_per_year.sqrt()
|
||
} else {
|
||
0.0
|
||
};
|
||
|
||
// Trade-level Sharpe (matches hyperopt's PerformanceMetrics::from_trades /
|
||
// calculate_sharpe_ratio in evaluation/metrics.rs):
|
||
// Only considers non-zero returns (actual trades), annualized with sqrt(252).
|
||
let trade_returns: Vec<f64> = returns.iter().copied().filter(|&r| r.abs() > 1e-12).collect();
|
||
let trade_sharpe_ratio = if trade_returns.len() > 1 {
|
||
let trade_mean: f64 = trade_returns.iter().sum::<f64>() / trade_returns.len() as f64;
|
||
let trade_var: f64 = trade_returns
|
||
.iter()
|
||
.map(|&r| (r - trade_mean).powi(2))
|
||
.sum::<f64>()
|
||
/ trade_returns.len() as f64;
|
||
let trade_std = trade_var.sqrt();
|
||
if trade_std > 1e-12 {
|
||
(trade_mean / trade_std) * 252.0_f64.sqrt()
|
||
} else {
|
||
0.0
|
||
}
|
||
} else {
|
||
0.0
|
||
};
|
||
|
||
// Max drawdown on cumulative equity curve
|
||
let mut equity = 1.0_f64;
|
||
let mut peak = 1.0_f64;
|
||
let mut max_drawdown = 0.0_f64;
|
||
|
||
for &ret in returns {
|
||
equity *= 1.0 + ret;
|
||
if equity > peak {
|
||
peak = equity;
|
||
}
|
||
let drawdown = if peak > 1e-12 {
|
||
(peak - equity) / peak
|
||
} else {
|
||
0.0
|
||
};
|
||
if drawdown > max_drawdown {
|
||
max_drawdown = drawdown;
|
||
}
|
||
}
|
||
let max_drawdown_pct = max_drawdown * 100.0;
|
||
|
||
// Win rate
|
||
let wins = returns.iter().filter(|&&r| r > 0.0).count();
|
||
let win_rate_pct = if num_trades > 0 {
|
||
(wins as f64 / num_trades as f64) * 100.0
|
||
} else {
|
||
0.0
|
||
};
|
||
|
||
// Profit factor
|
||
let gross_profit: f64 = returns.iter().filter(|&&r| r > 0.0).sum();
|
||
let gross_loss: f64 = returns.iter().filter(|&&r| r < 0.0).map(|&r| r.abs()).sum();
|
||
let profit_factor = if gross_loss > 1e-12 {
|
||
gross_profit / gross_loss
|
||
} else if gross_profit > 0.0 {
|
||
f64::INFINITY
|
||
} else {
|
||
0.0
|
||
};
|
||
|
||
// Total return (from compounded equity curve)
|
||
let total_return_pct = (equity - 1.0) * 100.0;
|
||
|
||
ComputedMetrics {
|
||
sharpe_ratio,
|
||
trade_sharpe_ratio,
|
||
max_drawdown_pct,
|
||
win_rate_pct,
|
||
profit_factor,
|
||
total_return_pct,
|
||
num_trades,
|
||
}
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// GPU-Batched Inference Helpers
|
||
// ---------------------------------------------------------------------------
|
||
|
||
/// Mutable portfolio state carried across chunks during evaluation.
|
||
struct PortfolioState {
|
||
equity: f64,
|
||
current_exposure: f64,
|
||
}
|
||
|
||
/// Build flat f32 state vectors for a chunk of bars.
|
||
///
|
||
/// Portfolio features (equity, exposure, spread) are frozen at the values from
|
||
/// the START of the chunk. Market features are per-bar. Returns a contiguous
|
||
/// `Vec<f32>` of length `chunk_len * feature_dim`.
|
||
fn build_chunk_states(
|
||
test_features: &[[f64; 51]],
|
||
test_bars: &[OHLCVBar],
|
||
chunk_start: usize,
|
||
chunk_end: usize,
|
||
feature_dim: usize,
|
||
portfolio: &PortfolioState,
|
||
tick_size: f64,
|
||
spread_ticks: f64,
|
||
) -> Vec<f32> {
|
||
let chunk_len = chunk_end - chunk_start;
|
||
let chunk_equity = portfolio.equity as f32;
|
||
let chunk_exposure = portfolio.current_exposure as f32;
|
||
let mut flat_states: Vec<f32> = Vec::with_capacity(chunk_len * feature_dim);
|
||
|
||
for bar_idx in chunk_start..chunk_end {
|
||
let Some(feat) = test_features.get(bar_idx) else {
|
||
flat_states.extend(std::iter::repeat_n(0.0_f32, feature_dim));
|
||
continue;
|
||
};
|
||
|
||
// 51 market features (f64 -> f32)
|
||
for &v in feat {
|
||
flat_states.push(v as f32);
|
||
}
|
||
|
||
// 3 portfolio features matching training's PortfolioTracker.get_portfolio_features():
|
||
// [0] normalized_value = portfolio_value / initial_capital (~1.0)
|
||
// [1] normalized_position = position_size / max_position (-1.0 to +1.0)
|
||
// [2] avg_spread = tick_size * spread_ticks / price (~0.0001)
|
||
let close = test_bars.get(bar_idx).map(|b| b.close).unwrap_or(1.0);
|
||
let spread_estimate = if close > 1e-12 {
|
||
(tick_size * spread_ticks / close) as f32
|
||
} else {
|
||
0.0001_f32
|
||
};
|
||
flat_states.push(chunk_equity); // normalized_value
|
||
flat_states.push(chunk_exposure); // normalized_position
|
||
flat_states.push(spread_estimate); // avg_spread
|
||
}
|
||
|
||
flat_states
|
||
}
|
||
|
||
/// Simulate trades for a chunk of action indices, updating portfolio state and
|
||
/// accumulating returns and action counts.
|
||
///
|
||
/// Processes action indices sequentially on CPU. Each action maps to a
|
||
/// `FactoredAction` with exposure, order type, and urgency. Transaction costs
|
||
/// are proportional to position delta and differentiated by order type/urgency.
|
||
fn simulate_chunk_trades(
|
||
action_indices: &[usize],
|
||
chunk_start: usize,
|
||
test_bars: &[OHLCVBar],
|
||
args: &Args,
|
||
portfolio: &mut PortfolioState,
|
||
returns: &mut Vec<f64>,
|
||
action_counts: &mut [usize; 3],
|
||
model_name: &str,
|
||
) {
|
||
for (i, &action_idx) in action_indices.iter().enumerate() {
|
||
let bar_idx = chunk_start + i;
|
||
|
||
let action = match FactoredAction::from_index(action_idx) {
|
||
Ok(fa) => fa,
|
||
Err(e) => {
|
||
warn!(" [{}] from_index({}) error at bar {}: {}", model_name, action_idx, bar_idx, e);
|
||
FactoredAction::new(ExposureLevel::Flat, ActionOrderType::Market, Urgency::Normal)
|
||
}
|
||
};
|
||
|
||
// Track action diversity by legacy category (buy/sell/hold)
|
||
let legacy_idx = if action.is_buy() { 0 } else if action.is_sell() { 1 } else { 2 };
|
||
if let Some(count) = action_counts.get_mut(legacy_idx) {
|
||
*count += 1;
|
||
}
|
||
|
||
// Compute percentage return, clamped to filter contract roll boundaries
|
||
let close_cur = test_bars.get(bar_idx).map(|b| b.close).unwrap_or(0.0);
|
||
let close_next = test_bars.get(bar_idx + 1).map(|b| b.close).unwrap_or(close_cur);
|
||
let pct_change = if close_cur.abs() > 1e-12 {
|
||
((close_next - close_cur) / close_cur).clamp(-args.max_bar_return, args.max_bar_return)
|
||
} else {
|
||
0.0
|
||
};
|
||
|
||
// Position delta: only incur costs when position actually changes
|
||
let target_exposure = action.target_exposure();
|
||
let position_delta = (target_exposure - portfolio.current_exposure).abs();
|
||
|
||
// Order-type and urgency-differentiated transaction cost:
|
||
// - Market orders: 15 bps, LimitMaker: 5 bps, IoC: 10 bps
|
||
// - Urgency scales cost: Patient=0.5x, Normal=1.0x, Aggressive=1.5x
|
||
// Cost is proportional to position delta (no cost for holding same position)
|
||
let tx_cost = if position_delta > 1e-12 {
|
||
let base_bps = args.tx_cost_bps + spread_cost_bps(close_cur, args.tick_size, args.spread_ticks);
|
||
let order_cost_frac = action.transaction_cost(); // 0.0015/0.0005/0.0010
|
||
let urgency_mult = action.urgency_weight(); // 0.5/1.0/1.5
|
||
position_delta * (base_bps * 0.0001 + order_cost_frac * urgency_mult)
|
||
} else {
|
||
0.0
|
||
};
|
||
|
||
// Exposure-weighted return minus differentiated transaction cost
|
||
let ret = target_exposure * pct_change - tx_cost;
|
||
returns.push(ret);
|
||
|
||
// Update portfolio state for next bar (and next chunk's frozen state)
|
||
portfolio.equity *= 1.0 + ret;
|
||
portfolio.current_exposure = target_exposure;
|
||
}
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// DQN Evaluation
|
||
// ---------------------------------------------------------------------------
|
||
|
||
/// Run DQN inference on test features using GPU-batched forward passes and return
|
||
/// per-bar trade returns and action counts (buy, sell, hold).
|
||
///
|
||
/// Bars are processed in chunks of `EVAL_CHUNK_SIZE` (1024). Each chunk:
|
||
/// 1. Builds state vectors on CPU with current portfolio features (equity, exposure, spread)
|
||
/// 2. Transfers the `[chunk_len, 54]` tensor to GPU in a single copy
|
||
/// 3. Executes one batched GPU forward pass via `batch_greedy_actions` (argmax of Q-values)
|
||
/// 4. Transfers only the action index vector (`Vec<usize>`) back to CPU
|
||
/// 5. Simulates trades sequentially on CPU to update portfolio state for the next chunk
|
||
///
|
||
/// This reduces GPU kernel launches from N (one per bar) to ceil(N/1024).
|
||
#[allow(clippy::cognitive_complexity)]
|
||
fn evaluate_dqn_fold(
|
||
fold: usize,
|
||
test_features: &[[f64; 51]],
|
||
test_bars: &[OHLCVBar],
|
||
models_dir: &Path,
|
||
args: &Args,
|
||
hp: &Option<Value>,
|
||
) -> Result<(Vec<f64>, [usize; 3])> {
|
||
// Prefer `_best` checkpoint; fall back to highest `_epoch{N}` if training
|
||
// early-stopped without marking a "best" (e.g. validation loss plateaued).
|
||
let best_path = models_dir.join(format!("dqn_fold{}_best.safetensors", fold));
|
||
let ckpt_path = if best_path.exists() {
|
||
best_path
|
||
} else {
|
||
// Glob for epoch checkpoints and pick the highest epoch number
|
||
let pattern = format!("dqn_fold{}_epoch", fold);
|
||
let mut candidates: Vec<_> = std::fs::read_dir(models_dir)
|
||
.ok()
|
||
.into_iter()
|
||
.flatten()
|
||
.filter_map(|e| e.ok())
|
||
.filter(|e| {
|
||
let name = e.file_name();
|
||
let s = name.to_string_lossy();
|
||
s.starts_with(&pattern) && s.ends_with(".safetensors")
|
||
})
|
||
.collect();
|
||
candidates.sort_by_key(|e| std::cmp::Reverse(e.file_name()));
|
||
match candidates.first() {
|
||
Some(entry) => {
|
||
let p = entry.path();
|
||
info!("DQN fold {} using fallback checkpoint: {}", fold, p.display());
|
||
p
|
||
}
|
||
None => {
|
||
anyhow::bail!(
|
||
"No DQN checkpoint found for fold {} in {}",
|
||
fold, models_dir.display()
|
||
);
|
||
}
|
||
}
|
||
};
|
||
|
||
// Create DQN with same config as training — all architecture-affecting params
|
||
// must match exactly, otherwise checkpoint loading fails (tensor shape mismatch).
|
||
#[allow(clippy::integer_division)]
|
||
let config = DQNConfig {
|
||
state_dim: args.feature_dim,
|
||
num_actions: args.num_actions,
|
||
hidden_dims: {
|
||
let base = hp_usize(hp, "hidden_dim_base").unwrap_or(256);
|
||
// Align to 8 for tensor cores (matches training)
|
||
let align = |x: usize| -> usize { x.div_ceil(8) * 8 };
|
||
vec![align(base), align(base / 2), align(base / 4)]
|
||
},
|
||
learning_rate: 1e-4,
|
||
gamma: hp_f64(hp, "gamma").unwrap_or(0.95) as f32,
|
||
epsilon_start: 0.0, // No exploration during evaluation
|
||
epsilon_end: 0.0,
|
||
epsilon_decay: 1.0,
|
||
replay_buffer_capacity: 100, // Minimal buffer, not used for eval
|
||
batch_size: 64,
|
||
min_replay_size: 64,
|
||
target_update_freq: 500,
|
||
warmup_steps: 0,
|
||
use_double_dqn: hp_bool(hp, "use_double_dqn").unwrap_or(true),
|
||
use_huber_loss: true,
|
||
use_per: false, // PER not needed for evaluation (no training)
|
||
// Architecture params — must match training checkpoint shapes
|
||
use_dueling: hp_bool(hp, "use_dueling").unwrap_or(true),
|
||
dueling_hidden_dim: hp_usize(hp, "dueling_hidden_dim").unwrap_or(128),
|
||
use_distributional: hp_bool(hp, "use_distributional").unwrap_or(true),
|
||
num_atoms: hp_usize(hp, "num_atoms").unwrap_or(51),
|
||
v_min: hp_f64(hp, "v_min").unwrap_or(-2.0) as f32,
|
||
v_max: hp_f64(hp, "v_max").unwrap_or(2.0) as f32,
|
||
use_noisy_nets: hp_bool(hp, "use_noisy_nets").unwrap_or(true),
|
||
use_cql: false, // CQL not needed for evaluation
|
||
// use_qr_dqn in hyperopt maps to use_iqn in DQNConfig
|
||
use_iqn: hp_bool(hp, "use_qr_dqn").unwrap_or(false),
|
||
iqn_num_quantiles: hp_usize(hp, "num_quantiles").unwrap_or(64),
|
||
use_cvar_action_selection: false,
|
||
..DQNConfig::default()
|
||
};
|
||
|
||
let mut dqn = DQN::new(config).context("Failed to create DQN model")?;
|
||
|
||
// Load trained weights (DQN::new auto-selects CUDA if available)
|
||
dqn.load_from_safetensors(&ckpt_path.to_string_lossy())
|
||
.with_context(|| format!("Failed to load DQN checkpoint: {}", ckpt_path.display()))?;
|
||
|
||
let device = dqn.device().clone();
|
||
info!(
|
||
" [DQN] Loaded checkpoint: {} (device={:?})",
|
||
ckpt_path.display(),
|
||
device,
|
||
);
|
||
|
||
// ── Chunked GPU-batched inference ──────────────────────────────────────
|
||
//
|
||
// Process bars in chunks of EVAL_CHUNK_SIZE. Each chunk:
|
||
// 1. Builds state vectors on CPU (with portfolio features frozen at chunk start)
|
||
// 2. Single GPU forward pass for the entire chunk via batch_greedy_actions
|
||
// 3. Sequential trade simulation on CPU (updates equity/exposure for next chunk)
|
||
//
|
||
// vs. the old per-bar loop: ~1000x fewer GPU kernel launches.
|
||
|
||
let eval_bars = test_features.len().saturating_sub(1); // last bar has no next-bar return
|
||
let num_chunks = eval_bars.div_ceil(EVAL_CHUNK_SIZE);
|
||
let feature_dim = args.feature_dim;
|
||
|
||
let mut returns = Vec::with_capacity(eval_bars);
|
||
let mut action_counts = [0_usize; 3]; // [buy, sell, hold]
|
||
let mut portfolio = PortfolioState { equity: 1.0, current_exposure: 0.0 };
|
||
|
||
info!(
|
||
" [DQN] GPU-batched eval: {} bars in {} chunks of {} (device: {:?})",
|
||
eval_bars, num_chunks, EVAL_CHUNK_SIZE, device,
|
||
);
|
||
|
||
for chunk_idx in 0..num_chunks {
|
||
let chunk_start = chunk_idx * EVAL_CHUNK_SIZE;
|
||
let chunk_end = (chunk_start + EVAL_CHUNK_SIZE).min(eval_bars);
|
||
let chunk_len = chunk_end - chunk_start;
|
||
|
||
// 1. Build state vectors on CPU — portfolio features frozen at chunk start
|
||
let flat_states = build_chunk_states(
|
||
test_features, test_bars, chunk_start, chunk_end,
|
||
feature_dim, &portfolio, args.tick_size, args.spread_ticks,
|
||
);
|
||
|
||
// 2. Single GPU forward pass — tensor on device, argmax on device.
|
||
// Only the action index vector (chunk_len usizes) crosses GPU->CPU boundary.
|
||
let batch_tensor = Tensor::from_slice(&flat_states, (chunk_len, feature_dim), &device)
|
||
.with_context(|| format!("Failed to create DQN batch tensor for chunk {}", chunk_idx))?;
|
||
|
||
let action_indices = dqn.batch_greedy_actions(&batch_tensor)
|
||
.with_context(|| format!("DQN batch_greedy_actions failed for chunk {}", chunk_idx))?;
|
||
|
||
// 3. Sequential trade simulation on CPU — updates portfolio state
|
||
simulate_chunk_trades(
|
||
&action_indices, chunk_start, test_bars, args,
|
||
&mut portfolio, &mut returns, &mut action_counts, "DQN",
|
||
);
|
||
}
|
||
|
||
Ok((returns, action_counts))
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// PPO Evaluation
|
||
// ---------------------------------------------------------------------------
|
||
|
||
/// Run PPO inference on test features using GPU-batched forward passes and return
|
||
/// per-bar trade returns and action counts (buy, sell, hold).
|
||
///
|
||
/// Same chunked strategy as DQN: bars are processed in chunks of `EVAL_CHUNK_SIZE`.
|
||
/// Each chunk builds a `[chunk_len, 54]` state tensor on GPU, runs the actor
|
||
/// network's `action_probabilities` in a single forward pass, computes argmax
|
||
/// on GPU to get greedy action indices, and transfers only the index vector to CPU
|
||
/// for sequential trade simulation.
|
||
fn evaluate_ppo_fold(
|
||
fold: usize,
|
||
test_features: &[[f64; 51]],
|
||
test_bars: &[OHLCVBar],
|
||
models_dir: &Path,
|
||
args: &Args,
|
||
hp: &Option<Value>,
|
||
) -> Result<(Vec<f64>, [usize; 3])> {
|
||
let actor_path = models_dir.join(format!("ppo_fold{}_actor.safetensors", fold));
|
||
let critic_path = models_dir.join(format!("ppo_fold{}_critic.safetensors", fold));
|
||
|
||
if !actor_path.exists() {
|
||
anyhow::bail!(
|
||
"PPO actor checkpoint not found: {}",
|
||
actor_path.display()
|
||
);
|
||
}
|
||
if !critic_path.exists() {
|
||
anyhow::bail!(
|
||
"PPO critic checkpoint not found: {}",
|
||
critic_path.display()
|
||
);
|
||
}
|
||
|
||
// Create PPO config matching training
|
||
#[allow(clippy::integer_division)]
|
||
let config = PPOConfig {
|
||
state_dim: args.feature_dim,
|
||
num_actions: args.num_actions,
|
||
policy_hidden_dims: {
|
||
let base = hp_usize(hp, "hidden_dim_base").unwrap_or(256);
|
||
let align = |x: usize| x.div_ceil(8) * 8; // tensor-core alignment (must match PpoTrainer)
|
||
vec![align(base), align(base / 2)]
|
||
},
|
||
value_hidden_dims: {
|
||
let base = hp_usize(hp, "hidden_dim_base").unwrap_or(256);
|
||
let align = |x: usize| x.div_ceil(8) * 8; // tensor-core alignment (must match PpoTrainer)
|
||
vec![align(base * 4), align(base * 3), align(base * 2), align(base), align(base / 2)]
|
||
},
|
||
policy_learning_rate: 3e-4,
|
||
value_learning_rate: 1e-3,
|
||
clip_epsilon: 0.2,
|
||
value_loss_coeff: 0.5,
|
||
entropy_coeff: 0.01,
|
||
batch_size: 64,
|
||
mini_batch_size: 64,
|
||
num_epochs: 4,
|
||
max_grad_norm: 0.5,
|
||
use_lstm: false,
|
||
..PPOConfig::default()
|
||
};
|
||
|
||
// Load PPO from checkpoint on GPU (CUDA if available, else CPU fallback)
|
||
let device = candle_core::Device::cuda_if_available(0)
|
||
.context("Failed to detect CUDA device for PPO")?;
|
||
|
||
let ppo = PPO::load_checkpoint(
|
||
&actor_path.to_string_lossy(),
|
||
&critic_path.to_string_lossy(),
|
||
config,
|
||
device.clone(),
|
||
)
|
||
.with_context(|| {
|
||
format!(
|
||
"Failed to load PPO checkpoint: actor={}, critic={}",
|
||
actor_path.display(),
|
||
critic_path.display()
|
||
)
|
||
})?;
|
||
|
||
info!(
|
||
" [PPO] Loaded checkpoint: {} (device={:?})",
|
||
actor_path.display(),
|
||
device,
|
||
);
|
||
|
||
// ── Chunked GPU-batched inference ──────────────────────────────────────
|
||
//
|
||
// Same strategy as DQN: process bars in chunks of EVAL_CHUNK_SIZE.
|
||
// PPO does not have batch_greedy_actions, so we:
|
||
// 1. Build [chunk_len, 54] tensor on GPU
|
||
// 2. Call actor.action_probabilities (single GPU forward pass + softmax)
|
||
// 3. GPU-side argmax → [chunk_len] action indices
|
||
// 4. Transfer only the index vector to CPU for trade simulation
|
||
|
||
let eval_bars = test_features.len().saturating_sub(1); // last bar has no next-bar return
|
||
let num_chunks = eval_bars.div_ceil(EVAL_CHUNK_SIZE);
|
||
let feature_dim = args.feature_dim;
|
||
|
||
let mut returns = Vec::with_capacity(eval_bars);
|
||
let mut action_counts = [0_usize; 3]; // [buy, sell, hold]
|
||
let mut portfolio = PortfolioState { equity: 1.0, current_exposure: 0.0 };
|
||
|
||
info!(
|
||
" [PPO] GPU-batched eval: {} bars in {} chunks of {} (device: {:?})",
|
||
eval_bars, num_chunks, EVAL_CHUNK_SIZE, device,
|
||
);
|
||
|
||
for chunk_idx in 0..num_chunks {
|
||
let chunk_start = chunk_idx * EVAL_CHUNK_SIZE;
|
||
let chunk_end = (chunk_start + EVAL_CHUNK_SIZE).min(eval_bars);
|
||
let chunk_len = chunk_end - chunk_start;
|
||
|
||
// 1. Build state vectors on CPU — portfolio features frozen at chunk start
|
||
let flat_states = build_chunk_states(
|
||
test_features, test_bars, chunk_start, chunk_end,
|
||
feature_dim, &portfolio, args.tick_size, args.spread_ticks,
|
||
);
|
||
|
||
// 2. Single GPU forward pass — tensor on device, softmax on device, argmax on device.
|
||
// Only the action index vector (chunk_len u32s) crosses the GPU->CPU boundary.
|
||
let batch_tensor = Tensor::from_slice(&flat_states, (chunk_len, feature_dim), &device)
|
||
.with_context(|| format!("Failed to create PPO batch tensor for chunk {}", chunk_idx))?;
|
||
|
||
// actor.action_probabilities: forward pass + softmax → [chunk_len, num_actions] on GPU
|
||
let probs_tensor = ppo.actor.action_probabilities(&batch_tensor)
|
||
.with_context(|| format!("PPO action_probabilities failed for chunk {}", chunk_idx))?;
|
||
|
||
// GPU-side argmax → [chunk_len] indices, then transfer to CPU as Vec<u32>
|
||
let action_indices: Vec<usize> = probs_tensor
|
||
.argmax(1)
|
||
.with_context(|| format!("PPO argmax failed for chunk {}", chunk_idx))?
|
||
.to_vec1::<u32>()
|
||
.with_context(|| format!("PPO action index transfer failed for chunk {}", chunk_idx))?
|
||
.into_iter()
|
||
.map(|a| a as usize)
|
||
.collect();
|
||
|
||
// 3. Sequential trade simulation on CPU — updates portfolio state
|
||
simulate_chunk_trades(
|
||
&action_indices, chunk_start, test_bars, args,
|
||
&mut portfolio, &mut returns, &mut action_counts, "PPO",
|
||
);
|
||
}
|
||
|
||
Ok((returns, action_counts))
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Aggregate & Sanity Checks
|
||
// ---------------------------------------------------------------------------
|
||
|
||
/// Compute average metrics for a specific model across all folds.
|
||
/// Returns `(avg_sharpe, avg_trade_sharpe, avg_drawdown, avg_win_rate)`.
|
||
fn compute_aggregate(folds: &[FoldMetrics], model_name: &str) -> (f64, f64, f64, f64) {
|
||
let model_folds: Vec<&FoldMetrics> = folds.iter().filter(|f| f.model == model_name).collect();
|
||
if model_folds.is_empty() {
|
||
return (0.0, 0.0, 0.0, 0.0);
|
||
}
|
||
let n = model_folds.len() as f64;
|
||
let avg_sharpe = model_folds.iter().map(|f| f.sharpe_ratio).sum::<f64>() / n;
|
||
let avg_trade_sharpe = model_folds.iter().map(|f| f.trade_sharpe_ratio).sum::<f64>() / n;
|
||
let avg_dd = model_folds.iter().map(|f| f.max_drawdown_pct).sum::<f64>() / n;
|
||
let avg_wr = model_folds.iter().map(|f| f.win_rate_pct).sum::<f64>() / n;
|
||
(avg_sharpe, avg_trade_sharpe, avg_dd, avg_wr)
|
||
}
|
||
|
||
/// Run sanity checks across all fold metrics.
|
||
fn run_sanity_checks(
|
||
folds: &[FoldMetrics],
|
||
all_action_counts: &[[usize; 3]],
|
||
) -> SanityChecks {
|
||
// beats_random: any model Sharpe > 0?
|
||
let beats_random = folds.iter().any(|f| f.sharpe_ratio > 0.0);
|
||
|
||
// action_diversity: all 3 actions used across all evaluations?
|
||
let mut total_actions = [0_usize; 3];
|
||
for counts in all_action_counts {
|
||
for (total, &count) in total_actions.iter_mut().zip(counts.iter()) {
|
||
*total += count;
|
||
}
|
||
}
|
||
let action_diversity = total_actions.iter().all(|&c| c > 0);
|
||
|
||
// fold_consistency: std(Sharpe) < 2 * |mean(Sharpe)| across all folds
|
||
let sharpe_values: Vec<f64> = folds.iter().map(|f| f.sharpe_ratio).collect();
|
||
let fold_consistency = if sharpe_values.is_empty() {
|
||
false
|
||
} else {
|
||
let n = sharpe_values.len() as f64;
|
||
let mean_sharpe = sharpe_values.iter().sum::<f64>() / n;
|
||
let var = sharpe_values
|
||
.iter()
|
||
.map(|&s| (s - mean_sharpe).powi(2))
|
||
.sum::<f64>()
|
||
/ n;
|
||
let std_sharpe = var.sqrt();
|
||
std_sharpe < 2.0 * mean_sharpe.abs()
|
||
};
|
||
|
||
SanityChecks {
|
||
beats_random,
|
||
action_diversity,
|
||
fold_consistency,
|
||
}
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Main
|
||
// ---------------------------------------------------------------------------
|
||
|
||
#[allow(clippy::cognitive_complexity, clippy::too_many_lines)]
|
||
fn main() -> Result<()> {
|
||
// Initialize tracing with optional OTLP export to Tempo
|
||
let otlp_endpoint = std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT").ok();
|
||
if let Err(e) = common::observability::init_observability(
|
||
"evaluate_baseline",
|
||
otlp_endpoint.as_deref(),
|
||
) {
|
||
eprintln!("Observability init failed (non-fatal): {e}");
|
||
}
|
||
|
||
tm::init();
|
||
metrics_server::start_metrics_server(9094);
|
||
tm::set_active_workers(1.0);
|
||
|
||
let args = Args::parse();
|
||
|
||
let eval_dqn = args.model == "dqn" || args.model == "both";
|
||
let eval_ppo = args.model == "ppo" || args.model == "both";
|
||
|
||
info!("=== Walk-Forward Baseline Evaluation ===");
|
||
info!(" Model(s): {}", args.model);
|
||
info!(" Symbol: {}", args.symbol);
|
||
info!(" Models dir: {}", args.models_dir.display());
|
||
info!(" Data dir: {}", args.data_dir.display());
|
||
info!(" Output: {}", args.output.display());
|
||
info!(" Feature dim: {}", args.feature_dim);
|
||
info!(" Num actions: {}", args.num_actions);
|
||
info!(" Max bar return: {:.2}%", args.max_bar_return * 100.0);
|
||
info!(" Tx cost: {:.1} bps commission + {:.1} tick spread (tick_size={:.4})",
|
||
args.tx_cost_bps, args.spread_ticks, args.tick_size);
|
||
if let Some(ref hp_path) = args.hyperopt_params {
|
||
info!(" Hyperopt params: {}", hp_path.display());
|
||
}
|
||
|
||
// 1. Load all OHLCV bars from DBN files
|
||
info!("Step 1/5: Loading OHLCV bars from DBN files...");
|
||
let data_load_start = std::time::Instant::now();
|
||
let bars = load_all_bars(&args.data_dir, &args.symbol)?;
|
||
let data_load_secs = data_load_start.elapsed().as_secs_f64();
|
||
if eval_dqn {
|
||
tm::record_data_load("dqn", data_load_secs);
|
||
}
|
||
if eval_ppo {
|
||
tm::record_data_load("ppo", data_load_secs);
|
||
}
|
||
if bars.is_empty() {
|
||
anyhow::bail!("No bars loaded from {}", args.data_dir.display());
|
||
}
|
||
info!(
|
||
" Loaded {} bars ({} to {})",
|
||
bars.len(),
|
||
bars.first().map(|b| b.timestamp.to_string()).unwrap_or_default(),
|
||
bars.last().map(|b| b.timestamp.to_string()).unwrap_or_default(),
|
||
);
|
||
|
||
// 2. Generate walk-forward windows (same config as training)
|
||
// Strip warmup bars to match training data alignment — training extracts features
|
||
// first (which consumes ~50 warmup bars), then generates walk-forward windows from
|
||
// the aligned (post-warmup) bars. We must do the same so fold boundaries match.
|
||
info!("Step 2/5: Generating walk-forward windows...");
|
||
let warmup_features = extract_ml_features(&bars)
|
||
.context("Feature extraction for warmup alignment failed")?;
|
||
let warmup_offset = bars.len().saturating_sub(warmup_features.len());
|
||
let aligned_bars = bars.get(warmup_offset..).unwrap_or(&bars);
|
||
info!(" Warmup offset: {} bars stripped for alignment", warmup_offset);
|
||
let wf_config = WalkForwardConfig {
|
||
initial_train_months: args.train_months,
|
||
val_months: args.val_months,
|
||
test_months: args.test_months,
|
||
step_months: args.step_months,
|
||
};
|
||
let windows = generate_walk_forward_windows(aligned_bars, &wf_config);
|
||
if windows.is_empty() {
|
||
anyhow::bail!(
|
||
"No walk-forward windows generated. Need at least {} months of data.",
|
||
wf_config.initial_train_months + wf_config.val_months + wf_config.test_months
|
||
);
|
||
}
|
||
info!(" Generated {} walk-forward folds", windows.len());
|
||
|
||
// 3. Evaluate each fold
|
||
info!("Step 3/5: Evaluating models on test data...");
|
||
let mut all_fold_metrics: Vec<FoldMetrics> = Vec::new();
|
||
let mut all_action_counts: Vec<[usize; 3]> = Vec::new();
|
||
|
||
for window in &windows {
|
||
info!(
|
||
"--- Fold {} --- Test: {} bars ({} to {})",
|
||
window.fold,
|
||
window.test.len(),
|
||
window.test
|
||
.first()
|
||
.map(|b| b.timestamp.to_string())
|
||
.unwrap_or_default(),
|
||
window.test
|
||
.last()
|
||
.map(|b| b.timestamp.to_string())
|
||
.unwrap_or_default(),
|
||
);
|
||
|
||
// Load NormStats from training
|
||
let norm_path = args
|
||
.models_dir
|
||
.join(format!("norm_stats_fold{}.json", window.fold));
|
||
let norm_stats: NormStats = if norm_path.exists() {
|
||
let norm_json = std::fs::read_to_string(&norm_path)
|
||
.with_context(|| format!("Failed to read {}", norm_path.display()))?;
|
||
serde_json::from_str(&norm_json)
|
||
.with_context(|| format!("Failed to parse {}", norm_path.display()))?
|
||
} else {
|
||
anyhow::bail!(
|
||
"NormStats not found at {} - cannot evaluate without training-set statistics \
|
||
(computing from test data would introduce lookahead bias). \
|
||
Run training first to generate this file.",
|
||
norm_path.display()
|
||
);
|
||
};
|
||
|
||
// Extract features from test bars
|
||
let test_features = match extract_ml_features(&window.test) {
|
||
Ok(f) => f,
|
||
Err(e) => {
|
||
warn!(
|
||
" Fold {} - test feature extraction failed: {}",
|
||
window.fold, e
|
||
);
|
||
continue;
|
||
}
|
||
};
|
||
|
||
if test_features.is_empty() {
|
||
warn!(" Fold {} - empty test features, skipping", window.fold);
|
||
continue;
|
||
}
|
||
|
||
// Normalize test features
|
||
let test_norm = norm_stats.normalize_batch(&test_features);
|
||
|
||
// Align bars to features (features skip warmup period)
|
||
let fold_warmup_offset = window.test.len().saturating_sub(test_norm.len());
|
||
let test_bars_aligned = window.test.get(fold_warmup_offset..).unwrap_or(&window.test);
|
||
|
||
// Test period date range for the report
|
||
let test_start = test_bars_aligned
|
||
.first()
|
||
.map(|b| b.timestamp.format("%Y-%m-%d").to_string())
|
||
.unwrap_or_default();
|
||
let test_end = test_bars_aligned
|
||
.last()
|
||
.map(|b| b.timestamp.format("%Y-%m-%d").to_string())
|
||
.unwrap_or_default();
|
||
|
||
// Evaluate DQN
|
||
if eval_dqn {
|
||
let hp = load_hyperopt_params(&args.hyperopt_params, "dqn");
|
||
match evaluate_dqn_fold(
|
||
window.fold,
|
||
&test_norm,
|
||
test_bars_aligned,
|
||
&args.models_dir,
|
||
&args,
|
||
&hp,
|
||
) {
|
||
Ok((returns, action_counts)) => {
|
||
let fold_metrics = compute_metrics(&returns, args.bars_per_year);
|
||
let fold_str = window.fold.to_string();
|
||
tm::set_epoch("dqn", &fold_str, window.fold as f64);
|
||
tm::set_eval_metrics(
|
||
"dqn",
|
||
&fold_str,
|
||
fold_metrics.win_rate_pct / 100.0,
|
||
fold_metrics.sharpe_ratio,
|
||
fold_metrics.profit_factor,
|
||
fold_metrics.total_return_pct / 100.0,
|
||
);
|
||
info!(
|
||
" [DQN] Fold {} - Sharpe(bar)={:.4} Sharpe(trade)={:.4} MaxDD={:.2}% WR={:.1}% PF={:.2} Return={:.4}% Trades={}",
|
||
window.fold,
|
||
fold_metrics.sharpe_ratio,
|
||
fold_metrics.trade_sharpe_ratio,
|
||
fold_metrics.max_drawdown_pct,
|
||
fold_metrics.win_rate_pct,
|
||
fold_metrics.profit_factor,
|
||
fold_metrics.total_return_pct,
|
||
fold_metrics.num_trades,
|
||
);
|
||
info!(
|
||
" [DQN] Actions - BUY={} SELL={} HOLD={}",
|
||
action_counts.first().copied().unwrap_or(0),
|
||
action_counts.get(1).copied().unwrap_or(0),
|
||
action_counts.get(2).copied().unwrap_or(0),
|
||
);
|
||
all_fold_metrics.push(FoldMetrics {
|
||
fold: window.fold,
|
||
model: "dqn".to_owned(),
|
||
sharpe_ratio: fold_metrics.sharpe_ratio,
|
||
trade_sharpe_ratio: fold_metrics.trade_sharpe_ratio,
|
||
max_drawdown_pct: fold_metrics.max_drawdown_pct,
|
||
win_rate_pct: fold_metrics.win_rate_pct,
|
||
profit_factor: fold_metrics.profit_factor,
|
||
total_return_pct: fold_metrics.total_return_pct,
|
||
num_trades: fold_metrics.num_trades,
|
||
test_start: test_start.clone(),
|
||
test_end: test_end.clone(),
|
||
});
|
||
all_action_counts.push(action_counts);
|
||
}
|
||
Err(e) => {
|
||
error!(" [DQN] Fold {} evaluation failed: {}", window.fold, e);
|
||
}
|
||
}
|
||
}
|
||
|
||
// Evaluate PPO
|
||
if eval_ppo {
|
||
let hp = load_hyperopt_params(&args.hyperopt_params, "ppo");
|
||
match evaluate_ppo_fold(
|
||
window.fold,
|
||
&test_norm,
|
||
test_bars_aligned,
|
||
&args.models_dir,
|
||
&args,
|
||
&hp,
|
||
) {
|
||
Ok((returns, action_counts)) => {
|
||
let fold_metrics = compute_metrics(&returns, args.bars_per_year);
|
||
let fold_str = window.fold.to_string();
|
||
tm::set_epoch("ppo", &fold_str, window.fold as f64);
|
||
tm::set_eval_metrics(
|
||
"ppo",
|
||
&fold_str,
|
||
fold_metrics.win_rate_pct / 100.0,
|
||
fold_metrics.sharpe_ratio,
|
||
fold_metrics.profit_factor,
|
||
fold_metrics.total_return_pct / 100.0,
|
||
);
|
||
info!(
|
||
" [PPO] Fold {} - Sharpe(bar)={:.4} Sharpe(trade)={:.4} MaxDD={:.2}% WR={:.1}% PF={:.2} Return={:.4}% Trades={}",
|
||
window.fold,
|
||
fold_metrics.sharpe_ratio,
|
||
fold_metrics.trade_sharpe_ratio,
|
||
fold_metrics.max_drawdown_pct,
|
||
fold_metrics.win_rate_pct,
|
||
fold_metrics.profit_factor,
|
||
fold_metrics.total_return_pct,
|
||
fold_metrics.num_trades,
|
||
);
|
||
info!(
|
||
" [PPO] Actions - BUY={} SELL={} HOLD={}",
|
||
action_counts.first().copied().unwrap_or(0),
|
||
action_counts.get(1).copied().unwrap_or(0),
|
||
action_counts.get(2).copied().unwrap_or(0),
|
||
);
|
||
all_fold_metrics.push(FoldMetrics {
|
||
fold: window.fold,
|
||
model: "ppo".to_owned(),
|
||
sharpe_ratio: fold_metrics.sharpe_ratio,
|
||
trade_sharpe_ratio: fold_metrics.trade_sharpe_ratio,
|
||
max_drawdown_pct: fold_metrics.max_drawdown_pct,
|
||
win_rate_pct: fold_metrics.win_rate_pct,
|
||
profit_factor: fold_metrics.profit_factor,
|
||
total_return_pct: fold_metrics.total_return_pct,
|
||
num_trades: fold_metrics.num_trades,
|
||
test_start: test_start.clone(),
|
||
test_end: test_end.clone(),
|
||
});
|
||
all_action_counts.push(action_counts);
|
||
}
|
||
Err(e) => {
|
||
error!(" [PPO] Fold {} evaluation failed: {}", window.fold, e);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// 4. Compute aggregate metrics
|
||
info!("Step 4/5: Computing aggregate metrics...");
|
||
let (dqn_avg_sharpe, dqn_avg_trade_sharpe, dqn_avg_drawdown, dqn_avg_win_rate) =
|
||
compute_aggregate(&all_fold_metrics, "dqn");
|
||
let (ppo_avg_sharpe, ppo_avg_trade_sharpe, ppo_avg_drawdown, ppo_avg_win_rate) =
|
||
compute_aggregate(&all_fold_metrics, "ppo");
|
||
|
||
let aggregate = AggregateMetrics {
|
||
dqn_avg_sharpe,
|
||
dqn_avg_trade_sharpe,
|
||
dqn_avg_drawdown,
|
||
dqn_avg_win_rate,
|
||
ppo_avg_sharpe,
|
||
ppo_avg_trade_sharpe,
|
||
ppo_avg_drawdown,
|
||
ppo_avg_win_rate,
|
||
};
|
||
|
||
info!(" DQN - avg Sharpe(bar)={:.4} avg Sharpe(trade)={:.4} avg MaxDD={:.2}% avg WR={:.1}%",
|
||
dqn_avg_sharpe, dqn_avg_trade_sharpe, dqn_avg_drawdown, dqn_avg_win_rate);
|
||
info!(" PPO - avg Sharpe(bar)={:.4} avg Sharpe(trade)={:.4} avg MaxDD={:.2}% avg WR={:.1}%",
|
||
ppo_avg_sharpe, ppo_avg_trade_sharpe, ppo_avg_drawdown, ppo_avg_win_rate);
|
||
|
||
// 5. Sanity checks & report
|
||
info!("Step 5/5: Running sanity checks and saving report...");
|
||
let sanity_checks = run_sanity_checks(&all_fold_metrics, &all_action_counts);
|
||
|
||
info!(" Beats random: {}", sanity_checks.beats_random);
|
||
info!(" Action diversity: {}", sanity_checks.action_diversity);
|
||
info!(" Fold consistency: {}", sanity_checks.fold_consistency);
|
||
|
||
let report = EvaluationReport {
|
||
folds: all_fold_metrics,
|
||
aggregate,
|
||
sanity_checks,
|
||
};
|
||
|
||
// Save report
|
||
if let Some(parent) = args.output.parent() {
|
||
std::fs::create_dir_all(parent)
|
||
.with_context(|| format!("Failed to create output dir: {}", parent.display()))?;
|
||
}
|
||
let report_json = serde_json::to_string_pretty(&report)
|
||
.context("Failed to serialize evaluation report")?;
|
||
std::fs::write(&args.output, &report_json)
|
||
.with_context(|| format!("Failed to write report to {}", args.output.display()))?;
|
||
|
||
info!("=== Evaluation Complete ===");
|
||
info!(" Report saved to: {}", args.output.display());
|
||
info!(" Total fold evaluations: {}", report.folds.len());
|
||
|
||
tm::set_active_workers(0.0);
|
||
|
||
Ok(())
|
||
}
|