gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(eval): train/eval parity - position tracking, cost differentiation, feature alignment

Browse Source 
- Wire position-delta cost model: costs only apply on position changes,
  proportional to delta (not flat per-bar)
- Differentiate order types: Market=15bps, LimitMaker=5bps, IoC=10bps
- Apply urgency weight: Patient=0.5x, Normal=1.0x, Aggressive=1.5x
- Append 3 portfolio features (position, unrealized PnL, drawdown) to match
  training's 54-dim state vector (was 51, causing shape mismatch on load)
- Fix default num_actions 3->45 to match training's factored action space
- Add --bars-per-year CLI flag for per-symbol annualization
  (ES/NQ=347760, 6E=345000, ZN=105840)

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-03-03 21:43:01 +01:00
parent 3ef80b9efe
commit 2f60e0cda0
1 changed files with 109 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

143
crates/ml/examples/evaluate_baseline.rs
View File

@@ -30,6 +30,7 @@ use serde_json::Value;
use tracing::{error, info, warn};
use common::metrics::{server as metrics_server, training_metrics as tm};
use ml::common::action::{ExposureLevel, OrderType as ActionOrderType, Urgency};
use ml::dqn::{DQNConfig, DQN};
#[allow(unreachable_pub)]
@@ -67,12 +68,12 @@ struct Args {
#[arg(long, default_value = "both")]
model: String,
/// Feature dimension (must match `extract_ml_features` output)
#[arg(long, default_value_t = 51)]
/// Feature dimension (51 market + 3 portfolio = 54, must match training config)
#[arg(long, default_value_t = 54)]
feature_dim: usize,
/// Number of actions for the DQN/PPO action space
#[arg(long, default_value_t = 3)]
/// Number of actions for the DQN/PPO action space (45 = 5 exposure x 3 order x 3 urgency)
#[arg(long, default_value_t = 45)]
num_actions: usize,
/// Walk-forward: initial training window in months
@@ -114,6 +115,11 @@ struct Args {
#[arg(long, default_value_t = 1.0)]
spread_ticks: f64,
/// Bars per year for Sharpe annualization (ES/NQ=347760, 6E=345000, ZN=105840).
/// Default 347760 = 252 trading days × 1380 bars/day (ES 23h session).
#[arg(long, default_value_t = 347_760.0)]
bars_per_year: f64,
/// Optional path to hyperopt results JSON -- overrides matching config fields
/// (must match the file used during training so network architecture is identical)
#[arg(long)]
@@ -240,7 +246,7 @@ struct ComputedMetrics {
/// - Profit factor: `gross_profit` / `gross_loss` (inf if no losses)
/// - Total return: sum of returns * 100 (as percentage)
/// - Num trades: count of non-zero returns (BUY or SELL actions)
fn compute_metrics(returns: &[f64]) -> ComputedMetrics {
fn compute_metrics(returns: &[f64], bars_per_year: f64) -> ComputedMetrics {
let n = returns.len();
if n == 0 {
return ComputedMetrics {
@@ -262,10 +268,6 @@ fn compute_metrics(returns: &[f64]) -> ComputedMetrics {
let variance: f64 = returns.iter().map(|&r| (r - mean).powi(2)).sum::<f64>() / n as f64;
let std = variance.sqrt();
// Annualized Sharpe ratio (1-minute bars for ES futures: ~23h/day × 60 = 1380 bars/day)
// CME E-mini futures trade Sun 5pm–Fri 4pm CT with a 1h daily halt.
let bars_per_year: f64 = 252.0 * 1380.0; // ~347,760 bars/year
let sharpe_ratio = if std > 1e-12 {
(mean / std) * bars_per_year.sqrt()
} else {
@@ -397,28 +399,42 @@ fn evaluate_dqn_fold(
dqn.get_epsilon()
);
// Run inference
// Run inference with position tracking and portfolio features
let n = test_features.len();
let mut returns = Vec::with_capacity(n);
let mut action_counts = [0_usize; 3]; // [buy, sell, hold]
let mut current_exposure: f64 = 0.0;
// Portfolio state for the 3 appended features (matching training's 54-dim input)
let mut equity: f64 = 1.0;
let mut peak_equity: f64 = 1.0;
let mut unrealized_pnl: f64 = 0.0;
for i in 0..n.saturating_sub(1) {
let Some(feat) = test_features.get(i) else {
continue;
};
let state: Vec<f32> = feat.iter().map(|&v| v as f32).collect();
// Append 3 portfolio features to match training's 54-dim state vector:
// [51 market features] + [position_size, unrealized_pnl, drawdown_pct]
let mut state: Vec<f32> = feat.iter().map(|&v| v as f32).collect();
let drawdown_pct = if peak_equity > 1e-12 {
((peak_equity - equity) / peak_equity).max(0.0)
} else {
0.0
};
state.push(current_exposure as f32); // position_size: -1.0 to +1.0
state.push(unrealized_pnl as f32); // unrealized P&L (fraction)
state.push(drawdown_pct as f32); // drawdown from peak (0.0 to 1.0)
// Select greedy action (returns FactoredAction)
let action = match dqn.select_action(&state) {
Ok(fa) => fa,
Err(e) => {
warn!(" [DQN] select_action error at step {}: {}", i, e);
// Default to Flat/Market/Normal (Hold equivalent)
ml::common::action::FactoredAction::new(
ml::common::action::ExposureLevel::Flat,
ml::common::action::OrderType::Market,
ml::common::action::Urgency::Normal,
ExposureLevel::Flat,
ActionOrderType::Market,
Urgency::Normal,
)
}
};
@@ -438,12 +454,38 @@ fn evaluate_dqn_fold(
0.0
};
// Exposure-weighted return minus order-type-specific transaction cost
let exposure = action.target_exposure() as f64;
let tx_cost_bps = args.tx_cost_bps + spread_cost_bps(close_cur, args.tick_size, args.spread_ticks);
let tx_cost = if action.is_hold() { 0.0 } else { tx_cost_bps * 0.0001 };
let ret = exposure * pct_change - tx_cost;
// Position delta: only incur costs when position actually changes
let target_exposure = action.target_exposure();
let position_delta = (target_exposure - current_exposure).abs();
// Order-type and urgency-differentiated transaction cost:
// - Market orders: 15 bps, LimitMaker: 5 bps, IoC: 10 bps
// - Urgency scales cost: Patient=0.5x, Normal=1.0x, Aggressive=1.5x
// Cost is proportional to position delta (no cost for holding same position)
let tx_cost = if position_delta > 1e-12 {
let base_bps = args.tx_cost_bps + spread_cost_bps(close_cur, args.tick_size, args.spread_ticks);
let order_cost_frac = action.transaction_cost(); // 0.0015/0.0005/0.0010
let urgency_mult = action.urgency_weight(); // 0.5/1.0/1.5
position_delta * (base_bps * 0.0001 + order_cost_frac * urgency_mult)
} else {
0.0
};
// Exposure-weighted return minus differentiated transaction cost
let ret = target_exposure * pct_change - tx_cost;
returns.push(ret);
// Update portfolio state for next step's features
equity *= 1.0 + ret;
if equity > peak_equity {
peak_equity = equity;
}
unrealized_pnl = if position_delta > 1e-12 {
0.0 // Position changed, reset unrealized P&L
} else {
unrealized_pnl + target_exposure * pct_change // Accumulate while holding
};
current_exposure = target_exposure;
}
Ok((returns, action_counts))
@@ -525,28 +567,40 @@ fn evaluate_ppo_fold(
actor_path.display()
);
// Run inference
// Run inference with position tracking and portfolio features
let n = test_features.len();
let mut returns = Vec::with_capacity(n);
let mut action_counts = [0_usize; 3]; // [buy, sell, hold]
let mut current_exposure: f64 = 0.0;
let mut equity: f64 = 1.0;
let mut peak_equity: f64 = 1.0;
let mut unrealized_pnl: f64 = 0.0;
for i in 0..n.saturating_sub(1) {
let Some(feat) = test_features.get(i) else {
continue;
};
let state: Vec<f32> = feat.iter().map(|&v| v as f32).collect();
// Append 3 portfolio features to match training's 54-dim state vector
let mut state: Vec<f32> = feat.iter().map(|&v| v as f32).collect();
let drawdown_pct = if peak_equity > 1e-12 {
((peak_equity - equity) / peak_equity).max(0.0)
} else {
0.0
};
state.push(current_exposure as f32);
state.push(unrealized_pnl as f32);
state.push(drawdown_pct as f32);
// Get greedy (deterministic) action from policy for reproducible evaluation
let action = match ppo.greedy_action(&state) {
Ok(fa) => fa,
Err(e) => {
warn!(" [PPO] greedy_action error at step {}: {}", i, e);
// Default to Flat/Market/Normal (Hold equivalent)
ml::common::action::FactoredAction::new(
ml::common::action::ExposureLevel::Flat,
ml::common::action::OrderType::Market,
ml::common::action::Urgency::Normal,
ExposureLevel::Flat,
ActionOrderType::Market,
Urgency::Normal,
)
}
};
@@ -566,12 +620,33 @@ fn evaluate_ppo_fold(
0.0
};
// Exposure-weighted return minus order-type-specific transaction cost
let exposure = action.target_exposure() as f64;
let tx_cost_bps = args.tx_cost_bps + spread_cost_bps(close_cur, args.tick_size, args.spread_ticks);
let tx_cost = if action.is_hold() { 0.0 } else { tx_cost_bps * 0.0001 };
let ret = exposure * pct_change - tx_cost;
// Position delta: only incur costs when position actually changes
let target_exposure = action.target_exposure();
let position_delta = (target_exposure - current_exposure).abs();
let tx_cost = if position_delta > 1e-12 {
let base_bps = args.tx_cost_bps + spread_cost_bps(close_cur, args.tick_size, args.spread_ticks);
let order_cost_frac = action.transaction_cost();
let urgency_mult = action.urgency_weight();
position_delta * (base_bps * 0.0001 + order_cost_frac * urgency_mult)
} else {
0.0
};
let ret = target_exposure * pct_change - tx_cost;
returns.push(ret);
// Update portfolio state for next step's features
equity *= 1.0 + ret;
if equity > peak_equity {
peak_equity = equity;
}
unrealized_pnl = if position_delta > 1e-12 {
0.0
} else {
unrealized_pnl + target_exposure * pct_change
};
current_exposure = target_exposure;
}
Ok((returns, action_counts))
@@ -742,7 +817,7 @@ fn main() -> Result<()> {
.with_context(|| format!("Failed to parse {}", norm_path.display()))?
} else {
anyhow::bail!(
"NormStats not found at {} — cannot evaluate without training-set statistics \
"NormStats not found at {} - cannot evaluate without training-set statistics \
(computing from test data would introduce lookahead bias). \
Run training first to generate this file.",
norm_path.display()
@@ -795,7 +870,7 @@ fn main() -> Result<()> {
&hp,
) {
Ok((returns, action_counts)) => {
let fold_metrics = compute_metrics(&returns);
let fold_metrics = compute_metrics(&returns, args.bars_per_year);
let fold_str = window.fold.to_string();
tm::set_epoch("dqn", &fold_str, window.fold as f64);
tm::set_eval_metrics(
@@ -854,7 +929,7 @@ fn main() -> Result<()> {
&hp,
) {
Ok((returns, action_counts)) => {
let fold_metrics = compute_metrics(&returns);
let fold_metrics = compute_metrics(&returns, args.bars_per_year);
let fold_str = window.fold.to_string();
tm::set_epoch("ppo", &fold_str, window.fold as f64);
tm::set_eval_metrics(