From 0ddc861708bd80f59041571fe1cc214b53c692d7 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Tue, 19 May 2026 08:44:46 +0200 Subject: [PATCH] feat(ml-backtesting): max_drawdown_pct + deployability verdict emitter (X16+X17) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X16: Adds Summary.max_drawdown_pct field as |max_drawdown_usd| / STARTING_CAPITAL_USD (pinned at $35k per project_ml_alpha_starting_capital memory — realistic ES single-contract small-account anchor). Used by the verdict emitter as the capital-deployability hard gate (median across windows < 20%). X17: Adds VerdictTier enum (Pass-robust / Pass-nominal / Fail-inconclusive / Fail / Fail-degenerate), AnchorSpec / AnchorReport / DeployabilityVerdict types, classify_verdict (tiered logic per spec §3.5), and emit_deployability_verdict that reads per-cell summary.json files at both realistic (1.0 tick, 200 ms) and stress (1.5 tick, 400 ms) anchors, computes median Sharpe / Sortino / max_dd_pct / profit_factor across walk-forward windows, and applies the gates. Per spec §3.2 (X16), §3.5 (X17). --- crates/ml-backtesting/src/aggregate.rs | 187 +++++++++++++++++++++++++ crates/ml-backtesting/src/artifacts.rs | 14 ++ 2 files changed, 201 insertions(+) diff --git a/crates/ml-backtesting/src/aggregate.rs b/crates/ml-backtesting/src/aggregate.rs index 65bfa9be7..3571cafc9 100644 --- a/crates/ml-backtesting/src/aggregate.rs +++ b/crates/ml-backtesting/src/aggregate.rs @@ -16,6 +16,193 @@ use std::path::Path; use std::sync::Arc; use crate::artifacts::Summary; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +/// X17: Tiered verdict for the deployability validation. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum VerdictTier { + PassRobust, + PassNominal, + FailInconclusive, + Fail, + FailDegenerate, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct AnchorSpec { + pub name: String, + pub cost_tick: f32, + pub latency_ms: u32, +} + +impl AnchorSpec { + pub fn realistic() -> Self { + Self { name: "realistic".to_string(), cost_tick: 1.0, latency_ms: 200 } + } + pub fn stress() -> Self { + Self { name: "stress".to_string(), cost_tick: 1.5, latency_ms: 400 } + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct AnchorReport { + pub anchor: AnchorSpec, + pub sharpe_per_window: Vec, + pub sortino_per_window: Vec, + pub max_dd_pct_per_window: Vec, + pub profit_factor_per_window: Vec, + pub median_sharpe: f32, + pub median_sortino: f32, + pub median_max_dd_pct: f32, + pub median_profit_factor: f32, + pub gate_sharpe: bool, + pub gate_max_dd: bool, + pub pass: bool, + pub status: String, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct DeployabilityVerdict { + pub verdict: VerdictTier, + pub realistic: AnchorReport, + pub stress: AnchorReport, + pub threshold: f32, + pub windows: Vec, + pub training_sha: String, + pub spec_sha: String, + pub timestamp_utc: String, +} + +/// X17: tiered verdict classifier. Spec §3.5. +pub fn classify_verdict(realistic: &AnchorReport, stress: &AnchorReport) -> VerdictTier { + if realistic.status != "ok" { + return VerdictTier::FailDegenerate; + } + if realistic.pass && stress.status == "ok" && stress.pass { + return VerdictTier::PassRobust; + } + if realistic.pass { + return VerdictTier::PassNominal; + } + let sharpe_grey = realistic.median_sharpe >= 0.8 && realistic.median_sharpe < 1.0; + let max_dd_grey = realistic.median_max_dd_pct >= 0.20 && realistic.median_max_dd_pct < 0.25; + if sharpe_grey || max_dd_grey { + return VerdictTier::FailInconclusive; + } + VerdictTier::Fail +} + +fn median(xs: &[f32]) -> f32 { + let mut v = xs.to_vec(); + v.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); + let n = v.len(); + if n == 0 { return f32::NAN; } + if n % 2 == 1 { v[n / 2] } else { 0.5 * (v[n / 2 - 1] + v[n / 2]) } +} + +/// Parse a cell directory name like +/// `cell_0042_cost1.00_lat200_th0.75_W1` into its (cost_tick, latency_ms, +/// threshold, window) tuple. Returns None on non-matching names so +/// callers can skip them. +fn parse_cell_name(name: &str) -> Option<(f32, u32, f32, String)> { + let parts: Vec<&str> = name.split('_').collect(); + if parts.len() < 5 || parts[0] != "cell" { return None; } + let cost: f32 = parts.iter().find(|p| p.starts_with("cost"))?[4..].parse().ok()?; + let lat: u32 = parts.iter().find(|p| p.starts_with("lat"))?[3..].parse().ok()?; + let th: f32 = parts.iter().find(|p| p.starts_with("th"))?[2..].parse().ok()?; + let win = parts.iter().rev().find(|p| p.starts_with('W'))?.to_string(); + Some((cost, lat, th, win)) +} + +fn build_anchor_report( + sweep_dir: &Path, + anchor: AnchorSpec, + threshold: f32, + windows: &[&str], +) -> Result { + let mut by_window: BTreeMap = BTreeMap::new(); + for entry in std::fs::read_dir(sweep_dir) + .with_context(|| format!("read_dir {}", sweep_dir.display()))? + { + let path = entry?.path(); + let name = path.file_name().and_then(|n| n.to_str()).unwrap_or("").to_string(); + let (cell_cost, cell_lat, cell_th, cell_win) = match parse_cell_name(&name) { + Some(t) => t, + None => continue, + }; + if (cell_cost - anchor.cost_tick).abs() > 1e-6 { continue; } + if cell_lat != anchor.latency_ms { continue; } + if (cell_th - threshold).abs() > 1e-6 { continue; } + if !windows.iter().any(|w| **w == cell_win) { continue; } + let summary_path = path.join("summary.json"); + let s: Summary = serde_json::from_reader( + std::fs::File::open(&summary_path) + .with_context(|| format!("open {}", summary_path.display()))?, + )?; + by_window.insert(cell_win.clone(), s); + } + let mut sharpe = Vec::new(); + let mut sortino = Vec::new(); + let mut max_dd = Vec::new(); + let mut pf = Vec::new(); + let mut status = String::from("ok"); + for w in windows { + let s = by_window.get(*w).ok_or_else(|| + anyhow::anyhow!("missing cell for anchor={} window={}", anchor.name, w) + )?; + if s.n_trades == 0 || !s.sharpe_ann.is_finite() || !s.sortino_ann.is_finite() { + status = format!("fail-degenerate: {w} trades={} sharpe={}", s.n_trades, s.sharpe_ann); + } + sharpe.push(s.sharpe_ann); + sortino.push(s.sortino_ann); + max_dd.push(s.max_drawdown_pct); + pf.push(s.profit_factor); + } + let median_sharpe = median(&sharpe); + let median_sortino = median(&sortino); + let median_max_dd_pct = median(&max_dd); + let median_profit_factor = median(&pf); + let gate_sharpe = median_sharpe > 1.0; + let gate_max_dd = median_max_dd_pct < 0.20; + let pass = gate_sharpe && gate_max_dd && status == "ok"; + Ok(AnchorReport { + anchor, + sharpe_per_window: sharpe, + sortino_per_window: sortino, + max_dd_pct_per_window: max_dd, + profit_factor_per_window: pf, + median_sharpe, median_sortino, median_max_dd_pct, median_profit_factor, + gate_sharpe, gate_max_dd, pass, + status, + }) +} + +/// X17: emit the full deployability verdict by aggregating per-cell +/// summary.json files at both anchors against the supplied threshold + +/// walk-forward window IDs. Spec §3.5. +pub fn emit_deployability_verdict( + sweep_dir: &Path, + threshold: f32, + windows: &[&str], + training_sha: &str, + spec_sha: &str, +) -> Result { + let realistic = build_anchor_report(sweep_dir, AnchorSpec::realistic(), threshold, windows)?; + let stress = build_anchor_report(sweep_dir, AnchorSpec::stress(), threshold, windows)?; + let verdict = classify_verdict(&realistic, &stress); + Ok(DeployabilityVerdict { + verdict, + realistic, + stress, + threshold, + windows: windows.iter().map(|s| s.to_string()).collect(), + training_sha: training_sha.to_string(), + spec_sha: spec_sha.to_string(), + timestamp_utc: chrono::Utc::now().to_rfc3339(), + }) +} /// One row of the aggregate parquet: cell name + a flattened subset of /// `Summary` fields the user is most likely to filter / sort by. diff --git a/crates/ml-backtesting/src/artifacts.rs b/crates/ml-backtesting/src/artifacts.rs index aac876ea7..08e124f08 100644 --- a/crates/ml-backtesting/src/artifacts.rs +++ b/crates/ml-backtesting/src/artifacts.rs @@ -25,6 +25,11 @@ pub struct Summary { pub sharpe_ann: f32, pub sortino_ann: f32, pub max_drawdown_usd: f32, + /// X16: max drawdown as a fraction of starting capital. Computed + /// in compute_summary using STARTING_CAPITAL_USD ($35k per project + /// convention; see memory `project_ml_alpha_starting_capital`). + /// Range [0.0, 1.0]; 0.20 = 20% drawdown. + pub max_drawdown_pct: f32, pub calmar: f32, pub n_trades: u64, pub win_rate: f32, @@ -48,6 +53,13 @@ pub struct Summary { /// `aggregate` reads from the same constant if it ever recomputes. pub const ANNUALISATION_SQRT_FACTOR: f32 = 28.722_815; +/// X16: starting capital in USD used as the base for max_drawdown_pct. +/// $35k is the project's chosen base for ES single-contract analysis +/// — realistic for a small-account ES trader (margin ~$15k initial, +/// ~$20k headroom for adverse moves before a margin call). See memory +/// `project_ml_alpha_starting_capital`. +pub const STARTING_CAPITAL_USD: f32 = 35_000.0; + /// Convert TradeRecord fixed-point USD ×100 to plain USD float. #[inline] fn fp_to_usd(fp: i32) -> f32 { @@ -146,11 +158,13 @@ pub fn compute_summary(records: &[TradeRecord], pnl_curve_usd: &[f32]) -> Summar 0.0 }; + let max_drawdown_pct = max_drawdown_usd.abs() / STARTING_CAPITAL_USD; Summary { total_pnl_usd, sharpe_ann, sortino_ann, max_drawdown_usd, + max_drawdown_pct, calmar, n_trades, win_rate,