From e190ecfa61b77aca68a74c4c17e06db5a432591f Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Fri, 15 May 2026 09:44:55 +0200 Subject: [PATCH] feat(ml-alpha): backtest cost sweep + annualised Sharpe (Phase 1d.4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Initial single-cost backtest at τ=0.25 cost=0.25 revealed the binding constraint: mean_ret = -0.176, pre-cost EV ≈ +0.074, so the model has a real directional edge but K=6000 price moves are too small to clear 1-tick round-trip cost. The cost-vs-edge balance is the real Phase 1d.4 verdict question, not whether the model has signal. Two enhancements per the insight block in the previous run: 1. **Cost sweep**: GpuBacktest::run now takes `costs: &[f32]` and produces (C × T) rows instead of T. The smoke runs at five costs: - 0.0 frictionless upper bound (theoretical max Sharpe) - 0.0625 quarter-tick (very aggressive execution) - 0.125 half-tick (professional desk) - 0.25 1 tick = $12.50/contract (retail / pessimistic) - 0.50 2 ticks (very pessimistic) Tells us the break-even cost where Sharpe crosses zero. 2. **Annualised Sharpe**: per-trade Sharpe × sqrt(trades_per_year). trades_per_year = n_trades × (seconds_per_year / test_time_span_seconds). test_time_span_seconds derived from first/last test sequence end-bar timestamps via FxCacheReader::record_timestamp. Standard Sharpe-time- scaling assumption (trades roughly i.i.d.); imperfect when signals cluster in correlated regimes, but the right ballpark for comparison with industry benchmarks. Output adds per-cost-band "best operating point" tables plus a clear "REALISTIC VERDICT" line at cost=0.125 (half-tick — what a professional desk would actually pay) with three gates: - Sharpe_ann > 2.0 → deployable - 0.5 < Sharpe_ann ≤ 2.0 → marginal - Sharpe_ann ≤ 0.5 → fail at realistic cost The "FRICTIONLESS UPPER BOUND" line reports the intrinsic edge — what the model could theoretically achieve at zero cost. Even if realistic Sharpe fails, this number tells us whether the model has anything to optimise toward at deployment. Co-Authored-By: Claude Opus 4.7 --- .../ml-alpha/examples/phase1d_long_horizon.rs | 100 +++++++++++++----- crates/ml-alpha/src/backtest.rs | 74 +++++++++++-- 2 files changed, 135 insertions(+), 39 deletions(-) diff --git a/crates/ml-alpha/examples/phase1d_long_horizon.rs b/crates/ml-alpha/examples/phase1d_long_horizon.rs index a728ded3b..6554b7a04 100644 --- a/crates/ml-alpha/examples/phase1d_long_horizon.rs +++ b/crates/ml-alpha/examples/phase1d_long_horizon.rs @@ -474,41 +474,85 @@ fn main() -> Result<()> { let prices_t_dev = stream.clone_htod(&prices_t_host)?; let prices_kt_dev = stream.clone_htod(&prices_kt_host)?; let bt = GpuBacktest::from_block(&block)?; - let thresholds: Vec = vec![0.00, 0.02, 0.05, 0.10, 0.15, 0.20, 0.25]; + + // Compute the test-set wall-clock time span for annualisation. + // Use the first and last end-bar timestamps from the val sequences. + // FxCacheReader stores timestamps in nanoseconds. + let first_test_end_bar = + labels.valid_indices[val_pos[stacker_train_n]]; + let last_test_end_bar = + labels.valid_indices[val_pos[val_pos.len() - 1]]; + let first_ts_ns = reader.record_timestamp(first_test_end_bar); + let last_ts_ns = reader.record_timestamp(last_test_end_bar); + let test_time_span_s = ((last_ts_ns - first_ts_ns) as f64) * 1e-9; + println!(); + println!("--- GPU backtest sweep (n_test={}, span={:.2}s ≈ {:.2}h) ---", + stacker_test_n, test_time_span_s, test_time_span_s / 3600.0); + + // Cost sweep: frictionless / quarter-tick / half-tick / 1-tick / 2-tick. + // 0.25 = 1 tick = $12.50/contract round-trip on ES.FUT. + let thresholds: Vec = vec![0.00, 0.05, 0.10, 0.15, 0.20, 0.25]; + let costs: Vec = vec![0.0, 0.0625, 0.125, 0.25, 0.50]; let bt_stats = bt.run( &probs_dev, &prices_t_dev, &prices_kt_dev, - &thresholds, cli.cost_per_trade, + &thresholds, &costs, test_time_span_s, )?; - println!(); - println!("--- GPU backtest sweep (cost={:.4} per round-trip, n_test={}) ---", - cli.cost_per_trade, stacker_test_n); - println!(" {:>5} {:>9} {:>10} {:>10} {:>9} {:>9} {:>12}", - "τ", "n_trades", "mean_ret", "std_ret", "Sharpe", "hit_rate", "total_pnl"); + + println!(" {:>6} {:>5} {:>9} {:>10} {:>10} {:>9} {:>9} {:>10} {:>11} {:>11}", + "cost", "τ", "n_trades", "mean_ret", "std_ret", + "Sharpe", "hit_rate", "trades/yr", "Sharpe_ann", "total_pnl"); for s in &bt_stats { - println!(" {:>5.2} {:>9} {:>10.5} {:>10.5} {:>9.4} {:>9.4} {:>12.2}", - s.threshold, s.n_trades, s.mean_ret, s.std_ret, - s.sharpe_per_trade, s.hit_rate, s.total_pnl); + println!(" {:>6.4} {:>5.2} {:>9} {:>10.5} {:>10.5} {:>9.4} {:>9.4} {:>10.0} {:>11.4} {:>11.2}", + s.cost, s.threshold, s.n_trades, s.mean_ret, s.std_ret, + s.sharpe_per_trade, s.hit_rate, s.trades_per_year, + s.sharpe_annualised, s.total_pnl); } - // Identify best Sharpe (with at least 100 trades for stat power). - let best = bt_stats.iter() - .filter(|s| s.n_trades >= 100) - .max_by(|a, b| a.sharpe_per_trade - .partial_cmp(&b.sharpe_per_trade) - .unwrap_or(std::cmp::Ordering::Equal)); - if let Some(best) = best { - println!(); - println!("BEST OPERATING POINT (Sharpe-maximising, n_trades ≥ 100):"); - println!(" threshold={:.2} n_trades={} mean_ret={:.5} Sharpe={:.4} hit_rate={:.4} total_pnl={:.2}", - best.threshold, best.n_trades, best.mean_ret, - best.sharpe_per_trade, best.hit_rate, best.total_pnl); - if best.sharpe_per_trade > 1.5 { - println!(" BACKTEST GATE PASS: per-trade Sharpe > 1.5 — FoxhuntQ-Δ is deployable."); - } else if best.sharpe_per_trade > 0.5 { - println!(" BACKTEST MARGINAL: 0.5 ≤ Sharpe ≤ 1.5 — tune cost model, threshold, or holding logic."); - } else { - println!(" BACKTEST GATE FAIL: Sharpe < 0.5 — signal exists but costs eat it."); + // Identify best annualised Sharpe per cost band (≥ 100 trades for stat power). + println!(); + println!("BEST OPERATING POINT per cost band (annualised-Sharpe-maximising, n_trades ≥ 100):"); + println!(" {:>6} {:>5} {:>9} {:>11} {:>9} {:>11}", + "cost", "τ", "n_trades", "Sharpe_ann", "hit_rate", "total_pnl"); + for &c in &costs { + let best = bt_stats.iter() + .filter(|s| (s.cost - c).abs() < 1e-6 && s.n_trades >= 100) + .max_by(|a, b| a.sharpe_annualised + .partial_cmp(&b.sharpe_annualised) + .unwrap_or(std::cmp::Ordering::Equal)); + if let Some(b) = best { + println!(" {:>6.4} {:>5.2} {:>9} {:>11.4} {:>9.4} {:>11.2}", + b.cost, b.threshold, b.n_trades, + b.sharpe_annualised, b.hit_rate, b.total_pnl); } } + // Overall verdict at the most realistic professional cost: 0.125 (half-tick). + let realistic_best = bt_stats.iter() + .filter(|s| (s.cost - 0.125).abs() < 1e-6 && s.n_trades >= 100) + .max_by(|a, b| a.sharpe_annualised + .partial_cmp(&b.sharpe_annualised) + .unwrap_or(std::cmp::Ordering::Equal)); + println!(); + if let Some(best) = realistic_best { + println!("REALISTIC VERDICT (cost=0.125, half-tick — professional execution):"); + println!(" threshold={:.2} n_trades={} Sharpe_ann={:.4} hit_rate={:.4}", + best.threshold, best.n_trades, best.sharpe_annualised, best.hit_rate); + if best.sharpe_annualised > 2.0 { + println!(" BACKTEST GATE PASS: annualised Sharpe > 2.0 at half-tick cost — deployable."); + } else if best.sharpe_annualised > 0.5 { + println!(" BACKTEST MARGINAL: 0.5 < Sharpe_ann ≤ 2.0 — improve execution or thresholds."); + } else { + println!(" BACKTEST FAIL at realistic cost: signal exists but doesn't survive half-tick friction."); + } + } + // Frictionless upper bound as a diagnostic. + let frictionless = bt_stats.iter() + .filter(|s| (s.cost - 0.0).abs() < 1e-6 && s.n_trades >= 100) + .max_by(|a, b| a.sharpe_annualised + .partial_cmp(&b.sharpe_annualised) + .unwrap_or(std::cmp::Ordering::Equal)); + if let Some(f) = frictionless { + println!("FRICTIONLESS UPPER BOUND (cost=0.0): Sharpe_ann={:.4} at τ={:.2} ({} trades) — model's intrinsic edge", + f.sharpe_annualised, f.threshold, f.n_trades); + } println!(); println!("--- Stacker stratified accuracy (test half, by Block-S feature) ---"); diff --git a/crates/ml-alpha/src/backtest.rs b/crates/ml-alpha/src/backtest.rs index d813bb351..a01141f15 100644 --- a/crates/ml-alpha/src/backtest.rs +++ b/crates/ml-alpha/src/backtest.rs @@ -27,9 +27,11 @@ use cudarc::driver::{CudaFunction, CudaSlice, CudaStream, LaunchConfig, PushKern use crate::mamba2_block::Mamba2Block; -/// One row of the threshold-sweep table. +/// One row of the (cost × threshold) sweep table. #[derive(Debug, Clone)] pub struct BacktestStats { + /// Round-trip cost in price units for this row. + pub cost: f32, /// The confidence threshold this row corresponds to. pub threshold: f32, /// Number of trades taken across the test set. @@ -38,15 +40,26 @@ pub struct BacktestStats { pub mean_ret: f32, /// Sample standard deviation of per-trade PnL. pub std_ret: f32, - /// Mean / std = per-trade Sharpe (unannualised). Multiply by sqrt(trades/year) - /// for an annualised Sharpe at the caller's discretion. + /// Mean / std = per-trade Sharpe (unannualised). pub sharpe_per_trade: f32, + /// Annualised Sharpe = sharpe_per_trade × sqrt(trades_per_year). + /// Assumes trades are roughly i.i.d. across time — overestimates when + /// overlapping signals are taken; underestimates when the model + /// concentrates trades in correlated regimes. Use as a comparable + /// ballpark, not a deployment forecast. + pub sharpe_annualised: f32, + /// Realised trade rate scaled to a calendar year — gives the + /// annualisation factor context. + pub trades_per_year: f32, /// Fraction of trades with positive PnL after cost. pub hit_rate: f32, /// Total PnL across all trades. pub total_pnl: f32, } +/// Seconds in a calendar year, used for annualisation: 365.25 × 86400. +const SECONDS_PER_YEAR: f64 = 365.25 * 86_400.0; + /// GPU-native backtest evaluator. Holds the kernel handles + a scratch /// allocator on the provided CUDA stream. Reusable across multiple /// `run` calls (e.g. for parameter sweeps). @@ -98,24 +111,51 @@ impl GpuBacktest { }) } - /// Run the threshold sweep. + /// Run the (cost × threshold) sweep. /// /// Arguments: - /// - `probs` : `[N]` stacker sigmoid output ∈ [0, 1] (already on GPU) - /// - `prices_t` : `[N]` mid-prices at sequence end-bar (already on GPU) - /// - `prices_kt` : `[N]` mid-prices at end-bar + horizon (already on GPU) - /// - `thresholds` : `[T]` confidence thresholds (uploaded inside) - /// - `cost_per_trade`: round-trip cost in price units + /// - `probs` : `[N]` stacker sigmoid output ∈ [0, 1] (already on GPU) + /// - `prices_t` : `[N]` mid-prices at sequence end-bar (already on GPU) + /// - `prices_kt` : `[N]` mid-prices at end-bar + horizon (already on GPU) + /// - `thresholds` : `[T]` confidence thresholds (uploaded inside) + /// - `costs` : `[C]` round-trip cost values in price units + /// - `test_time_span_s` : wall-clock seconds spanned by the test set + /// (last end-bar ts − first end-bar ts); used for annualised Sharpe /// - /// Returns `T` `BacktestStats` rows, one per threshold. + /// Returns `C × T` `BacktestStats` rows, in (cost-major, threshold-minor) order. #[allow(clippy::too_many_arguments)] pub fn run( + &self, + probs: &CudaSlice, + prices_t: &CudaSlice, + prices_kt: &CudaSlice, + thresholds: &[f32], + costs: &[f32], + test_time_span_s: f64, + ) -> Result> { + let mut all_stats: Vec = + Vec::with_capacity(costs.len() * thresholds.len()); + for &cost in costs { + let stats = self.run_single_cost( + probs, prices_t, prices_kt, thresholds, cost, test_time_span_s, + )?; + all_stats.extend(stats); + } + Ok(all_stats) + } + + /// Single-cost backtest. The (cost × threshold) sweep wraps this in + /// an outer loop over costs; pulled out so we can keep all kernel + /// launches and allocations local. + #[allow(clippy::too_many_arguments)] + fn run_single_cost( &self, probs: &CudaSlice, prices_t: &CudaSlice, prices_kt: &CudaSlice, thresholds: &[f32], cost_per_trade: f32, + test_time_span_s: f64, ) -> Result> { let n = probs.len() as i32; let t = thresholds.len() as i32; @@ -247,7 +287,6 @@ impl GpuBacktest { let (mean_ret, std_ret, sharpe, hit_rate) = if nt_trades > 0 { let nf = nt_trades as f32; let mean = total / nf; - // var = sum_sq/n - mean^2 (sum_sq over traded bars; non-traded are 0 → contribute 0). let var = (sum_sq_h[ti] / nf - mean * mean).max(0.0); let sd = var.sqrt(); let sh = if sd > 1e-9 { mean / sd } else { 0.0 }; @@ -256,12 +295,25 @@ impl GpuBacktest { } else { (0.0, 0.0, 0.0, 0.0) }; + // Annualisation: trades_per_year = n_trades × (year_seconds / test_span_seconds). + // Annualised Sharpe = per_trade_Sharpe × sqrt(trades_per_year) + // — standard practice for per-event Sharpe (Sharpe-time-scaling). + let (trades_per_year, sharpe_annualised) = if test_time_span_s > 0.0 && nt_trades > 0 { + let tpy = (nt_trades as f64) * (SECONDS_PER_YEAR / test_time_span_s); + let ann = sharpe as f64 * tpy.sqrt(); + (tpy as f32, ann as f32) + } else { + (0.0, 0.0) + }; out.push(BacktestStats { + cost: cost_per_trade, threshold: thresholds[ti], n_trades: nt_trades, mean_ret, std_ret, sharpe_per_trade: sharpe, + sharpe_annualised, + trades_per_year, hit_rate, total_pnl: total, });