feat(fxt-backtest): verdict subcommand wraps emit_deployability_verdict
Adds `fxt-backtest verdict <sweep_dir> --threshold X --windows W1,W2,W3,W4 --training-sha SHA --spec-sha SHA --out deployability_verdict.json` that reads per-cell summary.json files at the realistic (1 tick, 200ms) + stress (1.5 tick, 400ms) anchors against the pre-registered threshold, computes median Sharpe/max_dd/Sortino/profit_factor across windows, classifies into Pass-robust / Pass-nominal / Fail-inconclusive / Fail / Fail-degenerate per spec §3.5, and writes the audit JSON. Adds serde_json workspace dep to fxt-backtest's Cargo.toml (was already a transitive dep but not declared at this layer). End-to-end CLI for Phase 2 runtime is now: argo-train.sh → argo-lob-sweep.sh (smoke / threshold-tuning / deployability) → fxt-backtest aggregate → fxt-backtest verdict → commit deployability_verdict.json.
This commit is contained in:
@@ -20,6 +20,7 @@ ml-core.workspace = true
|
||||
anyhow.workspace = true
|
||||
clap = { workspace = true, features = ["derive", "env"] }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
serde_json.workspace = true
|
||||
serde_yaml.workspace = true
|
||||
tracing.workspace = true
|
||||
tracing-subscriber.workspace = true
|
||||
|
||||
@@ -17,7 +17,7 @@ use clap::{Parser, Subcommand};
|
||||
use ml_alpha::cfc::trunk::CfcConfig;
|
||||
use ml_alpha::heads::N_HORIZONS;
|
||||
use ml_alpha::trainer::perception::{PerceptionTrainer, PerceptionTrainerConfig};
|
||||
use ml_backtesting::aggregate::aggregate_sweep_dir;
|
||||
use ml_backtesting::aggregate::{aggregate_sweep_dir, emit_deployability_verdict};
|
||||
use ml_backtesting::harness::{BacktestHarness, BacktestHarnessConfig};
|
||||
use ml_backtesting::policy::Strategy;
|
||||
use ml_core::device::MlDevice;
|
||||
@@ -43,6 +43,13 @@ enum Cmd {
|
||||
/// All cells run sequentially on the same GPU (single-machine);
|
||||
/// for cluster fan-out use `argo-alpha-sweep.sh` (see infra/argo).
|
||||
Sweep(SweepArgs),
|
||||
/// Emit a deployability verdict from a completed sweep. Reads the
|
||||
/// per-cell summary.json files at the realistic (1.0 tick, 200 ms)
|
||||
/// and stress (1.5 tick, 400 ms) anchors against the supplied
|
||||
/// pre-registered threshold + walk-forward window IDs, classifies
|
||||
/// into Pass-robust / Pass-nominal / Fail-inconclusive / Fail /
|
||||
/// Fail-degenerate per spec §3.5, and writes the audit JSON.
|
||||
Verdict(VerdictArgs),
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
@@ -104,6 +111,32 @@ struct AggregateArgs {
|
||||
sweep_dir: PathBuf,
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
struct VerdictArgs {
|
||||
/// Sweep directory containing one subdirectory per cell (named
|
||||
/// `cell_<NNNN>_cost<C>_lat<L>_th<T>_<W>`), each with a summary.json.
|
||||
sweep_dir: PathBuf,
|
||||
/// Pre-registered threshold (from the threshold-tuning sweep on W0).
|
||||
/// Read from `config/ml/v2_prod_thresholds.json` in production usage.
|
||||
#[arg(long)]
|
||||
threshold: f32,
|
||||
/// Walk-forward window IDs, comma-separated. Typical:
|
||||
/// "W1,W2,W3,W4" matching the 4 held-out quarters in
|
||||
/// `config/ml/sweep_deployability.yaml`.
|
||||
#[arg(long, default_value = "W1,W2,W3,W4")]
|
||||
windows: String,
|
||||
/// Training-run commit SHA (recorded in the audit JSON).
|
||||
#[arg(long)]
|
||||
training_sha: String,
|
||||
/// Spec-revision commit SHA (recorded in the audit JSON).
|
||||
#[arg(long)]
|
||||
spec_sha: String,
|
||||
/// Output JSON path. Convention: `deployability_verdict.json` at
|
||||
/// repo root, committed as audit record.
|
||||
#[arg(long)]
|
||||
out: PathBuf,
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
struct SweepArgs {
|
||||
/// Sweep grid YAML file. See SweepGrid struct in main.rs for the
|
||||
@@ -182,6 +215,31 @@ fn main() -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
Cmd::Sweep(args) => sweep(args),
|
||||
Cmd::Verdict(args) => {
|
||||
let windows: Vec<&str> = args.windows.split(',').map(|s| s.trim()).collect();
|
||||
let verdict = emit_deployability_verdict(
|
||||
&args.sweep_dir,
|
||||
args.threshold,
|
||||
&windows,
|
||||
&args.training_sha,
|
||||
&args.spec_sha,
|
||||
)
|
||||
.context("emit_deployability_verdict")?;
|
||||
let json = serde_json::to_string_pretty(&verdict)
|
||||
.context("serialize deployability_verdict")?;
|
||||
std::fs::write(&args.out, &json)
|
||||
.with_context(|| format!("write {}", args.out.display()))?;
|
||||
println!(
|
||||
"verdict: {:?} (realistic median sharpe = {:.3}, max_dd_pct = {:.3}; stress median sharpe = {:.3}, max_dd_pct = {:.3}) -> {}",
|
||||
verdict.verdict,
|
||||
verdict.realistic.median_sharpe,
|
||||
verdict.realistic.median_max_dd_pct,
|
||||
verdict.stress.median_sharpe,
|
||||
verdict.stress.median_max_dd_pct,
|
||||
args.out.display(),
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user