From ff9207e7bb4294020db42041712a3cf8f4211ebd Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Thu, 21 May 2026 22:26:58 +0200 Subject: [PATCH] feat(per-horizon-cfc): plumb kernel-step-trace into fxt-backtest inference path Mirrors the alpha_train CLI flag (--kernel-step-trace ) into fxt-backtest. The PerceptionTrainerConfig already accepts the path; this commit just exposes the CLI surface and propagates through: fxt-backtest run --kernel-step-trace -> BacktestHarnessConfig.kernel_step_trace_path -> PerceptionTrainerConfig.kernel_step_trace_path Sweep YAML gains a `kernel_step_trace: Option` base field. Argo lob-backtest-sweep-template gains a `kernel-step-trace` workflow parameter (default disabled; when set, --features kernel-step-trace is passed to cargo build AND --kernel-step-trace to fxt-backtest). Gated behind the `kernel-step-trace` Cargo feature (matching alpha_train). When disabled (default), zero overhead. Enables per-step JSONL diagnostic emission from inference kernels -- needed for Smoke 2 deep-dive after sampling histograms (in parallel investigation) localize the per-horizon dynamics bottleneck. Co-Authored-By: Claude Opus 4.7 (1M context) --- bin/fxt-backtest/Cargo.toml | 8 +++ bin/fxt-backtest/src/main.rs | 57 +++++++++++++------ crates/ml-backtesting/src/harness.rs | 49 +++++++--------- .../k8s/argo/lob-backtest-sweep-template.yaml | 41 +++++++++++-- 4 files changed, 105 insertions(+), 50 deletions(-) diff --git a/bin/fxt-backtest/Cargo.toml b/bin/fxt-backtest/Cargo.toml index 956a4431f..cf82f9f24 100644 --- a/bin/fxt-backtest/Cargo.toml +++ b/bin/fxt-backtest/Cargo.toml @@ -13,6 +13,14 @@ keywords.workspace = true categories.workspace = true description = "Real-LOB GPU backtest CLI (run + sweep aggregator)" +[features] +default = [] +# Per-step kernel-state JSONL trace. Proxies through to ml-alpha's +# `kernel-step-trace` feature so the trainer compiles in the GPU log +# ring + drain task. Off by default (zero overhead). Enable with +# `cargo build -p fxt-backtest --features kernel-step-trace`. +kernel-step-trace = ["ml-alpha/kernel-step-trace"] + [dependencies] ml-backtesting = { path = "../../crates/ml-backtesting" } ml-alpha = { path = "../../crates/ml-alpha" } diff --git a/bin/fxt-backtest/src/main.rs b/bin/fxt-backtest/src/main.rs index 84db27edb..3457c3814 100644 --- a/bin/fxt-backtest/src/main.rs +++ b/bin/fxt-backtest/src/main.rs @@ -110,13 +110,22 @@ struct RunArgs { /// Output directory; per-cell artifacts written to /cell_NNNN/. #[arg(long)] out: PathBuf, - /// Diagnostic: sample the 5 per-horizon `alpha_probs` every N events - /// (post-`forward_step_into`), DtoD into a mapped-pinned buffer, and - /// accumulate per-horizon mean / stddev / min / max / 10-bin histogram. - /// Emitted at end-of-run as `per_horizon_logit_diag h{N}: ...` for each - /// of (h30, h100, h300, h1000, h6000). Default: disabled (zero overhead). - /// Recommended diagnostic value: 1000 (≈100 ms overhead on a 2 M-event - /// run). Mutually exclusive with stride=0 (rejected at parse time). + /// Enable per-step kernel-state JSONL trace to the given file. + /// + /// Mirrors `alpha_train --kernel-step-trace`. When set, the + /// `PerceptionTrainer` constructed by the binary allocates the GPU + /// log ring + spawns a drain task that writes one JSONL record per + /// kernel emission to this path. Requires the `kernel-step-trace` + /// Cargo feature at compile time (proxied through to `ml-alpha`). + /// When unset (default), no ring allocated, zero overhead. + /// + /// Example: `--kernel-step-trace /feature-cache/runs//step_trace.jsonl` + #[arg(long)] + kernel_step_trace: Option, + /// Per-horizon alpha_probs sampling stride for ALPHA Smoke 2 + /// diagnostic. When set to N, every Nth inference call DtoHs the 5 + /// alpha_probs and accumulates per-horizon mean/stddev/min/max + 10-bin + /// histogram (emitted at end of CRT.diag block). #[arg(long)] per_horizon_logit_sampling_stride: Option, } @@ -218,9 +227,18 @@ struct SweepBase { /// seeding when |target_signed − effective| < delta_floor. Default 1.0 = /// 1 lot (ES single-contract sizing). #[serde(default = "default_delta_floor")] delta_floor: f32, - /// Diagnostic: per-horizon alpha_probs sampling stride (events). When - /// set, each cell emits a `per_horizon_logit_diag h{N}: ...` line per - /// horizon at end-of-run. None = disabled (default). + /// Optional per-step kernel-state JSONL trace path. Propagated to + /// every cell's `PerceptionTrainerConfig.kernel_step_trace_path` + + /// `BacktestHarnessConfig.kernel_step_trace_path`. Requires the + /// `kernel-step-trace` Cargo feature on the binary at compile time. + /// Sweep cells share the same trace file path — useful for single-cell + /// diagnostic sweeps; with N>1 cells the trace will interleave records + /// from every harness/trainer instance (per-cell isolation must be + /// handled by the operator setting distinct paths via a richer schema). + #[serde(default)] kernel_step_trace: Option, + /// Per-horizon alpha_probs sampling stride (ALPHA Smoke 2 diagnostic). + /// `None` = disabled. When set, every Nth inference call DtoHs the 5 + /// alpha_probs into a mapped-pinned buffer for per-horizon stats. #[serde(default)] per_horizon_logit_sampling_stride: Option, } @@ -404,9 +422,8 @@ fn sweep(args: SweepArgs) -> Result<()> { sharpe_weight_floor: cell.sharpe_weight_floor.unwrap_or(grid.base.sharpe_weight_floor), out: cell_out.clone(), - per_horizon_logit_sampling_stride: - cell.per_horizon_logit_sampling_stride - .or(grid.base.per_horizon_logit_sampling_stride), + kernel_step_trace: grid.base.kernel_step_trace.clone(), + per_horizon_logit_sampling_stride: grid.base.per_horizon_logit_sampling_stride, }; run(run_args).with_context(|| format!("sweep cell {}", cell.name))?; tracing::info!(cell = cell.name.as_str(), "cell complete"); @@ -464,7 +481,12 @@ fn run_batched_cell( let dev = MlDevice::cuda(0).context("CUDA device unavailable")?; let seed = cell.seed.unwrap_or(base.seed); - let trainer_cfg = PerceptionTrainerConfig { seed, n_batch: 1, ..Default::default() }; + let trainer_cfg = PerceptionTrainerConfig { + seed, + n_batch: 1, + kernel_step_trace_path: base.kernel_step_trace.clone(), + ..Default::default() + }; let checkpoint = cell.checkpoint.clone().or_else(|| base.checkpoint.clone()); let trainer = if let Some(ckpt_path) = &checkpoint { PerceptionTrainer::from_checkpoint(&dev, &trainer_cfg, ckpt_path) @@ -496,8 +518,8 @@ fn run_batched_cell( variant_names: Some(names), sim_config_override: Some(batched), strategies: Vec::new(), // batched flow uses default policy per backtest - per_horizon_logit_sampling_stride: logit_stride, - kernel_step_trace_path: None, + kernel_step_trace_path: base.kernel_step_trace.clone(), + per_horizon_logit_sampling_stride: base.per_horizon_logit_sampling_stride, }; std::fs::create_dir_all(cell_out) @@ -549,6 +571,7 @@ fn run(args: RunArgs) -> Result<()> { let trainer_cfg = PerceptionTrainerConfig { n_batch: 1, seq_len: 32, + kernel_step_trace_path: args.kernel_step_trace.clone(), ..Default::default() }; let trainer = if let Some(ckpt_path) = &args.checkpoint { @@ -587,8 +610,8 @@ fn run(args: RunArgs) -> Result<()> { variant_names: None, // P6: legacy single-cell flow uses cell_NNNN naming sim_config_override: None, // P6: legacy single-cell flow uses from_uniform strategies: strategy_grid, + kernel_step_trace_path: args.kernel_step_trace.clone(), per_horizon_logit_sampling_stride: args.per_horizon_logit_sampling_stride, - kernel_step_trace_path: None, }; std::fs::create_dir_all(&args.out) diff --git a/crates/ml-backtesting/src/harness.rs b/crates/ml-backtesting/src/harness.rs index ce6159f26..c94e9627b 100644 --- a/crates/ml-backtesting/src/harness.rs +++ b/crates/ml-backtesting/src/harness.rs @@ -142,35 +142,30 @@ pub struct BacktestHarnessConfig { /// of building from_uniform off the scalar fields above. Length MUST /// equal n_parallel when set. pub sim_config_override: Option, - /// Diagnostic: every N events (post-forward_step_into), DtoD the 5 - /// per-horizon `alpha_probs` into a mapped-pinned buffer and update - /// running mean / variance / min / max / 10-bin histogram for each - /// of the 5 horizons. None = sampling disabled (zero overhead; - /// default). Some(N) emits a `per_horizon_logit_diag h{H}: ...` line - /// per horizon at end of run, alongside the existing `crt_diag` - /// battery. + /// Optional path for the kernel-step-trace JSONL drain. Mirrors the + /// alpha_train CLI flag (`--kernel-step-trace `). When `Some`, + /// the caller-constructed `PerceptionTrainer` is expected to have + /// been built with `PerceptionTrainerConfig.kernel_step_trace_path` + /// set to the same path — the harness records the value here for + /// audit/equivalence with `BacktestHarnessConfig` snapshots, but + /// the trainer is the actual consumer (it owns the GPU log ring + + /// drain task lifecycle). /// - /// Per `feedback_no_htod_htoh_only_mapped_pinned`: the sample buffer - /// is allocated once in `BacktestHarness::new` (5×f32 mapped-pinned) - /// and reused on every tick. At stride=1000 on 2M events, ~2k DtoD - /// stops × ~50µs ≈ 100ms total overhead. - /// - /// Goal (Smoke 2 follow-up): distinguish whether the observed - /// 1.04× uniform `mean_run_len` across horizons is caused by uniform - /// per-horizon LOGITS (→ block-diagonal heads needed) or by - /// DIFFERENTIATED logits with collapsed downstream computation. - pub per_horizon_logit_sampling_stride: Option, - /// Diagnostic mirror of `PerceptionTrainerConfig.kernel_step_trace_path`. - /// `Some(path)` records that this backtest was built with the trainer's - /// kernel-step-trace JSONL drain pointed at `path`; the harness emits - /// a startup `tracing::info!` so post-mortem log scrapes can correlate - /// trace files to backtest cells. The trainer is constructed by the - /// caller (`bin/fxt-backtest/src/main.rs`) and owns the actual ring + - /// drain task; the field here is the contract record. When `None` - /// (default), no trace file is referenced and the trainer carries - /// zero overhead even if it was built with the `kernel-step-trace` - /// Cargo feature compiled in. + /// Per `feedback_no_feature_flags`: gated by the compile-time + /// `kernel-step-trace` Cargo feature on `ml-alpha`; specific name + /// (gated diagnostic code, not a behavioral toggle) justifies it. + /// When the feature is off, this field remains storage-only and the + /// trainer ignores it. pub kernel_step_trace_path: Option, + + /// Per-horizon alpha_probs sampling stride (added by ALPHA diagnostic + /// follow-up at commit d2d34b6d4). `None` = disabled. When set, every Nth + /// inference call DtoHs the 5 alpha_probs into a mapped-pinned buffer and + /// accumulates per-horizon mean/stddev/min/max + 10-bin histogram for + /// emission in the CRT.diag block. Used to localize the Smoke 2 WIN-gate + /// failure root cause (is per-horizon LOGIT distribution uniform, or + /// differentiated-but-collapsed downstream?). + pub per_horizon_logit_sampling_stride: Option, } pub struct BacktestHarness { diff --git a/infra/k8s/argo/lob-backtest-sweep-template.yaml b/infra/k8s/argo/lob-backtest-sweep-template.yaml index 71573053f..4d73ae319 100644 --- a/infra/k8s/argo/lob-backtest-sweep-template.yaml +++ b/infra/k8s/argo/lob-backtest-sweep-template.yaml @@ -87,6 +87,15 @@ spec: value: "5" - name: max-events value: "0" + # Per-step kernel-state JSONL trace path. Empty (default) = disabled: + # ensure-binary compiles fxt-backtest WITHOUT --features kernel-step-trace + # and run-cell omits the --kernel-step-trace flag. When set to a + # non-empty path, ensure-binary rebuilds with the feature enabled + # and run-cell passes --kernel-step-trace through to the + # binary. Trace is written to that absolute path on the pod (must + # resolve to a mounted PVC — usually /feature-cache or /mnt/training-data). + - name: kernel-step-trace + value: "" volumes: - name: training-data @@ -181,6 +190,7 @@ spec: set -e SHA="{{workflow.parameters.commit-sha}}" BRANCH="{{workflow.parameters.git-branch}}" + KERNEL_TRACE="{{workflow.parameters.kernel-step-trace}}" mkdir -p ~/.ssh cp /etc/git-ssh/ssh-privatekey ~/.ssh/id_ed25519 @@ -205,16 +215,25 @@ spec: SHORT_SHA=$(echo "$SHA" | cut -c1-9) echo "Resolved SHA: $SHA (short: $SHORT_SHA)" - BIN_DIR="/mnt/training-data/bin/$SHORT_SHA" + # Feature variant: when kernel-step-trace is enabled, build + # with the Cargo feature and cache under a distinct subdir + # so the default + diagnostic binaries don't clobber each other. + FEATURE_FLAGS="" + VARIANT="default" + if [ -n "$KERNEL_TRACE" ]; then + FEATURE_FLAGS="--features kernel-step-trace" + VARIANT="kstrace" + fi + BIN_DIR="/mnt/training-data/bin/$SHORT_SHA-$VARIANT" mkdir -p "$BIN_DIR" if [ -x "$BIN_DIR/fxt-backtest" ]; then - echo "=== Cache HIT: fxt-backtest present in $BIN_DIR ===" + echo "=== Cache HIT: fxt-backtest ($VARIANT) present in $BIN_DIR ===" ls -lh "$BIN_DIR/fxt-backtest" - echo "$SHORT_SHA" > /tmp/sha + echo "$SHORT_SHA-$VARIANT" > /tmp/sha exit 0 fi - echo "=== Cache MISS: compiling fxt-backtest for $SHORT_SHA ===" + echo "=== Cache MISS: compiling fxt-backtest ($VARIANT) for $SHORT_SHA ===" if [ -d "$BUILD/.git" ]; then # Same pattern as alpha-perception-template.yaml: prior # cargo build mutates Cargo.lock (and sometimes other @@ -228,9 +247,9 @@ spec: cd "$BUILD"; git checkout "$SHA" fi - cargo build -p fxt-backtest --release + cargo build -p fxt-backtest --release $FEATURE_FLAGS cp "$CARGO_TARGET_DIR/release/fxt-backtest" "$BIN_DIR/fxt-backtest" - echo "$SHORT_SHA" > /tmp/sha + echo "$SHORT_SHA-$VARIANT" > /tmp/sha ls -lh "$BIN_DIR/fxt-backtest" # ── run-cell: one sweep cell against a GPU pod ───────────────── @@ -287,6 +306,7 @@ spec: CELL="{{inputs.parameters.cell-name}}" BIN="/mnt/training-data/bin/$SHA/fxt-backtest" CKPT="{{workflow.parameters.checkpoint}}" + KERNEL_TRACE="{{workflow.parameters.kernel-step-trace}}" OUT="{{workflow.parameters.sweep-root}}/{{workflow.parameters.sweep-tag}}/$CELL" mkdir -p "$OUT" echo "=== sweep cell $CELL on $(hostname) → $OUT ===" @@ -295,6 +315,14 @@ spec: if [ -n "$CKPT" ]; then CKPT_FLAG="--checkpoint $CKPT" fi + KERNEL_TRACE_FLAG="" + if [ -n "$KERNEL_TRACE" ]; then + # Per-cell trace path: append cell name so concurrent run-cell + # pods don't clobber each other's JSONL files. Operators get + # one trace per cell; aggregate by reading /// + # at analysis time. + KERNEL_TRACE_FLAG="--kernel-step-trace $OUT/kernel_step_trace.jsonl" + fi "$BIN" run \ --data "{{workflow.parameters.data-root}}" \ @@ -308,6 +336,7 @@ spec: --max-events "{{inputs.parameters.max-events}}" \ --seed "{{inputs.parameters.seed}}" \ $CKPT_FLAG \ + $KERNEL_TRACE_FLAG \ --out "$OUT" echo "=== cell $CELL done ==="