feat(per-horizon-cfc): plumb kernel-step-trace into fxt-backtest inference path

Mirrors the alpha_train CLI flag (--kernel-step-trace <path>) into fxt-backtest. The PerceptionTrainerConfig already accepts the path; this commit just exposes the CLI surface and propagates through: fxt-backtest run --kernel-step-trace <path> -> BacktestHarnessConfig.kernel_step_trace_path -> PerceptionTrainerConfig.kernel_step_trace_path Sweep YAML gains a `kernel_step_trace: Option<PathBuf>` base field. Argo lob-backtest-sweep-template gains a `kernel-step-trace` workflow parameter (default disabled; when set, --features kernel-step-trace is passed to cargo build AND --kernel-step-trace to fxt-backtest). Gated behind the `kernel-step-trace` Cargo feature (matching alpha_train). When disabled (default), zero overhead. Enables per-step JSONL diagnostic emission from inference kernels -- needed for Smoke 2 deep-dive after sampling histograms (in parallel investigation) localize the per-horizon dynamics bottleneck. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-21 22:26:58 +02:00
parent d2d34b6d4e
commit ff9207e7bb
4 changed files with 105 additions and 50 deletions
--- a/bin/fxt-backtest/Cargo.toml
+++ b/bin/fxt-backtest/Cargo.toml
@@ -13,6 +13,14 @@ keywords.workspace = true
 categories.workspace = true
 description = "Real-LOB GPU backtest CLI (run + sweep aggregator)"

+[features]
+default = []
+# Per-step kernel-state JSONL trace. Proxies through to ml-alpha's
+# `kernel-step-trace` feature so the trainer compiles in the GPU log
+# ring + drain task. Off by default (zero overhead). Enable with
+# `cargo build -p fxt-backtest --features kernel-step-trace`.
+kernel-step-trace = ["ml-alpha/kernel-step-trace"]
+
 [dependencies]
 ml-backtesting = { path = "../../crates/ml-backtesting" }
 ml-alpha = { path = "../../crates/ml-alpha" }
--- a/bin/fxt-backtest/src/main.rs
+++ b/bin/fxt-backtest/src/main.rs
@@ -110,13 +110,22 @@ struct RunArgs {
    /// Output directory; per-cell artifacts written to <out>/cell_NNNN/.
    #[arg(long)]
    out: PathBuf,
-    /// Diagnostic: sample the 5 per-horizon `alpha_probs` every N events
-    /// (post-`forward_step_into`), DtoD into a mapped-pinned buffer, and
-    /// accumulate per-horizon mean / stddev / min / max / 10-bin histogram.
-    /// Emitted at end-of-run as `per_horizon_logit_diag h{N}: ...` for each
-    /// of (h30, h100, h300, h1000, h6000). Default: disabled (zero overhead).
-    /// Recommended diagnostic value: 1000 (≈100 ms overhead on a 2 M-event
-    /// run). Mutually exclusive with stride=0 (rejected at parse time).
+    /// Enable per-step kernel-state JSONL trace to the given file.
+    ///
+    /// Mirrors `alpha_train --kernel-step-trace`. When set, the
+    /// `PerceptionTrainer` constructed by the binary allocates the GPU
+    /// log ring + spawns a drain task that writes one JSONL record per
+    /// kernel emission to this path. Requires the `kernel-step-trace`
+    /// Cargo feature at compile time (proxied through to `ml-alpha`).
+    /// When unset (default), no ring allocated, zero overhead.
+    ///
+    /// Example: `--kernel-step-trace /feature-cache/runs/<sha>/step_trace.jsonl`
+    #[arg(long)]
+    kernel_step_trace: Option<PathBuf>,
+    /// Per-horizon alpha_probs sampling stride for ALPHA Smoke 2
+    /// diagnostic. When set to N, every Nth inference call DtoHs the 5
+    /// alpha_probs and accumulates per-horizon mean/stddev/min/max + 10-bin
+    /// histogram (emitted at end of CRT.diag block).
    #[arg(long)]
    per_horizon_logit_sampling_stride: Option<u32>,
 }
@@ -218,9 +227,18 @@ struct SweepBase {
    /// seeding when |target_signed − effective| < delta_floor. Default 1.0 =
    /// 1 lot (ES single-contract sizing).
    #[serde(default = "default_delta_floor")] delta_floor: f32,
-    /// Diagnostic: per-horizon alpha_probs sampling stride (events). When
-    /// set, each cell emits a `per_horizon_logit_diag h{N}: ...` line per
-    /// horizon at end-of-run. None = disabled (default).
+    /// Optional per-step kernel-state JSONL trace path. Propagated to
+    /// every cell's `PerceptionTrainerConfig.kernel_step_trace_path` +
+    /// `BacktestHarnessConfig.kernel_step_trace_path`. Requires the
+    /// `kernel-step-trace` Cargo feature on the binary at compile time.
+    /// Sweep cells share the same trace file path — useful for single-cell
+    /// diagnostic sweeps; with N>1 cells the trace will interleave records
+    /// from every harness/trainer instance (per-cell isolation must be
+    /// handled by the operator setting distinct paths via a richer schema).
+    #[serde(default)] kernel_step_trace: Option<PathBuf>,
+    /// Per-horizon alpha_probs sampling stride (ALPHA Smoke 2 diagnostic).
+    /// `None` = disabled. When set, every Nth inference call DtoHs the 5
+    /// alpha_probs into a mapped-pinned buffer for per-horizon stats.
    #[serde(default)] per_horizon_logit_sampling_stride: Option<u32>,
 }

@@ -404,9 +422,8 @@ fn sweep(args: SweepArgs) -> Result<()> {
            sharpe_weight_floor:
                cell.sharpe_weight_floor.unwrap_or(grid.base.sharpe_weight_floor),
            out:        cell_out.clone(),
-            per_horizon_logit_sampling_stride:
-                cell.per_horizon_logit_sampling_stride
-                    .or(grid.base.per_horizon_logit_sampling_stride),
+            kernel_step_trace: grid.base.kernel_step_trace.clone(),
+            per_horizon_logit_sampling_stride: grid.base.per_horizon_logit_sampling_stride,
        };
        run(run_args).with_context(|| format!("sweep cell {}", cell.name))?;
        tracing::info!(cell = cell.name.as_str(), "cell complete");
@@ -464,7 +481,12 @@ fn run_batched_cell(

    let dev = MlDevice::cuda(0).context("CUDA device unavailable")?;
    let seed = cell.seed.unwrap_or(base.seed);
-    let trainer_cfg = PerceptionTrainerConfig { seed, n_batch: 1, ..Default::default() };
+    let trainer_cfg = PerceptionTrainerConfig {
+        seed,
+        n_batch: 1,
+        kernel_step_trace_path: base.kernel_step_trace.clone(),
+        ..Default::default()
+    };
    let checkpoint = cell.checkpoint.clone().or_else(|| base.checkpoint.clone());
    let trainer = if let Some(ckpt_path) = &checkpoint {
        PerceptionTrainer::from_checkpoint(&dev, &trainer_cfg, ckpt_path)
@@ -496,8 +518,8 @@ fn run_batched_cell(
        variant_names: Some(names),
        sim_config_override: Some(batched),
        strategies: Vec::new(),          // batched flow uses default policy per backtest
-        per_horizon_logit_sampling_stride: logit_stride,
-        kernel_step_trace_path: None,
+        kernel_step_trace_path: base.kernel_step_trace.clone(),
+        per_horizon_logit_sampling_stride: base.per_horizon_logit_sampling_stride,
    };

    std::fs::create_dir_all(cell_out)
@@ -549,6 +571,7 @@ fn run(args: RunArgs) -> Result<()> {
    let trainer_cfg = PerceptionTrainerConfig {
        n_batch: 1,
        seq_len: 32,
+        kernel_step_trace_path: args.kernel_step_trace.clone(),
        ..Default::default()
    };
    let trainer = if let Some(ckpt_path) = &args.checkpoint {
@@ -587,8 +610,8 @@ fn run(args: RunArgs) -> Result<()> {
        variant_names: None,                // P6: legacy single-cell flow uses cell_NNNN naming
        sim_config_override: None,          // P6: legacy single-cell flow uses from_uniform
        strategies: strategy_grid,
+        kernel_step_trace_path: args.kernel_step_trace.clone(),
        per_horizon_logit_sampling_stride: args.per_horizon_logit_sampling_stride,
-        kernel_step_trace_path: None,
    };

    std::fs::create_dir_all(&args.out)
--- a/crates/ml-backtesting/src/harness.rs
+++ b/crates/ml-backtesting/src/harness.rs
@@ -142,35 +142,30 @@ pub struct BacktestHarnessConfig {
    /// of building from_uniform off the scalar fields above. Length MUST
    /// equal n_parallel when set.
    pub sim_config_override: Option<crate::sim::BatchedSimConfig>,
-    /// Diagnostic: every N events (post-forward_step_into), DtoD the 5
-    /// per-horizon `alpha_probs` into a mapped-pinned buffer and update
-    /// running mean / variance / min / max / 10-bin histogram for each
-    /// of the 5 horizons. None = sampling disabled (zero overhead;
-    /// default). Some(N) emits a `per_horizon_logit_diag h{H}: ...` line
-    /// per horizon at end of run, alongside the existing `crt_diag`
-    /// battery.
+    /// Optional path for the kernel-step-trace JSONL drain. Mirrors the
+    /// alpha_train CLI flag (`--kernel-step-trace <path>`). When `Some`,
+    /// the caller-constructed `PerceptionTrainer` is expected to have
+    /// been built with `PerceptionTrainerConfig.kernel_step_trace_path`
+    /// set to the same path — the harness records the value here for
+    /// audit/equivalence with `BacktestHarnessConfig` snapshots, but
+    /// the trainer is the actual consumer (it owns the GPU log ring +
+    /// drain task lifecycle).
    ///
-    /// Per `feedback_no_htod_htoh_only_mapped_pinned`: the sample buffer
-    /// is allocated once in `BacktestHarness::new` (5×f32 mapped-pinned)
-    /// and reused on every tick. At stride=1000 on 2M events, ~2k DtoD
-    /// stops × ~50µs ≈ 100ms total overhead.
-    ///
-    /// Goal (Smoke 2 follow-up): distinguish whether the observed
-    /// 1.04× uniform `mean_run_len` across horizons is caused by uniform
-    /// per-horizon LOGITS (→ block-diagonal heads needed) or by
-    /// DIFFERENTIATED logits with collapsed downstream computation.
-    pub per_horizon_logit_sampling_stride: Option<u32>,
-    /// Diagnostic mirror of `PerceptionTrainerConfig.kernel_step_trace_path`.
-    /// `Some(path)` records that this backtest was built with the trainer's
-    /// kernel-step-trace JSONL drain pointed at `path`; the harness emits
-    /// a startup `tracing::info!` so post-mortem log scrapes can correlate
-    /// trace files to backtest cells. The trainer is constructed by the
-    /// caller (`bin/fxt-backtest/src/main.rs`) and owns the actual ring +
-    /// drain task; the field here is the contract record. When `None`
-    /// (default), no trace file is referenced and the trainer carries
-    /// zero overhead even if it was built with the `kernel-step-trace`
-    /// Cargo feature compiled in.
+    /// Per `feedback_no_feature_flags`: gated by the compile-time
+    /// `kernel-step-trace` Cargo feature on `ml-alpha`; specific name
+    /// (gated diagnostic code, not a behavioral toggle) justifies it.
+    /// When the feature is off, this field remains storage-only and the
+    /// trainer ignores it.
    pub kernel_step_trace_path: Option<PathBuf>,
+
+    /// Per-horizon alpha_probs sampling stride (added by ALPHA diagnostic
+    /// follow-up at commit d2d34b6d4). `None` = disabled. When set, every Nth
+    /// inference call DtoHs the 5 alpha_probs into a mapped-pinned buffer and
+    /// accumulates per-horizon mean/stddev/min/max + 10-bin histogram for
+    /// emission in the CRT.diag block. Used to localize the Smoke 2 WIN-gate
+    /// failure root cause (is per-horizon LOGIT distribution uniform, or
+    /// differentiated-but-collapsed downstream?).
+    pub per_horizon_logit_sampling_stride: Option<u32>,
 }

 pub struct BacktestHarness {
--- a/infra/k8s/argo/lob-backtest-sweep-template.yaml
+++ b/infra/k8s/argo/lob-backtest-sweep-template.yaml
@@ -87,6 +87,15 @@ spec:
        value: "5"
      - name: max-events
        value: "0"
+      # Per-step kernel-state JSONL trace path. Empty (default) = disabled:
+      # ensure-binary compiles fxt-backtest WITHOUT --features kernel-step-trace
+      # and run-cell omits the --kernel-step-trace flag. When set to a
+      # non-empty path, ensure-binary rebuilds with the feature enabled
+      # and run-cell passes --kernel-step-trace <path> through to the
+      # binary. Trace is written to that absolute path on the pod (must
+      # resolve to a mounted PVC — usually /feature-cache or /mnt/training-data).
+      - name: kernel-step-trace
+        value: ""

  volumes:
    - name: training-data
@@ -181,6 +190,7 @@ spec:
            set -e
            SHA="{{workflow.parameters.commit-sha}}"
            BRANCH="{{workflow.parameters.git-branch}}"
+            KERNEL_TRACE="{{workflow.parameters.kernel-step-trace}}"

            mkdir -p ~/.ssh
            cp /etc/git-ssh/ssh-privatekey ~/.ssh/id_ed25519
@@ -205,16 +215,25 @@ spec:
            SHORT_SHA=$(echo "$SHA" | cut -c1-9)
            echo "Resolved SHA: $SHA (short: $SHORT_SHA)"

-            BIN_DIR="/mnt/training-data/bin/$SHORT_SHA"
+            # Feature variant: when kernel-step-trace is enabled, build
+            # with the Cargo feature and cache under a distinct subdir
+            # so the default + diagnostic binaries don't clobber each other.
+            FEATURE_FLAGS=""
+            VARIANT="default"
+            if [ -n "$KERNEL_TRACE" ]; then
+              FEATURE_FLAGS="--features kernel-step-trace"
+              VARIANT="kstrace"
+            fi
+            BIN_DIR="/mnt/training-data/bin/$SHORT_SHA-$VARIANT"
            mkdir -p "$BIN_DIR"
            if [ -x "$BIN_DIR/fxt-backtest" ]; then
-              echo "=== Cache HIT: fxt-backtest present in $BIN_DIR ==="
+              echo "=== Cache HIT: fxt-backtest ($VARIANT) present in $BIN_DIR ==="
              ls -lh "$BIN_DIR/fxt-backtest"
-              echo "$SHORT_SHA" > /tmp/sha
+              echo "$SHORT_SHA-$VARIANT" > /tmp/sha
              exit 0
            fi

-            echo "=== Cache MISS: compiling fxt-backtest for $SHORT_SHA ==="
+            echo "=== Cache MISS: compiling fxt-backtest ($VARIANT) for $SHORT_SHA ==="
            if [ -d "$BUILD/.git" ]; then
              # Same pattern as alpha-perception-template.yaml: prior
              # cargo build mutates Cargo.lock (and sometimes other
@@ -228,9 +247,9 @@ spec:
              cd "$BUILD"; git checkout "$SHA"
            fi

-            cargo build -p fxt-backtest --release
+            cargo build -p fxt-backtest --release $FEATURE_FLAGS
            cp "$CARGO_TARGET_DIR/release/fxt-backtest" "$BIN_DIR/fxt-backtest"
-            echo "$SHORT_SHA" > /tmp/sha
+            echo "$SHORT_SHA-$VARIANT" > /tmp/sha
            ls -lh "$BIN_DIR/fxt-backtest"

    # ── run-cell: one sweep cell against a GPU pod ─────────────────
@@ -287,6 +306,7 @@ spec:
            CELL="{{inputs.parameters.cell-name}}"
            BIN="/mnt/training-data/bin/$SHA/fxt-backtest"
            CKPT="{{workflow.parameters.checkpoint}}"
+            KERNEL_TRACE="{{workflow.parameters.kernel-step-trace}}"
            OUT="{{workflow.parameters.sweep-root}}/{{workflow.parameters.sweep-tag}}/$CELL"
            mkdir -p "$OUT"
            echo "=== sweep cell $CELL on $(hostname) → $OUT ==="
@@ -295,6 +315,14 @@ spec:
            if [ -n "$CKPT" ]; then
              CKPT_FLAG="--checkpoint $CKPT"
            fi
+            KERNEL_TRACE_FLAG=""
+            if [ -n "$KERNEL_TRACE" ]; then
+              # Per-cell trace path: append cell name so concurrent run-cell
+              # pods don't clobber each other's JSONL files. Operators get
+              # one trace per cell; aggregate by reading <root>/<sweep>/<cell>/
+              # at analysis time.
+              KERNEL_TRACE_FLAG="--kernel-step-trace $OUT/kernel_step_trace.jsonl"
+            fi

            "$BIN" run \
              --data "{{workflow.parameters.data-root}}" \
@@ -308,6 +336,7 @@ spec:
              --max-events "{{inputs.parameters.max-events}}" \
              --seed "{{inputs.parameters.seed}}" \
              $CKPT_FLAG \
+              $KERNEL_TRACE_FLAG \
              --out "$OUT"

            echo "=== cell $CELL done ==="