foxhunt/crates/ml-alpha/tests/eval_diag_emission.rs

//! Phase A schema-parity test (2026-05-31 checkpoints+eval-diag plan).
//!
//! Validates the contract of `IntegratedTrainer::build_diag_value`:
//! the per-step record emitted in the EVAL phase has the SAME 643-leaf
//! schema (642 scalars + 1 boolean — `jq paths(scalars)` only counts
//! the scalars) as the TRAIN phase. Pre-implementation, eval emitted ZERO
//! records — see cluster run alpha-rl-8ll7j (+$61k pnl, -$444k DD,
//! eval trajectory invisible). Post-implementation, `eval_diag.jsonl`
//! lines parse with a leaf-path set identical to `diag.jsonl`.
//!
//! Run with:
//!   `cargo test -p ml-alpha --test eval_diag_emission -- --ignored --nocapture`
//!
//! Marked `#[ignore]` because it requires:
//!   * CUDA device 0 (RTX 3050 Ti locally / L40S on cluster)
//!   * Pre-built `target/release/examples/alpha_rl_train` binary
//!   * MBP-10 test data with predecoded sidecars at the path below
//!
//! Per `feedback_no_cpu_test_fallbacks`: this test runs the GPU
//! oracle binary end-to-end. There is no host-side reimplementation
//! of `build_diag_value` to cross-check against.

use anyhow::{Context, Result};
use serde_json::Value;
use std::collections::BTreeSet;
use std::path::{Path, PathBuf};
use std::process::Command;

/// Walk a JSON value and collect dotted paths to every leaf.
/// Includes scalars (Number, String, Bool, Null) — array indices are
/// part of the path so positional drift is caught. Unlike jq's
/// `paths(scalars)` this does NOT drop falsy values, so the count is
/// stable across step-to-step value changes (the schema is what we
/// care about, not which fields happen to be truthy on a given step).
fn collect_leaf_paths(v: &Value, prefix: &str, out: &mut BTreeSet<String>) {
    match v {
        Value::Object(map) => {
            for (k, child) in map {
                let next = if prefix.is_empty() {
                    k.clone()
                } else {
                    format!("{prefix}.{k}")
                };
                collect_leaf_paths(child, &next, out);
            }
        }
        Value::Array(arr) => {
            for (i, child) in arr.iter().enumerate() {
                let next = if prefix.is_empty() {
                    i.to_string()
                } else {
                    format!("{prefix}.{i}")
                };
                collect_leaf_paths(child, &next, out);
            }
        }
        // Every scalar leaf — boolean, number, string, null all count.
        Value::Null
        | Value::Bool(_)
        | Value::Number(_)
        | Value::String(_) => {
            out.insert(prefix.to_string());
        }
    }
}

fn first_line(path: &Path) -> Result<String> {
    let s = std::fs::read_to_string(path)
        .with_context(|| format!("read {}", path.display()))?;
    let first = s
        .lines()
        .next()
        .with_context(|| format!("{} has no lines", path.display()))?;
    Ok(first.to_string())
}

fn line_count(path: &Path) -> Result<usize> {
    let s = std::fs::read_to_string(path)
        .with_context(|| format!("read {}", path.display()))?;
    Ok(s.lines().count())
}

fn binary_path() -> PathBuf {
    // CARGO_MANIFEST_DIR = crates/ml-alpha
    let crate_root = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
    crate_root
        .parent()
        .and_then(|p| p.parent())
        .map(|root| root.join("target/release/examples/alpha_rl_train"))
        .unwrap_or_else(|| PathBuf::from("target/release/examples/alpha_rl_train"))
}

fn data_dir() -> PathBuf {
    // Allow override via env (matches the FOXHUNT_TEST_DATA pattern in
    // memory.md), default to the foxhunt-convention test data layout
    // rooted at the workspace root (two parents up from CARGO_MANIFEST_DIR =
    // `crates/ml-alpha`). ES.FUT subdir holds the all-instrument
    // predecoded files; pair with --instrument-mode=all for consistency.
    if let Ok(p) = std::env::var("FOXHUNT_EVAL_DIAG_DATA") {
        return PathBuf::from(p);
    }
    let crate_root = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
    crate_root
        .parent()
        .and_then(|p| p.parent())
        .map(|root| root.join("test_data/futures-baseline/ES.FUT"))
        .unwrap_or_else(|| PathBuf::from("test_data/futures-baseline/ES.FUT"))
}

#[test]
#[ignore = "requires CUDA + pre-built release binary + MBP-10 test data"]
fn eval_diag_jsonl_emitted_with_train_schema_parity() -> Result<()> {
    let bin = binary_path();
    anyhow::ensure!(
        bin.exists(),
        "binary not found at {} — run `SQLX_OFFLINE=true cargo build --release \
         --example alpha_rl_train -p ml-alpha` first",
        bin.display()
    );
    let data = data_dir();
    anyhow::ensure!(
        data.exists(),
        "test data dir not found at {} — set FOXHUNT_EVAL_DIAG_DATA to override",
        data.display()
    );

    let out = std::env::temp_dir().join("foxhunt-eval-diag-emission-test");
    if out.exists() {
        std::fs::remove_dir_all(&out).context("rm -rf out")?;
    }
    std::fs::create_dir_all(&out).context("mkdir -p out")?;

    // Run a minimal 100-step train + 50-step eval smoke.
    //
    // 2026-06-01 (B-8 followup): migrated from fold-idx=0/n_folds=2 to
    // fold-idx=1/n_folds=3. The original config split 9 files as 4+4,
    // and the first 4 didn't satisfy the loader's 1033-snapshot minimum
    // ("no files had enough snapshots across 4 input files"). The 6+3
    // split (fold 1 of 3) is the same config used by the B-6 invariant
    // test and reliably loads. The old comment ("n_folds=2 because local
    // test_data only has 2 MBP-10-bearing files") was stale — test_data
    // now ships 9 files.
    let n_steps: usize = 100;
    let n_eval_steps: usize = 50;
    let status = Command::new(&bin)
        .args([
            "--n-steps", &n_steps.to_string(),
            "--n-eval-steps", &n_eval_steps.to_string(),
            "--fold-idx", "1",
            "--n-folds", "3",
            "--mbp10-data-dir", &data.display().to_string(),
            "--predecoded-dir", &data.display().to_string(),
            "--out", &out.display().to_string(),
            "--instrument-mode", "all",
            "--n-backtests", "16",
            "--log-every", "50",
            "--seed", "42",
        ])
        .env("SQLX_OFFLINE", "true")
        .status()
        .context("spawn alpha_rl_train")?;
    anyhow::ensure!(status.success(), "alpha_rl_train exited with {status}");

    // ── Existence + line counts ─────────────────────────────────────
    let diag_path = out.join("diag.jsonl");
    let eval_diag_path = out.join("eval_diag.jsonl");
    anyhow::ensure!(diag_path.exists(), "missing {}", diag_path.display());
    anyhow::ensure!(
        eval_diag_path.exists(),
        "missing {} — Phase A eval-diag emission did not fire",
        eval_diag_path.display()
    );
    let n_train = line_count(&diag_path)?;
    let n_eval = line_count(&eval_diag_path)?;
    anyhow::ensure!(
        n_train == n_steps,
        "diag.jsonl line count {n_train} != n_steps {n_steps}"
    );
    anyhow::ensure!(
        n_eval == n_eval_steps,
        "eval_diag.jsonl line count {n_eval} != n_eval_steps {n_eval_steps}"
    );

    // ── Leaf-path parity ────────────────────────────────────────────
    let train_first: Value = serde_json::from_str(&first_line(&diag_path)?)
        .context("parse train first line")?;
    let eval_first: Value = serde_json::from_str(&first_line(&eval_diag_path)?)
        .context("parse eval first line")?;
    let mut train_paths = BTreeSet::new();
    let mut eval_paths = BTreeSet::new();
    collect_leaf_paths(&train_first, "", &mut train_paths);
    collect_leaf_paths(&eval_first, "", &mut eval_paths);

    let only_in_train: Vec<_> = train_paths.difference(&eval_paths).collect();
    let only_in_eval: Vec<_> = eval_paths.difference(&train_paths).collect();
    if !only_in_train.is_empty() || !only_in_eval.is_empty() {
        let summary = format!(
            "schema parity FAILED\n  only in train ({}): {:?}\n  only in eval ({}): {:?}",
            only_in_train.len(),
            only_in_train.iter().take(20).collect::<Vec<_>>(),
            only_in_eval.len(),
            only_in_eval.iter().take(20).collect::<Vec<_>>(),
        );
        anyhow::bail!("{summary}");
    }

    // ── Baseline-cardinality check ───────────────────────────────────
    // 642-leaf schema documented in `/tmp/diag_keys_baseline.txt` (jq
    // `paths(scalars)` semantics — falsy-leaf-dropping). This test
    // counts ALL leaves including booleans + nulls, so the expected
    // value is 643 (the +1 is `pyramid.max_units_reached`, a bool the
    // jq baseline drops when false). If the schema drifts
    // (intentional + plan-doc-blessed), update EXPECTED_LEAVES.
    //
    // 2026-05-31 addendum: +1 leaf for `isv_outputs.reward_scale_warmed_flag`
    // (eval-boundary boot_floor decoupling). New total: 644.
    // 2026-05-31 B-2: +3 leaves for pos_max_ema cold_start/growth_cap_base/cv_gain
    // (ISV-driven cold-start redesign). New total: 647.
    // 2026-05-31 B-3: +1 leaf for kelly_bootstrap_floor (fractional-trust schedule).
    // New total: 648.
    // 2026-06-01 B-4: +1 leaf for wr_ema_max_delta (Hoeffding-bound additive cap).
    // New total: 649. B-5 reused slot 721 (renamed wr_ema_max_delta → ema_alpha_slow,
    // diag leaf renamed; count unchanged).
    // 2026-06-01 B-6: +2 leaves for ema_trust_full_threshold + ema_cv_gain
    // (Bayesian shrinkage adaptive trust schedule + Welford-CV gain).
    // New total: 651.
    // 2026-06-01 B-7: +1 leaf for realized_pnl_cum_usd (raw-reward-based
    // cumulative pnl counter — exposes reward-clamp truncation gap when it
    // diverges from pnl_cum_usd). New total: 652.
    // 2026-06-01 B-8: +1 leaf for popart.sigma_welford (Welford-only σ
    // before F4 envelope floor; disaggregates σ shocks under B-7).
    // New total: 653.
    // 2026-06-01 B-9: +4 leaves for C51 Bellman-target saturation
    // observability under risk_stack.atom_calibration.{target_top_saturation_rate,
    // target_bot_saturation_rate, target_max_pre_proj, target_min_pre_proj}.
    // New total: 657.
    // 2026-06-01 B-10: +14 leaves under policy_diagnostic — 13 from new
    // slots 730-742 (G1 Q-distribution informativeness, G2 Q-distill
    // effectiveness, G3 PPO advantage normalization, G4 PPO surrogate
    // decomposition) + 1 from exposing existing slot 407 (rl_q_pi_agree_b
    // canonical Q-collapse discriminator that was writing-but-not-diagged).
    // Post-B-10: 671. CMDP fleet-fraction diagnostic adds 4 leaves
    // (frac_in_cooldown / frac_consec_near_limit / frac_dd_triggered /
    // frac_session_neg) → 675. Edge-decay Phase 1 adds 3 leaves
    // (edge_ph_mean / edge_ph_frac_alerted / edge_ph_frac_warmup) → 678.
    // Reward-policy realignment adds 1 leaf (rewards.pure_pnl_mode) → 679.
    // 2026-06-02 Determinism Phase 1: +15 leaves under top-level
    // `checksums` key (encoder_output, q_logits, pi_logits, v_value,
    // replay_sample_indices, rewards_after_shape, advantages, q_grad,
    // pi_grad, v_grad, encoder_grad, adam_m_sum, adam_v_sum, isv_state,
    // popart_sigma_state). New total: 694.
    // 2026-06-02 Determinism Phase 2.2: +17 leaves under `checksums`
    // for encoder localisation — mamba2_l1 weights (w_in/w_a/w_b/w_c) +
    // mamba2_l2 weights (w_in/w_c) + vsn_w + cfc_w_in + cfc_w_rec +
    // attn_q + encoder activations (vsn_out, mamba2_l1_out, ln_a_out,
    // mamba2_l2_out, ln_b_out, attn_context) + mamba2_grad_w_c_l1.
    // New total: 711.
    const EXPECTED_LEAVES: usize = 711;
    anyhow::ensure!(
        train_paths.len() == EXPECTED_LEAVES,
        "train leaf count {} != expected {EXPECTED_LEAVES} (schema drifted; \
         update EXPECTED_LEAVES in this test + the plan/spec)",
        train_paths.len()
    );

    // ── Eval step axis monotonicity ──────────────────────────────────
    // Verify the eval phase emits a step axis that continues past the
    // training horizon: eval[0].step == n_steps,
    // eval[N-1].step == n_steps + n_eval_steps - 1. Catches a regression
    // where the eval loop reused the training step counter (0..n_eval).
    let eval_lines: Vec<Value> = std::fs::read_to_string(&eval_diag_path)
        .with_context(|| format!("read {}", eval_diag_path.display()))?
        .lines()
        .map(|l| serde_json::from_str(l).expect("parse eval line"))
        .collect();
    anyhow::ensure!(
        eval_lines.len() == n_eval_steps,
        "eval_diag.jsonl parsed {} lines != n_eval_steps {n_eval_steps}",
        eval_lines.len()
    );
    let first_step = eval_lines
        .first()
        .unwrap()
        .get("step")
        .and_then(|v| v.as_u64())
        .context("eval[0].step missing or not u64")?;
    let last_step = eval_lines
        .last()
        .unwrap()
        .get("step")
        .and_then(|v| v.as_u64())
        .context("eval[N-1].step missing or not u64")?;
    let expected_first = n_steps as u64;
    let expected_last = (n_steps + n_eval_steps - 1) as u64;
    anyhow::ensure!(
        first_step == expected_first,
        "eval[0].step expected n_steps={expected_first}, got {first_step}"
    );
    anyhow::ensure!(
        last_step == expected_last,
        "eval[N-1].step expected n_steps+n_eval-1={expected_last}, got {last_step}"
    );

    eprintln!(
        "eval_diag_emission OK: train={} eval={} leaves (both files), {} train + {} eval lines",
        train_paths.len(),
        eval_paths.len(),
        n_train,
        n_eval,
    );
    Ok(())
}