foxhunt/scripts/validation/check_tier1.py

#!/usr/bin/env python3
"""Tier 1 convergence exit check (Plan 5 Task 4).

Consumes the aggregated metrics JSON produced by
`scripts/aggregate-multi-seed-metrics.py` (Plan 5 Task 1B.2).

Pass criteria (per spec §2 Tier 1 — convergence):
  1. std/mean ≤ 0.15 on stable epochs (>= warmup_end) for `val_sharpe`,
     `avg_q_value`, `train_loss`.
  2. avg_q_value.max ≤ 500 across all epochs/streams (no fold-1 explosion;
     the fold-1 explosion bug from task #31 produced max_q ~ 1e4+).
  3. Q-saturation placeholder (see A.6 audit — full check landed elsewhere).
  4. Hot-path DtoH placeholder (enforced by pre-commit hook).

Exits 0 on PASS, 1 on FAIL. Each check prints a `PASS:` / `FAIL:` line.
"""
from __future__ import annotations

import argparse
import json
import sys
from statistics import mean


def _stable(series, warmup_end):
    """Return entries with epoch >= warmup_end."""
    return [x for x in series if x["epoch"] >= warmup_end]


def check_std_over_mean(agg, warmup_end):
    """std/mean ≤ 0.15 on stable epochs for the three Tier 1 metrics.

    Defensive: if a metric is missing from the aggregate, return False with
    a clear message so the run can be flagged for re-aggregation rather
    than silently passing.
    """
    for metric in ("val_sharpe", "avg_q_value", "train_loss"):
        series = agg.get(metric)
        if not series:
            return False, f"{metric} missing from aggregate JSON"
        stable = _stable(series, warmup_end)
        if not stable:
            return False, (
                f"No stable epochs for {metric} past warmup_end={warmup_end}"
            )
        m = mean(x["mean"] for x in stable)
        s = mean(x["std"] for x in stable)
        ratio = abs(s / (m + 1e-9)) if abs(m) > 1e-6 else float("inf")
        if ratio > 0.15:
            return False, (
                f"{metric} std/mean = {ratio:.3f} > 0.15 on stable epochs"
            )
    return True, "std/mean <= 0.15 on all Tier 1 metrics"


def check_no_fold1_explosion(agg):
    """avg_q_value.max ≤ 500 — catches the fold-1 loss explosion (task #31)."""
    series = agg.get("avg_q_value", [])
    if not series:
        # Defensive: missing series cannot prove there was no explosion;
        # failing is the safe direction.
        return False, "avg_q_value missing from aggregate JSON"
    max_q = max((x["max"] for x in series), default=0)
    if abs(max_q) > 500:
        return False, f"avg_q_value max={max_q}, possible fold-1 explosion"
    return True, "no fold-1 explosion detected"


def check_no_q_saturation(_agg):
    """Q-saturation placeholder.

    Full per-bin saturation check (q_full / q_half / q_quarter within 1%
    of ±abs_half) is tracked under the A.6 audit rather than embedded in
    the tier1 script — the relevant signal lives in HEALTH_DIAG `mag`
    block and is best validated end-to-end on a real run.
    """
    return True, "no Q-saturation check (placeholder — see A.6 audit)"


def check_no_hot_path_dtoh(_agg):
    """Hot-path DtoH enforcement is a pre-commit responsibility.

    `scripts/gpu-hotpath-guard.sh` rejects any new `DeviceCopy`-style
    DtoH inside the per-step training path. Reaching this script implies
    that guard already passed, so we surface the invariant here as PASS
    rather than re-deriving it from runtime metrics.
    """
    return True, "hot-path audit enforced by pre-commit"


def main():
    ap = argparse.ArgumentParser(
        description="Tier 1 convergence exit check (Plan 5 Task 4).",
    )
    ap.add_argument(
        "metrics_json",
        help="Aggregated metrics JSON (output of aggregate-multi-seed-metrics.py).",
    )
    ap.add_argument(
        "--warmup-end",
        type=int,
        default=15,
        help="Epoch at which warmup ends (stable epochs are > this); default 15.",
    )
    args = ap.parse_args()

    try:
        with open(args.metrics_json) as fh:
            data = json.load(fh)
    except (OSError, json.JSONDecodeError) as exc:
        print(f"FAIL: cannot read {args.metrics_json}: {exc}")
        sys.exit(1)

    agg = data.get("aggregates")
    if not isinstance(agg, dict):
        print("FAIL: missing or malformed `aggregates` block in metrics JSON")
        sys.exit(1)

    checks = [
        check_std_over_mean(agg, args.warmup_end),
        check_no_fold1_explosion(agg),
        check_no_q_saturation(agg),
        check_no_hot_path_dtoh(agg),
    ]

    all_pass = all(ok for ok, _ in checks)
    for ok, msg in checks:
        print(("PASS" if ok else "FAIL") + ": " + msg)
    sys.exit(0 if all_pass else 1)


if __name__ == "__main__":
    main()