foxhunt/scripts/audit-diag.sh

#!/usr/bin/env bash
# audit-diag.sh — enforce "every observable surfaces in diag JSONL"
# per SP20 §0.3.
#
# Runs a local 100-step smoke (small footprint, b_size=4, predecoded
# test data) and parses the produced diag JSONL. For each entry in
# scripts/audit-manifest/diag-fields.txt: verify the jq-syntax path
# resolves to a non-null, non-NaN value in at least one sampled row.
#
# Exit codes:
#   0  — zero missing fields
#   1  — one or more missing fields
#   2  — usage / smoke-run failure

set -euo pipefail

ROOT=$(git rev-parse --show-toplevel)
MANIFEST="$ROOT/scripts/audit-manifest/diag-fields.txt"
BINARY="$ROOT/target/release/examples/alpha_rl_train"
SMOKE_OUT_DIR="${TMPDIR:-/tmp}/audit-diag-smoke-$$"
DIAG_FILE="$SMOKE_OUT_DIR/diag.jsonl"

if ! command -v jq >/dev/null 2>&1; then
  echo "ERROR: jq required (apt install jq)" >&2
  exit 2
fi

if [[ ! -f "$MANIFEST" ]]; then
  echo "ERROR: manifest not found: $MANIFEST" >&2
  exit 2
fi

mapfile -t fields < <(grep -vE '^\s*(#|$)' "$MANIFEST")

if [[ ${#fields[@]} -eq 0 ]]; then
  echo "audit-diag: manifest empty — no diag fields to audit. PASS."
  exit 0
fi

# ── Build binary if missing/stale ─────────────────────────────
if [[ ! -x "$BINARY" || "$BINARY" -ot "$ROOT/crates/ml-alpha/src/trainer/integrated.rs" ]]; then
  echo "audit-diag: building alpha_rl_train release binary..."
  (cd "$ROOT" && SQLX_OFFLINE=true cargo build -p ml-alpha --example alpha_rl_train --release 2>&1 | tail -5)
fi

# ── Local smoke: 100 steps, b_size=4, single fold, single seed ───
mkdir -p "$SMOKE_OUT_DIR"
echo "audit-diag: running 100-step local smoke (b_size=4) → $SMOKE_OUT_DIR"

# Inputs — use test-data MBP-10 baseline
TEST_DATA="${FOXHUNT_TEST_DATA:-$ROOT/test_data/futures-baseline}"
if [[ ! -d "$TEST_DATA/ES.FUT" ]]; then
  echo "ERROR: test data missing: $TEST_DATA/ES.FUT" >&2
  echo "Set FOXHUNT_TEST_DATA env var to a directory containing ES.FUT/ subdirectory" >&2
  exit 2
fi

PREDECODED="$TEST_DATA/predecoded"
if [[ ! -d "$PREDECODED" ]]; then
  PREDECODED="$TEST_DATA"
fi

"$BINARY" \
  --mbp10-data-dir "$TEST_DATA/ES.FUT" \
  --trades-data-dir "$TEST_DATA/ES.FUT" \
  --predecoded-dir "$PREDECODED" \
  --output-dir "$SMOKE_OUT_DIR" \
  --n-steps 100 \
  --n-backtests 4 \
  --seq-len 32 \
  --per-capacity 256 \
  --seed 16962 \
  --instrument-mode front-month \
  --n-folds 1 \
  --fold-idx 0 \
  --n-eval-steps 0 \
  > "$SMOKE_OUT_DIR/stdout.log" 2> "$SMOKE_OUT_DIR/stderr.log" \
  || { echo "ERROR: local smoke failed; see $SMOKE_OUT_DIR/stderr.log"; tail -20 "$SMOKE_OUT_DIR/stderr.log"; exit 2; }

if [[ ! -f "$DIAG_FILE" ]]; then
  echo "ERROR: smoke completed but diag.jsonl not produced at $DIAG_FILE" >&2
  exit 2
fi

n_rows=$(wc -l < "$DIAG_FILE")
echo "audit-diag: smoke produced $n_rows diag rows"

if [[ $n_rows -lt 10 ]]; then
  echo "ERROR: too few diag rows ($n_rows); smoke may have crashed early" >&2
  exit 2
fi

# Sample 3 rows: first, middle, last
mid_row=$((n_rows / 2))
last_row=$n_rows
sample_rows=(1 $mid_row $last_row)

# ── Verify each manifest field present + non-trivial ─────────
violations=0
echo
echo "audit-diag: validating ${#fields[@]} field path(s)..."
for field in "${fields[@]}"; do
  found_any=0
  bad_any=0
  for r in "${sample_rows[@]}"; do
    val=$(sed -n "${r}p" "$DIAG_FILE" | jq -c "$field // empty" 2>/dev/null || echo "<jq-error>")
    if [[ "$val" == "<jq-error>" ]]; then
      bad_any=1
      continue
    fi
    if [[ -n "$val" && "$val" != "null" ]]; then
      # Check for NaN (jq emits null for NaN typically; double-check)
      if [[ "$val" != *"NaN"* && "$val" != *"nan"* ]]; then
        found_any=1
      fi
    fi
  done
  if [[ $found_any -eq 0 ]]; then
    echo "  VIOLATION field $field: not present (or null/NaN) in sampled rows"
    violations=$((violations + 1))
  fi
done

echo
echo "audit-diag: $violations missing field(s)"

if [[ $violations -gt 0 ]]; then
  echo
  echo "Per SP20 §0.3: every observable state MUST be in diag JSONL same commit as the mechanic."
  echo "Add the field to the diag JSONL emission in crates/ml-alpha/examples/alpha_rl_train.rs"
  exit 1
fi

echo "audit-diag: PASS (sampled rows: ${sample_rows[*]})"
# Cleanup smoke dir on success
rm -rf "$SMOKE_OUT_DIR"
exit 0