plan5(task4): tiered-exit validation script suite (tier1/2/3 checks)
Creates scripts/validation/ with per-tier exit checks consuming the
aggregate JSON from scripts/aggregate-multi-seed-metrics.py (P5T1B):
check_tier1.py — convergence (std/mean ≤ 0.15 on val_sharpe /
avg_q_value / train_loss; avg_q_value max ≤ 500 fold-1 explosion
guard; placeholders for Q-saturation + hot-path-DtoH per spec).
check_tier2.py — behavioural (val_trades_per_bar ≥ 0.005,
val_active_frac > 0.2, dir argmax entropy > 0.8·log4 with
val_dir_entropy primary + val_dir_dist_* fallback).
check_tier3.py — profitability (val_sharpe_annualised > 1.0 with
val_sharpe per-bar fallback, val_win_rate ≥ 0.52 gated on
>500 trades, val_profit_factor mean ≥ 1.1 AND cross-seed std < 0.3).
check_all_tiers.py — subprocess wrapper, exits 0 only if all pass.
Stdlib-only (statistics / argparse / json / subprocess) — no new deps.
Defensive missing-metric handling: each check FAILs with an explanatory
message when its required aggregate key is absent rather than silently
passing, so missing HEALTH_DIAG metrics are surfaced loudly.
Test harness scripts/validation/tests/test_tier_checks.sh exercises
good + bad fixtures across all four scripts and against the wrapper.
Audit row added to docs/dqn-wire-up-audit.md documenting the suite +
the deferred metrics list (val_trades_per_bar, val_active_frac,
val_dir_entropy/_dist_*, val_sharpe_annualised, val_win_rate,
val_profit_factor, val_trade_count) that HEALTH_DIAG must emit before
tiers 2/3 can ever PASS on real data — tracked for Plan 5 Task 5
pre-flight wire-up.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
jgrusewski
28
scripts/validation/tests/fixtures/bad_tier1.json
vendored
Normal file
28
scripts/validation/tests/fixtures/bad_tier1.json
vendored
Normal file
@@ -0,0 +1,28 @@
|
||||
{
|
||||
"tag": "synthetic-bad-tier1",
|
||||
"multi_seed": 3,
|
||||
"folds": 1,
|
||||
"warmup_end_epoch": 15,
|
||||
"streams_seen": 3,
|
||||
"aggregates": {
|
||||
"val_sharpe": [
|
||||
{"epoch": 0, "mean": 0.10, "std": 0.50, "median": 0.10, "min": -0.5, "max": 0.7, "n_samples": 3},
|
||||
{"epoch": 15, "mean": 1.00, "std": 10.0, "median": 1.00, "min": -9.0, "max": 11.0, "n_samples": 3},
|
||||
{"epoch": 20, "mean": 1.05, "std": 9.5, "median": 1.05, "min": -8.5, "max": 10.5, "n_samples": 3},
|
||||
{"epoch": 25, "mean": 0.95, "std": 9.7, "median": 0.95, "min": -8.7, "max": 10.7, "n_samples": 3},
|
||||
{"epoch": 29, "mean": 1.00, "std": 10.2, "median": 1.00, "min": -9.0, "max": 11.0, "n_samples": 3}
|
||||
],
|
||||
"avg_q_value": [
|
||||
{"epoch": 0, "mean": 1.0, "std": 5.0, "median": 1.0, "min": -4.0, "max": 6.0, "n_samples": 3},
|
||||
{"epoch": 5, "mean": 50.0, "std": 30.0, "median": 50.0, "min": 20.0, "max": 80.0, "n_samples": 3},
|
||||
{"epoch": 10, "mean": 500.0, "std": 300.0, "median": 500.0, "min": 200.0, "max": 800.0, "n_samples": 3},
|
||||
{"epoch": 15, "mean": 5000.0, "std": 3000.0,"median": 5000.0,"min": 2000.0, "max": 10000.0, "n_samples": 3}
|
||||
],
|
||||
"train_loss": [
|
||||
{"epoch": 0, "mean": 5.00, "std": 2.00, "median": 5.00, "min": 3.00, "max": 7.00, "n_samples": 3},
|
||||
{"epoch": 15, "mean": 50.0, "std": 100.0, "median": 50.0, "min": -50.0, "max": 150.0, "n_samples": 3},
|
||||
{"epoch": 20, "mean": 60.0, "std": 110.0, "median": 60.0, "min": -50.0, "max": 170.0, "n_samples": 3},
|
||||
{"epoch": 29, "mean": 55.0, "std": 105.0, "median": 55.0, "min": -50.0, "max": 160.0, "n_samples": 3}
|
||||
]
|
||||
}
|
||||
}
|
||||
36
scripts/validation/tests/fixtures/good_tier1.json
vendored
Normal file
36
scripts/validation/tests/fixtures/good_tier1.json
vendored
Normal file
@@ -0,0 +1,36 @@
|
||||
{
|
||||
"tag": "synthetic-good-tier1",
|
||||
"multi_seed": 3,
|
||||
"folds": 1,
|
||||
"warmup_end_epoch": 15,
|
||||
"streams_seen": 3,
|
||||
"aggregates": {
|
||||
"val_sharpe": [
|
||||
{"epoch": 0, "mean": 0.10, "std": 0.50, "median": 0.10, "min": -0.5, "max": 0.7, "n_samples": 3},
|
||||
{"epoch": 5, "mean": 0.80, "std": 0.30, "median": 0.80, "min": 0.5, "max": 1.1, "n_samples": 3},
|
||||
{"epoch": 10, "mean": 1.50, "std": 0.20, "median": 1.50, "min": 1.3, "max": 1.7, "n_samples": 3},
|
||||
{"epoch": 15, "mean": 2.00, "std": 0.05, "median": 2.00, "min": 1.95, "max": 2.05, "n_samples": 3},
|
||||
{"epoch": 20, "mean": 2.05, "std": 0.06, "median": 2.05, "min": 1.99, "max": 2.11, "n_samples": 3},
|
||||
{"epoch": 25, "mean": 2.02, "std": 0.04, "median": 2.02, "min": 1.98, "max": 2.06, "n_samples": 3},
|
||||
{"epoch": 29, "mean": 2.04, "std": 0.05, "median": 2.04, "min": 1.99, "max": 2.09, "n_samples": 3}
|
||||
],
|
||||
"avg_q_value": [
|
||||
{"epoch": 0, "mean": 1.0, "std": 5.0, "median": 1.0, "min": -4.0, "max": 6.0, "n_samples": 3},
|
||||
{"epoch": 5, "mean": 5.0, "std": 2.0, "median": 5.0, "min": 3.0, "max": 7.0, "n_samples": 3},
|
||||
{"epoch": 10, "mean": 8.0, "std": 1.0, "median": 8.0, "min": 7.0, "max": 9.0, "n_samples": 3},
|
||||
{"epoch": 15, "mean": 10.0, "std": 1.0, "median": 10.0, "min": 9.0, "max": 11.0, "n_samples": 3},
|
||||
{"epoch": 20, "mean": 10.2, "std": 0.8, "median": 10.2, "min": 9.4, "max": 11.0, "n_samples": 3},
|
||||
{"epoch": 25, "mean": 10.1, "std": 0.9, "median": 10.1, "min": 9.2, "max": 11.0, "n_samples": 3},
|
||||
{"epoch": 29, "mean": 10.3, "std": 1.0, "median": 10.3, "min": 9.3, "max": 11.3, "n_samples": 3}
|
||||
],
|
||||
"train_loss": [
|
||||
{"epoch": 0, "mean": 5.00, "std": 2.00, "median": 5.00, "min": 3.00, "max": 7.00, "n_samples": 3},
|
||||
{"epoch": 5, "mean": 2.00, "std": 0.50, "median": 2.00, "min": 1.50, "max": 2.50, "n_samples": 3},
|
||||
{"epoch": 10, "mean": 1.00, "std": 0.20, "median": 1.00, "min": 0.80, "max": 1.20, "n_samples": 3},
|
||||
{"epoch": 15, "mean": 0.50, "std": 0.05, "median": 0.50, "min": 0.45, "max": 0.55, "n_samples": 3},
|
||||
{"epoch": 20, "mean": 0.48, "std": 0.06, "median": 0.48, "min": 0.42, "max": 0.54, "n_samples": 3},
|
||||
{"epoch": 25, "mean": 0.49, "std": 0.04, "median": 0.49, "min": 0.45, "max": 0.53, "n_samples": 3},
|
||||
{"epoch": 29, "mean": 0.50, "std": 0.05, "median": 0.50, "min": 0.45, "max": 0.55, "n_samples": 3}
|
||||
]
|
||||
}
|
||||
}
|
||||
46
scripts/validation/tests/test_tier_checks.sh
Executable file
46
scripts/validation/tests/test_tier_checks.sh
Executable file
@@ -0,0 +1,46 @@
|
||||
#!/usr/bin/env bash
|
||||
# Test: tier validation scripts accept good fixtures and reject bad ones.
|
||||
#
|
||||
# This is the validation surface for the Plan 5 Task 4 tier exit suite —
|
||||
# verifies the shape of pass/fail logic on synthetic aggregate JSONs
|
||||
# without needing a real training run.
|
||||
set -euo pipefail
|
||||
|
||||
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
SCRIPT_DIR="$(cd "${HERE}/.." && pwd)"
|
||||
FIXTURES="${HERE}/fixtures"
|
||||
|
||||
# ── Tier 1 ─────────────────────────────────────────────────────────────────
|
||||
# Bad metrics JSON should fail (exit code 1, masked by `if`).
|
||||
if python3 "${SCRIPT_DIR}/check_tier1.py" "${FIXTURES}/bad_tier1.json" --warmup-end 15; then
|
||||
echo "FAIL: tier1 check accepted bad metrics"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Known-good metrics JSON should pass (exit code 0).
|
||||
python3 "${SCRIPT_DIR}/check_tier1.py" "${FIXTURES}/good_tier1.json" --warmup-end 15
|
||||
|
||||
# ── Tier 2 ─────────────────────────────────────────────────────────────────
|
||||
# Bad fixture lacks val_trades_per_bar / val_active_frac / dir-entropy entirely
|
||||
# → tier2 must FAIL with the missing-metric explanation.
|
||||
if python3 "${SCRIPT_DIR}/check_tier2.py" "${FIXTURES}/bad_tier1.json" --warmup-end 15; then
|
||||
echo "FAIL: tier2 check accepted aggregate without behavioural metrics"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ── Tier 3 ─────────────────────────────────────────────────────────────────
|
||||
# Bad fixture lacks val_sharpe_annualised / val_win_rate / val_profit_factor
|
||||
# entirely → tier3 must FAIL with the missing-metric explanation.
|
||||
if python3 "${SCRIPT_DIR}/check_tier3.py" "${FIXTURES}/bad_tier1.json" --warmup-end 15; then
|
||||
echo "FAIL: tier3 check accepted aggregate without profitability metrics"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ── all-tiers wrapper ──────────────────────────────────────────────────────
|
||||
# Wrapper must reject any aggregate that fails any tier (uses bad fixture).
|
||||
if python3 "${SCRIPT_DIR}/check_all_tiers.py" "${FIXTURES}/bad_tier1.json" --warmup-end 15; then
|
||||
echo "FAIL: check_all_tiers.py accepted aggregate that fails tier1+2+3"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "PASS"
|
||||
Reference in New Issue
Block a user