test(sp15-p2b): 17 behavioral tests (Group 1 + Group 2) as #[ignore] contracts
Per spec §7.3 + plan v2 Phase 2B differential table. Each test documents its behavioral contract and its enabling phase. Tests are #[ignore]'d so the pre-commit-hook-gated suite does not fail; per-test #[ignore] is removed by the commit that lands its enabling phase. Group 1 trading behavior (7 tests): 2.1 flat_holds, 2.2 trend_directional, 2.3 mean_revert_reverses, 2.4 cost_sensitivity, 2.6 regime_silences, 2.7 stop_discipline, 2.18 action_latency. Group 2 state/arch grounding (10 tests): 2.8 hold_vs_flat_semantics, 2.9 eval_fold_isolation, 2.13 eval_train_consistency, 2.14 magnitude_uses_ all_buckets, 2.15 fold_transition_stability, 2.16 q_value_bounded, 2.17 gradient_flow_to_all_heads, 2.19 kelly_calibration, 2.20 state_input_ grounding, 2.21 egf_gate_opens (re-export hook for sp14_oracle_tests.rs). Group 3 (Phase 2C — 5 tests) lands paired with Phase 3.5 commits. Harness invocations use trainer = () placeholder; Phase 2B.b extends BehavioralResult/harness with per-bar actions, position-state inspection, forced-action stepping, and per-head grad-norm snapshots. Each test's ignore reason references which Phase X enabling work + harness extension makes it green. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -4,4 +4,31 @@
|
||||
mod synthetic_markets;
|
||||
mod oracle;
|
||||
mod harness;
|
||||
// Phase 2B will add: mod test_2_1_*; ... mod test_2_22_*;
|
||||
|
||||
// Group 1: trading behavior (7 tests)
|
||||
mod test_2_1_flat_market_holds;
|
||||
mod test_2_2_trend_market_directional;
|
||||
mod test_2_3_mean_revert_reverses;
|
||||
mod test_2_4_cost_sensitivity;
|
||||
mod test_2_6_regime_silences;
|
||||
mod test_2_7_stop_discipline;
|
||||
mod test_2_18_action_latency;
|
||||
|
||||
// Group 2: state/arch grounding (10 tests)
|
||||
mod test_2_8_hold_vs_flat_semantics;
|
||||
mod test_2_9_eval_fold_isolation;
|
||||
mod test_2_13_eval_train_consistency;
|
||||
mod test_2_14_magnitude_uses_all_buckets;
|
||||
mod test_2_15_fold_transition_stability;
|
||||
mod test_2_16_q_value_bounded;
|
||||
mod test_2_17_gradient_flow_to_all_heads;
|
||||
mod test_2_19_kelly_calibration;
|
||||
mod test_2_20_state_input_grounding;
|
||||
mod test_2_21_egf_gate_opens;
|
||||
|
||||
// Group 3 (Phase 2C) lands paired with Phase 3.5 commits:
|
||||
// mod test_2_5_drawdown_de_risks;
|
||||
// mod test_2_10_recovery_after_streak;
|
||||
// mod test_2_11_dd_state_aware_sizing;
|
||||
// mod test_2_12_cooldown_engagement;
|
||||
// mod test_2_22_plasticity_cooldown_interlock;
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
//! KL(train-mode, eval-mode action dists) < 0.5.
|
||||
#[test]
|
||||
#[ignore = "Phase 2B contract — requires train/eval mode toggle harness"]
|
||||
fn test_2_13_eval_train_consistency() {
|
||||
// Phase 2B.b: trainer.set_eval_mode(bool) + paired evaluation.
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
//! Each magnitude bucket > 5% of trade events.
|
||||
use crate::synthetic_markets::drift_market;
|
||||
use crate::harness::{evaluate_policy_on_market, BehavioralResult};
|
||||
|
||||
#[test]
|
||||
#[ignore = "Phase 2B contract — green when Phase 3.5 entropy-aware Hold + magnitude lands"]
|
||||
fn test_2_14_magnitude_uses_all_buckets() {
|
||||
let bars = drift_market(20_000, 0.5, 0.5, 0.25, 42);
|
||||
let mut trainer = ();
|
||||
let result: BehavioralResult = evaluate_policy_on_market(&mut trainer, &bars[5_000..], 0);
|
||||
for key in ["long_quarter", "long_half", "long_full"] {
|
||||
let pct = result.action_dist.get(key).copied().unwrap_or(0.0);
|
||||
assert!(pct >= 0.05, "test 2.14 FAIL: mag bucket {} only {:.2}%", key, pct * 100.0);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
//! Sharpe collapse at fold boundary < 50%.
|
||||
#[test]
|
||||
#[ignore = "Phase 2B contract — green via existing StateResetRegistry"]
|
||||
fn test_2_15_fold_transition_stability() {
|
||||
// Train on synthetic fold 1; capture sharpe at end. Reset state. Train fold 2;
|
||||
// capture sharpe at start. assert!((end - start).abs() / end.max(0.01) < 0.50);
|
||||
//
|
||||
// Phase 2B.b: requires fold-boundary scaffolding the harness doesn't yet expose.
|
||||
}
|
||||
6
crates/ml/tests/behavioral/test_2_16_q_value_bounded.rs
Normal file
6
crates/ml/tests/behavioral/test_2_16_q_value_bounded.rs
Normal file
@@ -0,0 +1,6 @@
|
||||
//! Q-values stay in [V_MIN, V_MAX] under K=1000 replay updates.
|
||||
#[test]
|
||||
#[ignore = "Phase 2B contract — green via existing C51 V-range; requires Q snapshot harness"]
|
||||
fn test_2_16_q_value_bounded() {
|
||||
// Phase 2B.b: trainer.snapshot_q_values() + replay-update loop.
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
//! Non-zero grad reaches every head (dir/mag/ord/urg/aux/CQL/IQN/C51/ens).
|
||||
#[test]
|
||||
#[ignore = "Phase 2B contract — green via architecture; requires per-head grad-norm harness"]
|
||||
fn test_2_17_gradient_flow_to_all_heads() {
|
||||
// Phase 2B.b: trainer.snapshot_per_head_grad_norms() after one backward pass.
|
||||
}
|
||||
27
crates/ml/tests/behavioral/test_2_18_action_latency.rs
Normal file
27
crates/ml/tests/behavioral/test_2_18_action_latency.rs
Normal file
@@ -0,0 +1,27 @@
|
||||
//! SP15 Phase 2B Group 1 — action latency: Hold→Long within 5 bars of +5σ signal.
|
||||
//! Anchor for: Phase 3 (responsiveness via teaching).
|
||||
|
||||
use crate::synthetic_markets::flat_market;
|
||||
use crate::harness::BehavioralResult;
|
||||
use ml::cuda_pipeline::lob_bar::LobBar;
|
||||
|
||||
#[test]
|
||||
#[ignore = "Phase 2B contract — requires per-bar action access from harness (Phase 2B.b)"]
|
||||
fn test_2_18_action_latency() {
|
||||
// Flat for 5000 bars, then step-function +drift starts at bar 5000
|
||||
let mut bars = flat_market(5_000, 0.1, 0.25, 42);
|
||||
let drift_start_price = bars.last().unwrap().price;
|
||||
for i in 0..5_000 {
|
||||
let p = drift_start_price + 0.5 * (i as f32);
|
||||
bars.push(LobBar::new(p, 0.25, 1.0));
|
||||
}
|
||||
|
||||
// Once harness exposes per-bar actions:
|
||||
// let switch_bar = result.actions.iter().skip(5_000)
|
||||
// .position(|a| matches!(a, ActionEnum::Long(_)))
|
||||
// .map(|p| p + 5_000);
|
||||
// assert!(switch_bar.unwrap_or(usize::MAX) < 5_005, ...);
|
||||
|
||||
let _ = bars; // contract document
|
||||
let _ = BehavioralResult::default();
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
//! kelly_f monotonically decreasing in Q-variance.
|
||||
#[test]
|
||||
#[ignore = "Phase 2B contract — green via existing Kelly; requires Q-variance sweep harness"]
|
||||
fn test_2_19_kelly_calibration() {
|
||||
// Phase 2B.b: trainer.set_q_variance_for_test(v) + read kelly_f from ISV.
|
||||
// Sweep v ∈ {0.01, 0.1, 1.0}, assert kelly_f[0] > kelly_f[1] > kelly_f[2].
|
||||
}
|
||||
18
crates/ml/tests/behavioral/test_2_1_flat_market_holds.rs
Normal file
18
crates/ml/tests/behavioral/test_2_1_flat_market_holds.rs
Normal file
@@ -0,0 +1,18 @@
|
||||
//! SP15 Phase 2B Group 1 — flat market should produce ≥80% Hold.
|
||||
//! Anchor for: Phase 3.5 confidence-aware Hold floor (sigmoid).
|
||||
|
||||
use crate::synthetic_markets::flat_market;
|
||||
use crate::harness::{evaluate_policy_on_market, BehavioralResult};
|
||||
|
||||
#[test]
|
||||
#[ignore = "Phase 2B contract — green when Phase 3.5 Hold floor lands"]
|
||||
fn test_2_1_flat_market_holds() {
|
||||
let bars = flat_market(10_000, 0.1, 0.25, 42);
|
||||
// Phase 2B.b will replace this with minimal_trainer_for_tests() once that helper exists.
|
||||
let mut trainer = ();
|
||||
let result: BehavioralResult = evaluate_policy_on_market(&mut trainer, &bars[5_000..], 0);
|
||||
|
||||
let hold_pct = result.action_dist.get("hold").copied().unwrap_or(0.0);
|
||||
assert!(hold_pct >= 0.80,
|
||||
"test 2.1 FAIL: flat market Hold rate = {:.2}, expected >= 0.80", hold_pct);
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
//! KL(uptrend-context, downtrend-context action dists) > 0.5.
|
||||
#[test]
|
||||
#[ignore = "Phase 2B contract — green via Phase 1.5 dd_pct trunk concat (already landed) + Phase 1.5.b state_dim bump"]
|
||||
fn test_2_20_state_input_grounding() {
|
||||
// Run policy on uptrend then downtrend (same dd_pct=0); compute KL.
|
||||
// Phase 2B.b: action-dist KL helper.
|
||||
}
|
||||
15
crates/ml/tests/behavioral/test_2_21_egf_gate_opens.rs
Normal file
15
crates/ml/tests/behavioral/test_2_21_egf_gate_opens.rs
Normal file
@@ -0,0 +1,15 @@
|
||||
//! Phase 0 anchor: aux/Q deliberate disagreement → gate1 opens within 100 steps.
|
||||
//! ALREADY IMPLEMENTED in crates/ml/tests/sp14_oracle_tests.rs (Task 0.C will land it
|
||||
//! once Phase 0.B retune completes). This file is a thin re-export hook so the
|
||||
//! behavioral_suite test target picks it up.
|
||||
|
||||
#[test]
|
||||
#[ignore = "Phase 2B contract — green via Phase 0.B (test lives in sp14_oracle_tests.rs)"]
|
||||
fn test_2_21_egf_gate_opens() {
|
||||
// The actual implementation lives in:
|
||||
// crates/ml/tests/sp14_oracle_tests.rs::gpu::egf_gate_opens_under_disagreement
|
||||
//
|
||||
// This re-export hook documents that test 2.21 EXISTS in the canonical
|
||||
// behavioral_suite contract. The implementation is reachable via the GPU
|
||||
// oracle test path; this stub is the catalog entry.
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
//! SP15 Phase 2B Group 1 — trend market should produce ≥70% direction-of-trend.
|
||||
//! Anchor for: Phase 1+3 (cost + trader teachings).
|
||||
|
||||
use crate::synthetic_markets::drift_market;
|
||||
use crate::harness::{evaluate_policy_on_market, BehavioralResult};
|
||||
|
||||
#[test]
|
||||
#[ignore = "Phase 2B contract — green when Phase 1+3 land"]
|
||||
fn test_2_2_trend_market_directional() {
|
||||
let bars = drift_market(10_000, 0.5, 0.1, 0.25, 42);
|
||||
let mut trainer = ();
|
||||
let result: BehavioralResult = evaluate_policy_on_market(&mut trainer, &bars[5_000..], 0);
|
||||
|
||||
let long_pct: f32 = result.action_dist.iter()
|
||||
.filter(|(k, _)| k.starts_with("long_"))
|
||||
.map(|(_, v)| *v)
|
||||
.sum();
|
||||
assert!(long_pct >= 0.70,
|
||||
"test 2.2 FAIL: trend market Long rate = {:.2}, expected >= 0.70 on +drift", long_pct);
|
||||
}
|
||||
32
crates/ml/tests/behavioral/test_2_3_mean_revert_reverses.rs
Normal file
32
crates/ml/tests/behavioral/test_2_3_mean_revert_reverses.rs
Normal file
@@ -0,0 +1,32 @@
|
||||
//! SP15 Phase 2B Group 1 — mean-revert market should reverse direction at ±2σ within 5 bars.
|
||||
//! Anchor for: Phase 1+3.
|
||||
|
||||
use crate::synthetic_markets::ou_market;
|
||||
use crate::harness::{evaluate_policy_on_market, BehavioralResult};
|
||||
|
||||
#[test]
|
||||
#[ignore = "Phase 2B contract — green when Phase 1+3 land + harness exposes per-bar actions"]
|
||||
fn test_2_3_mean_revert_reverses() {
|
||||
let mu: f32 = 4500.0;
|
||||
let sigma: f32 = 1.0;
|
||||
let bars = ou_market(10_000, 0.05, sigma, mu, 0.25, 42);
|
||||
let mut trainer = ();
|
||||
let _result: BehavioralResult = evaluate_policy_on_market(&mut trainer, &bars[5_000..], 0);
|
||||
|
||||
// The full assertion requires per-bar action access from BehavioralResult,
|
||||
// which the Phase 2A harness stub does not yet expose. Phase 2B.b will
|
||||
// extend BehavioralResult with `actions: Vec<ActionEnum>`. Until then,
|
||||
// this test serves as the contract specification.
|
||||
//
|
||||
// Once per-bar actions are exposed:
|
||||
// for (i, bar) in bars.iter().enumerate() {
|
||||
// let z = (bar.price - mu) / sigma;
|
||||
// if z.abs() > 2.0 {
|
||||
// let next_5 = &result.actions[i+1..(i+6).min(result.actions.len())];
|
||||
// let expected_dir = if z > 0.0 { /* Short */ } else { /* Long */ };
|
||||
// assert!(next_5.iter().any(|a| matches_direction(a, expected_dir)),
|
||||
// "no reverse at z={:.2}", z);
|
||||
// }
|
||||
// }
|
||||
// For now the test compiles + runs (no-op past the smoke).
|
||||
}
|
||||
24
crates/ml/tests/behavioral/test_2_4_cost_sensitivity.rs
Normal file
24
crates/ml/tests/behavioral/test_2_4_cost_sensitivity.rs
Normal file
@@ -0,0 +1,24 @@
|
||||
//! SP15 Phase 2B Group 1 — cost sensitivity: trade freq drops ≥40% under high spread.
|
||||
//! Anchor for: Phase 3.2 explicit cost in r_quality.
|
||||
|
||||
use crate::synthetic_markets::drift_market;
|
||||
use crate::harness::{evaluate_policy_on_market, BehavioralResult};
|
||||
|
||||
#[test]
|
||||
#[ignore = "Phase 2B contract — green when Phase 3.2 lands"]
|
||||
fn test_2_4_cost_sensitivity() {
|
||||
// Same +drift series at different mean spreads
|
||||
let bars_low = drift_market(10_000, 0.5, 0.1, /*mean_spread*/ 0.1, 42);
|
||||
let bars_high = drift_market(10_000, 0.5, 0.1, /*mean_spread*/ 2.0, 42);
|
||||
|
||||
let mut trainer = ();
|
||||
let result_low: BehavioralResult = evaluate_policy_on_market(&mut trainer, &bars_low[5_000..], 0);
|
||||
let result_high: BehavioralResult = evaluate_policy_on_market(&mut trainer, &bars_high[5_000..], 0);
|
||||
|
||||
let low = result_low.trade_count as f32;
|
||||
let high = result_high.trade_count as f32;
|
||||
assert!(low > 0.0, "low-spread trade_count = 0; cannot compute sensitivity");
|
||||
let drop = (low - high) / low;
|
||||
assert!(drop >= 0.40,
|
||||
"test 2.4 FAIL: trade-freq drop = {:.2}, expected >= 0.40 (high-spread)", drop);
|
||||
}
|
||||
18
crates/ml/tests/behavioral/test_2_6_regime_silences.rs
Normal file
18
crates/ml/tests/behavioral/test_2_6_regime_silences.rs
Normal file
@@ -0,0 +1,18 @@
|
||||
//! SP15 Phase 2B Group 1 — high-noise no-edge regime: trade freq ≤20%.
|
||||
//! Anchor for: Phase 3.4 regret + Phase 3.5 confidence-aware Hold floor.
|
||||
|
||||
use crate::synthetic_markets::flat_market;
|
||||
use crate::harness::{evaluate_policy_on_market, BehavioralResult};
|
||||
|
||||
#[test]
|
||||
#[ignore = "Phase 2B contract — green when Phase 3.4 + 3.5 land"]
|
||||
fn test_2_6_regime_silences() {
|
||||
// High-noise flat market: no edge to extract
|
||||
let bars = flat_market(10_000, /*noise_sigma*/ 5.0, 0.25, 42);
|
||||
let mut trainer = ();
|
||||
let result: BehavioralResult = evaluate_policy_on_market(&mut trainer, &bars[5_000..], 0);
|
||||
|
||||
let trade_freq = result.trade_count as f32 / 5_000.0;
|
||||
assert!(trade_freq <= 0.20,
|
||||
"test 2.6 FAIL: trade freq = {:.2}, expected <= 0.20 in no-edge regime", trade_freq);
|
||||
}
|
||||
22
crates/ml/tests/behavioral/test_2_7_stop_discipline.rs
Normal file
22
crates/ml/tests/behavioral/test_2_7_stop_discipline.rs
Normal file
@@ -0,0 +1,22 @@
|
||||
//! SP15 Phase 2B Group 1 — adverse excursion past trail dist → close within 1 bar.
|
||||
//! Anchor for: existing trail-stop logic.
|
||||
|
||||
use crate::harness::BehavioralResult;
|
||||
|
||||
#[test]
|
||||
#[ignore = "Phase 2B contract — requires forced-action harness extension (Phase 2B.b)"]
|
||||
fn test_2_7_stop_discipline() {
|
||||
// This test requires driving a forced Long entry, then injecting an
|
||||
// adverse-excursion price move past trail distance, then verifying the
|
||||
// policy emits Flat within 1 bar of the breach.
|
||||
//
|
||||
// Harness extensions needed (Phase 2B.b):
|
||||
// - trainer.step_with_forced_action(action: ActionEnum) -> Result<BarMetrics, MLError>
|
||||
// - bar-by-bar action observation
|
||||
//
|
||||
// Contract: after a trail breach detected at bar T, action[T+1] == Flat.
|
||||
|
||||
// Until the harness exposes forced-action stepping, this test is a
|
||||
// contract document. Phase 2B.b will populate the assertion body.
|
||||
let _ = BehavioralResult::default();
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
//! Hold preserves pre_pos; Flat zeros it. Anchor for: existing semantics.
|
||||
#[test]
|
||||
#[ignore = "Phase 2B contract — requires forced-action + position-state inspection harness"]
|
||||
fn test_2_8_hold_vs_flat_semantics() {
|
||||
// Drive entry → Hold → assert pre_pos != 0
|
||||
// Drive entry → Flat → assert pre_pos == 0
|
||||
// Phase 2B.b harness extension required.
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
//! KL(shuffled-future, unshuffled-future) < 0.01 — proves no temporal leak.
|
||||
#[test]
|
||||
#[ignore = "Phase 2B contract — green via StateResetRegistry coverage; requires KL helper"]
|
||||
fn test_2_9_eval_fold_isolation() {
|
||||
// Phase 2B.b: harness exposes action_distribution_kl(actions_a, actions_b).
|
||||
}
|
||||
Reference in New Issue
Block a user