From dff6e666ee5f348031f335939fdfa350b732a4d7 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Wed, 6 May 2026 11:29:22 +0200 Subject: [PATCH] feat(sp15-p2a.2): oracle policies + evaluator harness + pre-commit hook MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 2A scaffolding complete (per spec §7.2): - oracle.rs: OracleAction enum + 3 oracle policies (flat, drift, OU) - harness.rs: BehavioralResult + evaluate_policy_on_market stub. Trainer-side methods (eval_actions_on_features, read_isv_for_test) are documented gap #7; Phase 2B tests add them per-test as needed. - pre-commit hook: cargo test -p ml --test behavioral_suite gates argo-train.sh per spec §4.5 discipline rule. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../helpers/pre_commit_behavioral_suite.sh | 21 +++++ .claude/settings.json | 12 +++ crates/ml/tests/behavioral/harness.rs | 53 ++++++++++++ crates/ml/tests/behavioral/main.rs | 3 +- crates/ml/tests/behavioral/oracle.rs | 80 +++++++++++++++++++ 5 files changed, 168 insertions(+), 1 deletion(-) create mode 100755 .claude/helpers/pre_commit_behavioral_suite.sh create mode 100644 crates/ml/tests/behavioral/harness.rs create mode 100644 crates/ml/tests/behavioral/oracle.rs diff --git a/.claude/helpers/pre_commit_behavioral_suite.sh b/.claude/helpers/pre_commit_behavioral_suite.sh new file mode 100755 index 000000000..ce7ff1fe1 --- /dev/null +++ b/.claude/helpers/pre_commit_behavioral_suite.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# .claude/helpers/pre_commit_behavioral_suite.sh +# +# SP15 Phase 2 — behavioral suite gate. Blocks argo-train.sh if any +# behavioral test fails. Suite runs <10 min on dev RTX 3050 Ti. +set -euo pipefail + +cd "$(git rev-parse --show-toplevel)" + +echo "Running SP15 behavioral suite (gate for argo-train.sh)..." + +if ! SQLX_OFFLINE=true CUDA_COMPUTE_CAP=86 \ + cargo test -p ml --test behavioral_suite --features cuda \ + -- --include-ignored --test-threads=1 2>&1 \ + | tee /tmp/sp15_behavioral_suite.log; then + echo "Behavioral suite FAILED. argo-train.sh blocked." + echo " See /tmp/sp15_behavioral_suite.log for details." + exit 1 +fi + +echo "Behavioral suite PASS — argo-train.sh unblocked." diff --git a/.claude/settings.json b/.claude/settings.json index 1f6467424..99423beb4 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -1,5 +1,17 @@ { "hooks": { + "PreToolUse": [ + { + "matcher": "Bash", + "hooks": [ + { + "type": "command", + "command": "if echo \"${CLAUDE_TOOL_INPUT:-}\" | grep -qE 'argo-train\\.sh'; then \"$CLAUDE_PROJECT_DIR/.claude/helpers/pre_commit_behavioral_suite.sh\"; fi", + "timeout": 600 + } + ] + } + ], "SessionStart": [ { "hooks": [ diff --git a/crates/ml/tests/behavioral/harness.rs b/crates/ml/tests/behavioral/harness.rs new file mode 100644 index 000000000..84c4cda7d --- /dev/null +++ b/crates/ml/tests/behavioral/harness.rs @@ -0,0 +1,53 @@ +//! SP15 Phase 2A — behavioral evaluator harness. +//! +//! Runs a policy on a synthetic market, captures action distribution + +//! sharpe_net + max_dd. Trainer-side methods (eval_actions_on_features etc) +//! are not yet implemented — see plan IMPORTANT 7. Each Phase 2B test +//! adds the specific trainer methods it needs. + +use ml::cuda_pipeline::lob_bar::LobBar; +use std::collections::HashMap; + +#[derive(Debug, Default)] +pub struct BehavioralResult { + /// Action distribution keys: "hold", "flat", "long_quarter", + /// "long_half", "long_full", "short_quarter", "short_half", "short_full". + pub action_dist: HashMap<&'static str, f32>, + pub sharpe_net: f32, + pub max_dd: f32, + pub trade_count: usize, +} + +/// Runs the policy on `bars`, captures action distribution + sharpe_net +/// + max_dd. `n_warm` warmup bars precede action collection. +/// +/// Phase 2A contract (locked here): trainer-side `eval_actions_on_features` +/// is added per-test in Phase 2B as each behavioral test needs it (plan +/// execution-time gap #7). This function returns a default BehavioralResult; +/// each Phase 2B test rewrites the body to call the trainer methods it +/// requires. The signature here is the load-bearing piece — keep it stable. +#[allow(dead_code, unused_variables)] +pub fn evaluate_policy_on_market( + // Phase 2B swaps `&mut ()` for `&mut GpuDqnTrainer` once the per-test + // trainer methods land. Until then the unit param keeps the signature + // callable in tests that don't yet exercise the trainer path. + _trainer: &mut (), + _bars: &[LobBar], + _n_warm: usize, +) -> BehavioralResult { + // Phase 2B wires this to trainer.eval_actions_on_features + + // launch_sp15_cost_net_sharpe + ISV DD-slot reads. + BehavioralResult::default() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn behavioral_result_default_is_empty() { + let r = BehavioralResult::default(); + assert_eq!(r.trade_count, 0); + assert!(r.action_dist.is_empty()); + } +} diff --git a/crates/ml/tests/behavioral/main.rs b/crates/ml/tests/behavioral/main.rs index 77b9f17f8..76a59fd4a 100644 --- a/crates/ml/tests/behavioral/main.rs +++ b/crates/ml/tests/behavioral/main.rs @@ -2,5 +2,6 @@ //! Pre-commit hook gates argo-train.sh: failing test blocks L40S deploy. mod synthetic_markets; -// Phase 2A.2 will add: mod oracle; mod harness; +mod oracle; +mod harness; // Phase 2B will add: mod test_2_1_*; ... mod test_2_22_*; diff --git a/crates/ml/tests/behavioral/oracle.rs b/crates/ml/tests/behavioral/oracle.rs new file mode 100644 index 000000000..9a61f777b --- /dev/null +++ b/crates/ml/tests/behavioral/oracle.rs @@ -0,0 +1,80 @@ +//! SP15 Phase 2A — oracle policies for behavioral test harness. +//! Each oracle is a known-correct action sequence for a given synthetic +//! market type; tests assert the trained policy approaches the oracle. + +use ml::cuda_pipeline::lob_bar::LobBar; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum MagBucket { Quarter, Half, Full } + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum OracleAction { + Hold, + Long(MagBucket), + Short(MagBucket), + Flat, +} + +/// Oracle for flat markets: always Hold. +pub fn flat_market_oracle(bars: &[LobBar]) -> Vec { + vec![OracleAction::Hold; bars.len()] +} + +/// Oracle for drift markets: take direction-of-trend, mag = Half. +pub fn drift_market_oracle(bars: &[LobBar]) -> Vec { + let mut actions = Vec::with_capacity(bars.len()); + actions.push(OracleAction::Hold); // first bar: no prior + for w in bars.windows(2) { + let action = if w[1].price > w[0].price + 0.05 { + OracleAction::Long(MagBucket::Half) + } else if w[1].price < w[0].price - 0.05 { + OracleAction::Short(MagBucket::Half) + } else { + OracleAction::Hold + }; + actions.push(action); + } + actions +} + +/// Oracle for OU markets: reverse at ±2σ extremes. +pub fn ou_market_oracle(bars: &[LobBar], mu: f32, sigma: f32) -> Vec { + bars.iter().map(|b| { + let z = (b.price - mu) / sigma.max(1e-6); + if z > 2.0 { OracleAction::Short(MagBucket::Half) } + else if z < -2.0 { OracleAction::Long(MagBucket::Half) } + else { OracleAction::Hold } + }).collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::synthetic_markets::*; + + #[test] + fn flat_oracle_all_hold() { + let bars = flat_market(100, 0.1, 0.25, 42); + let actions = flat_market_oracle(&bars); + assert_eq!(actions.len(), 100); + assert!(actions.iter().all(|a| *a == OracleAction::Hold)); + } + + #[test] + fn drift_oracle_takes_uptrend() { + let bars = drift_market(1000, 0.5, 0.1, 0.25, 42); + let actions = drift_market_oracle(&bars); + let long_count = actions.iter().filter(|a| matches!(a, OracleAction::Long(_))).count(); + // Strong uptrend → most actions should be Long + assert!(long_count > 700, "drift_market_oracle Long count = {}, expected > 700", long_count); + } + + #[test] + fn ou_oracle_reverses_at_extremes() { + let bars = ou_market(1000, 0.05, 1.0, 4500.0, 0.25, 42); + let actions = ou_market_oracle(&bars, 4500.0, 1.0); + // OU oracle should produce SOME reversion actions (not all Hold) + let action_count = actions.iter().filter(|a| !matches!(a, OracleAction::Hold)).count(); + assert!(action_count > 0, "OU oracle produced zero non-Hold actions"); + } +}