feat(sp15-p2a.2): oracle policies + evaluator harness + pre-commit hook
Phase 2A scaffolding complete (per spec §7.2): - oracle.rs: OracleAction enum + 3 oracle policies (flat, drift, OU) - harness.rs: BehavioralResult + evaluate_policy_on_market stub. Trainer-side methods (eval_actions_on_features, read_isv_for_test) are documented gap #7; Phase 2B tests add them per-test as needed. - pre-commit hook: cargo test -p ml --test behavioral_suite gates argo-train.sh per spec §4.5 discipline rule. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
21
.claude/helpers/pre_commit_behavioral_suite.sh
Executable file
21
.claude/helpers/pre_commit_behavioral_suite.sh
Executable file
@@ -0,0 +1,21 @@
|
||||
#!/bin/bash
|
||||
# .claude/helpers/pre_commit_behavioral_suite.sh
|
||||
#
|
||||
# SP15 Phase 2 — behavioral suite gate. Blocks argo-train.sh if any
|
||||
# behavioral test fails. Suite runs <10 min on dev RTX 3050 Ti.
|
||||
set -euo pipefail
|
||||
|
||||
cd "$(git rev-parse --show-toplevel)"
|
||||
|
||||
echo "Running SP15 behavioral suite (gate for argo-train.sh)..."
|
||||
|
||||
if ! SQLX_OFFLINE=true CUDA_COMPUTE_CAP=86 \
|
||||
cargo test -p ml --test behavioral_suite --features cuda \
|
||||
-- --include-ignored --test-threads=1 2>&1 \
|
||||
| tee /tmp/sp15_behavioral_suite.log; then
|
||||
echo "Behavioral suite FAILED. argo-train.sh blocked."
|
||||
echo " See /tmp/sp15_behavioral_suite.log for details."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Behavioral suite PASS — argo-train.sh unblocked."
|
||||
@@ -1,5 +1,17 @@
|
||||
{
|
||||
"hooks": {
|
||||
"PreToolUse": [
|
||||
{
|
||||
"matcher": "Bash",
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "if echo \"${CLAUDE_TOOL_INPUT:-}\" | grep -qE 'argo-train\\.sh'; then \"$CLAUDE_PROJECT_DIR/.claude/helpers/pre_commit_behavioral_suite.sh\"; fi",
|
||||
"timeout": 600
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"SessionStart": [
|
||||
{
|
||||
"hooks": [
|
||||
|
||||
53
crates/ml/tests/behavioral/harness.rs
Normal file
53
crates/ml/tests/behavioral/harness.rs
Normal file
@@ -0,0 +1,53 @@
|
||||
//! SP15 Phase 2A — behavioral evaluator harness.
|
||||
//!
|
||||
//! Runs a policy on a synthetic market, captures action distribution +
|
||||
//! sharpe_net + max_dd. Trainer-side methods (eval_actions_on_features etc)
|
||||
//! are not yet implemented — see plan IMPORTANT 7. Each Phase 2B test
|
||||
//! adds the specific trainer methods it needs.
|
||||
|
||||
use ml::cuda_pipeline::lob_bar::LobBar;
|
||||
use std::collections::HashMap;
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct BehavioralResult {
|
||||
/// Action distribution keys: "hold", "flat", "long_quarter",
|
||||
/// "long_half", "long_full", "short_quarter", "short_half", "short_full".
|
||||
pub action_dist: HashMap<&'static str, f32>,
|
||||
pub sharpe_net: f32,
|
||||
pub max_dd: f32,
|
||||
pub trade_count: usize,
|
||||
}
|
||||
|
||||
/// Runs the policy on `bars`, captures action distribution + sharpe_net
|
||||
/// + max_dd. `n_warm` warmup bars precede action collection.
|
||||
///
|
||||
/// Phase 2A contract (locked here): trainer-side `eval_actions_on_features`
|
||||
/// is added per-test in Phase 2B as each behavioral test needs it (plan
|
||||
/// execution-time gap #7). This function returns a default BehavioralResult;
|
||||
/// each Phase 2B test rewrites the body to call the trainer methods it
|
||||
/// requires. The signature here is the load-bearing piece — keep it stable.
|
||||
#[allow(dead_code, unused_variables)]
|
||||
pub fn evaluate_policy_on_market(
|
||||
// Phase 2B swaps `&mut ()` for `&mut GpuDqnTrainer` once the per-test
|
||||
// trainer methods land. Until then the unit param keeps the signature
|
||||
// callable in tests that don't yet exercise the trainer path.
|
||||
_trainer: &mut (),
|
||||
_bars: &[LobBar],
|
||||
_n_warm: usize,
|
||||
) -> BehavioralResult {
|
||||
// Phase 2B wires this to trainer.eval_actions_on_features +
|
||||
// launch_sp15_cost_net_sharpe + ISV DD-slot reads.
|
||||
BehavioralResult::default()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn behavioral_result_default_is_empty() {
|
||||
let r = BehavioralResult::default();
|
||||
assert_eq!(r.trade_count, 0);
|
||||
assert!(r.action_dist.is_empty());
|
||||
}
|
||||
}
|
||||
@@ -2,5 +2,6 @@
|
||||
//! Pre-commit hook gates argo-train.sh: failing test blocks L40S deploy.
|
||||
|
||||
mod synthetic_markets;
|
||||
// Phase 2A.2 will add: mod oracle; mod harness;
|
||||
mod oracle;
|
||||
mod harness;
|
||||
// Phase 2B will add: mod test_2_1_*; ... mod test_2_22_*;
|
||||
|
||||
80
crates/ml/tests/behavioral/oracle.rs
Normal file
80
crates/ml/tests/behavioral/oracle.rs
Normal file
@@ -0,0 +1,80 @@
|
||||
//! SP15 Phase 2A — oracle policies for behavioral test harness.
|
||||
//! Each oracle is a known-correct action sequence for a given synthetic
|
||||
//! market type; tests assert the trained policy approaches the oracle.
|
||||
|
||||
use ml::cuda_pipeline::lob_bar::LobBar;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum MagBucket { Quarter, Half, Full }
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum OracleAction {
|
||||
Hold,
|
||||
Long(MagBucket),
|
||||
Short(MagBucket),
|
||||
Flat,
|
||||
}
|
||||
|
||||
/// Oracle for flat markets: always Hold.
|
||||
pub fn flat_market_oracle(bars: &[LobBar]) -> Vec<OracleAction> {
|
||||
vec![OracleAction::Hold; bars.len()]
|
||||
}
|
||||
|
||||
/// Oracle for drift markets: take direction-of-trend, mag = Half.
|
||||
pub fn drift_market_oracle(bars: &[LobBar]) -> Vec<OracleAction> {
|
||||
let mut actions = Vec::with_capacity(bars.len());
|
||||
actions.push(OracleAction::Hold); // first bar: no prior
|
||||
for w in bars.windows(2) {
|
||||
let action = if w[1].price > w[0].price + 0.05 {
|
||||
OracleAction::Long(MagBucket::Half)
|
||||
} else if w[1].price < w[0].price - 0.05 {
|
||||
OracleAction::Short(MagBucket::Half)
|
||||
} else {
|
||||
OracleAction::Hold
|
||||
};
|
||||
actions.push(action);
|
||||
}
|
||||
actions
|
||||
}
|
||||
|
||||
/// Oracle for OU markets: reverse at ±2σ extremes.
|
||||
pub fn ou_market_oracle(bars: &[LobBar], mu: f32, sigma: f32) -> Vec<OracleAction> {
|
||||
bars.iter().map(|b| {
|
||||
let z = (b.price - mu) / sigma.max(1e-6);
|
||||
if z > 2.0 { OracleAction::Short(MagBucket::Half) }
|
||||
else if z < -2.0 { OracleAction::Long(MagBucket::Half) }
|
||||
else { OracleAction::Hold }
|
||||
}).collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::synthetic_markets::*;
|
||||
|
||||
#[test]
|
||||
fn flat_oracle_all_hold() {
|
||||
let bars = flat_market(100, 0.1, 0.25, 42);
|
||||
let actions = flat_market_oracle(&bars);
|
||||
assert_eq!(actions.len(), 100);
|
||||
assert!(actions.iter().all(|a| *a == OracleAction::Hold));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn drift_oracle_takes_uptrend() {
|
||||
let bars = drift_market(1000, 0.5, 0.1, 0.25, 42);
|
||||
let actions = drift_market_oracle(&bars);
|
||||
let long_count = actions.iter().filter(|a| matches!(a, OracleAction::Long(_))).count();
|
||||
// Strong uptrend → most actions should be Long
|
||||
assert!(long_count > 700, "drift_market_oracle Long count = {}, expected > 700", long_count);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ou_oracle_reverses_at_extremes() {
|
||||
let bars = ou_market(1000, 0.05, 1.0, 4500.0, 0.25, 42);
|
||||
let actions = ou_market_oracle(&bars, 4500.0, 1.0);
|
||||
// OU oracle should produce SOME reversion actions (not all Hold)
|
||||
let action_count = actions.iter().filter(|a| !matches!(a, OracleAction::Hold)).count();
|
||||
assert!(action_count > 0, "OU oracle produced zero non-Hold actions");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user