From 94f8f65571a3062e58f585f8027ae24be1ce3983 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Fri, 15 May 2026 12:25:23 +0200 Subject: [PATCH] feat(alpha): 9-action discrete execution action space + legality gating MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase E Task 3. Introduces crates/ml/src/env/ for the execution-policy environment. This commit lands the action space only; fill_model (Task 4) and execution_env (Task 6) append to mod.rs in their respective commits per feedback_wire_everything_up (no orphan declarations). - 9-action discrete space: Wait + {Buy,Sell,Flat} × {Market,L1,L2 / L1} - Soft action masking: illegal actions degrade to Wait in env::step (not by masking Q-output) per design memo — preserves clean C51 categorical targets and Munchausen term (Task 10) - `is_legal(position)` with position ∈ {-1, 0, +1} (sign only; magnitude is decoupled into the Kelly layer in Task 11) - repr(u8) discriminants round-trip with from_u8 for replay-buffer storage by the existing GPU DQN trainer 4 unit tests: - n_actions_matches_enum_cardinality - legality_gating_by_position (covers flat/long/short × all 9 actions) - decode_closing_flag_is_only_flat - repr_u8_round_trip (replay-buffer contract) `cargo test -p ml --lib env::action_space`: 4 passed. --- crates/ml/src/env/action_space.rs | 206 ++++++++++++++++++++++++++++++ crates/ml/src/env/mod.rs | 14 ++ crates/ml/src/lib.rs | 1 + 3 files changed, 221 insertions(+) create mode 100644 crates/ml/src/env/action_space.rs create mode 100644 crates/ml/src/env/mod.rs diff --git a/crates/ml/src/env/action_space.rs b/crates/ml/src/env/action_space.rs new file mode 100644 index 000000000..974b972d7 --- /dev/null +++ b/crates/ml/src/env/action_space.rs @@ -0,0 +1,206 @@ +//! Phase E execution-policy action space. +//! +//! 9 discrete actions per the design memo. Size is decoupled (handled by +//! the Kelly layer in Task 11); the action decides ONLY placement and side. +//! `Wait` emits no order; the others emit one limit/market order at the +//! specified level with sign = side. +//! +//! Mapping: +//! +//! 0 Wait +//! 1 BuyMarket (aggressive, crosses spread) +//! 2 BuyL1 (passive, joins quote) +//! 3 BuyL2 (passive, behind quote) +//! 4 SellMarket (aggressive, crosses spread) +//! 5 SellL1 (passive, joins quote) +//! 6 SellL2 (passive, behind quote) +//! 7 FlatMarket (close current position aggressively) +//! 8 FlatL1 (close current position passively at L1) +//! +//! Action gating: actions 1-3 are only legal when `position <= 0` (not long); +//! actions 4-6 are only legal when `position >= 0` (not short); 7-8 are only +//! legal when `position != 0`. Illegal actions degrade to Wait inside the env, +//! NOT by masking the network's Q-output — soft masking preserves clean +//! C51 distributional targets and the Munchausen term (Task 10). + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(u8)] +pub enum ExecAction { + Wait = 0, + BuyMarket = 1, + BuyL1 = 2, + BuyL2 = 3, + SellMarket = 4, + SellL1 = 5, + SellL2 = 6, + FlatMarket = 7, + FlatL1 = 8, +} + +pub const N_ACTIONS: usize = 9; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Side { + None, + Buy, + Sell, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Placement { + None, + Market, + L1, + L2, +} + +#[derive(Debug, Clone, Copy)] +pub struct DecodedAction { + pub side: Side, + pub placement: Placement, + pub closing: bool, +} + +impl ExecAction { + pub fn from_u8(i: u8) -> Option { + match i { + 0 => Some(Self::Wait), + 1 => Some(Self::BuyMarket), + 2 => Some(Self::BuyL1), + 3 => Some(Self::BuyL2), + 4 => Some(Self::SellMarket), + 5 => Some(Self::SellL1), + 6 => Some(Self::SellL2), + 7 => Some(Self::FlatMarket), + 8 => Some(Self::FlatL1), + _ => None, + } + } + + pub fn decode(self) -> DecodedAction { + match self { + Self::Wait => DecodedAction { + side: Side::None, + placement: Placement::None, + closing: false, + }, + Self::BuyMarket => DecodedAction { + side: Side::Buy, + placement: Placement::Market, + closing: false, + }, + Self::BuyL1 => DecodedAction { + side: Side::Buy, + placement: Placement::L1, + closing: false, + }, + Self::BuyL2 => DecodedAction { + side: Side::Buy, + placement: Placement::L2, + closing: false, + }, + Self::SellMarket => DecodedAction { + side: Side::Sell, + placement: Placement::Market, + closing: false, + }, + Self::SellL1 => DecodedAction { + side: Side::Sell, + placement: Placement::L1, + closing: false, + }, + Self::SellL2 => DecodedAction { + side: Side::Sell, + placement: Placement::L2, + closing: false, + }, + Self::FlatMarket => DecodedAction { + side: Side::None, + placement: Placement::Market, + closing: true, + }, + Self::FlatL1 => DecodedAction { + side: Side::None, + placement: Placement::L1, + closing: true, + }, + } + } + + /// `position ∈ {-1, 0, +1}` (sign only; magnitude lives in the Kelly layer). + /// At position=0, both Buy and Sell sides are legal (the network picks one). + pub fn is_legal(self, position: i32) -> bool { + match self { + Self::Wait => true, + Self::BuyMarket | Self::BuyL1 | Self::BuyL2 => position <= 0, + Self::SellMarket | Self::SellL1 | Self::SellL2 => position >= 0, + Self::FlatMarket | Self::FlatL1 => position != 0, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn n_actions_matches_enum_cardinality() { + assert_eq!(N_ACTIONS, 9); + for i in 0..N_ACTIONS as u8 { + assert!(ExecAction::from_u8(i).is_some(), "action {i} missing"); + } + assert!(ExecAction::from_u8(9).is_none()); + } + + #[test] + fn legality_gating_by_position() { + // Flat (position=0): can buy or sell, can't close. + assert!(ExecAction::BuyMarket.is_legal(0)); + assert!(ExecAction::SellMarket.is_legal(0)); + assert!(ExecAction::BuyL1.is_legal(0)); + assert!(ExecAction::SellL2.is_legal(0)); + assert!(!ExecAction::FlatMarket.is_legal(0)); + assert!(!ExecAction::FlatL1.is_legal(0)); + + // Long (position=+1): can sell or close, can't buy more. + assert!(!ExecAction::BuyMarket.is_legal(1)); + assert!(!ExecAction::BuyL1.is_legal(1)); + assert!(!ExecAction::BuyL2.is_legal(1)); + assert!(ExecAction::SellMarket.is_legal(1)); + assert!(ExecAction::FlatMarket.is_legal(1)); + assert!(ExecAction::FlatL1.is_legal(1)); + + // Short (position=-1): can buy or close, can't sell more. + assert!(ExecAction::BuyMarket.is_legal(-1)); + assert!(!ExecAction::SellMarket.is_legal(-1)); + assert!(!ExecAction::SellL1.is_legal(-1)); + assert!(!ExecAction::SellL2.is_legal(-1)); + assert!(ExecAction::FlatMarket.is_legal(-1)); + assert!(ExecAction::FlatL1.is_legal(-1)); + + // Wait is always legal. + for p in [-1, 0, 1] { + assert!(ExecAction::Wait.is_legal(p)); + } + } + + #[test] + fn decode_closing_flag_is_only_flat() { + for i in 0..N_ACTIONS as u8 { + let a = ExecAction::from_u8(i).expect("valid action index"); + let d = a.decode(); + let is_flat = matches!(a, ExecAction::FlatMarket | ExecAction::FlatL1); + assert_eq!(d.closing, is_flat, "closing flag mismatch for {a:?}"); + } + } + + #[test] + fn repr_u8_round_trip() { + // The discriminant matches from_u8 — important because the GPU + // trainer writes action indices as u8 into the replay buffer. + for i in 0..N_ACTIONS as u8 { + let a = ExecAction::from_u8(i).expect("valid action index"); + assert_eq!(a as u8, i); + } + } +} diff --git a/crates/ml/src/env/mod.rs b/crates/ml/src/env/mod.rs new file mode 100644 index 000000000..51f3f8e8c --- /dev/null +++ b/crates/ml/src/env/mod.rs @@ -0,0 +1,14 @@ +//! Phase E execution-policy environment. +//! +//! Snapshot-stream replay with a Poisson-regression fill simulator and +//! discrete action space. Trains the existing GPU DQN trainer on order- +//! placement decisions for a given alpha signal + LOB state. +//! +//! Module manifest (lands incrementally per the Phase E plan): +//! - `action_space` (Task 3, this commit) — 9 discrete actions +//! - `fill_model` (Task 4) — Poisson regression fill simulator +//! - `execution_env` (Task 6) — 10-dim observation, step/reset loop + +pub mod action_space; + +pub use action_space::{DecodedAction, ExecAction, N_ACTIONS, Placement, Side}; diff --git a/crates/ml/src/lib.rs b/crates/ml/src/lib.rs index a32a33c10..e8639f8d3 100644 --- a/crates/ml/src/lib.rs +++ b/crates/ml/src/lib.rs @@ -231,6 +231,7 @@ pub mod dqn; // gpu re-exported from ml-core (see below) pub mod diffusion; pub mod ensemble; +pub mod env; // Phase E execution-policy env (action space, fill model, ExecutionEnv) pub mod evaluation; // DQN evaluation engine (backtest metrics, Sharpe ratio) pub mod flash_attention; pub mod hyperopt; // Bayesian hyperparameter optimization (egobox)