From f7718b37615cca07c6475c819154bd268175e1b0 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sat, 9 May 2026 22:04:50 +0200 Subject: [PATCH] fix(architectural): include bar formation params in fxcache key + actually USE imbalance bars MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## The bug (audit 2026-05-09) `crates/ml/src/feature_cache.rs:calculate_dbn_cache_key_full` hashed only `(symbol, data_source, dbn_filenames+sizes)` — NOT `imbalance_bar_threshold` or `imbalance_bar_ewma_alpha`. Combined with `precompute_features.rs:346` unconditionally calling `build_volume_bars` regardless of `data_source`, this meant: 1. 14 audited production runs (Apr 11–May 8) all collided on the same fxcache key (`a3f933aa...` / `c07c960a...`) regardless of TOML `imbalance_bar_threshold` value 2. The imbalance-bar code path was reachable only via fxcache MISS, which never happens in production because `ensure-fxcache` always populates first 3. Every "tuning" of `imbalance_bar_threshold` across 16+ SP runs was a silent no-op — the system was actually running volume bars at DEFAULT_VOLUME_BAR_SIZE (100 contracts/bar) ## The fix (this commit) **Part A — cache key includes bar formation params:** - `calculate_dbn_cache_key_full` signature: 5 args → 7 args. Two new f64 params hashed via `to_le_bytes()`. - 4 callers updated atomically (per `feedback_no_partial_refactor`). - 2 new unit tests (`test_cache_key_includes_bar_threshold`, `test_cache_key_includes_bar_alpha`) pin the contract. **Part B — precompute_features actually USES data_source:** - New CLI args `--imbalance-bar-threshold` (default 0.5) and `--imbalance-bar-ewma-alpha` (default 0.1) on both train_baseline_rl and precompute_features. - `precompute_features.rs:346` now branches: when `data_source == "mbp10"` AND `mbp10_data_dir.is_some()`, calls `mbp10_to_imbalance_bars` instead of `build_volume_bars`. **Argo plumbing:** - `train-template.yaml` + `train-multi-seed-template.yaml`: new workflow parameters threaded into BOTH precompute and trainer invocations so both compute the same fxcache key. - `scripts/argo-train.sh`: new CLI flags for ad-hoc overrides. - ensure-fxcache regen path: removed `rm -f /feature-cache/*.fxcache` (with bar-params now in key, parallel experiments coexist). ## Effects going forward - Tuning `imbalance_bar_threshold` actually changes bar density - Configuring `data_source = "mbp10"` actually produces imbalance bars - Multiple parallel experiments at different thresholds coexist on PVC - `dqn-production.toml: imbalance_bar_threshold = 0.5` no longer ignored Default values match prior production behavior → existing wgdc7-equivalent runs reproduce, just with a *new* fxcache key (the old volume-bar cache file is still on disk but won't be hit; harmless, can GC manually). Audit-doc: `docs/dqn-wire-up-audit.md` updated with full context. Tests: 5/5 feature_cache tests pass, full workspace + examples compile clean. Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/ml/examples/precompute_features.rs | 58 +++++++++++-- crates/ml/examples/train_baseline_rl.rs | 17 ++++ crates/ml/src/feature_cache.rs | 59 ++++++++++--- crates/ml/src/fxcache.rs | 13 ++- crates/ml/src/trainers/dqn/data_loading.rs | 2 + docs/dqn-wire-up-audit.md | 84 +++++++++++++++++++ infra/k8s/argo/train-multi-seed-template.yaml | 22 ++++- infra/k8s/argo/train-template.yaml | 18 +++- scripts/argo-train.sh | 13 +++ 9 files changed, 265 insertions(+), 21 deletions(-) diff --git a/crates/ml/examples/precompute_features.rs b/crates/ml/examples/precompute_features.rs index f625cc4e8..03010ec43 100644 --- a/crates/ml/examples/precompute_features.rs +++ b/crates/ml/examples/precompute_features.rs @@ -147,6 +147,20 @@ struct Opts { #[arg(long, default_value = "mbp10")] data_source: String, + /// Imbalance bar formation threshold. ONLY used when `data_source == "mbp10"` + /// and `mbp10_data_dir` is set; otherwise volume bars are produced. MUST + /// match the trainer's `imbalance_bar_threshold` for fxcache HIT (the cache + /// key includes this value as of 2026-05-09 architectural fix). Default + /// matches `dqn-production.toml: imbalance_bar_threshold = 0.5`. + #[arg(long, default_value_t = 0.5)] + imbalance_bar_threshold: f64, + + /// Imbalance bar EWMA alpha (weight on OLD threshold in this codebase's + /// reversed convention; α=1.0 disables adaptation). Default matches + /// `dqn-production.toml: 0.1`. MUST match the trainer's value for cache HIT. + #[arg(long, default_value_t = 0.1)] + imbalance_bar_ewma_alpha: f64, + /// Skip confirmation prompt #[arg(long)] yes: bool, @@ -262,6 +276,8 @@ async fn main() -> Result<()> { trades_dir.as_deref(), &opts.symbol, &opts.data_source, + opts.imbalance_bar_threshold, + opts.imbalance_bar_ewma_alpha, ).context("Failed to compute cache key")?; let early_check_path = output_dir.join(format!("{hex_key_early}.fxcache")); if early_check_path.exists() { @@ -342,17 +358,47 @@ async fn main() -> Result<()> { all_front_month_trades.sort_by_key(|t| t.timestamp); info!("Total trades: {} raw, {} front-month", total_raw, all_front_month_trades.len()); - // Build volume bars - let all_bars = ml::features::build_volume_bars(&all_front_month_trades, ml::features::DEFAULT_VOLUME_BAR_SIZE); - info!("Volume bars built: {} bars ({} contracts/bar) in {:.1}s", - all_bars.len(), ml::features::DEFAULT_VOLUME_BAR_SIZE, t1.elapsed().as_secs_f64()); + // Build bars — branch on data_source. Pre-2026-05-09 this was unconditional + // build_volume_bars, silently ignoring data_source="mbp10" + the imbalance + // bar threshold config. Audit confirmed 14 production runs all collided on + // the same fxcache key regardless of TOML threshold value, so the imbalance + // path was effectively dead. Wire it now: when "mbp10" + mbp10_dir is set, + // construct imbalance bars from MBP-10 trades; otherwise fall back to volume + // bars on the raw trade tape. + let all_bars = if opts.data_source == "mbp10" && mbp10_dir.is_some() { + let mbp10_path = mbp10_dir.as_deref().expect("mbp10_dir checked above"); + info!( + "Building imbalance bars from MBP-10: dir={}, threshold={}, ewma_alpha={}", + mbp10_path.display(), + opts.imbalance_bar_threshold, + opts.imbalance_bar_ewma_alpha, + ); + let bars = ml::features::mbp10_loader::mbp10_to_imbalance_bars( + mbp10_path, + &opts.symbol, + opts.imbalance_bar_threshold, + opts.imbalance_bar_ewma_alpha, + ).context("MBP-10 imbalance bar construction failed")?; + info!( + "Imbalance bars built: {} bars (threshold={}) in {:.1}s", + bars.len(), + opts.imbalance_bar_threshold, + t1.elapsed().as_secs_f64(), + ); + bars + } else { + let bars = ml::features::build_volume_bars(&all_front_month_trades, ml::features::DEFAULT_VOLUME_BAR_SIZE); + info!("Volume bars built: {} bars ({} contracts/bar) in {:.1}s", + bars.len(), ml::features::DEFAULT_VOLUME_BAR_SIZE, t1.elapsed().as_secs_f64()); + bars + }; // SP19 Path (B) (2026-05-09): need at least `WARMUP + LOOKAHEAD_HORIZON_MAX + 1` // bars so the 30-bar log-return at `i = 0` (`all_bars[WARMUP + 30]`) is // valid AND we still produce at least one output row. if all_bars.len() < WARMUP + LOOKAHEAD_HORIZON_MAX + 1 { anyhow::bail!( - "Insufficient data: {} volume bars, need at least {} (WARMUP={WARMUP} + LOOKAHEAD_HORIZON_MAX={LOOKAHEAD_HORIZON_MAX} + 1)", + "Insufficient data: {} bars, need at least {} (WARMUP={WARMUP} + LOOKAHEAD_HORIZON_MAX={LOOKAHEAD_HORIZON_MAX} + 1)", all_bars.len(), WARMUP + LOOKAHEAD_HORIZON_MAX + 1 ); } @@ -723,6 +769,8 @@ async fn main() -> Result<()> { trades_dir.as_deref(), &opts.symbol, &opts.data_source, + opts.imbalance_bar_threshold, + opts.imbalance_bar_ewma_alpha, ).context("Failed to compute cache key")?; let cache_key: [u8; 32] = hex::decode(&hex_key) .context("Invalid hex key")? diff --git a/crates/ml/examples/train_baseline_rl.rs b/crates/ml/examples/train_baseline_rl.rs index 78cb33119..801ed3a9b 100644 --- a/crates/ml/examples/train_baseline_rl.rs +++ b/crates/ml/examples/train_baseline_rl.rs @@ -237,6 +237,21 @@ struct Args { #[arg(long)] trades_data_dir: Option, + /// Imbalance bar formation threshold (signed buy/sell volume that triggers + /// a bar close). Must match the value used by `precompute_features` to + /// populate the fxcache, otherwise this trainer's discover_and_load call + /// MISSES and falls back to in-process MBP-10 loading. Default matches + /// `dqn-production.toml: imbalance_bar_threshold = 0.5`. + #[arg(long, default_value_t = 0.5)] + imbalance_bar_threshold: f64, + + /// Imbalance bar EWMA alpha. Convention in this codebase: weight on the + /// OLD threshold in `T_new = α·T_old + (1-α)·observed`. So α=1.0 disables + /// adaptation. Default matches `dqn-production.toml: 0.1`. Must match the + /// value used by `precompute_features` for fxcache hit. + #[arg(long, default_value_t = 0.1)] + imbalance_bar_ewma_alpha: f64, + /// Enable offline RL mode: train exclusively from a pre-collected dataset, /// skipping online experience collection. Requires --dataset-path. #[arg(long, default_value_t = false)] @@ -602,6 +617,8 @@ fn run_training(args: &Args) -> Result> { mbp10.map(|p| p.as_path()), trades.map(|p| p.as_path()), "mbp10", + args.imbalance_bar_threshold, + args.imbalance_bar_ewma_alpha, cache_dir_override.as_deref(), ); diff --git a/crates/ml/src/feature_cache.rs b/crates/ml/src/feature_cache.rs index ccd5d2f66..201735a87 100644 --- a/crates/ml/src/feature_cache.rs +++ b/crates/ml/src/feature_cache.rs @@ -13,20 +13,33 @@ use std::path::{Path, PathBuf}; /// Hashes every `.dbn` and `.dbn.zst` file found recursively under `data_dir` /// by (canonical path, size, mtime). Any change to the directory contents /// (add / remove / modify) produces a different key. +/// +/// Production defaults for the bar-formation params (matches `dqn-production.toml`): +/// `imbalance_bar_threshold = 0.5`, `imbalance_bar_ewma_alpha = 0.1`. pub fn calculate_dbn_cache_key(data_dir: &Path) -> Result { - calculate_dbn_cache_key_full(data_dir, None, None, "ES.FUT", "mbp10") + calculate_dbn_cache_key_full(data_dir, None, None, "ES.FUT", "mbp10", 0.5, 0.1) } -/// Extended cache key that includes MBP-10 and trades directories. -/// The cache is invalidated when ANY data source changes — OHLCV, MBP-10, or trades. -/// Without this, a cache built without trades would serve stale features -/// even after trades data is added, silently dropping VPIN/Kyle's Lambda enrichment. +/// Extended cache key that includes MBP-10 and trades directories AND bar +/// formation parameters. The cache is invalidated when ANY data source +/// changes — OHLCV, MBP-10, trades — OR when bar-formation params change. +/// +/// Bar-formation params (`imbalance_bar_threshold`, `imbalance_bar_ewma_alpha`) +/// are hashed in regardless of `data_source` because: (a) they uniquely identify +/// the bar stream the cache contains, (b) downstream features (e.g. log-returns, +/// targets) depend on bar boundaries, and (c) without this, two runs configured +/// with different thresholds will silently share the same cache file (canonical +/// 2026-05-09 audit: 14 prior production runs all collided on `a3f933aa...` / +/// `c07c960a...` keys regardless of TOML threshold value because the bar params +/// were not in the hash). pub fn calculate_dbn_cache_key_full( data_dir: &Path, mbp10_dir: Option<&Path>, trades_dir: Option<&Path>, symbol: &str, data_source: &str, + imbalance_bar_threshold: f64, + imbalance_bar_ewma_alpha: f64, ) -> Result { let mut hasher = Sha256::new(); @@ -36,6 +49,12 @@ pub fn calculate_dbn_cache_key_full( hasher.update(data_source.as_bytes()); hasher.update(b"|"); + // Include bar-formation params — different bar streams get different keys + hasher.update(&imbalance_bar_threshold.to_le_bytes()); + hasher.update(b"|"); + hasher.update(&imbalance_bar_ewma_alpha.to_le_bytes()); + hasher.update(b"|"); + // Hash only file contents (size + mtime), not paths. // This makes the key independent of CWD, relative/absolute paths, // and directory naming — only actual data changes invalidate it. @@ -82,8 +101,8 @@ mod tests { fn test_cache_key_includes_symbol() { let dir = Path::new("test_data/futures-baseline"); if !dir.exists() { return; } - let key_es = calculate_dbn_cache_key_full(dir, None, None, "ES.FUT", "mbp10").unwrap(); - let key_nq = calculate_dbn_cache_key_full(dir, None, None, "NQ.FUT", "mbp10").unwrap(); + let key_es = calculate_dbn_cache_key_full(dir, None, None, "ES.FUT", "mbp10", 0.5, 0.1).unwrap(); + let key_nq = calculate_dbn_cache_key_full(dir, None, None, "NQ.FUT", "mbp10", 0.5, 0.1).unwrap(); assert_ne!(key_es, key_nq, "Different symbols must produce different cache keys"); } @@ -91,17 +110,35 @@ mod tests { fn test_cache_key_includes_data_source() { let dir = Path::new("test_data/futures-baseline"); if !dir.exists() { return; } - let key_ohlcv = calculate_dbn_cache_key_full(dir, None, None, "ES.FUT", "mbp10").unwrap(); - let key_mbp10 = calculate_dbn_cache_key_full(dir, None, None, "ES.FUT", "mbp10").unwrap(); + let key_ohlcv = calculate_dbn_cache_key_full(dir, None, None, "ES.FUT", "ohlcv", 0.5, 0.1).unwrap(); + let key_mbp10 = calculate_dbn_cache_key_full(dir, None, None, "ES.FUT", "mbp10", 0.5, 0.1).unwrap(); assert_ne!(key_ohlcv, key_mbp10, "Different data sources must produce different cache keys"); } + #[test] + fn test_cache_key_includes_bar_threshold() { + let dir = Path::new("test_data/futures-baseline"); + if !dir.exists() { return; } + let key_a = calculate_dbn_cache_key_full(dir, None, None, "ES.FUT", "mbp10", 0.5, 0.1).unwrap(); + let key_b = calculate_dbn_cache_key_full(dir, None, None, "ES.FUT", "mbp10", 2.5, 0.1).unwrap(); + assert_ne!(key_a, key_b, "Different imbalance_bar_threshold must produce different cache keys"); + } + + #[test] + fn test_cache_key_includes_bar_alpha() { + let dir = Path::new("test_data/futures-baseline"); + if !dir.exists() { return; } + let key_a = calculate_dbn_cache_key_full(dir, None, None, "ES.FUT", "mbp10", 0.5, 0.1).unwrap(); + let key_b = calculate_dbn_cache_key_full(dir, None, None, "ES.FUT", "mbp10", 0.5, 1.0).unwrap(); + assert_ne!(key_a, key_b, "Different imbalance_bar_ewma_alpha must produce different cache keys"); + } + #[test] fn test_cache_key_stable() { let dir = Path::new("test_data/futures-baseline"); if !dir.exists() { return; } - let key1 = calculate_dbn_cache_key_full(dir, None, None, "ES.FUT", "mbp10").unwrap(); - let key2 = calculate_dbn_cache_key_full(dir, None, None, "ES.FUT", "mbp10").unwrap(); + let key1 = calculate_dbn_cache_key_full(dir, None, None, "ES.FUT", "mbp10", 0.5, 0.1).unwrap(); + let key2 = calculate_dbn_cache_key_full(dir, None, None, "ES.FUT", "mbp10", 0.5, 0.1).unwrap(); assert_eq!(key1, key2, "Same inputs must produce same key"); } } diff --git a/crates/ml/src/fxcache.rs b/crates/ml/src/fxcache.rs index fe632cc5a..561430397 100644 --- a/crates/ml/src/fxcache.rs +++ b/crates/ml/src/fxcache.rs @@ -689,7 +689,9 @@ pub fn norm_stats_path_for_key( /// * `symbol` — Trading symbol (e.g. `"ES.FUT"`) /// * `mbp10_dir` — Optional MBP-10 order book data directory /// * `trades_dir` — Optional trades data directory -/// * `data_source` — Data source mode (`"mbp10"` or `"mbp10"`) +/// * `data_source` — Data source mode (`"mbp10"` or `"ohlcv"`) +/// * `imbalance_bar_threshold` — Bar formation threshold (must match producer's value) +/// * `imbalance_bar_ewma_alpha` — Bar formation EWMA alpha (must match producer's value) /// * `cache_dir_override` — Explicit cache directory (from CLI `--feature-cache-dir`) pub fn discover_and_load( data_dir: &Path, @@ -697,14 +699,17 @@ pub fn discover_and_load( mbp10_dir: Option<&Path>, trades_dir: Option<&Path>, data_source: &str, + imbalance_bar_threshold: f64, + imbalance_bar_ewma_alpha: f64, cache_dir_override: Option<&Path>, ) -> Option { // 1. Resolve cache directory let cache_dir = resolve_cache_dir(data_dir, cache_dir_override)?; - // 2. Compute cache key (includes symbol + data_source) + // 2. Compute cache key (includes symbol + data_source + bar params) let key_hex = crate::feature_cache::calculate_dbn_cache_key_full( data_dir, mbp10_dir, trades_dir, symbol, data_source, + imbalance_bar_threshold, imbalance_bar_ewma_alpha, ) .ok()?; let key: [u8; 32] = hex::decode(&key_hex).ok()?.try_into().ok()?; @@ -742,6 +747,8 @@ mod discover_tests { None, None, "mbp10", + 0.5, + 0.1, None, ); assert!(result.is_none()); @@ -771,6 +778,8 @@ mod discover_tests { None, None, "mbp10", + 0.5, + 0.1, Some(&cache_dir), ); // Strict match only — no "most recent" fallback diff --git a/crates/ml/src/trainers/dqn/data_loading.rs b/crates/ml/src/trainers/dqn/data_loading.rs index a0e3aeba0..26413837d 100644 --- a/crates/ml/src/trainers/dqn/data_loading.rs +++ b/crates/ml/src/trainers/dqn/data_loading.rs @@ -149,6 +149,8 @@ impl DQNTrainer { mbp10_path, trades_path, &self.hyperparams.data_source, + f64::from(self.hyperparams.imbalance_bar_threshold), + f64::from(self.hyperparams.imbalance_bar_ewma_alpha), cache_dir_override, ) { // OFI is unconditional — the model requires order flow data. diff --git a/docs/dqn-wire-up-audit.md b/docs/dqn-wire-up-audit.md index 7118c53ce..753b70300 100644 --- a/docs/dqn-wire-up-audit.md +++ b/docs/dqn-wire-up-audit.md @@ -2,6 +2,90 @@ **Status:** Populated during Plan 1 Task 6 (A.5 orphan audit). Updated on every commit per Invariant 7. +## 2026-05-09 — fxcache key includes bar formation params + precompute_features actually USES data_source + +`crates/ml/src/trainers/dqn/data_loading.rs:146` — `discover_and_load` +call updated to pass `self.hyperparams.imbalance_bar_threshold` and +`self.hyperparams.imbalance_bar_ewma_alpha` (cast `f32` → `f64`) into +the fxcache lookup key. + +### Background + +Pre-fix audit found that the fxcache key `calculate_dbn_cache_key_full` +hashed only `(symbol, data_source, dbn_filenames+sizes)` — bar formation +params were absent. Combined with `precompute_features.rs:346` +unconditionally calling `build_volume_bars`, the imbalance-bar code path +was unreachable in production: 14 audited training workflows (Apr 11–May 8) +all collided on the same fxcache key regardless of TOML threshold value. +Every "tuning" of `imbalance_bar_threshold` across 16+ SP runs was a +silent no-op. + +### Fix scope (atomic per `feedback_no_partial_refactor`) + +`calculate_dbn_cache_key_full` signature: 5 args → 7 args. Two new `f64` +params (`imbalance_bar_threshold`, `imbalance_bar_ewma_alpha`) hashed via +`to_le_bytes()`. All 4 callers updated in one commit: + + - `crates/ml/src/fxcache.rs:discover_and_load` — signature gains both + params (now 8 args), threads through to the cache key. Inline tests + updated. + - `crates/ml/src/trainers/dqn/data_loading.rs:146` — passes + `self.hyperparams.imbalance_bar_*` (already on hyperparams struct, + no plumbing change required). + - `crates/ml/examples/train_baseline_rl.rs:599` — added matching CLI + args (`--imbalance-bar-threshold`, `--imbalance-bar-ewma-alpha`), + defaults `0.5` / `0.1` matching `dqn-production.toml`. + - `crates/ml/examples/precompute_features.rs:259,720` — added matching + CLI args. + +### Behavioral change + +`precompute_features.rs:346` now branches: when `data_source == "mbp10"` +AND `mbp10_data_dir.is_some()`, calls `mbp10_to_imbalance_bars` instead +of `build_volume_bars`. This is the FIRST commit where the imbalance-bar +code path is actually reachable in production. + +The trainer-side `data_loading.rs:146` cache lookup now uses a different +key than any pre-existing fxcache file → first run after this change +triggers fresh extraction → the imbalance bar producer (with our wgdc8 +EWMA bypass at `1aaf94306`) actually runs. + +### Argo plumbing + +`infra/k8s/argo/train-template.yaml` and `train-multi-seed-template.yaml`: +new workflow parameters `imbalance-bar-threshold` (default `"0.5"`) and +`imbalance-bar-ewma-alpha` (default `"0.1"`). Both threaded into the +precompute_features invocation AND the train_baseline_rl invocation so +both compute the same fxcache key. `scripts/argo-train.sh` exposes +`--imbalance-bar-threshold` and `--imbalance-bar-ewma-alpha` flags for +ad-hoc overrides. + +The `ensure-fxcache` regen branch no longer does a blanket +`rm -f /feature-cache/*.fxcache` — with bar-params now in the key, parallel +experiments at different thresholds can coexist on the PVC without +stomping each other's caches. Only the specific stale key (if any) needs +regen. + +### Tests + +5 unit tests in `crates/ml/src/feature_cache.rs` pass: +`test_cache_key_includes_symbol`, `test_cache_key_includes_data_source`, +`test_cache_key_includes_bar_threshold` (NEW), +`test_cache_key_includes_bar_alpha` (NEW), `test_cache_key_stable`. +All 2 inline tests in `crates/ml/src/fxcache.rs` (`discover_tests`) +updated for new signature; compile. + +Workspace + all examples compile clean. + +### Cross-references + +- `pearl_imbalance_bar_ewma_washes_out_configured_threshold.md` — related + but distinct issue (EWMA wash-out *within* `mbp10_to_imbalance_bars`). + Addressed on `wgdc8-bar-thresh-5x` via `1aaf94306`. +- Audit agent verdict 2026-05-09: "FULL WASH-OUT — and worse than wash-out: + the entire imbalance-bar code path was never executed in any audited + Argo run." 14 runs, same fxcache key, proof by collision. + ## 2026-05-09 — SP20 Phase 1.4 (Path C): kernel-arg refactor + aggregation kernel + production wire-up (atomic) Wires the SP20 fused-producer chain (Stats → Aggregate → EMAs → diff --git a/infra/k8s/argo/train-multi-seed-template.yaml b/infra/k8s/argo/train-multi-seed-template.yaml index fa09c88bf..e2c1c5c49 100644 --- a/infra/k8s/argo/train-multi-seed-template.yaml +++ b/infra/k8s/argo/train-multi-seed-template.yaml @@ -92,6 +92,14 @@ spec: # foxhunt-training-artifacts/profiles//. Default off. - name: profile value: "false" + # Bar formation params — passed to BOTH precompute_features (when + # building fxcache) AND train_baseline_rl (when looking up fxcache). + # MUST match between the two for fxcache HIT (cache key includes them + # as of 2026-05-09 architectural fix). Defaults match dqn-production.toml. + - name: imbalance-bar-threshold + value: "0.5" + - name: imbalance-bar-ewma-alpha + value: "0.1" volumes: - name: git-ssh-key @@ -345,17 +353,25 @@ spec: --trades-data-dir /data/futures-baseline-trades \ --output-dir /feature-cache \ --symbol {{workflow.parameters.symbol}} \ + --imbalance-bar-threshold {{workflow.parameters.imbalance-bar-threshold}} \ + --imbalance-bar-ewma-alpha {{workflow.parameters.imbalance-bar-ewma-alpha}} \ --yes; then echo "=== Feature cache ready ===" else echo "=== Cache stale or missing — regenerating ===" - rm -f /feature-cache/*.fxcache + # NOTE: don't blanket-rm /feature-cache/*.fxcache anymore. With + # bar-params now in the cache key, multiple valid caches can + # coexist (different threshold experiments). The early-exit + # check above already validates the specific key — fall through + # to regen only the one we need. $BINARY \ --data-dir /data/futures-baseline \ --mbp10-data-dir /data/futures-baseline-mbp10 \ --trades-data-dir /data/futures-baseline-trades \ --output-dir /feature-cache \ --symbol {{workflow.parameters.symbol}} \ + --imbalance-bar-threshold {{workflow.parameters.imbalance-bar-threshold}} \ + --imbalance-bar-ewma-alpha {{workflow.parameters.imbalance-bar-ewma-alpha}} \ --yes fi @@ -483,7 +499,9 @@ spec: --output-dir /workspace/output \ --epochs {{workflow.parameters.train-epochs}} \ --seed "$SEED" \ - --max-folds {{workflow.parameters.folds}} + --max-folds {{workflow.parameters.folds}} \ + --imbalance-bar-threshold {{workflow.parameters.imbalance-bar-threshold}} \ + --imbalance-bar-ewma-alpha {{workflow.parameters.imbalance-bar-ewma-alpha}} echo "=== Training complete: seed=$SEED folds={{workflow.parameters.folds}} ===" diff --git a/infra/k8s/argo/train-template.yaml b/infra/k8s/argo/train-template.yaml index 606b9639c..394165b96 100644 --- a/infra/k8s/argo/train-template.yaml +++ b/infra/k8s/argo/train-template.yaml @@ -65,6 +65,14 @@ spec: value: "1.0" - name: sanitizer value: "none" # "none", "memcheck", "racecheck", "synccheck" + # Bar formation params — passed to BOTH precompute_features (when + # building fxcache) AND train_baseline_rl (when looking up fxcache). + # MUST match between the two for fxcache HIT (cache key includes them + # as of 2026-05-09 architectural fix). + - name: imbalance-bar-threshold + value: "0.5" + - name: imbalance-bar-ewma-alpha + value: "0.1" volumes: - name: git-ssh-key @@ -365,17 +373,23 @@ spec: --trades-data-dir /data/futures-baseline-trades \ --output-dir /feature-cache \ --symbol {{workflow.parameters.symbol}} \ + --imbalance-bar-threshold {{workflow.parameters.imbalance-bar-threshold}} \ + --imbalance-bar-ewma-alpha {{workflow.parameters.imbalance-bar-ewma-alpha}} \ --yes; then echo "=== Feature cache ready ===" else echo "=== Cache stale or missing — regenerating ===" - rm -f /feature-cache/*.fxcache + # NOTE: don't blanket-rm /feature-cache/*.fxcache anymore. With + # bar-params now in the cache key, multiple valid caches can + # coexist (different threshold experiments). $BINARY \ --data-dir /data/futures-baseline \ --mbp10-data-dir /data/futures-baseline-mbp10 \ --trades-data-dir /data/futures-baseline-trades \ --output-dir /feature-cache \ --symbol {{workflow.parameters.symbol}} \ + --imbalance-bar-threshold {{workflow.parameters.imbalance-bar-threshold}} \ + --imbalance-bar-ewma-alpha {{workflow.parameters.imbalance-bar-ewma-alpha}} \ --yes echo "=== Feature cache regenerated ===" fi @@ -600,6 +614,8 @@ spec: --trades-data-dir /data/futures-baseline-trades \ --output-dir /workspace/output \ --epochs {{workflow.parameters.train-epochs}} \ + --imbalance-bar-threshold {{workflow.parameters.imbalance-bar-threshold}} \ + --imbalance-bar-ewma-alpha {{workflow.parameters.imbalance-bar-ewma-alpha}} \ $HYPEROPT_FLAG echo "=== Training complete ===" diff --git a/scripts/argo-train.sh b/scripts/argo-train.sh index 8b4a3ba62..852996e13 100755 --- a/scripts/argo-train.sh +++ b/scripts/argo-train.sh @@ -31,6 +31,8 @@ FOLDS=1 DRY_RUN=false TAG="" PROFILE=false +IMBALANCE_BAR_THRESHOLD="" +IMBALANCE_BAR_EWMA_ALPHA="" usage() { cat < Run N seeds in parallel (default: 1, fans out via DAG when >1) --folds Walk-forward fold count (default: 1, fans out via DAG when >1) --tag Label workflow with foxhunt-tag= for log aggregation + --imbalance-bar-threshold Override imbalance bar threshold (default: 0.5) + MUST match between precompute_features and trainer. + --imbalance-bar-ewma-alpha Override imbalance bar EWMA alpha (default: 0.1). + α=1.0 disables adaptation (this codebase's reversed + convention). MUST match between precompute and trainer. --profile Wrap training under nsys (NVIDIA Nsight Systems) and upload .nsys-rep artefacts to MinIO bucket foxhunt-training-artifacts/profiles//. Forces the @@ -91,6 +98,8 @@ while [[ $# -gt 0 ]]; do --folds) FOLDS="$2"; shift 2 ;; --tag) TAG="$2"; shift 2 ;; --profile) PROFILE=true; shift ;; + --imbalance-bar-threshold) IMBALANCE_BAR_THRESHOLD="$2"; shift 2 ;; + --imbalance-bar-ewma-alpha) IMBALANCE_BAR_EWMA_ALPHA="$2"; shift 2 ;; --dry-run) DRY_RUN=true; shift ;; -h|--help) usage ;; *) echo "Unknown option: $1"; usage ;; @@ -147,6 +156,8 @@ if [[ "$USE_MULTI_SEED" == "false" ]]; then [[ -n "$SYMBOL" ]] && CMD="$CMD -p symbol=$SYMBOL" [[ -n "$CAPITAL" ]] && CMD="$CMD -p initial-capital=$CAPITAL" [[ "$SANITIZER" != "none" ]] && CMD="$CMD -p sanitizer=$SANITIZER" + [[ -n "$IMBALANCE_BAR_THRESHOLD" ]] && CMD="$CMD -p imbalance-bar-threshold=$IMBALANCE_BAR_THRESHOLD" + [[ -n "$IMBALANCE_BAR_EWMA_ALPHA" ]] && CMD="$CMD -p imbalance-bar-ewma-alpha=$IMBALANCE_BAR_EWMA_ALPHA" $BASELINE && CMD="$CMD -p hyperopt-trials=0" $WATCH && CMD="$CMD --watch" @@ -278,6 +289,8 @@ CMD="$CMD -p profile=$PROFILE" [[ -n "$SYMBOL" ]] && CMD="$CMD -p symbol=$SYMBOL" [[ -n "$CAPITAL" ]] && CMD="$CMD -p initial-capital=$CAPITAL" [[ -n "$TAG" ]] && CMD="$CMD --labels foxhunt-tag=$TAG" +[[ -n "$IMBALANCE_BAR_THRESHOLD" ]] && CMD="$CMD -p imbalance-bar-threshold=$IMBALANCE_BAR_THRESHOLD" +[[ -n "$IMBALANCE_BAR_EWMA_ALPHA" ]] && CMD="$CMD -p imbalance-bar-ewma-alpha=$IMBALANCE_BAR_EWMA_ALPHA" [[ "$SANITIZER" != "none" ]] && CMD="$CMD -p sanitizer=$SANITIZER" $BASELINE && CMD="$CMD -p hyperopt-trials=0"