diff --git a/crates/ml/examples/precompute_features.rs b/crates/ml/examples/precompute_features.rs index 03010ec43..6d8032e00 100644 --- a/crates/ml/examples/precompute_features.rs +++ b/crates/ml/examples/precompute_features.rs @@ -161,6 +161,12 @@ struct Opts { #[arg(long, default_value_t = 0.1)] imbalance_bar_ewma_alpha: f64, + /// Volume bar size: contracts of one-sided volume per bar. Used when + /// `data_source != "mbp10"`. Default 100 matches `DEFAULT_VOLUME_BAR_SIZE`. + /// MUST match the trainer's value for cache HIT. + #[arg(long, default_value_t = 100)] + volume_bar_size: u64, + /// Skip confirmation prompt #[arg(long)] yes: bool, @@ -278,6 +284,7 @@ async fn main() -> Result<()> { &opts.data_source, opts.imbalance_bar_threshold, opts.imbalance_bar_ewma_alpha, + opts.volume_bar_size, ).context("Failed to compute cache key")?; let early_check_path = output_dir.join(format!("{hex_key_early}.fxcache")); if early_check_path.exists() { @@ -387,9 +394,9 @@ async fn main() -> Result<()> { ); bars } else { - let bars = ml::features::build_volume_bars(&all_front_month_trades, ml::features::DEFAULT_VOLUME_BAR_SIZE); + let bars = ml::features::build_volume_bars(&all_front_month_trades, opts.volume_bar_size); info!("Volume bars built: {} bars ({} contracts/bar) in {:.1}s", - bars.len(), ml::features::DEFAULT_VOLUME_BAR_SIZE, t1.elapsed().as_secs_f64()); + bars.len(), opts.volume_bar_size, t1.elapsed().as_secs_f64()); bars }; @@ -528,7 +535,17 @@ async fn main() -> Result<()> { all_snapshots.sort_by_key(|s| s.timestamp); info!("Loaded {} MBP-10 snapshots in {:.1}s", all_snapshots.len(), t2.elapsed().as_secs_f64()); - // Load trades (parallel) + // Load trades (parallel) and filter front-month per-file. + // + // Pre-2026-05-09: this branch loaded trades unfiltered, leaking + // off-contract trade activity into the per-bar OFI/VPIN/Kyle's + // Lambda computations during contract rollover windows. Volume + // bar formation already filters front-month at line 354 of this + // file; the OFI side did not. Audit confirmed the latent bug + // affected every prior MBP-10+trades production run. + // + // Fix: mirror the per-file `filter_front_month` call from the + // volume bar path so OFI windows only see in-month trades. let all_trades = if let Some(ref tdir) = trades_dir { let mut trade_files = collect_dbn_files_recursive(tdir); trade_files.sort(); @@ -536,7 +553,13 @@ async fn main() -> Result<()> { use rayon::prelude::*; trade_files.par_iter().filter_map(|path| { match load_trades_sync(path) { - Ok(t) => { info!(" {} trades from {:?}", t.len(), path.file_name()); Some(t) } + Ok(t) => { + let raw_count = t.len(); + let filtered = ml::features::filter_front_month(&t); + info!(" {} trades from {:?} ({} front-month)", + raw_count, path.file_name(), filtered.len()); + Some(filtered) + } Err(e) => { tracing::warn!(" Failed {:?}: {e}", path.file_name()); None } } }).collect() @@ -771,6 +794,7 @@ async fn main() -> Result<()> { &opts.data_source, opts.imbalance_bar_threshold, opts.imbalance_bar_ewma_alpha, + opts.volume_bar_size, ).context("Failed to compute cache key")?; let cache_key: [u8; 32] = hex::decode(&hex_key) .context("Invalid hex key")? diff --git a/crates/ml/examples/train_baseline_rl.rs b/crates/ml/examples/train_baseline_rl.rs index 801ed3a9b..affd958ca 100644 --- a/crates/ml/examples/train_baseline_rl.rs +++ b/crates/ml/examples/train_baseline_rl.rs @@ -252,6 +252,12 @@ struct Args { #[arg(long, default_value_t = 0.1)] imbalance_bar_ewma_alpha: f64, + /// Volume bar size: contracts of one-sided volume per bar. Used when + /// `data_source != "mbp10"`. Default 100. MUST match precompute's value + /// for cache HIT. + #[arg(long, default_value_t = 100)] + volume_bar_size: u64, + /// Enable offline RL mode: train exclusively from a pre-collected dataset, /// skipping online experience collection. Requires --dataset-path. #[arg(long, default_value_t = false)] @@ -619,6 +625,7 @@ fn run_training(args: &Args) -> Result> { "mbp10", args.imbalance_bar_threshold, args.imbalance_bar_ewma_alpha, + args.volume_bar_size, cache_dir_override.as_deref(), ); diff --git a/crates/ml/src/feature_cache.rs b/crates/ml/src/feature_cache.rs index 201735a87..fd6834ffa 100644 --- a/crates/ml/src/feature_cache.rs +++ b/crates/ml/src/feature_cache.rs @@ -15,23 +15,24 @@ use std::path::{Path, PathBuf}; /// (add / remove / modify) produces a different key. /// /// Production defaults for the bar-formation params (matches `dqn-production.toml`): -/// `imbalance_bar_threshold = 0.5`, `imbalance_bar_ewma_alpha = 0.1`. +/// `imbalance_bar_threshold = 0.5`, `imbalance_bar_ewma_alpha = 0.1`, +/// `volume_bar_size = 100` (matches `DEFAULT_VOLUME_BAR_SIZE`). pub fn calculate_dbn_cache_key(data_dir: &Path) -> Result { - calculate_dbn_cache_key_full(data_dir, None, None, "ES.FUT", "mbp10", 0.5, 0.1) + calculate_dbn_cache_key_full(data_dir, None, None, "ES.FUT", "mbp10", 0.5, 0.1, 100) } /// Extended cache key that includes MBP-10 and trades directories AND bar /// formation parameters. The cache is invalidated when ANY data source /// changes — OHLCV, MBP-10, trades — OR when bar-formation params change. /// -/// Bar-formation params (`imbalance_bar_threshold`, `imbalance_bar_ewma_alpha`) -/// are hashed in regardless of `data_source` because: (a) they uniquely identify -/// the bar stream the cache contains, (b) downstream features (e.g. log-returns, -/// targets) depend on bar boundaries, and (c) without this, two runs configured -/// with different thresholds will silently share the same cache file (canonical -/// 2026-05-09 audit: 14 prior production runs all collided on `a3f933aa...` / -/// `c07c960a...` keys regardless of TOML threshold value because the bar params -/// were not in the hash). +/// Bar-formation params (`imbalance_bar_threshold`, `imbalance_bar_ewma_alpha`, +/// `volume_bar_size`) are hashed in regardless of `data_source` because: +/// (a) they uniquely identify the bar stream the cache contains, (b) downstream +/// features (e.g. log-returns, targets) depend on bar boundaries, and +/// (c) without this, two runs configured with different params will silently +/// share the same cache file (canonical 2026-05-09 audit: 14 prior production +/// runs all collided on `a3f933aa...` / `c07c960a...` keys regardless of TOML +/// threshold value because the bar params were not in the hash). pub fn calculate_dbn_cache_key_full( data_dir: &Path, mbp10_dir: Option<&Path>, @@ -40,6 +41,7 @@ pub fn calculate_dbn_cache_key_full( data_source: &str, imbalance_bar_threshold: f64, imbalance_bar_ewma_alpha: f64, + volume_bar_size: u64, ) -> Result { let mut hasher = Sha256::new(); @@ -54,6 +56,8 @@ pub fn calculate_dbn_cache_key_full( hasher.update(b"|"); hasher.update(&imbalance_bar_ewma_alpha.to_le_bytes()); hasher.update(b"|"); + hasher.update(&volume_bar_size.to_le_bytes()); + hasher.update(b"|"); // Hash only file contents (size + mtime), not paths. // This makes the key independent of CWD, relative/absolute paths, @@ -101,8 +105,8 @@ mod tests { fn test_cache_key_includes_symbol() { let dir = Path::new("test_data/futures-baseline"); if !dir.exists() { return; } - let key_es = calculate_dbn_cache_key_full(dir, None, None, "ES.FUT", "mbp10", 0.5, 0.1).unwrap(); - let key_nq = calculate_dbn_cache_key_full(dir, None, None, "NQ.FUT", "mbp10", 0.5, 0.1).unwrap(); + let key_es = calculate_dbn_cache_key_full(dir, None, None, "ES.FUT", "mbp10", 0.5, 0.1, 100).unwrap(); + let key_nq = calculate_dbn_cache_key_full(dir, None, None, "NQ.FUT", "mbp10", 0.5, 0.1, 100).unwrap(); assert_ne!(key_es, key_nq, "Different symbols must produce different cache keys"); } @@ -110,8 +114,8 @@ mod tests { fn test_cache_key_includes_data_source() { let dir = Path::new("test_data/futures-baseline"); if !dir.exists() { return; } - let key_ohlcv = calculate_dbn_cache_key_full(dir, None, None, "ES.FUT", "ohlcv", 0.5, 0.1).unwrap(); - let key_mbp10 = calculate_dbn_cache_key_full(dir, None, None, "ES.FUT", "mbp10", 0.5, 0.1).unwrap(); + let key_ohlcv = calculate_dbn_cache_key_full(dir, None, None, "ES.FUT", "ohlcv", 0.5, 0.1, 100).unwrap(); + let key_mbp10 = calculate_dbn_cache_key_full(dir, None, None, "ES.FUT", "mbp10", 0.5, 0.1, 100).unwrap(); assert_ne!(key_ohlcv, key_mbp10, "Different data sources must produce different cache keys"); } @@ -119,8 +123,8 @@ mod tests { fn test_cache_key_includes_bar_threshold() { let dir = Path::new("test_data/futures-baseline"); if !dir.exists() { return; } - let key_a = calculate_dbn_cache_key_full(dir, None, None, "ES.FUT", "mbp10", 0.5, 0.1).unwrap(); - let key_b = calculate_dbn_cache_key_full(dir, None, None, "ES.FUT", "mbp10", 2.5, 0.1).unwrap(); + let key_a = calculate_dbn_cache_key_full(dir, None, None, "ES.FUT", "mbp10", 0.5, 0.1, 100).unwrap(); + let key_b = calculate_dbn_cache_key_full(dir, None, None, "ES.FUT", "mbp10", 2.5, 0.1, 100).unwrap(); assert_ne!(key_a, key_b, "Different imbalance_bar_threshold must produce different cache keys"); } @@ -128,17 +132,26 @@ mod tests { fn test_cache_key_includes_bar_alpha() { let dir = Path::new("test_data/futures-baseline"); if !dir.exists() { return; } - let key_a = calculate_dbn_cache_key_full(dir, None, None, "ES.FUT", "mbp10", 0.5, 0.1).unwrap(); - let key_b = calculate_dbn_cache_key_full(dir, None, None, "ES.FUT", "mbp10", 0.5, 1.0).unwrap(); + let key_a = calculate_dbn_cache_key_full(dir, None, None, "ES.FUT", "mbp10", 0.5, 0.1, 100).unwrap(); + let key_b = calculate_dbn_cache_key_full(dir, None, None, "ES.FUT", "mbp10", 0.5, 1.0, 100).unwrap(); assert_ne!(key_a, key_b, "Different imbalance_bar_ewma_alpha must produce different cache keys"); } + #[test] + fn test_cache_key_includes_volume_bar_size() { + let dir = Path::new("test_data/futures-baseline"); + if !dir.exists() { return; } + let key_a = calculate_dbn_cache_key_full(dir, None, None, "ES.FUT", "ohlcv", 0.5, 0.1, 100).unwrap(); + let key_b = calculate_dbn_cache_key_full(dir, None, None, "ES.FUT", "ohlcv", 0.5, 0.1, 500).unwrap(); + assert_ne!(key_a, key_b, "Different volume_bar_size must produce different cache keys"); + } + #[test] fn test_cache_key_stable() { let dir = Path::new("test_data/futures-baseline"); if !dir.exists() { return; } - let key1 = calculate_dbn_cache_key_full(dir, None, None, "ES.FUT", "mbp10", 0.5, 0.1).unwrap(); - let key2 = calculate_dbn_cache_key_full(dir, None, None, "ES.FUT", "mbp10", 0.5, 0.1).unwrap(); + let key1 = calculate_dbn_cache_key_full(dir, None, None, "ES.FUT", "mbp10", 0.5, 0.1, 100).unwrap(); + let key2 = calculate_dbn_cache_key_full(dir, None, None, "ES.FUT", "mbp10", 0.5, 0.1, 100).unwrap(); assert_eq!(key1, key2, "Same inputs must produce same key"); } } diff --git a/crates/ml/src/fxcache.rs b/crates/ml/src/fxcache.rs index 561430397..9ecda0e4a 100644 --- a/crates/ml/src/fxcache.rs +++ b/crates/ml/src/fxcache.rs @@ -690,8 +690,9 @@ pub fn norm_stats_path_for_key( /// * `mbp10_dir` — Optional MBP-10 order book data directory /// * `trades_dir` — Optional trades data directory /// * `data_source` — Data source mode (`"mbp10"` or `"ohlcv"`) -/// * `imbalance_bar_threshold` — Bar formation threshold (must match producer's value) -/// * `imbalance_bar_ewma_alpha` — Bar formation EWMA alpha (must match producer's value) +/// * `imbalance_bar_threshold` — Imbalance bar formation threshold (must match producer's value) +/// * `imbalance_bar_ewma_alpha` — Imbalance bar EWMA alpha (must match producer's value) +/// * `volume_bar_size` — Volume bar contracts/bar (must match producer's value) /// * `cache_dir_override` — Explicit cache directory (from CLI `--feature-cache-dir`) pub fn discover_and_load( data_dir: &Path, @@ -701,6 +702,7 @@ pub fn discover_and_load( data_source: &str, imbalance_bar_threshold: f64, imbalance_bar_ewma_alpha: f64, + volume_bar_size: u64, cache_dir_override: Option<&Path>, ) -> Option { // 1. Resolve cache directory @@ -709,7 +711,7 @@ pub fn discover_and_load( // 2. Compute cache key (includes symbol + data_source + bar params) let key_hex = crate::feature_cache::calculate_dbn_cache_key_full( data_dir, mbp10_dir, trades_dir, symbol, data_source, - imbalance_bar_threshold, imbalance_bar_ewma_alpha, + imbalance_bar_threshold, imbalance_bar_ewma_alpha, volume_bar_size, ) .ok()?; let key: [u8; 32] = hex::decode(&key_hex).ok()?.try_into().ok()?; @@ -749,6 +751,7 @@ mod discover_tests { "mbp10", 0.5, 0.1, + 100, None, ); assert!(result.is_none()); @@ -780,6 +783,7 @@ mod discover_tests { "mbp10", 0.5, 0.1, + 100, Some(&cache_dir), ); // Strict match only — no "most recent" fallback diff --git a/crates/ml/src/trainers/dqn/config.rs b/crates/ml/src/trainers/dqn/config.rs index ac0cfba09..840e6514b 100644 --- a/crates/ml/src/trainers/dqn/config.rs +++ b/crates/ml/src/trainers/dqn/config.rs @@ -994,6 +994,13 @@ pub struct DQNHyperparameters { /// Only used when `data_source = "mbp10"`. pub imbalance_bar_ewma_alpha: f32, + /// Volume bar size: contracts of one-sided volume per bar. Higher = + /// fewer/longer bars. Used when `data_source != "mbp10"`. Default: 100 + /// (matches `DEFAULT_VOLUME_BAR_SIZE` for backwards compatibility with + /// pre-2026-05-09 caches). Must match between `precompute_features` and + /// trainer for fxcache HIT (cache key includes this value). + pub volume_bar_size: u64, + // Offline RL mode (CQL/IQL on fixed datasets) /// Offline RL mode: skip online experience collection, train from a fixed dataset. /// Requires `dataset_path` to point to a pre-collected experience dataset. @@ -1587,6 +1594,7 @@ impl DQNHyperparameters { max_bars: 0, // 0 = unlimited imbalance_bar_threshold: 100.0, imbalance_bar_ewma_alpha: 0.1, + volume_bar_size: 100, // Offline RL: disabled by default (online training with experience collection) offline_mode: false, diff --git a/crates/ml/src/trainers/dqn/data_loading.rs b/crates/ml/src/trainers/dqn/data_loading.rs index 26413837d..10a230c4f 100644 --- a/crates/ml/src/trainers/dqn/data_loading.rs +++ b/crates/ml/src/trainers/dqn/data_loading.rs @@ -151,6 +151,7 @@ impl DQNTrainer { &self.hyperparams.data_source, f64::from(self.hyperparams.imbalance_bar_threshold), f64::from(self.hyperparams.imbalance_bar_ewma_alpha), + self.hyperparams.volume_bar_size, cache_dir_override, ) { // OFI is unconditional — the model requires order flow data. @@ -276,10 +277,11 @@ impl DQNTrainer { all_front_month.sort_by_key(|t| t.timestamp); let bars = crate::features::build_volume_bars( - &all_front_month, crate::features::DEFAULT_VOLUME_BAR_SIZE, + &all_front_month, self.hyperparams.volume_bar_size, ); - info!("Volume bars: {} bars from {} trades ({} front-month)", - bars.len(), total_raw, all_front_month.len()); + info!("Volume bars: {} bars from {} trades ({} front-month, size={})", + bars.len(), total_raw, all_front_month.len(), + self.hyperparams.volume_bar_size); if bars.is_empty() { return Err(anyhow::anyhow!( diff --git a/crates/ml/src/training_profile.rs b/crates/ml/src/training_profile.rs index b5987afd0..eda064098 100644 --- a/crates/ml/src/training_profile.rs +++ b/crates/ml/src/training_profile.rs @@ -95,6 +95,10 @@ pub struct TrainingSection { /// EWMA alpha for adaptive imbalance threshold. Default: 0.1. pub imbalance_bar_ewma_alpha: Option, + /// Volume bar size (contracts/bar). Used when `data_source != "mbp10"`. + /// Default: 100. Must match between precompute and trainer for cache HIT. + pub volume_bar_size: Option, + /// Maximum bars to load for training. 0 = unlimited (load all data). /// Useful for fast smoketests on large datasets. pub max_bars: Option, @@ -811,6 +815,9 @@ impl DqnTrainingProfile { if let Some(v) = t.imbalance_bar_ewma_alpha { hp.imbalance_bar_ewma_alpha = v as f32; } + if let Some(v) = t.volume_bar_size { + hp.volume_bar_size = v; + } } // [exploration] diff --git a/docs/dqn-wire-up-audit.md b/docs/dqn-wire-up-audit.md index 753b70300..721a87464 100644 --- a/docs/dqn-wire-up-audit.md +++ b/docs/dqn-wire-up-audit.md @@ -2,6 +2,56 @@ **Status:** Populated during Plan 1 Task 6 (A.5 orphan audit). Updated on every commit per Invariant 7. +## 2026-05-09 — volume_bar_size in cache key + OFI front-month filter fix + +`crates/ml/src/trainers/dqn/data_loading.rs` updated in two places: +1. Line 146: `discover_and_load` call now passes + `self.hyperparams.volume_bar_size` as the new bar-formation cache-key param + (8 → 9 args). +2. Line 279: `build_volume_bars` now uses `self.hyperparams.volume_bar_size` + instead of the `DEFAULT_VOLUME_BAR_SIZE` constant. Tuning this knob via + TOML now actually changes bar density (was previously a no-op via the + same fossilization pattern as `imbalance_bar_threshold` pre-fix). + +### New CLI / TOML / Argo plumbing + +- `Hyperparams.volume_bar_size: u64` field added (default 100, matches + `DEFAULT_VOLUME_BAR_SIZE`). TrainingProfile loader reads `volume_bar_size` + TOML key. +- `--volume-bar-size` CLI arg on `precompute_features` and + `train_baseline_rl` (default 100). Wired into Argo workflow params + `volume-bar-size` (default `"100"`) on both `train-template.yaml` and + `train-multi-seed-template.yaml`. `scripts/argo-train.sh` exposes + `--volume-bar-size ` for ad-hoc overrides. +- New Argo workflow param `data-source` (default `"mbp10"`) threaded + through to `precompute_features --data-source`. Exposed as + `--data-source ` on `argo-train.sh`. Lets experiments toggle volume + vs imbalance bar mode without TOML edits. + +### Cache key + +`calculate_dbn_cache_key_full` signature: 7 args → 8 args (added +`volume_bar_size: u64`). Hashed via `to_le_bytes()` after the imbalance +params. New unit test `test_cache_key_includes_volume_bar_size` pins the +contract; passes alongside the 5 existing tests. + +### Latent OFI bug fix (atomic with this commit) + +`crates/ml/examples/precompute_features.rs:539-557` (the OFI/VPIN/Kyle's +Lambda computation branch) was loading trades unfiltered for per-bar +microstructure feature computation. Pre-fix: the volume bar formation +path filtered front-month per-file (line 354), but the OFI path did not, +leaking off-contract trade activity into per-bar microstructure features +during contract rollover windows. + +Fix: mirror the per-file `filter_front_month` call from the volume bar +path. Volume bar formation and OFI computation now both see the same +in-month trade tape. Severity in production: small (front-month dominates +ES.FUT trade volume by 10-100×, contamination ≪ 1%) but real and present +in every prior MBP-10+trades production run. Filed alongside the +volume_bar_size cache-key fix because both are architectural cleanups +the wgdc8 experiment surfaced. + ## 2026-05-09 — fxcache key includes bar formation params + precompute_features actually USES data_source `crates/ml/src/trainers/dqn/data_loading.rs:146` — `discover_and_load` diff --git a/infra/k8s/argo/train-multi-seed-template.yaml b/infra/k8s/argo/train-multi-seed-template.yaml index e2c1c5c49..934cc2cad 100644 --- a/infra/k8s/argo/train-multi-seed-template.yaml +++ b/infra/k8s/argo/train-multi-seed-template.yaml @@ -100,6 +100,15 @@ spec: value: "0.5" - name: imbalance-bar-ewma-alpha value: "0.1" + # Volume bar size (contracts/bar). Used when data-source != "mbp10". + # Default 100 matches DEFAULT_VOLUME_BAR_SIZE. Cache key includes this. + - name: volume-bar-size + value: "100" + # Data source mode: "mbp10" (imbalance bars from MBP-10) or "ohlcv" + # (volume bars from trades). Threaded into both precompute_features and + # the trainer so cache keys align. + - name: data-source + value: "mbp10" volumes: - name: git-ssh-key @@ -353,8 +362,10 @@ spec: --trades-data-dir /data/futures-baseline-trades \ --output-dir /feature-cache \ --symbol {{workflow.parameters.symbol}} \ + --data-source {{workflow.parameters.data-source}} \ --imbalance-bar-threshold {{workflow.parameters.imbalance-bar-threshold}} \ --imbalance-bar-ewma-alpha {{workflow.parameters.imbalance-bar-ewma-alpha}} \ + --volume-bar-size {{workflow.parameters.volume-bar-size}} \ --yes; then echo "=== Feature cache ready ===" else @@ -370,8 +381,10 @@ spec: --trades-data-dir /data/futures-baseline-trades \ --output-dir /feature-cache \ --symbol {{workflow.parameters.symbol}} \ + --data-source {{workflow.parameters.data-source}} \ --imbalance-bar-threshold {{workflow.parameters.imbalance-bar-threshold}} \ --imbalance-bar-ewma-alpha {{workflow.parameters.imbalance-bar-ewma-alpha}} \ + --volume-bar-size {{workflow.parameters.volume-bar-size}} \ --yes fi @@ -501,7 +514,8 @@ spec: --seed "$SEED" \ --max-folds {{workflow.parameters.folds}} \ --imbalance-bar-threshold {{workflow.parameters.imbalance-bar-threshold}} \ - --imbalance-bar-ewma-alpha {{workflow.parameters.imbalance-bar-ewma-alpha}} + --imbalance-bar-ewma-alpha {{workflow.parameters.imbalance-bar-ewma-alpha}} \ + --volume-bar-size {{workflow.parameters.volume-bar-size}} echo "=== Training complete: seed=$SEED folds={{workflow.parameters.folds}} ===" diff --git a/infra/k8s/argo/train-template.yaml b/infra/k8s/argo/train-template.yaml index 394165b96..c1ca49a3a 100644 --- a/infra/k8s/argo/train-template.yaml +++ b/infra/k8s/argo/train-template.yaml @@ -73,6 +73,10 @@ spec: value: "0.5" - name: imbalance-bar-ewma-alpha value: "0.1" + - name: volume-bar-size + value: "100" + - name: data-source + value: "mbp10" volumes: - name: git-ssh-key @@ -373,8 +377,10 @@ spec: --trades-data-dir /data/futures-baseline-trades \ --output-dir /feature-cache \ --symbol {{workflow.parameters.symbol}} \ + --data-source {{workflow.parameters.data-source}} \ --imbalance-bar-threshold {{workflow.parameters.imbalance-bar-threshold}} \ --imbalance-bar-ewma-alpha {{workflow.parameters.imbalance-bar-ewma-alpha}} \ + --volume-bar-size {{workflow.parameters.volume-bar-size}} \ --yes; then echo "=== Feature cache ready ===" else @@ -388,8 +394,10 @@ spec: --trades-data-dir /data/futures-baseline-trades \ --output-dir /feature-cache \ --symbol {{workflow.parameters.symbol}} \ + --data-source {{workflow.parameters.data-source}} \ --imbalance-bar-threshold {{workflow.parameters.imbalance-bar-threshold}} \ --imbalance-bar-ewma-alpha {{workflow.parameters.imbalance-bar-ewma-alpha}} \ + --volume-bar-size {{workflow.parameters.volume-bar-size}} \ --yes echo "=== Feature cache regenerated ===" fi @@ -616,6 +624,7 @@ spec: --epochs {{workflow.parameters.train-epochs}} \ --imbalance-bar-threshold {{workflow.parameters.imbalance-bar-threshold}} \ --imbalance-bar-ewma-alpha {{workflow.parameters.imbalance-bar-ewma-alpha}} \ + --volume-bar-size {{workflow.parameters.volume-bar-size}} \ $HYPEROPT_FLAG echo "=== Training complete ===" diff --git a/scripts/argo-train.sh b/scripts/argo-train.sh index 852996e13..e257bb456 100755 --- a/scripts/argo-train.sh +++ b/scripts/argo-train.sh @@ -33,6 +33,8 @@ TAG="" PROFILE=false IMBALANCE_BAR_THRESHOLD="" IMBALANCE_BAR_EWMA_ALPHA="" +VOLUME_BAR_SIZE="" +DATA_SOURCE="" usage() { cat < Override imbalance bar EWMA alpha (default: 0.1). α=1.0 disables adaptation (this codebase's reversed convention). MUST match between precompute and trainer. + --volume-bar-size Override volume bar size in contracts (default: 100). + Used when --data-source != "mbp10". Cache key includes. + --data-source Data source mode: "mbp10" or "ohlcv" (default: "mbp10"). + Threaded into both precompute and trainer. --profile Wrap training under nsys (NVIDIA Nsight Systems) and upload .nsys-rep artefacts to MinIO bucket foxhunt-training-artifacts/profiles//. Forces the @@ -100,6 +106,8 @@ while [[ $# -gt 0 ]]; do --profile) PROFILE=true; shift ;; --imbalance-bar-threshold) IMBALANCE_BAR_THRESHOLD="$2"; shift 2 ;; --imbalance-bar-ewma-alpha) IMBALANCE_BAR_EWMA_ALPHA="$2"; shift 2 ;; + --volume-bar-size) VOLUME_BAR_SIZE="$2"; shift 2 ;; + --data-source) DATA_SOURCE="$2"; shift 2 ;; --dry-run) DRY_RUN=true; shift ;; -h|--help) usage ;; *) echo "Unknown option: $1"; usage ;; @@ -158,6 +166,8 @@ if [[ "$USE_MULTI_SEED" == "false" ]]; then [[ "$SANITIZER" != "none" ]] && CMD="$CMD -p sanitizer=$SANITIZER" [[ -n "$IMBALANCE_BAR_THRESHOLD" ]] && CMD="$CMD -p imbalance-bar-threshold=$IMBALANCE_BAR_THRESHOLD" [[ -n "$IMBALANCE_BAR_EWMA_ALPHA" ]] && CMD="$CMD -p imbalance-bar-ewma-alpha=$IMBALANCE_BAR_EWMA_ALPHA" + [[ -n "$VOLUME_BAR_SIZE" ]] && CMD="$CMD -p volume-bar-size=$VOLUME_BAR_SIZE" + [[ -n "$DATA_SOURCE" ]] && CMD="$CMD -p data-source=$DATA_SOURCE" $BASELINE && CMD="$CMD -p hyperopt-trials=0" $WATCH && CMD="$CMD --watch" @@ -291,6 +301,8 @@ CMD="$CMD -p profile=$PROFILE" [[ -n "$TAG" ]] && CMD="$CMD --labels foxhunt-tag=$TAG" [[ -n "$IMBALANCE_BAR_THRESHOLD" ]] && CMD="$CMD -p imbalance-bar-threshold=$IMBALANCE_BAR_THRESHOLD" [[ -n "$IMBALANCE_BAR_EWMA_ALPHA" ]] && CMD="$CMD -p imbalance-bar-ewma-alpha=$IMBALANCE_BAR_EWMA_ALPHA" +[[ -n "$VOLUME_BAR_SIZE" ]] && CMD="$CMD -p volume-bar-size=$VOLUME_BAR_SIZE" +[[ -n "$DATA_SOURCE" ]] && CMD="$CMD -p data-source=$DATA_SOURCE" [[ "$SANITIZER" != "none" ]] && CMD="$CMD -p sanitizer=$SANITIZER" $BASELINE && CMD="$CMD -p hyperopt-trials=0"