From 8b0122a5ac10326fe2b6dc024f30dc3e11ff38c0 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sat, 4 Apr 2026 09:48:38 +0200 Subject: [PATCH] feat: has_ofi flag in fxcache header (explicit, no zero-detection) Adds `has_ofi: bool` to `FxCacheData` and the fxcache binary header (stored in `reserved[0]`). Propagates through load_fxcache, write_fxcache, all callers (precompute_features, train_baseline_rl, hyperopt dqn adapter), existing roundtrip tests, and adds two new has_ofi-specific roundtrip tests. Co-Authored-By: Claude Sonnet 4.6 --- crates/ml/examples/precompute_features.rs | 3 +- crates/ml/examples/train_baseline_rl.rs | 1 + crates/ml/src/fxcache.rs | 54 +++++++++++++++++++++-- crates/ml/src/hyperopt/adapters/dqn.rs | 2 + crates/ml/tests/fxcache_roundtrip_test.rs | 8 ++-- 5 files changed, 60 insertions(+), 8 deletions(-) diff --git a/crates/ml/examples/precompute_features.rs b/crates/ml/examples/precompute_features.rs index 9741a78cd..068f44cbe 100644 --- a/crates/ml/examples/precompute_features.rs +++ b/crates/ml/examples/precompute_features.rs @@ -430,6 +430,7 @@ async fn main() -> Result<()> { let t2 = Instant::now(); info!("Writing .fxcache to {}...", output_path.display()); + let has_ofi = mbp10_dir.is_some(); let bytes_written = ml::fxcache::write_fxcache( &output_path, &features, @@ -438,6 +439,7 @@ async fn main() -> Result<()> { ×tamps, cache_key, opts.bf16, + has_ofi, ) .context("Failed to write .fxcache file")?; @@ -450,7 +452,6 @@ async fn main() -> Result<()> { println!("PRECOMPUTE SUMMARY"); println!("================================================================================"); println!(); - let has_ofi = ofi.iter().any(|o| o.iter().any(|&v| v != 0.0)); println!("Bars: {}", total_len); println!("Features: 42-dim"); println!("Targets: 4-dim"); diff --git a/crates/ml/examples/train_baseline_rl.rs b/crates/ml/examples/train_baseline_rl.rs index 3d92d440d..830131e0b 100644 --- a/crates/ml/examples/train_baseline_rl.rs +++ b/crates/ml/examples/train_baseline_rl.rs @@ -631,6 +631,7 @@ fn run_training(args: &Args) -> Result> { ofi, cache_key: [0u8; 32], bar_count: n, + has_ofi: false, } }; diff --git a/crates/ml/src/fxcache.rs b/crates/ml/src/fxcache.rs index f1c1473d2..bf7eed06b 100644 --- a/crates/ml/src/fxcache.rs +++ b/crates/ml/src/fxcache.rs @@ -80,7 +80,9 @@ pub struct FxCacheHeader { impl FxCacheHeader { /// Create a new header with the given parameters. - pub fn new(version: u16, bar_count: u64, cache_key: [u8; 32]) -> Self { + pub fn new(version: u16, bar_count: u64, cache_key: [u8; 32], has_ofi: bool) -> Self { + let mut reserved = [0u8; 8]; + reserved[0] = if has_ofi { 1 } else { 0 }; Self { magic: FXCACHE_MAGIC, version, @@ -89,7 +91,7 @@ impl FxCacheHeader { ofi_dim: OFI_DIM as u16, bar_count, cache_key, - reserved: [0u8; 8], + reserved, } } @@ -198,6 +200,9 @@ pub struct FxCacheData { pub cache_key: [u8; 32], /// Number of bars. pub bar_count: usize, + /// Explicit flag: true if OFI was computed from real MBP-10 data. + /// False means OFI is zero-filled (no MBP-10 data was available during precompute). + pub has_ofi: bool, } // ── Writer ─────────────────────────────────────────────────────────────────── @@ -213,6 +218,7 @@ pub struct FxCacheData { /// * `timestamps` — Per-bar timestamps (nanoseconds since Unix epoch) /// * `cache_key` — SHA256 key (raw 32 bytes) /// * `bf16` — If true, write version 2 (bf16); otherwise version 1 (f64) +/// * `has_ofi` — If true, OFI was computed from real MBP-10 data; false means zero-filled /// /// # Returns /// @@ -225,6 +231,7 @@ pub fn write_fxcache( timestamps: &[i64], cache_key: [u8; 32], bf16: bool, + has_ofi: bool, ) -> Result { let bar_count = features.len(); if targets.len() != bar_count || ofi.len() != bar_count || timestamps.len() != bar_count { @@ -247,7 +254,7 @@ pub fn write_fxcache( } let version: u16 = if bf16 { 2 } else { 1 }; - let header = FxCacheHeader::new(version, bar_count as u64, cache_key); + let header = FxCacheHeader::new(version, bar_count as u64, cache_key, has_ofi); header.validate()?; let file = std::fs::File::create(path) @@ -405,6 +412,8 @@ pub fn load_fxcache(path: &Path) -> Result { path ); + let has_ofi = header.reserved[0] == 1; + Ok(FxCacheData { timestamps, features, @@ -412,6 +421,7 @@ pub fn load_fxcache(path: &Path) -> Result { ofi, cache_key: header.cache_key, bar_count, + has_ofi, }) } @@ -504,6 +514,44 @@ fn read_body_bf16( // ── Finder ─────────────────────────────────────────────────────────────────── +// ── Tests ──────────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod has_ofi_tests { + use super::*; + + #[test] + fn test_has_ofi_roundtrip_true() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("test_ofi_true.fxcache"); + let features = vec![[1.0_f64; 42]; 10]; + let targets = vec![[0.0_f64; 4]; 10]; + let ofi = vec![[0.5_f64; 8]; 10]; + let timestamps = vec![1_i64; 10]; + let key = [0u8; 32]; + + write_fxcache(&path, &features, &targets, &ofi, ×tamps, key, false, true).unwrap(); + let loaded = load_fxcache(&path).unwrap(); + assert!(loaded.has_ofi, "has_ofi should be true"); + assert_eq!(loaded.bar_count, 10); + } + + #[test] + fn test_has_ofi_roundtrip_false() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("test_ofi_false.fxcache"); + let features = vec![[1.0_f64; 42]; 10]; + let targets = vec![[0.0_f64; 4]; 10]; + let ofi = vec![[0.0_f64; 8]; 10]; + let timestamps = vec![1_i64; 10]; + let key = [0u8; 32]; + + write_fxcache(&path, &features, &targets, &ofi, ×tamps, key, false, false).unwrap(); + let loaded = load_fxcache(&path).unwrap(); + assert!(!loaded.has_ofi, "has_ofi should be false"); + } +} + /// Find an `.fxcache` file by hex-encoded cache key in a cache directory. /// /// Looks for a file named `.fxcache` in `cache_dir`. diff --git a/crates/ml/src/hyperopt/adapters/dqn.rs b/crates/ml/src/hyperopt/adapters/dqn.rs index de6e78924..1dbab5b0f 100644 --- a/crates/ml/src/hyperopt/adapters/dqn.rs +++ b/crates/ml/src/hyperopt/adapters/dqn.rs @@ -890,6 +890,7 @@ impl DQNTrainer { } targets.push(t); } + let has_ofi = ofi_features.is_some(); let ofi: Vec<[f64; 8]> = if let Some(ref ofi_arc) = ofi_features { ofi_arc.iter().cloned().collect() } else { @@ -906,6 +907,7 @@ impl DQNTrainer { ofi, cache_key: [0u8; 32], bar_count: total, + has_ofi, } }; diff --git a/crates/ml/tests/fxcache_roundtrip_test.rs b/crates/ml/tests/fxcache_roundtrip_test.rs index 9688b45d2..5df5ea9b5 100644 --- a/crates/ml/tests/fxcache_roundtrip_test.rs +++ b/crates/ml/tests/fxcache_roundtrip_test.rs @@ -79,7 +79,7 @@ fn test_fxcache_f64_roundtrip() { let key = test_cache_key(); let bytes_written = - write_fxcache(&path, &features, &targets, &ofi, ×tamps, key, false).unwrap(); + write_fxcache(&path, &features, &targets, &ofi, ×tamps, key, false, false).unwrap(); assert!(bytes_written > 0, "expected nonzero bytes written"); let data: FxCacheData = load_fxcache(&path).unwrap(); @@ -123,7 +123,7 @@ fn test_fxcache_bf16_roundtrip() { let key = test_cache_key(); let bytes_written = - write_fxcache(&path, &features, &targets, &ofi, ×tamps, key, true).unwrap(); + write_fxcache(&path, &features, &targets, &ofi, ×tamps, key, true, false).unwrap(); assert!(bytes_written > 0, "expected nonzero bytes written"); let data: FxCacheData = load_fxcache(&path).unwrap(); @@ -185,7 +185,7 @@ fn test_fxcache_find() { let ofi = make_ofi(5); let timestamps = make_timestamps(5); - write_fxcache(&path, &features, &targets, &ofi, ×tamps, key, false).unwrap(); + write_fxcache(&path, &features, &targets, &ofi, ×tamps, key, false, false).unwrap(); // Should find the file. let found = find_fxcache(dir.path(), &key); @@ -231,7 +231,7 @@ fn test_fxcache_empty() { let timestamps: Vec = vec![]; let key = test_cache_key(); - let err = write_fxcache(&path, &features, &targets, &ofi, ×tamps, key, false).unwrap_err(); + let err = write_fxcache(&path, &features, &targets, &ofi, ×tamps, key, false, false).unwrap_err(); let msg = format!("{err:#}"); assert!( msg.contains("0 bars") || msg.contains("empty"),