From 00bbe76b05ff10e7b431d9b3e6bf6f54cbb6541e Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Fri, 10 Apr 2026 18:45:49 +0200 Subject: [PATCH] =?UTF-8?q?refactor:=20fxcache=20single=20f32=20format=20?= =?UTF-8?q?=E2=80=94=20delete=20bf16=20version,=20remove=20--bf16=20CLI=20?= =?UTF-8?q?flag?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.6 (1M context) --- crates/ml/examples/precompute_features.rs | 14 +- crates/ml/src/fxcache.rs | 173 +++++----------------- crates/ml/tests/fxcache_roundtrip_test.rs | 66 ++------- 3 files changed, 53 insertions(+), 200 deletions(-) diff --git a/crates/ml/examples/precompute_features.rs b/crates/ml/examples/precompute_features.rs index 79f77a2f6..0fcbeed2b 100644 --- a/crates/ml/examples/precompute_features.rs +++ b/crates/ml/examples/precompute_features.rs @@ -85,7 +85,6 @@ //! --trades-data-dir test_data/futures-baseline-trades \ //! --output-dir test_data/feature-cache \ //! --symbol ES.FUT \ -//! --bf16 \ //! --yes use std::path::{Path, PathBuf}; @@ -130,10 +129,6 @@ struct Opts { #[arg(long, default_value = "ES.FUT")] symbol: String, - /// Write version 2 (bf16) instead of version 1 (f64) - #[arg(long)] - bf16: bool, - /// Skip confirmation prompt #[arg(long)] yes: bool, @@ -189,7 +184,7 @@ async fn main() -> Result<()> { } println!("Output dir: {}", output_dir.display()); println!("Symbol: {}", opts.symbol); - println!("Format: {}", if opts.bf16 { "bf16 (v2)" } else { "f64 (v1)" }); + println!("Format: f32 (v1)"); println!(); // ── Confirmation ───────────────────────────────────────────────────────── @@ -454,8 +449,8 @@ async fn main() -> Result<()> { .map_err(|_| anyhow::anyhow!("Cache key wrong length"))?; // ── Normalize features (z-score) ────────────────────────────────────────── - // Raw features contain OHLCV prices (up to 25,000) that overflow bf16 in - // cuBLAS SGEMM. Normalize once at precompute time so every consumer + // Raw features contain OHLCV prices (up to 25,000). Normalize once at + // precompute time so every consumer // (training, hyperopt, inference) gets consistent normalized features. let norm_stats = ml::walk_forward::NormStats::from_features(&features); let features = norm_stats.normalize_batch(&features); @@ -486,7 +481,6 @@ async fn main() -> Result<()> { &ofi, ×tamps, cache_key, - opts.bf16, has_ofi, ) .context("Failed to write .fxcache file")?; @@ -504,7 +498,7 @@ async fn main() -> Result<()> { println!("Features: 42-dim"); println!("Targets: 4-dim"); println!("OFI: 8-dim ({})", if has_ofi { "from MBP-10" } else { "zero-padded" }); - println!("Format: {} (v{})", if opts.bf16 { "bf16" } else { "f64" }, if opts.bf16 { 2 } else { 1 }); + println!("Format: f32 (v1)"); println!("Cache key: {}", hex_key); println!("Output: {}", output_path.display()); println!("Size: {:.2} MB", bytes_written as f64 / 1_048_576.0); diff --git a/crates/ml/src/fxcache.rs b/crates/ml/src/fxcache.rs index bfe5adb23..56ad0e52f 100644 --- a/crates/ml/src/fxcache.rs +++ b/crates/ml/src/fxcache.rs @@ -10,7 +10,7 @@ //! ┌───────────────────────────────────────────────────┐ //! │ FxCacheHeader (64 bytes) │ //! │ magic [u8; 8] = b"FXCACHE\0" │ -//! │ version u16 = 1 (f64) | 2 (bf16) │ +//! │ version u16 = 1 (f32) │ //! │ feat_dim u16 = 42 │ //! │ target_dim u16 = 4 │ //! │ ofi_dim u16 = 8 │ @@ -20,14 +20,11 @@ //! └───────────────────────────────────────────────────┘ //! │ Body (bar_count records) │ //! │ Each record starts with an i64 timestamp (ns). │ -//! │ Version 1: [i64 ts][54 × f64] = 440 bytes/bar │ -//! │ Version 2: [i64 ts][56 × bf16] = 120 bytes/bar │ -//! │ (54 data + 2 zero-padding) │ +//! │ Version 1: [i64 ts][54 × f32] = 224 bytes/bar │ //! └───────────────────────────────────────────────────┘ //! ``` use anyhow::{bail, Context, Result}; -use half::f16; use std::io::{BufReader, BufWriter, Read, Write}; use std::path::{Path, PathBuf}; use tracing::{debug, info}; @@ -52,8 +49,8 @@ const OFI_DIM: usize = 8; /// Total f64 values per record: features + targets + OFI = 42 + 4 + 8 = 54. const RECORD_F64_COUNT: usize = FEAT_DIM + TARGET_DIM + OFI_DIM; -/// bf16 record width including 2 zero-padding values for 4-byte alignment. -const RECORD_BF16_COUNT: usize = RECORD_F64_COUNT + 2; +/// Total f32 values per record (same count as f64, no padding needed). +const RECORD_F32_COUNT: usize = RECORD_F64_COUNT; // ── Header ─────────────────────────────────────────────────────────────────── @@ -62,7 +59,7 @@ const RECORD_BF16_COUNT: usize = RECORD_F64_COUNT + 2; pub struct FxCacheHeader { /// Magic bytes: `b"FXCACHE\0"`. pub magic: [u8; 8], - /// Format version: 1 = f64, 2 = bf16. + /// Format version: 1 = f32. pub version: u16, /// Feature dimension (42). pub feat_dim: u16, @@ -104,9 +101,9 @@ impl FxCacheHeader { self.magic ); } - if self.version != 1 && self.version != 2 { + if self.version != 1 { bail!( - "Unsupported FxCache version: {} (expected 1 or 2)", + "Unsupported FxCache version: {} (expected 1)", self.version ); } @@ -217,7 +214,6 @@ pub struct FxCacheData { /// * `ofi` — Slice of 8-element OFI vectors /// * `timestamps` — Per-bar timestamps (nanoseconds since Unix epoch) /// * `cache_key` — SHA256 key (raw 32 bytes) -/// * `bf16` — If true, write version 2 (bf16); otherwise version 1 (f64) /// * `has_ofi` — If true, OFI was computed from real MBP-10 data; false means zero-filled /// /// # Returns @@ -230,7 +226,6 @@ pub fn write_fxcache( ofi: &[[f64; OFI_DIM]], timestamps: &[i64], cache_key: [u8; 32], - bf16: bool, has_ofi: bool, ) -> Result { let bar_count = features.len(); @@ -253,8 +248,7 @@ pub fn write_fxcache( .with_context(|| format!("Failed to create parent dirs for {:?}", path))?; } - let version: u16 = if bf16 { 2 } else { 1 }; - let header = FxCacheHeader::new(version, bar_count as u64, cache_key, has_ofi); + let header = FxCacheHeader::new(1, bar_count as u64, cache_key, has_ofi); header.validate()?; let file = std::fs::File::create(path) @@ -266,22 +260,16 @@ pub fn write_fxcache( .write_all(&header.to_bytes()) .context("Failed to write FxCache header")?; - // Write body - let body_bytes: u64 = if bf16 { - write_body_bf16(&mut writer, features, targets, ofi, timestamps)? - } else { - write_body_f64(&mut writer, features, targets, ofi, timestamps)? - }; + // Write body (f32 format) + let body_bytes = write_body_f32(&mut writer, features, targets, ofi, timestamps)?; writer.flush().context("Failed to flush FxCache writer")?; let total_bytes = HEADER_SIZE as u64 + body_bytes; info!( - "FxCache written: {} bars, v{} ({}), {:.2} MB -> {:?}", + "FxCache written: {} bars, v1 (f32), {:.2} MB -> {:?}", bar_count, - version, - if bf16 { "bf16" } else { "f64" }, total_bytes as f64 / 1_048_576.0, path ); @@ -289,72 +277,39 @@ pub fn write_fxcache( Ok(total_bytes) } -/// Write body in f64 format (version 1): [i64 ts] + 54 f64 values = 440 bytes per bar. -fn write_body_f64( +/// Write body in f32 format (version 1): [i64 ts] + 54 × f32 = 224 bytes per bar. +fn write_body_f32( writer: &mut BufWriter, features: &[[f64; FEAT_DIM]], targets: &[[f64; TARGET_DIM]], ofi: &[[f64; OFI_DIM]], timestamps: &[i64], ) -> Result { - let bytes_per_bar = 8 + RECORD_F64_COUNT * 8; // i64 timestamp + f64 data + let bytes_per_bar = 8 + RECORD_F32_COUNT * 4; // i64 timestamp + f32 data let total = features.len() as u64 * bytes_per_bar as u64; for i in 0..features.len() { writer.write_all(×tamps[i].to_le_bytes())?; for &v in &features[i] { - writer.write_all(&v.to_le_bytes())?; + writer.write_all(&(v as f32).to_le_bytes())?; } for &v in &targets[i] { - writer.write_all(&v.to_le_bytes())?; + writer.write_all(&(v as f32).to_le_bytes())?; } for &v in &ofi[i] { - writer.write_all(&v.to_le_bytes())?; + writer.write_all(&(v as f32).to_le_bytes())?; } } Ok(total) } -/// Write body in bf16 format (version 2): [i64 ts] + 56 bf16 values = 120 bytes per bar. -/// (54 data values + 2 zero-padding for 4-byte alignment.) -/// Timestamp is always i64 (8 bytes) — nanosecond precision requires 64 bits. -fn write_body_bf16( - writer: &mut BufWriter, - features: &[[f64; FEAT_DIM]], - targets: &[[f64; TARGET_DIM]], - ofi: &[[f64; OFI_DIM]], - timestamps: &[i64], -) -> Result { - let bytes_per_bar = 8 + RECORD_BF16_COUNT * 2; // i64 timestamp + bf16 data - let total = features.len() as u64 * bytes_per_bar as u64; - let zero = f16::ZERO; - - for i in 0..features.len() { - writer.write_all(×tamps[i].to_le_bytes())?; - for &v in &features[i] { - writer.write_all(&f16::from_f64(v).to_le_bytes())?; - } - for &v in &targets[i] { - writer.write_all(&f16::from_f64(v).to_le_bytes())?; - } - for &v in &ofi[i] { - writer.write_all(&f16::from_f64(v).to_le_bytes())?; - } - // 2 zero-padding bf16 values for alignment - writer.write_all(&zero.to_le_bytes())?; - writer.write_all(&zero.to_le_bytes())?; - } - - Ok(total) -} - // ── Reader ─────────────────────────────────────────────────────────────────── /// Load an `.fxcache` file into memory. /// -/// Reads the 64-byte header, validates it, then reads the body according to -/// the version (f64 or bf16). bf16 values are up-converted to f64 on load. +/// Reads the 64-byte header, validates it, then reads the f32 body, +/// converting each f32 back to f64 on load. /// /// # Arguments /// @@ -382,12 +337,8 @@ pub fn load_fxcache(path: &Path) -> Result { let bar_count = header.bar_count as usize; - // Sanity-check file size (each record has an i64 timestamp prefix) - let expected_body = if header.version == 1 { - bar_count as u64 * (8 + RECORD_F64_COUNT as u64 * 8) - } else { - bar_count as u64 * (8 + RECORD_BF16_COUNT as u64 * 2) - }; + // Sanity-check file size: [i64 ts] + 54 × f32 = 224 bytes/bar + let expected_body = bar_count as u64 * (8 + RECORD_F32_COUNT as u64 * 4); let expected_total = HEADER_SIZE as u64 + expected_body; if file_len < expected_total { bail!( @@ -397,18 +348,12 @@ pub fn load_fxcache(path: &Path) -> Result { ); } - // Read body - let (timestamps, features, targets, ofi) = if header.version == 1 { - read_body_f64(&mut reader, bar_count)? - } else { - read_body_bf16(&mut reader, bar_count)? - }; + // Read body (f32 format) + let (timestamps, features, targets, ofi) = read_body_f32(&mut reader, bar_count)?; info!( - "FxCache loaded: {} bars, v{} ({}) from {:?}", + "FxCache loaded: {} bars, v1 (f32) from {:?}", bar_count, - header.version, - if header.version == 1 { "f64" } else { "bf16" }, path ); @@ -425,8 +370,8 @@ pub fn load_fxcache(path: &Path) -> Result { }) } -/// Read body in f64 format (version 1). -fn read_body_f64( +/// Read body in f32 format (version 1), converting f32 back to f64 on load. +fn read_body_f32( reader: &mut BufReader, bar_count: usize, ) -> Result<(Vec, Vec<[f64; FEAT_DIM]>, Vec<[f64; TARGET_DIM]>, Vec<[f64; OFI_DIM]>)> { @@ -435,7 +380,7 @@ fn read_body_f64( let mut targets = Vec::with_capacity(bar_count); let mut ofi = Vec::with_capacity(bar_count); let mut i64_buf = [0u8; 8]; - let mut f64_buf = [0u8; 8]; + let mut f32_buf = [0u8; 4]; for _ in 0..bar_count { reader.read_exact(&mut i64_buf)?; @@ -443,22 +388,22 @@ fn read_body_f64( let mut feat = [0.0_f64; FEAT_DIM]; for slot in &mut feat { - reader.read_exact(&mut f64_buf)?; - *slot = f64::from_le_bytes(f64_buf); + reader.read_exact(&mut f32_buf)?; + *slot = f32::from_le_bytes(f32_buf) as f64; } features.push(feat); let mut tgt = [0.0_f64; TARGET_DIM]; for slot in &mut tgt { - reader.read_exact(&mut f64_buf)?; - *slot = f64::from_le_bytes(f64_buf); + reader.read_exact(&mut f32_buf)?; + *slot = f32::from_le_bytes(f32_buf) as f64; } targets.push(tgt); let mut ofi_row = [0.0_f64; OFI_DIM]; for slot in &mut ofi_row { - reader.read_exact(&mut f64_buf)?; - *slot = f64::from_le_bytes(f64_buf); + reader.read_exact(&mut f32_buf)?; + *slot = f32::from_le_bytes(f32_buf) as f64; } ofi.push(ofi_row); } @@ -466,52 +411,6 @@ fn read_body_f64( Ok((timestamps, features, targets, ofi)) } -/// Read body in bf16 format (version 2), converting to f64 on load. -/// Timestamp is always read as i64 (8 bytes) regardless of bf16 mode. -fn read_body_bf16( - reader: &mut BufReader, - bar_count: usize, -) -> Result<(Vec, Vec<[f64; FEAT_DIM]>, Vec<[f64; TARGET_DIM]>, Vec<[f64; OFI_DIM]>)> { - let mut timestamps = Vec::with_capacity(bar_count); - let mut features = Vec::with_capacity(bar_count); - let mut targets = Vec::with_capacity(bar_count); - let mut ofi = Vec::with_capacity(bar_count); - let mut i64_buf = [0u8; 8]; - let mut bf16_buf = [0u8; 2]; - - for _ in 0..bar_count { - reader.read_exact(&mut i64_buf)?; - timestamps.push(i64::from_le_bytes(i64_buf)); - - let mut feat = [0.0_f64; FEAT_DIM]; - for slot in &mut feat { - reader.read_exact(&mut bf16_buf)?; - *slot = f16::from_le_bytes(bf16_buf).to_f64(); - } - features.push(feat); - - let mut tgt = [0.0_f64; TARGET_DIM]; - for slot in &mut tgt { - reader.read_exact(&mut bf16_buf)?; - *slot = f16::from_le_bytes(bf16_buf).to_f64(); - } - targets.push(tgt); - - let mut ofi_row = [0.0_f64; OFI_DIM]; - for slot in &mut ofi_row { - reader.read_exact(&mut bf16_buf)?; - *slot = f16::from_le_bytes(bf16_buf).to_f64(); - } - ofi.push(ofi_row); - - // Skip 2 padding bf16 values - reader.read_exact(&mut bf16_buf)?; - reader.read_exact(&mut bf16_buf)?; - } - - Ok((timestamps, features, targets, ofi)) -} - // ── Finder ─────────────────────────────────────────────────────────────────── #[cfg(test)] @@ -528,7 +427,7 @@ mod has_ofi_tests { let timestamps = vec![1_i64; 10]; let key = [0u8; 32]; - write_fxcache(&path, &features, &targets, &ofi, ×tamps, key, false, true).unwrap(); + write_fxcache(&path, &features, &targets, &ofi, ×tamps, key, true).unwrap(); let loaded = load_fxcache(&path).unwrap(); assert!(loaded.has_ofi, "has_ofi should be true"); assert_eq!(loaded.bar_count, 10); @@ -544,7 +443,7 @@ mod has_ofi_tests { let timestamps = vec![1_i64; 10]; let key = [0u8; 32]; - write_fxcache(&path, &features, &targets, &ofi, ×tamps, key, false, false).unwrap(); + write_fxcache(&path, &features, &targets, &ofi, ×tamps, key, false).unwrap(); let loaded = load_fxcache(&path).unwrap(); assert!(!loaded.has_ofi, "has_ofi should be false"); } @@ -693,7 +592,7 @@ mod discover_tests { let targets = vec![[0.0_f64; 4]; 5]; let ofi = vec![[0.0_f64; 8]; 5]; let timestamps = vec![1_i64; 5]; - write_fxcache(&path, &features, &targets, &ofi, ×tamps, [0u8; 32], false, false) + write_fxcache(&path, &features, &targets, &ofi, ×tamps, [0u8; 32], false) .unwrap(); // Try to discover with a real data_dir (different key) — should NOT match diff --git a/crates/ml/tests/fxcache_roundtrip_test.rs b/crates/ml/tests/fxcache_roundtrip_test.rs index 5df5ea9b5..d14573203 100644 --- a/crates/ml/tests/fxcache_roundtrip_test.rs +++ b/crates/ml/tests/fxcache_roundtrip_test.rs @@ -1,6 +1,6 @@ //! Roundtrip tests for the `.fxcache` binary format. //! -//! Covers f64/bf16 serialization, cache-key lookup, header validation, +//! Covers f32 serialization, cache-key lookup, header validation, //! and the empty-bars error path. use ml::fxcache::{find_fxcache, load_fxcache, write_fxcache, FxCacheData}; @@ -65,11 +65,11 @@ fn test_cache_key() -> [u8; 32] { // ── Tests ─────────────────────────────────────────────────────────────────── -/// Write 100 bars as f64 (version 1), read back, verify bit-exact match. +/// Write 100 bars as f32 (version 1), read back, verify within f32 tolerance. #[test] -fn test_fxcache_f64_roundtrip() { +fn test_fxcache_f32_roundtrip() { let dir = TempDir::new().unwrap(); - let path = dir.path().join("test_f64.fxcache"); + let path = dir.path().join("test_f32.fxcache"); let n = 100; let features = make_features(n); @@ -79,7 +79,7 @@ fn test_fxcache_f64_roundtrip() { let key = test_cache_key(); let bytes_written = - write_fxcache(&path, &features, &targets, &ofi, ×tamps, key, false, false).unwrap(); + write_fxcache(&path, &features, &targets, &ofi, ×tamps, key, false).unwrap(); assert!(bytes_written > 0, "expected nonzero bytes written"); let data: FxCacheData = load_fxcache(&path).unwrap(); @@ -90,47 +90,7 @@ fn test_fxcache_f64_roundtrip() { assert_eq!(data.ofi.len(), n); assert_eq!(data.timestamps.len(), n); - // Bit-exact match for f64 roundtrip. - for i in 0..n { - assert_eq!( - data.timestamps[i], timestamps[i], - "timestamp mismatch at bar {i}" - ); - assert_eq!( - data.features[i], features[i], - "feature mismatch at bar {i}" - ); - assert_eq!( - data.targets[i], targets[i], - "target mismatch at bar {i}" - ); - assert_eq!(data.ofi[i], ofi[i], "ofi mismatch at bar {i}"); - } -} - -/// Write 50 bars as bf16 (version 2), read back, verify within tolerance. -/// bf16 has limited precision: ~3 significant digits. -#[test] -fn test_fxcache_bf16_roundtrip() { - let dir = TempDir::new().unwrap(); - let path = dir.path().join("test_bf16.fxcache"); - - let n = 50; - let features = make_features(n); - let targets = make_targets(n); - let ofi = make_ofi(n); - let timestamps = make_timestamps(n); - let key = test_cache_key(); - - let bytes_written = - write_fxcache(&path, &features, &targets, &ofi, ×tamps, key, true, false).unwrap(); - assert!(bytes_written > 0, "expected nonzero bytes written"); - - let data: FxCacheData = load_fxcache(&path).unwrap(); - assert_eq!(data.bar_count, n); - assert_eq!(data.cache_key, key); - - // Timestamps are always i64 — bit-exact even in bf16 mode. + // Timestamps are always i64 — bit-exact. for i in 0..n { assert_eq!( data.timestamps[i], timestamps[i], @@ -138,12 +98,12 @@ fn test_fxcache_bf16_roundtrip() { ); } - // bf16 tolerance checks. + // f32 roundtrip tolerance checks (~7 significant digits). for i in 0..n { for j in 0..42 { let diff = (data.features[i][j] - features[i][j]).abs(); assert!( - diff < 0.02, + diff < 1e-4, "feature[{i}][{j}]: expected {}, got {}, diff={diff}", features[i][j], data.features[i][j] @@ -151,9 +111,9 @@ fn test_fxcache_bf16_roundtrip() { } for j in 0..4 { let diff = (data.targets[i][j] - targets[i][j]).abs(); - // bf16 at ~5000 has resolution of ~4, so allow up to 4.0. + // f32 at ~5000 has resolution of ~0.0005, allow generous margin. assert!( - diff <= 4.0, + diff < 0.01, "target[{i}][{j}]: expected {}, got {}, diff={diff}", targets[i][j], data.targets[i][j] @@ -162,7 +122,7 @@ fn test_fxcache_bf16_roundtrip() { for j in 0..8 { let diff = (data.ofi[i][j] - ofi[i][j]).abs(); assert!( - diff < 0.02, + diff < 1e-4, "ofi[{i}][{j}]: expected {}, got {}, diff={diff}", ofi[i][j], data.ofi[i][j] @@ -185,7 +145,7 @@ fn test_fxcache_find() { let ofi = make_ofi(5); let timestamps = make_timestamps(5); - write_fxcache(&path, &features, &targets, &ofi, ×tamps, key, false, false).unwrap(); + write_fxcache(&path, &features, &targets, &ofi, ×tamps, key, false).unwrap(); // Should find the file. let found = find_fxcache(dir.path(), &key); @@ -231,7 +191,7 @@ fn test_fxcache_empty() { let timestamps: Vec = vec![]; let key = test_cache_key(); - let err = write_fxcache(&path, &features, &targets, &ofi, ×tamps, key, false, false).unwrap_err(); + let err = write_fxcache(&path, &features, &targets, &ofi, ×tamps, key, false).unwrap_err(); let msg = format!("{err:#}"); assert!( msg.contains("0 bars") || msg.contains("empty"),