gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor: fxcache single f32 format — delete bf16 version, remove --bf16 CLI flag

Browse Source 
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-04-10 18:45:49 +02:00
parent d11b9870ff
commit 00bbe76b05
3 changed files with 53 additions and 200 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
crates/ml/examples/precompute_features.rs
View File

@@ -85,7 +85,6 @@
//! --trades-data-dir test_data/futures-baseline-trades \
//! --output-dir test_data/feature-cache \
//! --symbol ES.FUT \
//! --bf16 \
//! --yes
use std::path::{Path, PathBuf};
@@ -130,10 +129,6 @@ struct Opts {
#[arg(long, default_value = "ES.FUT")]
symbol: String,
/// Write version 2 (bf16) instead of version 1 (f64)
#[arg(long)]
bf16: bool,
/// Skip confirmation prompt
#[arg(long)]
yes: bool,
@@ -189,7 +184,7 @@ async fn main() -> Result<()> {
}
println!("Output dir: {}", output_dir.display());
println!("Symbol: {}", opts.symbol);
println!("Format: {}", if opts.bf16 { "bf16 (v2)" } else { "f64 (v1)" });
println!("Format: f32 (v1)");
println!();
// ── Confirmation ─────────────────────────────────────────────────────────
@@ -454,8 +449,8 @@ async fn main() -> Result<()> {
.map_err(|_| anyhow::anyhow!("Cache key wrong length"))?;
// ── Normalize features (z-score) ──────────────────────────────────────────
// Raw features contain OHLCV prices (up to 25,000) that overflow bf16 in
// cuBLAS SGEMM. Normalize once at precompute time so every consumer
// Raw features contain OHLCV prices (up to 25,000). Normalize once at
// precompute time so every consumer
// (training, hyperopt, inference) gets consistent normalized features.
let norm_stats = ml::walk_forward::NormStats::from_features(&features);
let features = norm_stats.normalize_batch(&features);
@@ -486,7 +481,6 @@ async fn main() -> Result<()> {
&ofi,
&timestamps,
cache_key,
opts.bf16,
has_ofi,
)
.context("Failed to write .fxcache file")?;
@@ -504,7 +498,7 @@ async fn main() -> Result<()> {
println!("Features: 42-dim");
println!("Targets: 4-dim");
println!("OFI: 8-dim ({})", if has_ofi { "from MBP-10" } else { "zero-padded" });
println!("Format: {} (v{})", if opts.bf16 { "bf16" } else { "f64" }, if opts.bf16 { 2 } else { 1 });
println!("Format: f32 (v1)");
println!("Cache key: {}", hex_key);
println!("Output: {}", output_path.display());
println!("Size: {:.2} MB", bytes_written as f64 / 1_048_576.0);

173
crates/ml/src/fxcache.rs
View File

@@ -10,7 +10,7 @@
//! ┌───────────────────────────────────────────────────┐
//! │ FxCacheHeader (64 bytes) │
//! │ magic [u8; 8] = b"FXCACHE\0" │
//! │ version u16 = 1 (f64) | 2 (bf16) │
//! │ version u16 = 1 (f32) │
//! │ feat_dim u16 = 42 │
//! │ target_dim u16 = 4 │
//! │ ofi_dim u16 = 8 │
@@ -20,14 +20,11 @@
//! └───────────────────────────────────────────────────┘
//! │ Body (bar_count records) │
//! │ Each record starts with an i64 timestamp (ns). │
//! │ Version 1: [i64 ts][54 × f64] = 440 bytes/bar │
//! │ Version 2: [i64 ts][56 × bf16] = 120 bytes/bar │
//! │ (54 data + 2 zero-padding) │
//! │ Version 1: [i64 ts][54 × f32] = 224 bytes/bar │
//! └───────────────────────────────────────────────────┘
//! ```
use anyhow::{bail, Context, Result};
use half::f16;
use std::io::{BufReader, BufWriter, Read, Write};
use std::path::{Path, PathBuf};
use tracing::{debug, info};
@@ -52,8 +49,8 @@ const OFI_DIM: usize = 8;
/// Total f64 values per record: features + targets + OFI = 42 + 4 + 8 = 54.
const RECORD_F64_COUNT: usize = FEAT_DIM + TARGET_DIM + OFI_DIM;
/// bf16 record width including 2 zero-padding values for 4-byte alignment.
const RECORD_BF16_COUNT: usize = RECORD_F64_COUNT + 2;
/// Total f32 values per record (same count as f64, no padding needed).
const RECORD_F32_COUNT: usize = RECORD_F64_COUNT;
// ── Header ───────────────────────────────────────────────────────────────────
@@ -62,7 +59,7 @@ const RECORD_BF16_COUNT: usize = RECORD_F64_COUNT + 2;
pub struct FxCacheHeader {
/// Magic bytes: `b"FXCACHE\0"`.
pub magic: [u8; 8],
/// Format version: 1 = f64, 2 = bf16.
/// Format version: 1 = f32.
pub version: u16,
/// Feature dimension (42).
pub feat_dim: u16,
@@ -104,9 +101,9 @@ impl FxCacheHeader {
self.magic
);
}
if self.version != 1 && self.version != 2 {
if self.version != 1 {
bail!(
"Unsupported FxCache version: {} (expected 1 or 2)",
"Unsupported FxCache version: {} (expected 1)",
self.version
);
}
@@ -217,7 +214,6 @@ pub struct FxCacheData {
/// * `ofi` — Slice of 8-element OFI vectors
/// * `timestamps` — Per-bar timestamps (nanoseconds since Unix epoch)
/// * `cache_key` — SHA256 key (raw 32 bytes)
/// * `bf16` — If true, write version 2 (bf16); otherwise version 1 (f64)
/// * `has_ofi` — If true, OFI was computed from real MBP-10 data; false means zero-filled
///
/// # Returns
@@ -230,7 +226,6 @@ pub fn write_fxcache(
ofi: &[[f64; OFI_DIM]],
timestamps: &[i64],
cache_key: [u8; 32],
bf16: bool,
has_ofi: bool,
) -> Result<u64> {
let bar_count = features.len();
@@ -253,8 +248,7 @@ pub fn write_fxcache(
.with_context(|| format!("Failed to create parent dirs for {:?}", path))?;
}
let version: u16 = if bf16 { 2 } else { 1 };
let header = FxCacheHeader::new(version, bar_count as u64, cache_key, has_ofi);
let header = FxCacheHeader::new(1, bar_count as u64, cache_key, has_ofi);
header.validate()?;
let file = std::fs::File::create(path)
@@ -266,22 +260,16 @@ pub fn write_fxcache(
.write_all(&header.to_bytes())
.context("Failed to write FxCache header")?;
// Write body
let body_bytes: u64 = if bf16 {
write_body_bf16(&mut writer, features, targets, ofi, timestamps)?
} else {
write_body_f64(&mut writer, features, targets, ofi, timestamps)?
};
// Write body (f32 format)
let body_bytes = write_body_f32(&mut writer, features, targets, ofi, timestamps)?;
writer.flush().context("Failed to flush FxCache writer")?;
let total_bytes = HEADER_SIZE as u64 + body_bytes;
info!(
"FxCache written: {} bars, v{} ({}), {:.2} MB -> {:?}",
"FxCache written: {} bars, v1 (f32), {:.2} MB -> {:?}",
bar_count,
version,
if bf16 { "bf16" } else { "f64" },
total_bytes as f64 / 1_048_576.0,
path
);
@@ -289,72 +277,39 @@ pub fn write_fxcache(
Ok(total_bytes)
}
/// Write body in f64 format (version 1): [i64 ts] + 54 f64 values = 440 bytes per bar.
fn write_body_f64(
/// Write body in f32 format (version 1): [i64 ts] + 54 × f32 = 224 bytes per bar.
fn write_body_f32(
writer: &mut BufWriter<std::fs::File>,
features: &[[f64; FEAT_DIM]],
targets: &[[f64; TARGET_DIM]],
ofi: &[[f64; OFI_DIM]],
timestamps: &[i64],
) -> Result<u64> {
let bytes_per_bar = 8 + RECORD_F64_COUNT * 8; // i64 timestamp + f64 data
let bytes_per_bar = 8 + RECORD_F32_COUNT * 4; // i64 timestamp + f32 data
let total = features.len() as u64 * bytes_per_bar as u64;
for i in 0..features.len() {
writer.write_all(&timestamps[i].to_le_bytes())?;
for &v in &features[i] {
writer.write_all(&v.to_le_bytes())?;
writer.write_all(&(v as f32).to_le_bytes())?;
}
for &v in &targets[i] {
writer.write_all(&v.to_le_bytes())?;
writer.write_all(&(v as f32).to_le_bytes())?;
}
for &v in &ofi[i] {
writer.write_all(&v.to_le_bytes())?;
writer.write_all(&(v as f32).to_le_bytes())?;
}
}
Ok(total)
}
/// Write body in bf16 format (version 2): [i64 ts] + 56 bf16 values = 120 bytes per bar.
/// (54 data values + 2 zero-padding for 4-byte alignment.)
/// Timestamp is always i64 (8 bytes) — nanosecond precision requires 64 bits.
fn write_body_bf16(
writer: &mut BufWriter<std::fs::File>,
features: &[[f64; FEAT_DIM]],
targets: &[[f64; TARGET_DIM]],
ofi: &[[f64; OFI_DIM]],
timestamps: &[i64],
) -> Result<u64> {
let bytes_per_bar = 8 + RECORD_BF16_COUNT * 2; // i64 timestamp + bf16 data
let total = features.len() as u64 * bytes_per_bar as u64;
let zero = f16::ZERO;
for i in 0..features.len() {
writer.write_all(&timestamps[i].to_le_bytes())?;
for &v in &features[i] {
writer.write_all(&f16::from_f64(v).to_le_bytes())?;
}
for &v in &targets[i] {
writer.write_all(&f16::from_f64(v).to_le_bytes())?;
}
for &v in &ofi[i] {
writer.write_all(&f16::from_f64(v).to_le_bytes())?;
}
// 2 zero-padding bf16 values for alignment
writer.write_all(&zero.to_le_bytes())?;
writer.write_all(&zero.to_le_bytes())?;
}
Ok(total)
}
// ── Reader ───────────────────────────────────────────────────────────────────
/// Load an `.fxcache` file into memory.
///
/// Reads the 64-byte header, validates it, then reads the body according to
/// the version (f64 or bf16). bf16 values are up-converted to f64 on load.
/// Reads the 64-byte header, validates it, then reads the f32 body,
/// converting each f32 back to f64 on load.
///
/// # Arguments
///
@@ -382,12 +337,8 @@ pub fn load_fxcache(path: &Path) -> Result<FxCacheData> {
let bar_count = header.bar_count as usize;
// Sanity-check file size (each record has an i64 timestamp prefix)
let expected_body = if header.version == 1 {
bar_count as u64 * (8 + RECORD_F64_COUNT as u64 * 8)
} else {
bar_count as u64 * (8 + RECORD_BF16_COUNT as u64 * 2)
};
// Sanity-check file size: [i64 ts] + 54 × f32 = 224 bytes/bar
let expected_body = bar_count as u64 * (8 + RECORD_F32_COUNT as u64 * 4);
let expected_total = HEADER_SIZE as u64 + expected_body;
if file_len < expected_total {
bail!(
@@ -397,18 +348,12 @@ pub fn load_fxcache(path: &Path) -> Result<FxCacheData> {
);
}
// Read body
let (timestamps, features, targets, ofi) = if header.version == 1 {
read_body_f64(&mut reader, bar_count)?
} else {
read_body_bf16(&mut reader, bar_count)?
};
// Read body (f32 format)
let (timestamps, features, targets, ofi) = read_body_f32(&mut reader, bar_count)?;
info!(
"FxCache loaded: {} bars, v{} ({}) from {:?}",
"FxCache loaded: {} bars, v1 (f32) from {:?}",
bar_count,
header.version,
if header.version == 1 { "f64" } else { "bf16" },
path
);
@@ -425,8 +370,8 @@ pub fn load_fxcache(path: &Path) -> Result<FxCacheData> {
})
}
/// Read body in f64 format (version 1).
fn read_body_f64(
/// Read body in f32 format (version 1), converting f32 back to f64 on load.
fn read_body_f32(
reader: &mut BufReader<std::fs::File>,
bar_count: usize,
) -> Result<(Vec<i64>, Vec<[f64; FEAT_DIM]>, Vec<[f64; TARGET_DIM]>, Vec<[f64; OFI_DIM]>)> {
@@ -435,7 +380,7 @@ fn read_body_f64(
let mut targets = Vec::with_capacity(bar_count);
let mut ofi = Vec::with_capacity(bar_count);
let mut i64_buf = [0u8; 8];
let mut f64_buf = [0u8; 8];
let mut f32_buf = [0u8; 4];
for _ in 0..bar_count {
reader.read_exact(&mut i64_buf)?;
@@ -443,22 +388,22 @@ fn read_body_f64(
let mut feat = [0.0_f64; FEAT_DIM];
for slot in &mut feat {
reader.read_exact(&mut f64_buf)?;
*slot = f64::from_le_bytes(f64_buf);
reader.read_exact(&mut f32_buf)?;
*slot = f32::from_le_bytes(f32_buf) as f64;
}
features.push(feat);
let mut tgt = [0.0_f64; TARGET_DIM];
for slot in &mut tgt {
reader.read_exact(&mut f64_buf)?;
*slot = f64::from_le_bytes(f64_buf);
reader.read_exact(&mut f32_buf)?;
*slot = f32::from_le_bytes(f32_buf) as f64;
}
targets.push(tgt);
let mut ofi_row = [0.0_f64; OFI_DIM];
for slot in &mut ofi_row {
reader.read_exact(&mut f64_buf)?;
*slot = f64::from_le_bytes(f64_buf);
reader.read_exact(&mut f32_buf)?;
*slot = f32::from_le_bytes(f32_buf) as f64;
}
ofi.push(ofi_row);
}
@@ -466,52 +411,6 @@ fn read_body_f64(
Ok((timestamps, features, targets, ofi))
}
/// Read body in bf16 format (version 2), converting to f64 on load.
/// Timestamp is always read as i64 (8 bytes) regardless of bf16 mode.
fn read_body_bf16(
reader: &mut BufReader<std::fs::File>,
bar_count: usize,
) -> Result<(Vec<i64>, Vec<[f64; FEAT_DIM]>, Vec<[f64; TARGET_DIM]>, Vec<[f64; OFI_DIM]>)> {
let mut timestamps = Vec::with_capacity(bar_count);
let mut features = Vec::with_capacity(bar_count);
let mut targets = Vec::with_capacity(bar_count);
let mut ofi = Vec::with_capacity(bar_count);
let mut i64_buf = [0u8; 8];
let mut bf16_buf = [0u8; 2];
for _ in 0..bar_count {
reader.read_exact(&mut i64_buf)?;
timestamps.push(i64::from_le_bytes(i64_buf));
let mut feat = [0.0_f64; FEAT_DIM];
for slot in &mut feat {
reader.read_exact(&mut bf16_buf)?;
*slot = f16::from_le_bytes(bf16_buf).to_f64();
}
features.push(feat);
let mut tgt = [0.0_f64; TARGET_DIM];
for slot in &mut tgt {
reader.read_exact(&mut bf16_buf)?;
*slot = f16::from_le_bytes(bf16_buf).to_f64();
}
targets.push(tgt);
let mut ofi_row = [0.0_f64; OFI_DIM];
for slot in &mut ofi_row {
reader.read_exact(&mut bf16_buf)?;
*slot = f16::from_le_bytes(bf16_buf).to_f64();
}
ofi.push(ofi_row);
// Skip 2 padding bf16 values
reader.read_exact(&mut bf16_buf)?;
reader.read_exact(&mut bf16_buf)?;
}
Ok((timestamps, features, targets, ofi))
}
// ── Finder ───────────────────────────────────────────────────────────────────
#[cfg(test)]
@@ -528,7 +427,7 @@ mod has_ofi_tests {
let timestamps = vec![1_i64; 10];
let key = [0u8; 32];
write_fxcache(&path, &features, &targets, &ofi, &timestamps, key, false, true).unwrap();
write_fxcache(&path, &features, &targets, &ofi, &timestamps, key, true).unwrap();
let loaded = load_fxcache(&path).unwrap();
assert!(loaded.has_ofi, "has_ofi should be true");
assert_eq!(loaded.bar_count, 10);
@@ -544,7 +443,7 @@ mod has_ofi_tests {
let timestamps = vec![1_i64; 10];
let key = [0u8; 32];
write_fxcache(&path, &features, &targets, &ofi, &timestamps, key, false, false).unwrap();
write_fxcache(&path, &features, &targets, &ofi, &timestamps, key, false).unwrap();
let loaded = load_fxcache(&path).unwrap();
assert!(!loaded.has_ofi, "has_ofi should be false");
}
@@ -693,7 +592,7 @@ mod discover_tests {
let targets = vec![[0.0_f64; 4]; 5];
let ofi = vec![[0.0_f64; 8]; 5];
let timestamps = vec![1_i64; 5];
write_fxcache(&path, &features, &targets, &ofi, &timestamps, [0u8; 32], false, false)
write_fxcache(&path, &features, &targets, &ofi, &timestamps, [0u8; 32], false)
.unwrap();
// Try to discover with a real data_dir (different key) — should NOT match

66
crates/ml/tests/fxcache_roundtrip_test.rs
View File

@@ -1,6 +1,6 @@
//! Roundtrip tests for the `.fxcache` binary format.
//!
//! Covers f64/bf16 serialization, cache-key lookup, header validation,
//! Covers f32 serialization, cache-key lookup, header validation,
//! and the empty-bars error path.
use ml::fxcache::{find_fxcache, load_fxcache, write_fxcache, FxCacheData};
@@ -65,11 +65,11 @@ fn test_cache_key() -> [u8; 32] {
// ── Tests ───────────────────────────────────────────────────────────────────
/// Write 100 bars as f64 (version 1), read back, verify bit-exact match.
/// Write 100 bars as f32 (version 1), read back, verify within f32 tolerance.
#[test]
fn test_fxcache_f64_roundtrip() {
fn test_fxcache_f32_roundtrip() {
let dir = TempDir::new().unwrap();
let path = dir.path().join("test_f64.fxcache");
let path = dir.path().join("test_f32.fxcache");
let n = 100;
let features = make_features(n);
@@ -79,7 +79,7 @@ fn test_fxcache_f64_roundtrip() {
let key = test_cache_key();
let bytes_written =
write_fxcache(&path, &features, &targets, &ofi, &timestamps, key, false, false).unwrap();
write_fxcache(&path, &features, &targets, &ofi, &timestamps, key, false).unwrap();
assert!(bytes_written > 0, "expected nonzero bytes written");
let data: FxCacheData = load_fxcache(&path).unwrap();
@@ -90,47 +90,7 @@ fn test_fxcache_f64_roundtrip() {
assert_eq!(data.ofi.len(), n);
assert_eq!(data.timestamps.len(), n);
// Bit-exact match for f64 roundtrip.
for i in 0..n {
assert_eq!(
data.timestamps[i], timestamps[i],
"timestamp mismatch at bar {i}"
);
assert_eq!(
data.features[i], features[i],
"feature mismatch at bar {i}"
);
assert_eq!(
data.targets[i], targets[i],
"target mismatch at bar {i}"
);
assert_eq!(data.ofi[i], ofi[i], "ofi mismatch at bar {i}");
}
}
/// Write 50 bars as bf16 (version 2), read back, verify within tolerance.
/// bf16 has limited precision: ~3 significant digits.
#[test]
fn test_fxcache_bf16_roundtrip() {
let dir = TempDir::new().unwrap();
let path = dir.path().join("test_bf16.fxcache");
let n = 50;
let features = make_features(n);
let targets = make_targets(n);
let ofi = make_ofi(n);
let timestamps = make_timestamps(n);
let key = test_cache_key();
let bytes_written =
write_fxcache(&path, &features, &targets, &ofi, &timestamps, key, true, false).unwrap();
assert!(bytes_written > 0, "expected nonzero bytes written");
let data: FxCacheData = load_fxcache(&path).unwrap();
assert_eq!(data.bar_count, n);
assert_eq!(data.cache_key, key);
// Timestamps are always i64 — bit-exact even in bf16 mode.
// Timestamps are always i64 — bit-exact.
for i in 0..n {
assert_eq!(
data.timestamps[i], timestamps[i],
@@ -138,12 +98,12 @@ fn test_fxcache_bf16_roundtrip() {
);
}
// bf16 tolerance checks.
// f32 roundtrip tolerance checks (~7 significant digits).
for i in 0..n {
for j in 0..42 {
let diff = (data.features[i][j] - features[i][j]).abs();
assert!(
diff < 0.02,
diff < 1e-4,
"feature[{i}][{j}]: expected {}, got {}, diff={diff}",
features[i][j],
data.features[i][j]
@@ -151,9 +111,9 @@ fn test_fxcache_bf16_roundtrip() {
}
for j in 0..4 {
let diff = (data.targets[i][j] - targets[i][j]).abs();
// bf16 at ~5000 has resolution of ~4, so allow up to 4.0.
// f32 at ~5000 has resolution of ~0.0005, allow generous margin.
assert!(
diff <= 4.0,
diff < 0.01,
"target[{i}][{j}]: expected {}, got {}, diff={diff}",
targets[i][j],
data.targets[i][j]
@@ -162,7 +122,7 @@ fn test_fxcache_bf16_roundtrip() {
for j in 0..8 {
let diff = (data.ofi[i][j] - ofi[i][j]).abs();
assert!(
diff < 0.02,
diff < 1e-4,
"ofi[{i}][{j}]: expected {}, got {}, diff={diff}",
ofi[i][j],
data.ofi[i][j]
@@ -185,7 +145,7 @@ fn test_fxcache_find() {
let ofi = make_ofi(5);
let timestamps = make_timestamps(5);
write_fxcache(&path, &features, &targets, &ofi, &timestamps, key, false, false).unwrap();
write_fxcache(&path, &features, &targets, &ofi, &timestamps, key, false).unwrap();
// Should find the file.
let found = find_fxcache(dir.path(), &key);
@@ -231,7 +191,7 @@ fn test_fxcache_empty() {
let timestamps: Vec<i64> = vec![];
let key = test_cache_key();
let err = write_fxcache(&path, &features, &targets, &ofi, &timestamps, key, false, false).unwrap_err();
let err = write_fxcache(&path, &features, &targets, &ofi, &timestamps, key, false).unwrap_err();
let msg = format!("{err:#}");
assert!(
msg.contains("0 bars") || msg.contains("empty"),