a4ba8bddaa
Each record now starts with an i64 timestamp (nanoseconds since epoch) before the feature/target/OFI data. v1 records grow from 432 to 440 bytes, v2 from 112 to 120 bytes. The timestamp is always i64 even in bf16 mode. train_baseline_rl reconstructs bars with real timestamps instead of placeholders so the walk-forward windower can split by month. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
530 lines
18 KiB
Rust
530 lines
18 KiB
Rust
//! FxCache — Flat Binary Feature Cache for Zero-Overhead GPU Loading
|
||
//!
|
||
//! Provides a compact binary format for pre-computed DQN training data
|
||
//! (features + targets + OFI vectors) designed for direct GPU upload
|
||
//! without parsing overhead.
|
||
//!
|
||
//! ## Format
|
||
//!
|
||
//! ```text
|
||
//! ┌───────────────────────────────────────────────────┐
|
||
//! │ FxCacheHeader (64 bytes) │
|
||
//! │ magic [u8; 8] = b"FXCACHE\0" │
|
||
//! │ version u16 = 1 (f64) | 2 (bf16) │
|
||
//! │ feat_dim u16 = 42 │
|
||
//! │ target_dim u16 = 4 │
|
||
//! │ ofi_dim u16 = 8 │
|
||
//! │ bar_count u64 │
|
||
//! │ cache_key [u8; 32] (SHA256 raw bytes) │
|
||
//! │ reserved [u8; 8] │
|
||
//! └───────────────────────────────────────────────────┘
|
||
//! │ Body (bar_count records) │
|
||
//! │ Each record starts with an i64 timestamp (ns). │
|
||
//! │ Version 1: [i64 ts][54 × f64] = 440 bytes/bar │
|
||
//! │ Version 2: [i64 ts][56 × bf16] = 120 bytes/bar │
|
||
//! │ (54 data + 2 zero-padding) │
|
||
//! └───────────────────────────────────────────────────┘
|
||
//! ```
|
||
|
||
use anyhow::{bail, Context, Result};
|
||
use half::f16;
|
||
use std::io::{BufReader, BufWriter, Read, Write};
|
||
use std::path::{Path, PathBuf};
|
||
use tracing::{debug, info};
|
||
|
||
// ── Constants ────────────────────────────────────────────────────────────────
|
||
|
||
/// Magic bytes identifying an FxCache file.
|
||
const FXCACHE_MAGIC: [u8; 8] = *b"FXCACHE\0";
|
||
|
||
/// Header size in bytes (fixed).
|
||
const HEADER_SIZE: usize = 64;
|
||
|
||
/// Feature vector dimensionality.
|
||
const FEAT_DIM: usize = 42;
|
||
|
||
/// Target vector dimensionality (close, next_close, raw_close, raw_next).
|
||
const TARGET_DIM: usize = 4;
|
||
|
||
/// OFI vector dimensionality (8 MBP-10 order-flow imbalance levels).
|
||
const OFI_DIM: usize = 8;
|
||
|
||
/// Total f64 values per record: features + targets + OFI = 42 + 4 + 8 = 54.
|
||
const RECORD_F64_COUNT: usize = FEAT_DIM + TARGET_DIM + OFI_DIM;
|
||
|
||
/// bf16 record width including 2 zero-padding values for 4-byte alignment.
|
||
const RECORD_BF16_COUNT: usize = RECORD_F64_COUNT + 2;
|
||
|
||
// ── Header ───────────────────────────────────────────────────────────────────
|
||
|
||
/// 64-byte fixed header for `.fxcache` files.
|
||
#[derive(Debug, Clone)]
|
||
pub struct FxCacheHeader {
|
||
/// Magic bytes: `b"FXCACHE\0"`.
|
||
pub magic: [u8; 8],
|
||
/// Format version: 1 = f64, 2 = bf16.
|
||
pub version: u16,
|
||
/// Feature dimension (42).
|
||
pub feat_dim: u16,
|
||
/// Target dimension (4).
|
||
pub target_dim: u16,
|
||
/// OFI dimension (8).
|
||
pub ofi_dim: u16,
|
||
/// Number of bars (records) in the file.
|
||
pub bar_count: u64,
|
||
/// SHA256 cache key (raw 32 bytes).
|
||
pub cache_key: [u8; 32],
|
||
/// Reserved for future use.
|
||
pub reserved: [u8; 8],
|
||
}
|
||
|
||
impl FxCacheHeader {
|
||
/// Create a new header with the given parameters.
|
||
pub fn new(version: u16, bar_count: u64, cache_key: [u8; 32]) -> Self {
|
||
Self {
|
||
magic: FXCACHE_MAGIC,
|
||
version,
|
||
feat_dim: FEAT_DIM as u16,
|
||
target_dim: TARGET_DIM as u16,
|
||
ofi_dim: OFI_DIM as u16,
|
||
bar_count,
|
||
cache_key,
|
||
reserved: [0u8; 8],
|
||
}
|
||
}
|
||
|
||
/// Validate header integrity.
|
||
pub fn validate(&self) -> Result<()> {
|
||
if self.magic != FXCACHE_MAGIC {
|
||
bail!(
|
||
"Invalid FxCache magic: expected {:?}, got {:?}",
|
||
FXCACHE_MAGIC,
|
||
self.magic
|
||
);
|
||
}
|
||
if self.version != 1 && self.version != 2 {
|
||
bail!(
|
||
"Unsupported FxCache version: {} (expected 1 or 2)",
|
||
self.version
|
||
);
|
||
}
|
||
if self.feat_dim as usize != FEAT_DIM {
|
||
bail!(
|
||
"Feature dimension mismatch: expected {}, got {}",
|
||
FEAT_DIM,
|
||
self.feat_dim
|
||
);
|
||
}
|
||
if self.target_dim as usize != TARGET_DIM {
|
||
bail!(
|
||
"Target dimension mismatch: expected {}, got {}",
|
||
TARGET_DIM,
|
||
self.target_dim
|
||
);
|
||
}
|
||
if self.ofi_dim as usize != OFI_DIM {
|
||
bail!(
|
||
"OFI dimension mismatch: expected {}, got {}",
|
||
OFI_DIM,
|
||
self.ofi_dim
|
||
);
|
||
}
|
||
if self.bar_count == 0 {
|
||
bail!("FxCache bar_count is zero — empty cache files are not valid");
|
||
}
|
||
Ok(())
|
||
}
|
||
|
||
/// Serialize header to 64 bytes (little-endian).
|
||
fn to_bytes(&self) -> [u8; HEADER_SIZE] {
|
||
let mut buf = [0u8; HEADER_SIZE];
|
||
buf[0..8].copy_from_slice(&self.magic);
|
||
buf[8..10].copy_from_slice(&self.version.to_le_bytes());
|
||
buf[10..12].copy_from_slice(&self.feat_dim.to_le_bytes());
|
||
buf[12..14].copy_from_slice(&self.target_dim.to_le_bytes());
|
||
buf[14..16].copy_from_slice(&self.ofi_dim.to_le_bytes());
|
||
buf[16..24].copy_from_slice(&self.bar_count.to_le_bytes());
|
||
buf[24..56].copy_from_slice(&self.cache_key);
|
||
buf[56..64].copy_from_slice(&self.reserved);
|
||
buf
|
||
}
|
||
|
||
/// Deserialize header from 64 bytes (little-endian).
|
||
fn from_bytes(buf: &[u8; HEADER_SIZE]) -> Self {
|
||
let mut magic = [0u8; 8];
|
||
magic.copy_from_slice(&buf[0..8]);
|
||
|
||
let version = u16::from_le_bytes([buf[8], buf[9]]);
|
||
let feat_dim = u16::from_le_bytes([buf[10], buf[11]]);
|
||
let target_dim = u16::from_le_bytes([buf[12], buf[13]]);
|
||
let ofi_dim = u16::from_le_bytes([buf[14], buf[15]]);
|
||
let bar_count = u64::from_le_bytes([
|
||
buf[16], buf[17], buf[18], buf[19], buf[20], buf[21], buf[22], buf[23],
|
||
]);
|
||
|
||
let mut cache_key = [0u8; 32];
|
||
cache_key.copy_from_slice(&buf[24..56]);
|
||
|
||
let mut reserved = [0u8; 8];
|
||
reserved.copy_from_slice(&buf[56..64]);
|
||
|
||
Self {
|
||
magic,
|
||
version,
|
||
feat_dim,
|
||
target_dim,
|
||
ofi_dim,
|
||
bar_count,
|
||
cache_key,
|
||
reserved,
|
||
}
|
||
}
|
||
}
|
||
|
||
// ── Data ─────────────────────────────────────────────────────────────────────
|
||
|
||
/// In-memory representation of an FxCache file's contents.
|
||
#[derive(Debug)]
|
||
pub struct FxCacheData {
|
||
/// Per-bar timestamps (nanoseconds since Unix epoch).
|
||
pub timestamps: Vec<i64>,
|
||
/// Feature vectors, one per bar (42 elements each).
|
||
pub features: Vec<[f64; FEAT_DIM]>,
|
||
/// Target vectors, one per bar (4 elements each).
|
||
pub targets: Vec<[f64; TARGET_DIM]>,
|
||
/// OFI vectors, one per bar (8 elements each).
|
||
pub ofi: Vec<[f64; OFI_DIM]>,
|
||
/// SHA256 cache key (raw 32 bytes).
|
||
pub cache_key: [u8; 32],
|
||
/// Number of bars.
|
||
pub bar_count: usize,
|
||
}
|
||
|
||
// ── Writer ───────────────────────────────────────────────────────────────────
|
||
|
||
/// Write feature/target/OFI data to an `.fxcache` binary file.
|
||
///
|
||
/// # Arguments
|
||
///
|
||
/// * `path` — Output file path (parent directories are created automatically)
|
||
/// * `features` — Slice of 42-element feature vectors
|
||
/// * `targets` — Slice of 4-element target vectors
|
||
/// * `ofi` — Slice of 8-element OFI vectors
|
||
/// * `timestamps` — Per-bar timestamps (nanoseconds since Unix epoch)
|
||
/// * `cache_key` — SHA256 key (raw 32 bytes)
|
||
/// * `bf16` — If true, write version 2 (bf16); otherwise version 1 (f64)
|
||
///
|
||
/// # Returns
|
||
///
|
||
/// Total bytes written (header + body).
|
||
pub fn write_fxcache(
|
||
path: &Path,
|
||
features: &[[f64; FEAT_DIM]],
|
||
targets: &[[f64; TARGET_DIM]],
|
||
ofi: &[[f64; OFI_DIM]],
|
||
timestamps: &[i64],
|
||
cache_key: [u8; 32],
|
||
bf16: bool,
|
||
) -> Result<u64> {
|
||
let bar_count = features.len();
|
||
if targets.len() != bar_count || ofi.len() != bar_count || timestamps.len() != bar_count {
|
||
bail!(
|
||
"Length mismatch: features={}, targets={}, ofi={}, timestamps={}",
|
||
bar_count,
|
||
targets.len(),
|
||
ofi.len(),
|
||
timestamps.len()
|
||
);
|
||
}
|
||
if bar_count == 0 {
|
||
bail!("Cannot write empty FxCache (0 bars)");
|
||
}
|
||
|
||
// Create parent directories
|
||
if let Some(parent) = path.parent() {
|
||
std::fs::create_dir_all(parent)
|
||
.with_context(|| format!("Failed to create parent dirs for {:?}", path))?;
|
||
}
|
||
|
||
let version: u16 = if bf16 { 2 } else { 1 };
|
||
let header = FxCacheHeader::new(version, bar_count as u64, cache_key);
|
||
header.validate()?;
|
||
|
||
let file = std::fs::File::create(path)
|
||
.with_context(|| format!("Failed to create FxCache file {:?}", path))?;
|
||
let mut writer = BufWriter::new(file);
|
||
|
||
// Write header
|
||
writer
|
||
.write_all(&header.to_bytes())
|
||
.context("Failed to write FxCache header")?;
|
||
|
||
// Write body
|
||
let body_bytes: u64 = if bf16 {
|
||
write_body_bf16(&mut writer, features, targets, ofi, timestamps)?
|
||
} else {
|
||
write_body_f64(&mut writer, features, targets, ofi, timestamps)?
|
||
};
|
||
|
||
writer.flush().context("Failed to flush FxCache writer")?;
|
||
|
||
let total_bytes = HEADER_SIZE as u64 + body_bytes;
|
||
|
||
info!(
|
||
"FxCache written: {} bars, v{} ({}), {:.2} MB -> {:?}",
|
||
bar_count,
|
||
version,
|
||
if bf16 { "bf16" } else { "f64" },
|
||
total_bytes as f64 / 1_048_576.0,
|
||
path
|
||
);
|
||
|
||
Ok(total_bytes)
|
||
}
|
||
|
||
/// Write body in f64 format (version 1): [i64 ts] + 54 f64 values = 440 bytes per bar.
|
||
fn write_body_f64(
|
||
writer: &mut BufWriter<std::fs::File>,
|
||
features: &[[f64; FEAT_DIM]],
|
||
targets: &[[f64; TARGET_DIM]],
|
||
ofi: &[[f64; OFI_DIM]],
|
||
timestamps: &[i64],
|
||
) -> Result<u64> {
|
||
let bytes_per_bar = 8 + RECORD_F64_COUNT * 8; // i64 timestamp + f64 data
|
||
let total = features.len() as u64 * bytes_per_bar as u64;
|
||
|
||
for i in 0..features.len() {
|
||
writer.write_all(×tamps[i].to_le_bytes())?;
|
||
for &v in &features[i] {
|
||
writer.write_all(&v.to_le_bytes())?;
|
||
}
|
||
for &v in &targets[i] {
|
||
writer.write_all(&v.to_le_bytes())?;
|
||
}
|
||
for &v in &ofi[i] {
|
||
writer.write_all(&v.to_le_bytes())?;
|
||
}
|
||
}
|
||
|
||
Ok(total)
|
||
}
|
||
|
||
/// Write body in bf16 format (version 2): [i64 ts] + 56 bf16 values = 120 bytes per bar.
|
||
/// (54 data values + 2 zero-padding for 4-byte alignment.)
|
||
/// Timestamp is always i64 (8 bytes) — nanosecond precision requires 64 bits.
|
||
fn write_body_bf16(
|
||
writer: &mut BufWriter<std::fs::File>,
|
||
features: &[[f64; FEAT_DIM]],
|
||
targets: &[[f64; TARGET_DIM]],
|
||
ofi: &[[f64; OFI_DIM]],
|
||
timestamps: &[i64],
|
||
) -> Result<u64> {
|
||
let bytes_per_bar = 8 + RECORD_BF16_COUNT * 2; // i64 timestamp + bf16 data
|
||
let total = features.len() as u64 * bytes_per_bar as u64;
|
||
let zero = f16::ZERO;
|
||
|
||
for i in 0..features.len() {
|
||
writer.write_all(×tamps[i].to_le_bytes())?;
|
||
for &v in &features[i] {
|
||
writer.write_all(&f16::from_f64(v).to_le_bytes())?;
|
||
}
|
||
for &v in &targets[i] {
|
||
writer.write_all(&f16::from_f64(v).to_le_bytes())?;
|
||
}
|
||
for &v in &ofi[i] {
|
||
writer.write_all(&f16::from_f64(v).to_le_bytes())?;
|
||
}
|
||
// 2 zero-padding bf16 values for alignment
|
||
writer.write_all(&zero.to_le_bytes())?;
|
||
writer.write_all(&zero.to_le_bytes())?;
|
||
}
|
||
|
||
Ok(total)
|
||
}
|
||
|
||
// ── Reader ───────────────────────────────────────────────────────────────────
|
||
|
||
/// Load an `.fxcache` file into memory.
|
||
///
|
||
/// Reads the 64-byte header, validates it, then reads the body according to
|
||
/// the version (f64 or bf16). bf16 values are up-converted to f64 on load.
|
||
///
|
||
/// # Arguments
|
||
///
|
||
/// * `path` — Path to the `.fxcache` file
|
||
///
|
||
/// # Returns
|
||
///
|
||
/// Fully parsed `FxCacheData` with features, targets, OFI, cache key, and bar count.
|
||
pub fn load_fxcache(path: &Path) -> Result<FxCacheData> {
|
||
let file = std::fs::File::open(path)
|
||
.with_context(|| format!("Failed to open FxCache file {:?}", path))?;
|
||
let file_len = file
|
||
.metadata()
|
||
.with_context(|| format!("Failed to stat FxCache file {:?}", path))?
|
||
.len();
|
||
let mut reader = BufReader::new(file);
|
||
|
||
// Read header
|
||
let mut header_buf = [0u8; HEADER_SIZE];
|
||
reader
|
||
.read_exact(&mut header_buf)
|
||
.context("Failed to read FxCache header")?;
|
||
let header = FxCacheHeader::from_bytes(&header_buf);
|
||
header.validate()?;
|
||
|
||
let bar_count = header.bar_count as usize;
|
||
|
||
// Sanity-check file size (each record has an i64 timestamp prefix)
|
||
let expected_body = if header.version == 1 {
|
||
bar_count as u64 * (8 + RECORD_F64_COUNT as u64 * 8)
|
||
} else {
|
||
bar_count as u64 * (8 + RECORD_BF16_COUNT as u64 * 2)
|
||
};
|
||
let expected_total = HEADER_SIZE as u64 + expected_body;
|
||
if file_len < expected_total {
|
||
bail!(
|
||
"FxCache file truncated: expected {} bytes, got {}",
|
||
expected_total,
|
||
file_len
|
||
);
|
||
}
|
||
|
||
// Read body
|
||
let (timestamps, features, targets, ofi) = if header.version == 1 {
|
||
read_body_f64(&mut reader, bar_count)?
|
||
} else {
|
||
read_body_bf16(&mut reader, bar_count)?
|
||
};
|
||
|
||
info!(
|
||
"FxCache loaded: {} bars, v{} ({}) from {:?}",
|
||
bar_count,
|
||
header.version,
|
||
if header.version == 1 { "f64" } else { "bf16" },
|
||
path
|
||
);
|
||
|
||
Ok(FxCacheData {
|
||
timestamps,
|
||
features,
|
||
targets,
|
||
ofi,
|
||
cache_key: header.cache_key,
|
||
bar_count,
|
||
})
|
||
}
|
||
|
||
/// Read body in f64 format (version 1).
|
||
fn read_body_f64(
|
||
reader: &mut BufReader<std::fs::File>,
|
||
bar_count: usize,
|
||
) -> Result<(Vec<i64>, Vec<[f64; FEAT_DIM]>, Vec<[f64; TARGET_DIM]>, Vec<[f64; OFI_DIM]>)> {
|
||
let mut timestamps = Vec::with_capacity(bar_count);
|
||
let mut features = Vec::with_capacity(bar_count);
|
||
let mut targets = Vec::with_capacity(bar_count);
|
||
let mut ofi = Vec::with_capacity(bar_count);
|
||
let mut i64_buf = [0u8; 8];
|
||
let mut f64_buf = [0u8; 8];
|
||
|
||
for _ in 0..bar_count {
|
||
reader.read_exact(&mut i64_buf)?;
|
||
timestamps.push(i64::from_le_bytes(i64_buf));
|
||
|
||
let mut feat = [0.0_f64; FEAT_DIM];
|
||
for slot in &mut feat {
|
||
reader.read_exact(&mut f64_buf)?;
|
||
*slot = f64::from_le_bytes(f64_buf);
|
||
}
|
||
features.push(feat);
|
||
|
||
let mut tgt = [0.0_f64; TARGET_DIM];
|
||
for slot in &mut tgt {
|
||
reader.read_exact(&mut f64_buf)?;
|
||
*slot = f64::from_le_bytes(f64_buf);
|
||
}
|
||
targets.push(tgt);
|
||
|
||
let mut ofi_row = [0.0_f64; OFI_DIM];
|
||
for slot in &mut ofi_row {
|
||
reader.read_exact(&mut f64_buf)?;
|
||
*slot = f64::from_le_bytes(f64_buf);
|
||
}
|
||
ofi.push(ofi_row);
|
||
}
|
||
|
||
Ok((timestamps, features, targets, ofi))
|
||
}
|
||
|
||
/// Read body in bf16 format (version 2), converting to f64 on load.
|
||
/// Timestamp is always read as i64 (8 bytes) regardless of bf16 mode.
|
||
fn read_body_bf16(
|
||
reader: &mut BufReader<std::fs::File>,
|
||
bar_count: usize,
|
||
) -> Result<(Vec<i64>, Vec<[f64; FEAT_DIM]>, Vec<[f64; TARGET_DIM]>, Vec<[f64; OFI_DIM]>)> {
|
||
let mut timestamps = Vec::with_capacity(bar_count);
|
||
let mut features = Vec::with_capacity(bar_count);
|
||
let mut targets = Vec::with_capacity(bar_count);
|
||
let mut ofi = Vec::with_capacity(bar_count);
|
||
let mut i64_buf = [0u8; 8];
|
||
let mut bf16_buf = [0u8; 2];
|
||
|
||
for _ in 0..bar_count {
|
||
reader.read_exact(&mut i64_buf)?;
|
||
timestamps.push(i64::from_le_bytes(i64_buf));
|
||
|
||
let mut feat = [0.0_f64; FEAT_DIM];
|
||
for slot in &mut feat {
|
||
reader.read_exact(&mut bf16_buf)?;
|
||
*slot = f16::from_le_bytes(bf16_buf).to_f64();
|
||
}
|
||
features.push(feat);
|
||
|
||
let mut tgt = [0.0_f64; TARGET_DIM];
|
||
for slot in &mut tgt {
|
||
reader.read_exact(&mut bf16_buf)?;
|
||
*slot = f16::from_le_bytes(bf16_buf).to_f64();
|
||
}
|
||
targets.push(tgt);
|
||
|
||
let mut ofi_row = [0.0_f64; OFI_DIM];
|
||
for slot in &mut ofi_row {
|
||
reader.read_exact(&mut bf16_buf)?;
|
||
*slot = f16::from_le_bytes(bf16_buf).to_f64();
|
||
}
|
||
ofi.push(ofi_row);
|
||
|
||
// Skip 2 padding bf16 values
|
||
reader.read_exact(&mut bf16_buf)?;
|
||
reader.read_exact(&mut bf16_buf)?;
|
||
}
|
||
|
||
Ok((timestamps, features, targets, ofi))
|
||
}
|
||
|
||
// ── Finder ───────────────────────────────────────────────────────────────────
|
||
|
||
/// Find an `.fxcache` file by hex-encoded cache key in a cache directory.
|
||
///
|
||
/// Looks for a file named `<hex_key>.fxcache` in `cache_dir`.
|
||
///
|
||
/// # Arguments
|
||
///
|
||
/// * `cache_dir` — Directory to search
|
||
/// * `cache_key` — Raw 32-byte SHA256 cache key
|
||
///
|
||
/// # Returns
|
||
///
|
||
/// `Some(path)` if the file exists, `None` otherwise.
|
||
pub fn find_fxcache(cache_dir: &Path, cache_key: &[u8; 32]) -> Option<PathBuf> {
|
||
let hex_key = hex::encode(cache_key);
|
||
let candidate = cache_dir.join(format!("{hex_key}.fxcache"));
|
||
if candidate.exists() {
|
||
debug!("FxCache found: {:?}", candidate);
|
||
Some(candidate)
|
||
} else {
|
||
debug!("FxCache miss: {:?}", candidate);
|
||
None
|
||
}
|
||
}
|