foxhunt/crates/ml/src/fxcache.rs

//! FxCache — Flat Binary Feature Cache for Zero-Overhead GPU Loading
//!
//! Provides a compact binary format for pre-computed DQN training data
//! (features + targets + OFI vectors) designed for direct GPU upload
//! without parsing overhead.
//!
//! ## Format
//!
//! ```text
//! ┌─────────────────────────────────────────────────────────────┐
//! │ FxCacheHeader (72 bytes)                                    │
//! │  magic               [u8; 8]   = b"FXCACHE\0"               │
//! │  version             u16       = 6 (f32 + schema-hash)      │
//! │  feat_dim            u16       = 42                         │
//! │  target_dim          u16       = 6                          │
//! │  ofi_dim             u16       = 32                         │
//! │  bar_count           u64                                    │
//! │  cache_key           [u8; 32]  (SHA256 raw bytes)           │
//! │  feature_schema_hash u64       (FNV-1a from build.rs)       │
//! │  reserved            [u8; 8]                                │
//! └─────────────────────────────────────────────────────────────┘
//! │ Body (bar_count records)                                    │
//! │  Each record starts with an i64 timestamp (ns).             │
//! │  Version 6: [i64 ts][80 × f32] = 328 bytes/bar              │
//! └─────────────────────────────────────────────────────────────┘
//! ```

use anyhow::{bail, Context, Result};
use std::io::{BufReader, BufWriter, Read, Write};
use std::path::{Path, PathBuf};
use tracing::{debug, info};

// ── Constants ────────────────────────────────────────────────────────────────

/// Magic bytes identifying an FxCache file.
const FXCACHE_MAGIC: [u8; 8] = *b"FXCACHE\0";

/// Header size in bytes (fixed).
const HEADER_SIZE: usize = 72;

/// Cache format version. Bump on ANY *wire-format* change (header layout,
/// body record shape). Schema-level changes (feature column semantics,
/// dimensionality) are tracked automatically by `FEATURE_SCHEMA_HASH` —
/// callers do NOT need to bump this constant when they edit
/// `features/extraction.rs` or `ml-core::state_layout`.
/// Stale cache files with wrong version OR wrong schema hash are
/// auto-detected and rejected by `validate()`. The ensure-fxcache Argo step
/// catches the error and regenerates.
/// v5: OFI_DIM 20→32 — adds 10 MicrostructureState slots (ofi_trajectory,
///     realized_variance, hawkes_intensity, book_pressure, spread_dynamics,
///     aggression_ratio, queue_depletion_asymmetry, order_count_flux,
///     intra_bar_momentum, regime_score) + 2 TLOB-novel slots
///     (order_count_imbalance, microprice_residual).
/// v6: header grows 64→72 bytes, adds `feature_schema_hash: u64` between
///     `cache_key` and `reserved`. Stamped at write time from the
///     compile-time `FEATURE_SCHEMA_HASH` const (built from the bytes of
///     extraction.rs / fxcache.rs / state_layout.rs by `build.rs`).
///     Recover-from-stale path is identical: validate() bails, Argo regens.
/// v7: OFI window alignment fix — OFI features at bar t now use the formation
///     interval `(close(bar_{t-1}), close(bar_t)]` instead of the leaked
///     `[close(bar_t), close(bar_{t+1}))`. Wire-format unchanged; semantic
///     contract changed. Per audit `docs/lookahead-bias-audit-2026-04-28.md`
///     §3 the previous layout leaked bar t+1's microstructure into 31/32 OFI
///     dims at bar t. Caches written before this fix carry contaminated OFI
///     and MUST be regenerated. Strict-checked via `validate()`.
/// v8: Target column semantic fix — slots [0:1] (`preproc_close`/`preproc_next`)
///     now hold log-return-normalized values per the documented contract in
///     `experience_kernels.cu:1556` and `cuda_pipeline/mod.rs:508`. Prior writers
///     (`precompute_features.rs:360`, `data_loading.rs:510`) stored raw prices
///     in all 6 target slots, leaving the network-input columns at ~$5000+ ES
///     futures price level instead of unit-scale log returns. Wire-format
///     unchanged; semantic contract enforced. Caches written before this fix
///     carry raw prices in `preproc_*` columns and MUST be regenerated.
pub const FXCACHE_VERSION: u16 = 8;

/// Compile-time fingerprint over the feature-schema source files. Bumps
/// automatically whenever `features/extraction.rs`, `fxcache.rs`, or
/// `ml-core::state_layout` change (any byte — whitespace counts).
///
/// Encoded into the fxcache header at write time and strict-checked at load
/// time, so caches built against a different schema fail validation and
/// trigger automatic regeneration via `precompute_features`. This removes
/// the manual "remember to bump `FXCACHE_VERSION` on schema change" ritual.
///
/// The actual value is set by `build.rs` (FNV-1a 64-bit). Stable across
/// rust versions and machines (unlike `std::hash::DefaultHasher`).
pub const FEATURE_SCHEMA_HASH: u64 = {
    // build.rs emits the hash as a decimal u64 string; `from_str_radix` is
    // const since rust 1.83 and the workspace MSRV is 1.85.
    match u64::from_str_radix(env!("FEATURE_SCHEMA_HASH"), 10) {
        Ok(v) => v,
        Err(_) => panic!("build.rs emitted invalid u64 for FEATURE_SCHEMA_HASH"),
    }
};

/// Feature vector dimensionality.
const FEAT_DIM: usize = 42;

/// Target vector dimensionality. Mirrors the named column constants below.
///
/// Public so consumer kernels and host-side readers can index by name rather
/// than literal stride. Two latent bugs converged in the cancelled 50-epoch
/// run `train-multi-seed-p5qzw` because consumers hardcoded stride 4 (left
/// over from pre-`063fd2716`) and column 0 (pre-`5a5dd0fed` raw_close, post-
/// fix preproc_close). Centralizing the layout here so future renames force
/// every consumer to update at the same call site.
pub const TARGET_DIM: usize = 6;

/// Column 0: `preproc_close` — log-return-normalized close (network input).
pub const TARGET_PREPROC_CLOSE: usize = 0;

/// Column 1: `preproc_next` — log-return-normalized next-bar close.
pub const TARGET_PREPROC_NEXT: usize = 1;

/// Column 2: `raw_close` — raw close price for portfolio simulation P&L + tx.
pub const TARGET_RAW_CLOSE: usize = 2;

/// Column 3: `raw_next` — raw next-bar close. **FUTURE information** — must
/// not be used for any reward, P&L, equity, or portfolio computation. Kept
/// in the layout for legacy back-compat; consumers should `(void)` this slot
/// to suppress unused-element warnings.
pub const TARGET_RAW_NEXT: usize = 3;

/// Column 4: `raw_open` — raw open price (OHLCV).
pub const TARGET_RAW_OPEN: usize = 4;

/// Column 5: `mid_price_open` — MBP-10 midpoint at bar open; falls back to
/// `raw_open` when no MBP-10 snapshot is available at the bar boundary.
pub const TARGET_MID_OPEN: usize = 5;

/// OFI vector dimensionality — mirrors `ml_core::state_layout::OFI_DIM`
/// (20 legacy slots + 12 new microstructure slots).
pub const OFI_DIM: usize = ml_core::state_layout::OFI_DIM;

/// Total f64 values per record: features + targets + OFI = 42 + 6 + 20 = 68.
const RECORD_F64_COUNT: usize = FEAT_DIM + TARGET_DIM + OFI_DIM;

/// Total f32 values per record (same count as f64, no padding needed).
const RECORD_F32_COUNT: usize = RECORD_F64_COUNT;

// ── Header ───────────────────────────────────────────────────────────────────

/// 72-byte fixed header for `.fxcache` files (v6+).
#[derive(Debug, Clone)]
pub struct FxCacheHeader {
    /// Magic bytes: `b"FXCACHE\0"`.
    pub magic: [u8; 8],
    /// Format version — see `FXCACHE_VERSION`.
    pub version: u16,
    /// Feature dimension (42).
    pub feat_dim: u16,
    /// Target dimension (6).
    pub target_dim: u16,
    /// OFI dimension (see `OFI_DIM`).
    pub ofi_dim: u16,
    /// Number of bars (records) in the file.
    pub bar_count: u64,
    /// SHA256 cache key (raw 32 bytes).
    pub cache_key: [u8; 32],
    /// Compile-time feature-schema fingerprint — see `FEATURE_SCHEMA_HASH`.
    /// Mismatch on load means the cache was built against different feature
    /// extraction / state-layout / fxcache-format source than the current
    /// binary. Strict-checked in `validate()`; regen via `precompute_features`.
    pub feature_schema_hash: u64,
    /// Reserved for future use.
    pub reserved: [u8; 8],
}

impl FxCacheHeader {
    /// Create a new header with the given parameters.
    pub fn new(version: u16, bar_count: u64, cache_key: [u8; 32], has_ofi: bool) -> Self {
        let mut reserved = [0u8; 8];
        reserved[0] = if has_ofi { 1 } else { 0 };
        Self {
            magic: FXCACHE_MAGIC,
            version,
            feat_dim: FEAT_DIM as u16,
            target_dim: TARGET_DIM as u16,
            ofi_dim: OFI_DIM as u16,
            bar_count,
            cache_key,
            feature_schema_hash: FEATURE_SCHEMA_HASH,
            reserved,
        }
    }

    /// Validate header integrity.
    ///
    /// Strictly enforces the current `FXCACHE_VERSION` AND the current
    /// `FEATURE_SCHEMA_HASH`. Any older version (including v4 with
    /// OFI_DIM=20, v5 without schema-hash) or any cache built against a
    /// different feature-extractor / state-layout source is rejected —
    /// regenerate via `precompute_features`.
    pub fn validate(&self) -> Result<()> {
        if self.magic != FXCACHE_MAGIC {
            bail!(
                "Invalid FxCache magic: expected {:?}, got {:?}",
                FXCACHE_MAGIC,
                self.magic
            );
        }
        if self.version != FXCACHE_VERSION {
            bail!(
                "Stale FxCache version: {} (expected {}). Delete and regenerate.",
                self.version, FXCACHE_VERSION
            );
        }
        if self.feat_dim as usize != FEAT_DIM {
            bail!(
                "Feature dimension mismatch: expected {}, got {}",
                FEAT_DIM,
                self.feat_dim
            );
        }
        let td = self.target_dim as usize;
        if td != TARGET_DIM {
            bail!(
                "Target dimension mismatch: expected {}, got {}",
                TARGET_DIM,
                td
            );
        }
        if self.ofi_dim as usize != OFI_DIM {
            bail!(
                "OFI dimension mismatch: expected {}, got {}",
                OFI_DIM,
                self.ofi_dim
            );
        }
        if self.feature_schema_hash != FEATURE_SCHEMA_HASH {
            bail!(
                "Stale FxCache feature schema: hash {:#018x} (expected {:#018x}). \
                 Source files defining feature extraction / state layout / \
                 fxcache format have changed since this cache was built. \
                 Delete and regenerate via precompute_features.",
                self.feature_schema_hash, FEATURE_SCHEMA_HASH
            );
        }
        if self.bar_count == 0 {
            bail!("FxCache bar_count is zero — empty cache files are not valid");
        }
        Ok(())
    }

    /// Serialize header to 72 bytes (little-endian).
    fn to_bytes(&self) -> [u8; HEADER_SIZE] {
        let mut buf = [0u8; HEADER_SIZE];
        buf[0..8].copy_from_slice(&self.magic);
        buf[8..10].copy_from_slice(&self.version.to_le_bytes());
        buf[10..12].copy_from_slice(&self.feat_dim.to_le_bytes());
        buf[12..14].copy_from_slice(&self.target_dim.to_le_bytes());
        buf[14..16].copy_from_slice(&self.ofi_dim.to_le_bytes());
        buf[16..24].copy_from_slice(&self.bar_count.to_le_bytes());
        buf[24..56].copy_from_slice(&self.cache_key);
        buf[56..64].copy_from_slice(&self.feature_schema_hash.to_le_bytes());
        buf[64..72].copy_from_slice(&self.reserved);
        buf
    }

    /// Deserialize header from 72 bytes (little-endian).
    fn from_bytes(buf: &[u8; HEADER_SIZE]) -> Self {
        let mut magic = [0u8; 8];
        magic.copy_from_slice(&buf[0..8]);

        let version = u16::from_le_bytes([buf[8], buf[9]]);
        let feat_dim = u16::from_le_bytes([buf[10], buf[11]]);
        let target_dim = u16::from_le_bytes([buf[12], buf[13]]);
        let ofi_dim = u16::from_le_bytes([buf[14], buf[15]]);
        let bar_count = u64::from_le_bytes([
            buf[16], buf[17], buf[18], buf[19], buf[20], buf[21], buf[22], buf[23],
        ]);

        let mut cache_key = [0u8; 32];
        cache_key.copy_from_slice(&buf[24..56]);

        let feature_schema_hash = u64::from_le_bytes([
            buf[56], buf[57], buf[58], buf[59], buf[60], buf[61], buf[62], buf[63],
        ]);

        let mut reserved = [0u8; 8];
        reserved.copy_from_slice(&buf[64..72]);

        Self {
            magic,
            version,
            feat_dim,
            target_dim,
            ofi_dim,
            bar_count,
            cache_key,
            feature_schema_hash,
            reserved,
        }
    }
}

// ── Data ─────────────────────────────────────────────────────────────────────

/// In-memory representation of an FxCache file's contents.
#[derive(Debug)]
pub struct FxCacheData {
    /// Per-bar timestamps (nanoseconds since Unix epoch).
    pub timestamps: Vec<i64>,
    /// Feature vectors, one per bar (42 elements each).
    pub features: Vec<[f64; FEAT_DIM]>,
    /// Target vectors, one per bar (6 elements each).
    pub targets: Vec<[f64; TARGET_DIM]>,
    /// OFI vectors, one per bar (`OFI_DIM` elements each).
    pub ofi: Vec<[f64; OFI_DIM]>,
    /// SHA256 cache key (raw 32 bytes).
    pub cache_key: [u8; 32],
    /// Number of bars.
    pub bar_count: usize,
    /// Explicit flag: true if OFI was computed from real MBP-10 data.
    /// False means OFI is zero-filled (no MBP-10 data was available during precompute).
    pub has_ofi: bool,
}

// ── Writer ───────────────────────────────────────────────────────────────────

/// Write feature/target/OFI data to an `.fxcache` binary file.
///
/// # Arguments
///
/// * `path`       — Output file path (parent directories are created automatically)
/// * `features`   — Slice of 42-element feature vectors
/// * `targets`    — Slice of 6-element target vectors
/// * `ofi`        — Slice of `OFI_DIM`-element OFI vectors
/// * `timestamps` — Per-bar timestamps (nanoseconds since Unix epoch)
/// * `cache_key`  — SHA256 key (raw 32 bytes)
/// * `has_ofi`    — If true, OFI was computed from real MBP-10 data; false means zero-filled
///
/// # Returns
///
/// Total bytes written (header + body).
pub fn write_fxcache(
    path: &Path,
    features: &[[f64; FEAT_DIM]],
    targets: &[[f64; TARGET_DIM]],
    ofi: &[[f64; OFI_DIM]],
    timestamps: &[i64],
    cache_key: [u8; 32],
    has_ofi: bool,
) -> Result<u64> {
    let bar_count = features.len();
    if targets.len() != bar_count || ofi.len() != bar_count || timestamps.len() != bar_count {
        bail!(
            "Length mismatch: features={}, targets={}, ofi={}, timestamps={}",
            bar_count,
            targets.len(),
            ofi.len(),
            timestamps.len()
        );
    }
    if bar_count == 0 {
        bail!("Cannot write empty FxCache (0 bars)");
    }

    // Create parent directories
    if let Some(parent) = path.parent() {
        std::fs::create_dir_all(parent)
            .with_context(|| format!("Failed to create parent dirs for {:?}", path))?;
    }

    let header = FxCacheHeader::new(FXCACHE_VERSION, bar_count as u64, cache_key, has_ofi);
    header.validate()?;

    let file = std::fs::File::create(path)
        .with_context(|| format!("Failed to create FxCache file {:?}", path))?;
    let mut writer = BufWriter::new(file);

    // Write header
    writer
        .write_all(&header.to_bytes())
        .context("Failed to write FxCache header")?;

    // Write body (f32 format)
    let body_bytes = write_body_f32(&mut writer, features, targets, ofi, timestamps)?;

    writer.flush().context("Failed to flush FxCache writer")?;

    let total_bytes = HEADER_SIZE as u64 + body_bytes;

    info!(
        "FxCache written: {} bars, v{} (f32, OFI_DIM={}), {:.2} MB -> {:?}",
        bar_count,
        FXCACHE_VERSION,
        OFI_DIM,
        total_bytes as f64 / 1_048_576.0,
        path
    );

    Ok(total_bytes)
}

/// Write body in f32 format: [i64 ts] + (FEAT_DIM+TARGET_DIM+OFI_DIM) × f32 per bar.
fn write_body_f32(
    writer: &mut BufWriter<std::fs::File>,
    features: &[[f64; FEAT_DIM]],
    targets: &[[f64; TARGET_DIM]],
    ofi: &[[f64; OFI_DIM]],
    timestamps: &[i64],
) -> Result<u64> {
    let bytes_per_bar = 8 + RECORD_F32_COUNT * 4; // i64 timestamp + f32 data
    let total = features.len() as u64 * bytes_per_bar as u64;

    for i in 0..features.len() {
        writer.write_all(&timestamps[i].to_le_bytes())?;
        for &v in &features[i] {
            writer.write_all(&(v as f32).to_le_bytes())?;
        }
        for &v in &targets[i] {
            writer.write_all(&(v as f32).to_le_bytes())?;
        }
        for &v in &ofi[i] {
            writer.write_all(&(v as f32).to_le_bytes())?;
        }
    }

    Ok(total)
}

// ── Reader ───────────────────────────────────────────────────────────────────

/// Load an `.fxcache` file into memory.
///
/// Reads the 72-byte header, validates it, then reads the f32 body,
/// converting each f32 back to f64 on load.
///
/// # Arguments
///
/// * `path` — Path to the `.fxcache` file
///
/// # Returns
///
/// Fully parsed `FxCacheData` with features, targets, OFI, cache key, and bar count.
pub fn load_fxcache(path: &Path) -> Result<FxCacheData> {
    let file = std::fs::File::open(path)
        .with_context(|| format!("Failed to open FxCache file {:?}", path))?;
    let file_len = file
        .metadata()
        .with_context(|| format!("Failed to stat FxCache file {:?}", path))?
        .len();
    let mut reader = BufReader::new(file);

    // Read header
    let mut header_buf = [0u8; HEADER_SIZE];
    reader
        .read_exact(&mut header_buf)
        .context("Failed to read FxCache header")?;
    let header = FxCacheHeader::from_bytes(&header_buf);
    header.validate()?;

    let bar_count = header.bar_count as usize;

    // Sanity-check file size — current version only; legacy versions rejected by validate()
    let on_disk_record_f32 = FEAT_DIM + TARGET_DIM + OFI_DIM;
    let expected_body = bar_count as u64 * (8 + on_disk_record_f32 as u64 * 4);
    let expected_total = HEADER_SIZE as u64 + expected_body;
    if file_len < expected_total {
        bail!(
            "FxCache file truncated: expected {} bytes, got {}",
            expected_total,
            file_len
        );
    }

    let (timestamps, features, targets, ofi) = read_body_f32(&mut reader, bar_count)?;

    info!(
        "FxCache loaded: {} bars, v{} from {:?}",
        bar_count, header.version, path
    );

    let has_ofi = header.reserved[0] == 1;

    Ok(FxCacheData {
        timestamps,
        features,
        targets,
        ofi,
        cache_key: header.cache_key,
        bar_count,
        has_ofi,
    })
}

/// Read body in f32 format (current `FXCACHE_VERSION`), converting f32 back to f64 on load.
fn read_body_f32(
    reader: &mut BufReader<std::fs::File>,
    bar_count: usize,
) -> Result<(Vec<i64>, Vec<[f64; FEAT_DIM]>, Vec<[f64; TARGET_DIM]>, Vec<[f64; OFI_DIM]>)> {
    let mut timestamps = Vec::with_capacity(bar_count);
    let mut features = Vec::with_capacity(bar_count);
    let mut targets = Vec::with_capacity(bar_count);
    let mut ofi = Vec::with_capacity(bar_count);
    let mut i64_buf = [0u8; 8];
    let mut f32_buf = [0u8; 4];

    for _ in 0..bar_count {
        reader.read_exact(&mut i64_buf)?;
        timestamps.push(i64::from_le_bytes(i64_buf));

        let mut feat = [0.0_f64; FEAT_DIM];
        for slot in &mut feat {
            reader.read_exact(&mut f32_buf)?;
            *slot = f32::from_le_bytes(f32_buf) as f64;
        }
        features.push(feat);

        let mut tgt = [0.0_f64; TARGET_DIM];
        for slot in &mut tgt {
            reader.read_exact(&mut f32_buf)?;
            *slot = f32::from_le_bytes(f32_buf) as f64;
        }
        targets.push(tgt);

        let mut ofi_row = [0.0_f64; OFI_DIM];
        for slot in &mut ofi_row {
            reader.read_exact(&mut f32_buf)?;
            *slot = f32::from_le_bytes(f32_buf) as f64;
        }
        ofi.push(ofi_row);
    }

    Ok((timestamps, features, targets, ofi))
}

// ── Finder ───────────────────────────────────────────────────────────────────

#[cfg(test)]
mod target_layout_tests {
    use super::*;

    /// Compile-time guard: the named target columns are dense and exhaust
    /// the layout. If a future patch reorders or removes a slot, this fires.
    #[test]
    fn target_columns_dense_and_exhaustive() {
        let cols = [
            TARGET_PREPROC_CLOSE,
            TARGET_PREPROC_NEXT,
            TARGET_RAW_CLOSE,
            TARGET_RAW_NEXT,
            TARGET_RAW_OPEN,
            TARGET_MID_OPEN,
        ];
        for (i, &c) in cols.iter().enumerate() {
            assert_eq!(c, i, "Column constant out of order at index {i}");
        }
        assert_eq!(cols.len(), TARGET_DIM);
    }
}

#[cfg(test)]
mod has_ofi_tests {
    use super::*;

    #[test]
    fn test_has_ofi_roundtrip_true() {
        let dir = tempfile::tempdir().unwrap();
        let path = dir.path().join("test_ofi_true.fxcache");
        let features = vec![[1.0_f64; 42]; 10];
        let targets = vec![[0.0_f64; 6]; 10];
        let ofi = vec![[0.5_f64; OFI_DIM]; 10];
        let timestamps = vec![1_i64; 10];
        let key = [0u8; 32];

        write_fxcache(&path, &features, &targets, &ofi, &timestamps, key, true).unwrap();
        let loaded = load_fxcache(&path).unwrap();
        assert!(loaded.has_ofi, "has_ofi should be true");
        assert_eq!(loaded.bar_count, 10);
    }

    #[test]
    fn test_has_ofi_roundtrip_false() {
        let dir = tempfile::tempdir().unwrap();
        let path = dir.path().join("test_ofi_false.fxcache");
        let features = vec![[1.0_f64; 42]; 10];
        let targets = vec![[0.0_f64; 6]; 10];
        let ofi = vec![[0.0_f64; OFI_DIM]; 10];
        let timestamps = vec![1_i64; 10];
        let key = [0u8; 32];

        write_fxcache(&path, &features, &targets, &ofi, &timestamps, key, false).unwrap();
        let loaded = load_fxcache(&path).unwrap();
        assert!(!loaded.has_ofi, "has_ofi should be false");
    }
}

/// Find an `.fxcache` file by hex-encoded cache key in a cache directory.
///
/// Looks for a file named `<hex_key>.fxcache` in `cache_dir`.
///
/// # Arguments
///
/// * `cache_dir` — Directory to search
/// * `cache_key` — Raw 32-byte SHA256 cache key
///
/// # Returns
///
/// `Some(path)` if the file exists, `None` otherwise.
pub fn find_fxcache(cache_dir: &Path, cache_key: &[u8; 32]) -> Option<PathBuf> {
    let hex_key = hex::encode(cache_key);
    let candidate = cache_dir.join(format!("{hex_key}.fxcache"));
    if candidate.exists() {
        debug!("FxCache found: {:?}", candidate);
        Some(candidate)
    } else {
        debug!("FxCache miss: {:?}", candidate);
        None
    }
}

/// Resolve fxcache directory: explicit override > env var > walk-up sibling.
pub fn resolve_cache_dir(data_dir: &Path, override_dir: Option<&Path>) -> Option<PathBuf> {
    // 1. Explicit override (CLI --feature-cache-dir)
    if let Some(dir) = override_dir {
        if dir.exists() {
            return Some(dir.to_path_buf());
        }
    }

    // 2. Environment variable
    if let Ok(dir) = std::env::var("FOXHUNT_FEATURE_CACHE_DIR") {
        let p = PathBuf::from(dir);
        if p.exists() {
            return Some(p);
        }
    }

    // 3. Walk up from data_dir to find sibling feature-cache/
    let mut dir = data_dir;
    loop {
        if let Some(parent) = dir.parent() {
            let candidate = parent.join("feature-cache");
            if candidate.exists() {
                return Some(candidate);
            }
            if parent == dir {
                break;
            }
            dir = parent;
        } else {
            break;
        }
    }

    None
}

/// Returns the path to the `{hex_key}.norm_stats.json` file that sits
/// alongside the `.fxcache` for the given cache key.
///
/// `precompute_features` writes this file at the same time as the fxcache
/// (see `precompute_features.rs:607`). Supervised training writes a per-fold
/// copy into the output dir for evaluation; RL training should use this
/// helper to produce the same per-fold copies (`norm_stats_fold{N}.json`)
/// so `evaluate_baseline` can find them. Returns `None` if the cache
/// directory can't be resolved.
pub fn norm_stats_path_for_key(
    data_dir: &Path,
    cache_dir_override: Option<&Path>,
    cache_key: &[u8; 32],
) -> Option<PathBuf> {
    let cache_dir = resolve_cache_dir(data_dir, cache_dir_override)?;
    let hex_key = hex::encode(cache_key);
    Some(cache_dir.join(format!("{hex_key}.norm_stats.json")))
}

/// Discover and load an fxcache file. Single source of truth for all callers.
///
/// Cache dir priority: `cache_dir_override` > `FOXHUNT_FEATURE_CACHE_DIR` env > walk-up sibling.
/// Strict key match only — returns `None` on miss. No "most recent" fallback.
///
/// # Arguments
/// * `data_dir` — Base data directory (e.g. `test_data/futures-baseline`)
/// * `symbol` — Trading symbol (e.g. `"ES.FUT"`)
/// * `mbp10_dir` — Optional MBP-10 order book data directory
/// * `trades_dir` — Optional trades data directory
/// * `data_source` — Data source mode (`"mbp10"` or `"mbp10"`)
/// * `cache_dir_override` — Explicit cache directory (from CLI `--feature-cache-dir`)
pub fn discover_and_load(
    data_dir: &Path,
    symbol: &str,
    mbp10_dir: Option<&Path>,
    trades_dir: Option<&Path>,
    data_source: &str,
    cache_dir_override: Option<&Path>,
) -> Option<FxCacheData> {
    // 1. Resolve cache directory
    let cache_dir = resolve_cache_dir(data_dir, cache_dir_override)?;

    // 2. Compute cache key (includes symbol + data_source)
    let key_hex = crate::feature_cache::calculate_dbn_cache_key_full(
        data_dir, mbp10_dir, trades_dir, symbol, data_source,
    )
    .ok()?;
    let key: [u8; 32] = hex::decode(&key_hex).ok()?.try_into().ok()?;

    // 3. Strict key match — no fallback to "most recent"
    let path = find_fxcache(&cache_dir, &key)?;

    // 4. Load and return
    match load_fxcache(&path) {
        Ok(data) => {
            info!(
                "fxcache hit: {} bars, OFI={} from {:?}",
                data.bar_count, data.has_ofi, path
            );
            Some(data)
        }
        Err(e) => {
            tracing::warn!("fxcache load failed: {e}");
            None
        }
    }
}

// ── Tests ────────────────────────────────────────────────────────────────────

#[cfg(test)]
mod discover_tests {
    use super::*;

    #[test]
    fn test_discover_returns_none_for_nonexistent_dir() {
        let result = discover_and_load(
            Path::new("/nonexistent/path"),
            "ES.FUT",
            None,
            None,
            "mbp10",
            None,
        );
        assert!(result.is_none());
    }

    #[test]
    fn test_discover_returns_none_on_key_mismatch() {
        let dir = tempfile::tempdir().unwrap();
        let cache_dir = dir.path().join("feature-cache");
        std::fs::create_dir_all(&cache_dir).unwrap();

        // Write a cache file with a known dummy key
        let path = cache_dir.join(
            "0000000000000000000000000000000000000000000000000000000000000000.fxcache",
        );
        let features = vec![[1.0_f64; 42]; 5];
        let targets = vec![[0.0_f64; 6]; 5];
        let ofi = vec![[0.0_f64; OFI_DIM]; 5];
        let timestamps = vec![1_i64; 5];
        write_fxcache(&path, &features, &targets, &ofi, &timestamps, [0u8; 32], false)
            .unwrap();

        // Try to discover with a real data_dir (different key) — should NOT match
        let result = discover_and_load(
            Path::new("test_data/futures-baseline"),
            "ES.FUT",
            None,
            None,
            "mbp10",
            Some(&cache_dir),
        );
        // Strict match only — no "most recent" fallback
        assert!(result.is_none(), "Wrong key should not match (strict mode)");
    }

    #[test]
    fn test_resolve_cache_dir_explicit_override() {
        let dir = tempfile::tempdir().unwrap();
        let result = resolve_cache_dir(Path::new("/some/data"), Some(dir.path()));
        assert_eq!(result, Some(dir.path().to_path_buf()));
    }

    #[test]
    fn test_resolve_cache_dir_nonexistent_override_falls_through() {
        let result = resolve_cache_dir(
            Path::new("/some/data"),
            Some(Path::new("/nonexistent/override")),
        );
        // Falls through to env var / walk-up (both will fail here)
        assert!(result.is_none());
    }
}