foxhunt/crates/ml/src/env/loaders.rs

//! Phase E loader helpers shared by `alpha_dqn_h600_smoke` and
//! `alpha_compose_backtest` (and any future Phase E binary).
//!
//! Three loaders:
//!
//!   - [`load_alpha_cache`] — binary cache produced by
//!     `alpha_train_stacker --alpha-cache-out`. Format: `[u32 n] [f32; n]`
//!     little-endian. Each f32 is the per-bar stacker logit; bars
//!     before `seq_len - 1` are 0.0 (Pearl A sentinel).
//!
//!   - [`load_fill_model_from_json`] — parses the FillCoeffs JSON
//!     emitted by `alpha_fit_fill_model`. Expects `bid_coeffs` and
//!     `ask_coeffs` as 3-row arrays of 5-element f32 vectors.
//!
//!   - [`load_snapshots_from_fxcache`] — builds the env's
//!     `Vec<SnapshotRow>` from a precomputed snapshot fxcache. Mid
//!     comes from `raw_close`; bid/ask synthesized at fixed half-tick
//!     offsets (the parser bug for L2-L9 is fixed but the fxcache
//!     itself doesn't store the LOB so we synthesize). 81-dim Block-S
//!     features feed `spread_bps`, `l1_imbalance`, `ofi_sum_5`,
//!     `time_since_trade_s`, `book_event_rate`, `mid_drift_5`.
//!     `alpha_logit` is filled from the optional alpha cache (else 0.0).

use std::io::Read;
use std::path::Path;

use anyhow::{Context, Result};
use ml_alpha::fxcache_reader::{COL_RAW_CLOSE, FEAT_DIM, FxCacheReader};
use serde_json::Value;
use tracing::{info, warn};

use crate::env::execution_env::SnapshotRow;
use crate::env::fill_model::{FillCoeffs, FillModel};

/// ES futures minimum price increment.
const TICK: f32 = 0.25;

/// Load the alpha-logit cache (`[u32 n] [f32; n]` little-endian binary).
/// Each index aligns to the fxcache bar at the same index.
pub fn load_alpha_cache(path: &Path) -> Result<Vec<f32>> {
    let mut f = std::fs::File::open(path)
        .with_context(|| format!("open alpha cache {}", path.display()))?;
    let mut len_bytes = [0u8; 4];
    f.read_exact(&mut len_bytes).context("read alpha-cache header")?;
    let n = u32::from_le_bytes(len_bytes) as usize;
    let mut buf = vec![0u8; n * 4];
    f.read_exact(&mut buf).context("read alpha-cache body")?;
    let mut out = Vec::with_capacity(n);
    for i in 0..n {
        let off = i * 4;
        let v = f32::from_le_bytes([buf[off], buf[off + 1], buf[off + 2], buf[off + 3]]);
        out.push(v);
    }
    Ok(out)
}

/// Parse FillModel from JSON (the artifact produced by
/// `alpha_fit_fill_model`). Expects `bid_coeffs` and `ask_coeffs` keys
/// each holding a 3-row × 5-element f32 array.
pub fn load_fill_model_from_json(path: &Path) -> Result<FillModel> {
    let s = std::fs::read_to_string(path)
        .with_context(|| format!("read fill coeffs JSON at {}", path.display()))?;
    let v: Value = serde_json::from_str(&s)
        .with_context(|| format!("parse fill coeffs JSON at {}", path.display()))?;
    let parse_levels = |key: &str| -> Result<[FillCoeffs; 3]> {
        let arr = v[key]
            .as_array()
            .ok_or_else(|| anyhow::anyhow!("missing/non-array `{}`", key))?;
        if arr.len() != 3 {
            anyhow::bail!("{} must have 3 entries, got {}", key, arr.len());
        }
        let mut out: [FillCoeffs; 3] = [FillCoeffs { beta: [0.0; 5] }; 3];
        for (i, lvl) in arr.iter().enumerate() {
            let vv = lvl
                .as_array()
                .ok_or_else(|| anyhow::anyhow!("{}[{}] not array", key, i))?;
            if vv.len() != 5 {
                anyhow::bail!("{}[{}] must have 5 floats, got {}", key, i, vv.len());
            }
            for k in 0..5 {
                out[i].beta[k] = vv[k]
                    .as_f64()
                    .ok_or_else(|| anyhow::anyhow!("{}[{}][{}] not number", key, i, k))?
                    as f32;
            }
        }
        Ok(out)
    };
    Ok(FillModel {
        bid_coeffs: parse_levels("bid_coeffs")?,
        ask_coeffs: parse_levels("ask_coeffs")?,
    })
}

/// Build `Vec<SnapshotRow>` from a precomputed fxcache. Mid from
/// `raw_close`; bid/ask synthesized at fixed half-tick offsets; 81-dim
/// Block-S features feed the runtime feature fields. If `alpha_cache`
/// is `Some`, each `SnapshotRow.alpha_logit` is populated from the
/// cache (and `alpha_confidence = |sigmoid(z) − 0.5|`); otherwise 0.0.
/// Maximum spread we'll accept from `spread_bps` (in price units) when
/// `use_real_spread` is true. Above this we cap to the max — protects
/// against fxcache feature outliers / NaN / unrealistic wide ticks.
/// 10 ticks = 2.50 in ES futures price = ~5.6 bps at mid=4500.
const MAX_REAL_SPREAD_PRICE: f32 = 10.0 * TICK;

/// Build `Vec<SnapshotRow>` from a precomputed fxcache. Mid from
/// `raw_close`; 81-dim Block-S features feed the runtime feature fields.
///
/// **Bid/ask synthesis:**
///   - `use_real_spread = false` (legacy): bid_l1 = mid − 0.125, ask_l1
///     = mid + 0.125. Fixed half-tick spread, equivalent to the original
///     loader behaviour and matched the Phase E.1/2/3 smoke/backtest runs.
///   - `use_real_spread = true` (Path 3): spread_price derived from
///     fxcache `features[78]` (`spread_bps`); bid_l1 = mid − spread/2,
///     ask_l1 = mid + spread/2. Variable per-bar spread reflecting
///     actual market state. Floored at 1 tick (=0.25), capped at 10 ticks
///     to protect against feature outliers.
///
/// L2/L3 are always synthesized at ±TICK offsets from L1 — the fxcache
/// doesn't store depth beyond L1 spread+imbalance, and L2/L3 fills are
/// rare (most policy decisions hinge on L1 + market crosses).
///
/// If `alpha_cache` is `Some`, each row's `alpha_logit` is populated
/// from the cache (and `alpha_confidence = |sigmoid(z) − 0.5|`).
pub fn load_snapshots_from_fxcache(
    fxcache_path: &Path,
    max_snapshots: usize,
    alpha_cache: Option<&[f32]>,
    use_real_spread: bool,
) -> Result<Vec<SnapshotRow>> {
    let reader = FxCacheReader::open(fxcache_path)
        .with_context(|| format!("open fxcache {}", fxcache_path.display()))?;
    let alpha_dim = reader
        .alpha_feature_dim()
        .ok_or_else(|| anyhow::anyhow!("fxcache lacks alpha column"))?;
    if alpha_dim < 81 {
        anyhow::bail!(
            "fxcache alpha_dim={} but expected ≥81 (snapshot_pipeline layout)",
            alpha_dim
        );
    }
    let n_bars_total = reader.bar_count();
    let n = n_bars_total.min(max_snapshots);
    info!("fxcache: {} total bars, taking {} for the env", n_bars_total, n);
    info!(
        "fxcache loader: spread mode = {}",
        if use_real_spread { "REAL (derived from features[78] spread_bps)" }
        else                  { "FIXED ±0.125-tick" }
    );

    if let Some(cache) = alpha_cache {
        if cache.len() < n {
            anyhow::bail!(
                "alpha cache has {} entries but env wants {} bars",
                cache.len(),
                n
            );
        }
    }

    let mut rows: Vec<SnapshotRow> = Vec::with_capacity(n);
    let mut n_degenerate = 0_usize;
    // Diagnostic: spread distribution (real mode only).
    let mut sum_spread = 0.0_f64;
    let mut min_spread = f32::INFINITY;
    let mut max_spread = f32::NEG_INFINITY;
    let mut n_floor_hits = 0_usize;
    let mut n_cap_hits = 0_usize;
    for i in 0..n {
        let rec = reader.record(i);
        let mid = rec.targets[COL_RAW_CLOSE - FEAT_DIM];
        if !mid.is_finite() || mid <= 0.0 {
            n_degenerate += 1;
            continue;
        }
        let features = reader
            .alpha_features(i)
            .ok_or_else(|| anyhow::anyhow!("missing alpha row at bar {}", i))?;

        let spread_bps = features[78];
        let l1_imbalance = features[79];
        let ofi_sum_5 = features[0..5].iter().sum::<f32>();
        let mid_drift_5 = features[80];
        let time_since_trade_s = features[75];
        let book_event_rate = features[77];

        let (bid_l1, ask_l1) = if use_real_spread {
            // spread_bps = 10000 × (ask − bid) / mid → spread_price = bps × mid / 10000
            let raw = if spread_bps.is_finite() && spread_bps > 0.0 {
                spread_bps / 10_000.0 * mid
            } else {
                TICK // sentinel: fall back to 1-tick spread if bps is non-finite
            };
            let mut clamped = raw;
            if clamped < TICK {
                clamped = TICK;
                n_floor_hits += 1;
            }
            if clamped > MAX_REAL_SPREAD_PRICE {
                clamped = MAX_REAL_SPREAD_PRICE;
                n_cap_hits += 1;
            }
            sum_spread += clamped as f64;
            if clamped < min_spread { min_spread = clamped; }
            if clamped > max_spread { max_spread = clamped; }
            let half = clamped * 0.5;
            (mid - half, mid + half)
        } else {
            (mid - 0.125, mid + 0.125)
        };
        // Phase E.4.A.3: extend to L1-L10 depth. L2-L10 synthesized at
        // ±TICK offsets from L1; real L4-L10 from MBP-10 lands in
        // Task 5 follow-on (the loader doesn't have MBP-10 access yet).
        let mut bid_l = [0.0_f32; 10];
        let mut ask_l = [0.0_f32; 10];
        for k in 0..10 {
            bid_l[k] = bid_l1 - (k as f32) * TICK;
            ask_l[k] = ask_l1 + (k as f32) * TICK;
        }

        let alpha_logit = alpha_cache.map(|c| c[i]).unwrap_or(0.0);
        let alpha_confidence = {
            let p = 1.0_f32 / (1.0 + (-alpha_logit.clamp(-50.0, 50.0)).exp());
            (p - 0.5).abs()
        };

        rows.push(SnapshotRow {
            mid_price: mid,
            bid_l,
            ask_l,
            alpha_logit,
            alpha_confidence,
            spread_bps,
            l1_imbalance,
            ofi_sum_5,
            mid_drift_5,
            time_since_trade_s,
            book_event_rate,
        });
    }
    if n_degenerate > 0 {
        warn!(
            "fxcache loader: skipped {} degenerate bars (non-finite or zero mid)",
            n_degenerate
        );
    }
    if use_real_spread && !rows.is_empty() {
        let mean_spread = (sum_spread / rows.len() as f64) as f32;
        info!(
            "fxcache loader: real-spread stats — mean={:.4} ({:.2} ticks), min={:.4}, max={:.4}, floor_hits={}/{}, cap_hits={}/{}",
            mean_spread, mean_spread / TICK,
            min_spread, max_spread,
            n_floor_hits, rows.len(),
            n_cap_hits, rows.len(),
        );
    }
    Ok(rows)
}