//! `.fxcache` file reader (Arrow IPC, FXCACHE_VERSION=10).
//!
//! Decoupled from `crates/ml/src/fxcache.rs` to keep `ml-alpha`'s dependency
//! footprint tight, but uses the **same Apache Arrow IPC wire format** as the
//! production writer in `ml::fxcache::write_fxcache`. Schema/version/dim
//! validation happens against the Arrow schema metadata embedded in the file —
//! no compile-time constant matching needed, no off-by-N alignment risk.
//!
//! ## File layout
//!
//! Arrow IPC `.arrow` file with two columns and metadata:
//!
//! ```text
//! Schema:
//!   ts_ns:  Int64                         (nullable=false)
//!   record: FixedSizeBinary(N)            (nullable=false)
//!     where N = 4 × (feat_dim + target_dim + ofi_dim)
//!
//! Schema metadata (all String values):
//!   fxcache_version:      "10"
//!   feat_dim:             "42"
//!   target_dim:           "6"
//!   ofi_dim:              "32"
//!   has_ofi:              "true" | "false"
//!   cache_key_hex:        64-char hex of the 32-byte SHA256 key
//!   feature_schema_hash:  16-char hex of the FNV-1a feature-schema hash
//!
//! Row blob layout (the FixedSizeBinary bytes for each bar):
//!   [feat_dim × f32 LE][target_dim × f32 LE][ofi_dim × f32 LE]
//! ```
//!
//! ## Why we materialize at `open()` rather than mmap
//!
//! V9 (custom binary) used `memmap2` for zero-copy slice views. Alpha (Arrow IPC)
//! reads all batches into memory at `open()`. At our scale (175k bars × 80
//! f32 ≈ 56 MB), the time difference vs. mmap is <100 ms, well below the GPU
//! upload cost. The trade-off buys us: schema-in-file, multi-language tooling
//! (Python/polars can read directly), and elimination of the off-by-N
//! alignment bugs that plagued the custom-binary format.

use std::fs::File;
use std::path::Path;

use anyhow::{anyhow, bail, Context, Result};
use arrow::array::{Array, FixedSizeBinaryArray, Int64Array};
use arrow::ipc::reader::FileReader;

/// Alpha Arrow IPC format version. Mirrors `ml::fxcache::FXCACHE_VERSION`.
pub const FXCACHE_VERSION: u16 = 10;

/// Alpha feature block dimensionality. Mirrors `ml::fxcache::ALPHA_FEATURE_DIM`.
/// Files with the `alpha_feature_dim` schema-metadata key contain a third Arrow
/// column (`alpha_features: FixedSizeBinary(ALPHA_FEATURE_DIM × 4)`) with the
/// FoxhuntQ-Δ Phase 1c modern microstructure features.
pub const ALPHA_FEATURE_DIM: usize = 134;

/// Feature dimension (per-bar). Mirrors `ml::fxcache::FEAT_DIM`.
pub const FEAT_DIM: usize = 42;

/// Target dimension (per-bar). Includes `preproc_close[0]`, `preproc_next[1]`,
/// `raw_close[2]`, `raw_next[3]`, `raw_open[4]`, `mid_open[5]`.
pub const TARGET_DIM: usize = 6;

/// OFI dimension (per-bar) — order-flow imbalance microstructure features.
pub const OFI_DIM: usize = 32;

/// Total f32 values per record.
pub const RECORD_F32_COUNT: usize = FEAT_DIM + TARGET_DIM + OFI_DIM;

/// Column index of `preproc_close` within the target slice. Used for label
/// generation (binary direction prediction at horizon H).
pub const COL_PREPROC_CLOSE: usize = FEAT_DIM;

/// Column index of `raw_close` within the target slice. Used for honest P&L
/// signal calculation (preproc is log-return normalized which loses sign info
/// in some configurations).
pub const COL_RAW_CLOSE: usize = FEAT_DIM + 2;

/// Parsed fxcache header metadata, derived from the Arrow schema's
/// `metadata: HashMap<String, String>`.
#[derive(Debug, Clone)]
pub struct FxCacheMetadata {
    pub version: u16,
    pub bar_count: usize,
    pub feat_dim: usize,
    pub target_dim: usize,
    pub ofi_dim: usize,
    pub cache_key_hex: String,
    pub feature_schema_hash: u64,
    pub has_ofi: bool,
}

/// One fxcache record (zero-copy slice views into the reader's cached f32
/// buffer). Cheap to construct.
#[derive(Debug, Clone, Copy)]
pub struct FxCacheRecord<'a> {
    /// Feature vector (42-dim base OHLCV + technical features).
    pub features: &'a [f32],
    /// Target slice (6 columns: preproc_close, preproc_next, raw_close,
    /// raw_next, raw_open, mid_open).
    pub targets: &'a [f32],
    /// Order-flow imbalance microstructure features (32-dim).
    pub ofi: &'a [f32],
}

/// fxcache reader — owns the materialized f32 + timestamp arrays.
///
/// `record(i)` returns slice views into the owned buffer; the reader must
/// outlive any record borrow. Same observed API as the previous mmap-based
/// reader (callers don't need to change).
///
/// When the file contains the optional alpha_features column, the reader
/// additionally exposes `alpha_features(i)` for the Phase 1c modern feature
/// set.
pub struct FxCacheReader {
    metadata: FxCacheMetadata,
    /// Flat f32 storage: `bar_count × RECORD_F32_COUNT`, row-major.
    f32_data: Vec<f32>,
    timestamps: Vec<i64>,
    /// Optional flat alpha storage: `bar_count × alpha_dim`, row-major.
    /// `None` for files without the alpha column.
    alpha_data: Option<Vec<f32>>,
    /// Detected alpha-feature width per row (from `alpha_feature_dim` metadata).
    /// `None` iff `alpha_data` is `None`. The fxcache schema carries the
    /// variable-width 134-dim bar-level stack or the 81-dim per-snapshot
    /// stack via the same column with this metadata-declared dim.
    alpha_dim: Option<usize>,
}

impl FxCacheReader {
    /// Open and validate an fxcache file (Arrow IPC format).
    pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
        let path_ref = path.as_ref();
        let file = File::open(path_ref)
            .with_context(|| format!("opening fxcache: {}", path_ref.display()))?;
        let mut reader = FileReader::try_new(file, None)
            .with_context(|| format!("Arrow IPC reader init for {}", path_ref.display()))?;
        let schema = reader.schema();
        let meta_map = schema.metadata();

        // ── Validate schema metadata against current ml-alpha constants ──
        let version: u16 = meta_map
            .get("fxcache_version")
            .ok_or_else(|| anyhow!("fxcache: missing schema metadata 'fxcache_version'"))?
            .parse()
            .context("parse fxcache_version")?;
        if version != FXCACHE_VERSION {
            bail!(
                "fxcache version mismatch: file={}, ml-alpha expects {}",
                version, FXCACHE_VERSION
            );
        }
        let feat_dim: usize = meta_map
            .get("feat_dim")
            .ok_or_else(|| anyhow!("missing 'feat_dim'"))?
            .parse()
            .context("parse feat_dim")?;
        let target_dim: usize = meta_map
            .get("target_dim")
            .ok_or_else(|| anyhow!("missing 'target_dim'"))?
            .parse()
            .context("parse target_dim")?;
        let ofi_dim: usize = meta_map
            .get("ofi_dim")
            .ok_or_else(|| anyhow!("missing 'ofi_dim'"))?
            .parse()
            .context("parse ofi_dim")?;
        let has_ofi: bool = meta_map
            .get("has_ofi")
            .ok_or_else(|| anyhow!("missing 'has_ofi'"))?
            .parse()
            .context("parse has_ofi")?;
        let feature_schema_hash = u64::from_str_radix(
            meta_map
                .get("feature_schema_hash")
                .ok_or_else(|| anyhow!("missing 'feature_schema_hash'"))?,
            16,
        )
        .context("parse feature_schema_hash hex")?;
        let cache_key_hex = meta_map
            .get("cache_key_hex")
            .ok_or_else(|| anyhow!("missing 'cache_key_hex'"))?
            .clone();

        if feat_dim != FEAT_DIM || target_dim != TARGET_DIM || ofi_dim != OFI_DIM {
            bail!(
                "fxcache dim mismatch: schema(feat={feat_dim}, target={target_dim}, ofi={ofi_dim}) \
                 vs ml-alpha consts ({FEAT_DIM}, {TARGET_DIM}, {OFI_DIM})"
            );
        }

        // Detect optional alpha_features column (FoxhuntQ-Δ Phase 1c). The
        // dim is variable: 134 for the bar-level alpha stack, 81 for the
        // snapshot stack, or anything else the writer chose. The reader
        // honors whatever the file declares; downstream consumers
        // (training.rs) read `reader.alpha_feature_dim()` to size their
        // model accordingly.
        let alpha_dim: Option<usize> = if meta_map.contains_key("alpha_feature_dim") {
            let declared: usize = meta_map
                .get("alpha_feature_dim")
                .ok_or_else(|| anyhow!("alpha_feature_dim metadata key absent during guard race"))?
                .parse()
                .context("parse alpha_feature_dim")?;
            if declared == 0 {
                bail!("fxcache declares alpha_feature_dim = 0 (degenerate)");
            }
            Some(declared)
        } else {
            None
        };
        let has_alpha = alpha_dim.is_some();

        // ── Materialize all batches into flat f32 buffer ──
        let expected_blob_size = RECORD_F32_COUNT * 4;
        let expected_alpha_blob_size = alpha_dim.map(|d| d * 4).unwrap_or(0);
        let mut timestamps: Vec<i64> = Vec::new();
        let mut f32_data: Vec<f32> = Vec::new();
        let mut alpha_data: Option<Vec<f32>> = if has_alpha { Some(Vec::new()) } else { None };

        for batch_result in reader.by_ref() {
            let batch = batch_result.context("read Arrow batch")?;
            let ts_arr = batch
                .column(0)
                .as_any()
                .downcast_ref::<Int64Array>()
                .ok_or_else(|| {
                    anyhow!(
                        "fxcache column 0 should be Int64, got {:?}",
                        batch.column(0).data_type()
                    )
                })?;
            let blob_arr = batch
                .column(1)
                .as_any()
                .downcast_ref::<FixedSizeBinaryArray>()
                .ok_or_else(|| {
                    anyhow!(
                        "fxcache column 1 should be FixedSizeBinary, got {:?}",
                        batch.column(1).data_type()
                    )
                })?;
            let alpha_arr_opt = if has_alpha {
                if batch.num_columns() < 3 {
                    bail!(
                        "fxcache claims alpha_feature_dim but has {} columns (expected 3)",
                        batch.num_columns()
                    );
                }
                Some(
                    batch
                        .column(2)
                        .as_any()
                        .downcast_ref::<FixedSizeBinaryArray>()
                        .ok_or_else(|| {
                            anyhow!(
                                "fxcache alpha column should be FixedSizeBinary, got {:?}",
                                batch.column(2).data_type()
                            )
                        })?,
                )
            } else {
                None
            };

            for i in 0..batch.num_rows() {
                timestamps.push(ts_arr.value(i));
                let blob: &[u8] = blob_arr.value(i);
                if blob.len() != expected_blob_size {
                    bail!(
                        "fxcache row blob size {} != expected {} (RECORD_F32_COUNT * 4)",
                        blob.len(),
                        expected_blob_size
                    );
                }
                // Decode RECORD_F32_COUNT little-endian f32 values into the flat buffer.
                for j in 0..RECORD_F32_COUNT {
                    let off = j * 4;
                    let value = f32::from_le_bytes([
                        blob[off],
                        blob[off + 1],
                        blob[off + 2],
                        blob[off + 3],
                    ]);
                    f32_data.push(value);
                }
                if let (Some(alpha_arr), Some(dst), Some(dim)) =
                    (alpha_arr_opt, alpha_data.as_mut(), alpha_dim)
                {
                    let alpha_blob: &[u8] = alpha_arr.value(i);
                    if alpha_blob.len() != expected_alpha_blob_size {
                        bail!(
                            "fxcache alpha row blob size {} != expected {} (alpha_feature_dim × 4)",
                            alpha_blob.len(),
                            expected_alpha_blob_size
                        );
                    }
                    for j in 0..dim {
                        let off = j * 4;
                        dst.push(f32::from_le_bytes([
                            alpha_blob[off],
                            alpha_blob[off + 1],
                            alpha_blob[off + 2],
                            alpha_blob[off + 3],
                        ]));
                    }
                }
            }
        }

        let bar_count = timestamps.len();
        if bar_count == 0 {
            bail!("fxcache is empty: 0 bars");
        }

        let metadata = FxCacheMetadata {
            version,
            bar_count,
            feat_dim,
            target_dim,
            ofi_dim,
            cache_key_hex,
            feature_schema_hash,
            has_ofi,
        };

        Ok(Self {
            metadata,
            f32_data,
            timestamps,
            alpha_data,
            alpha_dim,
        })
    }

    /// Whether the file contains the alpha_features column.
    pub fn has_alpha_features(&self) -> bool {
        self.alpha_data.is_some()
    }

    /// Width of the alpha-features row stored in the file. `None` if the
    /// fxcache was written without the alpha column. Variable across files:
    /// 134 for the bar-level stack, 81 for the per-snapshot stack.
    pub fn alpha_feature_dim(&self) -> Option<usize> {
        self.alpha_dim
    }

    /// Alpha feature row for bar `i`, if the file contains the alpha column.
    /// Returns `None` if the file is alpha-without-alphafeatures (legacy write).
    /// Panics if `i >= bar_count`.
    pub fn alpha_features(&self, i: usize) -> Option<&[f32]> {
        assert!(
            i < self.metadata.bar_count,
            "alpha_features({i}) >= bar_count({})",
            self.metadata.bar_count
        );
        let dim = self.alpha_dim?;
        self.alpha_data.as_ref().map(|buf| {
            let start = i * dim;
            &buf[start..start + dim]
        })
    }

    /// Borrow header metadata.
    pub fn metadata(&self) -> &FxCacheMetadata {
        &self.metadata
    }

    /// Number of bars (records) in the file.
    pub fn bar_count(&self) -> usize {
        self.metadata.bar_count
    }

    /// Return the timestamp (nanoseconds since Unix epoch, signed) for record
    /// `i`. Panics if `i >= bar_count`.
    pub fn record_timestamp(&self, i: usize) -> i64 {
        assert!(
            i < self.metadata.bar_count,
            "record_timestamp({i}) >= bar_count({})",
            self.metadata.bar_count
        );
        self.timestamps[i]
    }

    /// Return zero-copy slice views of record `i`. Panics if `i >= bar_count`.
    ///
    /// # Performance
    ///
    /// O(1) — slice indexing into the owned `f32_data` buffer. The buffer
    /// was materialized once at `open()`.
    pub fn record(&self, i: usize) -> FxCacheRecord<'_> {
        assert!(
            i < self.metadata.bar_count,
            "record({i}) >= bar_count({})",
            self.metadata.bar_count
        );
        let start = i * RECORD_F32_COUNT;
        let row = &self.f32_data[start..start + RECORD_F32_COUNT];
        FxCacheRecord {
            features: &row[..FEAT_DIM],
            targets: &row[FEAT_DIM..FEAT_DIM + TARGET_DIM],
            ofi: &row[FEAT_DIM + TARGET_DIM..],
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    /// Smoke test: open the local Phase 1a test fxcache and verify metadata
    /// + that the timestamp + feature values look plausible. Ignored by
    /// default because it depends on the local test_data symlink AND the
    /// file must be in alpha Arrow IPC format (regenerate via
    /// `crates/ml/examples/precompute_features.rs` after the format change).
    /// Run with: `cargo test -p ml-alpha --lib -- --ignored fxcache_local_smoke`.
    #[test]
    #[ignore]
    fn fxcache_local_smoke() {
        let path = "/home/jgrusewski/Work/foxhunt/test_data/feature-cache/13c0b086a975cc7e2384377a2cd0e97738c9410292fcfecb5807c29bf885cb48.fxcache";
        let reader = FxCacheReader::open(path).expect("local fxcache should open");
        let m = reader.metadata();
        assert_eq!(m.version, FXCACHE_VERSION);
        assert_eq!(m.feat_dim, FEAT_DIM);
        assert_eq!(m.target_dim, TARGET_DIM);
        assert_eq!(m.ofi_dim, OFI_DIM);
        assert!(m.has_ofi);
        assert!(m.bar_count > 0);

        // Bounds check: first + last record readable.
        let first = reader.record(0);
        assert_eq!(first.features.len(), FEAT_DIM);
        assert_eq!(first.targets.len(), TARGET_DIM);
        assert_eq!(first.ofi.len(), OFI_DIM);

        let last = reader.record(m.bar_count - 1);
        assert_eq!(last.features.len(), FEAT_DIM);

        // Timestamps should be plausible (post-2020 nanoseconds since epoch:
        // between ~1.6e18 and ~2.0e18) AND monotonically non-decreasing.
        let t0 = reader.record_timestamp(0);
        let t_last = reader.record_timestamp(m.bar_count - 1);
        assert!(
            t0 > 1_500_000_000_000_000_000 && t0 < 2_500_000_000_000_000_000,
            "first timestamp {t0} ns is implausible"
        );
        assert!(
            t_last >= t0,
            "timestamps not monotonic: t_last={t_last} < t0={t0}"
        );

        // raw_close (target column 2) should be a plausible price magnitude
        // (futures contracts: O(10) to O(1e5), never NaN/Inf or O(1e30)).
        let raw_close_first = first.targets[2];
        assert!(
            raw_close_first.is_finite() && raw_close_first.abs() < 1.0e6,
            "first raw_close {raw_close_first} looks broken (sentinel-magnitude?)"
        );
    }
}