foxhunt/crates/data/tests/real_data_helpers.rs

//! Real Market Data Helpers for Feature Engineering Tests
//!
//! Utilities to load real BTC/ETH Parquet data and convert it to formats
//! used by feature engineering tests (OHLCV bars, PricePoints).

#![allow(
    clippy::absurd_extreme_comparisons,
    clippy::assertions_on_constants,
    clippy::assertions_on_result_states,
    clippy::clone_on_copy,
    clippy::decimal_literal_representation,
    clippy::doc_lazy_continuation,
    clippy::doc_markdown,
    clippy::double_comparisons,
    clippy::else_if_without_else,
    clippy::empty_drop,
    clippy::expect_used,
    clippy::field_reassign_with_default,
    clippy::format_push_string,
    clippy::if_then_some_else_none,
    clippy::indexing_slicing,
    clippy::integer_division,
    clippy::len_zero,
    clippy::let_and_return,
    clippy::let_underscore_must_use,
    clippy::manual_let_else,
    clippy::manual_range_contains,
    clippy::map_err_ignore,
    clippy::missing_const_for_fn,
    clippy::modulo_arithmetic,
    clippy::needless_range_loop,
    clippy::new_without_default,
    clippy::non_ascii_literal,
    clippy::nonminimal_bool,
    clippy::octal_escapes,
    clippy::overly_complex_bool_expr,
    clippy::redundant_clone,
    clippy::shadow_reuse,
    clippy::shadow_unrelated,
    clippy::similar_names,
    clippy::single_component_path_imports,
    clippy::single_match_else,
    clippy::str_to_string,
    clippy::string_slice,
    clippy::tests_outside_test_module,
    clippy::unnecessary_cast,
    clippy::unnecessary_get_then_check,
    clippy::unnecessary_unwrap,
    clippy::unseparated_literal_suffix,
    clippy::unwrap_or_default,
    clippy::unwrap_used,
    clippy::use_debug,
    clippy::useless_vec,
    clippy::wildcard_enum_match_arm,
    dead_code,
    deprecated,
    unreachable_pub,
    unused_assignments,
    unused_comparisons,
    unused_imports,
    unused_variables
)]

use anyhow::{Context, Result};
use chrono::{DateTime, Utc};
use data::features::PricePoint;
use data::parquet_persistence::{MarketDataEvent, ParquetMarketDataReader};
use std::path::PathBuf;

/// Path to real test data directory (relative to workspace root)
const REAL_DATA_PATH: &str = "test_data/real/parquet";
const BTC_FILE: &str = "BTC-USD_30day_2024-09.parquet";
const ETH_FILE: &str = "ETH-USD_30day_2024-09.parquet";

/// Real market data loader for feature engineering tests
pub(crate) struct RealDataLoader {
    base_path: String,
}

impl RealDataLoader {
    /// Create new loader with workspace-relative path
    pub(crate) fn new() -> Self {
        let manifest_dir = env!("CARGO_MANIFEST_DIR");
        let base_path = PathBuf::from(manifest_dir)
            .parent()
            .expect("Failed to get workspace root")
            .join(REAL_DATA_PATH);

        Self {
            base_path: base_path.to_string_lossy().to_string(),
        }
    }

    /// Check if real data files exist
    pub(crate) fn files_exist(&self) -> bool {
        let btc_path = PathBuf::from(&self.base_path).join(BTC_FILE);
        let eth_path = PathBuf::from(&self.base_path).join(ETH_FILE);
        btc_path.exists() && eth_path.exists()
    }

    /// Load BTC price data as PricePoints
    pub(crate) async fn load_btc_prices(&self, count: usize) -> Result<Vec<PricePoint>> {
        self.load_prices(BTC_FILE, count).await
    }

    /// Load ETH price data as PricePoints
    pub(crate) async fn load_eth_prices(&self, count: usize) -> Result<Vec<PricePoint>> {
        self.load_prices(ETH_FILE, count).await
    }

    /// Load BTC close prices as f64 vector (for simple technical indicator tests)
    pub(crate) async fn load_btc_close_prices(&self, count: usize) -> Result<Vec<f64>> {
        self.load_close_prices(BTC_FILE, count).await
    }

    /// Load ETH close prices as f64 vector (for simple technical indicator tests)
    pub(crate) async fn load_eth_close_prices(&self, count: usize) -> Result<Vec<f64>> {
        self.load_close_prices(ETH_FILE, count).await
    }

    /// Load price data from a specific file as PricePoints
    async fn load_prices(&self, filename: &str, count: usize) -> Result<Vec<PricePoint>> {
        let reader = ParquetMarketDataReader::new(self.base_path.clone());
        let events = reader
            .read_file(filename)
            .await
            .with_context(|| format!("Failed to load {}", filename))?;

        // Take only the requested number of events
        let events_subset = events.into_iter().take(count).collect::<Vec<_>>();

        // Convert MarketDataEvent to PricePoint
        let price_points = events_subset
            .into_iter()
            .map(|event| self.event_to_price_point(&event))
            .collect::<Result<Vec<_>>>()?;

        Ok(price_points)
    }

    /// Load close prices as f64 vector for simple indicator calculations
    async fn load_close_prices(&self, filename: &str, count: usize) -> Result<Vec<f64>> {
        let reader = ParquetMarketDataReader::new(self.base_path.clone());
        let events = reader
            .read_file(filename)
            .await
            .with_context(|| format!("Failed to load {}", filename))?;

        // Take only the requested number of events and extract close prices
        let close_prices = events
            .into_iter()
            .take(count)
            .filter_map(|event| event.price) // Use price as close price
            .collect::<Vec<_>>();

        Ok(close_prices)
    }

    /// Convert MarketDataEvent to PricePoint
    fn event_to_price_point(&self, event: &MarketDataEvent) -> Result<PricePoint> {
        let timestamp = self.ns_to_datetime(event.timestamp_ns)?;
        let close = event.price.context("Price is None")?;

        // If OHLC data is available, use it; otherwise derive from close price
        let open = event.open.unwrap_or(close);
        let high = event.high.unwrap_or(close.max(open));
        let low = event.low.unwrap_or(close.min(open));

        Ok(PricePoint {
            timestamp,
            open,
            high,
            low,
            close,
        })
    }

    /// Convert nanosecond timestamp to DateTime<Utc>
    fn ns_to_datetime(&self, timestamp_ns: u64) -> Result<DateTime<Utc>> {
        let timestamp_ms = (timestamp_ns / 1_000_000) as i64;
        DateTime::from_timestamp_millis(timestamp_ms).context("Invalid timestamp")
    }
}

impl Default for RealDataLoader {
    fn default() -> Self {
        Self::new()
    }
}