foxhunt/data/tests/real_data_helpers.rs

//! Real Market Data Helpers for Feature Engineering Tests
//!
//! Utilities to load real BTC/ETH Parquet data and convert it to formats
//! used by feature engineering tests (OHLCV bars, PricePoints).

use anyhow::{Context, Result};
use chrono::{DateTime, Utc};
use data::features::PricePoint;
use data::parquet_persistence::{MarketDataEvent, ParquetMarketDataReader};
use std::path::PathBuf;

/// Path to real test data directory (relative to workspace root)
const REAL_DATA_PATH: &str = "test_data/real/parquet";
const BTC_FILE: &str = "BTC-USD_30day_2024-09.parquet";
const ETH_FILE: &str = "ETH-USD_30day_2024-09.parquet";

/// Real market data loader for feature engineering tests
pub struct RealDataLoader {
    base_path: String,
}

impl RealDataLoader {
    /// Create new loader with workspace-relative path
    pub fn new() -> Self {
        let manifest_dir = env!("CARGO_MANIFEST_DIR");
        let base_path = PathBuf::from(manifest_dir)
            .parent()
            .expect("Failed to get workspace root")
            .join(REAL_DATA_PATH);

        Self {
            base_path: base_path.to_string_lossy().to_string(),
        }
    }

    /// Check if real data files exist
    pub fn files_exist(&self) -> bool {
        let btc_path = PathBuf::from(&self.base_path).join(BTC_FILE);
        let eth_path = PathBuf::from(&self.base_path).join(ETH_FILE);
        btc_path.exists() && eth_path.exists()
    }

    /// Load BTC price data as PricePoints
    pub async fn load_btc_prices(&self, count: usize) -> Result<Vec<PricePoint>> {
        self.load_prices(BTC_FILE, count).await
    }

    /// Load ETH price data as PricePoints
    pub async fn load_eth_prices(&self, count: usize) -> Result<Vec<PricePoint>> {
        self.load_prices(ETH_FILE, count).await
    }

    /// Load BTC close prices as f64 vector (for simple technical indicator tests)
    pub async fn load_btc_close_prices(&self, count: usize) -> Result<Vec<f64>> {
        self.load_close_prices(BTC_FILE, count).await
    }

    /// Load ETH close prices as f64 vector (for simple technical indicator tests)
    pub async fn load_eth_close_prices(&self, count: usize) -> Result<Vec<f64>> {
        self.load_close_prices(ETH_FILE, count).await
    }

    /// Load price data from a specific file as PricePoints
    async fn load_prices(&self, filename: &str, count: usize) -> Result<Vec<PricePoint>> {
        let reader = ParquetMarketDataReader::new(self.base_path.clone());
        let events = reader
            .read_file(filename)
            .await
            .with_context(|| format!("Failed to load {}", filename))?;

        // Take only the requested number of events
        let events_subset = events.into_iter().take(count).collect::<Vec<_>>();

        // Convert MarketDataEvent to PricePoint
        let price_points = events_subset
            .into_iter()
            .map(|event| self.event_to_price_point(&event))
            .collect::<Result<Vec<_>>>()?;

        Ok(price_points)
    }

    /// Load close prices as f64 vector for simple indicator calculations
    async fn load_close_prices(&self, filename: &str, count: usize) -> Result<Vec<f64>> {
        let reader = ParquetMarketDataReader::new(self.base_path.clone());
        let events = reader
            .read_file(filename)
            .await
            .with_context(|| format!("Failed to load {}", filename))?;

        // Take only the requested number of events and extract close prices
        let close_prices = events
            .into_iter()
            .take(count)
            .filter_map(|event| event.price) // Use price as close price
            .collect::<Vec<_>>();

        Ok(close_prices)
    }

    /// Convert MarketDataEvent to PricePoint
    fn event_to_price_point(&self, event: &MarketDataEvent) -> Result<PricePoint> {
        let timestamp = self.ns_to_datetime(event.timestamp_ns)?;
        let close = event.price.context("Price is None")?;

        // If OHLC data is available, use it; otherwise derive from close price
        let open = event.open.unwrap_or(close);
        let high = event.high.unwrap_or(close.max(open));
        let low = event.low.unwrap_or(close.min(open));

        Ok(PricePoint {
            timestamp,
            open,
            high,
            low,
            close,
        })
    }

    /// Convert nanosecond timestamp to DateTime<Utc>
    fn ns_to_datetime(&self, timestamp_ns: u64) -> Result<DateTime<Utc>> {
        let timestamp_ms = (timestamp_ns / 1_000_000) as i64;
        DateTime::from_timestamp_millis(timestamp_ms).context("Invalid timestamp")
    }
}

impl Default for RealDataLoader {
    fn default() -> Self {
        Self::new()
    }
}

/// Extract a specific time range from price data
pub fn extract_time_range(
    data: &[PricePoint],
    start: DateTime<Utc>,
    end: DateTime<Utc>,
) -> Vec<PricePoint> {
    data.iter()
        .filter(|point| point.timestamp >= start && point.timestamp <= end)
        .cloned()
        .collect()
}

/// Extract a specific number of bars from the middle of the dataset
/// This helps avoid edge effects in technical indicators
pub fn extract_middle_bars(data: &[PricePoint], count: usize) -> Vec<PricePoint> {
    if data.len() <= count {
        return data.to_vec();
    }

    let start_idx = (data.len() - count) / 2;
    let end_idx = start_idx + count;
    data[start_idx..end_idx].to_vec()
}