foxhunt/adaptive-strategy/tests/real_data_helpers.rs

//! Real Market Data Helpers for Strategy Testing
//!
//! Utilities to load real BTC/ETH Parquet data and convert it to formats
//! used by adaptive strategy tests.

use adaptive_strategy::regime::{PricePoint, VolumePoint};
use anyhow::{Context, Result};
use chrono::{DateTime, Duration, Utc};
use data::parquet_persistence::{MarketDataEvent, ParquetMarketDataReader};
use std::path::PathBuf;

/// Path to real test data directory (relative to workspace root)
const REAL_DATA_PATH: &str = "test_data/real/parquet";
const BTC_FILE: &str = "BTC-USD_30day_2024-09.parquet";
const ETH_FILE: &str = "ETH-USD_30day_2024-09.parquet";

/// Path to DBN data (futures market data - better for regime detection)
const DBN_DATA_PATH: &str = "test_data/real/databento";
const ES_FUT_FILE: &str = "ES.FUT_ohlcv-1m_2024-01-02.dbn";

/// Real market data loader for strategy tests
pub struct RealDataLoader {
    base_path: String,
    dbn_base_path: String,
}

impl RealDataLoader {
    /// Create new loader with workspace-relative path
    pub fn new() -> Self {
        let manifest_dir = env!("CARGO_MANIFEST_DIR");
        let workspace_root = PathBuf::from(manifest_dir)
            .parent()
            .expect("Failed to get workspace root");

        let base_path = workspace_root.join(REAL_DATA_PATH);
        let dbn_base_path = workspace_root.join(DBN_DATA_PATH);

        Self {
            base_path: base_path.to_string_lossy().to_string(),
            dbn_base_path: dbn_base_path.to_string_lossy().to_string(),
        }
    }

    /// Check if real data files exist (prefer DBN, fallback to Parquet)
    pub fn files_exist(&self) -> bool {
        // Check DBN files first (better for regime detection)
        let es_fut_path = PathBuf::from(&self.dbn_base_path).join(ES_FUT_FILE);
        if es_fut_path.exists() {
            return true;
        }

        // Fallback to Parquet
        let btc_path = PathBuf::from(&self.base_path).join(BTC_FILE);
        let eth_path = PathBuf::from(&self.base_path).join(ETH_FILE);
        btc_path.exists() && eth_path.exists()
    }

    /// Check if DBN data files exist
    pub fn dbn_files_exist(&self) -> bool {
        let es_fut_path = PathBuf::from(&self.dbn_base_path).join(ES_FUT_FILE);
        es_fut_path.exists()
    }

    /// Load BTC price data
    pub async fn load_btc_prices(&self, count: usize) -> Result<Vec<PricePoint>> {
        self.load_prices(BTC_FILE, count).await
    }

    /// Load ETH price data
    pub async fn load_eth_prices(&self, count: usize) -> Result<Vec<PricePoint>> {
        self.load_prices(ETH_FILE, count).await
    }

    /// Load BTC volume data
    pub async fn load_btc_volume(&self, count: usize) -> Result<Vec<VolumePoint>> {
        self.load_volume(BTC_FILE, count).await
    }

    /// Load ETH volume data
    pub async fn load_eth_volume(&self, count: usize) -> Result<Vec<VolumePoint>> {
        self.load_volume(ETH_FILE, count).await
    }

    /// Load price data from a specific file
    async fn load_prices(&self, filename: &str, count: usize) -> Result<Vec<PricePoint>> {
        let reader = ParquetMarketDataReader::new(self.base_path.clone());
        let events = reader.read_file(filename).await
            .with_context(|| format!("Failed to load {}", filename))?;

        // Take only the requested number of events
        let events_subset = events.into_iter().take(count).collect::<Vec<_>>();

        // Convert MarketDataEvent to PricePoint
        let price_points = events_subset
            .into_iter()
            .map(|event| self.event_to_price_point(&event))
            .collect::<Result<Vec<_>>>()?;

        Ok(price_points)
    }

    /// Load volume data from a specific file
    async fn load_volume(&self, filename: &str, count: usize) -> Result<Vec<VolumePoint>> {
        let reader = ParquetMarketDataReader::new(self.base_path.clone());
        let events = reader.read_file(filename).await
            .with_context(|| format!("Failed to load {}", filename))?;

        // Take only the requested number of events
        let events_subset = events.into_iter().take(count).collect::<Vec<_>>();

        // Convert MarketDataEvent to VolumePoint
        let volume_points = events_subset
            .into_iter()
            .map(|event| self.event_to_volume_point(&event))
            .collect::<Result<Vec<_>>>()?;

        Ok(volume_points)
    }

    /// Convert MarketDataEvent to PricePoint
    fn event_to_price_point(&self, event: &MarketDataEvent) -> Result<PricePoint> {
        let timestamp = self.ns_to_datetime(event.timestamp_ns)?;
        let price = event.price.context("Price is None")?;

        // If OHLC data is available, use it; otherwise derive from price
        let high = event.high.unwrap_or(price * 1.001); // 0.1% above close
        let low = event.low.unwrap_or(price * 0.999);   // 0.1% below close
        let open = event.open.unwrap_or(price);

        Ok(PricePoint {
            timestamp,
            price,
            high,
            low,
            open,
        })
    }

    /// Convert MarketDataEvent to VolumePoint
    fn event_to_volume_point(&self, event: &MarketDataEvent) -> Result<VolumePoint> {
        let timestamp = self.ns_to_datetime(event.timestamp_ns)?;
        let volume = event.quantity.context("Quantity is None")?;

        // Estimate dollar volume (volume * price)
        let price = event.price.unwrap_or(1.0);
        let dollar_volume = volume * price;

        Ok(VolumePoint {
            timestamp,
            volume,
            dollar_volume,
        })
    }

    /// Convert nanosecond timestamp to DateTime<Utc>
    fn ns_to_datetime(&self, timestamp_ns: u64) -> Result<DateTime<Utc>> {
        let timestamp_ms = (timestamp_ns / 1_000_000) as i64;
        DateTime::from_timestamp_millis(timestamp_ms)
            .context("Invalid timestamp")
    }
}

impl Default for RealDataLoader {
    fn default() -> Self {
        Self::new()
    }
}

/// Extract a specific time range from price data
pub fn extract_time_range(
    data: &[PricePoint],
    start: DateTime<Utc>,
    duration: Duration,
) -> Vec<PricePoint> {
    let end = start + duration;
    data.iter()
        .filter(|p| p.timestamp >= start && p.timestamp <= end)
        .cloned()
        .collect()
}

/// Extract trending segment from price data (consistent directional movement)
///
/// Identifies segments with strong, consistent trends by combining:
/// 1. Linear regression slope (directional strength)
/// 2. R-squared (trend consistency)
/// 3. Low volatility perpendicular to trend (cleaner trend)
pub fn extract_trending_segment(data: &[PricePoint], count: usize) -> Vec<PricePoint> {
    if data.len() < count {
        return data.to_vec();
    }

    let mut best_start = 0;
    let mut best_score = f64::NEG_INFINITY;

    // Scan for best trending segment
    for i in 0..data.len().saturating_sub(count) {
        let segment = &data[i..i + count];
        if segment.len() < 10 {
            continue;
        }

        // Calculate linear regression
        let (slope, r_squared) = calculate_linear_regression(segment);

        // Calculate normalized slope (per data point)
        let avg_price = segment.iter().map(|p| p.price).sum::<f64>() / segment.len() as f64;
        let normalized_slope = slope.abs() / avg_price;

        // Calculate volatility perpendicular to trend
        let residual_vol = calculate_residual_volatility(segment, slope);

        // Scoring:
        // - High absolute slope (strong trend)
        // - High R² (consistent trend)
        // - Low residual volatility (clean trend)
        let trend_score = normalized_slope * 1000.0 * r_squared * (1.0 / (1.0 + residual_vol));

        if trend_score > best_score {
            best_score = trend_score;
            best_start = i;
        }
    }

    data[best_start..best_start + count].to_vec()
}

/// Calculate linear regression slope and R-squared for price data
fn calculate_linear_regression(segment: &[PricePoint]) -> (f64, f64) {
    let n = segment.len() as f64;
    let mut sum_x = 0.0;
    let mut sum_y = 0.0;
    let mut sum_xy = 0.0;
    let mut sum_x2 = 0.0;

    for (i, point) in segment.iter().enumerate() {
        let x = i as f64;
        let y = point.price;
        sum_x += x;
        sum_y += y;
        sum_xy += x * y;
        sum_x2 += x * x;
    }

    let mean_x = sum_x / n;
    let mean_y = sum_y / n;

    // Slope = Σ((x - mean_x)(y - mean_y)) / Σ((x - mean_x)²)
    let numerator = sum_xy - n * mean_x * mean_y;
    let denominator = sum_x2 - n * mean_x * mean_x;
    let slope = if denominator.abs() > 1e-10 {
        numerator / denominator
    } else {
        0.0
    };

    // Calculate R²
    let mut ss_tot = 0.0;
    let mut ss_res = 0.0;
    for (i, point) in segment.iter().enumerate() {
        let x = i as f64;
        let y = point.price;
        let y_pred = mean_y + slope * (x - mean_x);
        ss_tot += (y - mean_y).powi(2);
        ss_res += (y - y_pred).powi(2);
    }

    let r_squared = if ss_tot > 1e-10 {
        1.0 - (ss_res / ss_tot)
    } else {
        0.0
    };

    (slope, r_squared.max(0.0).min(1.0))
}

/// Calculate residual volatility (deviations from linear trend)
fn calculate_residual_volatility(segment: &[PricePoint], slope: f64) -> f64 {
    let n = segment.len() as f64;
    let mean_y = segment.iter().map(|p| p.price).sum::<f64>() / n;
    let mean_x = (segment.len() - 1) as f64 / 2.0;

    let mut sum_sq_residuals = 0.0;
    for (i, point) in segment.iter().enumerate() {
        let x = i as f64;
        let y_pred = mean_y + slope * (x - mean_x);
        sum_sq_residuals += (point.price - y_pred).powi(2);
    }

    (sum_sq_residuals / n).sqrt()
}

/// Extract ranging segment from price data (sideways, mean-reverting)
///
/// Identifies segments with:
/// 1. Low linear trend (minimal slope)
/// 2. Tight price range (bounded oscillation)
/// 3. High mean reversion (price returns to average)
pub fn extract_ranging_segment(data: &[PricePoint], count: usize) -> Vec<PricePoint> {
    if data.len() < count {
        return data.to_vec();
    }

    let mut best_start = 0;
    let mut best_score = f64::NEG_INFINITY;

    for i in 0..data.len().saturating_sub(count) {
        let segment = &data[i..i + count];
        if segment.len() < 10 {
            continue;
        }

        // Calculate linear regression
        let (slope, _r_squared) = calculate_linear_regression(segment);

        // Calculate price range
        let prices: Vec<f64> = segment.iter().map(|p| p.price).collect();
        let max = prices.iter().cloned().fold(f64::NEG_INFINITY, f64::max);
        let min = prices.iter().cloned().fold(f64::INFINITY, f64::min);
        let range = max - min;
        let avg_price = prices.iter().sum::<f64>() / prices.len() as f64;
        let normalized_range = range / avg_price;

        // Calculate mean reversion (how often price crosses the mean)
        let mean = avg_price;
        let mut crossings = 0;
        for window in segment.windows(2) {
            let prev_above = window[0].price > mean;
            let curr_above = window[1].price > mean;
            if prev_above != curr_above {
                crossings += 1;
            }
        }
        let crossing_rate = crossings as f64 / segment.len() as f64;

        // Normalized slope (should be very low for ranging)
        let normalized_slope = slope.abs() / avg_price;

        // Scoring:
        // - Low slope (sideways)
        // - Low range (bounded)
        // - High crossing rate (mean-reverting)
        let ranging_score = crossing_rate * 100.0 / (1.0 + normalized_slope * 1000.0 + normalized_range * 10.0);

        if ranging_score > best_score {
            best_score = ranging_score;
            best_start = i;
        }
    }

    data[best_start..best_start + count].to_vec()
}

/// Extract volatile segment from price data (high volatility)
pub fn extract_volatile_segment(data: &[PricePoint], count: usize) -> Vec<PricePoint> {
    // Find segment with highest volatility
    let mut best_start = 0;
    let mut best_volatility = 0.0;

    for i in 0..data.len().saturating_sub(count) {
        let segment = &data[i..i + count];
        if segment.len() < 2 {
            continue;
        }

        // Calculate simple volatility as standard deviation of returns
        let returns: Vec<f64> = segment
            .windows(2)
            .map(|w| (w[1].price - w[0].price) / w[0].price)
            .collect();

        let mean = returns.iter().sum::<f64>() / returns.len() as f64;
        let variance = returns
            .iter()
            .map(|r| (r - mean).powi(2))
            .sum::<f64>()
            / returns.len() as f64;
        let volatility = variance.sqrt();

        if volatility > best_volatility {
            best_volatility = volatility;
            best_start = i;
        }
    }

    data[best_start..best_start + count].to_vec()
}

/// Extract stable segment from price data (very low volatility)
pub fn extract_stable_segment(data: &[PricePoint], count: usize) -> Vec<PricePoint> {
    // Find segment with lowest volatility
    let mut best_start = 0;
    let mut best_volatility = f64::MAX;

    for i in 0..data.len().saturating_sub(count) {
        let segment = &data[i..i + count];
        if segment.len() < 2 {
            continue;
        }

        // Calculate simple volatility
        let returns: Vec<f64> = segment
            .windows(2)
            .map(|w| (w[1].price - w[0].price) / w[0].price)
            .collect();

        let mean = returns.iter().sum::<f64>() / returns.len() as f64;
        let variance = returns
            .iter()
            .map(|r| (r - mean).powi(2))
            .sum::<f64>()
            / returns.len() as f64;
        let volatility = variance.sqrt();

        if volatility < best_volatility {
            best_volatility = volatility;
            best_start = i;
        }
    }

    data[best_start..best_start + count].to_vec()
}

#[cfg(test)]
mod tests {
    use super::*;

    #[tokio::test]
    async fn test_real_data_loader_creation() {
        let loader = RealDataLoader::new();
        assert!(!loader.base_path.is_empty());
    }

    #[tokio::test]
    #[ignore = "requires real Parquet files"]
    async fn test_load_btc_prices() {
        let loader = RealDataLoader::new();
        if !loader.files_exist() {
            eprintln!("Skipping test: real data files not found");
            return;
        }

        let prices = loader.load_btc_prices(100).await.unwrap();
        assert_eq!(prices.len(), 100);
        assert!(prices[0].price > 0.0);
        assert!(prices[0].high >= prices[0].low);
    }

    #[tokio::test]
    #[ignore = "requires real Parquet files"]
    async fn test_load_btc_volume() {
        let loader = RealDataLoader::new();
        if !loader.files_exist() {
            eprintln!("Skipping test: real data files not found");
            return;
        }

        let volume = loader.load_btc_volume(100).await.unwrap();
        assert_eq!(volume.len(), 100);
        assert!(volume[0].volume >= 0.0);
        assert!(volume[0].dollar_volume >= 0.0);
    }
}