foxhunt/check_validation_data.rs

//! Verify chronological sequence of training and unseen validation data
use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
use parquet::arrow::arrow_reader::ParquetRecordBatchReader;
use arrow::array::*;
use std::fs::File;

fn main() -> Result<(), Box<dyn std::error::Error>> {
    println!("=" .repeat(60));
    println!("TRAINING DATA (ES_FUT_180d.parquet)");
    println!("=" .repeat(60));

    // Read training data
    let train_file = File::open("test_data/ES_FUT_180d.parquet")?;
    let train_builder = ParquetRecordBatchReaderBuilder::try_new(train_file)?;
    let train_metadata = train_builder.metadata();

    let mut train_rows = 0;
    for i in 0..train_metadata.num_row_groups() {
        train_rows += train_metadata.row_group(i).num_rows();
    }

    let mut train_reader = train_builder.build()?;
    let train_batch = train_reader.next().unwrap()?;

    // Get ts_event column
    let ts_col_idx = train_batch.schema().index_of("ts_event")?;
    let ts_array = train_batch.column(ts_col_idx)
        .as_any()
        .downcast_ref::<TimestampNanosecondArray>()
        .expect("ts_event is not TimestampNanosecondArray");

    let train_start = ts_array.value(0);

    // Get last batch
    let mut last_batch = train_batch.clone();
    for batch in train_reader {
        last_batch = batch?;
    }

    let ts_col_idx = last_batch.schema().index_of("ts_event")?;
    let ts_array = last_batch.column(ts_col_idx)
        .as_any()
        .downcast_ref::<TimestampNanosecondArray>()
        .expect("ts_event is not TimestampNanosecondArray");

    let train_end = ts_array.value(ts_array.len() - 1);

    println!("Rows: {}", train_rows);
    println!("File size: 2.9M");
    println!("\nDate range:");
    println!("  Start: {}", chrono::DateTime::from_timestamp_nanos(train_start));
    println!("  End:   {}", chrono::DateTime::from_timestamp_nanos(train_end));

    // Read unseen data
    println!("\n{}", "=".repeat(60));
    println!("UNSEEN VALIDATION DATA (ES_FUT_unseen.parquet)");
    println!("{}", "=".repeat(60));

    let unseen_file = File::open("test_data/ES_FUT_unseen.parquet")?;
    let unseen_builder = ParquetRecordBatchReaderBuilder::try_new(unseen_file)?;
    let unseen_metadata = unseen_builder.metadata();

    let mut unseen_rows = 0;
    for i in 0..unseen_metadata.num_row_groups() {
        unseen_rows += unseen_metadata.row_group(i).num_rows();
    }

    let mut unseen_reader = unseen_builder.build()?;
    let unseen_batch = unseen_reader.next().unwrap()?;

    let ts_col_idx = unseen_batch.schema().index_of("ts_event")?;
    let ts_array = unseen_batch.column(ts_col_idx)
        .as_any()
        .downcast_ref::<TimestampNanosecondArray>()
        .expect("ts_event is not TimestampNanosecondArray");

    let unseen_start = ts_array.value(0);

    // Get last batch
    let mut last_batch = unseen_batch.clone();
    for batch in unseen_reader {
        last_batch = batch?;
    }

    let ts_col_idx = last_batch.schema().index_of("ts_event")?;
    let ts_array = last_batch.column(ts_col_idx)
        .as_any()
        .downcast_ref::<TimestampNanosecondArray>()
        .expect("ts_event is not TimestampNanosecondArray");

    let unseen_end = ts_array.value(ts_array.len() - 1);

    println!("Rows: {}", unseen_rows);
    println!("File size: 224K");
    println!("\nDate range:");
    println!("  Start: {}", chrono::DateTime::from_timestamp_nanos(unseen_start));
    println!("  End:   {}", chrono::DateTime::from_timestamp_nanos(unseen_end));

    // Check chronological sequence
    println!("\n{}", "=".repeat(60));
    println!("CHRONOLOGICAL SEQUENCE CHECK");
    println!("{}", "=".repeat(60));

    let gap_ns = unseen_start - train_end;
    let gap_seconds = gap_ns as f64 / 1_000_000_000.0;
    let gap_days = gap_seconds / 86400.0;

    println!("Training ends:     {}", chrono::DateTime::from_timestamp_nanos(train_end));
    println!("Unseen starts:     {}", chrono::DateTime::from_timestamp_nanos(unseen_start));
    println!("Gap:               {} ns ({:.0} seconds = {:.1} days)", gap_ns, gap_seconds, gap_days);

    let status = if gap_ns < 0 {
        println!("⚠️  OVERLAP: Unseen data starts BEFORE training ends!");
        "FAILED"
    } else if gap_ns == 0 {
        println!("✅ PERFECT: No gap, immediate continuation");
        "EXISTS"
    } else if gap_seconds <= 86400.0 {
        println!("✅ ACCEPTABLE: Gap is less than 1 day");
        "EXISTS"
    } else {
        println!("⚠️  GAP: {:.1} days between training and unseen data", gap_days);
        "FAILED"
    };

    // Calculate unseen duration
    let unseen_duration_ns = unseen_end - unseen_start;
    let unseen_days = unseen_duration_ns as f64 / (86400.0 * 1_000_000_000.0);

    println!("\nUnseen data duration: {:.1} days", unseen_days);

    let short_data = if unseen_days < 30.0 {
        println!("⚠️  WARNING: Only {:.1} days (recommended: 30-90 days)", unseen_days);
        true
    } else {
        println!("✅ GOOD: {:.1} days of validation data", unseen_days);
        false
    };

    // Final verdict
    println!("\n{}", "=".repeat(60));
    println!("FINAL VERDICT");
    println!("{}", "=".repeat(60));

    let ready = if status == "FAILED" {
        println!("❌ FAILED: Need to re-download - chronological gap/overlap issue");
        "NO"
    } else if short_data {
        println!("⚠️  WARNING: Data exists but only {:.1} days (recommend 30-90)", unseen_days);
        "PARTIAL"
    } else {
        println!("✅ PASSED: Data is valid and ready for backtest");
        "YES"
    };

    println!("\nStatus: {}", status);
    println!("Ready for backtest: {}", ready);

    Ok(())
}