foxhunt/crates/ml/examples/validate_225_features_databento.rs

//! 54-Feature Extraction Validation - Wave 12 Agent W12-11
//!
//! Validates production feature extraction on all 4 downloaded Databento datasets.
//! Verifies zero NaN/Inf errors and Wave D feature activation across all symbols.
//!
//! ## Test Coverage
//! - ES.FUT: 180-day S&P 500 E-mini futures (~174k bars)
//! - NQ.FUT: 180-day NASDAQ E-mini futures (~262k bars)
//! - 6E.FUT: 180-day Euro FX futures (~204k bars)
//! - ZN.FUT: 90-day 10-Year Treasury Note futures (~142k bars)
//!
//! ## Validation Criteria
//! 1. Feature count: Exactly 54 per bar
//! 2. NaN count: 0 across all features
//! 3. Inf count: 0 across all features
//! 4. Wave D activation: ≥40% non-zero (indices 201-224)
//! 5. Extraction time: <10ms per bar
//!
//! ## Usage
//! ```bash
//! cargo run -p ml --example validate_54_features_databento --release
//! ```

use anyhow::{Context, Result};
use dbn::decode::{DbnDecoder, DecodeRecordRef};
use dbn::OhlcvMsg;
use ml::features::extraction::{extract_ml_features, OHLCVBar};
use serde::{Deserialize, Serialize};
use std::path::Path;
use std::time::Instant;

/// Symbol validation result
#[derive(Debug, Clone, Serialize, Deserialize)]
struct SymbolValidation {
    symbol: String,
    file_path: String,
    bar_count: usize,
    feature_count: usize,
    nan_count: usize,
    inf_count: usize,
    wave_d_nonzero_count: usize,
    wave_d_total_count: usize,
    wave_d_activation_pct: f64,
    extraction_time_ms: f64,
    ms_per_bar: f64,
    validation_passed: bool,
    error_message: Option<String>,
}

/// Overall validation report
#[derive(Debug, Clone, Serialize, Deserialize)]
struct ValidationReport {
    total_symbols: usize,
    passed: usize,
    failed: usize,
    total_bars: usize,
    total_features: usize,
    total_nan: usize,
    total_inf: usize,
    avg_wave_d_activation_pct: f64,
    avg_ms_per_bar: f64,
    total_time_ms: f64,
    results: Vec<SymbolValidation>,
}

/// Load OHLCV bars from DBN file
fn load_dbn_bars(file_path: &Path) -> Result<Vec<OHLCVBar>> {
    let mut decoder = DbnDecoder::from_file(file_path)
        .with_context(|| format!("Failed to open DBN file: {}", file_path.display()))?;

    let mut bars = Vec::new();

    while let Some(record_ref) = decoder
        .decode_record_ref()
        .with_context(|| format!("Failed to decode DBN record at bar {}", bars.len()))?
    {
        if let Some(ohlcv) = record_ref.get::<OhlcvMsg>() {
            // Convert DBN fixed-point prices (9 decimal places) to f64
            let open = ohlcv.open as f64 / 1_000_000_000.0;
            let high = ohlcv.high as f64 / 1_000_000_000.0;
            let low = ohlcv.low as f64 / 1_000_000_000.0;
            let close = ohlcv.close as f64 / 1_000_000_000.0;
            let volume = ohlcv.volume as f64;

            // Convert timestamp (nanoseconds since epoch) to DateTime
            let timestamp =
                chrono::DateTime::<chrono::Utc>::from_timestamp_nanos(ohlcv.hd.ts_event as i64);

            bars.push(OHLCVBar {
                timestamp,
                open,
                high,
                low,
                close,
                volume,
            });
        }
    }

    Ok(bars)
}

/// Validate feature extraction on a single symbol
fn validate_symbol(symbol: &str, file_path: &str) -> Result<SymbolValidation> {
    let start_time = Instant::now();

    let mut result = SymbolValidation {
        symbol: symbol.to_string(),
        file_path: file_path.to_string(),
        bar_count: 0,
        feature_count: 0,
        nan_count: 0,
        inf_count: 0,
        wave_d_nonzero_count: 0,
        wave_d_total_count: 0,
        wave_d_activation_pct: 0.0,
        extraction_time_ms: 0.0,
        ms_per_bar: 0.0,
        validation_passed: false,
        error_message: None,
    };

    // Load DBN bars
    let path = Path::new(file_path);
    if !path.exists() {
        result.error_message = Some(format!("File not found: {}", file_path));
        return Ok(result);
    }

    let bars = match load_dbn_bars(path) {
        Ok(b) => b,
        Err(e) => {
            result.error_message = Some(format!("Failed to load DBN file: {}", e));
            return Ok(result);
        },
    };

    result.bar_count = bars.len();

    if bars.len() < 50 {
        result.error_message = Some(format!(
            "Insufficient bars: {} (need ≥50 for warmup)",
            bars.len()
        ));
        return Ok(result);
    }

    // Extract features
    let extraction_start = Instant::now();
    let features = match extract_ml_features(&bars) {
        Ok(f) => f,
        Err(e) => {
            result.error_message = Some(format!("Feature extraction failed: {}", e));
            return Ok(result);
        },
    };
    let extraction_time = extraction_start.elapsed();

    result.extraction_time_ms = extraction_time.as_secs_f64() * 1000.0;
    result.feature_count = features.len();

    if features.is_empty() {
        result.error_message = Some("No features extracted".to_string());
        return Ok(result);
    }

    // Verify feature dimensions
    let expected_features_per_bar = 54;
    let actual_features_per_bar = features[0].len();

    if actual_features_per_bar != expected_features_per_bar {
        result.error_message = Some(format!(
            "Feature dimension mismatch: expected {}, got {}",
            expected_features_per_bar, actual_features_per_bar
        ));
        return Ok(result);
    }

    // Check for NaN/Inf
    for feature_vec in &features {
        for &val in feature_vec.iter() {
            if val.is_nan() {
                result.nan_count += 1;
            }
            if val.is_infinite() {
                result.inf_count += 1;
            }
        }
    }

    // Check Wave D feature activation (indices 201-224)
    result.wave_d_total_count = features.len() * 24;
    result.wave_d_nonzero_count = features
        .iter()
        .flat_map(|fv| &fv[201..224])
        .filter(|&&v| v.abs() > 1e-9)
        .count();

    result.wave_d_activation_pct =
        (result.wave_d_nonzero_count as f64 / result.wave_d_total_count as f64) * 100.0;

    // Calculate extraction time per bar
    result.ms_per_bar = result.extraction_time_ms / result.bar_count as f64;

    // Overall validation
    result.validation_passed = result.nan_count == 0
        && result.inf_count == 0
        && result.wave_d_activation_pct >= 40.0
        && result.ms_per_bar < 10.0;

    let total_time = start_time.elapsed();
    result.extraction_time_ms = total_time.as_secs_f64() * 1000.0;

    Ok(result)
}

fn main() -> Result<()> {
    tracing_subscriber::fmt::init();

    println!("═════════════════════════════════════════════════════════════════");
    println!("  54-Feature Extraction Validation - Wave 12 Agent W12-11");
    println!("═════════════════════════════════════════════════════════════════\n");

    let start_time = Instant::now();

    // Define test symbols and expected files (absolute paths - use decompressed versions)
    let base_path = "/home/jgrusewski/Work/foxhunt/test_data";
    let symbols = vec![
        (
            "ES.FUT",
            format!("{}/ES_FUT_180d_decompressed.dbn", base_path),
        ),
        (
            "NQ.FUT",
            format!("{}/NQ_FUT_180d_uncompressed.dbn", base_path),
        ),
        (
            "6E.FUT",
            format!("{}/6E_FUT_180d_decompressed.dbn", base_path),
        ),
        ("ZN.FUT", format!("{}/ZN_FUT_90d.dbn", base_path)),
    ];

    let mut results = Vec::new();
    let mut total_bars = 0;
    let mut total_features = 0;

    for (symbol, file_path) in &symbols {
        println!("Validating {} ({})...", symbol, file_path);

        match validate_symbol(symbol, file_path) {
            Ok(result) => {
                if result.validation_passed {
                    println!(
                        "  ✅ PASS - {} bars, {} features",
                        result.bar_count, result.feature_count
                    );
                    println!(
                        "     NaN: {}, Inf: {}, Wave D: {:.1}%, Time: {:.3}ms/bar",
                        result.nan_count,
                        result.inf_count,
                        result.wave_d_activation_pct,
                        result.ms_per_bar
                    );
                } else {
                    println!("  ❌ FAIL");
                    if let Some(err) = &result.error_message {
                        println!("     Error: {}", err);
                    }
                    if result.bar_count > 0 {
                        println!(
                            "     Bars: {}, Features: {}, NaN: {}, Inf: {}, Wave D: {:.1}%",
                            result.bar_count,
                            result.feature_count,
                            result.nan_count,
                            result.inf_count,
                            result.wave_d_activation_pct
                        );
                        println!("     Extraction: {:.3}ms/bar", result.ms_per_bar);
                    }
                }

                total_bars += result.bar_count;
                total_features += result.feature_count;
                results.push(result);
            },
            Err(e) => {
                println!("  ❌ ERROR: {}", e);
                results.push(SymbolValidation {
                    symbol: symbol.to_string(),
                    file_path: file_path.to_string(),
                    bar_count: 0,
                    feature_count: 0,
                    nan_count: 0,
                    inf_count: 0,
                    wave_d_nonzero_count: 0,
                    wave_d_total_count: 0,
                    wave_d_activation_pct: 0.0,
                    extraction_time_ms: 0.0,
                    ms_per_bar: 0.0,
                    validation_passed: false,
                    error_message: Some(e.to_string()),
                });
            },
        }
        println!();
    }

    let total_time_ms = start_time.elapsed().as_secs_f64() * 1000.0;

    // Calculate summary statistics
    let passed = results.iter().filter(|r| r.validation_passed).count();
    let failed = results.len() - passed;

    let total_nan: usize = results.iter().map(|r| r.nan_count).sum();
    let total_inf: usize = results.iter().map(|r| r.inf_count).sum();

    let avg_wave_d_activation_pct = if passed > 0 {
        results
            .iter()
            .filter(|r| r.validation_passed)
            .map(|r| r.wave_d_activation_pct)
            .sum::<f64>()
            / passed as f64
    } else {
        0.0
    };

    let avg_ms_per_bar = if total_bars > 0 {
        results.iter().map(|r| r.extraction_time_ms).sum::<f64>() / total_bars as f64
    } else {
        0.0
    };

    let report = ValidationReport {
        total_symbols: results.len(),
        passed,
        failed,
        total_bars,
        total_features,
        total_nan,
        total_inf,
        avg_wave_d_activation_pct,
        avg_ms_per_bar,
        total_time_ms,
        results,
    };

    println!("═════════════════════════════════════════════════════════════════");
    println!("  VALIDATION SUMMARY");
    println!("═════════════════════════════════════════════════════════════════");
    println!("  Total Symbols: {}", report.total_symbols);
    println!("  Passed: {} ✅", report.passed);
    println!("  Failed: {} ❌", report.failed);
    println!("  Total Bars: {}", report.total_bars);
    println!("  Total Features: {}", report.total_features);
    println!("  Total NaN: {}", report.total_nan);
    println!("  Total Inf: {}", report.total_inf);
    println!(
        "  Avg Wave D Activation: {:.1}%",
        report.avg_wave_d_activation_pct
    );
    println!("  Avg Extraction Time: {:.3}ms/bar", report.avg_ms_per_bar);
    println!("  Total Validation Time: {:.0}ms", report.total_time_ms);
    println!("═════════════════════════════════════════════════════════════════\n");

    // Save JSON report
    let report_json = serde_json::to_string_pretty(&report)?;
    std::fs::write("/tmp/w12_11_validation_report.json", &report_json)?;
    println!("✅ Validation report saved to: /tmp/w12_11_validation_report.json");

    // Generate agent report
    let mut agent_report = String::new();
    agent_report.push_str("# Agent W12-11: 54-Feature Extraction Validation\n\n");
    agent_report.push_str(&format!(
        "**Date**: {}\n",
        chrono::Utc::now().format("%Y-%m-%d %H:%M:%S UTC")
    ));
    agent_report.push_str(&format!(
        "**Total Validation Time**: {:.0}ms\n\n",
        report.total_time_ms
    ));
    agent_report.push_str("---\n\n");
    agent_report.push_str("## Executive Summary\n\n");
    agent_report.push_str(&format!("- **Total Symbols**: {}\n", report.total_symbols));
    agent_report.push_str(&format!("- **Passed**: {} ✅\n", report.passed));
    agent_report.push_str(&format!("- **Failed**: {} ❌\n", report.failed));
    agent_report.push_str(&format!("- **Total Bars**: {}\n", report.total_bars));
    agent_report.push_str(&format!(
        "- **Total Features**: {}\n",
        report.total_features
    ));
    agent_report.push_str(&format!("- **Total NaN**: {}\n", report.total_nan));
    agent_report.push_str(&format!("- **Total Inf**: {}\n", report.total_inf));
    agent_report.push_str(&format!(
        "- **Avg Wave D Activation**: {:.1}%\n",
        report.avg_wave_d_activation_pct
    ));
    agent_report.push_str(&format!(
        "- **Avg Extraction Time**: {:.3}ms/bar\n",
        report.avg_ms_per_bar
    ));
    agent_report.push_str(&format!(
        "- **Overall Status**: {}\n\n",
        if report.failed == 0 {
            "✅ ALL PASSED"
        } else {
            "❌ VALIDATION FAILURES"
        }
    ));
    agent_report.push_str("---\n\n");
    agent_report.push_str("## Validation Results by Symbol\n\n");

    for result in &report.results {
        agent_report.push_str(&format!(
            "### {} - {}\n\n",
            result.symbol,
            if result.validation_passed {
                "✅ PASS"
            } else {
                "❌ FAIL"
            }
        ));
        agent_report.push_str(&format!("- **File**: `{}`\n", result.file_path));
        agent_report.push_str(&format!("- **Bar Count**: {}\n", result.bar_count));
        agent_report.push_str(&format!("- **Feature Count**: {}\n", result.feature_count));
        agent_report.push_str(&format!("- **NaN Count**: {}\n", result.nan_count));
        agent_report.push_str(&format!("- **Inf Count**: {}\n", result.inf_count));
        agent_report.push_str(&format!(
            "- **Wave D Activation**: {:.1}% ({}/{} non-zero)\n",
            result.wave_d_activation_pct, result.wave_d_nonzero_count, result.wave_d_total_count
        ));
        agent_report.push_str(&format!(
            "- **Extraction Time**: {:.3}ms/bar ({:.2}ms total)\n",
            result.ms_per_bar, result.extraction_time_ms
        ));
        if let Some(err) = &result.error_message {
            agent_report.push_str(&format!("- **Error**: {}\n", err));
        }
        agent_report.push_str("\n");
    }

    agent_report.push_str("---\n\n");
    agent_report.push_str("## Success Criteria\n\n");
    agent_report.push_str(&format!(
        "- [{}] Feature count: 54 per bar\n",
        if report.passed > 0 { "x" } else { " " }
    ));
    agent_report.push_str(&format!(
        "- [{}] NaN count: 0 across all features\n",
        if report.total_nan == 0 { "x" } else { " " }
    ));
    agent_report.push_str(&format!(
        "- [{}] Inf count: 0 across all features\n",
        if report.total_inf == 0 { "x" } else { " " }
    ));
    agent_report.push_str(&format!(
        "- [{}] Wave D activation: ≥40% non-zero (indices 201-224)\n",
        if report.avg_wave_d_activation_pct >= 40.0 {
            "x"
        } else {
            " "
        }
    ));
    agent_report.push_str(&format!(
        "- [{}] Extraction time: <10ms per bar\n",
        if report.avg_ms_per_bar < 10.0 {
            "x"
        } else {
            " "
        }
    ));
    agent_report.push_str(&format!(
        "- [{}] All 4 symbols validated\n\n",
        if report.passed == 4 { "x" } else { " " }
    ));

    agent_report.push_str("---\n\n");
    agent_report.push_str("## Next Steps\n\n");
    if report.failed > 0 {
        agent_report.push_str("❌ **Action Required**: Some validations failed.\n\n");
        for result in &report.results {
            if !result.validation_passed {
                if let Some(err) = &result.error_message {
                    agent_report.push_str(&format!("- {}: {}\n", result.symbol, err));
                }
            }
        }
    } else {
        agent_report.push_str("✅ **All validations passed!** Ready to proceed with:\n\n");
        agent_report.push_str("- Wave 12 Agent W12-12: Feature statistics analysis\n");
        agent_report.push_str("- Wave 12 Agent W12-13: ML model retraining initialization\n");
        agent_report.push_str("- Wave 13: Model retraining with 54 features\n");
    }

    std::fs::write("/tmp/w12_11_agent_report.md", &agent_report)?;
    println!("✅ Agent report saved to: /tmp/w12_11_agent_report.md\n");

    // Exit with appropriate status code
    if report.failed > 0 {
        eprintln!("❌ Validation failed for {} symbols", report.failed);
        std::process::exit(1);
    } else {
        println!("✅ ALL 4 SYMBOLS VALIDATED - 54 FEATURES OPERATIONAL");
        Ok(())
    }
}