1120 lines
38 KiB
Rust
1120 lines
38 KiB
Rust
//! Comprehensive DBN Data Quality Validation Tool
|
|
//!
|
|
//! This tool performs systematic validation across all DBN market data files:
|
|
//! - Multi-file validation with quality scoring
|
|
//! - Cross-symbol validation (price correlations, time alignment)
|
|
//! - Anomaly detection (automated spike detection)
|
|
//! - Summary statistics and quality reports
|
|
//! - Exit codes for CI/CD integration
|
|
//!
|
|
//! ## Usage
|
|
//!
|
|
//! ```bash
|
|
//! # Validate single symbol
|
|
//! cargo run --bin validate_dbn_data -- --symbol ES.FUT
|
|
//!
|
|
//! # Validate all symbols
|
|
//! cargo run --bin validate_dbn_data -- --all
|
|
//!
|
|
//! # Generate HTML report
|
|
//! cargo run --bin validate_dbn_data -- --all --output report.html
|
|
//!
|
|
//! # CI/CD mode (exit 1 if quality fails)
|
|
//! cargo run --bin validate_dbn_data -- --all --fail-on-poor-quality
|
|
//! ```
|
|
|
|
use anyhow::{Context, Result};
|
|
use backtesting_service::dbn_data_source::DbnDataSource;
|
|
use backtesting_service::strategy_engine::MarketData;
|
|
use chrono::Utc;
|
|
use clap::Parser;
|
|
use rust_decimal::prelude::*;
|
|
use rust_decimal::Decimal;
|
|
use serde::{Deserialize, Serialize};
|
|
use std::collections::HashMap;
|
|
use std::path::{Path, PathBuf};
|
|
use std::time::Instant;
|
|
|
|
#[derive(Parser, Debug)]
|
|
#[clap(name = "validate_dbn_data")]
|
|
#[clap(about = "Comprehensive DBN data quality validation tool")]
|
|
struct Args {
|
|
/// Validate specific symbol
|
|
#[clap(long, short)]
|
|
symbol: Option<String>,
|
|
|
|
/// Validate all symbols in test_data
|
|
#[clap(long, short)]
|
|
all: bool,
|
|
|
|
/// Output format: text, json, html
|
|
#[clap(long, short, default_value = "text")]
|
|
format: String,
|
|
|
|
/// Output file path (stdout if not specified)
|
|
#[clap(long, short)]
|
|
output: Option<PathBuf>,
|
|
|
|
/// Exit with error code if quality is poor
|
|
#[clap(long)]
|
|
fail_on_poor_quality: bool,
|
|
|
|
/// Minimum quality score (0-100) to pass
|
|
#[clap(long, default_value = "70")]
|
|
min_quality_score: u8,
|
|
|
|
/// Enable verbose output
|
|
#[clap(long, short)]
|
|
verbose: bool,
|
|
|
|
/// Test data directory
|
|
#[clap(long, default_value = "test_data/real/databento")]
|
|
data_dir: String,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
struct ValidationReport {
|
|
timestamp: String,
|
|
total_symbols: usize,
|
|
total_files: usize,
|
|
total_bars: usize,
|
|
duration_ms: u64,
|
|
symbols: Vec<SymbolReport>,
|
|
cross_symbol_analysis: Option<CrossSymbolAnalysis>,
|
|
overall_quality: QualityScore,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
struct SymbolReport {
|
|
symbol: String,
|
|
files: Vec<String>,
|
|
total_bars: usize,
|
|
statistics: Statistics,
|
|
quality_checks: QualityChecks,
|
|
quality_score: QualityScore,
|
|
anomalies: Vec<Anomaly>,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
struct Statistics {
|
|
// Price statistics
|
|
min_close: String,
|
|
max_close: String,
|
|
median_close: String,
|
|
avg_close: String,
|
|
price_range: String,
|
|
price_std_dev: String,
|
|
|
|
// Volume statistics
|
|
total_volume: String,
|
|
avg_volume: String,
|
|
min_volume: String,
|
|
max_volume: String,
|
|
volume_std_dev: String,
|
|
|
|
// Time coverage
|
|
first_timestamp: String,
|
|
last_timestamp: String,
|
|
duration_hours: i64,
|
|
expected_bars: usize,
|
|
actual_bars: usize,
|
|
completeness_pct: f64,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
struct QualityChecks {
|
|
ohlcv_violations: usize,
|
|
zero_volumes: usize,
|
|
timestamp_gaps: usize,
|
|
price_spikes: usize,
|
|
duplicate_timestamps: usize,
|
|
out_of_order_timestamps: usize,
|
|
negative_prices: usize,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
struct QualityScore {
|
|
score: u8,
|
|
rating: String,
|
|
issues: Vec<String>,
|
|
recommendations: Vec<String>,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
struct Anomaly {
|
|
anomaly_type: String,
|
|
bar_index: usize,
|
|
timestamp: String,
|
|
description: String,
|
|
severity: String,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
struct CrossSymbolAnalysis {
|
|
symbols_analyzed: Vec<String>,
|
|
correlation_matrix: HashMap<String, HashMap<String, f64>>,
|
|
time_alignment_issues: Vec<String>,
|
|
}
|
|
|
|
#[tokio::main]
|
|
async fn main() -> Result<()> {
|
|
let args = Args::parse();
|
|
|
|
// Initialize tracing
|
|
if args.verbose {
|
|
tracing_subscriber::fmt()
|
|
.with_max_level(tracing::Level::DEBUG)
|
|
.init();
|
|
} else {
|
|
tracing_subscriber::fmt()
|
|
.with_max_level(tracing::Level::INFO)
|
|
.init();
|
|
}
|
|
|
|
let start = Instant::now();
|
|
|
|
// Discover DBN files
|
|
let data_dir = Path::new(&args.data_dir);
|
|
let symbols_files = if args.all {
|
|
discover_dbn_files(data_dir)?
|
|
} else if let Some(ref symbol) = args.symbol {
|
|
let files = find_symbol_files(data_dir, symbol)?;
|
|
if files.is_empty() {
|
|
anyhow::bail!("No DBN files found for symbol: {}", symbol);
|
|
}
|
|
let mut map = HashMap::new();
|
|
map.insert(symbol.clone(), files);
|
|
map
|
|
} else {
|
|
anyhow::bail!("Either --symbol or --all must be specified");
|
|
};
|
|
|
|
if symbols_files.is_empty() {
|
|
anyhow::bail!("No DBN files found in: {}", data_dir.display());
|
|
}
|
|
|
|
println!("Validating {} symbols...\n", symbols_files.len());
|
|
|
|
// Load and validate each symbol
|
|
let mut symbol_reports = Vec::new();
|
|
let mut total_bars = 0;
|
|
|
|
for (symbol, files) in symbols_files.iter() {
|
|
println!("Validating {}...", symbol);
|
|
|
|
let report = validate_symbol(symbol, files).await?;
|
|
total_bars += report.total_bars;
|
|
|
|
if args.verbose {
|
|
print_symbol_summary(&report);
|
|
}
|
|
|
|
symbol_reports.push(report);
|
|
}
|
|
|
|
// Cross-symbol analysis
|
|
let cross_symbol = if symbol_reports.len() > 1 {
|
|
Some(analyze_cross_symbol(&symbol_reports).await?)
|
|
} else {
|
|
None
|
|
};
|
|
|
|
// Calculate overall quality
|
|
let overall_quality = calculate_overall_quality(&symbol_reports);
|
|
|
|
// Create validation report
|
|
let report = ValidationReport {
|
|
timestamp: Utc::now().to_rfc3339(),
|
|
total_symbols: symbols_files.len(),
|
|
total_files: symbols_files.values().map(|f| f.len()).sum(),
|
|
total_bars,
|
|
duration_ms: start.elapsed().as_millis() as u64,
|
|
symbols: symbol_reports,
|
|
cross_symbol_analysis: cross_symbol,
|
|
overall_quality: overall_quality.clone(),
|
|
};
|
|
|
|
// Output report
|
|
match args.format.as_str() {
|
|
"json" => output_json(&report, args.output.as_ref())?,
|
|
"html" => output_html(&report, args.output.as_ref())?,
|
|
_ => output_text(&report),
|
|
}
|
|
|
|
// Exit with error if quality check fails
|
|
if args.fail_on_poor_quality && overall_quality.score < args.min_quality_score {
|
|
eprintln!(
|
|
"\n❌ VALIDATION FAILED: Quality score {} < minimum {}",
|
|
overall_quality.score, args.min_quality_score
|
|
);
|
|
std::process::exit(1);
|
|
}
|
|
|
|
if overall_quality.score >= args.min_quality_score {
|
|
println!(
|
|
"\n✅ VALIDATION PASSED: Quality score {}/100",
|
|
overall_quality.score
|
|
);
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
/// Discover all DBN files in directory
|
|
fn discover_dbn_files(data_dir: &Path) -> Result<HashMap<String, Vec<String>>> {
|
|
let mut symbols_files: HashMap<String, Vec<String>> = HashMap::new();
|
|
|
|
if !data_dir.exists() {
|
|
anyhow::bail!("Data directory not found: {}", data_dir.display());
|
|
}
|
|
|
|
for entry in std::fs::read_dir(data_dir)? {
|
|
let entry = entry?;
|
|
let path = entry.path();
|
|
|
|
if path.extension().and_then(|s| s.to_str()) == Some("dbn") {
|
|
if let Some(file_name) = path.file_name().and_then(|s| s.to_str()) {
|
|
// Extract symbol from filename (e.g., "ES.FUT_ohlcv-1m_2024-01-02.dbn" -> "ES.FUT")
|
|
if let Some(symbol) = file_name.split('_').next() {
|
|
symbols_files
|
|
.entry(symbol.to_string())
|
|
.or_default()
|
|
.push(path.to_string_lossy().to_string());
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Sort files for each symbol
|
|
for files in symbols_files.values_mut() {
|
|
files.sort();
|
|
}
|
|
|
|
Ok(symbols_files)
|
|
}
|
|
|
|
/// Find all DBN files for a specific symbol
|
|
fn find_symbol_files(data_dir: &Path, symbol: &str) -> Result<Vec<String>> {
|
|
let mut files = Vec::new();
|
|
|
|
if !data_dir.exists() {
|
|
return Ok(files);
|
|
}
|
|
|
|
for entry in std::fs::read_dir(data_dir)? {
|
|
let entry = entry?;
|
|
let path = entry.path();
|
|
|
|
if path.extension().and_then(|s| s.to_str()) == Some("dbn") {
|
|
if let Some(file_name) = path.file_name().and_then(|s| s.to_str()) {
|
|
if file_name.starts_with(symbol) {
|
|
files.push(path.to_string_lossy().to_string());
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
files.sort();
|
|
Ok(files)
|
|
}
|
|
|
|
/// Validate single symbol across all files
|
|
async fn validate_symbol(symbol: &str, files: &[String]) -> Result<SymbolReport> {
|
|
// Create data source with all files for symbol
|
|
let mut file_mapping = HashMap::new();
|
|
file_mapping.insert(symbol.to_string(), files.to_vec());
|
|
|
|
let data_source = DbnDataSource::new_multi_file(file_mapping).await?;
|
|
|
|
// Load all bars
|
|
let bars = data_source.load_ohlcv_bars_all(symbol).await?;
|
|
|
|
if bars.is_empty() {
|
|
anyhow::bail!("No bars loaded for symbol: {}", symbol);
|
|
}
|
|
|
|
// Calculate statistics
|
|
let statistics = calculate_statistics(&bars);
|
|
|
|
// Perform quality checks
|
|
let (quality_checks, anomalies) = perform_quality_checks(&bars);
|
|
|
|
// Calculate quality score
|
|
let quality_score = calculate_quality_score(&bars, &quality_checks, &statistics);
|
|
|
|
Ok(SymbolReport {
|
|
symbol: symbol.to_string(),
|
|
files: files.to_vec(),
|
|
total_bars: bars.len(),
|
|
statistics,
|
|
quality_checks,
|
|
quality_score,
|
|
anomalies,
|
|
})
|
|
}
|
|
|
|
/// Calculate comprehensive statistics
|
|
fn calculate_statistics(bars: &[MarketData]) -> Statistics {
|
|
let mut prices: Vec<Decimal> = bars.iter().map(|b| b.close).collect();
|
|
prices.sort();
|
|
|
|
let min_close = *prices.first().expect("INVARIANT: Collection should be non-empty");
|
|
let max_close = *prices.last().expect("INVARIANT: Collection should be non-empty");
|
|
let median_close = prices[prices.len() / 2];
|
|
let sum_prices: Decimal = prices.iter().sum();
|
|
let avg_close = sum_prices / Decimal::from(prices.len());
|
|
|
|
// Calculate standard deviation
|
|
let variance: Decimal = prices
|
|
.iter()
|
|
.map(|p| (*p - avg_close).powi(2))
|
|
.sum::<Decimal>()
|
|
/ Decimal::from(prices.len());
|
|
let std_dev = variance.sqrt().unwrap_or(Decimal::ZERO);
|
|
|
|
// Volume statistics
|
|
let total_volume: Decimal = bars.iter().map(|b| b.volume).sum();
|
|
let avg_volume = total_volume / Decimal::from(bars.len());
|
|
let max_volume = bars.iter().map(|b| b.volume).max().unwrap_or(Decimal::ZERO);
|
|
let min_volume = bars.iter().map(|b| b.volume).min().unwrap_or(Decimal::ZERO);
|
|
|
|
let volumes: Vec<Decimal> = bars.iter().map(|b| b.volume).collect();
|
|
let volume_variance: Decimal = volumes
|
|
.iter()
|
|
.map(|v| (*v - avg_volume).powi(2))
|
|
.sum::<Decimal>()
|
|
/ Decimal::from(volumes.len());
|
|
let volume_std_dev = volume_variance.sqrt().unwrap_or(Decimal::ZERO);
|
|
|
|
// Time coverage
|
|
let first_ts = bars.first().expect("INVARIANT: Collection should be non-empty").timestamp;
|
|
let last_ts = bars.last().expect("INVARIANT: Collection should be non-empty").timestamp;
|
|
let duration_seconds = (last_ts - first_ts).num_seconds();
|
|
let duration_hours = duration_seconds / 3600;
|
|
|
|
// Expected bars (1-minute bars, assuming ~6.5 hour trading day)
|
|
let expected_bars = (duration_hours as f64 / 6.5 * 390.0) as usize;
|
|
let completeness_pct = (bars.len() as f64 / expected_bars.max(1) as f64) * 100.0;
|
|
|
|
Statistics {
|
|
min_close: format!("{:.2}", min_close),
|
|
max_close: format!("{:.2}", max_close),
|
|
median_close: format!("{:.2}", median_close),
|
|
avg_close: format!("{:.2}", avg_close),
|
|
price_range: format!("{:.2}", max_close - min_close),
|
|
price_std_dev: format!("{:.2}", std_dev),
|
|
total_volume: format!("{:.0}", total_volume),
|
|
avg_volume: format!("{:.0}", avg_volume),
|
|
min_volume: format!("{:.0}", min_volume),
|
|
max_volume: format!("{:.0}", max_volume),
|
|
volume_std_dev: format!("{:.0}", volume_std_dev),
|
|
first_timestamp: first_ts.to_rfc3339(),
|
|
last_timestamp: last_ts.to_rfc3339(),
|
|
duration_hours,
|
|
expected_bars,
|
|
actual_bars: bars.len(),
|
|
completeness_pct,
|
|
}
|
|
}
|
|
|
|
/// Perform comprehensive quality checks
|
|
fn perform_quality_checks(bars: &[MarketData]) -> (QualityChecks, Vec<Anomaly>) {
|
|
let mut checks = QualityChecks {
|
|
ohlcv_violations: 0,
|
|
zero_volumes: 0,
|
|
timestamp_gaps: 0,
|
|
price_spikes: 0,
|
|
duplicate_timestamps: 0,
|
|
out_of_order_timestamps: 0,
|
|
negative_prices: 0,
|
|
};
|
|
|
|
let mut anomalies = Vec::new();
|
|
let mut timestamps_seen = std::collections::HashSet::new();
|
|
|
|
for i in 0..bars.len() {
|
|
let bar = &bars[i];
|
|
|
|
// OHLCV relationship check
|
|
if !(bar.high >= bar.low
|
|
&& bar.high >= bar.open
|
|
&& bar.high >= bar.close
|
|
&& bar.low <= bar.open
|
|
&& bar.low <= bar.close)
|
|
{
|
|
checks.ohlcv_violations += 1;
|
|
anomalies.push(Anomaly {
|
|
anomaly_type: "OHLCV Violation".to_string(),
|
|
bar_index: i,
|
|
timestamp: bar.timestamp.to_rfc3339(),
|
|
description: format!(
|
|
"Invalid OHLCV relationship: O={} H={} L={} C={}",
|
|
bar.open, bar.high, bar.low, bar.close
|
|
),
|
|
severity: "HIGH".to_string(),
|
|
});
|
|
}
|
|
|
|
// Zero volume check
|
|
if bar.volume == Decimal::ZERO {
|
|
checks.zero_volumes += 1;
|
|
}
|
|
|
|
// Negative price check
|
|
if bar.open < Decimal::ZERO
|
|
|| bar.high < Decimal::ZERO
|
|
|| bar.low < Decimal::ZERO
|
|
|| bar.close < Decimal::ZERO
|
|
{
|
|
checks.negative_prices += 1;
|
|
anomalies.push(Anomaly {
|
|
anomaly_type: "Negative Price".to_string(),
|
|
bar_index: i,
|
|
timestamp: bar.timestamp.to_rfc3339(),
|
|
description: format!(
|
|
"Negative price detected: O={} H={} L={} C={}",
|
|
bar.open, bar.high, bar.low, bar.close
|
|
),
|
|
severity: "CRITICAL".to_string(),
|
|
});
|
|
}
|
|
|
|
// Duplicate timestamp check
|
|
if !timestamps_seen.insert(bar.timestamp) {
|
|
checks.duplicate_timestamps += 1;
|
|
anomalies.push(Anomaly {
|
|
anomaly_type: "Duplicate Timestamp".to_string(),
|
|
bar_index: i,
|
|
timestamp: bar.timestamp.to_rfc3339(),
|
|
description: "Duplicate timestamp found".to_string(),
|
|
severity: "MEDIUM".to_string(),
|
|
});
|
|
}
|
|
|
|
if i > 0 {
|
|
let prev_bar = &bars[i - 1];
|
|
|
|
// Out of order timestamp check
|
|
if bar.timestamp < prev_bar.timestamp {
|
|
checks.out_of_order_timestamps += 1;
|
|
anomalies.push(Anomaly {
|
|
anomaly_type: "Out of Order".to_string(),
|
|
bar_index: i,
|
|
timestamp: bar.timestamp.to_rfc3339(),
|
|
description: format!(
|
|
"Timestamp out of order: current={} < previous={}",
|
|
bar.timestamp, prev_bar.timestamp
|
|
),
|
|
severity: "HIGH".to_string(),
|
|
});
|
|
}
|
|
|
|
// Timestamp gap check (>2 minutes for 1-minute bars)
|
|
let gap_seconds = (bar.timestamp - prev_bar.timestamp).num_seconds();
|
|
if gap_seconds > 120 {
|
|
checks.timestamp_gaps += 1;
|
|
if gap_seconds > 3600 {
|
|
// Gaps > 1 hour are anomalies
|
|
anomalies.push(Anomaly {
|
|
anomaly_type: "Large Gap".to_string(),
|
|
bar_index: i,
|
|
timestamp: bar.timestamp.to_rfc3339(),
|
|
description: format!(
|
|
"Gap of {} seconds ({} minutes)",
|
|
gap_seconds,
|
|
gap_seconds / 60
|
|
),
|
|
severity: "LOW".to_string(),
|
|
});
|
|
}
|
|
}
|
|
|
|
// Price spike check (>10% move)
|
|
let price_change_pct =
|
|
((bar.close - prev_bar.close) / prev_bar.close).abs() * Decimal::from(100);
|
|
if price_change_pct > Decimal::from(10) {
|
|
checks.price_spikes += 1;
|
|
anomalies.push(Anomaly {
|
|
anomaly_type: "Price Spike".to_string(),
|
|
bar_index: i,
|
|
timestamp: bar.timestamp.to_rfc3339(),
|
|
description: format!(
|
|
"{:.2}% price change (${:.2} -> ${:.2})",
|
|
price_change_pct, prev_bar.close, bar.close
|
|
),
|
|
severity: "MEDIUM".to_string(),
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
// Sort anomalies by severity
|
|
anomalies.sort_by(|a, b| {
|
|
let severity_order = |s: &str| match s {
|
|
"CRITICAL" => 0,
|
|
"HIGH" => 1,
|
|
"MEDIUM" => 2,
|
|
"LOW" => 3,
|
|
_ => 4,
|
|
};
|
|
severity_order(&a.severity).cmp(&severity_order(&b.severity))
|
|
});
|
|
|
|
(checks, anomalies)
|
|
}
|
|
|
|
/// Calculate quality score (0-100)
|
|
fn calculate_quality_score(
|
|
bars: &[MarketData],
|
|
checks: &QualityChecks,
|
|
stats: &Statistics,
|
|
) -> QualityScore {
|
|
let mut score = 100u8;
|
|
let mut issues = Vec::new();
|
|
let mut recommendations = Vec::new();
|
|
|
|
// Critical issues (-20 points each)
|
|
if checks.ohlcv_violations > 0 {
|
|
score = score.saturating_sub(20);
|
|
issues.push(format!("{} OHLCV violations", checks.ohlcv_violations));
|
|
recommendations
|
|
.push("Fix OHLCV relationship violations - data corruption likely".to_string());
|
|
}
|
|
|
|
if checks.negative_prices > 0 {
|
|
score = score.saturating_sub(20);
|
|
issues.push(format!("{} negative prices", checks.negative_prices));
|
|
recommendations.push("Remove bars with negative prices".to_string());
|
|
}
|
|
|
|
if checks.out_of_order_timestamps > 0 {
|
|
score = score.saturating_sub(15);
|
|
issues.push(format!(
|
|
"{} out-of-order timestamps",
|
|
checks.out_of_order_timestamps
|
|
));
|
|
recommendations.push("Sort bars chronologically".to_string());
|
|
}
|
|
|
|
// High severity issues (-10 points each)
|
|
if checks.duplicate_timestamps > 0 {
|
|
score = score.saturating_sub(10);
|
|
issues.push(format!(
|
|
"{} duplicate timestamps",
|
|
checks.duplicate_timestamps
|
|
));
|
|
recommendations.push("Remove duplicate bars".to_string());
|
|
}
|
|
|
|
// Medium severity issues (-5 points)
|
|
if checks.price_spikes > bars.len() / 100 {
|
|
score = score.saturating_sub(5);
|
|
issues.push(format!("{} price spikes (>10%)", checks.price_spikes));
|
|
recommendations.push("Investigate price spikes for data quality".to_string());
|
|
}
|
|
|
|
// Low severity issues (-2 points)
|
|
if checks.zero_volumes > bars.len() / 10 {
|
|
score = score.saturating_sub(2);
|
|
issues.push(format!("{} zero volume bars", checks.zero_volumes));
|
|
recommendations.push("Zero volumes may indicate illiquid periods".to_string());
|
|
}
|
|
|
|
if checks.timestamp_gaps > bars.len() / 20 {
|
|
score = score.saturating_sub(2);
|
|
issues.push(format!("{} timestamp gaps", checks.timestamp_gaps));
|
|
recommendations.push("Timestamp gaps expected during market close/open".to_string());
|
|
}
|
|
|
|
// Data completeness penalty
|
|
if stats.completeness_pct < 80.0 {
|
|
score = score.saturating_sub(10);
|
|
issues.push(format!("Low completeness: {:.1}%", stats.completeness_pct));
|
|
recommendations.push("Consider acquiring more complete data".to_string());
|
|
}
|
|
|
|
// Determine rating
|
|
let rating = if score >= 90 {
|
|
"EXCELLENT"
|
|
} else if score >= 75 {
|
|
"GOOD"
|
|
} else if score >= 60 {
|
|
"ACCEPTABLE"
|
|
} else if score >= 40 {
|
|
"POOR"
|
|
} else {
|
|
"CRITICAL"
|
|
};
|
|
|
|
QualityScore {
|
|
score,
|
|
rating: rating.to_string(),
|
|
issues,
|
|
recommendations,
|
|
}
|
|
}
|
|
|
|
/// Calculate overall quality across all symbols
|
|
fn calculate_overall_quality(symbol_reports: &[SymbolReport]) -> QualityScore {
|
|
if symbol_reports.is_empty() {
|
|
return QualityScore {
|
|
score: 0,
|
|
rating: "NO DATA".to_string(),
|
|
issues: vec!["No symbols validated".to_string()],
|
|
recommendations: vec![],
|
|
};
|
|
}
|
|
|
|
// Average score across symbols
|
|
let avg_score = symbol_reports
|
|
.iter()
|
|
.map(|r| r.quality_score.score as u32)
|
|
.sum::<u32>()
|
|
/ symbol_reports.len() as u32;
|
|
|
|
// Aggregate issues
|
|
let mut all_issues = Vec::new();
|
|
let mut all_recommendations = Vec::new();
|
|
|
|
for report in symbol_reports {
|
|
if report.quality_score.score < 70 {
|
|
all_issues.push(format!(
|
|
"{}: {} (score: {})",
|
|
report.symbol, report.quality_score.rating, report.quality_score.score
|
|
));
|
|
}
|
|
all_recommendations.extend(report.quality_score.recommendations.clone());
|
|
}
|
|
|
|
// Deduplicate recommendations
|
|
all_recommendations.sort();
|
|
all_recommendations.dedup();
|
|
|
|
let rating = if avg_score >= 90 {
|
|
"EXCELLENT"
|
|
} else if avg_score >= 75 {
|
|
"GOOD"
|
|
} else if avg_score >= 60 {
|
|
"ACCEPTABLE"
|
|
} else if avg_score >= 40 {
|
|
"POOR"
|
|
} else {
|
|
"CRITICAL"
|
|
};
|
|
|
|
QualityScore {
|
|
score: avg_score as u8,
|
|
rating: rating.to_string(),
|
|
issues: all_issues,
|
|
recommendations: all_recommendations,
|
|
}
|
|
}
|
|
|
|
/// Cross-symbol analysis (correlations, time alignment)
|
|
///
|
|
/// Pearson correlation of close prices requires raw bar data (`Vec<MarketData>`)
|
|
/// which is not available from `SymbolReport` (only string-formatted statistics).
|
|
/// To add real correlation, pass `HashMap<String, Vec<MarketData>>` alongside
|
|
/// `symbol_reports` and compute `pearson_correlation(&closes_a, &closes_b)` on
|
|
/// overlapping timestamps.
|
|
///
|
|
/// Time alignment is implemented below using the first/last timestamps from
|
|
/// each symbol's statistics.
|
|
async fn analyze_cross_symbol(symbol_reports: &[SymbolReport]) -> Result<CrossSymbolAnalysis> {
|
|
let symbols: Vec<String> = symbol_reports.iter().map(|r| r.symbol.clone()).collect();
|
|
|
|
// Correlation requires raw close-price vectors which SymbolReport does not carry.
|
|
// Placeholder: callers should extend this once raw bars are passed through.
|
|
let correlation_matrix = HashMap::new();
|
|
|
|
// Time alignment: check whether symbol pairs share the same time window.
|
|
let mut time_alignment_issues = Vec::new();
|
|
|
|
for i in 0..symbol_reports.len() {
|
|
for j in (i + 1)..symbol_reports.len() {
|
|
let a = &symbol_reports[i];
|
|
let b = &symbol_reports[j];
|
|
|
|
// Compare first-timestamp alignment (>1 hour difference = issue)
|
|
let a_first = chrono::DateTime::parse_from_rfc3339(&a.statistics.first_timestamp)
|
|
.ok()
|
|
.map(|dt| dt.with_timezone(&chrono::Utc));
|
|
let b_first = chrono::DateTime::parse_from_rfc3339(&b.statistics.first_timestamp)
|
|
.ok()
|
|
.map(|dt| dt.with_timezone(&chrono::Utc));
|
|
let a_last = chrono::DateTime::parse_from_rfc3339(&a.statistics.last_timestamp)
|
|
.ok()
|
|
.map(|dt| dt.with_timezone(&chrono::Utc));
|
|
let b_last = chrono::DateTime::parse_from_rfc3339(&b.statistics.last_timestamp)
|
|
.ok()
|
|
.map(|dt| dt.with_timezone(&chrono::Utc));
|
|
|
|
if let (Some(af), Some(bf)) = (a_first, b_first) {
|
|
let gap_secs = (af - bf).num_seconds().unsigned_abs();
|
|
if gap_secs > 3600 {
|
|
time_alignment_issues.push(format!(
|
|
"{} vs {}: start-time gap of {} seconds ({} hours)",
|
|
a.symbol,
|
|
b.symbol,
|
|
gap_secs,
|
|
gap_secs / 3600
|
|
));
|
|
}
|
|
}
|
|
|
|
if let (Some(al), Some(bl)) = (a_last, b_last) {
|
|
let gap_secs = (al - bl).num_seconds().unsigned_abs();
|
|
if gap_secs > 3600 {
|
|
time_alignment_issues.push(format!(
|
|
"{} vs {}: end-time gap of {} seconds ({} hours)",
|
|
a.symbol,
|
|
b.symbol,
|
|
gap_secs,
|
|
gap_secs / 3600
|
|
));
|
|
}
|
|
}
|
|
|
|
// Check overlapping coverage
|
|
if let (Some(af), Some(al), Some(bf), Some(bl)) = (a_first, a_last, b_first, b_last) {
|
|
let overlap_start = af.max(bf);
|
|
let overlap_end = al.min(bl);
|
|
if overlap_start > overlap_end {
|
|
time_alignment_issues.push(format!(
|
|
"{} vs {}: NO overlapping time window",
|
|
a.symbol, b.symbol
|
|
));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
Ok(CrossSymbolAnalysis {
|
|
symbols_analyzed: symbols,
|
|
correlation_matrix,
|
|
time_alignment_issues,
|
|
})
|
|
}
|
|
|
|
/// Print symbol summary to console
|
|
fn print_symbol_summary(report: &SymbolReport) {
|
|
println!(" Symbol: {}", report.symbol);
|
|
println!(" Files: {}", report.files.len());
|
|
println!(" Bars: {}", report.total_bars);
|
|
println!(
|
|
" Quality: {} ({})",
|
|
report.quality_score.score, report.quality_score.rating
|
|
);
|
|
if !report.anomalies.is_empty() {
|
|
println!(" Anomalies: {}", report.anomalies.len());
|
|
}
|
|
println!();
|
|
}
|
|
|
|
/// Output report as text
|
|
fn output_text(report: &ValidationReport) {
|
|
println!("\n═══════════════════════════════════════════════════════");
|
|
println!("DBN DATA QUALITY VALIDATION REPORT");
|
|
println!("═══════════════════════════════════════════════════════");
|
|
println!("Timestamp: {}", report.timestamp);
|
|
println!("Duration: {}ms", report.duration_ms);
|
|
println!();
|
|
|
|
println!("📊 SUMMARY");
|
|
println!(" Total Symbols: {}", report.total_symbols);
|
|
println!(" Total Files: {}", report.total_files);
|
|
println!(" Total Bars: {}", report.total_bars);
|
|
println!(
|
|
" Overall Quality: {} ({})",
|
|
report.overall_quality.score, report.overall_quality.rating
|
|
);
|
|
println!();
|
|
|
|
for symbol_report in &report.symbols {
|
|
println!("─────────────────────────────────────────────────────");
|
|
println!("SYMBOL: {}", symbol_report.symbol);
|
|
println!("─────────────────────────────────────────────────────");
|
|
println!();
|
|
|
|
println!("📈 Statistics:");
|
|
println!(" Bars: {}", symbol_report.total_bars);
|
|
println!(
|
|
" Price Range: ${} - ${}",
|
|
symbol_report.statistics.min_close, symbol_report.statistics.max_close
|
|
);
|
|
println!(" Avg Close: ${}", symbol_report.statistics.avg_close);
|
|
println!(" Std Dev: ${}", symbol_report.statistics.price_std_dev);
|
|
println!(" Total Volume: {}", symbol_report.statistics.total_volume);
|
|
println!(" Avg Volume: {}", symbol_report.statistics.avg_volume);
|
|
println!(
|
|
" Duration: {} hours",
|
|
symbol_report.statistics.duration_hours
|
|
);
|
|
println!(
|
|
" Completeness: {:.1}%",
|
|
symbol_report.statistics.completeness_pct
|
|
);
|
|
println!();
|
|
|
|
println!("✅ Quality Checks:");
|
|
println!(
|
|
" OHLCV Violations: {}",
|
|
symbol_report.quality_checks.ohlcv_violations
|
|
);
|
|
println!(
|
|
" Zero Volumes: {}",
|
|
symbol_report.quality_checks.zero_volumes
|
|
);
|
|
println!(
|
|
" Timestamp Gaps: {}",
|
|
symbol_report.quality_checks.timestamp_gaps
|
|
);
|
|
println!(
|
|
" Price Spikes: {}",
|
|
symbol_report.quality_checks.price_spikes
|
|
);
|
|
println!(
|
|
" Duplicate Timestamps: {}",
|
|
symbol_report.quality_checks.duplicate_timestamps
|
|
);
|
|
println!(
|
|
" Out of Order: {}",
|
|
symbol_report.quality_checks.out_of_order_timestamps
|
|
);
|
|
println!(
|
|
" Negative Prices: {}",
|
|
symbol_report.quality_checks.negative_prices
|
|
);
|
|
println!();
|
|
|
|
println!(
|
|
"🎯 Quality Score: {} ({})",
|
|
symbol_report.quality_score.score, symbol_report.quality_score.rating
|
|
);
|
|
|
|
if !symbol_report.quality_score.issues.is_empty() {
|
|
println!(" Issues:");
|
|
for issue in &symbol_report.quality_score.issues {
|
|
println!(" • {}", issue);
|
|
}
|
|
}
|
|
|
|
if !symbol_report.anomalies.is_empty() {
|
|
println!();
|
|
println!("⚠️ Anomalies (showing first 5):");
|
|
for anomaly in symbol_report.anomalies.iter().take(5) {
|
|
println!(
|
|
" [{:8}] {} at bar {}: {}",
|
|
anomaly.severity, anomaly.anomaly_type, anomaly.bar_index, anomaly.description
|
|
);
|
|
}
|
|
if symbol_report.anomalies.len() > 5 {
|
|
println!(" ... and {} more", symbol_report.anomalies.len() - 5);
|
|
}
|
|
}
|
|
|
|
println!();
|
|
}
|
|
|
|
println!("═══════════════════════════════════════════════════════");
|
|
println!(
|
|
"OVERALL QUALITY: {} ({})",
|
|
report.overall_quality.score, report.overall_quality.rating
|
|
);
|
|
|
|
if !report.overall_quality.recommendations.is_empty() {
|
|
println!();
|
|
println!("💡 Recommendations:");
|
|
for rec in &report.overall_quality.recommendations {
|
|
println!(" • {}", rec);
|
|
}
|
|
}
|
|
|
|
println!("═══════════════════════════════════════════════════════");
|
|
}
|
|
|
|
/// Output report as JSON
|
|
fn output_json(report: &ValidationReport, output_path: Option<&PathBuf>) -> Result<()> {
|
|
let json =
|
|
serde_json::to_string_pretty(report).context("Failed to serialize report to JSON")?;
|
|
|
|
if let Some(path) = output_path {
|
|
std::fs::write(path, json).context("Failed to write JSON report")?;
|
|
println!("JSON report written to: {}", path.display());
|
|
} else {
|
|
println!("{}", json);
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
/// Output report as HTML
|
|
fn output_html(report: &ValidationReport, output_path: Option<&PathBuf>) -> Result<()> {
|
|
let html = generate_html_report(report);
|
|
|
|
if let Some(path) = output_path {
|
|
std::fs::write(path, html).context("Failed to write HTML report")?;
|
|
println!("HTML report written to: {}", path.display());
|
|
} else {
|
|
println!("{}", html);
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
/// Generate HTML report
|
|
fn generate_html_report(report: &ValidationReport) -> String {
|
|
let mut html = String::new();
|
|
|
|
html.push_str("<!DOCTYPE html>\n<html>\n<head>\n");
|
|
html.push_str("<meta charset='UTF-8'>\n");
|
|
html.push_str("<title>DBN Data Quality Validation Report</title>\n");
|
|
html.push_str("<style>\n");
|
|
html.push_str("body { font-family: Arial, sans-serif; margin: 20px; background: #f5f5f5; }\n");
|
|
html.push_str("h1 { color: #333; }\n");
|
|
html.push_str("h2 { color: #666; border-bottom: 2px solid #ddd; padding-bottom: 5px; }\n");
|
|
html.push_str(".summary { background: white; padding: 20px; border-radius: 8px; margin-bottom: 20px; box-shadow: 0 2px 4px rgba(0,0,0,0.1); }\n");
|
|
html.push_str(".symbol-report { background: white; padding: 20px; border-radius: 8px; margin-bottom: 20px; box-shadow: 0 2px 4px rgba(0,0,0,0.1); }\n");
|
|
html.push_str(".quality-excellent { color: #28a745; font-weight: bold; }\n");
|
|
html.push_str(".quality-good { color: #5cb85c; font-weight: bold; }\n");
|
|
html.push_str(".quality-acceptable { color: #f0ad4e; font-weight: bold; }\n");
|
|
html.push_str(".quality-poor { color: #d9534f; font-weight: bold; }\n");
|
|
html.push_str(".quality-critical { color: #c82333; font-weight: bold; }\n");
|
|
html.push_str(".stats-table { width: 100%; border-collapse: collapse; margin: 10px 0; }\n");
|
|
html.push_str(".stats-table th, .stats-table td { padding: 8px; text-align: left; border-bottom: 1px solid #ddd; }\n");
|
|
html.push_str(".stats-table th { background: #f8f9fa; font-weight: bold; }\n");
|
|
html.push_str(".anomaly { padding: 10px; margin: 5px 0; border-left: 4px solid; }\n");
|
|
html.push_str(".anomaly-critical { border-color: #c82333; background: #f8d7da; }\n");
|
|
html.push_str(".anomaly-high { border-color: #d9534f; background: #f8d7da; }\n");
|
|
html.push_str(".anomaly-medium { border-color: #f0ad4e; background: #fcf8e3; }\n");
|
|
html.push_str(".anomaly-low { border-color: #5bc0de; background: #d9edf7; }\n");
|
|
html.push_str("</style>\n");
|
|
html.push_str("</head>\n<body>\n");
|
|
|
|
html.push_str("<h1>DBN Data Quality Validation Report</h1>\n");
|
|
html.push_str(&format!(
|
|
"<p><strong>Generated:</strong> {}</p>\n",
|
|
report.timestamp
|
|
));
|
|
html.push_str(&format!(
|
|
"<p><strong>Duration:</strong> {}ms</p>\n",
|
|
report.duration_ms
|
|
));
|
|
|
|
html.push_str("<div class='summary'>\n");
|
|
html.push_str("<h2>Summary</h2>\n");
|
|
html.push_str("<table class='stats-table'>\n");
|
|
html.push_str(&format!(
|
|
"<tr><th>Total Symbols</th><td>{}</td></tr>\n",
|
|
report.total_symbols
|
|
));
|
|
html.push_str(&format!(
|
|
"<tr><th>Total Files</th><td>{}</td></tr>\n",
|
|
report.total_files
|
|
));
|
|
html.push_str(&format!(
|
|
"<tr><th>Total Bars</th><td>{}</td></tr>\n",
|
|
report.total_bars
|
|
));
|
|
|
|
let quality_class = match report.overall_quality.rating.as_str() {
|
|
"EXCELLENT" => "quality-excellent",
|
|
"GOOD" => "quality-good",
|
|
"ACCEPTABLE" => "quality-acceptable",
|
|
"POOR" => "quality-poor",
|
|
_ => "quality-critical",
|
|
};
|
|
html.push_str(&format!(
|
|
"<tr><th>Overall Quality</th><td><span class='{}'>{} ({})</span></td></tr>\n",
|
|
quality_class, report.overall_quality.score, report.overall_quality.rating
|
|
));
|
|
html.push_str("</table>\n");
|
|
html.push_str("</div>\n");
|
|
|
|
for symbol_report in &report.symbols {
|
|
html.push_str("<div class='symbol-report'>\n");
|
|
html.push_str(&format!("<h2>Symbol: {}</h2>\n", symbol_report.symbol));
|
|
|
|
html.push_str("<h3>Statistics</h3>\n");
|
|
html.push_str("<table class='stats-table'>\n");
|
|
html.push_str(&format!(
|
|
"<tr><th>Total Bars</th><td>{}</td></tr>\n",
|
|
symbol_report.total_bars
|
|
));
|
|
html.push_str(&format!(
|
|
"<tr><th>Price Range</th><td>${} - ${}</td></tr>\n",
|
|
symbol_report.statistics.min_close, symbol_report.statistics.max_close
|
|
));
|
|
html.push_str(&format!(
|
|
"<tr><th>Average Close</th><td>${}</td></tr>\n",
|
|
symbol_report.statistics.avg_close
|
|
));
|
|
html.push_str(&format!(
|
|
"<tr><th>Completeness</th><td>{:.1}%</td></tr>\n",
|
|
symbol_report.statistics.completeness_pct
|
|
));
|
|
html.push_str("</table>\n");
|
|
|
|
html.push_str("<h3>Quality Checks</h3>\n");
|
|
html.push_str("<table class='stats-table'>\n");
|
|
html.push_str(&format!(
|
|
"<tr><th>OHLCV Violations</th><td>{}</td></tr>\n",
|
|
symbol_report.quality_checks.ohlcv_violations
|
|
));
|
|
html.push_str(&format!(
|
|
"<tr><th>Zero Volumes</th><td>{}</td></tr>\n",
|
|
symbol_report.quality_checks.zero_volumes
|
|
));
|
|
html.push_str(&format!(
|
|
"<tr><th>Price Spikes</th><td>{}</td></tr>\n",
|
|
symbol_report.quality_checks.price_spikes
|
|
));
|
|
html.push_str(&format!(
|
|
"<tr><th>Timestamp Gaps</th><td>{}</td></tr>\n",
|
|
symbol_report.quality_checks.timestamp_gaps
|
|
));
|
|
html.push_str("</table>\n");
|
|
|
|
let quality_class = match symbol_report.quality_score.rating.as_str() {
|
|
"EXCELLENT" => "quality-excellent",
|
|
"GOOD" => "quality-good",
|
|
"ACCEPTABLE" => "quality-acceptable",
|
|
"POOR" => "quality-poor",
|
|
_ => "quality-critical",
|
|
};
|
|
html.push_str(&format!(
|
|
"<h3>Quality Score: <span class='{}'>{} ({})</span></h3>\n",
|
|
quality_class, symbol_report.quality_score.score, symbol_report.quality_score.rating
|
|
));
|
|
|
|
if !symbol_report.anomalies.is_empty() {
|
|
html.push_str("<h3>Anomalies</h3>\n");
|
|
for anomaly in symbol_report.anomalies.iter().take(10) {
|
|
let anomaly_class = match anomaly.severity.as_str() {
|
|
"CRITICAL" => "anomaly-critical",
|
|
"HIGH" => "anomaly-high",
|
|
"MEDIUM" => "anomaly-medium",
|
|
_ => "anomaly-low",
|
|
};
|
|
html.push_str(&format!("<div class='anomaly {}'>\n", anomaly_class));
|
|
html.push_str(&format!(
|
|
"<strong>[{}]</strong> {} at bar {}: {}\n",
|
|
anomaly.severity, anomaly.anomaly_type, anomaly.bar_index, anomaly.description
|
|
));
|
|
html.push_str("</div>\n");
|
|
}
|
|
if symbol_report.anomalies.len() > 10 {
|
|
html.push_str(&format!(
|
|
"<p><em>... and {} more anomalies</em></p>\n",
|
|
symbol_report.anomalies.len() - 10
|
|
));
|
|
}
|
|
}
|
|
|
|
html.push_str("</div>\n");
|
|
}
|
|
|
|
html.push_str("</body>\n</html>");
|
|
html
|
|
}
|