foxhunt/ml/examples/verify_dbn_loader_zero_free.rs

//! DBN Sequence Loader Zero-Padding Verification
//!
//! Wave 5 Agent 26: Verifies that MAMBA-2 data loader produces zero-free features
//! by using the production extract_ml_features() pipeline.
//!
//! Expected results:
//! - 0% zero-padding (all 225 features are real)
//! - Sequences have shape [batch, seq_len, 225]
//! - All feature values are non-zero (except for actual market conditions)

use anyhow::Result;
use ml::data_loaders::DbnSequenceLoader;
use tracing::{info, warn, Level};

#[tokio::main]
async fn main() -> Result<()> {
    // Initialize logging
    tracing_subscriber::fmt().with_max_level(Level::INFO).init();

    info!("=== DBN Sequence Loader Zero-Padding Verification ===");
    info!("");

    // Create loader with Wave D configuration (225 features)
    info!("Creating DbnSequenceLoader with 225 features...");
    let feature_config = ml::features::config::FeatureConfig::wave_d();
    let seq_len = 60;

    let mut loader = DbnSequenceLoader::with_feature_config(seq_len, feature_config.clone())
        .await
        .map_err(|e| anyhow::anyhow!("Failed to create loader: {}", e))?;

    info!("✓ Loader created");
    info!("  Feature config: {:?}", feature_config.phase);
    info!("  Feature count: {}", feature_config.feature_count());
    info!("  Sequence length: {}", seq_len);
    info!("");

    // Load sequences from test data
    info!("Loading sequences from test data...");
    let dbn_dir = "test_data/real/databento/ml_training_small";
    let train_split = 0.9;

    let (train_data, val_data) = loader
        .load_sequences(dbn_dir, train_split)
        .await
        .map_err(|e| anyhow::anyhow!("Failed to load sequences: {}", e))?;

    info!("✓ Sequences loaded");
    info!("  Training sequences: {}", train_data.len());
    info!("  Validation sequences: {}", val_data.len());
    info!("");

    // Analyze a sample sequence for zero-padding
    if let Some((input, _target)) = train_data.first() {
        info!("Analyzing first training sequence...");
        let dims = input.dims();
        info!("  Input shape: {:?}", dims);

        // Expected shape: [1, 60, 225]
        assert_eq!(dims.len(), 3, "Expected 3D tensor");
        assert_eq!(dims[0], 1, "Expected batch size of 1");
        assert_eq!(dims[1], seq_len, "Expected sequence length of {}", seq_len);
        assert_eq!(dims[2], 225, "Expected 225 features");

        info!("  ✓ Shape is correct: [1, {}, 225]", seq_len);

        // Convert to Vec for analysis
        let values: Vec<f64> = input.flatten_all()?.to_vec1()?;
        let total_values = values.len();
        let zero_count = values.iter().filter(|&&x| x == 0.0).count();
        let zero_percentage = (zero_count as f64 / total_values as f64) * 100.0;

        info!("");
        info!("Zero-Padding Analysis:");
        info!("  Total values: {}", total_values);
        info!("  Zero values: {} ({:.2}%)", zero_count, zero_percentage);
        info!(
            "  Non-zero values: {} ({:.2}%)",
            total_values - zero_count,
            100.0 - zero_percentage
        );

        if zero_percentage > 10.0 {
            warn!("⚠️  High zero percentage detected: {:.2}%", zero_percentage);
            warn!("   This suggests zero-padding is still present!");
        } else {
            info!(
                "  ✓ Zero-padding eliminated ({}% < 10% threshold)",
                zero_percentage
            );
        }

        // Check per-feature zero counts
        info!("");
        info!("Per-Feature Zero Analysis:");
        let mut features_with_zeros = Vec::new();

        for feature_idx in 0..225 {
            let feature_values: Vec<f64> = (0..seq_len)
                .map(|t| values[t * 225 + feature_idx])
                .collect();

            let feature_zeros = feature_values.iter().filter(|&&x| x == 0.0).count();
            if feature_zeros > 0 {
                features_with_zeros.push((feature_idx, feature_zeros, seq_len));
            }
        }

        if features_with_zeros.is_empty() {
            info!("  ✓ No features have all zeros (100% real features)");
        } else {
            info!("  Features with zeros:");
            for (idx, zeros, total) in features_with_zeros.iter().take(10) {
                let pct = (*zeros as f64 / *total as f64) * 100.0;
                info!(
                    "    Feature {}: {}/{} zeros ({:.1}%)",
                    idx, zeros, total, pct
                );
            }
            if features_with_zeros.len() > 10 {
                info!(
                    "    ... and {} more features",
                    features_with_zeros.len() - 10
                );
            }
        }
    } else {
        warn!("⚠️  No training sequences found!");
    }

    info!("");
    info!("=== VERIFICATION COMPLETE ===");

    Ok(())
}