foxhunt/ml/examples/validate_checkpoints.rs

//! Checkpoint Validation Tool
//!
//! Validates that trained model checkpoints contain real weights (not placeholders).
//! This script inspects SafeTensors files to ensure they:
//! 1. Have valid SafeTensors format (JSON header)
//! 2. Contain real tensor data (not all zeros or text placeholders)
//! 3. Have reasonable file sizes
//! 4. Match expected model architecture

use anyhow::{Context, Result};
use candle_core::safetensors::load as safetensors_load;
use std::collections::HashMap;
use std::fs;
use std::path::{Path, PathBuf};

#[derive(Debug)]
struct CheckpointReport {
    path: PathBuf,
    file_size_bytes: u64,
    is_valid_safetensors: bool,
    tensor_count: usize,
    tensors: Vec<TensorInfo>,
    is_all_zeros: bool,
    is_text_placeholder: bool,
}

#[derive(Debug)]
struct TensorInfo {
    name: String,
    shape: Vec<usize>,
    dtype: String,
    element_count: usize,
}

impl CheckpointReport {
    fn new(path: PathBuf) -> Result<Self> {
        let metadata = fs::metadata(&path)
            .with_context(|| format!("Failed to read metadata for {:?}", path))?;
        let file_size_bytes = metadata.len();

        let bytes = fs::read(&path)
            .with_context(|| format!("Failed to read file {:?}", path))?;

        // Check if all zeros
        let is_all_zeros = bytes.iter().all(|&b| b == 0);

        // Check if text placeholder
        let is_text_placeholder = if let Ok(text) = String::from_utf8(bytes[..bytes.len().min(100)].to_vec()) {
            text.contains("placeholder") || text.contains("Placeholder")
        } else {
            false
        };

        // Try to parse as SafeTensors
        let (is_valid_safetensors, tensor_count, tensors) = match safetensors_load(&path, &candle_core::Device::Cpu) {
            Ok(tensors_map) => {
                let tensor_count = tensors_map.len();

                let tensor_infos: Vec<TensorInfo> = tensors_map
                    .iter()
                    .map(|(name, tensor)| {
                        let shape = tensor.shape().dims().to_vec();
                        let element_count: usize = shape.iter().product();
                        TensorInfo {
                            name: name.clone(),
                            shape,
                            dtype: format!("{:?}", tensor.dtype()),
                            element_count,
                        }
                    })
                    .collect();

                (true, tensor_count, tensor_infos)
            }
            Err(_) => (false, 0, Vec::new()),
        };

        Ok(Self {
            path,
            file_size_bytes,
            is_valid_safetensors,
            tensor_count,
            tensors,
            is_all_zeros,
            is_text_placeholder,
        })
    }

    fn is_valid(&self) -> bool {
        self.is_valid_safetensors
            && !self.is_all_zeros
            && !self.is_text_placeholder
            && self.tensor_count > 0
            && self.file_size_bytes > 1024 // Must be >1KB
    }

    fn status(&self) -> &str {
        if self.is_all_zeros {
            "❌ ALL ZEROS"
        } else if self.is_text_placeholder {
            "❌ TEXT PLACEHOLDER"
        } else if !self.is_valid_safetensors {
            "❌ INVALID FORMAT"
        } else if self.tensor_count == 0 {
            "❌ NO TENSORS"
        } else if self.file_size_bytes <= 1024 {
            "⚠️  SUSPICIOUSLY SMALL"
        } else {
            "✅ VALID"
        }
    }

    fn print_summary(&self) {
        println!("\n{}", "=".repeat(80));
        println!("File: {}", self.path.display());
        println!("Size: {} bytes ({} KB)", self.file_size_bytes, self.file_size_bytes / 1024);
        println!("Status: {}", self.status());
        println!("Valid SafeTensors: {}", self.is_valid_safetensors);
        println!("Tensor count: {}", self.tensor_count);
        println!("All zeros: {}", self.is_all_zeros);
        println!("Text placeholder: {}", self.is_text_placeholder);

        if !self.tensors.is_empty() {
            println!("\nTensors:");
            for tensor in &self.tensors {
                println!(
                    "  - {} (shape: {:?}, dtype: {}, elements: {})",
                    tensor.name, tensor.shape, tensor.dtype, tensor.element_count
                );
            }
        }
    }
}

fn validate_directory(dir: &Path, model_name: &str) -> Result<HashMap<String, CheckpointReport>> {
    println!("\n{}", "=".repeat(80));
    println!("Validating {} checkpoints in: {}", model_name, dir.display());
    println!("{}", "=".repeat(80));

    let mut reports = HashMap::new();

    if !dir.exists() {
        println!("❌ Directory does not exist");
        return Ok(reports);
    }

    let entries: Vec<_> = fs::read_dir(dir)?
        .filter_map(|e| e.ok())
        .filter(|e| {
            e.path()
                .extension()
                .and_then(|s| s.to_str())
                == Some("safetensors")
        })
        .collect();

    println!("Found {} checkpoint files", entries.len());

    for (i, entry) in entries.iter().enumerate() {
        let path = entry.path();
        println!("\n[{}/{}] Validating: {}", i + 1, entries.len(), path.display());

        match CheckpointReport::new(path.clone()) {
            Ok(report) => {
                let file_name = path
                    .file_name()
                    .and_then(|s| s.to_str())
                    .unwrap_or("unknown")
                    .to_string();
                report.print_summary();
                reports.insert(file_name, report);
            }
            Err(e) => {
                println!("❌ Failed to validate: {}", e);
            }
        }
    }

    Ok(reports)
}

fn print_comparison_table(
    dqn_reports: &HashMap<String, CheckpointReport>,
    ppo_reports: &HashMap<String, CheckpointReport>,
) {
    println!("\n{}", "=".repeat(80));
    println!("COMPARISON: Agent 57 (Wave 160 Phase 2) vs Current");
    println!("{}", "=".repeat(80));

    println!("\n{:<30} | {:<20} | {:<20}", "Metric", "DQN", "PPO");
    println!("{}", "-".repeat(80));

    let dqn_total = dqn_reports.len();
    let ppo_total = ppo_reports.len();
    println!("{:<30} | {:<20} | {:<20}", "Total Files", dqn_total, ppo_total);

    let dqn_valid = dqn_reports.values().filter(|r| r.is_valid()).count();
    let ppo_valid = ppo_reports.values().filter(|r| r.is_valid()).count();
    println!("{:<30} | {:<20} | {:<20}", "Valid Files", dqn_valid, ppo_valid);

    let dqn_zeros = dqn_reports.values().filter(|r| r.is_all_zeros).count();
    let ppo_zeros = ppo_reports.values().filter(|r| r.is_all_zeros).count();
    println!("{:<30} | {:<20} | {:<20}", "All Zeros", dqn_zeros, ppo_zeros);

    let dqn_placeholders = dqn_reports.values().filter(|r| r.is_text_placeholder).count();
    let ppo_placeholders = ppo_reports.values().filter(|r| r.is_text_placeholder).count();
    println!("{:<30} | {:<20} | {:<20}", "Text Placeholders", dqn_placeholders, ppo_placeholders);

    let dqn_avg_size: u64 = if !dqn_reports.is_empty() {
        dqn_reports.values().map(|r| r.file_size_bytes).sum::<u64>() / dqn_reports.len() as u64
    } else {
        0
    };
    let ppo_avg_size: u64 = if !ppo_reports.is_empty() {
        ppo_reports.values().map(|r| r.file_size_bytes).sum::<u64>() / ppo_reports.len() as u64
    } else {
        0
    };
    println!(
        "{:<30} | {:<20} | {:<20}",
        "Average Size (KB)",
        dqn_avg_size / 1024,
        ppo_avg_size / 1024
    );

    println!("\n{}", "=".repeat(80));
    println!("AGENT 57 BASELINE (Wave 160 Phase 2)");
    println!("{}", "=".repeat(80));
    println!("DQN: 51 files of 1,024 bytes (all zeros) ❌");
    println!("PPO: 50 files of 26 bytes (text placeholders) ❌");

    println!("\n{}", "=".repeat(80));
    println!("CURRENT STATUS (Wave 160 Phase 3+)");
    println!("{}", "=".repeat(80));
    println!(
        "DQN: {} files, {} valid, {} all zeros {}",
        dqn_total,
        dqn_valid,
        dqn_zeros,
        if dqn_zeros > 0 { "❌" } else { "✅" }
    );
    println!(
        "PPO: {} files, {} valid, {} placeholders {}",
        ppo_total,
        ppo_valid,
        ppo_placeholders,
        if ppo_valid == ppo_total { "✅" } else { "❌" }
    );
}

fn main() -> Result<()> {
    // Validate DQN checkpoints
    let dqn_dir = Path::new("ml/trained_models/production/dqn_real_data");
    let dqn_reports = validate_directory(dqn_dir, "DQN")?;

    // Validate PPO checkpoints
    let ppo_dir = Path::new("ml/trained_models/production/ppo_real_data");
    let ppo_reports = validate_directory(ppo_dir, "PPO")?;

    // Print comparison table
    print_comparison_table(&dqn_reports, &ppo_reports);

    // Print final summary
    println!("\n{}", "=".repeat(80));
    println!("FINAL ASSESSMENT");
    println!("{}", "=".repeat(80));

    let dqn_ready = dqn_reports.values().all(|r| r.is_valid());
    let ppo_ready = ppo_reports.values().all(|r| r.is_valid());

    println!("\nDQN Production Ready: {}", if dqn_ready { "✅ YES" } else { "❌ NO" });
    println!("PPO Production Ready: {}", if ppo_ready { "✅ YES" } else { "❌ NO" });

    if !dqn_ready {
        println!("\n⚠️  DQN ISSUE DETECTED:");
        println!("   Root cause: ml/src/trainers/dqn.rs:765");
        println!("   serialize_model() returns vec![0u8; 1024] (placeholder)");
        println!("   Fix required: Implement real SafeTensors serialization like PPO");
        println!("   Reference: ml/src/trainers/ppo.rs:555 (model.actor.vars().save())");
    }

    if ppo_ready {
        println!("\n✅ PPO SUCCESS:");
        println!("   All {} checkpoints valid", ppo_reports.len());
        println!("   Real SafeTensors format with actor/critic networks");
        println!("   Average size: ~42 KB per checkpoint");
        println!("   Ready for production inference");
    }

    Ok(())
}