foxhunt/ml/examples/verify_mamba2_dimensions.rs

//! MAMBA-2 Dimension Verification Script
//!
//! Verifies that MAMBA-2 correctly handles 225 input features with d_state=16
//! This script demonstrates that d_model (input) and d_state (SSM) are independent.

use anyhow::Result;
use candle_core::{Device, Tensor};
use tracing::{info, Level};

use ml::mamba::{Mamba2Config, Mamba2SSM};

fn main() -> Result<()> {
    // Initialize logging
    tracing_subscriber::fmt()
        .with_max_level(Level::INFO)
        .init();

    info!("=== MAMBA-2 Dimension Verification ===");
    info!("");

    // Create MAMBA-2 config with 225 input features and 16 SSM state dimension
    let config = Mamba2Config {
        d_model: 225,        // INPUT: 225 features (Wave C + Wave D)
        d_state: 16,         // SSM: 16-dimensional state space
        d_head: 28,          // 225 / 8 ≈ 28
        num_heads: 8,
        expand: 2,           // d_inner = 225 * 2 = 450
        num_layers: 6,
        dropout: 0.1,
        use_ssd: true,
        use_selective_state: true,
        hardware_aware: true,
        target_latency_us: 5,
        max_seq_len: 128,
        learning_rate: 0.0001,
        weight_decay: 1e-4,
        grad_clip: 1.0,
        warmup_steps: 1000,
        batch_size: 32,
        seq_len: 60,
    };

    info!("Configuration:");
    info!("  d_model (input features): {}", config.d_model);
    info!("  d_state (SSM dimension): {}", config.d_state);
    info!("  d_inner (internal): {} (d_model × expand = {} × {})",
        config.d_model * config.expand, config.d_model, config.expand);
    info!("  num_layers: {}", config.num_layers);
    info!("");

    // Create device (CPU for quick verification)
    let device = Device::Cpu;
    info!("Device: {:?}", device);
    info!("");

    // Create model
    info!("Creating MAMBA-2 model...");
    let mut model = Mamba2SSM::new(config.clone(), &device)
        .map_err(|e| anyhow::anyhow!("Failed to create model: {}", e))?;

    info!("✓ Model created successfully");
    info!("  Parameters: {}", model.metadata.num_parameters);
    info!("  Input dim: {}", model.metadata.input_dim);
    info!("  Output dim: {}", model.metadata.output_dim);
    info!("");

    // Test 1: Single sample inference
    info!("Test 1: Single Sample Inference");
    info!("  Input shape: [1, 60, 225] (batch=1, seq=60, features=225)");

    let batch_size = 1;
    let seq_len = 60;
    let features = 225;

    // Create dummy input data
    let input_data: Vec<f64> = (0..batch_size * seq_len * features)
        .map(|i| (i as f64) * 0.01)
        .collect();

    let input = Tensor::from_vec(
        input_data,
        (batch_size, seq_len, features),
        &device,
    )?;

    info!("  Input tensor created: {:?}", input.dims());

    // Forward pass
    let output = model.forward(&input)
        .map_err(|e| anyhow::anyhow!("Forward pass failed: {}", e))?;

    info!("  Output shape: {:?}", output.dims());
    info!("  ✓ Forward pass successful");
    info!("");

    // Test 2: Batch inference
    info!("Test 2: Batch Inference");
    info!("  Input shape: [32, 60, 225] (batch=32, seq=60, features=225)");

    let batch_size = 32;
    let input_data: Vec<f64> = (0..batch_size * seq_len * features)
        .map(|i| (i as f64) * 0.01)
        .collect();

    let input_batch = Tensor::from_vec(
        input_data,
        (batch_size, seq_len, features),
        &device,
    )?;

    info!("  Input tensor created: {:?}", input_batch.dims());

    let output_batch = model.forward(&input_batch)
        .map_err(|e| anyhow::anyhow!("Batch forward pass failed: {}", e))?;

    info!("  Output shape: {:?}", output_batch.dims());
    info!("  ✓ Batch forward pass successful");
    info!("");

    // Test 3: SSM state verification
    info!("Test 3: SSM State Verification");
    for (i, ssm_state) in model.state.ssm_states.iter().enumerate() {
        info!("  Layer {}:", i);
        info!("    A matrix: {:?} (state transition)", ssm_state.A.dims());
        info!("    B matrix: {:?} (input-to-state)", ssm_state.B.dims());
        info!("    C matrix: {:?} (state-to-output)", ssm_state.C.dims());
        info!("    Delta: {:?} (discretization)", ssm_state.delta.dims());
        info!("    Hidden: {:?} (current state)", ssm_state.hidden.dims());

        // Verify dimensions
        let a_dims = ssm_state.A.dims();
        let b_dims = ssm_state.B.dims();
        let c_dims = ssm_state.C.dims();

        assert_eq!(a_dims[0], config.d_state, "A matrix row dimension mismatch");
        assert_eq!(a_dims[1], config.d_state, "A matrix col dimension mismatch");
        assert_eq!(b_dims[0], config.d_state, "B matrix row dimension mismatch");
        assert_eq!(b_dims[1], config.d_model * config.expand, "B matrix col dimension mismatch");
        assert_eq!(c_dims[0], config.d_model * config.expand, "C matrix row dimension mismatch");
        assert_eq!(c_dims[1], config.d_state, "C matrix col dimension mismatch");
    }
    info!("  ✓ All SSM matrices have correct dimensions");
    info!("");

    // Test 4: Memory estimation
    info!("Test 4: Memory Estimation");
    let d_inner = config.d_model * config.expand;
    let params_per_layer =
        config.d_state * config.d_state + // A matrix
        config.d_state * d_inner +         // B matrix
        d_inner * config.d_state +         // C matrix
        config.d_model;                     // Delta
    let total_ssm_params = params_per_layer * config.num_layers;
    let ssm_memory_mb = (total_ssm_params * 8) as f64 / (1024.0 * 1024.0); // 8 bytes per F64

    info!("  SSM parameters per layer: {}", params_per_layer);
    info!("  Total SSM parameters: {}", total_ssm_params);
    info!("  SSM memory (F64): {:.2} MB", ssm_memory_mb);
    info!("");

    info!("  Comparison with d_state=225:");
    let alt_params_per_layer =
        225 * 225 +      // A matrix
        225 * d_inner +  // B matrix
        d_inner * 225 +  // C matrix
        config.d_model;  // Delta
    let alt_total_params = alt_params_per_layer * config.num_layers;
    let alt_memory_mb = (alt_total_params * 8) as f64 / (1024.0 * 1024.0);

    info!("  Alternative SSM parameters per layer: {}", alt_params_per_layer);
    info!("  Alternative total SSM parameters: {}", alt_total_params);
    info!("  Alternative SSM memory (F64): {:.2} MB", alt_memory_mb);
    info!("  Memory increase: {:.1}x", alt_memory_mb / ssm_memory_mb);
    info!("");

    // Summary
    info!("=== VERIFICATION SUMMARY ===");
    info!("✓ MAMBA-2 correctly handles 225 input features with d_state=16");
    info!("✓ Input projection: [batch, seq, 225] → [batch, seq, 450]");
    info!("✓ SSM processing: 16-dimensional state space");
    info!("✓ Output projection: [batch, seq, 450] → [batch, seq, 1]");
    info!("✓ Memory efficient: {:.2} MB SSM matrices (vs {:.2} MB with d_state=225)",
        ssm_memory_mb, alt_memory_mb);
    info!("");
    info!("CONCLUSION: Configuration is CORRECT and OPTIMAL");

    Ok(())
}