foxhunt/ml/examples/test_dqn_init.rs

//! Test DQN Initialization Non-Determinism
//!
//! Creates a DQN model and prints initial Q-values to verify
//! that network weights are randomly initialized (not deterministic).
//!
//! # Usage
//!
//! ```bash
//! # Run 3 times and compare Q-values
//! cargo run -p ml --example test_dqn_init --release --features cuda
//! cargo run -p ml --example test_dqn_init --release --features cuda
//! cargo run -p ml --example test_dqn_init --release --features cuda
//! ```

use anyhow::Result;
use candle_core::{Device, Tensor};
use ml::dqn::{RewardSystem, WorkingDQN, WorkingDQNConfig};

fn main() -> Result<()> {
    // Initialize tracing
    tracing_subscriber::fmt()
        .with_max_level(tracing::Level::DEBUG)
        .init();

    println!("=== DQN Initialization Test ===\n");

    // Create DQN config
    let config = WorkingDQNConfig {
        state_dim: 128,
        hidden_dims: vec![256, 128, 64],
        num_actions: 3,
        learning_rate: 0.0001,
        gamma: 0.99,
        epsilon_start: 1.0,
        epsilon_end: 0.05,
        epsilon_decay: 0.995,
        replay_buffer_capacity: 10000,
        batch_size: 32,
        min_replay_size: 1000,
        target_update_freq: 10000,
        use_double_dqn: true,
        use_huber_loss: false,
        huber_delta: 1.0,
        gradient_clip_norm: 10.0,
        leaky_relu_alpha: 0.01,
        tau: 0.001,
        use_soft_updates: false,
        warmup_steps: 1000,
        temperature_start: 1.0,
        temperature_min: 0.1,
        temperature_decay: 0.995,
        target_temperature_fraction: 0.75,
        variance_multiplier: 0.5,
        use_adaptive_temperature: false,
        loss_improvement_threshold: 0.999,
        plateau_window: 10,
        temp_increase_factor: 1.05,
        temperature_slow_decay: 0.998,
        reward_system: RewardSystem::Elite,
    };

    println!("Creating DQN model...");
    let dqn = WorkingDQN::new(config)?;
    println!("✓ DQN model created\n");

    // Create a test state (all zeros)
    let device = dqn.device();
    let test_state = Tensor::zeros((1, 128), candle_core::DType::F32, device)?;

    println!("Computing initial Q-values for zero state...");
    let q_values = dqn.forward(&test_state)?;

    // Extract Q-values
    let q_vec = q_values.squeeze(0)?.to_vec1::<f32>()?;

    println!("\n=== INITIAL Q-VALUES (Step 0) ===");
    println!("  BUY (Action 0):  {:+.6}", q_vec[0]);
    println!("  SELL (Action 1): {:+.6}", q_vec[1]);
    println!("  HOLD (Action 2): {:+.6}", q_vec[2]);
    println!("\n=== Q-Value Differences ===");
    println!("  HOLD - BUY:  {:+.6}", q_vec[2] - q_vec[0]);
    println!("  HOLD - SELL: {:+.6}", q_vec[2] - q_vec[1]);
    println!("  BUY - SELL:  {:+.6}", q_vec[0] - q_vec[1]);

    // Check for deterministic initialization (209% HOLD bias)
    let hold_bias = (q_vec[2] - q_vec[0]) / q_vec[0].abs();
    println!("\n=== Bias Analysis ===");
    println!("  HOLD bias: {:.1}%", hold_bias * 100.0);

    if hold_bias.abs() > 1.5 {
        println!(
            "  ⚠️  WARNING: Large HOLD bias detected (>{:.0}%)",
            hold_bias.abs() * 100.0
        );
    } else {
        println!("  ✓ HOLD bias within acceptable range (<150%)");
    }

    println!("\n=== VALIDATION ===");
    println!("Run this example 3 times in parallel:");
    println!("  cargo run -p ml --example test_dqn_init --release --features cuda &");
    println!("  cargo run -p ml --example test_dqn_init --release --features cuda &");
    println!("  cargo run -p ml --example test_dqn_init --release --features cuda &");
    println!("  wait");
    println!("\nSUCCESS: If Q-values are DIFFERENT across runs");
    println!("FAILURE: If Q-values are IDENTICAL across runs");

    Ok(())
}