c645e6222d
CRITICAL FINDINGS from 3-trial validation: - 85,120 gradient clipping warnings (81.6% of logs) - REGRESSION - Rainbow features DISABLED: use_dueling=false, use_distributional=false, use_noisy_nets=false - Negative Q-values confirmed: HOLD -1000 to -3250 - Performance: Sharpe 0.29 (target 0.77) Changes: - Fixed N-Step compilation (7/7 tests passing) - Fixed Distributional compilation (6/6 tests passing) - Fixed Dueling CUDA errors (10/10 tests passing) - Added TDD validation for state_dim=225 - Total: 23/23 Wave 11 tests passing (100%) Issues requiring investigation: 1. Why are Dueling/Distributional/Noisy disabled in hyperopt? 2. Why gradient explosion despite previous fixes? 3. Test coverage gaps - unit tests pass but integration fails 🤖 Generated with Claude Code Co-Authored-By: Claude <noreply@anthropic.com>
110 lines
3.7 KiB
Rust
110 lines
3.7 KiB
Rust
//! Test DQN Initialization Non-Determinism
|
|
//!
|
|
//! Creates a DQN model and prints initial Q-values to verify
|
|
//! that network weights are randomly initialized (not deterministic).
|
|
//!
|
|
//! # Usage
|
|
//!
|
|
//! ```bash
|
|
//! # Run 3 times and compare Q-values
|
|
//! cargo run -p ml --example test_dqn_init --release --features cuda
|
|
//! cargo run -p ml --example test_dqn_init --release --features cuda
|
|
//! cargo run -p ml --example test_dqn_init --release --features cuda
|
|
//! ```
|
|
|
|
use anyhow::Result;
|
|
use candle_core::{Device, Tensor};
|
|
use ml::dqn::{RewardSystem, WorkingDQN, WorkingDQNConfig};
|
|
|
|
fn main() -> Result<()> {
|
|
// Initialize tracing
|
|
tracing_subscriber::fmt()
|
|
.with_max_level(tracing::Level::DEBUG)
|
|
.init();
|
|
|
|
println!("=== DQN Initialization Test ===\n");
|
|
|
|
// Create DQN config
|
|
let config = WorkingDQNConfig {
|
|
state_dim: 225, // Wave 6.1: Updated to match Migration 045
|
|
hidden_dims: vec![256, 128, 64],
|
|
num_actions: 3,
|
|
learning_rate: 0.0001,
|
|
gamma: 0.99,
|
|
epsilon_start: 1.0,
|
|
epsilon_end: 0.05,
|
|
epsilon_decay: 0.995,
|
|
replay_buffer_capacity: 10000,
|
|
batch_size: 32,
|
|
min_replay_size: 1000,
|
|
target_update_freq: 10000,
|
|
use_double_dqn: true,
|
|
use_huber_loss: false,
|
|
huber_delta: 1.0,
|
|
gradient_clip_norm: 10.0,
|
|
leaky_relu_alpha: 0.01,
|
|
tau: 0.001,
|
|
use_soft_updates: false,
|
|
warmup_steps: 1000,
|
|
temperature_start: 1.0,
|
|
temperature_min: 0.1,
|
|
temperature_decay: 0.995,
|
|
target_temperature_fraction: 0.75,
|
|
variance_multiplier: 0.5,
|
|
use_adaptive_temperature: false,
|
|
loss_improvement_threshold: 0.999,
|
|
plateau_window: 10,
|
|
temp_increase_factor: 1.05,
|
|
temperature_slow_decay: 0.998,
|
|
reward_system: RewardSystem::Elite,
|
|
};
|
|
|
|
println!("Creating DQN model...");
|
|
let dqn = WorkingDQN::new(config)?;
|
|
println!("✓ DQN model created\n");
|
|
|
|
// Create a test state (all zeros)
|
|
let device = dqn.device();
|
|
let test_state = Tensor::zeros((1, 128), candle_core::DType::F32, device)?;
|
|
|
|
println!("Computing initial Q-values for zero state...");
|
|
let q_values = dqn.forward(&test_state)?;
|
|
|
|
// Extract Q-values
|
|
let q_vec = q_values.squeeze(0)?.to_vec1::<f32>()?;
|
|
|
|
println!("\n=== INITIAL Q-VALUES (Step 0) ===");
|
|
println!(" BUY (Action 0): {:+.6}", q_vec[0]);
|
|
println!(" SELL (Action 1): {:+.6}", q_vec[1]);
|
|
println!(" HOLD (Action 2): {:+.6}", q_vec[2]);
|
|
println!("\n=== Q-Value Differences ===");
|
|
println!(" HOLD - BUY: {:+.6}", q_vec[2] - q_vec[0]);
|
|
println!(" HOLD - SELL: {:+.6}", q_vec[2] - q_vec[1]);
|
|
println!(" BUY - SELL: {:+.6}", q_vec[0] - q_vec[1]);
|
|
|
|
// Check for deterministic initialization (209% HOLD bias)
|
|
let hold_bias = (q_vec[2] - q_vec[0]) / q_vec[0].abs();
|
|
println!("\n=== Bias Analysis ===");
|
|
println!(" HOLD bias: {:.1}%", hold_bias * 100.0);
|
|
|
|
if hold_bias.abs() > 1.5 {
|
|
println!(
|
|
" ⚠️ WARNING: Large HOLD bias detected (>{:.0}%)",
|
|
hold_bias.abs() * 100.0
|
|
);
|
|
} else {
|
|
println!(" ✓ HOLD bias within acceptable range (<150%)");
|
|
}
|
|
|
|
println!("\n=== VALIDATION ===");
|
|
println!("Run this example 3 times in parallel:");
|
|
println!(" cargo run -p ml --example test_dqn_init --release --features cuda &");
|
|
println!(" cargo run -p ml --example test_dqn_init --release --features cuda &");
|
|
println!(" cargo run -p ml --example test_dqn_init --release --features cuda &");
|
|
println!(" wait");
|
|
println!("\nSUCCESS: If Q-values are DIFFERENT across runs");
|
|
println!("FAILURE: If Q-values are IDENTICAL across runs");
|
|
|
|
Ok(())
|
|
}
|