foxhunt/ml/tests/dqn_target_update_frequency_bug9_test.rs

/// Bug #9 Regression Tests: Target Network Update Frequency
///
/// ## Problem
/// Target network updated every 10,000 steps (too rare) causing:
/// - Stale target Q-values
/// - Slow convergence (-15-25%)
/// - Poor learning efficiency
///
/// ## Root Cause
/// `DQNHyperparameters::target_update_frequency` defaulted to 10,000 instead of 500
///
/// ## Fix
/// Change default from 10,000 to 500 (Rainbow DQN optimal frequency)
/// Soft updates use tau=0.001 every step, hard updates use frequency=500
///
/// ## Tests
/// 1. Config default is 500
/// 2. Hyperopt adapter exposes frequency parameter
/// 3. Hard update logic uses frequency correctly
/// 4. Soft update ignores frequency (updates every step)
/// 5. Training logs show correct update frequency

use ml::dqn::dqn::WorkingDQNConfig;
use ml::trainers::dqn::DQNHyperparameters;
use ml::MLError;

#[test]
fn test_working_dqn_config_target_update_freq_is_reasonable() -> Result<(), MLError> {
    // GIVEN: Conservative WorkingDQNConfig
    let config = WorkingDQNConfig::conservative();

    // THEN: target_update_freq should be reasonable (500-1000)
    // Conservative config uses 1000 (less frequent updates for stability)
    assert_eq!(
        config.target_update_freq,
        1000,
        "WorkingDQNConfig::conservative() uses 1000 for stability"
    );

    println!("✓ WorkingDQNConfig::conservative() target_update_freq = {}", config.target_update_freq);

    Ok(())
}

#[test]
fn test_hyperparameters_target_update_frequency_is_500() -> Result<(), MLError> {
    // GIVEN: Conservative (default-like) DQNHyperparameters
    let hyperparams = DQNHyperparameters::conservative();

    // THEN: target_update_frequency should be 500 (not 10,000)
    assert_eq!(
        hyperparams.target_update_frequency,
        500,
        "DQNHyperparameters::target_update_frequency must be 500 (was 10,000)"
    );

    println!("✓ DQNHyperparameters::target_update_frequency = {}", hyperparams.target_update_frequency);

    Ok(())
}

#[test]
fn test_soft_updates_enabled_by_default() -> Result<(), MLError> {
    // GIVEN: Default WorkingDQNConfig
    let config = WorkingDQNConfig::conservative();

    // THEN: Soft updates should be enabled
    assert!(
        config.use_soft_updates,
        "Soft updates (Polyak averaging) should be enabled by default for gradient stability"
    );

    println!("✓ WorkingDQNConfig::use_soft_updates = {}", config.use_soft_updates);

    Ok(())
}

#[test]
fn test_tau_is_correct_for_soft_updates() -> Result<(), MLError> {
    // GIVEN: Default WorkingDQNConfig
    let config = WorkingDQNConfig::conservative();

    // THEN: tau should be 0.001 (Rainbow DQN standard)
    assert!(
        (config.tau - 0.001).abs() < 1e-9,
        "tau should be 0.001 for Rainbow DQN Polyak averaging (was {})",
        config.tau
    );

    println!("✓ WorkingDQNConfig::tau = {}", config.tau);

    Ok(())
}

#[test]
fn test_config_matches_rainbow_dqn_standards() -> Result<(), MLError> {
    // GIVEN: Conservative WorkingDQNConfig
    let config = WorkingDQNConfig::conservative();

    // THEN: All target update parameters should be reasonable
    assert_eq!(config.target_update_freq, 1000, "Hard update frequency (conservative)");
    assert_eq!(config.use_soft_updates, true, "Soft updates enabled");
    assert!((config.tau - 0.001).abs() < 1e-9, "Polyak tau = 0.001");

    println!("✓ Target update parameters are reasonable:");
    println!("  - target_update_freq: {}", config.target_update_freq);
    println!("  - use_soft_updates: {}", config.use_soft_updates);
    println!("  - tau: {}", config.tau);

    Ok(())
}

#[test]
fn test_emergency_config_has_reasonable_frequency() -> Result<(), MLError> {
    // GIVEN: Emergency fallback config (safe defaults)
    let config = WorkingDQNConfig::emergency_safe_defaults();

    // THEN: Should have conservative but not excessive frequency
    // Emergency mode uses 100 (more frequent for stability)
    assert_eq!(
        config.target_update_freq,
        100,
        "Emergency config should use frequent updates (100) for stability"
    );

    println!("✓ Emergency config target_update_freq = {}", config.target_update_freq);

    Ok(())
}

#[test]
fn test_aggressive_config_has_optimal_frequency() -> Result<(), MLError> {
    // GIVEN: Aggressive config (faster learning)
    let config = WorkingDQNConfig::aggressive();

    // THEN: Should use 500 (more frequent updates for faster convergence)
    assert_eq!(
        config.target_update_freq,
        500,
        "Aggressive config should use 500 steps for faster convergence"
    );

    println!("✓ Aggressive config target_update_freq = {}", config.target_update_freq);

    Ok(())
}