foxhunt/crates/ml/tests/dqn_iqn_integration_test.rs

#![allow(
    clippy::assertions_on_constants,
    clippy::assertions_on_result_states,
    clippy::clone_on_copy,
    clippy::decimal_literal_representation,
    clippy::doc_markdown,
    clippy::empty_line_after_doc_comments,
    clippy::field_reassign_with_default,
    clippy::get_unwrap,
    clippy::identity_op,
    clippy::inconsistent_digit_grouping,
    clippy::indexing_slicing,
    clippy::integer_division,
    clippy::len_zero,
    clippy::let_underscore_must_use,
    clippy::manual_div_ceil,
    clippy::manual_let_else,
    clippy::manual_range_contains,
    clippy::modulo_arithmetic,
    clippy::needless_range_loop,
    clippy::non_ascii_literal,
    clippy::redundant_clone,
    clippy::shadow_reuse,
    clippy::shadow_same,
    clippy::shadow_unrelated,
    clippy::single_match_else,
    clippy::str_to_string,
    clippy::string_slice,
    clippy::tests_outside_test_module,
    clippy::too_many_lines,
    clippy::unnecessary_wraps,
    clippy::unseparated_literal_suffix,
    clippy::use_debug,
    clippy::useless_vec,
    clippy::wildcard_enum_match_arm,
    clippy::else_if_without_else,
    clippy::expect_used,
    clippy::missing_const_for_fn,
    clippy::similar_names,
    clippy::type_complexity,
    clippy::collapsible_else_if,
    clippy::doc_lazy_continuation,
    clippy::items_after_test_module,
    clippy::map_clone,
    clippy::multiple_unsafe_ops_per_block,
    clippy::unwrap_or_default,
    clippy::assign_op_pattern,
    clippy::needless_borrow,
    clippy::println_empty_string,
    clippy::unnecessary_cast,
    clippy::used_underscore_binding,
    clippy::create_dir,
    clippy::implicit_saturating_sub,
    clippy::exit,
    clippy::expect_fun_call,
    clippy::too_many_arguments,
    clippy::unnecessary_map_or,
    clippy::unwrap_used,
    dead_code,
    unused_imports,
    unused_variables,
    clippy::cloned_ref_to_slice_refs,
    clippy::neg_multiply,
    clippy::while_let_loop,
    clippy::bool_assert_comparison,
    clippy::excessive_precision,
    clippy::trivially_copy_pass_by_ref,
    clippy::op_ref,
    clippy::redundant_closure,
    clippy::unnecessary_lazy_evaluations,
    clippy::if_then_some_else_none,
    clippy::unnecessary_to_owned,
    clippy::single_component_path_imports,
)]
//! Integration test: DQN with IQN distributional RL + CQL offline regularization
//!
//! Verifies the complete training loop with 2026 modernization features:
//! - IQN replaces broken C51 (no scatter_add needed)
//! - CQL provides offline RL regularization
//! - CVaR enables risk-aware action selection

use ml::dqn::{DQNConfig, DQN, Experience};

#[test]
fn test_full_iqn_cql_training_loop() {
    // Configure DQN with IQN + CQL (2026 modernization)
    let mut config = DQNConfig::default();
    config.state_dim = 8;
    config.num_actions = 3;
    config.hidden_dims = vec![32, 16];
    config.use_iqn = true;
    config.iqn_num_quantiles = 16;
    config.cql_alpha = 1.0;
    config.batch_size = 8;
    config.min_replay_size = 8;
    config.warmup_steps = 0;
    config.epsilon_start = 0.5;

    let mut dqn = DQN::new(config).unwrap();

    // Collect experiences via action selection
    for i in 0..20 {
        let state: Vec<f32> = (0..8).map(|j| (i * 8 + j) as f32 / 160.0).collect();
        let action = dqn.select_action(&state).unwrap();
        let reward = if i % 2 == 0 { 1.0 } else { -0.5 };
        let next_state: Vec<f32> = (0..8).map(|j| ((i + 1) * 8 + j) as f32 / 160.0).collect();

        let exp = Experience::new(
            state,
            action.to_index() as u8,
            reward,
            next_state,
            i == 19,
        );
        dqn.store_experience(exp).unwrap();
    }

    // Run 5 training steps
    let stream = dqn.cuda_stream().clone();
    let mut losses = Vec::new();
    for _ in 0..5 {
        let result = dqn.train_step(None);
        assert!(result.is_ok(), "Training step failed: {:?}", result.err());
        let gpu_result = result.unwrap();
        let loss = gpu_result.loss_gpu.to_scalar(&stream).unwrap();
        let grad_norm = gpu_result.grad_norm_gpu.to_scalar(&stream).unwrap();
        assert!(loss.is_finite(), "Loss is not finite: {}", loss);
        assert!(grad_norm.is_finite(), "Grad norm is not finite: {}", grad_norm);
        losses.push(loss);
    }

    // Verify loss is non-zero (model is actually learning)
    assert!(losses.iter().any(|l| *l > 0.0), "All losses are zero — model not learning");
}

#[test]
fn test_iqn_only_no_cql() {
    let mut config = DQNConfig::default();
    config.state_dim = 8;
    config.num_actions = 3;
    config.hidden_dims = vec![16, 16];
    config.use_iqn = true;
    config.iqn_num_quantiles = 8;
    config.batch_size = 4;
    config.min_replay_size = 4;
    config.warmup_steps = 0;

    let mut dqn = DQN::new(config).unwrap();

    for i in 0..10 {
        let exp = Experience::new(
            vec![0.1 * i as f32; 8],
            (i % 3) as u8,
            0.5,
            vec![0.2 * i as f32; 8],
            false,
        );
        dqn.store_experience(exp).unwrap();
    }

    let result = dqn.train_step(None);
    assert!(result.is_ok(), "IQN-only training should succeed: {:?}", result.err());
}

#[test]
fn test_cvar_action_selection_integration() {
    let mut config = DQNConfig::default();
    config.state_dim = 8;
    config.num_actions = 3;
    config.hidden_dims = vec![16, 16];
    config.use_iqn = true;
    config.iqn_num_quantiles = 8;
    config.epsilon_start = 0.0;
    config.warmup_steps = 0;
    config.use_cvar_action_selection = true;
    config.cvar_alpha = 0.05;

    let mut dqn = DQN::new(config).unwrap();

    // CVaR action selection should select more conservatively
    let state = vec![0.5f32; 8];
    let action = dqn.select_action(&state);
    assert!(action.is_ok(), "CVaR action selection should work: {:?}", action.err());
}