#![allow(unsafe_code)] // Required for f32<->u32 CudaSlice reinterpret casts and safetensors serialization
//! DQN Configuration and Hyperparameters
//!
//! Contains all training configuration including learning rates,
//! batch sizes, exploration parameters, and risk management settings.

use ml_core::device::MlDevice;
use ml_core::cuda_autograd::GpuTensor;
use ml_core::cuda_autograd::GpuVarStore;

use crate::dqn::action_space::FactoredAction;
use crate::dqn::dqn::DQN;
use crate::dqn::regime_conditional::{RegimeConditionalDQN, RegimeMetrics, RegimeType};
use crate::dqn::Experience;
use crate::MLError;

/// Polymorphic DQN agent container supporting both standard and regime-conditional architectures.
///
/// Provides unified API for agent operations regardless of underlying architecture.
/// Allows switching between single-head (standard) and multi-head (regime-conditional)
/// Q-networks via configuration without code duplication.
pub enum DQNAgentType {
    /// Standard single-head Q-network
    Standard(DQN),
    /// Regime-conditional multi-head Q-network (3 heads: Trending, Ranging, Volatile)
    RegimeConditional(RegimeConditionalDQN),
}

impl std::fmt::Debug for DQNAgentType {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        match self {
            Self::Standard(_) => f.debug_tuple("DQNAgentType::Standard").finish(),
            Self::RegimeConditional(_) => f.debug_tuple("DQNAgentType::RegimeConditional").finish(),
        }
    }
}

impl DQNAgentType {
    /// Select action using appropriate agent type
    pub fn select_action(&mut self, state: &[f32]) -> Result<FactoredAction, MLError> {
        match self {
            Self::Standard(agent) => agent.select_action(state),
            Self::RegimeConditional(agent) => agent.select_action(state),
        }
    }

    /// Store experience in replay buffer
    pub fn store_experience(&self, experience: Experience) -> Result<(), MLError> {
        match self {
            Self::Standard(agent) => {
                agent.memory.add(experience)?;
                Ok(())
            }
            Self::RegimeConditional(agent) => agent.store_experience(experience),
        }
    }

    /// Training step — returns [`GpuTrainResult`] (zero CPU readback).
    ///
    /// Loss and grad norm stay as GPU rank-0 tensors. The caller accumulates
    /// on GPU and reads back once per epoch.
    pub fn train_step(&mut self, batch: Option<crate::dqn::replay_buffer_type::BatchSample>) -> Result<crate::dqn::dqn::GpuTrainResult, MLError> {
        match self {
            Self::Standard(agent) => agent.train_step(batch),
            Self::RegimeConditional(agent) => agent.train_step(batch),
        }
    }

    /// Batch greedy action selection — single forward pass per architecture head.
    ///
    /// Dispatches to the underlying DQN or RegimeConditionalDQN batch method.
    pub fn batch_greedy_actions(&self, states: &GpuTensor) -> Result<GpuTensor, MLError> {
        match self {
            Self::Standard(agent) => agent.batch_greedy_actions(states),
            Self::RegimeConditional(agent) => {
                let indices = agent.batch_greedy_actions(states)?;
                let f32_vals: Vec<f32> = indices.iter().map(|&v| v as f32).collect();
                let stream = agent.get_stream();
                GpuTensor::from_host(&f32_vals, vec![f32_vals.len()], stream)
            }
        }
    }

    /// Batch Q-value computation — returns raw Q-values tensor `[batch, num_actions]`.
    ///
    /// Used by `GpuBacktestEvaluator` for GPU-side argmax.
    pub fn batch_q_values(&self, states: &GpuTensor) -> Result<GpuTensor, MLError> {
        match self {
            Self::Standard(agent) => agent.q_values_for_batch(states),
            Self::RegimeConditional(agent) => agent.batch_q_values(states),
        }
    }

    /// Per-branch Q-values for branching DQN — regime-blended when using RegimeConditional.
    ///
    /// Returns `Some((exposure [batch,5], order [batch,3], urgency [batch,3]))` when
    /// branching is active, `None` otherwise. Used by the GPU action selector's
    /// `select_actions_branching()` for per-branch epsilon-greedy exploration.
    pub fn batch_branching_q_values(
        &self,
        states: &GpuTensor,
    ) -> Result<Option<(GpuTensor, GpuTensor, GpuTensor)>, MLError> {
        match self {
            Self::Standard(agent) => {
                if !true {
                    return Ok(None);
                }
                let branching_net = agent.branching_q_network.as_ref().ok_or_else(|| {
                    MLError::ModelError("Branching enabled but network not initialized".into())
                })?;
                let output = branching_net.forward_branches_eval(states)?;
                if output.advantages.len() < 3 {
                    return Err(MLError::ModelError(format!(
                        "Branching produced {} branches, expected 3",
                        output.advantages.len()
                    )));
                }
                Ok(Some((
                    output.advantages[0].clone(),
                    output.advantages[1].clone(),
                    output.advantages[2].clone(),
                )))
            }
            Self::RegimeConditional(agent) => agent.batch_branching_q_values(states),
        }
    }

    /// Batch softmax (Boltzmann) action selection -- dispatches to underlying DQN variant.
    pub fn batch_softmax_actions(
        &self,
        states: &GpuTensor,
        temperature: f64,
    ) -> Result<GpuTensor, MLError> {
        match self {
            Self::Standard(agent) => agent.batch_softmax_actions(states, temperature),
            Self::RegimeConditional(agent) => {
                let indices = agent.batch_softmax_actions(states, temperature)?;
                let f32_vals: Vec<f32> = indices.iter().map(|&v| v as f32).collect();
                let stream = agent.get_stream();
                GpuTensor::from_host(&f32_vals, vec![f32_vals.len()], stream)
            }
        }
    }

    /// Hierarchical factored softmax -- exposure-first, then order/urgency within.
    pub fn batch_hierarchical_softmax_actions(
        &self,
        states: &GpuTensor,
        temperature: f64,
    ) -> Result<GpuTensor, MLError> {
        match self {
            Self::Standard(agent) => {
                agent.batch_hierarchical_softmax_actions(states, temperature)
            }
            Self::RegimeConditional(agent) => {
                let indices = agent.batch_hierarchical_softmax_actions(states, temperature)?;
                let f32_vals: Vec<f32> = indices.iter().map(|&v| v as f32).collect();
                let stream = agent.get_stream();
                GpuTensor::from_host(&f32_vals, vec![f32_vals.len()], stream)
            }
        }
    }

    /// Set noise sigma scale for annealed noisy exploration
    pub fn set_noise_sigma_scale(&mut self, scale: f64) {
        match self {
            Self::Standard(agent) => agent.set_noise_sigma_scale(scale),
            Self::RegimeConditional(agent) => agent.set_noise_sigma_scale(scale),
        }
    }

    /// Reset count-based exploration bonus (call at epoch boundary)
    pub fn reset_count_bonus(&mut self) {
        match self {
            Self::Standard(agent) => agent.reset_count_bonus(),
            Self::RegimeConditional(agent) => agent.reset_count_bonus(),
        }
    }

    /// Get count bonuses for all actions (UCB exploration).
    /// Returns a Vec<f32> of length num_actions with exploration bonuses.
    pub fn get_count_bonuses(&self) -> Vec<f32> {
        match self {
            Self::Standard(agent) => agent.get_count_bonuses(),
            Self::RegimeConditional(agent) => agent.get_count_bonuses(),
        }
    }

    /// Get per-branch count bonuses for branching DQN (exposure [5], order [3], urgency [3]).
    pub fn get_count_bonuses_branched(&self) -> Option<(Vec<f64>, Vec<f64>, Vec<f64>)> {
        match self {
            Self::Standard(agent) => agent.get_count_bonuses_branched(),
            Self::RegimeConditional(_) => None, // RegimeConditional uses flat bonuses
        }
    }

    /// Whether count-based exploration bonus is enabled.
    pub fn use_count_bonus(&self) -> bool {
        match self {
            Self::Standard(agent) => agent.config.use_count_bonus,
            Self::RegimeConditional(agent) => agent.config().use_count_bonus,
        }
    }

    /// Number of discrete actions in the action space.
    pub fn num_actions(&self) -> usize {
        match self {
            Self::Standard(agent) => agent.config.num_actions,
            Self::RegimeConditional(agent) => agent.config().num_actions,
        }
    }

    /// Branch sizes for branching DQN: `(exposure, order_type, urgency)`.
    ///
    /// Returns the actual branch sizes from the DQN config. For non-branching
    /// DQN, all three branches equal `num_actions`.
    pub fn branch_sizes(&self) -> (usize, usize, usize) {
        match self {
            Self::Standard(agent) => (
                agent.config.num_actions,
                agent.config.num_order_types,
                agent.config.num_urgency_levels,
            ),
            Self::RegimeConditional(agent) => {
                let cfg = agent.config();
                (cfg.num_actions, cfg.num_order_types, cfg.num_urgency_levels)
            }
        }
    }

    /// Update epsilon for exploration decay
    pub fn update_epsilon(&mut self) {
        match self {
            Self::Standard(agent) => agent.update_epsilon(),
            Self::RegimeConditional(agent) => {
                // Update epsilon for all regime heads
                agent.update_epsilon(RegimeType::Trending);
                agent.update_epsilon(RegimeType::Ranging);
                agent.update_epsilon(RegimeType::Volatile);
            }
        }
    }

    /// Get current epsilon value
    pub fn get_epsilon(&self) -> f32 {
        match self {
            Self::Standard(agent) => agent.get_epsilon(),
            Self::RegimeConditional(agent) => {
                // Return trending head epsilon as representative value
                agent.get_epsilon(RegimeType::Trending)
            }
        }
    }

    /// Get the effective epsilon for exploration, respecting noisy_epsilon_floor.
    /// The batch training path must use this instead of get_epsilon() to avoid
    /// zero exploration when noisy nets are enabled.
    pub fn get_effective_epsilon(&self) -> f32 {
        match self {
            Self::Standard(agent) => agent.get_effective_epsilon(),
            Self::RegimeConditional(agent) => agent.primary_head().get_effective_epsilon(),
        }
    }

    /// Set epsilon value for exploration (all heads for regime-conditional)
    pub fn set_epsilon(&mut self, epsilon: f64) {
        match self {
            Self::Standard(agent) => agent.set_epsilon(epsilon),
            Self::RegimeConditional(agent) => {
                // BUG #40 FIX: Set epsilon on ALL regime heads
                agent.set_epsilon_all(epsilon);
            }
        }
    }

    /// Save checkpoint to disk with architecture metadata.
    ///
    /// Downloads GpuVarStore parameters to host and serializes via safetensors.
    pub fn save_checkpoint(&self, path: &str) -> Result<(), MLError> {
        match self {
            Self::Standard(agent) => {
                // Download parameters to host for safetensors serialization
                let vars = agent.get_q_network_vars();
                let stream = vars.stream();
                let mut host_tensors: std::collections::HashMap<String, Vec<f32>> =
                    std::collections::HashMap::new();
                let mut shapes: std::collections::HashMap<String, Vec<usize>> =
                    std::collections::HashMap::new();
                for (name, param) in vars.iter() {
                    let tensor = GpuTensor::new(param.data.clone(), param.shape.clone())
                        .map_err(|e| MLError::CheckpointError(format!("Checkpoint tensor '{name}': {e}")))?;
                    let host_data = tensor.to_host(stream)
                        .map_err(|e| MLError::CheckpointError(format!("Checkpoint DtoH '{name}': {e}")))?;
                    shapes.insert(name.to_string(), param.shape.clone());
                    host_tensors.insert(name.to_string(), host_data);
                }
                // Build safetensors-compatible data (raw bytes)
                let mut st_data = std::collections::HashMap::new();
                for (name, data) in &host_tensors {
                    let shape = shapes.get(name).cloned().unwrap_or_default();
                    // Safety: f32 slice to u8 slice -- same alignment, 4x length
                    let bytes: &[u8] = unsafe {
                        std::slice::from_raw_parts(
                            data.as_ptr() as *const u8,
                            data.len() * std::mem::size_of::<f32>(),
                        )
                    };
                    st_data.insert(
                        name.clone(),
                        safetensors::tensor::TensorView::new(
                            safetensors::Dtype::F32,
                            shape,
                            bytes,
                        ).map_err(|e| MLError::CheckpointError(format!("TensorView '{name}': {e}")))?,
                    );
                }
                let arch_metadata = agent.config.checkpoint_metadata();
                safetensors::serialize_to_file(st_data, Some(arch_metadata), std::path::Path::new(path))
                    .map_err(|e| MLError::CheckpointError(format!("Failed to save checkpoint: {e}")))?;
                Ok(())
            }
            Self::RegimeConditional(agent) => agent.save_checkpoint(path),
        }
    }

    /// Get regime-specific metrics (None for standard agent)
    pub fn get_regime_metrics(&self) -> Option<&std::collections::HashMap<RegimeType, RegimeMetrics>> {
        match self {
            Self::Standard(_) => None,
            Self::RegimeConditional(agent) => Some(agent.get_regime_metrics()),
        }
    }

    /// Get replay buffer size
    pub fn get_replay_buffer_size(&self) -> Result<usize, MLError> {
        match self {
            Self::Standard(agent) => agent.get_replay_buffer_size(),
            Self::RegimeConditional(agent) => agent.get_replay_buffer_size(),
        }
    }

    /// Forward pass through Q-network.
    ///
    /// For `RegimeConditional`, uses GPU regime classification masks to blend
    /// Q-values from all 3 heads — matching `batch_q_values()` semantics.
    /// Previously hard-coded to trending head only, causing ranging/volatile
    /// heads to be ignored during experience collection.
    pub fn forward(&self, state: &GpuTensor) -> Result<GpuTensor, MLError> {
        match self {
            Self::Standard(agent) => agent.forward(state),
            Self::RegimeConditional(agent) => agent.batch_q_values(state),
        }
    }

    /// Track action for diversity monitoring (only for standard agent)
    pub fn track_action(&mut self, action: FactoredAction) {
        match self {
            Self::Standard(agent) => agent.track_action(action),
            Self::RegimeConditional(_) => {
                // Regime-conditional agent doesn't use track_action
                // Action tracking happens via regime metrics instead
            }
        }
    }

    /// Track a batch of actions for diversity monitoring.
    ///
    /// Batched variant that allows the caller to acquire the write lock once
    /// and push all actions in a single critical section, reducing per-sample
    /// async RwLock overhead from O(n) to O(1).
    pub fn track_actions_batch(&mut self, actions: &[FactoredAction]) {
        match self {
            Self::Standard(agent) => agent.track_actions_batch(actions),
            Self::RegimeConditional(_) => {
                // Regime-conditional agent doesn't use track_action
            }
        }
    }

    /// Store a batch of experiences in the replay buffer with a single lock acquisition.
    ///
    /// For Standard agents this delegates to `ReplayBufferType::add_batch` which
    /// holds the inner parking_lot Mutex once for the entire batch.
    /// For RegimeConditional agents experiences are cloned to all three head buffers.
    pub fn store_experiences_batch(&self, experiences: Vec<Experience>) -> Result<(), MLError> {
        match self {
            Self::Standard(agent) => {
                agent.memory.add_batch(experiences)?;
                Ok(())
            }
            Self::RegimeConditional(agent) => {
                for exp in experiences {
                    agent.store_experience(exp)?;
                }
                Ok(())
            }
        }
    }



    /// Insert a batch of experience tensors directly into the GPU replay buffer.
    ///
    /// Feeds tensors straight to `GpuReplayBuffer::insert_batch()` -- zero CPU intermediaries.
    /// CPU replay buffer fallback is a hard error when cuda is enabled.
    ///
    /// Note: `actions` GpuTensor stores f32 values that represent u32 action indices.
    /// The underlying `insert_batch` API expects `CudaSlice<u32>`, so we reinterpret
    /// the f32 data as u32 (valid for action indices in [0, 44]).
    pub fn insert_batch_tensors(
        &self,
        states: &GpuTensor,
        next_states: &GpuTensor,
        actions: &GpuTensor,
        rewards: &GpuTensor,
        dones: &GpuTensor,
    ) -> Result<(), MLError> {
        // Convert Tensors to CudaSlices for the raw GPU PER insert API
        let batch_size = states.shape().first().copied().unwrap_or(0);
        let s_slice = states.data();
        let ns_slice = next_states.data();
        // actions are stored as f32 in GpuTensor but insert_batch wants u32.
        // Reinterpret the CudaSlice<f32> as CudaSlice<u32> (bit-compatible for small ints).
        let a_f32_slice = actions.data();
        let r_slice = rewards.data();
        let d_slice = dones.data();

        match self {
            Self::Standard(agent) => {
                let mut gpu_buf = agent.memory.as_gpu_buffer()
                    .ok_or_else(|| MLError::TrainingError(
                        "CPU replay buffer fallback disabled -- use GpuPrioritized when cuda is enabled".to_owned()
                    ))?;
                // Safety: f32 and u32 are same size. Action values [0..44] are
                // integers stored as f32 then bitwise reinterpreted as u32.
                let a_slice: &cudarc::driver::CudaSlice<u32> = unsafe {
                    &*(a_f32_slice as *const cudarc::driver::CudaSlice<f32>
                        as *const cudarc::driver::CudaSlice<u32>)
                };
                gpu_buf.gpu.insert_batch(s_slice, ns_slice, a_slice, r_slice, d_slice, batch_size)
            }
            Self::RegimeConditional(agent) => {
                let a_slice: &cudarc::driver::CudaSlice<u32> = unsafe {
                    &*(a_f32_slice as *const cudarc::driver::CudaSlice<f32>
                        as *const cudarc::driver::CudaSlice<u32>)
                };
                macro_rules! insert_head {
                    ($getter:ident) => {
                        if let Some(head) = agent.$getter() {
                            let mut gpu_buf = head.memory.as_gpu_buffer()
                                .ok_or_else(|| MLError::TrainingError(
                                    "CPU replay buffer fallback disabled -- use GpuPrioritized when cuda is enabled".to_owned()
                                ))?;
                            gpu_buf.gpu.insert_batch(s_slice, ns_slice, a_slice, r_slice, d_slice, batch_size)?;
                        }
                    };
                }

                insert_head!(get_trending_head);
                insert_head!(get_ranging_head);
                insert_head!(get_volatile_head);
                Ok(())
            }
        }
    }

    /// Check if agent can train (has enough replay buffer samples)
    pub fn can_train(&self) -> bool {
        match self {
            Self::Standard(agent) => agent.can_train(),
            Self::RegimeConditional(agent) => {
                // Check if replay buffer has minimum samples
                // Use 100 as min_replay_size (same as in regime_conditional.rs train_step)
                if let Ok(buffer) = agent.get_replay_buffer_size() {
                    buffer >= 100
                } else {
                    false
                }
            }
        }
    }

    /// Get reference to replay buffer memory
    ///
    /// For RegimeConditionalDQN, returns the trending head's buffer as representative sample.
    pub fn memory(&self) -> &crate::dqn::replay_buffer_type::ReplayBufferType {
        match self {
            Self::Standard(agent) => &agent.memory,
            Self::RegimeConditional(agent) => {
                // Regime conditional DQN has separate buffers per head
                // Return trending head's buffer as representative sample for Q-value monitoring
                agent.get_trending_head_memory()
            }
        }
    }

    /// WAVE 23 P0: Log diagnostics and check for gradient collapse (early stopping)
    /// Returns Err if gradient collapse detected for consecutive epochs
    pub fn log_diagnostics(&mut self, grad_norm: f32) -> Result<(), MLError> {
        match self {
            Self::Standard(agent) => agent.log_diagnostics(grad_norm),
            Self::RegimeConditional(agent) => agent.log_diagnostics(grad_norm),
        }
    }

    /// Gradient collapse check only -- no dead neuron detection (zero GPU->CPU sync).
    /// Use in CUDA guard hot path; dead neuron detection runs at epoch boundary.
    pub fn check_gradient_collapse(&mut self, grad_norm: f32) -> Result<(), MLError> {
        match self {
            Self::Standard(agent) => agent.check_gradient_collapse(grad_norm),
            Self::RegimeConditional(agent) => agent.check_gradient_collapse(grad_norm),
        }
    }

    /// Suppress or enable forward() monitoring (GPU→CPU sync avoidance in hot path).
    pub fn set_training_forward_active(&mut self, active: bool) {
        match self {
            Self::Standard(agent) => agent.set_training_forward_active(active),
            Self::RegimeConditional(_) => {}
        }
    }

    /// WAVE 23 P0: Log Q-values and check for Q-value divergence (early stopping)
    /// Returns Err if Q-value divergence detected for consecutive checks
    pub fn log_q_values(&mut self, states_tensor: &GpuTensor) -> Result<(), MLError> {
        match self {
            Self::Standard(agent) => agent.log_q_values(states_tensor),
            Self::RegimeConditional(_agent) => {
                // Regime-conditional doesn't implement Q-value divergence detection yet
                // Skip for now (can add in future)
                Ok(())
            }
        }
    }

    /// Log Q-values from pre-computed GPU statistics (zero tensor readback).
    /// Returns Err if Q-value divergence detected for consecutive checks.
    pub fn log_q_values_from_stats(
        &mut self,
        q_min: f32,
        q_max: f32,
        q_mean: f32,
        q_variance: f32,
        n_actions: usize,
    ) -> Result<(), MLError> {
        match self {
            Self::Standard(agent) => {
                agent.log_q_values_from_stats(q_min, q_max, q_mean, q_variance, n_actions)
            }
            Self::RegimeConditional(_agent) => {
                // Regime-conditional doesn't implement Q-value divergence detection yet
                Ok(())
            }
        }
    }

    /// Get device (CPU or CUDA)
    pub fn device(&self) -> MlDevice {
        match self {
            Self::Standard(agent) => {
                let stream = agent.cuda_stream();
                MlDevice::Cuda {
                    context: stream.context().clone(),
                    stream: std::sync::Arc::clone(stream),
                }
            }
            Self::RegimeConditional(agent) => agent.get_device(),
        }
    }

    /// Get Q-network variables for checkpoint saving
    pub fn get_q_network_vars(&self) -> &GpuVarStore {
        match self {
            Self::Standard(agent) => agent.get_q_network_vars(),
            Self::RegimeConditional(agent) => {
                // For regime-conditional, return trending head vars as representative
                agent.get_trending_head()
                    .map(|head| head.get_q_network_vars())
                    .unwrap_or_else(|| agent.primary_head().get_q_network_vars())
            }
        }
    }

    /// Build safetensors checkpoint metadata from the agent's config.
    pub fn checkpoint_metadata(&self) -> std::collections::HashMap<String, String> {
        match self {
            Self::Standard(agent) => agent.config.checkpoint_metadata(),
            Self::RegimeConditional(agent) => {
                // Use trending head config as representative for regime-conditional
                match agent.get_trending_head() {
                    Some(head) => head.config.checkpoint_metadata(),
                    None => std::collections::HashMap::new(),
                }
            }
        }
    }

    /// Get mutable reference to underlying standard agent (for methods not in unified API)
    pub fn as_standard_mut(&mut self) -> Option<&mut DQN> {
        match self {
            Self::Standard(agent) => Some(agent),
            Self::RegimeConditional(_) => None,
        }
    }

    /// Get mutable reference to the primary DQN head (works for both Standard and RegimeConditional).
    /// For Standard: returns the single DQN. For RegimeConditional: returns the primary head.
    pub fn primary_dqn_mut(&mut self) -> &mut DQN {
        match self {
            Self::Standard(agent) => agent,
            Self::RegimeConditional(regime) => regime.primary_head_mut(),
        }
    }

    /// Get state dimension from agent configuration
    ///
    /// WAVE 10.4: Added to fix hardcoded STATE_DIM bug
    /// Returns the actual state dimension (45 without OFI: 42 market + 3 portfolio, or 53 with OFI: +8 OFI)
    pub fn get_state_dim(&self) -> usize {
        match self {
            Self::Standard(agent) => agent.get_state_dim(),
            Self::RegimeConditional(agent) => agent.get_state_dim(),
        }
    }

    /// BUG #38 FIX: Clear replay buffer(s)
    pub fn clear_replay_buffer(&mut self) -> Result<(), MLError> {
        match self {
            Self::Standard(agent) => agent.clear_replay_buffer(),
            Self::RegimeConditional(agent) => agent.clear_replay_buffer(),
        }
    }

    /// BUG #38 FIX: Reset target network(s) to match main Q-network(s)
    pub fn reset_target_network(&mut self) -> Result<(), MLError> {
        match self {
            Self::Standard(agent) => agent.reset_target_network(),
            Self::RegimeConditional(agent) => agent.reset_target_network(),
        }
    }

    /// Compute gradients without stepping the optimizer (for true gradient accumulation).
    /// Only supported for Standard DQN. RegimeConditional returns an error.
    pub fn compute_gradients(
        &mut self,
        batch: Option<crate::dqn::replay_buffer_type::BatchSample>,
    ) -> Result<crate::dqn::dqn::GradientResult, MLError> {
        match self {
            Self::Standard(agent) => agent.compute_gradients(batch),
            Self::RegimeConditional(agent) => agent.compute_gradients(batch),
        }
    }

    /// Apply pre-accumulated gradients in a single optimizer step.
    pub fn apply_accumulated_gradients(
        &mut self,
        grads: &std::collections::BTreeMap<String, GpuTensor>,
    ) -> Result<(), MLError> {
        match self {
            Self::Standard(agent) => agent.apply_accumulated_gradients(grads),
            Self::RegimeConditional(agent) => agent.apply_accumulated_gradients(grads),
        }
    }

    /// Get optimizer variable references (for gradient accumulation merging).
    ///
    /// Returns `Err` -- the fused CUDA trainer and DQN use their own flat parameter buffers.
    /// Callers should use `apply_accumulated_gradients()` instead.
    pub fn optimizer_vars(&self) -> Result<Vec<GpuTensor>, MLError> {
        Err(MLError::TrainingError(
            "optimizer_vars not available -- use apply_accumulated_gradients".to_owned()
        ))
    }

    /// Update replay buffer priorities with new TD errors (PER only, no-op for uniform).
    pub fn update_priorities(&self, indices: &[usize], td_errors: &[f32]) -> Result<(), MLError> {
        self.memory().update_priorities(indices, td_errors)
    }

    /// Update replay buffer priorities from GPU-resident tensors (GpuPrioritized only).
    pub fn update_priorities_gpu(
        &self,
        indices: &GpuTensor,
        td_errors: &GpuTensor,
    ) -> Result<(), MLError> {
        self.memory().update_priorities_gpu(indices, td_errors)
    }

    /// Step the replay buffer training counter for beta annealing (PER only).
    pub fn step_replay_buffer(&self) {
        self.memory().step();
    }

    /// Post-step bookkeeping for the fused CUDA training path.
    ///
    /// Delegates to `DQN::fused_post_step` -- increments training_steps, updates
    /// PER priorities, steps beta annealing, and runs Polyak target update.
    /// Fused post-step: update priorities, increment steps, update target networks.
    pub fn fused_post_step(&mut self, td_errors: &[f32], indices: &[usize]) -> Result<(), crate::MLError> {
        self.primary_dqn_mut().fused_post_step(td_errors, indices)
    }

    /// Post-step bookkeeping WITHOUT target EMA (for GPU-native EMA path).
    pub fn fused_post_step_no_ema(&mut self, td_errors: &[f32], indices: &[usize]) -> Result<(), crate::MLError> {
        self.primary_dqn_mut().fused_post_step_no_ema(td_errors, indices)
    }

    /// Flush GPU-accumulated max priority to CPU (single scalar readback per epoch).
    pub fn flush_max_priority(&self) -> Result<(), crate::MLError> {
        self.memory().flush_max_priority()
    }

    /// GPU-only post-step bookkeeping: training_steps++ and PER beta annealing.
    pub fn fused_post_step_gpu_bookkeeping(&mut self) -> Result<(), crate::MLError> {
        let dqn = self.primary_dqn_mut();
        dqn.increment_training_steps();
        dqn.memory.step();
        Ok(())
    }

    /// PER alpha/epsilon for GPU-native priority update kernel.
    pub fn per_alpha_epsilon(&self) -> Option<(f32, f32)> {
        self.memory().per_alpha_epsilon()
    }

    /// Direct reference to the GPU-resident priorities tensor.
    ///
    /// Returns `Ok(None)` for non-GPU buffers. Used by fused training to pass
    /// the tensor to `GpuDqnTrainer::update_priorities_cuda()`.
    pub fn priorities_tensor(&self) -> Result<Option<GpuTensor>, crate::MLError> {
        Ok(self.memory().priorities_tensor())
    }

    /// Set max_priority from a CPU scalar (epoch-boundary flush from CUDA kernel).
    pub fn apply_max_priority_scalar(&self, max_prio: f32) -> Result<(), crate::MLError> {
        self.memory().apply_max_priority_scalar(max_prio)
    }

    /// Update learning rate for the optimizer(s) by applying a decay factor.
    ///
    /// For Standard agents, updates the single optimizer.
    /// For RegimeConditional agents, updates all three regime head optimizers.
    pub fn update_learning_rate(&mut self, decay_factor: f64) -> Result<(), crate::MLError> {
        match self {
            Self::Standard(agent) => agent.update_learning_rate(decay_factor),
            Self::RegimeConditional(agent) => agent.update_learning_rate(decay_factor),
        }
    }

    /// Whether the active agent architecture uses Branching DQN (3 independent heads).
    ///
    /// Determines which weight extraction function to use for pure-CUDA forward:
    /// - `true`  → `extract_dueling_weights_branching()` (reads `branch_0_fc.*` keys)
    /// - `false` → `extract_dueling_weights()`           (reads `advantage_fc.*` keys)
    pub fn is_using_branching(&self) -> bool {
        match self {
            Self::Standard(agent) => true && agent.branching_q_network.is_some(),
            Self::RegimeConditional(agent) => {
                agent.get_trending_head()
                    .is_some_and(|h| h.branching_q_network.is_some())
            }
        }
    }

    /// Network dimensions for pure-CUDA forward kernel: `(shared_h1, shared_h2, value_h, adv_h)`.
    ///
    /// These are injected as `#define` constants into the NVRTC-compiled forward kernel.
    /// The CUDA kernel assumes exactly 2 shared layers (matching DQN's `hidden_dims[0..2]`).
    pub fn network_dims(&self) -> (usize, usize, usize, usize) {
        match self {
            Self::Standard(agent) => {
                let h = &agent.config.hidden_dims;
                let shared_h1 = h.first().copied().unwrap_or(256);
                let shared_h2 = h.get(1).copied().unwrap_or(shared_h1);
                let value_h = agent.config.dueling_hidden_dim;
                let adv_h = if true {
                    agent.config.branch_hidden_dim
                } else {
                    agent.config.dueling_hidden_dim
                };
                (shared_h1, shared_h2, value_h, adv_h)
            }
            Self::RegimeConditional(agent) => {
                // Use trending head config as representative
                match agent.get_trending_head() {
                    Some(head) => {
                        let h = &head.config.hidden_dims;
                        let shared_h1 = h.first().copied().unwrap_or(256);
                        let shared_h2 = h.get(1).copied().unwrap_or(shared_h1);
                        let value_h = head.config.dueling_hidden_dim;
                        let adv_h = if true {
                            head.config.branch_hidden_dim
                        } else {
                            head.config.dueling_hidden_dim
                        };
                        (shared_h1, shared_h2, value_h, adv_h)
                    }
                    None => (256, 256, 128, 128), // safe defaults
                }
            }
        }
    }
}

/// DQN training hyperparameters from gRPC request
#[derive(Debug, Clone)]
pub struct DQNHyperparameters {
    /// Learning rate (typically 1e-4 to 1e-3)
    pub learning_rate: f64,
    /// Batch size (dynamically capped by AutoBatchSizer based on GPU VRAM)
    pub batch_size: usize,
    /// Discount factor (typically 0.95-0.99)
    pub gamma: f64,
    /// Initial exploration rate
    pub epsilon_start: f64,
    /// Final exploration rate
    pub epsilon_end: f64,
    /// Exploration decay rate
    pub epsilon_decay: f64,
    /// Replay buffer capacity
    pub buffer_size: usize,
    /// Fraction of free VRAM to allocate for replay buffer (0.0 = use static buffer_size).
    /// When > 0.0, overrides buffer_size with VRAM-optimal capacity.
    /// Typical: 0.70 (70% of free VRAM after model load).
    pub replay_buffer_vram_fraction: f64,
    /// Minimum replay buffer size before training starts
    pub min_replay_size: usize,
    /// Number of training epochs
    pub epochs: usize,
    /// Checkpoint save frequency (epochs)
    pub checkpoint_frequency: usize,
    /// Enable early stopping based on convergence criteria
    pub early_stopping_enabled: bool,
    /// Minimum Q-value threshold before stopping (default: 0.5)
    pub q_value_floor: f64,
    /// Minimum loss improvement percentage over window (default: 2.0%)
    pub min_loss_improvement_pct: f64,
    /// Window size for plateau detection (default: 30 epochs)
    pub plateau_window: usize,
    /// Minimum epochs before early stopping can trigger (default: 50)
    pub min_epochs_before_stopping: usize,
    /// Deprecated: replaced by `w_idle` in composite reward. Kept for hyperopt adapter compat.
    pub hold_penalty: f64,

    // GPU composite reward weights (8-component composite in CUDA kernel)
    /// DSR (Sharpe) weight for composite reward
    pub w_dsr: f64,
    /// Normalized PnL weight for composite reward
    pub w_pnl: f64,
    /// Drawdown penalty weight for composite reward
    pub w_dd: f64,
    /// Idle penalty weight for composite reward
    pub w_idle: f64,
    /// Drawdown tolerance before penalty kicks in (fraction, e.g. 0.02 = 2%)
    pub dd_threshold: f64,
    /// Asymmetric loss scaling factor (prospect theory, default 1.5)
    pub loss_aversion: f64,
    /// Position staleness rent per step held
    pub time_decay_rate: f64,
    /// Q-value gap threshold for trade conviction filter.
    /// When greedy Q(best_exposure) - Q(flat) < threshold, default to flat.
    /// 0.0 = disabled. Typical range [0.0, 0.5].
    pub q_gap_threshold: f64,
    /// Huber loss delta threshold (default: 1.0)
    pub huber_delta: f64,
    /// Gradient clipping max norm (None = disabled)
    pub gradient_clip_norm: Option<f64>,
    /// Deprecated: was HOLD action penalty weight, replaced by `w_idle` in GPU composite reward.
    /// Kept for backward compatibility with hyperopt adapters. Not used in reward computation.
    pub hold_penalty_weight: f64,
    /// Price movement threshold for HOLD penalty (as fraction, e.g., 0.02 = 2%)
    pub movement_threshold: f64,
    /// Enable preprocessing (log returns + normalization + outlier clipping)
    pub enable_preprocessing: bool,
    /// Preprocessing window size (default: 50)
    pub preprocessing_window: i64,
    /// Preprocessing clip sigma (default: 5.0)
    pub preprocessing_clip_sigma: f64,

    // WAVE 16 (Agent 36): Target update configuration
    /// Polyak averaging coefficient for soft target updates (default: 0.001)
    /// Rainbow DQN standard: τ=0.001 gives 693-step convergence half-life
    pub tau: f64,
    /// Target update mode: Soft (Polyak averaging) or Hard (periodic full copy)
    pub target_update_mode: crate::trainers::TargetUpdateMode,
    /// Target network hard update frequency in training steps (default: 10000)
    /// Used when target_update_mode = Hard. Rainbow DQN: 32K frames, Stable Baselines3: 10K steps
    pub target_update_frequency: usize,

    // Rainbow DQN warmup period
    /// Warmup steps for random exploration (Rainbow DQN standard: 80K for 50M+ steps)
    /// For short training (<200K steps), warmup=0 is recommended.
    /// Adaptive CLI defaults: 0 (<200K), 5% (200K-500K), 8% (500K-1M), 80K (>1M)
    pub warmup_steps: usize,

    // P2-A Enhancement: Initial capital for portfolio
    /// Initial capital for portfolio trading (default: $100,000)
    /// Minimum: $1,000 (validated at CLI layer)
    pub initial_capital: f32,

    /// Bars per trading day at current data resolution.
    /// 390 = 1-minute (6.5h × 60min), 78 = 5-minute, 1 = daily.
    /// Used for Sharpe/Sortino annualization and return scaling.
    /// Derived from data pipeline's BarSize when available.
    pub bars_per_day: f64,

    /// Minimum bars to hold a position before allowing exit or reversal.
    /// Prevents per-bar churning. Trailing stop can override after this period.
    /// Default: 5 (about 5 minutes for 1-min bars). Range: [2, 20].
    pub min_hold_bars: usize,

    // P2-B Enhancement: Cash reserve requirement
    /// Cash reserve requirement as percentage of portfolio value (0-100)
    /// Default: 0.0 (no reserve, backward compatible)
    pub cash_reserve_percent: f64,

    // WAVE 16S: Adaptive Risk Management Features
    /// Enable Kelly criterion position sizing
    pub enable_kelly_sizing: bool,
    /// Enable volatility-adjusted epsilon exploration
    pub enable_volatility_epsilon: bool,
    /// Enable risk-adjusted rewards (Sharpe ratio)
    pub enable_risk_adjusted_rewards: bool,
    /// Kelly fractional multiplier (0.5 = half-Kelly, conservative)
    pub kelly_fractional: f64,
    /// Maximum Kelly fraction (cap at 0.25 = 25% of portfolio)
    pub kelly_max_fraction: f64,
    /// Minimum trades required for Kelly calculation
    pub kelly_min_trades: usize,
    /// Volatility rolling window size
    pub volatility_window: usize,

    // WAVE 35: Advanced Features - Regime-Conditional Q-Networks + Compliance Engine
    /// Enable regime-conditional Q-network (3 heads: Trending, Ranging, Volatile)
    pub enable_regime_qnetwork: bool,
    /// Enable compliance engine (real-time regulatory validation)
    pub enable_compliance: bool,

    // WAVE 16: Core Risk Management Features
    /// Enable drawdown monitoring (15% max drawdown early stop)
    pub enable_drawdown_monitoring: bool,
    /// Enable position limits (3-tier: absolute ±10.0, notional $1M, concentration 10%)
    pub enable_position_limits: bool,
    /// Enable circuit breaker (5-failure trip mechanism)
    pub enable_circuit_breaker: bool,

    // Wave 16 Portfolio Features
    /// Enable action masking (filters invalid actions based on position limits)
    pub enable_action_masking: bool,
    /// Enable entropy regularization (prevents policy collapse)
    pub enable_entropy_regularization: bool,
    /// Enable stress testing (robustness validation)
    pub enable_stress_testing: bool,
    /// Maximum absolute position size for action masking (1.0-10.0 contracts)
    /// Default: 2.0 (matches current production behavior)
    pub max_position_absolute: f64,

    // WAVE 17: Hyperopt-tuned parameters
    /// SAC-style entropy coefficient for Q-value softmax regularization (default: 0.01)
    pub entropy_coefficient: Option<f64>,
    /// Minimum epsilon when noisy nets are enabled (default: 0.05)
    pub noisy_epsilon_floor: Option<f64>,
    /// Count-based exploration bonus coefficient (default: 0.1, range: [0.01, 0.5])
    pub count_bonus_coefficient: Option<f64>,
    /// Transaction cost multiplier for reward calculation
    pub transaction_cost_multiplier: f64,

    // Wave 3 (Phase 2): Triple Barrier Method
    /// Enable triple barrier method for multi-step reward labeling
    pub enable_triple_barrier: bool,
    pub triple_barrier_profit_target_bps: u32,
    pub triple_barrier_stop_loss_bps: u32,
    pub triple_barrier_max_holding_seconds: u64,

    // P0: Prioritized Experience Replay (always enabled)
    pub per_alpha: f64,
    pub per_beta_start: f64,

    // Wave 2.1: Dueling Networks (always enabled)
    /// Hidden dimension for dueling value/advantage streams
    pub dueling_hidden_dim: usize,

    // Wave 2.2: Multi-Step Returns (N-step TD)
    /// Number of steps for n-step returns (1-10, default: 3 for Rainbow DQN)
    /// Recommended: 3-5 for balance between bias and variance
    pub n_steps: usize,

    // Wave 2.3: Distributional RL (C51) — always enabled
    /// Number of atoms for value distribution (Rainbow DQN standard: 51)
    pub num_atoms: usize,
    /// Minimum value for distribution support (must be -v_max for symmetric support)
    pub v_min: f64,
    /// Maximum value for distribution support (v_min = -v_max enforced by hyperopt)
    pub v_max: f64,

    /// Reward scale factor for v_range computation.
    /// Q* ~ reward_scale / (1 - gamma). With 20% headroom and [20, 300] clamp:
    /// v_range = (reward_scale / (1 - gamma) * 1.2).clamp(20.0, 300.0)
    /// Default: 10.0 (reward v6 REWARD_SCALE).
    pub reward_scale: f64,

    // Wave 2.4: Noisy Networks for Exploration (always enabled)
    /// Initial noise std dev (Rainbow DQN standard: 0.5, scaled by 1/√in_features)
    pub noisy_sigma_init: f64,

    // Two-Phase Feature Normalization Configuration
    /// Ratio of total epochs to use for feature statistics collection (default: 0.3 = 30%)
    /// Phase 1 collects statistics for this percentage of training
    /// Example: 100 epochs * 0.3 = 30 epochs for stats collection
    pub feature_stats_collection_ratio: f32,
    /// Maximum number of epochs for stats collection (default: Some(10))
    /// Acts as a cap: min(epochs * ratio, max_epochs)
    /// Example: 100 epochs → min(30, 10) = 10; 200 epochs → min(60, 10) = 10
    /// Set to None for no cap (pure percentage-based)
    pub max_feature_stats_epochs: Option<usize>,

    // WAVE 23 P0: Early Stopping for Gradient Collapse
    /// Adaptive gradient collapse threshold multiplier (default: 100.0)
    /// Threshold = learning_rate × gradient_collapse_multiplier
    /// WAVE 23: Replaces hardcoded 0.1 threshold with learning-rate aware detection
    pub gradient_collapse_multiplier: f64,
    /// Consecutive epoch patience before early stopping (default: 5)
    /// Prevents false positives from single-epoch anomalies
    pub gradient_collapse_patience: usize,

    // WAVE 26 P0.6: Learning Rate Scheduling with Warmup
    // NOTE: warmup_steps is defined above (line 331) - using same field for both purposes
    /// Learning rate decay type after warmup
    pub lr_decay_type: super::lr_scheduler::LRDecayType,
    /// Minimum learning rate for Cosine decay
    pub min_learning_rate: f64,
    /// Minimum learning rate floor (absolute minimum, default: 1e-6)
    pub lr_min: f64,
    /// Steps for learning rate decay (default: 1000)
    pub lr_decay_steps: usize,
    /// Learning rate decay rate (default: 0.99)
    pub lr_decay_rate: f64,

    // WAVE 26 P1.4: Ensemble Uncertainty for Exploration (6D hyperopt expansion)
    /// Enable ensemble uncertainty-based exploration (replaces noisy networks)
    pub use_ensemble_uncertainty: bool,
    /// Number of Q-network heads in ensemble (3-10, default: 5)
    pub ensemble_size: usize,
    /// Variance penalty weight for uncertainty estimation (0.1-1.0)
    pub beta_variance: f64,
    /// Disagreement penalty weight for epistemic uncertainty (0.1-1.0)
    pub beta_disagreement: f64,
    /// Entropy bonus weight for exploration diversity (0.05-0.5)
    pub beta_entropy: f64,
    /// Variance cap to prevent over-penalization (0.1-2.0)
    pub variance_cap: f64,

    // WAVE 26 P1.8: Curiosity-Driven Exploration
    /// Curiosity weight for intrinsic reward (0.0 = disabled, typical range: 0.0-0.5)
    /// Balances extrinsic reward from trading vs intrinsic reward from novelty
    pub curiosity_weight: f64,

    // WAVE 26 P2.2: Gradient Accumulation
    /// Number of mini-batches to accumulate gradients over before optimizer step
    /// Default: 1 (no accumulation, standard training)
    /// Effective batch size = batch_size × gradient_accumulation_steps
    /// Example: batch_size=64, accumulation_steps=4 → effective_batch=256
    /// Benefits: Larger effective batch size without GPU memory constraints
    pub gradient_accumulation_steps: usize,

    // WAVE 26 P1: Advanced DQN Features Integration
    // P1.1: Spectral Normalization (in network config - see DQNConfig)
    // P1.2: Self-Attention (in network config - see DQNConfig)

    // P1.3: Sharpe Ratio Reward Component
    /// Enable Sharpe ratio as reward component (0.0 = disabled, typical: 0.2-0.5)
    pub sharpe_weight: f64,
    /// Rolling window size for Sharpe calculation (default: 20)
    pub sharpe_window: usize,

    // P1.6: Adaptive Dropout Scheduling
    /// Enable adaptive dropout scheduler (reduces dropout rate over training)
    pub enable_dropout_scheduler: bool,
    /// Initial dropout rate (default: 0.5)
    pub dropout_initial: f64,
    /// Final dropout rate (default: 0.1)
    pub dropout_final: f64,
    /// Steps for dropout annealing (default: 10000)
    pub dropout_anneal_steps: usize,

    // P1.7: Hindsight Experience Replay (HER)
    /// Enable HER for improved data efficiency (0.0 = disabled, typical: 0.3-0.8)
    pub her_ratio: f64,
    /// HER strategy: "final" or "future"
    pub her_strategy: String,

    // P1.8: Curiosity-Driven Exploration (already has curiosity_weight above)
    // Uses existing curiosity_weight parameter

    // P1.9: Generalized Advantage Estimation (GAE)
    /// Enable GAE for lower variance returns
    pub enable_gae: bool,
    /// GAE lambda parameter (default: 0.95)
    pub gae_lambda: f64,

    // P1.11: Noisy Network Sigma Scheduling
    /// Enable noisy sigma scheduler (anneals noise over training)
    pub enable_noisy_sigma_scheduler: bool,
    /// Initial sigma for noisy networks (default: 0.6)
    pub noisy_sigma_initial: f64,
    /// Final sigma for noisy networks (default: 0.4)
    pub noisy_sigma_final: f64,
    /// Steps for sigma annealing (default: 10000)
    pub noisy_sigma_anneal_steps: usize,

    // WAVE 30: L2 Weight Decay for Overfitting Prevention
    /// L2 weight decay for AdamW optimizer (default: 1e-4, range: [1e-5, 1e-3])
    /// Prevents overfitting by penalizing large weights via L2 regularization.
    /// Recommended: 1e-4 for standard training, 1e-3 for aggressive regularization.
    pub weight_decay: f64,

    // Conservative Q-Learning (CQL) — always enabled, alpha controls strength
    /// CQL regularization strength (default: 0.1, range 0.0-1.0)
    /// 0.0 = disabled, 0.1 = mild conservatism, 1.0 = full offline-RL strength
    pub cql_alpha: f64,

    // QR-DQN (Quantile Regression DQN) -- replaces disabled C51
    /// Enable QR-DQN (IQN) for distributional RL
    /// Number of quantiles for QR-DQN (32-200)
    pub num_quantiles: usize,
    /// Kappa for quantile Huber loss (0.5-2.0)
    pub qr_kappa: f64,
    /// Lambda weight for IQN quantile loss relative to C51 loss in the dual-head ensemble.
    /// L_total = L_c51 + iqn_lambda * L_iqn
    /// Range [0.0, 2.0]: 0.0 = C51 only, 0.5 = balanced, 1.0 = equal weight
    pub iqn_lambda: f64,
    /// Spectral norm σ_max — constrains ||W||_σ ≤ σ_max.
    /// Range [1.0, 10.0]. Default 3.0 (permits Xavier scaling, prevents Q-explosion).
    pub spectral_norm_sigma_max: f64,
    /// When true, IQN's bounded Huber loss is used for PER priorities instead of
    /// C51's unbounded cross-entropy. Prevents the PER feedback explosion.

    // Phase 3: GPU experience collection kernel configuration
    /// Enable the NVRTC-compiled GPU experience collection kernel.
    /// When false, experience collection uses the batched CPU path while
    /// the training step (forward/backward/optimizer) still runs on CUDA.
    /// Number of parallel episodes per GPU kernel launch (default: 128, scaled dynamically)
    /// Higher values improve GPU utilization on larger GPUs (e.g. H100: 8192)
    pub gpu_n_episodes: usize,
    /// Timesteps per episode in GPU kernel (default: 500, max: 1000)
    /// Higher values collect more experience per launch at the cost of VRAM
    pub gpu_timesteps_per_episode: usize,
    /// Average bid-ask spread for GPU portfolio simulation (default: 0.0001 = 1bp)
    pub avg_spread: f64,

    /// Maximum training steps per epoch (0 = unlimited, uses full dataset).
    /// Caps `training_data.len() / batch_size` to reduce epoch walltime.
    /// CI/smoke: set to 50-100 for fast validation without full dataset sweeps.
    /// Production: leave at 0 for full-dataset training.
    pub max_training_steps_per_epoch: usize,

    /// Hidden dimension base for GPU-dynamic network sizing.
    /// None = use default [256, 128, 64]. Some(base) = [base, base/2, base/4].
    pub hidden_dim_base: Option<usize>,

    /// Minimum profit factor for trade execution (BUG #7 fix)
    /// Trades must have profit > cost × this factor to be executed.
    /// Range [1.1, 2.0]: 1.1 = 10% margin above costs, 2.0 = 100% margin.
    pub minimum_profit_factor: f64,

    // Differential Sharpe Ratio (DSR) — always enabled in GPU composite reward kernel.
    /// Deprecated: DSR is always enabled. Kept for backward compatibility with hyperopt adapters.
    pub use_dsr: bool,
    /// DSR EMA decay rate (0.001=slow, 0.05=fast). Tunable in hyperopt.
    pub dsr_eta: f64,

    // OFI (Order Flow Imbalance) features from MBP-10 data
    /// Directory containing MBP-10 .dbn.zst files for OFI feature extraction.
    /// When Some, 8 OFI features are computed and appended to the state (state_dim += 8).
    /// When None, no OFI features are used (backward compatible).
    pub mbp10_data_dir: Option<String>,
    /// Directory containing trade .dbn.zst files for real VPIN/Kyle's Lambda.
    /// When Some, trades are loaded and fed to `OFICalculator::feed_trade()`.
    /// When None, VPIN/Kyle's Lambda use tick-rule proxy (backward compatible).
    pub trades_data_dir: Option<String>,

    // Offline RL mode (CQL/IQL on fixed datasets)
    /// Offline RL mode: skip online experience collection, train from a fixed dataset.
    /// Requires `dataset_path` to point to a pre-collected experience dataset.
    pub offline_mode: bool,
    /// Path to a pre-collected experience dataset (bincode format).
    /// Used with `offline_mode` to load a fixed dataset into the replay buffer.
    pub dataset_path: Option<String>,

    // IQL (Implicit Q-Learning) -- Kostrikov et al. 2021
    /// IQL expectile parameter tau (0.5-0.95, higher = more optimistic value estimates).
    /// Controls the asymmetry of the value function loss: tau > 0.5 penalizes
    /// underestimation more, effectively extracting the value of the best
    /// in-distribution action.
    pub iql_expectile_tau: f32,
    /// IQL advantage temperature beta (1.0-10.0, higher = more greedy policy).
    /// Controls how sharply the extracted policy concentrates on high-advantage
    /// actions during advantage-weighted regression.
    pub iql_advantage_temperature: f32,
    /// Enable IQL mode (Implicit Q-Learning for offline RL).
    /// When enabled, trains a separate value network V(s) with expectile regression
    /// and extracts the policy via advantage-weighted regression. Complementary to CQL.
    pub use_iql: bool,

    // Fill simulation parameters for GPU experience collector
    /// IoC base fill probability (default 0.85, range [0.5, 1.0])
    pub fill_ioc_fill_prob: f64,
    /// Minimum limit fill probability at wide spread (default 0.30, range [0.1, 0.5])
    pub fill_limit_fill_min: f64,
    /// Maximum limit fill probability at tight spread (default 0.80, range [0.5, 0.95])
    pub fill_limit_fill_max: f64,
    /// Fraction of spread paid by market orders (default 0.50, range [0.2, 0.8])
    pub fill_spread_cost_frac: f64,
    /// Fraction of spread captured by limit orders (default 0.50, range [0.2, 0.8])
    pub fill_spread_capture_frac: f64,

    // Q-value clipping for GPU experience collector
    /// Minimum Q-value clamp in GPU kernel (default -200.0)
    pub q_clip_min: f64,
    /// Maximum Q-value clamp in GPU kernel (default 200.0)
    pub q_clip_max: f64,

    // Phase C+: Branching DQN (Tavakoli et al., 2018) — always enabled
    /// Hidden dimension for each branching advantage head (64-256, step=64).
    /// Controls capacity of the per-dimension advantage streams in Branching DQN.
    /// Only used when `use_branching` is true.
    pub branch_hidden_dim: usize,

    // GPU Walk-Forward: expanding-window cross-validation entirely on GPU
    /// Enable GPU-resident walk-forward evaluation.
    /// When true, the entire dataset is uploaded to GPU VRAM once and fold
    /// transitions use index ranges (zero re-upload). Requires CUDA.
    pub enable_gpu_walk_forward: bool,
    /// Initial training window as a fraction of total data (default: 0.5).
    pub wf_initial_train_fraction: f64,
    /// Validation window as a fraction of total data (default: 0.125).
    pub wf_val_fraction: f64,
    /// Test window as a fraction of total data (default: 0.125).
    pub wf_test_fraction: f64,
    /// Step size between folds as a fraction of total data (default: 0.125).
    pub wf_step_fraction: f64,

    // Ensemble Agent Configuration
    /// Number of agents in ensemble (default: 1 = no ensemble).
    /// When > 1, trains multiple independent DQN agents and combines their
    /// Q-values for more robust action selection with uncertainty estimation.
    pub ensemble_count: usize,
    /// KL-divergence diversity loss weight for ensemble training (default: 0.01).
    /// Encourages each agent in the ensemble to learn different strategies
    /// by penalizing Q-value distribution similarity between ensemble members.
    pub ensemble_diversity_weight: f64,

    // Curriculum learning: difficulty-based walk-forward window filtering
    /// Curriculum phase for difficulty-based window filtering.
    /// 0 = Easy (ADX>30 windows only), 1 = Mixed (trending weighted 2x), 2 = Full (equal weight).
    /// Default: 2 (Full -- no filtering, backward compatible).
    pub curriculum_phase: usize,

    // Expert demonstration generation for imitation learning warmup
    /// Fraction of each training batch sampled from expert demonstrations (0.0-1.0).
    /// 0.0 = no demonstrations (pure RL), 0.5 = half batch from expert.
    /// Default: 0.0 (disabled, backward compatible).
    pub expert_demo_ratio: f64,
    /// Number of epochs over which expert_demo_ratio decays linearly to 0.
    /// After this many epochs, training is pure RL.
    /// Default: 20.
    pub expert_demo_decay_epochs: usize,

    // GPU Multi-Head Feature Attention
    /// Enable GPU multi-head feature attention over h_s2 (post-graph, 1-step lag).
    /// When true, 4-head self-attention is applied to the shared trunk output
    /// after each CUDA Graph replay, modifying save_h_s2 for the next step.
    /// Default: false.
    pub use_attention: bool,

    // C51 distributional loss warmup
    /// Number of epochs to use MSE loss on expected Q-values before switching to C51 cross-entropy.
    /// MSE provides stronger gradient signal during early training when C51 cross-entropy between
    /// nearly-identical distributions produces near-zero gradient for the value mean.
    /// Default: 5. Set to 0 to use C51 from the start.
    pub c51_warmup_epochs: usize,

    // Decision Transformer pre-training
    /// Decision Transformer pre-training epochs (0 = disabled).
    /// When > 0, runs DT pre-training on offline trajectories before the main DQN loop.
    /// The DT kernels and pretrain_step() are implemented in decision_transformer.rs.
    pub dt_pretrain_epochs: usize,
    /// DT context length (number of past timesteps per sequence).
    pub dt_context_len: usize,
    /// DT embedding dimension for states, actions, and returns-to-go.
    pub dt_embed_dim: usize,
    /// DT number of transformer layers.
    pub dt_num_layers: usize,
    /// DT target return-to-go for conditioning (in units of cumulative reward).
    pub dt_target_return: f64,

    // CVaR (Conditional Value at Risk) action selection for IQN
    /// Enable CVaR-aware action selection when IQN is active.
    /// When true, actions are selected by optimizing the worst-case quantile tail
    /// (risk-averse) instead of the mean quantile (risk-neutral).
    /// Default: true (enables risk-aware position scaling via IQN head).
    pub use_cvar_action_selection: bool,
    /// CVaR confidence level alpha (0.0-1.0). Lower alpha = more risk-averse.
    /// 0.05 = optimize for worst 5% outcomes (extreme caution).
    /// 0.25 = optimize for worst 25% outcomes (moderate risk aversion).
    /// Default: 0.05.
    pub cvar_alpha: f32,
}

impl Default for DQNHyperparameters {
    /// Default implementation uses conservative hyperparameters.
    /// WARNING: These are NOT optimized for production. Use hyperopt results instead.
    fn default() -> Self {
        Self::conservative()
    }
}

impl DQNHyperparameters {
    /// Compute v_min from reward_scale and gamma.
    /// Q* ~ reward_scale / (1 - gamma). With 20% headroom, clamped to [20, 300].
    pub fn computed_v_min(&self) -> f64 {
        let v_range = (self.reward_scale / (1.0 - self.gamma) * 1.2).clamp(20.0, 300.0);
        -v_range
    }

    /// Compute v_max from reward_scale and gamma.
    /// Q* ~ reward_scale / (1 - gamma). With 20% headroom, clamped to [20, 300].
    pub fn computed_v_max(&self) -> f64 {
        (self.reward_scale / (1.0 - self.gamma) * 1.2).clamp(20.0, 300.0)
    }

    /// Create conservative hyperparameters suitable for testing and development.
    /// WARNING: These are NOT optimized for production. Use hyperopt results instead.
    /// After DQN hyperopt completes, update ml/hyperparams/dqn_best.toml with optimal values.
    pub fn conservative() -> Self {
        Self {
            learning_rate: 0.00003,
            batch_size: 1024,  // H100 80GB target; AutoBatchSizer caps for smaller GPUs
            gamma: 0.95,
            epsilon_start: 1.0,
            epsilon_end: 0.01,
            epsilon_decay: 0.995,
            buffer_size: 500000,  // WAVE 24: Increased from 100K for better sample diversity (anti-overfitting)
            replay_buffer_vram_fraction: 0.70,
            min_replay_size: 1000,
            epochs: 100,
            checkpoint_frequency: 10,
            early_stopping_enabled: true,
            q_value_floor: -5.0,  // Wave 3 fix: allow normal negative Q-values, catch explosions only
            min_loss_improvement_pct: 2.0,
            plateau_window: 30,
            min_epochs_before_stopping: 50,
            hold_penalty: -0.001, // deprecated: kept for hyperopt adapter compat
            w_dsr: 1.0,
            w_pnl: 0.3,
            w_dd: 1.0,
            w_idle: 0.01,
            dd_threshold: 0.02,
            loss_aversion: 1.5,
            time_decay_rate: 0.0005,
            q_gap_threshold: 0.05, // Tier 2 default: mild conviction gating (hyperopt searches [0.0, 0.5])
            huber_delta: 100.0,   // BUG #12 FIX: Scale delta 100x for gradient explosion fix (was 1.0)
            gradient_clip_norm: Some(10.0), // C51 101-atoms × 3 branches → gradient naturally 300x larger than DQN
            hold_penalty_weight: 0.01,      // Default: 1% penalty weight
            movement_threshold: 0.02,       // Default: 2% price movement threshold
            enable_preprocessing: true,     // Default: preprocessing enabled (Wave 14 Agent 32)
            preprocessing_window: 50,       // Default: 50-bar rolling window
            preprocessing_clip_sigma: 5.0,  // Default: clip at ±5σ

            // WAVE 16 (Agent 36): Target update defaults (SOFT UPDATES for gradient stability)
            tau: 0.005, // Polyak averaging with 0.5% blend per step. Old 0.001 was too slow for
                        // reward v4's tight v_range=±0.5 — target couldn't track online, Q saturated v_max.
            target_update_mode: crate::trainers::TargetUpdateMode::Soft, // Soft updates (Rainbow DQN standard)
            target_update_frequency: 500, // BUG #9 FIX: Hard update frequency: 500 steps (optimal Rainbow DQN, was 10K)

            // Rainbow DQN warmup
            warmup_steps: 0, // Adaptive in CLI (0 for <200K, scaled 200K-1M, 80K for >1M)

            // P2-A Enhancement
            initial_capital: 100_000.0, // $100K default
            bars_per_day: 390.0, // 1-minute bars (6.5h × 60min)
            min_hold_bars: 5,

            // P2-B Enhancement
            cash_reserve_percent: 0.0, // Default: no reserve (backward compatible)

            // WAVE 16S: Adaptive Risk Management
            enable_kelly_sizing: true,           // Default: Kelly position sizing enabled
            enable_volatility_epsilon: true,     // Default: volatility-adjusted exploration enabled
            enable_risk_adjusted_rewards: true,  // Default: Sharpe-based rewards enabled
            kelly_fractional: 0.5,               // Default: half-Kelly (conservative)
            kelly_max_fraction: 0.25,            // Default: max 25% of portfolio
            kelly_min_trades: 20,                // Default: 20 trades minimum for statistics
            volatility_window: 20,               // Default: 20-period rolling window

            // WAVE 35: Advanced Features (WAVE 16S: NOW ENABLED BY DEFAULT)
            enable_regime_qnetwork: true,        // Default: regime-conditional Q-networks enabled
            enable_compliance: true,             // Default: compliance engine enabled

            // WAVE 16: Core Risk Management (default: ALL ENABLED)
            enable_drawdown_monitoring: true,    // Default: drawdown monitoring enabled (15% max)
            enable_position_limits: true,        // Default: 3-tier position limits enabled
            enable_circuit_breaker: true,        // Default: circuit breaker enabled (5-failure trip)

            // Wave 16 Portfolio Features (default: ALL ENABLED)
            enable_action_masking: true,         // Default: action masking enabled
            enable_entropy_regularization: true, // Default: entropy regularization enabled
            enable_stress_testing: true,         // Default: stress testing enabled
            max_position_absolute: 2.0,          // Default: ±2.0 position limit (matches production)

            // WAVE 17: Hyperopt-tuned parameters
            entropy_coefficient: Some(0.001), // Must be ≤ reward magnitude (~0.001). Old 0.01 was 10x reward → entropy dominated learning.
            noisy_epsilon_floor: Some(0.02), // NoisyNet handles exploration; low floor avoids interference
            count_bonus_coefficient: Some(0.1),
            transaction_cost_multiplier: 1.0,

            // Triple Barrier defaults
            enable_triple_barrier: false,
            triple_barrier_profit_target_bps: 100,
            triple_barrier_stop_loss_bps: 50,
            triple_barrier_max_holding_seconds: 3600,

            // P0: Prioritized Experience Replay (WAVE 6.4: ENABLED BY DEFAULT)

            per_alpha: 0.6,
            per_beta_start: 0.6,  // Higher start → faster IS weight correction (was 0.4)

            // Wave 2.1: Dueling Networks (WAVE 6.4: ENABLED BY DEFAULT)
            dueling_hidden_dim: 128,  // Default: 128 hidden units

            // Wave 2.2: Multi-Step Returns (WAVE 6.4: ENABLED BY DEFAULT)
            n_steps: 3,  // Default: 3 (Rainbow DQN standard)

            // Wave 2.3: Distributional RL (BUG #36 FIXED — scatter_add gradient flow verified)
            num_atoms: 51,  // Rainbow DQN standard: 51 atoms
            // v_min/v_max computed from reward_scale + gamma:
            // Q* ~ 10 / (1 - 0.95) * 1.2 = 240, clamped to [20, 300] → ±240
            reward_scale: 10.0,
            v_min: -(10.0_f64 / (1.0 - 0.95) * 1.2).clamp(20.0, 300.0), // = -240.0
            v_max: (10.0_f64 / (1.0 - 0.95) * 1.2).clamp(20.0, 300.0),  // = 240.0

            // Wave 2.4: Noisy Networks (WAVE 6.4: ENABLED BY DEFAULT)
            noisy_sigma_init: 0.5,  // Rainbow DQN standard: 0.5

            // Two-Phase Feature Normalization Configuration
            feature_stats_collection_ratio: 0.3,  // Default: 30% of epochs for stats collection
            max_feature_stats_epochs: Some(10),   // Default: cap at 10 epochs

            // WAVE 23 P0: Early Stopping for Gradient Collapse
            gradient_collapse_multiplier: 100.0,  // Adaptive threshold (LR × 100)
            gradient_collapse_patience: 5,        // 5 consecutive epochs before early stop

            // WAVE 26 P0.6: Learning Rate Scheduling
            // NOTE: warmup_steps already set above (line 500)
            lr_decay_type: super::lr_scheduler::LRDecayType::Constant, // Default: constant LR
            min_learning_rate: 1e-6,              // Default: min LR for Cosine decay
            lr_min: 1e-6,                         // Default: absolute minimum LR
            lr_decay_steps: 1000,                 // Default: decay every 1000 steps
            lr_decay_rate: 0.99,                  // Default: 1% decay per step

            // WAVE 26 P1.4: Ensemble Uncertainty (default: DISABLED for backward compatibility)
            use_ensemble_uncertainty: false,      // Default: disabled (use noisy networks instead)
            ensemble_size: 5,                     // Default: 5 Q-network heads
            beta_variance: 0.5,                   // Default: 50% variance penalty
            beta_disagreement: 0.5,               // Default: 50% disagreement penalty
            beta_entropy: 0.1,                    // Default: 10% entropy bonus
            variance_cap: 1.0,                    // Default: cap at 1.0 to prevent over-penalization

            // WAVE 26 P1.8: Curiosity-Driven Exploration
            curiosity_weight: 0.1,                // Default: enabled (ungates GPU experience collector; hyperopt tunes 0.01-0.5)

            // WAVE 26 P2.2: Gradient Accumulation
            gradient_accumulation_steps: 1,       // Default: no accumulation (standard training)

            // WAVE 26 P1: Advanced DQN Features Integration
            // P1.3: Sharpe Ratio Reward
            sharpe_weight: 0.0,                   // Default: disabled (hyperopt will tune 0.0-0.5)
            sharpe_window: 20,                    // Default: 20-step rolling window

            // P1.6: Adaptive Dropout
            enable_dropout_scheduler: false,      // Default: disabled (enable for overfitting prevention)
            dropout_initial: 0.5,                 // Default: 50% initial dropout
            dropout_final: 0.1,                   // Default: 10% final dropout
            dropout_anneal_steps: 10000,          // Default: 10K steps for annealing

            // P1.7: Hindsight Experience Replay
            her_ratio: 0.2,                       // Tier 2 default: 20% HER relabeling (hyperopt tunes 0.0-0.8)
            her_strategy: "future".to_owned(),   // Default: future strategy (better than final)

            // P1.9: Generalized Advantage Estimation
            enable_gae: false,                    // Default: disabled (enable for lower variance)
            gae_lambda: 0.95,                     // Default: 0.95 (standard PPO/A2C value)

            // P1.11: Noisy Sigma Scheduling
            enable_noisy_sigma_scheduler: true,    // Enabled by default for wider exploration
            noisy_sigma_initial: 0.8,             // Higher initial noise for 81-action exploration
            noisy_sigma_final: 0.4,               // Default: 40% final noise
            noisy_sigma_anneal_steps: 10000,      // Default: 10K steps for annealing

            // WAVE 30: L2 Weight Decay
            weight_decay: 1e-4,                   // Default: 0.0001 (standard regularization strength)

            // Conservative Q-Learning (CQL)
            cql_alpha: 0.1,                       // Default: mild conservatism (0.1 of 0.0-1.0 range)

            // QR-DQN (complementary to C51 — IQN for quantile estimation)
            num_quantiles: 32,                    // Default: 32 quantiles
            qr_kappa: 1.0,                        // Default: 1.0 (standard quantile Huber loss)
            iqn_lambda: 0.25,                     // Default: mild IQN regularization alongside C51
            spectral_norm_sigma_max: 3.0,        // Default: permits Xavier scaling [1.0, 10.0]

            // Phase 3: GPU experience collection
            gpu_n_episodes: 256,                  // Floor: optimal_n_episodes() scales up dynamically (H100→8192)
            gpu_timesteps_per_episode: 500,       // Default: 500 timesteps per episode
            avg_spread: 0.0001,                   // Default: 1bp (ES/NQ futures)
            max_training_steps_per_epoch: 0,      // Default: unlimited (full dataset training)

            // GPU-dynamic network sizing — None means auto-detect from hardware.
            // H100: optimal_n_episodes fills 132 SMs; hidden_dim_base expanded by hyperopt bounds.
            hidden_dim_base: None,                // Default: None (use [256, 128, 64])

            // BUG #7: Minimum profit factor for trade execution
            minimum_profit_factor: 1.5,   // Default: 50% margin above breakeven

            // DSR: enabled by default — directly optimizes Sharpe ratio (Moody & Saffell 2001)
            use_dsr: true,
            dsr_eta: 0.01,  // ~100-step effective EMA window

            // OFI: disabled by default (requires MBP-10 data)
            mbp10_data_dir: None,
            trades_data_dir: None,

            // Offline RL: disabled by default (online training with experience collection)
            offline_mode: false,
            dataset_path: None,

            // IQL: disabled by default (use CQL for standard offline RL)
            iql_expectile_tau: 0.7,
            iql_advantage_temperature: 3.0,
            use_iql: false,
            // Fill simulation: match ExperienceCollectorConfig::default()
            fill_ioc_fill_prob: 0.85,
            fill_limit_fill_min: 0.30,
            fill_limit_fill_max: 0.80,
            fill_spread_cost_frac: 0.50,
            fill_spread_capture_frac: 0.50,

            // Q-value clipping: match ExperienceCollectorConfig::default()
            q_clip_min: -200.0,
            q_clip_max: 200.0,

            branch_hidden_dim: 128,

            // GPU walk-forward: disabled by default (single-pass training)
            enable_gpu_walk_forward: false,
            wf_initial_train_fraction: 0.5,
            wf_val_fraction: 0.125,
            wf_test_fraction: 0.125,
            wf_step_fraction: 0.125,

            // Ensemble Agent: disabled by default (single agent training)
            ensemble_count: 1,                    // Default: 1 = no ensemble (single agent)
            ensemble_diversity_weight: 0.01,      // Default: 1% KL-divergence diversity loss weight

            // Curriculum learning: disabled by default (Full = no filtering)
            curriculum_phase: 2,                  // Default: 2 = Full (all windows, equal weight)

            // Expert demonstrations: disabled by default (pure RL)
            expert_demo_ratio: 0.0,               // Default: 0.0 = no expert demos
            expert_demo_decay_epochs: 20,          // Default: 20 epochs to decay to pure RL

            // GPU attention: always on (unconditional — no use_attention flag)
            use_attention: true,  // kept for backward compat with deserialized configs

            // C51 warmup: use MSE loss for first 5 epochs (stronger gradient than C51 cross-entropy)
            c51_warmup_epochs: 5,

            // Decision Transformer pre-training: disabled by default
            dt_pretrain_epochs: 0,    // 0 = disabled
            dt_context_len: 20,
            dt_embed_dim: 128,
            dt_num_layers: 3,
            dt_target_return: 2.0,

            // CVaR action selection: enabled by default (risk-aware IQN action scoring)
            use_cvar_action_selection: true,
            cvar_alpha: 0.05,         // Optimize for worst 5% quantile tail
        }
    }
}


// ============================================================================
// 2025-Optimized DQN Configuration Presets
// ============================================================================
//
// Production-ready defaults based on 2025 research and empirical results.
// Incorporates lessons from hyperopt trials, ablation studies, and real-world trading performance.
//
// Key Features:
// - Ensemble uncertainty estimation for better risk-aware exploration
// - Noisy networks with scheduled sigma decay for exploration
// - Soft target updates (Polyak averaging) for training stability
// - Prioritized experience replay with balanced alpha/beta
// - Moderate network capacity (512-256-128) for generalization
// - Conservative hyperparameters validated through production use

use crate::dqn::dqn::DQNConfig;

/// Production-ready 2025 DQN configuration
///
/// This configuration represents the best-known defaults as of 2025, incorporating:
/// - Results from 100+ hyperopt trials
/// - Ablation studies on Rainbow components
/// - Production trading performance data
/// - Modern RL research best practices
///
/// # Design Principles
///
/// 1. **Stability over speed**: Conservative learning rates and update frequencies
/// 2. **Ensemble-based exploration**: Use model uncertainty for exploration bonuses
/// 3. **Robust replay**: Prioritized replay with balanced diversity
/// 4. **Adaptive exploration**: Noisy networks + epsilon decay for multi-phase exploration
/// 5. **Risk-aware**: Integrated Sharpe ratio and curiosity-driven rewards
///
/// # Recommended Use Cases
///
/// - Production trading systems (live or paper)
/// - Baseline for hyperparameter tuning
/// - Reference for ablation studies
/// - Starting point for custom variants
///
/// # Not Recommended For
///
/// - HFT (use dqn_hft_config with attention layers enabled)
/// - Very small datasets (use dqn_conservative_config with reduced capacity)
/// - Offline RL (disable exploration)
pub(crate) fn dqn_default_config() -> DQNConfig {
    DQNConfig {
        // Network Architecture
        state_dim: 48,  // Default placeholder; overridden by constructor to (42+8+16+7)&!7=72
        num_actions: 9,  // 9 exposure levels (25% steps)
        hidden_dims: vec![512, 256, 128],  // Sufficient capacity without overfitting (Trial 19)
        leaky_relu_alpha: 0.01,

        // Training Hyperparameters
        learning_rate: 1e-4,
        warmup_steps: 5000,
        batch_size: 1024,  // H100 80GB easily fits 1024; AutoBatchSizer caps for smaller GPUs
        gamma: 0.95,
        gradient_clip_norm: 100.0,
        huber_delta: 10.0,

        // Replay Buffer
        replay_buffer_capacity: 500_000,
        min_replay_size: 10_000,
        per_alpha: 0.6,
        per_beta_start: 0.4,
        per_beta_max: 1.0,
        per_beta_annealing_steps: 100_000,
        per_max_memory_bytes: 0, // Constructor overrides via VRAM probe on CUDA

        // Exploration Strategy
        epsilon_start: 1.0,
        epsilon_end: 0.01,
        epsilon_decay: 0.9999,  // Reaches ~0.01 after 100k steps
        noisy_sigma_init: 0.5,

        // Target Network Updates
        target_update_freq: 1,
        use_soft_updates: true,
        tau: 0.001,
        tau_final: 0.0001,
        tau_anneal_steps: 100_000,

        // Rainbow DQN Components
        dueling_hidden_dim: 256,
        num_atoms: 51,
        // v_min/v_max from conservative defaults: reward_scale=10, gamma=0.95 → ±240
        v_min: -(10.0_f32 / (1.0 - 0.95) * 1.2).clamp(20.0, 300.0),
        v_max: (10.0_f32 / (1.0 - 0.95) * 1.2).clamp(20.0, 300.0),
        n_steps: 3,

        // Stability & Safety
        enable_q_value_clipping: true,
        q_value_clip_min: -5.0,   // Reward v4: Q-values in [-0.5, +0.5], 10x headroom
        q_value_clip_max: 5.0,
        gradient_collapse_multiplier: 100.0,
        gradient_collapse_patience: 5,

        // Portfolio Tracking
        initial_capital: 100_000.0,

        // CQL + IQN (2026 modernization)
        cql_alpha: 1.0,
        use_iqn: true,
        iqn_num_quantiles: 64,
        iqn_kappa: 1.0,
        iqn_embedding_dim: 64,
        use_cvar_action_selection: false,
        cvar_alpha: 0.05,

        minimum_profit_factor: 1.5,
        weight_decay: 1e-4,
        ..Default::default()
    }
}

/// HFT-optimized variant with attention layers and faster updates
///
/// Differences from default config:
/// - Attention layers enabled for temporal pattern recognition (requires network config)
/// - Faster target updates (tau = 0.005)
/// - Smaller warmup (1000 steps)
/// - Larger batch size (512) for throughput
pub(crate) fn dqn_hft_config() -> DQNConfig {
    let mut config = dqn_default_config();

    // HFT-specific overrides
    config.warmup_steps = 1000;
    config.batch_size = 1024;
    config.tau = 0.005;

    config
}

/// Conservative variant for small datasets or cautious exploration
///
/// Differences from default config:
/// - Smaller network (256-128-64)
/// - Lower learning rate (5e-5)
/// - More aggressive exploration decay
/// - Smaller batch size (256)
pub(crate) fn dqn_conservative_config() -> DQNConfig {
    let mut config = dqn_default_config();

    // Conservative overrides
    config.hidden_dims = vec![256, 128, 64];
    config.learning_rate = 5e-5;
    config.batch_size = 256;
    config.epsilon_decay = 0.999;  // Faster decay to conservative policy

    config
}

/// Aggressive variant for maximum exploration and learning
///
/// Differences from default config:
/// - Larger network (768-512-256)
/// - Higher learning rate (3e-4)
/// - Slower exploration decay
/// - Larger batch size (2048)
pub(crate) fn dqn_aggressive_config() -> DQNConfig {
    let mut config = dqn_default_config();

    // Aggressive overrides
    config.hidden_dims = vec![768, 512, 256];
    config.learning_rate = 3e-4;
    config.batch_size = 2048;
    config.epsilon_decay = 0.99995;  // Slower decay, more exploration

    config
}
