gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: three-phase pipeline + hyperopt search space expansion

Browse Source 
Task 9: Document ensemble consensus limitation — ensemble heads live on
FusedTrainingCtx (training-only), not accessible at inference time.
The ensemble already provides value via diversity gradient during training.

Task 10: Add three-phase training pipeline to the epoch loop:
- Phase 1 (Behavioral Cloning): C51 warmup + expert demos (already existed)
- Phase 2 (Full-Stack RL): all features, expert decay (already existed)
- Phase 3 (Refinement, last 20%): force expert_ratio=0, shrink-and-perturb
  at phase boundary for plasticity consolidation

Task 11: Expand hyperopt search space from 41D to 45D with 4 new dims:
- her_ratio [0.0, 0.5]: HER relabeling ratio (was hardcoded 0.0)
- curiosity_weight [0.0, 0.2]: intrinsic reward weight (was fixed 0.0)
- use_cvar_action_selection [0.0, 1.0]: risk-aware IQN action scoring
- cvar_alpha [0.01, 0.2]: CVaR confidence level
All wired through build_hyperparams to DQNHyperparameters.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-03-24 23:19:12 +01:00
parent bff4b2807c
commit 70e38508fc
4 changed files with 161 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

105
crates/ml/src/hyperopt/adapters/dqn.rs
View File

@@ -3,7 +3,7 @@
//! This module provides a production-ready adapter for optimizing DQN
//! hyperparameters using the generic optimization framework. It implements:
//!
//! - 39D continuous parameter space with log-scale handling
//! - 45D continuous parameter space with log-scale handling
//! - Training wrapper that integrates with existing DQN pipeline
//! - Backtest-based objective using Sharpe ratio (production metric)
//! - Prioritized Experience Replay (PER) tuning for Rainbow DQN performance (+25-40% convergence speed)
@@ -144,10 +144,9 @@ const TRIAL_VRAM_MB: f64 = 7000.0;
/// Corrected: was 0.02 (20KB) — 100x too high
const MB_PER_SAMPLE: f64 = 0.0005;
/// DQN hyperparameter space (39D continuous)
/// DQN hyperparameter space (45D continuous)
///
/// Fixed params (not in search space):
/// - curiosity_weight: 0.0 (conflicts with NoisyNet)
/// - noisy_epsilon_floor: 0.10 (minimum random exploration to prevent action collapse)
///
/// Tuned params restored in C1-C4:
@@ -413,6 +412,24 @@ pub struct DQNParams {
/// Decision Transformer pre-training epochs (0 = disabled).
/// Not in the hyperopt search space — fixed to 0 for all trials.
pub dt_pretrain_epochs: usize,
// === Task 11: New hyperopt-tunable parameters ===
/// HER relabeling ratio (0.0 = disabled, 0.5 = 50% relabeled)
/// Improves sample efficiency by relabeling failed episodes toward achieved goals.
pub her_ratio: f64,
/// Curiosity weight for intrinsic reward (0.0 = disabled, 0.2 = 20% intrinsic)
/// Encourages exploration of novel state-action pairs via prediction error.
pub curiosity_weight_tunable: f64,
/// Enable CVaR (Conditional Value-at-Risk) action selection (0.0 = off, 1.0 = on)
/// Stored as f64 for hyperopt, rounded to bool in build_hyperparams.
pub use_cvar_action_selection: f64,
/// CVaR alpha confidence level (0.01-0.20). Lower = more risk-averse.
/// 0.05 = optimize for worst 5% outcomes.
pub cvar_alpha: f64,
}
impl Default for DQNParams {
@@ -503,13 +520,18 @@ impl Default for DQNParams {
q_gap_threshold: 0.0,
c51_warmup_epochs: 5, // C51 warmup: MSE loss for first 5 epochs
dt_pretrain_epochs: 0, // Disabled by default; not in hyperopt search space
// Task 11: New hyperopt-tunable parameters
her_ratio: 0.2, // Default: 20% HER relabeling
curiosity_weight_tunable: 0.1, // Default: 10% curiosity-driven intrinsic reward
use_cvar_action_selection: 1.0, // Default: enabled (risk-aware IQN)
cvar_alpha: 0.05, // Default: worst 5% tail
}
}
}
impl ParameterSpace for DQNParams {
fn continuous_bounds() -> Vec<(f64, f64)> {
// 39D search space (C8: added 7 composite reward weights at indices 31-38)
// 45D search space (Task 11: +4 dims at indices 41-44)
// All bounds loaded from config/training/dqn-hyperopt.toml [search_space].
// To change search ranges, edit the TOML — no code changes needed.
// Log-scale transforms are applied here; TOML stores linear values.
@@ -558,6 +580,11 @@ impl ParameterSpace for DQNParams {
let mut rw_qgt = b("q_gap_threshold", (0.0, 0.5));
let sn_sigma = b("spectral_norm_sigma_max", (1.0, 10.0));
let cw = b("c51_warmup_epochs", (0.0, 20.0));
// Task 11: New hyperopt-tunable parameters
let hr = b("her_ratio", (0.0, 0.5));
let cwt = b("curiosity_weight_tunable", (0.0, 0.2));
let ucvar = b("use_cvar_action_selection", (0.0, 1.0));
let cva = b("cvar_alpha", (0.01, 0.2));
// Two-phase hyperopt: fix architecture OR dynamics params to single-point bounds.
// Single-point bounds (v, v) make the optimizer trivially converge on those
@@ -687,6 +714,11 @@ impl ParameterSpace for DQNParams {
rw_qgt, // 38: q_gap_threshold
sn_sigma, // 39: spectral_norm_sigma_max
cw, // 40: c51_warmup_epochs
// Task 11: New hyperopt-tunable parameters
hr, // 41: her_ratio
cwt, // 42: curiosity_weight_tunable
ucvar, // 43: use_cvar_action_selection
cva, // 44: cvar_alpha
];
// Phase FULL: fix all non-architecture dims to Phase 1 best values.
@@ -732,6 +764,11 @@ impl ParameterSpace for DQNParams {
if x.len() < 39 {
return Err(MLError::ConfigError(format!("Expected at least 39 continuous parameters, got {}", x.len())));
}
// Task 11: New parameters (indices 41-44) — backwards compatible via .get()
let her_ratio = x.get(41).copied().unwrap_or(0.2).clamp(0.0, 0.5);
let curiosity_weight_tunable = x.get(42).copied().unwrap_or(0.1).clamp(0.0, 0.2);
let use_cvar_action_selection = x.get(43).copied().unwrap_or(1.0).clamp(0.0, 1.0);
let cvar_alpha = x.get(44).copied().unwrap_or(0.05).clamp(0.01, 0.2);
// === 26 TUNED parameters (from search space) ===
// C2: Removed curiosity_weight and noisy_epsilon_floor (exploration stacking)
@@ -923,13 +960,18 @@ impl ParameterSpace for DQNParams {
q_gap_threshold,
c51_warmup_epochs, // Index 40: C51 warmup epochs [0, 20]
dt_pretrain_epochs: 0, // Fixed: DT pre-training not in hyperopt search space
// Task 11: New hyperopt-tunable parameters (indices 41-44)
her_ratio,
curiosity_weight_tunable,
use_cvar_action_selection,
cvar_alpha,
};
Ok(params)
}
fn to_continuous(&self) -> Vec<f64> {
// 39D search space (C8: added 7 composite reward weights at indices 31-38)
// 45D search space (Task 11: +4 dims at indices 41-44)
vec![
self.learning_rate.ln(), // 0
self.batch_size as f64, // 1
@@ -973,11 +1015,16 @@ impl ParameterSpace for DQNParams {
self.q_gap_threshold, // 38
self.spectral_norm_sigma_max, // 39
self.c51_warmup_epochs as f64, // 40
// Task 11: New hyperopt-tunable parameters
self.her_ratio, // 41
self.curiosity_weight_tunable, // 42
self.use_cvar_action_selection, // 43
self.cvar_alpha, // 44
]
}
fn param_names() -> Vec<&'static str> {
// 38 tuned parameters (C8: added 7 composite reward weights)
// 45 tuned parameters (Task 11: +4 new dims at indices 41-44)
vec![
"learning_rate", // 0
"batch_size", // 1
@@ -1021,6 +1068,11 @@ impl ParameterSpace for DQNParams {
"q_gap_threshold", // 38
"spectral_norm_sigma_max", // 39
"c51_warmup_epochs", // 40
// Task 11: New hyperopt-tunable parameters
"her_ratio", // 41
"curiosity_weight", // 42
"use_cvar_action_selection", // 43
"cvar_alpha", // 44
]
}
@@ -2618,7 +2670,7 @@ impl HyperparameterOptimizable for DQNTrainer {
// Fix 1: Clamp buffer size to max (4GB GPU constraint)
let clamped_buffer_size = params.buffer_size.min(self.buffer_size_max);
info!("Training DQN with parameters (C8: 39D search space):");
info!("Training DQN with parameters (45D search space):");
info!(" Learning rate: {:.6}", params.learning_rate);
info!(" Batch size: {}", params.batch_size);
info!(" Gamma: {:.3}", params.gamma);
@@ -2946,8 +2998,8 @@ impl HyperparameterOptimizable for DQNTrainer {
beta_entropy: params.beta_entropy,
variance_cap: 1.0, // Fixed in all trials (not in search space)
// WAVE 26 P1.8: Curiosity-Driven Exploration
curiosity_weight: params.curiosity_weight,
// WAVE 26 P1.8: Curiosity-Driven Exploration (Task 11: tunable via hyperopt)
curiosity_weight: params.curiosity_weight_tunable,
// WAVE 26 P1.12: Polyak Soft Update
tau: params.tau, // Use hyperopt-tunable tau
@@ -2965,8 +3017,8 @@ impl HyperparameterOptimizable for DQNTrainer {
dropout_final: 0.1, // Default: 10% final dropout
dropout_anneal_steps: 10000, // Default: 10K steps for annealing
// WAVE 26 P1.7: Hindsight Experience Replay
her_ratio: 0.0, // Default: disabled (not in search space yet)
// WAVE 26 P1.7: Hindsight Experience Replay (Task 11: now in search space)
her_ratio: params.her_ratio,
her_strategy: "future".to_owned(), // Default: future strategy
// WAVE 26 P1.9: Generalized Advantage Estimation
@@ -3058,6 +3110,9 @@ impl HyperparameterOptimizable for DQNTrainer {
c51_warmup_epochs: params.c51_warmup_epochs,
// Decision Transformer pre-training: disabled for all hyperopt trials
dt_pretrain_epochs: params.dt_pretrain_epochs,
// Task 11: CVaR action selection — wired from hyperopt search space (indices 43-44)
use_cvar_action_selection: params.use_cvar_action_selection.round() >= 1.0,
cvar_alpha: params.cvar_alpha as f32,
..Default::default()
};
@@ -3808,7 +3863,7 @@ mod tests {
#[test]
fn test_dqn_params_roundtrip() {
// Roundtrip test for the 39D search space (C8: 7 composite reward weights added)
// Roundtrip test for the 45D search space (Task 11: +4 new dims)
// Only the tuned parameters roundtrip; fixed params get defaults from from_continuous
let params = DQNParams {
learning_rate: 3.37e-05,
@@ -3879,10 +3934,15 @@ mod tests {
q_gap_threshold: 0.2,
c51_warmup_epochs: 5,
dt_pretrain_epochs: 0,
// Task 11: New hyperopt-tunable parameters
her_ratio: 0.3,
curiosity_weight_tunable: 0.15,
use_cvar_action_selection: 1.0,
cvar_alpha: 0.08,
};
let continuous = params.to_continuous();
assert_eq!(continuous.len(), 41, "to_continuous must return 41D vector");
assert_eq!(continuous.len(), 45, "to_continuous must return 45D vector");
let recovered = DQNParams::from_continuous(&continuous).unwrap();
// Tuned parameters must roundtrip exactly
@@ -3933,12 +3993,18 @@ mod tests {
assert!((recovered.dd_threshold - params.dd_threshold).abs() < 1e-6);
assert!((recovered.loss_aversion - params.loss_aversion).abs() < 1e-6);
assert!((recovered.time_decay_rate - params.time_decay_rate).abs() < 1e-6);
// Task 11: New hyperopt parameters roundtrip (indices 41-44)
assert!((recovered.her_ratio - params.her_ratio).abs() < 1e-6);
assert!((recovered.curiosity_weight_tunable - params.curiosity_weight_tunable).abs() < 1e-6);
assert!((recovered.use_cvar_action_selection - params.use_cvar_action_selection).abs() < 1e-6);
assert!((recovered.cvar_alpha - params.cvar_alpha).abs() < 1e-6);
}
#[test]
fn test_dqn_params_bounds() {
let bounds = DQNParams::continuous_bounds();
assert_eq!(bounds.len(), 41); // 41D: 39 base + spectral_norm_sigma_max + c51_warmup_epochs
assert_eq!(bounds.len(), 45); // 45D: 39 base + spectral_norm_sigma_max + c51_warmup_epochs + her_ratio + curiosity_weight + use_cvar + cvar_alpha
// Check log-scale bounds are reasonable
assert!(bounds[0].0 < bounds[0].1); // learning_rate
@@ -3998,7 +4064,7 @@ mod tests {
#[test]
fn test_param_names() {
let names = DQNParams::param_names();
assert_eq!(names.len(), 41); // 41D: 39 base + spectral_norm_sigma_max + c51_warmup_epochs
assert_eq!(names.len(), 45); // 45D: 39 base + spectral_norm_sigma_max + c51_warmup_epochs + her_ratio + curiosity_weight + use_cvar + cvar_alpha
assert_eq!(names[0], "learning_rate");
assert_eq!(names[1], "batch_size");
assert_eq!(names[2], "gamma");
@@ -4031,12 +4097,17 @@ mod tests {
assert_eq!(names[29], "gradient_accumulation_steps"); // C6
assert_eq!(names[39], "spectral_norm_sigma_max"); // 39
assert_eq!(names[40], "c51_warmup_epochs"); // 40
// Task 11: New hyperopt-tunable parameters
assert_eq!(names[41], "her_ratio"); // 41
assert_eq!(names[42], "curiosity_weight"); // 42
assert_eq!(names[43], "use_cvar_action_selection"); // 43
assert_eq!(names[44], "cvar_alpha"); // 44
}
#[test]
fn test_per_params_always_enabled() {
// Test that PER is always enabled with tunable alpha/beta parameters
// 39D search space (C8: added 7 composite reward weights)
// 45D search space (Task 11: +4 new dims)
let continuous = vec![
3e-5_f64.ln(), 92.0, 0.92, 97_273_f64.ln(), 4.0, 24.77_f64.ln(), 0.03, 1.0,
0.6, 0.4, // 8-9: per_alpha, per_beta_start
@@ -4443,7 +4514,7 @@ mod tests {
fn test_qr_dqn_roundtrip_continuous() {
let params = DQNParams::default();
let continuous = params.to_continuous();
assert_eq!(continuous.len(), 41, "Should have 41 continuous dimensions");
assert_eq!(continuous.len(), 45, "Should have 45 continuous dimensions");
let roundtrip = DQNParams::from_continuous(&continuous).unwrap();
// Default num_quantiles=32 (IQN default, used alongside C51)
assert_eq!(roundtrip.num_quantiles, 32);

18
crates/ml/src/trainers/dqn/trainer/action.rs
View File

@@ -130,6 +130,24 @@ impl DQNTrainer {
// select_actions_batch_gpu removed: zero callers, was a Candle forward path.
// GPU experience collection uses the experience_action_select kernel directly.
// ── Ensemble consensus action selection ──────────────────────────────
// Task 9 analysis: The ensemble heads (heads 1..K-1) live on FusedTrainingCtx
// which is only accessible during the training loop (not during inference).
// The ensemble already provides value via diversity gradient during training:
// each head learns different strategies, and the KL-divergence loss encourages
// disagreement. At inference time, head 0 (the primary network inside the
// CUDA Graph) is the sole Q-value source for action selection.
//
// To use ensemble consensus at inference, the extra head weights would need
// to be synced into separate forward-pass-capable networks — a significant
// memory + compute cost that is not justified until production A/B testing
// shows head-0-only inference is a bottleneck. The ensemble diversity gradient
// already improves head 0's policy during training.
//
// Future: if ensemble consensus is needed at inference, extract head weights
// at epoch boundary into lightweight forward-only modules and compute
// mean Q + variance scaling here in select_actions_batch().
pub(crate) async fn epsilon_greedy_action(&self, state: &GpuTensor) -> Result<usize> {
use rand::Rng;
let epsilon = self.get_epsilon().await? as f32;

12
crates/ml/src/trainers/dqn/trainer/tests.rs
View File

@@ -552,19 +552,23 @@ fn test_c2_five_actions_produce_distinct_exposures() {
assert_eq!(unique.len(), 5, "All 5 exposure levels must be distinct");
}
/// Verify hyperopt search space is 41D.
/// Verify hyperopt search space is 45D (Task 11: +4 new dims).
#[test]
fn test_c3_search_space_is_40d() {
let bounds = crate::hyperopt::adapters::dqn::DQNParams::continuous_bounds();
assert_eq!(bounds.len(), 41, "Search space must be 41D");
assert_eq!(bounds.len(), 45, "Search space must be 45D");
let names = crate::hyperopt::adapters::dqn::DQNParams::param_names();
assert_eq!(names.len(), 41);
assert_eq!(names.len(), 45);
assert!(names.contains(&"count_bonus_coefficient"), "count_bonus_coefficient must be in search space (C3)");
assert!(names.contains(&"sharpe_weight"), "sharpe_weight must be in search space (C4)");
assert!(names.contains(&"branch_hidden_dim"), "branch_hidden_dim must be in search space (L2)");
assert!(names.contains(&"c51_warmup_epochs"), "c51_warmup_epochs must be in search space");
assert!(!names.contains(&"curiosity_weight"), "curiosity_weight must not be in search space");
// Task 11: curiosity_weight is now in the search space
assert!(names.contains(&"curiosity_weight"), "curiosity_weight must be in search space (Task 11)");
assert!(names.contains(&"her_ratio"), "her_ratio must be in search space (Task 11)");
assert!(names.contains(&"use_cvar_action_selection"), "use_cvar_action_selection must be in search space (Task 11)");
assert!(names.contains(&"cvar_alpha"), "cvar_alpha must be in search space (Task 11)");
assert!(!names.contains(&"noisy_epsilon_floor"), "noisy_epsilon_floor must not be in search space");
}

49
crates/ml/src/trainers/dqn/trainer/training_loop.rs
View File

@@ -279,11 +279,56 @@ impl DQNTrainer {
// Sync GPU weights after training
self.sync_gpu_weights().await?;
// Shrink-and-Perturb for plasticity maintenance.
// ── Three-Phase Training Pipeline ──────────────────────────────
//
// Phase 1 (Behavioral Cloning): epochs 0..c51_warmup_epochs
// C51 alpha ramps 0→1 (MSE→C51 blend). Expert demos active.
// Already implemented via c51_alpha ramp + expert_demo_ratio above.
//
// Phase 2 (Full-Stack Online RL): c51_warmup_epochs..phase3_start
// All features active. Expert ratio decays to 0 via ExpertDemoGenerator.
// Already implemented via expert_demo_decay_epochs linear decay.
//
// Phase 3 (Refinement): last 20% of epochs
// Pure C51. No expert demos. Shrink-and-perturb for plasticity.
// Forces the network to consolidate learned policy without expert crutch.
let total_epochs = self.hyperparams.epochs;
let phase3_start = total_epochs * 80 / 100;
let in_phase3 = epoch >= phase3_start && total_epochs > 4;
if in_phase3 {
// Phase 3: Force expert demos off (pure RL refinement)
if let Some(ref mut c) = self.gpu_experience_collector {
c.set_expert_ratio(0.0);
}
// Phase 3: Shrink-and-perturb at the phase boundary for plasticity
if epoch == phase3_start {
if let Some(ref mut fused) = self.fused_ctx {
let alpha = 0.9_f32; // 90% old weights, 10% noise
let sigma = 0.01_f32; // noise scale
if let Err(e) = fused.shrink_and_perturb(alpha, sigma) {
tracing::warn!("Shrink-and-Perturb failed at Phase 3 start (non-fatal): {e}");
} else {
tracing::info!(
epoch,
phase3_start,
alpha,
sigma,
"Phase 3 (Refinement): Shrink-and-Perturb applied at phase boundary"
);
}
}
}
}
// Shrink-and-Perturb for plasticity maintenance (Phases 1-2).
// Applied every 25% of total epochs (at epoch boundaries only).
// Prevents the network from losing its ability to learn new patterns
// across market regime shifts during long training runs.
if self.hyperparams.epochs > 4 {
// In Phase 3, the boundary perturb above handles it — skip periodic.
if !in_phase3 && self.hyperparams.epochs > 4 {
let interval = (self.hyperparams.epochs / 4).max(1);
if epoch > 0 && epoch % interval == 0 {
if let Some(ref mut fused) = self.fused_ctx {