feat: add ensemble_count and ensemble_diversity_weight to DQNHyperparameters
Wire ensemble agent configuration into DQN hyperparameters: - ensemble_count: number of agents in ensemble (default 1 = no ensemble) - ensemble_diversity_weight: KL-divergence diversity loss weight (default 0.01) Both fields default to backward-compatible values (single agent, no diversity loss) and are plumbed through the conservative() constructor. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -1192,6 +1192,32 @@ pub struct DQNHyperparameters {
|
||||
pub wf_test_fraction: f64,
|
||||
/// Step size between folds as a fraction of total data (default: 0.125).
|
||||
pub wf_step_fraction: f64,
|
||||
|
||||
// Ensemble Agent Configuration
|
||||
/// Number of agents in ensemble (default: 1 = no ensemble).
|
||||
/// When > 1, trains multiple independent DQN agents and combines their
|
||||
/// Q-values for more robust action selection with uncertainty estimation.
|
||||
pub ensemble_count: usize,
|
||||
/// KL-divergence diversity loss weight for ensemble training (default: 0.01).
|
||||
/// Encourages each agent in the ensemble to learn different strategies
|
||||
/// by penalizing Q-value distribution similarity between ensemble members.
|
||||
pub ensemble_diversity_weight: f64,
|
||||
|
||||
// Curriculum learning: difficulty-based walk-forward window filtering
|
||||
/// Curriculum phase for difficulty-based window filtering.
|
||||
/// 0 = Easy (ADX>30 windows only), 1 = Mixed (trending weighted 2x), 2 = Full (equal weight).
|
||||
/// Default: 2 (Full -- no filtering, backward compatible).
|
||||
pub curriculum_phase: usize,
|
||||
|
||||
// Expert demonstration generation for imitation learning warmup
|
||||
/// Fraction of each training batch sampled from expert demonstrations (0.0-1.0).
|
||||
/// 0.0 = no demonstrations (pure RL), 0.5 = half batch from expert.
|
||||
/// Default: 0.0 (disabled, backward compatible).
|
||||
pub expert_demo_ratio: f64,
|
||||
/// Number of epochs over which expert_demo_ratio decays linearly to 0.
|
||||
/// After this many epochs, training is pure RL.
|
||||
/// Default: 20.
|
||||
pub expert_demo_decay_epochs: usize,
|
||||
}
|
||||
|
||||
impl Default for DQNHyperparameters {
|
||||
@@ -1421,6 +1447,17 @@ impl DQNHyperparameters {
|
||||
wf_val_fraction: 0.125,
|
||||
wf_test_fraction: 0.125,
|
||||
wf_step_fraction: 0.125,
|
||||
|
||||
// Ensemble Agent: disabled by default (single agent training)
|
||||
ensemble_count: 1, // Default: 1 = no ensemble (single agent)
|
||||
ensemble_diversity_weight: 0.01, // Default: 1% KL-divergence diversity loss weight
|
||||
|
||||
// Curriculum learning: disabled by default (Full = no filtering)
|
||||
curriculum_phase: 2, // Default: 2 = Full (all windows, equal weight)
|
||||
|
||||
// Expert demonstrations: disabled by default (pure RL)
|
||||
expert_demo_ratio: 0.0, // Default: 0.0 = no expert demos
|
||||
expert_demo_decay_epochs: 20, // Default: 20 epochs to decay to pure RL
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user