a400288ae0
Task 7: Update all three DQN TOML profiles with every configurable parameter. - dqn-smoketest: add [distributional], [advanced], [exploration] (noisy_sigma, entropy_coefficient, count_bonus), [risk] (max_position, loss_aversion), fix learning_rate 0.0003 -> 0.00003, add reward_scale + gamma - dqn-production: add reward_scale, exploration params (noisy/entropy/count_bonus), advanced params (n_steps/tau/c51_warmup/her/iqn_lambda/spectral_norm), remove hardcoded v_min/v_max (now computed), remove duplicate tau from [training] - dqn-hyperopt: add search space bounds for spectral_norm_sigma_max, c51_warmup_epochs, her_ratio Profile system additions (training_profile.rs): - TrainingSection: add reward_scale (recomputes v_min/v_max on apply) - ExplorationSection: add noisy_sigma_init, entropy_coefficient, count_bonus_coefficient, q_gap_threshold - AdvancedSection: add n_steps, tau, c51_warmup_epochs, her_ratio, iqn_lambda, spectral_norm_sigma_max, gradient_clip_norm - RiskSection: add max_position (alias for max_position_absolute), loss_aversion - RewardSection: add reward_scale - SearchSpaceSection: add spectral_norm_sigma_max, c51_warmup_epochs, her_ratio - apply_to: gamma change now recomputes v_min/v_max automatically Task 8: Config consistency integration tests (5 tests): - test_config_consistency_across_structs: v_range computed not hardcoded, fill simulation bounds, q_clip symmetry - test_toml_profile_applies_all_fields: smoketest profile applies every new section field correctly - test_production_profile_applies_all_sections: production profile end-to-end - test_hyperopt_search_space_has_new_bounds: new search space fields parse - test_reward_scale_recomputes_v_range: gamma override triggers v_range recomputation Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
136 lines
3.4 KiB
TOML
136 lines
3.4 KiB
TOML
# DQN Hyperopt Profile — PSO search space definition
|
|
# All bounds are [min, max] ranges. The adapter reads these at runtime.
|
|
# To constrain the search space, narrow the ranges here — no code changes needed.
|
|
|
|
[search_space]
|
|
# Base parameters
|
|
learning_rate = [0.00001, 0.0003] # log scale in adapter
|
|
batch_size = [64, 512]
|
|
gamma = [0.90, 0.99] # wider range — v_range computed dynamically from gamma
|
|
buffer_size = [50000, 100000] # log scale in adapter
|
|
max_position_absolute = [1.0, 4.0]
|
|
huber_delta = [10.0, 40.0] # log scale in adapter
|
|
entropy_coefficient = [0.05, 0.5]
|
|
transaction_cost_multiplier = [0.5, 2.0]
|
|
per_alpha = [0.4, 0.8]
|
|
per_beta_start = [0.2, 0.6]
|
|
|
|
# Rainbow DQN extensions
|
|
v_range = [1.0, 1.0] # IGNORED: v_range now computed dynamically from gamma
|
|
noisy_sigma_init = [0.1, 1.0] # log scale in adapter
|
|
dueling_hidden_dim = [128, 512]
|
|
n_steps = [3, 5]
|
|
num_atoms = [51, 101] # 51=C51 paper minimum, 101=higher resolution (2x slower but better signal)
|
|
|
|
# Weight decay
|
|
weight_decay = [0.0001, 0.01] # log scale in adapter
|
|
|
|
# Kelly risk parameters
|
|
kelly_fractional = [0.25, 0.75]
|
|
kelly_max_fraction = [0.1, 0.5]
|
|
|
|
# Volatility
|
|
volatility_window = [10, 30]
|
|
|
|
# Soft update
|
|
tau = [0.005, 0.01] # log scale in adapter
|
|
|
|
# Network sizing
|
|
hidden_dim_base = [128, 256] # capped at production default: 512 is 4x slower for marginal benefit
|
|
|
|
# CQL regularization
|
|
cql_alpha = [0.0, 1.0]
|
|
|
|
# Training dynamics
|
|
lr_decay_type = [0, 2] # discrete: 0=constant, 1=linear, 2=cosine
|
|
dsr_eta = [0.001, 0.05] # log scale in adapter
|
|
minimum_profit_factor = [1.1, 2.0]
|
|
|
|
# Exploration
|
|
count_bonus_coefficient = [0.0, 0.3]
|
|
|
|
# Risk-adjusted returns
|
|
sharpe_weight = [0.0, 0.5]
|
|
|
|
# Branching DQN
|
|
branch_hidden_dim = [64, 256]
|
|
|
|
# Gradient accumulation
|
|
gradient_accumulation_steps = [1, 1] # fixed at 1 for now
|
|
|
|
# IQN dual-head
|
|
iqn_lambda = [0.0, 2.0]
|
|
|
|
# Spectral normalization
|
|
spectral_norm_sigma_max = [1.0, 10.0]
|
|
|
|
# C51 warmup
|
|
c51_warmup_epochs = [0, 10]
|
|
|
|
# HER ratio
|
|
her_ratio = [0.0, 0.8]
|
|
|
|
# Composite reward weights
|
|
w_dsr = [0.1, 2.0]
|
|
w_pnl = [0.0, 1.0]
|
|
w_dd = [0.0, 5.0]
|
|
w_idle = [0.0, 0.1]
|
|
dd_threshold = [0.005, 0.03] # HFT: tight drawdown tolerance (0.5%-3%)
|
|
loss_aversion = [1.0, 3.0]
|
|
time_decay_rate = [0.0001, 0.005]
|
|
|
|
# Trade conviction filter
|
|
q_gap_threshold = [0.0, 0.5] # 0.0=trade every bar, 0.5=high conviction only
|
|
|
|
[experience]
|
|
initial_capital = 100000.0
|
|
|
|
[experience.fill_simulation]
|
|
ioc_fill_prob = 0.85
|
|
limit_fill_min = 0.30
|
|
limit_fill_max = 0.80
|
|
spread_cost_frac = 0.50
|
|
spread_capture_frac = 0.50
|
|
|
|
[risk]
|
|
q_clip_min = -200.0
|
|
q_clip_max = 200.0
|
|
|
|
[reward]
|
|
w_dsr = 1.0
|
|
w_pnl = 0.3
|
|
w_dd = 1.0
|
|
w_idle = 0.01
|
|
dd_threshold = 0.01
|
|
loss_aversion = 1.5
|
|
time_decay_rate = 0.0005
|
|
q_gap_threshold = 0.1
|
|
|
|
[fixed]
|
|
|
|
[pso]
|
|
swarm_size = 20
|
|
max_iterations = 50
|
|
inertia = 0.7
|
|
cognitive = 1.5
|
|
social = 1.5
|
|
|
|
# Two-phase hyperopt: Phase 1 fixes architecture to small network,
|
|
# searches only learning dynamics (~15D). Phase 2 fixes best dynamics
|
|
# from Phase 1 JSON, searches architecture (~5D).
|
|
# See: docs/superpowers/specs/2026-03-22-two-phase-hyperopt-design.md
|
|
[phase_fast]
|
|
hidden_dim_base = 128
|
|
num_atoms = 101
|
|
branch_hidden_dim = 64
|
|
dueling_hidden_dim = 128
|
|
v_range = 1.0
|
|
w_dsr = 1.0
|
|
w_pnl = 0.3
|
|
w_dd = 1.0
|
|
w_idle = 0.01
|
|
dd_threshold = 0.01
|
|
loss_aversion = 1.5
|
|
time_decay_rate = 0.0005
|
|
q_gap_threshold = 0.1
|