feat: reward config pipeline — DQNHyperparameters + TOML profiles

Add 7 composite reward fields to DQNHyperparameters: w_dsr, w_pnl,
w_dd, w_idle, dd_threshold, loss_aversion, time_decay_rate.

Add RewardSection to training_profile.rs with Option<f64> fields and
apply_to() mapping. Add [reward] section to all 3 DQN TOML profiles
(production, smoketest, hyperopt) with identical defaults.

Remove hold_reward from ExperienceSection (replaced by w_idle).
Add 7 reward search bounds to SearchSpaceSection and bound() match.
Add 7 reward phase_fast defaults to PhaseFastSection.

hold_penalty kept as deprecated field for hyperopt adapter compat
(Task 4 will clean it up).

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-03-22 20:50:23 +01:00
parent aa272666a1
commit af940671bc
5 changed files with 136 additions and 8 deletions

View File

@@ -61,6 +61,24 @@ gradient_accumulation_steps = [1, 1] # fixed at 1 for now
# IQN dual-head
iqn_lambda = [0.0, 2.0]
# Composite reward weights
w_dsr = [0.1, 2.0]
w_pnl = [0.0, 1.0]
w_dd = [0.0, 5.0]
w_idle = [0.0, 0.1]
dd_threshold = [0.01, 0.10]
loss_aversion = [1.0, 3.0]
time_decay_rate = [0.0001, 0.005]
[reward]
w_dsr = 1.0
w_pnl = 0.3
w_dd = 1.0
w_idle = 0.01
dd_threshold = 0.02
loss_aversion = 1.5
time_decay_rate = 0.0005
[fixed]
use_branching = true # GPU pipeline requires branching
use_double_dqn = true
@@ -84,3 +102,10 @@ num_atoms = 11
branch_hidden_dim = 64
dueling_hidden_dim = 128
v_range = 20.0
w_dsr = 1.0
w_pnl = 0.3
w_dd = 1.0
w_idle = 0.01
dd_threshold = 0.02
loss_aversion = 1.5
time_decay_rate = 0.0005

View File

@@ -63,4 +63,12 @@ min_loss_improvement_pct = 0.1
[experience]
initial_capital = 100000.0
tx_cost_multiplier = 0.0001
hold_reward = 0.001
[reward]
w_dsr = 1.0
w_pnl = 0.3
w_dd = 1.0
w_idle = 0.01
dd_threshold = 0.02
loss_aversion = 1.5
time_decay_rate = 0.0005

View File

@@ -27,3 +27,12 @@ min_epochs_before_stopping = 5
[experience]
gpu_n_episodes = 16
gpu_timesteps_per_episode = 50
[reward]
w_dsr = 1.0
w_pnl = 0.3
w_dd = 1.0
w_idle = 0.01
dd_threshold = 0.02
loss_aversion = 1.5
time_decay_rate = 0.0005