chore(alpha): H=600 DQN smoke verdict — FAIL on rvr (linear Q lock-in)
Phase E.1 Task 12. Stabilized H=600 DQN smoke ran end-to-end on full 500K-snapshot data. All three preconditions PASS but the rvr gate FAILS: Q_SPREAD_EMA = 35.54 ≥ 0.05 PASS ACTION_ENTROPY_EMA = 1.91 ≥ 1.099 PASS RETURN_VS_RANDOM_EMA = -2.58 ≥ 0.0 FAIL ← policy WORSE than random EARLY_Q_MOVEMENT_EMA = 0.096 ≥ 0.01 PASS rvr trajectory across 1000 episodes: ep 50 | rollout_R= -11049 | rvr = -1.18 (near random) ep 200 | rollout_R= -9620 | rvr = -0.97 (briefly improving) ep 600 | rollout_R= -18140 | rvr = -2.03 (degrading) ep 1000 | rollout_R= -18272 | rvr = -2.58 (deterministic-bad) Random baseline at H=600 = -5185 mean, std=4953. Trained policy loses 3.5× worse than random. Diagnostics performed: reward_scale=10000 → rvr=-2.37 (no help) alpha_m=0 (vanilla DQN, no Munchausen) → rvr=-2.44 (no help) Root cause: "first-best-action lock-in" of linear Q + ε-greedy. DQN's TD update only modifies Q[a] for the TAKEN action; with ε-decay, the argmax action self-reinforces while other actions' Q stays frozen at random Xavier init. Random policy samples all 9 uniformly → 11% chance of "lucky" close-position at any step → exits bad trades. Trained policy converges deterministic on one bad action → never exits. Per plan: pivot to NoisyNet (Task 19) — parameter-space noise breaks the lock-in. Alternative: wire alpha_logit from Phase 1d.3 stacker (currently hardcoded to 0.0 placeholder) so the policy has actual directional signal to work with. Kill-criteria gate worked as designed — correctly flagged that linear Q + ε-greedy on this env is insufficient without further intervention. Memory note: project_phase_e1_h600_smoke_verdict.md (full analysis + hypothesis tree + recommended next steps).
This commit is contained in:
jgrusewski
166
config/ml/alpha_dqn_h600_smoke.json
Normal file
166
config/ml/alpha_dqn_h600_smoke.json
Normal file
@@ -0,0 +1,166 @@
|
||||
{
|
||||
"action_entropy_ema": 1.9093124866485596,
|
||||
"all_pass": false,
|
||||
"alpha_m": 0.8999999761581421,
|
||||
"early_q_movement_ema": 0.09593381732702255,
|
||||
"eps_end": 0.05000000074505806,
|
||||
"eps_start": 0.5,
|
||||
"gamma": 0.9900000095367432,
|
||||
"grad_clip": 1.0,
|
||||
"horizon": 600,
|
||||
"kc_log": [
|
||||
{
|
||||
"early_mvmt": 0.010988299734890461,
|
||||
"entropy": 1.9636621475219727,
|
||||
"episode": 50,
|
||||
"q_spread": 28.91693115234375,
|
||||
"rvr": -1.1840312480926514
|
||||
},
|
||||
{
|
||||
"early_mvmt": 0.016329117119312286,
|
||||
"entropy": 1.961021900177002,
|
||||
"episode": 100,
|
||||
"q_spread": 37.871768951416016,
|
||||
"rvr": -1.072948932647705
|
||||
},
|
||||
{
|
||||
"early_mvmt": 0.022424953058362007,
|
||||
"entropy": 1.956502079963684,
|
||||
"episode": 150,
|
||||
"q_spread": 28.133586883544922,
|
||||
"rvr": -1.0141085386276245
|
||||
},
|
||||
{
|
||||
"early_mvmt": 0.028699442744255066,
|
||||
"entropy": 1.9538824558258057,
|
||||
"episode": 200,
|
||||
"q_spread": 21.6387996673584,
|
||||
"rvr": -0.9686362743377686
|
||||
},
|
||||
{
|
||||
"early_mvmt": 0.03473977372050285,
|
||||
"entropy": 1.951103687286377,
|
||||
"episode": 250,
|
||||
"q_spread": 22.248136520385742,
|
||||
"rvr": -1.1477679014205933
|
||||
},
|
||||
{
|
||||
"early_mvmt": 0.04057596996426582,
|
||||
"entropy": 1.9480493068695068,
|
||||
"episode": 300,
|
||||
"q_spread": 36.82394790649414,
|
||||
"rvr": -1.0388388633728027
|
||||
},
|
||||
{
|
||||
"early_mvmt": 0.04621242359280586,
|
||||
"entropy": 1.9451168775558472,
|
||||
"episode": 350,
|
||||
"q_spread": 36.524620056152344,
|
||||
"rvr": -1.1660404205322266
|
||||
},
|
||||
{
|
||||
"early_mvmt": 0.05157305300235748,
|
||||
"entropy": 1.9419623613357544,
|
||||
"episode": 400,
|
||||
"q_spread": 31.634666442871094,
|
||||
"rvr": -1.2641773223876953
|
||||
},
|
||||
{
|
||||
"early_mvmt": 0.056624751538038254,
|
||||
"entropy": 1.9387248754501343,
|
||||
"episode": 450,
|
||||
"q_spread": 25.271129608154297,
|
||||
"rvr": -1.3423638343811035
|
||||
},
|
||||
{
|
||||
"early_mvmt": 0.061381783336400986,
|
||||
"entropy": 1.9356876611709595,
|
||||
"episode": 500,
|
||||
"q_spread": 21.30175018310547,
|
||||
"rvr": -1.3799097537994385
|
||||
},
|
||||
{
|
||||
"early_mvmt": 0.06583097577095032,
|
||||
"entropy": 1.9327433109283447,
|
||||
"episode": 550,
|
||||
"q_spread": 22.904991149902344,
|
||||
"rvr": -1.4454447031021118
|
||||
},
|
||||
{
|
||||
"early_mvmt": 0.07003756612539291,
|
||||
"entropy": 1.929825782775879,
|
||||
"episode": 600,
|
||||
"q_spread": 25.839786529541016,
|
||||
"rvr": -2.0275630950927734
|
||||
},
|
||||
{
|
||||
"early_mvmt": 0.07398476451635361,
|
||||
"entropy": 1.927020788192749,
|
||||
"episode": 650,
|
||||
"q_spread": 39.29933166503906,
|
||||
"rvr": -1.894902229309082
|
||||
},
|
||||
{
|
||||
"early_mvmt": 0.07770879566669464,
|
||||
"entropy": 1.9242794513702393,
|
||||
"episode": 700,
|
||||
"q_spread": 28.634672164916992,
|
||||
"rvr": -1.7190972566604614
|
||||
},
|
||||
{
|
||||
"early_mvmt": 0.08122072368860245,
|
||||
"entropy": 1.921583652496338,
|
||||
"episode": 750,
|
||||
"q_spread": 23.793535232543945,
|
||||
"rvr": -1.9371918439865112
|
||||
},
|
||||
{
|
||||
"early_mvmt": 0.08452516794204712,
|
||||
"entropy": 1.9189369678497314,
|
||||
"episode": 800,
|
||||
"q_spread": 16.309627532958984,
|
||||
"rvr": -2.267064094543457
|
||||
},
|
||||
{
|
||||
"early_mvmt": 0.08763865381479263,
|
||||
"entropy": 1.9163986444473267,
|
||||
"episode": 850,
|
||||
"q_spread": 18.71721839904785,
|
||||
"rvr": -2.415508508682251
|
||||
},
|
||||
{
|
||||
"early_mvmt": 0.09058000147342682,
|
||||
"entropy": 1.9139772653579712,
|
||||
"episode": 900,
|
||||
"q_spread": 15.583666801452637,
|
||||
"rvr": -2.569871664047241
|
||||
},
|
||||
{
|
||||
"early_mvmt": 0.09333524852991104,
|
||||
"entropy": 1.911572813987732,
|
||||
"episode": 950,
|
||||
"q_spread": 14.705427169799805,
|
||||
"rvr": -2.4978740215301514
|
||||
},
|
||||
{
|
||||
"early_mvmt": 0.09593381732702255,
|
||||
"entropy": 1.9093124866485596,
|
||||
"episode": 1000,
|
||||
"q_spread": 35.54442596435547,
|
||||
"rvr": -2.57589054107666
|
||||
}
|
||||
],
|
||||
"lr": 0.00009999999747378752,
|
||||
"n_episodes": 1000,
|
||||
"pass_early": true,
|
||||
"pass_entropy": true,
|
||||
"pass_q_spread": true,
|
||||
"pass_rvr": false,
|
||||
"phase": "E.1 Task 12",
|
||||
"q_init_norm": 2.5735182762145996,
|
||||
"q_spread_ema": 35.54442596435547,
|
||||
"return_vs_random_ema": -2.57589054107666,
|
||||
"reward_scale": 1000.0,
|
||||
"target_update_every": 10,
|
||||
"tau": 0.029999999329447746
|
||||
}
|
||||
Reference in New Issue
Block a user