From 8548d126fd616efe49084c4e4a0d24e293f2b090 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Fri, 15 May 2026 16:09:28 +0200 Subject: [PATCH] =?UTF-8?q?chore(alpha):=20H=3D600=20DQN=20smoke=20verdict?= =?UTF-8?q?=20=E2=80=94=20FAIL=20on=20rvr=20(linear=20Q=20lock-in)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase E.1 Task 12. Stabilized H=600 DQN smoke ran end-to-end on full 500K-snapshot data. All three preconditions PASS but the rvr gate FAILS: Q_SPREAD_EMA = 35.54 ≥ 0.05 PASS ACTION_ENTROPY_EMA = 1.91 ≥ 1.099 PASS RETURN_VS_RANDOM_EMA = -2.58 ≥ 0.0 FAIL ← policy WORSE than random EARLY_Q_MOVEMENT_EMA = 0.096 ≥ 0.01 PASS rvr trajectory across 1000 episodes: ep 50 | rollout_R= -11049 | rvr = -1.18 (near random) ep 200 | rollout_R= -9620 | rvr = -0.97 (briefly improving) ep 600 | rollout_R= -18140 | rvr = -2.03 (degrading) ep 1000 | rollout_R= -18272 | rvr = -2.58 (deterministic-bad) Random baseline at H=600 = -5185 mean, std=4953. Trained policy loses 3.5× worse than random. Diagnostics performed: reward_scale=10000 → rvr=-2.37 (no help) alpha_m=0 (vanilla DQN, no Munchausen) → rvr=-2.44 (no help) Root cause: "first-best-action lock-in" of linear Q + ε-greedy. DQN's TD update only modifies Q[a] for the TAKEN action; with ε-decay, the argmax action self-reinforces while other actions' Q stays frozen at random Xavier init. Random policy samples all 9 uniformly → 11% chance of "lucky" close-position at any step → exits bad trades. Trained policy converges deterministic on one bad action → never exits. Per plan: pivot to NoisyNet (Task 19) — parameter-space noise breaks the lock-in. Alternative: wire alpha_logit from Phase 1d.3 stacker (currently hardcoded to 0.0 placeholder) so the policy has actual directional signal to work with. Kill-criteria gate worked as designed — correctly flagged that linear Q + ε-greedy on this env is insufficient without further intervention. Memory note: project_phase_e1_h600_smoke_verdict.md (full analysis + hypothesis tree + recommended next steps). --- config/ml/alpha_dqn_h600_smoke.json | 166 ++++++++++++++++++++++++++++ 1 file changed, 166 insertions(+) create mode 100644 config/ml/alpha_dqn_h600_smoke.json diff --git a/config/ml/alpha_dqn_h600_smoke.json b/config/ml/alpha_dqn_h600_smoke.json new file mode 100644 index 000000000..b59ef061d --- /dev/null +++ b/config/ml/alpha_dqn_h600_smoke.json @@ -0,0 +1,166 @@ +{ + "action_entropy_ema": 1.9093124866485596, + "all_pass": false, + "alpha_m": 0.8999999761581421, + "early_q_movement_ema": 0.09593381732702255, + "eps_end": 0.05000000074505806, + "eps_start": 0.5, + "gamma": 0.9900000095367432, + "grad_clip": 1.0, + "horizon": 600, + "kc_log": [ + { + "early_mvmt": 0.010988299734890461, + "entropy": 1.9636621475219727, + "episode": 50, + "q_spread": 28.91693115234375, + "rvr": -1.1840312480926514 + }, + { + "early_mvmt": 0.016329117119312286, + "entropy": 1.961021900177002, + "episode": 100, + "q_spread": 37.871768951416016, + "rvr": -1.072948932647705 + }, + { + "early_mvmt": 0.022424953058362007, + "entropy": 1.956502079963684, + "episode": 150, + "q_spread": 28.133586883544922, + "rvr": -1.0141085386276245 + }, + { + "early_mvmt": 0.028699442744255066, + "entropy": 1.9538824558258057, + "episode": 200, + "q_spread": 21.6387996673584, + "rvr": -0.9686362743377686 + }, + { + "early_mvmt": 0.03473977372050285, + "entropy": 1.951103687286377, + "episode": 250, + "q_spread": 22.248136520385742, + "rvr": -1.1477679014205933 + }, + { + "early_mvmt": 0.04057596996426582, + "entropy": 1.9480493068695068, + "episode": 300, + "q_spread": 36.82394790649414, + "rvr": -1.0388388633728027 + }, + { + "early_mvmt": 0.04621242359280586, + "entropy": 1.9451168775558472, + "episode": 350, + "q_spread": 36.524620056152344, + "rvr": -1.1660404205322266 + }, + { + "early_mvmt": 0.05157305300235748, + "entropy": 1.9419623613357544, + "episode": 400, + "q_spread": 31.634666442871094, + "rvr": -1.2641773223876953 + }, + { + "early_mvmt": 0.056624751538038254, + "entropy": 1.9387248754501343, + "episode": 450, + "q_spread": 25.271129608154297, + "rvr": -1.3423638343811035 + }, + { + "early_mvmt": 0.061381783336400986, + "entropy": 1.9356876611709595, + "episode": 500, + "q_spread": 21.30175018310547, + "rvr": -1.3799097537994385 + }, + { + "early_mvmt": 0.06583097577095032, + "entropy": 1.9327433109283447, + "episode": 550, + "q_spread": 22.904991149902344, + "rvr": -1.4454447031021118 + }, + { + "early_mvmt": 0.07003756612539291, + "entropy": 1.929825782775879, + "episode": 600, + "q_spread": 25.839786529541016, + "rvr": -2.0275630950927734 + }, + { + "early_mvmt": 0.07398476451635361, + "entropy": 1.927020788192749, + "episode": 650, + "q_spread": 39.29933166503906, + "rvr": -1.894902229309082 + }, + { + "early_mvmt": 0.07770879566669464, + "entropy": 1.9242794513702393, + "episode": 700, + "q_spread": 28.634672164916992, + "rvr": -1.7190972566604614 + }, + { + "early_mvmt": 0.08122072368860245, + "entropy": 1.921583652496338, + "episode": 750, + "q_spread": 23.793535232543945, + "rvr": -1.9371918439865112 + }, + { + "early_mvmt": 0.08452516794204712, + "entropy": 1.9189369678497314, + "episode": 800, + "q_spread": 16.309627532958984, + "rvr": -2.267064094543457 + }, + { + "early_mvmt": 0.08763865381479263, + "entropy": 1.9163986444473267, + "episode": 850, + "q_spread": 18.71721839904785, + "rvr": -2.415508508682251 + }, + { + "early_mvmt": 0.09058000147342682, + "entropy": 1.9139772653579712, + "episode": 900, + "q_spread": 15.583666801452637, + "rvr": -2.569871664047241 + }, + { + "early_mvmt": 0.09333524852991104, + "entropy": 1.911572813987732, + "episode": 950, + "q_spread": 14.705427169799805, + "rvr": -2.4978740215301514 + }, + { + "early_mvmt": 0.09593381732702255, + "entropy": 1.9093124866485596, + "episode": 1000, + "q_spread": 35.54442596435547, + "rvr": -2.57589054107666 + } + ], + "lr": 0.00009999999747378752, + "n_episodes": 1000, + "pass_early": true, + "pass_entropy": true, + "pass_q_spread": true, + "pass_rvr": false, + "phase": "E.1 Task 12", + "q_init_norm": 2.5735182762145996, + "q_spread_ema": 35.54442596435547, + "return_vs_random_ema": -2.57589054107666, + "reward_scale": 1000.0, + "target_update_every": 10, + "tau": 0.029999999329447746 +} \ No newline at end of file