diff --git a/config/ml/alpha_dqn_h600_smoke_e2.json b/config/ml/alpha_dqn_h600_smoke_e2.json new file mode 100644 index 000000000..f5cf79e53 --- /dev/null +++ b/config/ml/alpha_dqn_h600_smoke_e2.json @@ -0,0 +1,100 @@ +{ + "action_entropy_ema": 1.9570317268371582, + "all_pass": true, + "alpha_m": 0.8999999761581421, + "early_q_movement_ema": 0.12511339783668518, + "eps_end": 0.05000000074505806, + "eps_start": 0.5, + "final_stacker_kelly_attenuation": 0.10000000149011612, + "final_stacker_threshold": 0.5, + "final_trade_rate_observed_ema": 0.711693525314331, + "gamma": 0.9900000095367432, + "grad_clip": 1.0, + "horizon": 600, + "kc_log": [ + { + "early_mvmt": 0.022276392206549644, + "entropy": 2.0100932121276855, + "episode": 100, + "q_spread": 16.85750389099121, + "rvr": 1.0427366495132446 + }, + { + "early_mvmt": 0.0334387831389904, + "entropy": 2.00576114654541, + "episode": 200, + "q_spread": 18.60127067565918, + "rvr": 1.0427367687225342 + }, + { + "early_mvmt": 0.046235110610723495, + "entropy": 2.0008304119110107, + "episode": 300, + "q_spread": 29.530759811401367, + "rvr": 1.0427361726760864 + }, + { + "early_mvmt": 0.05913778021931648, + "entropy": 1.9954335689544678, + "episode": 400, + "q_spread": 17.126867294311523, + "rvr": 1.0427309274673462 + }, + { + "early_mvmt": 0.07171963155269623, + "entropy": 1.9893970489501953, + "episode": 500, + "q_spread": 17.392959594726562, + "rvr": 1.0427260398864746 + }, + { + "early_mvmt": 0.08374495804309845, + "entropy": 1.9831516742706299, + "episode": 600, + "q_spread": 15.180435180664062, + "rvr": 1.0427193641662598 + }, + { + "early_mvmt": 0.0951075330376625, + "entropy": 1.9764565229415894, + "episode": 700, + "q_spread": 18.398340225219727, + "rvr": 1.042712688446045 + }, + { + "early_mvmt": 0.10574351996183395, + "entropy": 1.9698795080184937, + "episode": 800, + "q_spread": 19.092670440673828, + "rvr": 1.0426955223083496 + }, + { + "early_mvmt": 0.1157108023762703, + "entropy": 1.9635075330734253, + "episode": 900, + "q_spread": 12.077985763549805, + "rvr": 1.0426815748214722 + }, + { + "early_mvmt": 0.12511339783668518, + "entropy": 1.9570317268371582, + "episode": 1000, + "q_spread": 10.57870864868164, + "rvr": 1.0426677465438843 + } + ], + "lr": 0.00009999999747378752, + "n_episodes": 1000, + "pass_early": true, + "pass_entropy": true, + "pass_q_spread": true, + "pass_rvr": true, + "phase": "E.1 Task 12", + "q_init_norm": 2.5735182762145996, + "q_spread_ema": 10.57870864868164, + "return_vs_random_ema": 1.0426677465438843, + "reward_scale": 1000.0, + "target_update_every": 10, + "tau": 0.029999999329447746, + "trade_rate_target": 0.07999999821186066 +} \ No newline at end of file diff --git a/crates/ml/examples/alpha_dqn_h600_smoke.rs b/crates/ml/examples/alpha_dqn_h600_smoke.rs index 75e914faf..f2b3404dc 100644 --- a/crates/ml/examples/alpha_dqn_h600_smoke.rs +++ b/crates/ml/examples/alpha_dqn_h600_smoke.rs @@ -139,6 +139,27 @@ struct Cli { /// Episodes between kill-criteria pipeline launches. #[arg(long, default_value_t = 50)] kill_criteria_every: usize, + /// Phase E.2 stacker-threshold controller — target trade rate (ISV[544]). + /// Set once at training start, never reset (TrainingPersist anchor). + #[arg(long, default_value_t = 0.08)] + trade_rate_target: f32, + /// Stacker-threshold controller P-gain on rate error. + #[arg(long, default_value_t = 0.01)] + k_threshold: f32, + /// Kelly-attenuation controller P-gain on Sharpe error. + #[arg(long, default_value_t = 0.005)] + k_atten: f32, + /// Target rollout Sharpe for the Kelly-attenuation controller. + #[arg(long, default_value_t = 0.5)] + target_sharpe: f32, + /// Pearl D Wiener-α floor on the observed-rate EMA (slot 545). 0.4 per + /// `pearl_wiener_alpha_floor_for_nonstationary` — controller co-adapts + /// with the policy, so the EMA needs to stay responsive. + #[arg(long, default_value_t = 0.4)] + wiener_alpha_floor: f32, + /// Meta-α for the Wiener variance updates inside the controller. + #[arg(long, default_value_t = 0.1)] + ctl_alpha_meta: f32, /// Reward normalization scale. Rewards are divided by this before /// being passed to the Munchausen target — the network learns /// normalized Q-values. Default 1000 ≈ |median reward| at H=600 @@ -512,6 +533,14 @@ fn main() -> Result<()> { let pearls_kernel = pearls_module.load_function("apply_pearls_ad_kernel") .context("pearls load")?; + // Phase E.2 Task 16: stacker-threshold + Kelly-attenuation controller + let ctl_module = ctx + .load_cubin(ml::cuda_pipeline::alpha_kernels::STACKER_THRESHOLD_CONTROLLER_CUBIN.to_vec()) + .context("controller cubin load")?; + let ctl_kernel = ctl_module + .load_function("stacker_threshold_controller_update") + .context("controller kernel load")?; + // --- Load env data --- let fill_model = load_fill_model(&cli.fill_coeffs)?; info!("Loaded FillModel from {}", cli.fill_coeffs.display()); @@ -595,12 +624,19 @@ fn main() -> Result<()> { // bug fixes. See alpha_random_baseline.json for the source numbers. isv_host[RANDOM_BASELINE_MEAN_INDEX] = -5191.53; isv_host[RANDOM_BASELINE_STD_INDEX] = 4963.62; + // Phase E.2 Task 17: trade-rate target anchor for the stacker-threshold + // controller. Set at training start, never reset across folds + // (TrainingPersist). Default 0.08 = ~8% per-step trade rate. + isv_host[ml::cuda_pipeline::alpha_isv_slots::TRADE_RATE_TARGET_INDEX] = + cli.trade_rate_target; let mut isv_dev = stream.clone_htod(&isv_host).context("upload isv")?; // Wiener state for the 4 kill-criteria slots: 4 × [sample_var, diff_var, x_lag] = 12 floats. let mut wiener_dev = stream.alloc_zeros::(12).context("alloc wiener")?; // Scratch buffer for kill-criteria producer output: 4 floats. let mut kc_scratch_dev = stream.alloc_zeros::(4).context("alloc kc_scratch")?; + // Phase E.2: Wiener state for slot 545 (observed-rate EMA): 3 floats. + let mut ctl_wiener_dev = stream.alloc_zeros::(3).context("alloc ctl wiener")?; // --- Allocate per-episode batch buffers (sized to horizon, reused) --- let h = cli.horizon as i32; @@ -822,6 +858,47 @@ fn main() -> Result<()> { } } + // Phase E.2 Task 17: Stacker-threshold controller at rollout end. + // Trade rate = (count of non-Wait actions) / total decisions in the + // episode. Realized Sharpe ≈ mean(returns) / std(returns) over the + // window — for a single rollout this collapses to the rollout's + // terminal reward divided by the random baseline std (a per-rollout + // analog of the rvr metric, lets the controller decide whether to + // tighten or loosen Kelly). + let trade_count_ep: f32 = actions_host + .iter() + .filter(|&&a| a != 0) // 0 == Wait + .count() as f32; + let decisions_ep = actions_host.len() as f32; + // Single-episode Sharpe proxy: terminal reward / baseline std. + // Note: terminal reward is the LAST element of rewards_host + // (sum of step rewards from env.step into ep.cumulative_pnl). + let ep_terminal_r = *rewards_host.last().unwrap_or(&0.0); + let baseline_std = isv_host[RANDOM_BASELINE_STD_INDEX].max(1e-6); + let rollout_sharpe = ep_terminal_r / baseline_std; + unsafe { + let (isv_ptr, _g0) = isv_dev.device_ptr_mut(&stream); + let (w_ptr, _g1) = ctl_wiener_dev.device_ptr_mut(&stream); + ml::cuda_pipeline::alpha_kernels::launch_stacker_threshold_controller( + &stream, + &ctl_kernel, + trade_count_ep, + decisions_ep, + rollout_sharpe, + cli.target_sharpe, + cli.k_threshold, + cli.k_atten, + cli.wiener_alpha_floor, + cli.ctl_alpha_meta, + ml::cuda_pipeline::alpha_isv_slots::STACKER_THRESHOLD_INDEX as i32, + ml::cuda_pipeline::alpha_isv_slots::TRADE_RATE_TARGET_INDEX as i32, + ml::cuda_pipeline::alpha_isv_slots::TRADE_RATE_OBSERVED_EMA_INDEX as i32, + ml::cuda_pipeline::alpha_isv_slots::STACKER_KELLY_ATTENUATION_INDEX as i32, + isv_ptr, + w_ptr, + )?; + } + // Target network hard update every K episodes: w_target ← w_online. // Standard DQN stabilizer — V_soft(s') bootstrap reads w_target, // so updating it slowly breaks the chase-its-own-tail divergence @@ -901,9 +978,14 @@ fn main() -> Result<()> { isv[RETURN_VS_RANDOM_EMA_INDEX], isv[EARLY_Q_MOVEMENT_EMA_INDEX], ]; + // Phase E.2: controller-driven slots for visibility. + let ctl_threshold = isv[ml::cuda_pipeline::alpha_isv_slots::STACKER_THRESHOLD_INDEX]; + let ctl_obs_rate = isv[ml::cuda_pipeline::alpha_isv_slots::TRADE_RATE_OBSERVED_EMA_INDEX]; + let ctl_atten = isv[ml::cuda_pipeline::alpha_isv_slots::STACKER_KELLY_ATTENUATION_INDEX]; info!( - "ep {:>4} | ε={:.3} | rollout_R_mean={:>+9.1} | KC: q_spread={:.4} entropy={:.4} rvr={:+.4} early_mvmt={:.4}", - ep + 1, eps, rollout_r_mean, kc[0], kc[1], kc[2], kc[3] + "ep {:>4} | ε={:.3} | R_mean={:>+9.1} | KC q={:.3} H={:.3} rvr={:+.3} ΔQ={:.3} | CTL thresh={:.4} obs={:.3} atten={:.3}", + ep + 1, eps, rollout_r_mean, kc[0], kc[1], kc[2], kc[3], + ctl_threshold, ctl_obs_rate, ctl_atten, ); kc_logs.push((ep + 1, kc)); } @@ -965,6 +1047,10 @@ fn main() -> Result<()> { "target_update_every": cli.target_update_every, "grad_clip": cli.grad_clip, "q_init_norm": q_init_norm, + "final_stacker_threshold": isv_final[ml::cuda_pipeline::alpha_isv_slots::STACKER_THRESHOLD_INDEX], + "final_trade_rate_observed_ema": isv_final[ml::cuda_pipeline::alpha_isv_slots::TRADE_RATE_OBSERVED_EMA_INDEX], + "final_stacker_kelly_attenuation": isv_final[ml::cuda_pipeline::alpha_isv_slots::STACKER_KELLY_ATTENUATION_INDEX], + "trade_rate_target": cli.trade_rate_target, "q_spread_ema": kc_final[0], "action_entropy_ema": kc_final[1], "return_vs_random_ema": kc_final[2],