diff --git a/config/ml/alpha_compose_backtest.json b/config/ml/alpha_compose_backtest.json index 2faef85ea..5b796bfaf 100644 --- a/config/ml/alpha_compose_backtest.json +++ b/config/ml/alpha_compose_backtest.json @@ -1,64 +1,424 @@ { "bins": [ { + "avg_n_trades": 477.0899963378906, "cost": 0.0, - "mean_reward": -11.089629173278809, - "n_episodes": 500, - "p05": -22.522815704345703, - "p50": -11.164673805236816, - "p95": 3.1911258697509766, - "sharpe_annualised": -39.49838638305664, - "sharpe_per_episode": -1.3801853656768799, - "std_reward": 8.034883499145508, - "win_rate": 0.09000000357627869 + "mean_reward": -11.580000877380371, + "n_episodes": 300, + "p05": -22.575578689575195, + "p50": -11.476005554199219, + "p95": 2.4081244468688965, + "sharpe_annualised": -41.78176498413086, + "sharpe_per_episode": -1.4599729776382446, + "std_reward": 7.931654453277588, + "threshold": 0.0, + "win_rate": 0.07666666805744171 }, { + "avg_n_trades": 476.27667236328125, "cost": 0.0625, - "mean_reward": -20.684438705444336, - "n_episodes": 500, - "p05": -36.075843811035156, - "p50": -19.98797607421875, - "p95": -7.718697547912598, - "sharpe_annualised": -59.89125061035156, - "sharpe_per_episode": -2.0927696228027344, - "std_reward": 9.88376235961914, - "win_rate": 0.012000000104308128 + "mean_reward": -19.83131217956543, + "n_episodes": 300, + "p05": -32.835304260253906, + "p50": -19.831050872802734, + "p95": -6.751925468444824, + "sharpe_annualised": -71.4566879272461, + "sharpe_per_episode": -2.496898651123047, + "std_reward": 7.942378044128418, + "threshold": 0.0, + "win_rate": 0.006666666828095913 }, { + "avg_n_trades": 481.7133483886719, "cost": 0.125, - "mean_reward": -29.383296966552734, - "n_episodes": 500, - "p05": -44.815711975097656, - "p50": -28.835330963134766, - "p95": -15.50295352935791, - "sharpe_annualised": -88.58140563964844, - "sharpe_per_episode": -3.095284938812256, - "std_reward": 9.492920875549316, - "win_rate": 0.0 + "mean_reward": -29.428951263427734, + "n_episodes": 300, + "p05": -46.65030288696289, + "p50": -28.805376052856445, + "p95": -13.693151473999023, + "sharpe_annualised": -86.77941131591797, + "sharpe_per_episode": -3.032318115234375, + "std_reward": 9.705100059509277, + "threshold": 0.0, + "win_rate": 0.0033333334140479565 }, { + "avg_n_trades": 480.3333435058594, "cost": 0.25, - "mean_reward": -48.22906494140625, - "n_episodes": 500, - "p05": -67.93758392333984, - "p50": -48.01160430908203, - "p95": -29.78984832763672, - "sharpe_annualised": -115.97515106201172, - "sharpe_per_episode": -4.052499771118164, - "std_reward": 11.9010648727417, + "mean_reward": -48.30219268798828, + "n_episodes": 300, + "p05": -67.57722473144531, + "p50": -46.67949676513672, + "p95": -31.478973388671875, + "sharpe_annualised": -108.57279968261719, + "sharpe_per_episode": -3.7938406467437744, + "std_reward": 12.73173999786377, + "threshold": 0.0, "win_rate": 0.0 }, { + "avg_n_trades": 478.3133239746094, "cost": 0.5, - "mean_reward": -84.58942413330078, - "n_episodes": 500, - "p05": -110.6795883178711, - "p50": -84.19437408447266, - "p95": -58.62353515625, - "sharpe_annualised": -138.9206085205078, - "sharpe_per_episode": -4.854279041290283, - "std_reward": 17.425745010375977, + "mean_reward": -83.0477294921875, + "n_episodes": 300, + "p05": -108.5387954711914, + "p50": -81.84159088134766, + "p95": -58.206626892089844, + "sharpe_annualised": -146.75875854492188, + "sharpe_per_episode": -5.128166198730469, + "std_reward": 16.19443130493164, + "threshold": 0.0, "win_rate": 0.0 + }, + { + "avg_n_trades": 401.3333435058594, + "cost": 0.0, + "mean_reward": -9.551131248474121, + "n_episodes": 300, + "p05": -23.94935417175293, + "p50": -8.9005765914917, + "p95": 5.565349578857422, + "sharpe_annualised": -27.86334800720215, + "sharpe_per_episode": -0.9736241698265076, + "std_reward": 9.80987548828125, + "threshold": 0.05000000074505806, + "win_rate": 0.1133333370089531 + }, + { + "avg_n_trades": 393.0733337402344, + "cost": 0.0625, + "mean_reward": -16.152053833007812, + "n_episodes": 300, + "p05": -30.313827514648438, + "p50": -15.583097457885742, + "p95": -2.770275592803955, + "sharpe_annualised": -53.90193176269531, + "sharpe_per_episode": -1.8834859132766724, + "std_reward": 8.575616836547852, + "threshold": 0.05000000074505806, + "win_rate": 0.02666666731238365 + }, + { + "avg_n_trades": 400.413330078125, + "cost": 0.125, + "mean_reward": -24.998205184936523, + "n_episodes": 300, + "p05": -42.52478790283203, + "p50": -23.242856979370117, + "p95": -9.703174591064453, + "sharpe_annualised": -68.45292663574219, + "sharpe_per_episode": -2.3919389247894287, + "std_reward": 10.451022148132324, + "threshold": 0.05000000074505806, + "win_rate": 0.0033333334140479565 + }, + { + "avg_n_trades": 395.1866760253906, + "cost": 0.25, + "mean_reward": -38.649471282958984, + "n_episodes": 300, + "p05": -57.62205123901367, + "p50": -38.203250885009766, + "p95": -23.15749740600586, + "sharpe_annualised": -104.59449005126953, + "sharpe_per_episode": -3.654827356338501, + "std_reward": 10.574910163879395, + "threshold": 0.05000000074505806, + "win_rate": 0.0 + }, + { + "avg_n_trades": 392.3766784667969, + "cost": 0.5, + "mean_reward": -68.999755859375, + "n_episodes": 300, + "p05": -95.44992065429688, + "p50": -66.09086608886719, + "p95": -49.909767150878906, + "sharpe_annualised": -127.27163696289062, + "sharpe_per_episode": -4.447230815887451, + "std_reward": 15.515217781066895, + "threshold": 0.05000000074505806, + "win_rate": 0.0 + }, + { + "avg_n_trades": 307.6400146484375, + "cost": 0.0, + "mean_reward": -6.366776466369629, + "n_episodes": 300, + "p05": -22.731773376464844, + "p50": -5.475202560424805, + "p95": 6.573000431060791, + "sharpe_annualised": -20.232799530029297, + "sharpe_per_episode": -0.7069912552833557, + "std_reward": 9.005453109741211, + "threshold": 0.10000000149011612, + "win_rate": 0.16333332657814026 + }, + { + "avg_n_trades": 315.8599853515625, + "cost": 0.0625, + "mean_reward": -12.57988452911377, + "n_episodes": 300, + "p05": -30.93537139892578, + "p50": -11.238504409790039, + "p95": -0.2330303192138672, + "sharpe_annualised": -39.50290298461914, + "sharpe_per_episode": -1.3803430795669556, + "std_reward": 9.113592147827148, + "threshold": 0.10000000149011612, + "win_rate": 0.046666666865348816 + }, + { + "avg_n_trades": 310.2099914550781, + "cost": 0.125, + "mean_reward": -17.49962043762207, + "n_episodes": 300, + "p05": -34.14094543457031, + "p50": -15.837596893310547, + "p95": -3.5142734050750732, + "sharpe_annualised": -53.26398468017578, + "sharpe_per_episode": -1.8611942529678345, + "std_reward": 9.402361869812012, + "threshold": 0.10000000149011612, + "win_rate": 0.013333333656191826 + }, + { + "avg_n_trades": 314.6833190917969, + "cost": 0.25, + "mean_reward": -30.126296997070312, + "n_episodes": 300, + "p05": -52.358428955078125, + "p50": -28.658782958984375, + "p95": -13.625950813293457, + "sharpe_annualised": -71.65260314941406, + "sharpe_per_episode": -2.503744602203369, + "std_reward": 12.032496452331543, + "threshold": 0.10000000149011612, + "win_rate": 0.0 + }, + { + "avg_n_trades": 309.2866516113281, + "cost": 0.5, + "mean_reward": -52.3604736328125, + "n_episodes": 300, + "p05": -79.04839324951172, + "p50": -50.56077194213867, + "p95": -27.740079879760742, + "sharpe_annualised": -82.7920913696289, + "sharpe_per_episode": -2.8929898738861084, + "std_reward": 18.09908676147461, + "threshold": 0.10000000149011612, + "win_rate": 0.0 + }, + { + "avg_n_trades": 236.6266632080078, + "cost": 0.0, + "mean_reward": -5.102373123168945, + "n_episodes": 300, + "p05": -18.235349655151367, + "p50": -3.8291523456573486, + "p95": 3.9433228969573975, + "sharpe_annualised": -16.905658721923828, + "sharpe_per_episode": -0.5907315015792847, + "std_reward": 8.637381553649902, + "threshold": 0.15000000596046448, + "win_rate": 0.20999999344348907 + }, + { + "avg_n_trades": 228.65333557128906, + "cost": 0.0625, + "mean_reward": -8.878447532653809, + "n_episodes": 300, + "p05": -25.199073791503906, + "p50": -6.852076053619385, + "p95": 2.099073886871338, + "sharpe_annualised": -28.601604461669922, + "sharpe_per_episode": -0.9994209408760071, + "std_reward": 8.88359260559082, + "threshold": 0.15000000596046448, + "win_rate": 0.07666666805744171 + }, + { + "avg_n_trades": 232.20333862304688, + "cost": 0.125, + "mean_reward": -12.617223739624023, + "n_episodes": 300, + "p05": -31.3784236907959, + "p50": -9.867401123046875, + "p95": -1.0055747032165527, + "sharpe_annualised": -34.65873718261719, + "sharpe_per_episode": -1.2110742330551147, + "std_reward": 10.418208122253418, + "threshold": 0.15000000596046448, + "win_rate": 0.036666665226221085 + }, + { + "avg_n_trades": 231.2066650390625, + "cost": 0.25, + "mean_reward": -20.333833694458008, + "n_episodes": 300, + "p05": -44.20069122314453, + "p50": -17.208972930908203, + "p95": -7.3099260330200195, + "sharpe_annualised": -50.19882583618164, + "sharpe_per_episode": -1.7540888786315918, + "std_reward": 11.59224796295166, + "threshold": 0.15000000596046448, + "win_rate": 0.013333333656191826 + }, + { + "avg_n_trades": 228.2899932861328, + "cost": 0.5, + "mean_reward": -36.5968132019043, + "n_episodes": 300, + "p05": -63.5013313293457, + "p50": -31.710899353027344, + "p95": -16.777376174926758, + "sharpe_annualised": -64.87017059326172, + "sharpe_per_episode": -2.266747236251831, + "std_reward": 16.145078659057617, + "threshold": 0.15000000596046448, + "win_rate": 0.0 + }, + { + "avg_n_trades": 168.1199951171875, + "cost": 0.0, + "mean_reward": -5.073331832885742, + "n_episodes": 300, + "p05": -21.23957061767578, + "p50": -2.783275604248047, + "p95": 4.150900363922119, + "sharpe_annualised": -15.721929550170898, + "sharpe_per_episode": -0.549368679523468, + "std_reward": 9.234840393066406, + "threshold": 0.20000000298023224, + "win_rate": 0.20333333313465118 + }, + { + "avg_n_trades": 166.10333251953125, + "cost": 0.0625, + "mean_reward": -6.372754096984863, + "n_episodes": 300, + "p05": -21.26305389404297, + "p50": -4.527249813079834, + "p95": 2.169199228286743, + "sharpe_annualised": -22.731422424316406, + "sharpe_per_episode": -0.7943001985549927, + "std_reward": 8.023104667663574, + "threshold": 0.20000000298023224, + "win_rate": 0.09333333373069763 + }, + { + "avg_n_trades": 167.32000732421875, + "cost": 0.125, + "mean_reward": -9.114399909973145, + "n_episodes": 300, + "p05": -27.226896286010742, + "p50": -6.687276363372803, + "p95": 0.4474220275878906, + "sharpe_annualised": -29.17302703857422, + "sharpe_per_episode": -1.0193880796432495, + "std_reward": 8.941049575805664, + "threshold": 0.20000000298023224, + "win_rate": 0.06333333253860474 + }, + { + "avg_n_trades": 166.086669921875, + "cost": 0.25, + "mean_reward": -15.02822208404541, + "n_episodes": 300, + "p05": -36.3570556640625, + "p50": -12.731026649475098, + "p95": -4.7999491691589355, + "sharpe_annualised": -46.5723991394043, + "sharpe_per_episode": -1.6273713111877441, + "std_reward": 9.234661102294922, + "threshold": 0.20000000298023224, + "win_rate": 0.006666666828095913 + }, + { + "avg_n_trades": 175.14666748046875, + "cost": 0.5, + "mean_reward": -27.366077423095703, + "n_episodes": 300, + "p05": -50.28740692138672, + "p50": -23.888004302978516, + "p95": -11.858697891235352, + "sharpe_annualised": -62.76474380493164, + "sharpe_per_episode": -2.1931777000427246, + "std_reward": 12.477821350097656, + "threshold": 0.20000000298023224, + "win_rate": 0.0 + }, + { + "avg_n_trades": 136.49000549316406, + "cost": 0.0, + "mean_reward": -5.5052690505981445, + "n_episodes": 300, + "p05": -21.544471740722656, + "p50": -2.841050148010254, + "p95": 2.903995990753174, + "sharpe_annualised": -18.832592010498047, + "sharpe_per_episode": -0.6580640077590942, + "std_reward": 8.365856170654297, + "threshold": 0.25, + "win_rate": 0.17000000178813934 + }, + { + "avg_n_trades": 138.30332946777344, + "cost": 0.0625, + "mean_reward": -7.428314685821533, + "n_episodes": 300, + "p05": -25.75029754638672, + "p50": -4.445875644683838, + "p95": 2.239424467086792, + "sharpe_annualised": -21.298498153686523, + "sharpe_per_episode": -0.7442297339439392, + "std_reward": 9.98121166229248, + "threshold": 0.25, + "win_rate": 0.10999999940395355 + }, + { + "avg_n_trades": 133.67333984375, + "cost": 0.125, + "mean_reward": -9.272025108337402, + "n_episodes": 300, + "p05": -26.37662696838379, + "p50": -6.8429741859436035, + "p95": -0.16450214385986328, + "sharpe_annualised": -33.544925689697266, + "sharpe_per_episode": -1.1721545457839966, + "std_reward": 7.910241603851318, + "threshold": 0.25, + "win_rate": 0.05000000074505806 + }, + { + "avg_n_trades": 132.44667053222656, + "cost": 0.25, + "mean_reward": -14.290428161621094, + "n_episodes": 300, + "p05": -33.351287841796875, + "p50": -10.576525688171387, + "p95": -4.033750057220459, + "sharpe_annualised": -42.11674118041992, + "sharpe_per_episode": -1.4716780185699463, + "std_reward": 9.710295677185059, + "threshold": 0.25, + "win_rate": 0.006666666828095913 + }, + { + "avg_n_trades": 136.05332946777344, + "cost": 0.5, + "mean_reward": -21.997268676757812, + "n_episodes": 300, + "p05": -41.8651237487793, + "p50": -20.465248107910156, + "p95": -7.810499668121338, + "sharpe_annualised": -54.85932159423828, + "sharpe_per_episode": -1.9169398546218872, + "std_reward": 11.475200653076172, + "threshold": 0.25, + "win_rate": 0.0033333334140479565 } ], "cost_grid": [ @@ -69,9 +429,17 @@ 0.5 ], "horizon": 600, - "n_eval_episodes": 500, + "n_eval_episodes": 300, "n_train_episodes": 1000, - "phase": "E.3 Task 23", + "phase": "E.3 Task 23 (2D sweep)", + "threshold_grid": [ + 0.0, + 0.05000000074505806, + 0.10000000149011612, + 0.15000000596046448, + 0.20000000298023224, + 0.25 + ], "train_cost": 0.0625, "train_frac": 0.800000011920929 } \ No newline at end of file diff --git a/crates/ml/examples/alpha_compose_backtest.rs b/crates/ml/examples/alpha_compose_backtest.rs index 3de2b1fc3..82c51b83d 100644 --- a/crates/ml/examples/alpha_compose_backtest.rs +++ b/crates/ml/examples/alpha_compose_backtest.rs @@ -84,6 +84,12 @@ struct Cli { /// Phase 1d.4 used [0.0, 0.0625, 0.125, 0.25, 0.50]. #[arg(long, value_delimiter = ',', default_value = "0.0,0.0625,0.125,0.25,0.5")] cost_grid: Vec, + /// Comma-separated alpha-confidence threshold grid for the gate at eval. + /// Direct analogue of Phase 1d.4's `--threshold` sweep — at each + /// threshold, the policy is forced to Wait when |sigmoid(alpha)−0.5| + /// < threshold. Threshold 0 = no gate (original Task 23 behaviour). + #[arg(long, value_delimiter = ',', default_value = "0.0,0.05,0.10,0.15,0.20,0.25")] + threshold_grid: Vec, #[arg(long, default_value_t = 1)] trade_size: i32, #[arg(long, default_value_t = 0xCAFEBABE_u64)] @@ -152,9 +158,25 @@ fn epsilon_greedy(q: &[f32], eps: f32, rng: &mut SmokeRng) -> u8 { } } +/// Phase E.3 confidence-gated ε-greedy. If `alpha_confidence < threshold`, +/// force action=0 (Wait). Otherwise standard ε-greedy. +fn epsilon_greedy_gated( + q: &[f32], + alpha_confidence: f32, + threshold: f32, + eps: f32, + rng: &mut SmokeRng, +) -> u8 { + if alpha_confidence < threshold { + return 0; + } + epsilon_greedy(q, eps, rng) +} + #[derive(Debug, Clone, serde::Serialize)] struct CostBin { cost: f32, + threshold: f32, n_episodes: usize, mean_reward: f32, std_reward: f32, @@ -163,6 +185,7 @@ struct CostBin { /// the eval window's mid_price index span × bar_seconds. sharpe_annualised: f32, win_rate: f32, + avg_n_trades: f32, p05: f32, p50: f32, p95: f32, @@ -434,16 +457,20 @@ fn main() -> Result<()> { info!("Training complete. Frozen policy ready for eval."); // --------------------------------------------------------------- - // Phase 2: COST SWEEP — frozen-policy greedy eval per cost. + // Phase 2: 2D SWEEP — frozen-policy greedy eval per (threshold, cost). + // Direct analogue of Phase 1d.4's threshold × cost table. // --------------------------------------------------------------- - info!("=== Eval phase: {} episodes per cost × {} costs ===", - cli.n_eval_episodes, cli.cost_grid.len()); + info!("=== Eval phase: {} episodes × {} thresholds × {} costs ===", + cli.n_eval_episodes, cli.threshold_grid.len(), cli.cost_grid.len()); let eval_max_start = (n_total - n_train).saturating_sub(cli.horizon + 1).max(1); - let mut bins: Vec = Vec::with_capacity(cli.cost_grid.len()); + let mut bins: Vec = + Vec::with_capacity(cli.cost_grid.len() * cli.threshold_grid.len()); + for &threshold in &cli.threshold_grid { for &cost in &cli.cost_grid { env.config.cost_per_contract = cost; let mut rewards: Vec = Vec::with_capacity(cli.n_eval_episodes); let mut win_count = 0_usize; + let mut trade_count_total: u64 = 0; for _ in 0..cli.n_eval_episodes { let start_cursor = n_train + (episode_rng.next_u64() as usize) % eval_max_start; @@ -451,6 +478,7 @@ fn main() -> Result<()> { env.reset_at(env_seed, start_cursor); let mut state = EpisodeState::new(); let mut terminal_r = 0.0_f32; + let mut ep_n_trades = 0_u32; loop { let s_vec = env.state(&state).to_vec(); stream.memcpy_htod(&s_vec, &mut single_state_dev)?; @@ -469,7 +497,16 @@ fn main() -> Result<()> { stream.synchronize()?; let q_host = stream.clone_dtoh(&single_q_dev)?; let mut greedy_rng = SmokeRng::new(0); // unused — eps=0 - let action = epsilon_greedy(&q_host, 0.0, &mut greedy_rng); + let action = epsilon_greedy_gated( + &q_host, + s_vec[1], // alpha_confidence + threshold, + 0.0, + &mut greedy_rng, + ); + if action != 0 { + ep_n_trades += 1; + } match env.step(action, &mut state) { Some((_, reward, done)) => { if done { @@ -481,6 +518,7 @@ fn main() -> Result<()> { } } rewards.push(terminal_r); + trade_count_total += ep_n_trades as u64; if terminal_r > 0.0 { win_count += 1; } @@ -494,14 +532,10 @@ fn main() -> Result<()> { / n; let std = var.sqrt().max(1e-6); let sharpe_per = (mean / std) as f32; - // Annualised Sharpe via per-episode Sharpe × sqrt(episodes/year). - // Episode duration ≈ horizon × bar_seconds. ES.FUT MBP-10 snapshots - // at snapshot_interval=50 ≈ 12 sec/bar on average; 600-bar horizon - // ≈ 7200 s ≈ 2 hours. Episodes per year ≈ 252 × 6.5 × 3600 / 7200 - // ≈ 819. Sharpe_ann ≈ Sharpe_per × sqrt(819) ≈ 28.6. let episodes_per_year = 252.0 * 6.5 * 3600.0 / (cli.horizon as f64 * 12.0); let sharpe_ann = (sharpe_per as f64 * episodes_per_year.sqrt()) as f32; let win_rate = win_count as f32 / cli.n_eval_episodes as f32; + let avg_n_trades = trade_count_total as f32 / cli.n_eval_episodes as f32; let mut sorted = rewards.clone(); sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); let pick = |q: f64| -> f32 { @@ -510,50 +544,72 @@ fn main() -> Result<()> { }; let bin = CostBin { cost, + threshold, n_episodes: cli.n_eval_episodes, mean_reward: mean as f32, std_reward: std as f32, sharpe_per_episode: sharpe_per, sharpe_annualised: sharpe_ann, win_rate, + avg_n_trades, p05: pick(0.05), p50: pick(0.50), p95: pick(0.95), }; info!( - " cost={:>6.4} mean={:>+9.2} std={:>8.2} Sharpe/ep={:+.3} Sharpe_ann={:+.3} win_rate={:.3}", - cost, mean, std, sharpe_per, sharpe_ann, win_rate + " τ={:.2} cost={:>6.4} mean={:>+9.2} Sharpe/ep={:+.3} Sharpe_ann={:+.3} win={:.3} trades/ep={:.1}", + threshold, cost, mean, sharpe_per, sharpe_ann, win_rate, avg_n_trades ); bins.push(bin); } + } // --- Print table --- info!(""); - info!("=== Phase E.3 cost-sweep table (vs Phase 1d.4 baseline) ==="); + info!("=== Phase E.3 2D sweep (threshold × cost, vs Phase 1d.4 baseline) ==="); info!( - " {:>6} {:>8} {:>9} {:>9} {:>10} {:>11} {:>8}", - "cost", "n_ep", "mean_R", "std_R", "Sharpe/ep", "Sharpe_ann", "win_rate" + " {:>6} {:>6} {:>9} {:>9} {:>10} {:>11} {:>8} {:>10}", + "τ", "cost", "mean_R", "std_R", "Sharpe/ep", "Sharpe_ann", "win_rate", "trades/ep" ); for b in &bins { info!( - " {:>6.4} {:>8} {:>9.2} {:>9.2} {:>10.4} {:>11.4} {:>8.3}", - b.cost, b.n_episodes, b.mean_reward, b.std_reward, - b.sharpe_per_episode, b.sharpe_annualised, b.win_rate + " {:>6.3} {:>6.4} {:>9.2} {:>9.2} {:>10.4} {:>11.4} {:>8.3} {:>10.2}", + b.threshold, b.cost, b.mean_reward, b.std_reward, + b.sharpe_per_episode, b.sharpe_annualised, b.win_rate, b.avg_n_trades ); } info!(""); + info!("BEST per-cost (max Sharpe_ann across all τ at each cost):"); + for &cost in &cli.cost_grid { + let best = bins + .iter() + .filter(|b| (b.cost - cost).abs() < 1e-6) + .max_by(|a, b| { + a.sharpe_annualised + .partial_cmp(&b.sharpe_annualised) + .unwrap_or(std::cmp::Ordering::Equal) + }); + if let Some(b) = best { + info!( + " cost={:>6.4} best τ={:.3} Sharpe_ann={:+.3} win={:.3} trades/ep={:.1}", + b.cost, b.threshold, b.sharpe_annualised, b.win_rate, b.avg_n_trades + ); + } + } + info!(""); info!("Phase 1d.4 baseline for comparison: +4.4 annualised at cost=0,"); info!(" -4.0 annualised at cost=0.125."); // --- Save JSON --- let json = serde_json::json!({ - "phase": "E.3 Task 23", + "phase": "E.3 Task 23 (2D sweep)", "horizon": cli.horizon, "train_frac": cli.train_frac, "n_train_episodes": cli.n_train_episodes, "n_eval_episodes": cli.n_eval_episodes, "train_cost": cli.train_cost, "cost_grid": cli.cost_grid, + "threshold_grid": cli.threshold_grid, "bins": bins, }); let mut f = File::create(&cli.out_path)?; diff --git a/crates/ml/examples/alpha_dqn_h600_smoke.rs b/crates/ml/examples/alpha_dqn_h600_smoke.rs index f721cb66d..5a6a76902 100644 --- a/crates/ml/examples/alpha_dqn_h600_smoke.rs +++ b/crates/ml/examples/alpha_dqn_h600_smoke.rs @@ -310,6 +310,24 @@ impl SmokeRng { } } +/// Phase E.3 confidence-gated ε-greedy. If `alpha_confidence < threshold`, +/// force action=0 (Wait) regardless of Q. Otherwise standard ε-greedy. +/// Both args are expected in [0, 0.5] (the state's +/// `alpha_confidence = |sigmoid(alpha_logit) − 0.5|`, and the controller +/// slot 543 which is clamped to [0, 0.5]). Threshold=0 disables the gate. +fn epsilon_greedy_gated( + q: &[f32], + alpha_confidence: f32, + threshold: f32, + eps: f32, + rng: &mut SmokeRng, +) -> u8 { + if alpha_confidence < threshold { + return 0; // Wait + } + epsilon_greedy(q, eps, rng) +} + /// Picks an action via ε-greedy: with probability `eps` random, otherwise /// argmax over `q`. Returns the action index in `[0, N_ACTIONS)`. fn epsilon_greedy(q: &[f32], eps: f32, rng: &mut SmokeRng) -> u8 { @@ -508,6 +526,12 @@ fn main() -> Result<()> { // --- Training loop --- let mut episode_rng = SmokeRng::new(cli.seed.wrapping_add(0xFEED)); let mut action_counts_host = vec![0_i32; N_ACTIONS]; + // Phase E.3: cached threshold from controller (slot 543). Updated after + // each rollout-end controller invocation; used in action selection for + // the NEXT episode. Initialised to 0 (no gate) so the first episode's + // exploration is unrestricted, letting the controller accumulate enough + // observations to drive a meaningful threshold. + let mut current_threshold: f32 = 0.0; let mut recent_returns: Vec = Vec::new(); let max_start = n_rows.saturating_sub(cli.horizon + 1).max(1); @@ -547,7 +571,13 @@ fn main() -> Result<()> { } stream.synchronize().context("sync after per-step fwd")?; let q_host = stream.clone_dtoh(&single_q_dev).context("dtoh Q")?; - let action = epsilon_greedy(&q_host, eps, &mut episode_rng); + let action = epsilon_greedy_gated( + &q_host, + s_vec[1], // alpha_confidence (state index 1) + current_threshold, + eps, + &mut episode_rng, + ); let (_next_state_arr, reward, done) = env.step(action, &mut state) .ok_or_else(|| anyhow::anyhow!("step returned None"))?; @@ -745,6 +775,12 @@ fn main() -> Result<()> { w_ptr, )?; } + // Phase E.3: refresh CPU-side threshold cache from device after the + // controller update, for the NEXT episode's gating decisions. + stream.synchronize().context("sync after controller")?; + let isv_now = stream.clone_dtoh(&isv_dev).context("dtoh isv post-controller")?; + current_threshold = + isv_now[ml::cuda_pipeline::alpha_isv_slots::STACKER_THRESHOLD_INDEX]; // Target network hard update every K episodes: w_target ← w_online. // Standard DQN stabilizer — V_soft(s') bootstrap reads w_target,