From eabc7ae36b6aedc3043abf819bfb293e1e71e810 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sun, 22 Mar 2026 21:45:00 +0100 Subject: [PATCH] =?UTF-8?q?feat:=20Phase=203=20reward=20tuning=20=E2=80=94?= =?UTF-8?q?=20fix=20dynamics+architecture,=20search=207=20reward=20dims?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three-phase hyperopt pipeline: Phase 1 (fast): fix architecture + reward, search dynamics (~17D) Phase 2 (full): fix dynamics, search architecture (~5D) Phase 3 (reward): fix dynamics + architecture, search reward weights (7D) Phase 3 is the fastest (~10s/trial) since only experience collection changes. CLI: --phase reward --hyperopt-params phase2_results.json Argo template runs all 3 phases sequentially. Phase 2 writes to _phase2_results.json, Phase 3 writes final _hyperopt_results.json. Co-Authored-By: Claude Opus 4.6 (1M context) --- crates/ml/examples/hyperopt_baseline_rl.rs | 55 +++++++++---------- crates/ml/src/hyperopt/adapters/dqn.rs | 20 ++++++- crates/ml/src/training_profile.rs | 9 ++- .../k8s/argo/compile-and-train-template.yaml | 27 ++++++++- 4 files changed, 76 insertions(+), 35 deletions(-) diff --git a/crates/ml/examples/hyperopt_baseline_rl.rs b/crates/ml/examples/hyperopt_baseline_rl.rs index e00288d51..e509c736d 100644 --- a/crates/ml/examples/hyperopt_baseline_rl.rs +++ b/crates/ml/examples/hyperopt_baseline_rl.rs @@ -195,14 +195,15 @@ struct Args { #[arg(long)] trades_data_dir: Option, - /// Two-phase hyperopt phase selection: - /// fast — Phase 1: fix architecture, search learning dynamics (~15D). DEFAULT. - /// full — Phase 2: fix dynamics from --hyperopt-params, search architecture (~5D). + /// Three-phase hyperopt phase selection: + /// fast — Phase 1: fix architecture + reward, search dynamics (~17D). DEFAULT. + /// full — Phase 2: fix dynamics, search architecture (~5D). + /// reward — Phase 3: fix dynamics + architecture, search reward weights only (7D, fastest). #[arg(long, default_value = "fast")] phase: String, - /// Path to Phase 1 JSON results (required for --phase full). - /// Contains the best continuous parameter vector from Phase 1's output. + /// Path to previous phase's JSON results (required for --phase full and --phase reward). + /// Contains the best continuous parameter vector from the previous phase. #[arg(long)] hyperopt_params: Option, } @@ -516,32 +517,28 @@ fn main() -> Result<()> { // This must happen before continuous_bounds() is called. { use ml::training_profile::{set_hyperopt_phase, HyperoptPhase}; + let load_best_vec = |phase_name: &str| -> Vec { + let params_path = args.hyperopt_params.as_ref() + .unwrap_or_else(|| panic!("--hyperopt-params required for --phase {phase_name}")); + let params_json = std::fs::read_to_string(params_path) + .unwrap_or_else(|e| panic!("Failed to read --hyperopt-params: {e}")); + let parsed: serde_json::Value = serde_json::from_str(¶ms_json) + .unwrap_or_else(|e| panic!("Failed to parse --hyperopt-params JSON: {e}")); + let dqn_result = parsed.get("dqn") + .expect("JSON missing 'dqn' key"); + dqn_result.get("best_continuous_vector") + .expect("JSON missing 'best_continuous_vector'") + .as_array() + .expect("best_continuous_vector must be an array") + .iter() + .map(|v| v.as_f64().expect("best_continuous_vector values must be f64")) + .collect() + }; let phase = match args.phase.as_str() { "fast" => HyperoptPhase::Fast, - "full" => { - let params_path = args.hyperopt_params.as_ref() - .expect("--hyperopt-params required for --phase full"); - let params_json = std::fs::read_to_string(params_path) - .expect("Failed to read --hyperopt-params JSON"); - let parsed: serde_json::Value = serde_json::from_str(¶ms_json) - .expect("Failed to parse --hyperopt-params JSON"); - - // Extract the best_params continuous vector from Phase 1 output. - // Phase 1 output format: { "dqn": { "best_params": { ... } } } - // We need the to_continuous() representation — stored as "continuous_vector". - let dqn_result = parsed.get("dqn") - .expect("Phase 1 JSON missing 'dqn' key"); - let best_vec = dqn_result.get("best_continuous_vector") - .expect("Phase 1 JSON missing 'best_continuous_vector' — was Phase 1 run with a compatible version?") - .as_array() - .expect("best_continuous_vector must be an array") - .iter() - .map(|v| v.as_f64().expect("best_continuous_vector values must be f64")) - .collect::>(); - assert_eq!(best_vec.len(), 31, "best_continuous_vector must have 31 elements"); - HyperoptPhase::Full(best_vec) - } - other => panic!("Invalid --phase value '{}'. Must be 'fast' or 'full'.", other), + "full" => HyperoptPhase::Full(load_best_vec("full")), + "reward" => HyperoptPhase::Reward(load_best_vec("reward")), + other => panic!("Invalid --phase '{}'. Must be 'fast', 'full', or 'reward'.", other), }; info!("Hyperopt phase: {:?}", args.phase); set_hyperopt_phase(phase); diff --git a/crates/ml/src/hyperopt/adapters/dqn.rs b/crates/ml/src/hyperopt/adapters/dqn.rs index a4cf760d3..b5395bc4e 100644 --- a/crates/ml/src/hyperopt/adapters/dqn.rs +++ b/crates/ml/src/hyperopt/adapters/dqn.rs @@ -601,12 +601,17 @@ impl ParameterSpace for DQNParams { HyperoptPhase::Full(ref best_vec) => { // Phase 2: fix learning dynamics from Phase 1 best params. // Search only architecture dims (~5D free, ~33D fixed). - // The best_vec is the raw continuous vector from Phase 1. - // Fix all non-architecture dims to Phase 1's best values. // Architecture dims (indices 10, 13, 15, 21, 28) stay free. tracing::info!("Phase FULL: fixing {} dynamics params from Phase 1 best", best_vec.len()); // Bounds will be applied after vec construction below. } + HyperoptPhase::Reward(ref best_vec) => { + // Phase 3: fix dynamics + architecture from Phase 2 best params. + // Search ONLY reward weights (indices 31-37, 7D). + // All other dims fixed to Phase 2's best values. + tracing::info!("Phase REWARD: fixing {} dynamics+architecture params, searching 7 reward dims", best_vec.len()); + // Bounds will be applied after vec construction below. + } } let mut bounds = vec![ @@ -663,6 +668,17 @@ impl ParameterSpace for DQNParams { } } + // Phase REWARD: fix ALL dims EXCEPT reward weights (indices 31-37). + // This is the fastest phase — only 7D search, ~10s/trial. + if let HyperoptPhase::Reward(ref best_vec) = phase { + let reward_dims: &[usize] = &[31, 32, 33, 34, 35, 36, 37]; + for (i, bv) in best_vec.iter().enumerate() { + if i < bounds.len() && !reward_dims.contains(&i) { + bounds[i] = (*bv, *bv); + } + } + } + bounds } diff --git a/crates/ml/src/training_profile.rs b/crates/ml/src/training_profile.rs index 1cae538a2..97af0a765 100644 --- a/crates/ml/src/training_profile.rs +++ b/crates/ml/src/training_profile.rs @@ -277,15 +277,18 @@ pub struct PhaseFastSection { pub time_decay_rate: Option, } -/// Two-phase hyperopt configuration. +/// Three-phase hyperopt configuration. /// Set via `HYPEROPT_PHASE` static before running optimization. #[derive(Debug, Clone, PartialEq)] pub enum HyperoptPhase { - /// Phase 1 (default): fix architecture, search learning dynamics (~15D). + /// Phase 1 (default): fix architecture + reward, search learning dynamics. Fast, - /// Phase 2: fix dynamics from Phase 1 JSON, search architecture (~5D). + /// Phase 2: fix dynamics from Phase 1 JSON, search architecture. /// Contains the best continuous parameter vector from Phase 1. Full(Vec), + /// Phase 3: fix dynamics + architecture from Phase 2 JSON, search only reward weights (7D). + /// Fastest phase (~10s/trial) — only experience collection changes, not network or training. + Reward(Vec), } impl Default for HyperoptPhase { diff --git a/infra/k8s/argo/compile-and-train-template.yaml b/infra/k8s/argo/compile-and-train-template.yaml index deb2da495..63419dee7 100644 --- a/infra/k8s/argo/compile-and-train-template.yaml +++ b/infra/k8s/argo/compile-and-train-template.yaml @@ -441,9 +441,34 @@ spec: --mbp10-data-dir {{workflow.parameters.mbp10-data-dir}} \ --trades-data-dir {{workflow.parameters.trades-data-dir}} \ --base-dir /workspace/output/hyperopt \ + --output /workspace/output/${MODEL}_phase2_results.json + + echo "=== Phase 2 complete ===" + cat /workspace/output/${MODEL}_phase2_results.json 2>/dev/null || echo "No Phase 2 results" + + # Phase 3 (reward): fix dynamics + architecture, search only reward weights (7D) + PHASE3_TRIALS=$(({{workflow.parameters.hyperopt-trials}} / 2)) + [ "$PHASE3_TRIALS" -lt 5 ] && PHASE3_TRIALS=5 + PHASE3_EPOCHS=$(({{workflow.parameters.hyperopt-epochs}} * 2)) + + ${BINARY} \ + --model "$MODEL" \ + --phase reward \ + --hyperopt-params /workspace/output/${MODEL}_phase2_results.json \ + --trials $PHASE3_TRIALS \ + --epochs $PHASE3_EPOCHS \ + --symbol {{workflow.parameters.symbol}} \ + --tx-cost-bps {{workflow.parameters.tx-cost-bps}} \ + --tick-size {{workflow.parameters.tick-size}} \ + --spread-ticks {{workflow.parameters.spread-ticks}} \ + --initial-capital {{workflow.parameters.initial-capital}} \ + --data-dir {{workflow.parameters.data-dir}} \ + --mbp10-data-dir {{workflow.parameters.mbp10-data-dir}} \ + --trades-data-dir {{workflow.parameters.trades-data-dir}} \ + --base-dir /workspace/output/hyperopt \ --output /workspace/output/${MODEL}_hyperopt_results.json - echo "=== Two-phase hyperopt complete ===" + echo "=== Three-phase hyperopt complete ===" cat /workspace/output/${MODEL}_hyperopt_results.json 2>/dev/null || echo "No results file" resources: requests: