feat: Phase 3 reward tuning — fix dynamics+architecture, search 7 reward dims

Three-phase hyperopt pipeline:
  Phase 1 (fast): fix architecture + reward, search dynamics (~17D)
  Phase 2 (full): fix dynamics, search architecture (~5D)
  Phase 3 (reward): fix dynamics + architecture, search reward weights (7D)

Phase 3 is the fastest (~10s/trial) since only experience collection
changes. CLI: --phase reward --hyperopt-params phase2_results.json

Argo template runs all 3 phases sequentially. Phase 2 writes to
_phase2_results.json, Phase 3 writes final _hyperopt_results.json.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

crates/ml/examples/hyperopt_baseline_rl.rs

@@ -195,14 +195,15 @@ struct Args {
     #[arg(long)]
     trades_data_dir: Option<PathBuf>,
 
-    /// Two-phase hyperopt phase selection:
-    ///   fast — Phase 1: fix architecture, search learning dynamics (~15D). DEFAULT.
-    ///   full — Phase 2: fix dynamics from --hyperopt-params, search architecture (~5D).
+    /// Three-phase hyperopt phase selection:
+    ///   fast   — Phase 1: fix architecture + reward, search dynamics (~17D). DEFAULT.
+    ///   full   — Phase 2: fix dynamics, search architecture (~5D).
+    ///   reward — Phase 3: fix dynamics + architecture, search reward weights only (7D, fastest).
     #[arg(long, default_value = "fast")]
     phase: String,
 
-    /// Path to Phase 1 JSON results (required for --phase full).
-    /// Contains the best continuous parameter vector from Phase 1's output.
+    /// Path to previous phase's JSON results (required for --phase full and --phase reward).
+    /// Contains the best continuous parameter vector from the previous phase.
     #[arg(long)]
     hyperopt_params: Option<PathBuf>,
 }
@@ -516,32 +517,28 @@ fn main() -> Result<()> {
     // This must happen before continuous_bounds() is called.
     {
         use ml::training_profile::{set_hyperopt_phase, HyperoptPhase};
+        let load_best_vec = |phase_name: &str| -> Vec<f64> {
+            let params_path = args.hyperopt_params.as_ref()
+                .unwrap_or_else(|| panic!("--hyperopt-params required for --phase {phase_name}"));
+            let params_json = std::fs::read_to_string(params_path)
+                .unwrap_or_else(|e| panic!("Failed to read --hyperopt-params: {e}"));
+            let parsed: serde_json::Value = serde_json::from_str(&params_json)
+                .unwrap_or_else(|e| panic!("Failed to parse --hyperopt-params JSON: {e}"));
+            let dqn_result = parsed.get("dqn")
+                .expect("JSON missing 'dqn' key");
+            dqn_result.get("best_continuous_vector")
+                .expect("JSON missing 'best_continuous_vector'")
+                .as_array()
+                .expect("best_continuous_vector must be an array")
+                .iter()
+                .map(|v| v.as_f64().expect("best_continuous_vector values must be f64"))
+                .collect()
+        };
         let phase = match args.phase.as_str() {
             "fast" => HyperoptPhase::Fast,
-            "full" => {
-                let params_path = args.hyperopt_params.as_ref()
-                    .expect("--hyperopt-params required for --phase full");
-                let params_json = std::fs::read_to_string(params_path)
-                    .expect("Failed to read --hyperopt-params JSON");
-                let parsed: serde_json::Value = serde_json::from_str(&params_json)
-                    .expect("Failed to parse --hyperopt-params JSON");
-
-                // Extract the best_params continuous vector from Phase 1 output.
-                // Phase 1 output format: { "dqn": { "best_params": { ... } } }
-                // We need the to_continuous() representation — stored as "continuous_vector".
-                let dqn_result = parsed.get("dqn")
-                    .expect("Phase 1 JSON missing 'dqn' key");
-                let best_vec = dqn_result.get("best_continuous_vector")
-                    .expect("Phase 1 JSON missing 'best_continuous_vector' — was Phase 1 run with a compatible version?")
-                    .as_array()
-                    .expect("best_continuous_vector must be an array")
-                    .iter()
-                    .map(|v| v.as_f64().expect("best_continuous_vector values must be f64"))
-                    .collect::<Vec<f64>>();
-                assert_eq!(best_vec.len(), 31, "best_continuous_vector must have 31 elements");
-                HyperoptPhase::Full(best_vec)
-            }
-            other => panic!("Invalid --phase value '{}'. Must be 'fast' or 'full'.", other),
+            "full" => HyperoptPhase::Full(load_best_vec("full")),
+            "reward" => HyperoptPhase::Reward(load_best_vec("reward")),
+            other => panic!("Invalid --phase '{}'. Must be 'fast', 'full', or 'reward'.", other),
         };
         info!("Hyperopt phase: {:?}", args.phase);
         set_hyperopt_phase(phase);

crates/ml/src/hyperopt/adapters/dqn.rs

@@ -601,12 +601,17 @@ impl ParameterSpace for DQNParams {
             HyperoptPhase::Full(ref best_vec) => {
                 // Phase 2: fix learning dynamics from Phase 1 best params.
                 // Search only architecture dims (~5D free, ~33D fixed).
-                // The best_vec is the raw continuous vector from Phase 1.
-                // Fix all non-architecture dims to Phase 1's best values.
                 // Architecture dims (indices 10, 13, 15, 21, 28) stay free.
                 tracing::info!("Phase FULL: fixing {} dynamics params from Phase 1 best", best_vec.len());
                 // Bounds will be applied after vec construction below.
             }
+            HyperoptPhase::Reward(ref best_vec) => {
+                // Phase 3: fix dynamics + architecture from Phase 2 best params.
+                // Search ONLY reward weights (indices 31-37, 7D).
+                // All other dims fixed to Phase 2's best values.
+                tracing::info!("Phase REWARD: fixing {} dynamics+architecture params, searching 7 reward dims", best_vec.len());
+                // Bounds will be applied after vec construction below.
+            }
         }
 
         let mut bounds = vec![
@@ -663,6 +668,17 @@ impl ParameterSpace for DQNParams {
             }
         }
 
+        // Phase REWARD: fix ALL dims EXCEPT reward weights (indices 31-37).
+        // This is the fastest phase — only 7D search, ~10s/trial.
+        if let HyperoptPhase::Reward(ref best_vec) = phase {
+            let reward_dims: &[usize] = &[31, 32, 33, 34, 35, 36, 37];
+            for (i, bv) in best_vec.iter().enumerate() {
+                if i < bounds.len() && !reward_dims.contains(&i) {
+                    bounds[i] = (*bv, *bv);
+                }
+            }
+        }
+
         bounds
     }
 

crates/ml/src/training_profile.rs

@@ -277,15 +277,18 @@ pub struct PhaseFastSection {
     pub time_decay_rate: Option<f64>,
 }
 
-/// Two-phase hyperopt configuration.
+/// Three-phase hyperopt configuration.
 /// Set via `HYPEROPT_PHASE` static before running optimization.
 #[derive(Debug, Clone, PartialEq)]
 pub enum HyperoptPhase {
-    /// Phase 1 (default): fix architecture, search learning dynamics (~15D).
+    /// Phase 1 (default): fix architecture + reward, search learning dynamics.
     Fast,
-    /// Phase 2: fix dynamics from Phase 1 JSON, search architecture (~5D).
+    /// Phase 2: fix dynamics from Phase 1 JSON, search architecture.
     /// Contains the best continuous parameter vector from Phase 1.
     Full(Vec<f64>),
+    /// Phase 3: fix dynamics + architecture from Phase 2 JSON, search only reward weights (7D).
+    /// Fastest phase (~10s/trial) — only experience collection changes, not network or training.
+    Reward(Vec<f64>),
 }
 
 impl Default for HyperoptPhase {

infra/k8s/argo/compile-and-train-template.yaml

@@ -441,9 +441,34 @@ spec:
               --mbp10-data-dir {{workflow.parameters.mbp10-data-dir}} \
               --trades-data-dir {{workflow.parameters.trades-data-dir}} \
               --base-dir /workspace/output/hyperopt \
+              --output /workspace/output/${MODEL}_phase2_results.json
+
+            echo "=== Phase 2 complete ==="
+            cat /workspace/output/${MODEL}_phase2_results.json 2>/dev/null || echo "No Phase 2 results"
+
+            # Phase 3 (reward): fix dynamics + architecture, search only reward weights (7D)
+            PHASE3_TRIALS=$(({{workflow.parameters.hyperopt-trials}} / 2))
+            [ "$PHASE3_TRIALS" -lt 5 ] && PHASE3_TRIALS=5
+            PHASE3_EPOCHS=$(({{workflow.parameters.hyperopt-epochs}} * 2))
+
+            ${BINARY} \
+              --model "$MODEL" \
+              --phase reward \
+              --hyperopt-params /workspace/output/${MODEL}_phase2_results.json \
+              --trials $PHASE3_TRIALS \
+              --epochs $PHASE3_EPOCHS \
+              --symbol {{workflow.parameters.symbol}} \
+              --tx-cost-bps {{workflow.parameters.tx-cost-bps}} \
+              --tick-size {{workflow.parameters.tick-size}} \
+              --spread-ticks {{workflow.parameters.spread-ticks}} \
+              --initial-capital {{workflow.parameters.initial-capital}} \
+              --data-dir {{workflow.parameters.data-dir}} \
+              --mbp10-data-dir {{workflow.parameters.mbp10-data-dir}} \
+              --trades-data-dir {{workflow.parameters.trades-data-dir}} \
+              --base-dir /workspace/output/hyperopt \
               --output /workspace/output/${MODEL}_hyperopt_results.json
 
-            echo "=== Two-phase hyperopt complete ==="
+            echo "=== Three-phase hyperopt complete ==="
             cat /workspace/output/${MODEL}_hyperopt_results.json 2>/dev/null || echo "No results file"
         resources:
           requests: