From ca94c0e8516c2c7df45c4855da6564cd2c3fb099 Mon Sep 17 00:00:00 2001
From: jgrusewski <jgrusewski@localhost>
Date: Sun, 22 Mar 2026 10:39:36 +0100
Subject: [PATCH] feat: wire two-phase hyperopt into Argo training workflows

Phase 1 writes ${MODEL}_phase1_results.json to PVC, Phase 2 reads it
via --hyperopt-params. Phase 2 uses half the trials but double epochs.
train-best downstream reads final ${MODEL}_hyperopt_results.json unchanged.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 .../k8s/argo/compile-and-train-template.yaml  | 29 +++++++++++++++++-
 .../k8s/argo/training-workflow-template.yaml  | 30 ++++++++++++++++++-
 2 files changed, 57 insertions(+), 2 deletions(-)

diff --git a/infra/k8s/argo/compile-and-train-template.yaml b/infra/k8s/argo/compile-and-train-template.yaml
index c17973c2d..1e5be9635 100644
--- a/infra/k8s/argo/compile-and-train-template.yaml
+++ b/infra/k8s/argo/compile-and-train-template.yaml
@@ -398,8 +398,10 @@ spec:
             echo "=== Running hyperopt: $MODEL (${BINARY}) ==="
             echo "  Trials: {{workflow.parameters.hyperopt-trials}}, Epochs: {{workflow.parameters.hyperopt-epochs}}"
 
+            # Phase 1 (fast): fix architecture, search learning dynamics
             ${BINARY} \
               --model "$MODEL" \
+              --phase fast \
               --trials {{workflow.parameters.hyperopt-trials}} \
               --epochs {{workflow.parameters.hyperopt-epochs}} \
               --symbol {{workflow.parameters.symbol}} \
@@ -411,9 +413,34 @@ spec:
               --mbp10-data-dir {{workflow.parameters.mbp10-data-dir}} \
               --trades-data-dir {{workflow.parameters.trades-data-dir}} \
               --base-dir /workspace/output/hyperopt \
+              --output /workspace/output/${MODEL}_phase1_results.json
+
+            echo "=== Phase 1 complete ==="
+            cat /workspace/output/${MODEL}_phase1_results.json 2>/dev/null || echo "No Phase 1 results"
+
+            # Phase 2 (full): fix dynamics from Phase 1, search architecture
+            PHASE2_TRIALS=$(({{workflow.parameters.hyperopt-trials}} / 2))
+            [ "$PHASE2_TRIALS" -lt 5 ] && PHASE2_TRIALS=5
+            PHASE2_EPOCHS=$(({{workflow.parameters.hyperopt-epochs}} * 2))
+
+            ${BINARY} \
+              --model "$MODEL" \
+              --phase full \
+              --hyperopt-params /workspace/output/${MODEL}_phase1_results.json \
+              --trials $PHASE2_TRIALS \
+              --epochs $PHASE2_EPOCHS \
+              --symbol {{workflow.parameters.symbol}} \
+              --tx-cost-bps {{workflow.parameters.tx-cost-bps}} \
+              --tick-size {{workflow.parameters.tick-size}} \
+              --spread-ticks {{workflow.parameters.spread-ticks}} \
+              --initial-capital {{workflow.parameters.initial-capital}} \
+              --data-dir {{workflow.parameters.data-dir}} \
+              --mbp10-data-dir {{workflow.parameters.mbp10-data-dir}} \
+              --trades-data-dir {{workflow.parameters.trades-data-dir}} \
+              --base-dir /workspace/output/hyperopt \
               --output /workspace/output/${MODEL}_hyperopt_results.json
 
-            echo "=== Hyperopt complete ==="
+            echo "=== Two-phase hyperopt complete ==="
             cat /workspace/output/${MODEL}_hyperopt_results.json 2>/dev/null || echo "No results file"
         resources:
           requests:
diff --git a/infra/k8s/argo/training-workflow-template.yaml b/infra/k8s/argo/training-workflow-template.yaml
index 1fa263bcd..35a0477da 100644
--- a/infra/k8s/argo/training-workflow-template.yaml
+++ b/infra/k8s/argo/training-workflow-template.yaml
@@ -221,8 +221,10 @@ spec:
             fi
 
             echo "Running $BINARY --model $MODEL ($TRIALS trials, $N_INITIAL initial x {{workflow.parameters.hyperopt-epochs}} epochs)"
+            # Phase 1 (fast): fix architecture, search learning dynamics
             $BINARY \
               --model "$MODEL" \
+              --phase fast \
               --trials "$TRIALS" \
               --n-initial "$N_INITIAL" \
               --epochs {{workflow.parameters.hyperopt-epochs}} \
@@ -236,9 +238,35 @@ spec:
               --mbp10-data-dir {{workflow.parameters.mbp10-data-dir}} \
               --trades-data-dir {{workflow.parameters.trades-data-dir}} \
               --base-dir /workspace/output/hyperopt \
+              --output /workspace/output/${MODEL}_phase1_results.json
+
+            echo "=== Phase 1 complete ==="
+            cat /workspace/output/${MODEL}_phase1_results.json 2>/dev/null || echo "No Phase 1 results"
+
+            # Phase 2 (full): fix dynamics from Phase 1, search architecture
+            PHASE2_TRIALS=$((TRIALS / 2))
+            [ "$PHASE2_TRIALS" -lt 5 ] && PHASE2_TRIALS=5
+            PHASE2_EPOCHS=$(({{workflow.parameters.hyperopt-epochs}} * 2))
+
+            $BINARY \
+              --model "$MODEL" \
+              --phase full \
+              --hyperopt-params /workspace/output/${MODEL}_phase1_results.json \
+              --trials $PHASE2_TRIALS \
+              --epochs $PHASE2_EPOCHS \
+              --parallel 0 \
+              --symbol {{workflow.parameters.symbol}} \
+              --tx-cost-bps {{workflow.parameters.tx-cost-bps}} \
+              --tick-size {{workflow.parameters.tick-size}} \
+              --spread-ticks {{workflow.parameters.spread-ticks}} \
+              --initial-capital {{workflow.parameters.initial-capital}} \
+              --data-dir {{workflow.parameters.data-dir}} \
+              --mbp10-data-dir {{workflow.parameters.mbp10-data-dir}} \
+              --trades-data-dir {{workflow.parameters.trades-data-dir}} \
+              --base-dir /workspace/output/hyperopt \
               --output /workspace/output/${MODEL}_hyperopt_results.json
 
-            echo "=== Hyperopt results ==="
+            echo "=== Two-phase hyperopt complete ==="
             cat /workspace/output/${MODEL}_hyperopt_results.json 2>/dev/null || echo "No results file"
         env:
           - name: RUST_LOG