From 914ecd6c890a0b944c80494a58486a4bbfddbafe Mon Sep 17 00:00:00 2001
From: jgrusewski <jgrusewski@localhost>
Date: Fri, 27 Feb 2026 11:05:58 +0100
Subject: [PATCH] fix(ci): ensure evaluate always runs after training, even on
 failure

Training and hyperopt jobs now capture exit codes and continue to
evaluation step regardless. Job still fails if training failed, but
eval output/artifacts are always available for debugging.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 .gitlab-ci.yml | 94 ++++++++++++++++++++++++++++----------------------
 1 file changed, 53 insertions(+), 41 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 8d721ca1d..930cf9dfb 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -461,21 +461,26 @@ train-validate-rl:
   extends: .train-validate-base
   needs: [build-training]
   script:
-    - /usr/local/bin/train_baseline_rl
-        --model both
-        --symbol ES.FUT
-        --data-dir /mnt/training-data/futures-baseline
-        --output-dir ${CI_PROJECT_DIR}/output
-        --max-steps-per-epoch 50
-    - echo "=== RL training complete, running evaluation ==="
-    - /usr/local/bin/evaluate_baseline
-        --model both
-        --models-dir ${CI_PROJECT_DIR}/output
-        --data-dir /mnt/training-data/futures-baseline
-        --output ${CI_PROJECT_DIR}/output/rl_eval_report.json
-        --symbol ES.FUT
-    - echo "RL validation + evaluation passed"
-    - cat ${CI_PROJECT_DIR}/output/rl_eval_report.json
+    - |
+      /usr/local/bin/train_baseline_rl \
+        --model both \
+        --symbol ES.FUT \
+        --data-dir /mnt/training-data/futures-baseline \
+        --output-dir ${CI_PROJECT_DIR}/output \
+        --max-steps-per-epoch 50 \
+      && TRAIN_OK=1 || TRAIN_OK=0
+      echo "=== RL training exit: $TRAIN_OK ==="
+      echo "=== Running evaluation ==="
+      /usr/local/bin/evaluate_baseline \
+        --model both \
+        --models-dir ${CI_PROJECT_DIR}/output \
+        --data-dir /mnt/training-data/futures-baseline \
+        --output ${CI_PROJECT_DIR}/output/rl_eval_report.json \
+        --symbol ES.FUT \
+      || true
+      echo "=== Eval report ==="
+      cat ${CI_PROJECT_DIR}/output/rl_eval_report.json 2>/dev/null || echo "No eval report"
+      [ "$TRAIN_OK" = "1" ] || exit 1
   artifacts:
     paths:
       - output/rl_eval_report.json
@@ -729,32 +734,39 @@ hyperopt-dqn:
   needs: [build-training]
   timeout: 4h
   script:
-    - /usr/local/bin/hyperopt_baseline_rl
-        --model dqn
-        --trials 20
-        --n-initial 5
-        --epochs 15
-        --symbol ES.FUT
-        --data-dir /mnt/training-data/futures-baseline
-        --base-dir ${CI_PROJECT_DIR}/output/hyperopt
-        --output ${CI_PROJECT_DIR}/output/dqn_hyperopt_results.json
-    - echo "=== DQN Hyperopt complete ==="
-    - cat ${CI_PROJECT_DIR}/output/dqn_hyperopt_results.json
-    - echo "=== Training DQN with best params ==="
-    - /usr/local/bin/train_baseline_rl
-        --model dqn
-        --symbol ES.FUT
-        --data-dir /mnt/training-data/futures-baseline
-        --output-dir ${CI_PROJECT_DIR}/output
-        --hyperopt-params ${CI_PROJECT_DIR}/output/dqn_hyperopt_results.json
-    - echo "=== Evaluating DQN with best params ==="
-    - /usr/local/bin/evaluate_baseline
-        --model dqn
-        --models-dir ${CI_PROJECT_DIR}/output
-        --data-dir /mnt/training-data/futures-baseline
-        --output ${CI_PROJECT_DIR}/output/dqn_eval_report.json
-        --symbol ES.FUT
-    - cat ${CI_PROJECT_DIR}/output/dqn_eval_report.json
+    - |
+      /usr/local/bin/hyperopt_baseline_rl \
+        --model dqn \
+        --trials 20 \
+        --n-initial 5 \
+        --epochs 15 \
+        --symbol ES.FUT \
+        --data-dir /mnt/training-data/futures-baseline \
+        --base-dir ${CI_PROJECT_DIR}/output/hyperopt \
+        --output ${CI_PROJECT_DIR}/output/dqn_hyperopt_results.json \
+      && HYPEROPT_OK=1 || HYPEROPT_OK=0
+      echo "=== DQN Hyperopt exit: $HYPEROPT_OK ==="
+      cat ${CI_PROJECT_DIR}/output/dqn_hyperopt_results.json 2>/dev/null || echo "No hyperopt results"
+      echo "=== Training DQN with best params ==="
+      /usr/local/bin/train_baseline_rl \
+        --model dqn \
+        --symbol ES.FUT \
+        --data-dir /mnt/training-data/futures-baseline \
+        --output-dir ${CI_PROJECT_DIR}/output \
+        --hyperopt-params ${CI_PROJECT_DIR}/output/dqn_hyperopt_results.json \
+      && TRAIN_OK=1 || TRAIN_OK=0
+      echo "=== Training exit: $TRAIN_OK ==="
+      echo "=== Evaluating DQN ==="
+      /usr/local/bin/evaluate_baseline \
+        --model dqn \
+        --models-dir ${CI_PROJECT_DIR}/output \
+        --data-dir /mnt/training-data/futures-baseline \
+        --output ${CI_PROJECT_DIR}/output/dqn_eval_report.json \
+        --symbol ES.FUT \
+      || true
+      echo "=== Eval report ==="
+      cat ${CI_PROJECT_DIR}/output/dqn_eval_report.json 2>/dev/null || echo "No eval report"
+      [ "$HYPEROPT_OK" = "1" ] && [ "$TRAIN_OK" = "1" ] || exit 1
   artifacts:
     paths:
       - output/dqn_hyperopt_results.json