gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(ci): ensure evaluate always runs after training, even on failure

Browse Source 
Training and hyperopt jobs now capture exit codes and continue to
evaluation step regardless. Job still fails if training failed, but
eval output/artifacts are always available for debugging.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-02-27 11:05:58 +01:00
parent 6e6cee733e
commit 914ecd6c89
1 changed files with 53 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

94
.gitlab-ci.yml
View File

@@ -461,21 +461,26 @@ train-validate-rl:
extends: .train-validate-base
needs: [build-training]
script:
- /usr/local/bin/train_baseline_rl
--model both
--symbol ES.FUT
--data-dir /mnt/training-data/futures-baseline
--output-dir ${CI_PROJECT_DIR}/output
--max-steps-per-epoch 50
- echo "=== RL training complete, running evaluation ==="
- /usr/local/bin/evaluate_baseline
--model both
--models-dir ${CI_PROJECT_DIR}/output
--data-dir /mnt/training-data/futures-baseline
--output ${CI_PROJECT_DIR}/output/rl_eval_report.json
--symbol ES.FUT
- echo "RL validation + evaluation passed"
- cat ${CI_PROJECT_DIR}/output/rl_eval_report.json
- |
/usr/local/bin/train_baseline_rl \
--model both \
--symbol ES.FUT \
--data-dir /mnt/training-data/futures-baseline \
--output-dir ${CI_PROJECT_DIR}/output \
--max-steps-per-epoch 50 \
&& TRAIN_OK=1 || TRAIN_OK=0
echo "=== RL training exit: $TRAIN_OK ==="
echo "=== Running evaluation ==="
/usr/local/bin/evaluate_baseline \
--model both \
--models-dir ${CI_PROJECT_DIR}/output \
--data-dir /mnt/training-data/futures-baseline \
--output ${CI_PROJECT_DIR}/output/rl_eval_report.json \
--symbol ES.FUT \
|| true
echo "=== Eval report ==="
cat ${CI_PROJECT_DIR}/output/rl_eval_report.json 2>/dev/null || echo "No eval report"
[ "$TRAIN_OK" = "1" ] || exit 1
artifacts:
paths:
- output/rl_eval_report.json
@@ -729,32 +734,39 @@ hyperopt-dqn:
needs: [build-training]
timeout: 4h
script:
- /usr/local/bin/hyperopt_baseline_rl
--model dqn
--trials 20
--n-initial 5
--epochs 15
--symbol ES.FUT
--data-dir /mnt/training-data/futures-baseline
--base-dir ${CI_PROJECT_DIR}/output/hyperopt
--output ${CI_PROJECT_DIR}/output/dqn_hyperopt_results.json
- echo "=== DQN Hyperopt complete ==="
- cat ${CI_PROJECT_DIR}/output/dqn_hyperopt_results.json
- echo "=== Training DQN with best params ==="
- /usr/local/bin/train_baseline_rl
--model dqn
--symbol ES.FUT
--data-dir /mnt/training-data/futures-baseline
--output-dir ${CI_PROJECT_DIR}/output
--hyperopt-params ${CI_PROJECT_DIR}/output/dqn_hyperopt_results.json
- echo "=== Evaluating DQN with best params ==="
- /usr/local/bin/evaluate_baseline
--model dqn
--models-dir ${CI_PROJECT_DIR}/output
--data-dir /mnt/training-data/futures-baseline
--output ${CI_PROJECT_DIR}/output/dqn_eval_report.json
--symbol ES.FUT
- cat ${CI_PROJECT_DIR}/output/dqn_eval_report.json
- |
/usr/local/bin/hyperopt_baseline_rl \
--model dqn \
--trials 20 \
--n-initial 5 \
--epochs 15 \
--symbol ES.FUT \
--data-dir /mnt/training-data/futures-baseline \
--base-dir ${CI_PROJECT_DIR}/output/hyperopt \
--output ${CI_PROJECT_DIR}/output/dqn_hyperopt_results.json \
&& HYPEROPT_OK=1 || HYPEROPT_OK=0
echo "=== DQN Hyperopt exit: $HYPEROPT_OK ==="
cat ${CI_PROJECT_DIR}/output/dqn_hyperopt_results.json 2>/dev/null || echo "No hyperopt results"
echo "=== Training DQN with best params ==="
/usr/local/bin/train_baseline_rl \
--model dqn \
--symbol ES.FUT \
--data-dir /mnt/training-data/futures-baseline \
--output-dir ${CI_PROJECT_DIR}/output \
--hyperopt-params ${CI_PROJECT_DIR}/output/dqn_hyperopt_results.json \
&& TRAIN_OK=1 || TRAIN_OK=0
echo "=== Training exit: $TRAIN_OK ==="
echo "=== Evaluating DQN ==="
/usr/local/bin/evaluate_baseline \
--model dqn \
--models-dir ${CI_PROJECT_DIR}/output \
--data-dir /mnt/training-data/futures-baseline \
--output ${CI_PROJECT_DIR}/output/dqn_eval_report.json \
--symbol ES.FUT \
|| true
echo "=== Eval report ==="
cat ${CI_PROJECT_DIR}/output/dqn_eval_report.json 2>/dev/null || echo "No eval report"
[ "$HYPEROPT_OK" = "1" ] && [ "$TRAIN_OK" = "1" ] || exit 1
artifacts:
paths:
- output/dqn_hyperopt_results.json