diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8d721ca1d..930cf9dfb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -461,21 +461,26 @@ train-validate-rl: extends: .train-validate-base needs: [build-training] script: - - /usr/local/bin/train_baseline_rl - --model both - --symbol ES.FUT - --data-dir /mnt/training-data/futures-baseline - --output-dir ${CI_PROJECT_DIR}/output - --max-steps-per-epoch 50 - - echo "=== RL training complete, running evaluation ===" - - /usr/local/bin/evaluate_baseline - --model both - --models-dir ${CI_PROJECT_DIR}/output - --data-dir /mnt/training-data/futures-baseline - --output ${CI_PROJECT_DIR}/output/rl_eval_report.json - --symbol ES.FUT - - echo "RL validation + evaluation passed" - - cat ${CI_PROJECT_DIR}/output/rl_eval_report.json + - | + /usr/local/bin/train_baseline_rl \ + --model both \ + --symbol ES.FUT \ + --data-dir /mnt/training-data/futures-baseline \ + --output-dir ${CI_PROJECT_DIR}/output \ + --max-steps-per-epoch 50 \ + && TRAIN_OK=1 || TRAIN_OK=0 + echo "=== RL training exit: $TRAIN_OK ===" + echo "=== Running evaluation ===" + /usr/local/bin/evaluate_baseline \ + --model both \ + --models-dir ${CI_PROJECT_DIR}/output \ + --data-dir /mnt/training-data/futures-baseline \ + --output ${CI_PROJECT_DIR}/output/rl_eval_report.json \ + --symbol ES.FUT \ + || true + echo "=== Eval report ===" + cat ${CI_PROJECT_DIR}/output/rl_eval_report.json 2>/dev/null || echo "No eval report" + [ "$TRAIN_OK" = "1" ] || exit 1 artifacts: paths: - output/rl_eval_report.json @@ -729,32 +734,39 @@ hyperopt-dqn: needs: [build-training] timeout: 4h script: - - /usr/local/bin/hyperopt_baseline_rl - --model dqn - --trials 20 - --n-initial 5 - --epochs 15 - --symbol ES.FUT - --data-dir /mnt/training-data/futures-baseline - --base-dir ${CI_PROJECT_DIR}/output/hyperopt - --output ${CI_PROJECT_DIR}/output/dqn_hyperopt_results.json - - echo "=== DQN Hyperopt complete ===" - - cat ${CI_PROJECT_DIR}/output/dqn_hyperopt_results.json - - echo "=== Training DQN with best params ===" - - /usr/local/bin/train_baseline_rl - --model dqn - --symbol ES.FUT - --data-dir /mnt/training-data/futures-baseline - --output-dir ${CI_PROJECT_DIR}/output - --hyperopt-params ${CI_PROJECT_DIR}/output/dqn_hyperopt_results.json - - echo "=== Evaluating DQN with best params ===" - - /usr/local/bin/evaluate_baseline - --model dqn - --models-dir ${CI_PROJECT_DIR}/output - --data-dir /mnt/training-data/futures-baseline - --output ${CI_PROJECT_DIR}/output/dqn_eval_report.json - --symbol ES.FUT - - cat ${CI_PROJECT_DIR}/output/dqn_eval_report.json + - | + /usr/local/bin/hyperopt_baseline_rl \ + --model dqn \ + --trials 20 \ + --n-initial 5 \ + --epochs 15 \ + --symbol ES.FUT \ + --data-dir /mnt/training-data/futures-baseline \ + --base-dir ${CI_PROJECT_DIR}/output/hyperopt \ + --output ${CI_PROJECT_DIR}/output/dqn_hyperopt_results.json \ + && HYPEROPT_OK=1 || HYPEROPT_OK=0 + echo "=== DQN Hyperopt exit: $HYPEROPT_OK ===" + cat ${CI_PROJECT_DIR}/output/dqn_hyperopt_results.json 2>/dev/null || echo "No hyperopt results" + echo "=== Training DQN with best params ===" + /usr/local/bin/train_baseline_rl \ + --model dqn \ + --symbol ES.FUT \ + --data-dir /mnt/training-data/futures-baseline \ + --output-dir ${CI_PROJECT_DIR}/output \ + --hyperopt-params ${CI_PROJECT_DIR}/output/dqn_hyperopt_results.json \ + && TRAIN_OK=1 || TRAIN_OK=0 + echo "=== Training exit: $TRAIN_OK ===" + echo "=== Evaluating DQN ===" + /usr/local/bin/evaluate_baseline \ + --model dqn \ + --models-dir ${CI_PROJECT_DIR}/output \ + --data-dir /mnt/training-data/futures-baseline \ + --output ${CI_PROJECT_DIR}/output/dqn_eval_report.json \ + --symbol ES.FUT \ + || true + echo "=== Eval report ===" + cat ${CI_PROJECT_DIR}/output/dqn_eval_report.json 2>/dev/null || echo "No eval report" + [ "$HYPEROPT_OK" = "1" ] && [ "$TRAIN_OK" = "1" ] || exit 1 artifacts: paths: - output/dqn_hyperopt_results.json