diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ea56dcc61..066e59fa9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,5 @@ # GitLab CI/CD — Foxhunt -# Runs on ci-build pool (L4 or H100, scale-to-zero), linker: clang+lld +# Runs on gpu-training pool (H100, scale-to-zero), linker: clang+lld # CI builder image: CUDA 12.4 + Rust 1.89 + protoc + sccache (hosted on Scaleway CR) # Service images: pushed to Scaleway Container Registry (Kaniko) @@ -20,6 +20,7 @@ stages: - test - compile - build + - train - deploy variables: @@ -433,6 +434,51 @@ build-training: --destination "${REGISTRY}/training:${CI_COMMIT_SHA}" --destination "${REGISTRY}/training:latest" +# -------------------------------------------------------------------------- +# Stage 3c: Training validation — quick smoke test on GPU +# Runs the training binary for a few steps to verify CUDA + model correctness. +# Triggered manually or on schedule (full ensemble training uses train.sh). +# -------------------------------------------------------------------------- +.train-validate-base: + stage: train + image: ${REGISTRY}/training:${CI_COMMIT_SHA} + tags: + - kapsule + - gpu + rules: + - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push" + when: manual + allow_failure: true + - if: $CI_PIPELINE_SOURCE == "schedule" && $TRAIN_VALIDATE == "true" + before_script: + - export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//') + - nvidia-smi + - mkdir -p /output + +train-validate-dqn: + extends: .train-validate-base + needs: [build-training] + script: + - /usr/local/bin/train_baseline_rl + --model dqn + --symbol ES.FUT + --data-dir /mnt/training-data/futures-baseline + --output-dir /output + --max-steps-per-epoch 50 + - echo "DQN validation passed" + +train-validate-tft: + extends: .train-validate-base + needs: [build-training] + script: + - /usr/local/bin/train_baseline_supervised + --model tft + --symbol ES.FUT + --data-dir /mnt/training-data/futures-baseline + --output-dir /output + --max-steps-per-epoch 50 + - echo "TFT validation passed" + # -------------------------------------------------------------------------- # IaC: Terragrunt plan on MR (runs on gitlab pool) # -------------------------------------------------------------------------- diff --git a/infra/k8s/gitlab/runner-values.yaml b/infra/k8s/gitlab/runner-values.yaml index b17b7236f..cdb8604dc 100644 --- a/infra/k8s/gitlab/runner-values.yaml +++ b/infra/k8s/gitlab/runner-values.yaml @@ -1,6 +1,6 @@ -# GitLab Runner — Kubernetes executor targeting ci-build pool (L4) +# GitLab Runner — Kubernetes executor targeting gpu-training pool (H100) # Runner manager pod lives on gitlab node pool -# Build pods spawn on ci-build pool (L4-1-24G, scale-to-zero — €0.75/hr vs H100 €3.50+/hr) +# Build pods spawn on gpu-training pool (H100-1-80G, scale-to-zero) gitlabUrl: http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181 # runnerToken set via --set at install time @@ -26,13 +26,13 @@ runners: namespace = "foxhunt" image = "rust:1.89-slim" privileged = false - # L4 scale-to-zero needs ~3-5 min to provision; default 180s times out + # GPU pool scale-to-zero needs ~3-5 min to provision; default 180s times out poll_timeout = 600 - # Resource limits for build pods (L4 node: 12 vCPU, 45.5GB) - cpu_request = "4000m" - cpu_limit = "10000m" - memory_request = "16Gi" - memory_limit = "38Gi" + # Resource limits for build pods (H100-1-80G node) + cpu_request = "8000m" + cpu_limit = "20000m" + memory_request = "32Gi" + memory_limit = "128Gi" helper_cpu_request = "100m" helper_cpu_limit = "500m" helper_memory_request = "128Mi" @@ -42,7 +42,7 @@ runners: runtime_class_name = "nvidia" # Sub-tables must come AFTER all scalar values (TOML rule) [runners.kubernetes.node_selector] - "k8s.scaleway.com/pool-name" = "ci-build" + "k8s.scaleway.com/pool-name" = "gpu-training" [runners.kubernetes.node_tolerations] "nvidia.com/gpu" = "NoSchedule" # Cilium CNI agent takes ~30s to initialize on fresh scale-from-zero nodes