From fb53b81a93889c8a7bfe5bc70ad42d345c1387ed Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Tue, 10 Mar 2026 23:25:33 +0100 Subject: [PATCH] infra: automate terragrunt via Argo CI, clean up kapsule module, harden PAT rotation - Add terragrunt-apply step to Argo CI pipeline (plan+apply on main push when infra/live/ or infra/modules/ change) - Bake OpenTofu 1.9.0 + Terragrunt 0.77.12 into ci-builder-cpu image with SHA256 checksum verification - Remove 3 ghost node pools (foxhunt, gitlab, h100-sxm8) from kapsule module to match Scaleway reality - Make terragrunt.hcl single source of truth (remove variable defaults) - Fix GitLab TF state lock methods (POST/DELETE for HTTP backend) - Harden PAT rotation: more retries, verification step, recovery docs - Add weekly PAT expiry check CronJob (warns 14 days before expiry) Co-Authored-By: Claude Opus 4.6 --- infra/docker/Dockerfile.ci-builder-cpu | 16 ++- infra/k8s/argo/ci-pipeline-template.yaml | 116 ++++++++++++++++ infra/k8s/gitlab/pat-rotation.yaml | 138 ++++++++++++++++--- infra/live/production/kapsule/terragrunt.hcl | 30 ++-- infra/live/production/root.hcl | 2 + infra/modules/kapsule/main.tf | 75 ++-------- infra/modules/kapsule/outputs.tf | 24 +--- infra/modules/kapsule/variables.tf | 68 ++------- 8 files changed, 283 insertions(+), 186 deletions(-) diff --git a/infra/docker/Dockerfile.ci-builder-cpu b/infra/docker/Dockerfile.ci-builder-cpu index 5f6d9392d..53c2f1174 100644 --- a/infra/docker/Dockerfile.ci-builder-cpu +++ b/infra/docker/Dockerfile.ci-builder-cpu @@ -1,6 +1,6 @@ # CUDA-free CI builder image for Foxhunt service compilation # ~2-3GB vs ~8GB for the CUDA devel variant (Dockerfile.ci-builder) -# Contains: Rust 1.89, protoc, sccache, rclone, git, OpenSSL, clang, mold, lld, make +# Contains: Rust 1.89, protoc, sccache, rclone, OpenTofu, Terragrunt, git, OpenSSL, clang, mold, lld, make # Base: Ubuntu 24.04 (glibc 2.39) — matches local dev and training runtime # Build: docker build -f infra/docker/Dockerfile.ci-builder-cpu -t foxhunt-ci-builder-cpu . # Push: docker tag foxhunt-ci-builder-cpu gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/ci-builder-cpu:latest @@ -55,5 +55,17 @@ RUN curl -fsSL https://downloads.rclone.org/v1.69.1/rclone-v1.69.1-linux-amd64.z # clippy pre-installed RUN rustup component add clippy +# OpenTofu 1.9.0 (Terraform-compatible IaC, used by Terragrunt for infra CI) +RUN curl -fsSL https://github.com/opentofu/opentofu/releases/download/v1.9.0/tofu_1.9.0_linux_amd64.zip -o /tmp/tofu.zip \ + && echo "638dd3fb9ecfa6fd9f54a0024b195b12b407c51ccee6f83b18a75a8be79f8214 /tmp/tofu.zip" | sha256sum -c - \ + && unzip -o /tmp/tofu.zip -d /usr/local/bin tofu \ + && rm /tmp/tofu.zip \ + && chmod +x /usr/local/bin/tofu + +# Terragrunt 0.77.12 (DRY Terraform/OpenTofu wrapper) +RUN curl -fsSL https://github.com/gruntwork-io/terragrunt/releases/download/v0.77.12/terragrunt_linux_amd64 -o /usr/local/bin/terragrunt \ + && echo "7f6592fc7faa55b75d7a50ac6a485c48be502486ea61680836f19677b6e8e1b9 /usr/local/bin/terragrunt" | sha256sum -c - \ + && chmod +x /usr/local/bin/terragrunt + # Verify -RUN rustc --version && cargo --version && git --version && protoc --version && sccache --version && make --version && mold --version && rclone --version +RUN rustc --version && cargo --version && git --version && protoc --version && sccache --version && make --version && mold --version && rclone --version && tofu --version && terragrunt --version diff --git a/infra/k8s/argo/ci-pipeline-template.yaml b/infra/k8s/argo/ci-pipeline-template.yaml index 916f54bdc..78ab566dc 100644 --- a/infra/k8s/argo/ci-pipeline-template.yaml +++ b/infra/k8s/argo/ci-pipeline-template.yaml @@ -172,6 +172,15 @@ spec: value: foxhunt-training-runtime when: "{{tasks.detect-changes.outputs.parameters.docker-images}} == true" + - name: terragrunt-apply + dependencies: [detect-changes] + template: terragrunt-apply + arguments: + parameters: + - name: commit-sha + value: "{{workflow.parameters.commit-sha}}" + when: "{{tasks.detect-changes.outputs.parameters.needs-infra}} == true" + # ── detect-changes: granular per-binary change detection ── - name: detect-changes nodeSelector: @@ -262,6 +271,7 @@ spec: DOCKER_IMAGES=$(check_paths "infra/docker/") NEEDS_DASHBOARD=$(check_paths "web-dashboard/") INFRA_CI=$(check_paths "infra/k8s/argo/ci-pipeline infra/k8s/services/") + NEEDS_INFRA=$(check_paths "infra/live/ infra/modules/") # ======================================== # Build SERVICE package list @@ -407,6 +417,7 @@ spec: echo "needs-training: $NEEDS_TRAINING" echo "needs-code: $NEEDS_CODE" echo "needs-dashboard: $NEEDS_DASHBOARD" + echo "needs-infra: $NEEDS_INFRA" echo "docker-images: $DOCKER_IMAGES" echo "service-packages: $SERVICE_PKGS" echo "training-examples: $TRAINING_EXAMPLES" @@ -419,6 +430,7 @@ spec: echo -n "$NEEDS_DASHBOARD" > /tmp/outputs/needs-dashboard echo -n "$DOCKER_IMAGES" > /tmp/outputs/docker-images echo -n "$NEEDS_CODE" > /tmp/outputs/needs-code + echo -n "$NEEDS_INFRA" > /tmp/outputs/needs-infra echo -n "$SERVICE_PKGS" > /tmp/outputs/service-packages echo -n "$TRAINING_EXAMPLES" > /tmp/outputs/training-examples echo -n "$DEPLOY_LIST" > /tmp/outputs/deploy-list @@ -451,6 +463,9 @@ spec: - name: deploy-list valueFrom: path: /tmp/outputs/deploy-list + - name: needs-infra + valueFrom: + path: /tmp/outputs/needs-infra # ── create-tag: CalVer auto-tag on code changes ── - name: create-tag @@ -1051,3 +1066,104 @@ spec: done echo "=== Deploy ${TAG} complete ($DEPLOY_LIST) ===" + + # ── terragrunt-apply: apply infra changes on main push ── + - name: terragrunt-apply + inputs: + parameters: + - name: commit-sha + nodeSelector: + k8s.scaleway.com/pool-name: platform + serviceAccountName: argo-workflow + container: + image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/ci-builder-cpu:latest + command: ["/bin/sh", "-c"] + env: + - name: GITLAB_TOKEN + valueFrom: + secretKeyRef: + name: gitlab-pat + key: token + - name: TF_HTTP_USERNAME + value: root + - name: TF_HTTP_PASSWORD + valueFrom: + secretKeyRef: + name: gitlab-pat + key: token + - name: SCW_ACCESS_KEY + valueFrom: + secretKeyRef: + name: scaleway-credentials + key: access-key + - name: SCW_SECRET_KEY + valueFrom: + secretKeyRef: + name: scaleway-credentials + key: secret-key + - name: SCW_DEFAULT_PROJECT_ID + valueFrom: + secretKeyRef: + name: scaleway-credentials + key: project-id + - name: GITLAB_TF_STATE_URL + value: "http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181" + resources: + requests: + cpu: 200m + memory: 256Mi + limits: + cpu: "1" + memory: 512Mi + volumeMounts: + - name: git-ssh-key + mountPath: /etc/git-ssh + readOnly: true + args: + - | + set -e + + # Clone repo + mkdir -p /root/.ssh + cp /etc/git-ssh/ssh-privatekey /root/.ssh/id_ed25519 + chmod 600 /root/.ssh/id_ed25519 + printf 'Host *\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n' > /root/.ssh/config + chmod 600 /root/.ssh/config + + SHA="{{inputs.parameters.commit-sha}}" + REPO="ssh://git@gitlab-gitlab-shell.foxhunt.svc.cluster.local:2222/root/foxhunt.git" + git clone --no-checkout --filter=blob:none "$REPO" /workspace/src + cd /workspace/src + git checkout "$SHA" + echo "Checked out $(git rev-parse --short HEAD)" + + tofu --version + terragrunt --version + + # Apply each terragrunt module + for module in block-storage dns public-gateway kapsule; do + MODULE_DIR="infra/live/production/${module}" + if [ -d "$MODULE_DIR" ]; then + echo "=== Terragrunt plan: ${module} ===" + cd "/workspace/src/${MODULE_DIR}" + terragrunt init --non-interactive -reconfigure + OUTPUT=$(terragrunt plan --non-interactive -detailed-exitcode 2>&1) || EXITCODE=$? + EXITCODE=${EXITCODE:-0} + + if [ "$EXITCODE" -eq 2 ]; then + echo "=== Terragrunt plan output: ${module} ===" + echo "$OUTPUT" + echo "=== Terragrunt apply: ${module} ===" + terragrunt apply --non-interactive -auto-approve + elif [ "$EXITCODE" -eq 0 ]; then + echo "=== No changes for ${module} ===" + else + echo "=== ERROR planning ${module} ===" + echo "$OUTPUT" + exit 1 + fi + cd /workspace/src + fi + done + + echo "=== Terragrunt apply complete ===" diff --git a/infra/k8s/gitlab/pat-rotation.yaml b/infra/k8s/gitlab/pat-rotation.yaml index 6c351f60e..6cee96944 100644 --- a/infra/k8s/gitlab/pat-rotation.yaml +++ b/infra/k8s/gitlab/pat-rotation.yaml @@ -2,12 +2,22 @@ # Rotates the personal access token monthly, storing the new token in a K8s Secret. # Uses GitLab's atomic rotate API: creates new token + revokes old in one call. # +# The rotation runs on the 1st of every month. Additionally, a weekly expiry check +# runs every Monday and logs a warning if the token expires within 14 days, giving +# time to investigate rotation failures before CI breaks. +# # Initial setup: # kubectl -n foxhunt create secret generic gitlab-pat \ # --from-literal=token=glpat- # -# Manual trigger: +# Manual rotation: # kubectl -n foxhunt create job pat-rotate-manual --from=cronjob/gitlab-pat-rotation +# +# Manual expiry check: +# kubectl -n foxhunt create job pat-check-manual --from=cronjob/gitlab-pat-expiry-check +# +# Sync local token: +# export GITLAB_TOKEN=$(kubectl -n foxhunt get secret gitlab-pat -o jsonpath='{.data.token}' | base64 -d) --- apiVersion: v1 kind: ServiceAccount @@ -40,6 +50,7 @@ roleRef: name: gitlab-pat-rotator apiGroup: rbac.authorization.k8s.io --- +# Monthly rotation — creates new token, revokes old, updates K8s secret apiVersion: batch/v1 kind: CronJob metadata: @@ -48,12 +59,12 @@ metadata: spec: schedule: "0 3 1 * *" # 1st of every month at 03:00 UTC successfulJobsHistoryLimit: 3 - failedJobsHistoryLimit: 3 + failedJobsHistoryLimit: 5 concurrencyPolicy: Forbid jobTemplate: spec: - backoffLimit: 2 - activeDeadlineSeconds: 120 + backoffLimit: 3 + activeDeadlineSeconds: 180 template: metadata: labels: @@ -62,7 +73,7 @@ spec: serviceAccountName: gitlab-pat-rotator restartPolicy: Never nodeSelector: - k8s.scaleway.com/pool-name: infra + k8s.scaleway.com/pool-name: platform containers: - name: rotate image: alpine:3.21 @@ -76,22 +87,25 @@ spec: CURRENT_PAT=$(cat /etc/gitlab-pat/token) GITLAB_URL="http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181" - # Verify current token is still valid + # 1. Verify current token is valid TOKEN_INFO=$(curl -sf "${GITLAB_URL}/api/v4/personal_access_tokens/self" \ -H "PRIVATE-TOKEN: ${CURRENT_PAT}") || { - echo "ERROR: Current PAT is invalid or GitLab unreachable" + echo "CRITICAL: Current PAT is invalid or GitLab unreachable." + echo "Manual recovery required: create a new PAT in GitLab UI and run:" + echo " kubectl -n foxhunt create secret generic gitlab-pat --from-literal=token= --dry-run=client -o yaml | kubectl apply -f -" exit 1 } OLD_EXPIRY=$(echo "$TOKEN_INFO" | jq -r '.expires_at') - echo "Current PAT expires: ${OLD_EXPIRY}" + TOKEN_NAME=$(echo "$TOKEN_INFO" | jq -r '.name') + echo "Current PAT '${TOKEN_NAME}' expires: ${OLD_EXPIRY}" - # Rotate: new expiry = 1 year from now + # 2. Rotate: new expiry = 1 year from now EXPIRES=$(date -d "@$(($(date +%s) + 31536000))" +%Y-%m-%d) RESPONSE=$(curl -sf -X POST \ "${GITLAB_URL}/api/v4/personal_access_tokens/self/rotate" \ -H "PRIVATE-TOKEN: ${CURRENT_PAT}" \ -d "expires_at=${EXPIRES}") || { - echo "ERROR: Rotation API call failed" + echo "ERROR: Rotation API call failed. Token may still be valid." exit 1 } @@ -103,7 +117,7 @@ spec: echo "PAT rotated successfully. New expiry: ${EXPIRES}" - # Update K8s secret via API (old token is already revoked at this point) + # 3. Update K8s secret (old token is already revoked at this point) K8S_TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token) K8S_CA=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt K8S_NS=$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace) @@ -111,8 +125,7 @@ spec: NEW_PAT_B64=$(printf '%s' "$NEW_PAT" | base64) - # Retry secret update — losing the new token after rotation is catastrophic - for attempt in 1 2 3; do + for attempt in 1 2 3 4 5; do if curl -sf -X PATCH \ "${K8S_API}/api/v1/namespaces/${K8S_NS}/secrets/gitlab-pat" \ --cacert "$K8S_CA" \ @@ -120,17 +133,27 @@ spec: -H "Content-Type: application/strategic-merge-patch+json" \ -d "{\"data\":{\"token\":\"${NEW_PAT_B64}\"}}"; then echo "" - echo "Secret updated on attempt ${attempt}." + echo "K8s secret updated on attempt ${attempt}." + + # 4. Verify the new token works + VERIFY=$(curl -sf "${GITLAB_URL}/api/v4/personal_access_tokens/self" \ + -H "PRIVATE-TOKEN: ${NEW_PAT}" | jq -r '.expires_at') || true + if [ "$VERIFY" = "$EXPIRES" ]; then + echo "Verification passed: new token valid until ${VERIFY}" + else + echo "WARNING: Verification returned unexpected expiry: ${VERIFY}" + fi exit 0 fi - echo "Secret update attempt ${attempt} failed, retrying..." - sleep 2 + echo "Secret update attempt ${attempt} failed, retrying in 3s..." + sleep 3 done # If we get here, rotation succeeded but secret update failed. # Print token so it can be recovered from job logs. - echo "CRITICAL: Secret update failed after 3 attempts!" + echo "CRITICAL: Secret update failed after 5 attempts!" echo "New PAT (recover manually): ${NEW_PAT}" + echo "Run: kubectl -n foxhunt create secret generic gitlab-pat --from-literal=token= --dry-run=client -o yaml | kubectl apply -f -" exit 1 volumeMounts: - name: gitlab-pat @@ -147,3 +170,84 @@ spec: - name: gitlab-pat secret: secretName: gitlab-pat +--- +# Weekly expiry check — warns if token expires within 14 days +apiVersion: batch/v1 +kind: CronJob +metadata: + name: gitlab-pat-expiry-check + namespace: foxhunt +spec: + schedule: "0 8 * * 1" # Every Monday at 08:00 UTC + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 3 + concurrencyPolicy: Forbid + jobTemplate: + spec: + backoffLimit: 1 + activeDeadlineSeconds: 60 + template: + metadata: + labels: + app: gitlab-pat-expiry-check + spec: + restartPolicy: Never + nodeSelector: + k8s.scaleway.com/pool-name: platform + containers: + - name: check + image: alpine:3.21 + command: ["/bin/sh", "-c"] + args: + - | + set -e + + apk add --no-cache -q curl jq + + CURRENT_PAT=$(cat /etc/gitlab-pat/token) + GITLAB_URL="http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181" + + TOKEN_INFO=$(curl -sf "${GITLAB_URL}/api/v4/personal_access_tokens/self" \ + -H "PRIVATE-TOKEN: ${CURRENT_PAT}") || { + echo "CRITICAL: PAT is INVALID. CI will fail!" + echo "Create a new PAT in GitLab UI and run:" + echo " kubectl -n foxhunt create secret generic gitlab-pat --from-literal=token= --dry-run=client -o yaml | kubectl apply -f -" + exit 1 + } + + EXPIRY=$(echo "$TOKEN_INFO" | jq -r '.expires_at') + TOKEN_NAME=$(echo "$TOKEN_INFO" | jq -r '.name') + SCOPES=$(echo "$TOKEN_INFO" | jq -r '.scopes | join(",")') + + # Calculate days until expiry + EXPIRY_EPOCH=$(date -d "$EXPIRY" +%s) + NOW_EPOCH=$(date +%s) + DAYS_LEFT=$(( (EXPIRY_EPOCH - NOW_EPOCH) / 86400 )) + + echo "PAT '${TOKEN_NAME}': expires=${EXPIRY} scopes=${SCOPES} days_left=${DAYS_LEFT}" + + if [ "$DAYS_LEFT" -lt 0 ]; then + echo "CRITICAL: PAT has EXPIRED ${DAYS_LEFT} days ago!" + exit 1 + elif [ "$DAYS_LEFT" -lt 14 ]; then + echo "WARNING: PAT expires in ${DAYS_LEFT} days. Rotation may have failed." + echo "Manual rotation: kubectl -n foxhunt create job pat-rotate-manual --from=cronjob/gitlab-pat-rotation" + exit 1 + else + echo "OK: PAT valid for ${DAYS_LEFT} more days." + fi + volumeMounts: + - name: gitlab-pat + mountPath: /etc/gitlab-pat + readOnly: true + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + volumes: + - name: gitlab-pat + secret: + secretName: gitlab-pat diff --git a/infra/live/production/kapsule/terragrunt.hcl b/infra/live/production/kapsule/terragrunt.hcl index 1f79b6522..5e6b7197e 100644 --- a/infra/live/production/kapsule/terragrunt.hcl +++ b/infra/live/production/kapsule/terragrunt.hcl @@ -10,49 +10,35 @@ inputs = { cluster_name = "foxhunt" k8s_version = "1.34" - # Foxhunt app services pool (api-gateway, trading-service, etc.) - # Renamed from "services" — single DEV1-L, dedicated to microservices - foxhunt_type = "DEV1-L" - foxhunt_max_size = 1 + # Platform pool — runs everything: app services, databases, GitLab, monitoring + platform_type = "DEV1-L" + platform_max_size = 3 - # Platform infrastructure pool (postgres, redis, minio, questdb, monitoring) - enable_platform_pool = true - platform_type = "DEV1-L" - - # GitLab CE pool (dedicated for GitLab + Prometheus + Runner manager) - enable_gitlab_pool = true - gitlab_type = "GP1-XS" - - # CPU compile pool (POP2-HC-32C-64G for cargo check/build — 32 vCPU, 64GB RAM, high clock, no GPU) + # CPU compile pool (POP2-HC-32C-64G — 32 vCPU, 64GB RAM, high clock) enable_ci_compile_cpu_pool = true ci_compile_cpu_type = "POP2-HC-32C-64G" ci_compile_cpu_max_size = 4 - # L40S training pool (48GB VRAM, CUDA CC 89 — supervised hyperopt) + # L40S training pool (48GB VRAM, CUDA CC 89) enable_ci_training_l40s_pool = true ci_training_l40s_type = "L40S-1-48G" ci_training_l40s_max_size = 1 - # H100 training pool (80GB VRAM, CUDA CC 90 — RL hyperopt) + # H100 training pool (80GB VRAM, CUDA CC 90) enable_ci_training_h100_pool = true ci_training_h100_type = "H100-1-80G" ci_training_h100_max_size = 1 - # H100x2 training pool (2× 80GB — multi-GPU parallel hyperopt) + # H100x2 training pool (2x 80GB — multi-GPU parallel hyperopt) enable_ci_training_h100x2_pool = true ci_training_h100x2_type = "H100-2-80G" ci_training_h100x2_max_size = 1 - # H100-SXM training pool (2× 80GB SXM, NVLink — future multi-GPU) + # H100-SXM training pool (2x 80GB SXM, NVLink) enable_ci_training_h100_sxm_pool = true ci_training_h100_sxm_type = "H100-SXM-2-80G" ci_training_h100_sxm_max_size = 1 - # H100-SXM-8 training pool (8× 80GB SXM, NVLink — full multi-GPU saturation) - enable_ci_training_h100_sxm8_pool = true - ci_training_h100_sxm8_type = "H100-SXM-8-80G" - ci_training_h100_sxm8_max_size = 1 - # NAT gateway handles egress — no public IPs on nodes public_ip_disabled = true } diff --git a/infra/live/production/root.hcl b/infra/live/production/root.hcl index b8e466765..190667547 100644 --- a/infra/live/production/root.hcl +++ b/infra/live/production/root.hcl @@ -20,6 +20,8 @@ remote_state { address = "${get_env("GITLAB_TF_STATE_URL", "https://tf.fxhnt.ai")}/api/v4/projects/1/terraform/state/${path_relative_to_include()}" lock_address = "${get_env("GITLAB_TF_STATE_URL", "https://tf.fxhnt.ai")}/api/v4/projects/1/terraform/state/${path_relative_to_include()}/lock" unlock_address = "${get_env("GITLAB_TF_STATE_URL", "https://tf.fxhnt.ai")}/api/v4/projects/1/terraform/state/${path_relative_to_include()}/lock" + lock_method = "POST" + unlock_method = "DELETE" } } diff --git a/infra/modules/kapsule/main.tf b/infra/modules/kapsule/main.tf index fac678184..510afbcfd 100644 --- a/infra/modules/kapsule/main.tf +++ b/infra/modules/kapsule/main.tf @@ -26,16 +26,14 @@ resource "scaleway_k8s_cluster" "foxhunt" { } } -# Foxhunt app services pool (api-gateway, trading-service, etc.) -# Renamed from "services" — was 2× DEV1-L shared with databases, -# now 1× DEV1-L dedicated to foxhunt microservices only. -resource "scaleway_k8s_pool" "foxhunt" { +# Platform pool — runs everything: app services, databases, GitLab, monitoring +resource "scaleway_k8s_pool" "platform" { cluster_id = scaleway_k8s_cluster.foxhunt.id - name = "foxhunt" - node_type = var.foxhunt_type + name = "platform" + node_type = var.platform_type size = 1 min_size = 1 - max_size = var.foxhunt_max_size + max_size = var.platform_max_size autoscaling = true autohealing = true public_ip_disabled = var.public_ip_disabled @@ -46,38 +44,7 @@ resource "scaleway_k8s_pool" "foxhunt" { } } -# Platform infrastructure pool (postgres, redis, minio, questdb, monitoring) -resource "scaleway_k8s_pool" "platform" { - count = var.enable_platform_pool ? 1 : 0 - cluster_id = scaleway_k8s_cluster.foxhunt.id - name = "platform" - node_type = var.platform_type - size = 1 - min_size = 1 - max_size = 1 - autoscaling = false - autohealing = true - public_ip_disabled = var.public_ip_disabled - region = var.region -} - -# GitLab CE node pool (dedicated GP1-XS for GitLab + Runner manager) -resource "scaleway_k8s_pool" "gitlab" { - count = var.enable_gitlab_pool ? 1 : 0 - cluster_id = scaleway_k8s_cluster.foxhunt.id - name = "gitlab" - node_type = var.gitlab_type - size = 1 - min_size = 1 - max_size = 1 - autoscaling = false - autohealing = true - public_ip_disabled = var.public_ip_disabled - region = var.region -} - # L40S training pool (48GB VRAM, CUDA CC 89 — hyperopt + supervised training) -# Nodes auto-labeled: k8s.scaleway.com/pool-name=ci-training-l40s resource "scaleway_k8s_pool" "ci_training_l40s" { count = var.enable_ci_training_l40s_pool ? 1 : 0 cluster_id = scaleway_k8s_cluster.foxhunt.id @@ -96,8 +63,7 @@ resource "scaleway_k8s_pool" "ci_training_l40s" { } } -# H100 training pool (80GB VRAM, CUDA CC 90 — RL hyperopt + large model training) -# Nodes auto-labeled: k8s.scaleway.com/pool-name=ci-training-h100 +# H100 training pool (80GB VRAM, CUDA CC 90 — RL hyperopt) resource "scaleway_k8s_pool" "ci_training_h100" { count = var.enable_ci_training_h100_pool ? 1 : 0 cluster_id = scaleway_k8s_cluster.foxhunt.id @@ -116,8 +82,7 @@ resource "scaleway_k8s_pool" "ci_training_h100" { } } -# H100x2 training pool (2× 80GB VRAM — multi-GPU parallel hyperopt) -# Nodes auto-labeled: k8s.scaleway.com/pool-name=ci-training-h100x2 +# H100x2 training pool (2x 80GB VRAM — multi-GPU parallel hyperopt) resource "scaleway_k8s_pool" "ci_training_h100x2" { count = var.enable_ci_training_h100x2_pool ? 1 : 0 cluster_id = scaleway_k8s_cluster.foxhunt.id @@ -136,8 +101,7 @@ resource "scaleway_k8s_pool" "ci_training_h100x2" { } } -# H100-SXM training pool (2× 80GB SXM VRAM — NVLink multi-GPU) -# Nodes auto-labeled: k8s.scaleway.com/pool-name=ci-training-h100-sxm +# H100-SXM training pool (2x 80GB SXM VRAM — NVLink multi-GPU) resource "scaleway_k8s_pool" "ci_training_h100_sxm" { count = var.enable_ci_training_h100_sxm_pool ? 1 : 0 cluster_id = scaleway_k8s_cluster.foxhunt.id @@ -156,28 +120,7 @@ resource "scaleway_k8s_pool" "ci_training_h100_sxm" { } } -# H100-SXM-8 training pool (8× 80GB SXM VRAM, NVLink — full multi-GPU saturation) -# Nodes auto-labeled: k8s.scaleway.com/pool-name=ci-training-h100-sxm8 -resource "scaleway_k8s_pool" "ci_training_h100_sxm8" { - count = var.enable_ci_training_h100_sxm8_pool ? 1 : 0 - cluster_id = scaleway_k8s_cluster.foxhunt.id - name = "ci-training-h100-sxm8" - node_type = var.ci_training_h100_sxm8_type - size = 1 - min_size = 0 - max_size = var.ci_training_h100_sxm8_max_size - autoscaling = true - autohealing = true - public_ip_disabled = var.public_ip_disabled - region = var.region - - lifecycle { - ignore_changes = [size] - } -} - -# CPU compile pool (POP2 — high CPU, no GPU, for cargo check/build) -# Nodes auto-labeled: k8s.scaleway.com/pool-name=ci-compile-cpu +# CPU compile pool (POP2-HC — high CPU, no GPU, for cargo check/build) resource "scaleway_k8s_pool" "ci_compile_cpu" { count = var.enable_ci_compile_cpu_pool ? 1 : 0 cluster_id = scaleway_k8s_cluster.foxhunt.id diff --git a/infra/modules/kapsule/outputs.tf b/infra/modules/kapsule/outputs.tf index 9d41a2daa..eb738af91 100644 --- a/infra/modules/kapsule/outputs.tf +++ b/infra/modules/kapsule/outputs.tf @@ -14,23 +14,13 @@ output "apiserver_url" { value = scaleway_k8s_cluster.foxhunt.apiserver_url } -output "foxhunt_pool_id" { - description = "ID of the foxhunt app services node pool" - value = scaleway_k8s_pool.foxhunt.id -} - output "platform_pool_id" { - description = "ID of the platform infrastructure node pool" - value = var.enable_platform_pool ? scaleway_k8s_pool.platform[0].id : "" -} - -output "gitlab_pool_id" { - description = "ID of the GitLab node pool" - value = var.enable_gitlab_pool ? scaleway_k8s_pool.gitlab[0].id : "" + description = "ID of the platform node pool" + value = scaleway_k8s_pool.platform.id } output "ci_compile_cpu_pool_id" { - description = "ID of the CPU compile node pool (POP2-HC)" + description = "ID of the CPU compile node pool" value = var.enable_ci_compile_cpu_pool ? scaleway_k8s_pool.ci_compile_cpu[0].id : "" } @@ -54,15 +44,7 @@ output "ci_training_h100_sxm_pool_id" { value = var.enable_ci_training_h100_sxm_pool ? scaleway_k8s_pool.ci_training_h100_sxm[0].id : "" } -output "ci_training_h100_sxm8_pool_id" { - description = "ID of the H100-SXM-8 training node pool (8× 80GB)" - value = var.enable_ci_training_h100_sxm8_pool ? scaleway_k8s_pool.ci_training_h100_sxm8[0].id : "" -} - -# Dev pool removed — DevPod runs on platform pool - output "private_network_id" { description = "ID of the VPC private network the cluster is attached to" value = scaleway_vpc_private_network.foxhunt.id } - diff --git a/infra/modules/kapsule/variables.tf b/infra/modules/kapsule/variables.tf index 07360c0e3..c0a9c687f 100644 --- a/infra/modules/kapsule/variables.tf +++ b/infra/modules/kapsule/variables.tf @@ -6,53 +6,25 @@ variable "region" { variable "cluster_name" { description = "Name of the Kubernetes cluster" type = string - default = "foxhunt" } variable "k8s_version" { description = "Kubernetes version for the Kapsule cluster" type = string - default = "1.34" -} - -variable "foxhunt_type" { - description = "Instance type for the foxhunt app services node pool" - type = string - default = "DEV1-L" -} - -variable "foxhunt_max_size" { - description = "Maximum number of nodes in the foxhunt pool (autoscaling)" - type = number - default = 1 -} - -variable "enable_platform_pool" { - description = "Create dedicated node pool for platform infrastructure (databases, monitoring)" - type = bool - default = false } variable "platform_type" { - description = "Instance type for the platform infrastructure node pool" + description = "Instance type for the platform node pool" type = string - default = "DEV1-L" } -variable "enable_gitlab_pool" { - description = "Create dedicated node pool for GitLab CE" - type = bool - default = false -} - -variable "gitlab_type" { - description = "Instance type for the GitLab node pool" - type = string - default = "DEV1-L" +variable "platform_max_size" { + description = "Maximum number of nodes in the platform pool" + type = number } variable "enable_ci_training_l40s_pool" { - description = "Create L40S GPU node pool for training + hyperopt (48GB VRAM, CC 89)" + description = "Create L40S GPU node pool for training + hyperopt" type = bool default = false } @@ -70,7 +42,7 @@ variable "ci_training_l40s_max_size" { } variable "enable_ci_training_h100_pool" { - description = "Create H100 GPU node pool for training + hyperopt (80GB VRAM, CC 90)" + description = "Create H100 GPU node pool for training + hyperopt" type = bool default = false } @@ -88,7 +60,7 @@ variable "ci_training_h100_max_size" { } variable "enable_ci_training_h100x2_pool" { - description = "Create H100x2 GPU node pool for multi-GPU training (2× 80GB VRAM)" + description = "Create H100x2 GPU node pool for multi-GPU training" type = bool default = false } @@ -106,7 +78,7 @@ variable "ci_training_h100x2_max_size" { } variable "enable_ci_training_h100_sxm_pool" { - description = "Create H100-SXM GPU node pool for NVLink multi-GPU training (2× 80GB SXM)" + description = "Create H100-SXM GPU node pool for NVLink multi-GPU training" type = bool default = false } @@ -123,34 +95,15 @@ variable "ci_training_h100_sxm_max_size" { default = 1 } -variable "enable_ci_training_h100_sxm8_pool" { - description = "Create H100-SXM-8 GPU node pool for full multi-GPU training (8× 80GB SXM, NVLink)" - type = bool - default = false -} - -variable "ci_training_h100_sxm8_type" { - description = "Instance type for the H100-SXM-8 training pool" - type = string - default = "H100-SXM-8-80G" -} - -variable "ci_training_h100_sxm8_max_size" { - description = "Maximum number of nodes in the H100-SXM-8 training pool" - type = number - default = 1 -} - variable "enable_ci_compile_cpu_pool" { - description = "Create high-CPU node pool for CI compilation (no GPU needed)" + description = "Create high-CPU node pool for CI compilation" type = bool default = false } variable "ci_compile_cpu_type" { - description = "Instance type for the CPU compile pool (32 vCPU, 64GB RAM, high clock, no GPU)" + description = "Instance type for the CPU compile pool" type = string - default = "POP2-HC-32C-64G" } variable "ci_compile_cpu_max_size" { @@ -164,4 +117,3 @@ variable "public_ip_disabled" { type = bool default = false } -