infra: automate terragrunt via Argo CI, clean up kapsule module, harden PAT rotation

- Add terragrunt-apply step to Argo CI pipeline (plan+apply on main push
  when infra/live/ or infra/modules/ change)
- Bake OpenTofu 1.9.0 + Terragrunt 0.77.12 into ci-builder-cpu image
  with SHA256 checksum verification
- Remove 3 ghost node pools (foxhunt, gitlab, h100-sxm8) from kapsule
  module to match Scaleway reality
- Make terragrunt.hcl single source of truth (remove variable defaults)
- Fix GitLab TF state lock methods (POST/DELETE for HTTP backend)
- Harden PAT rotation: more retries, verification step, recovery docs
- Add weekly PAT expiry check CronJob (warns 14 days before expiry)

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-03-10 23:25:33 +01:00
parent 53eb82c193
commit fb53b81a93
8 changed files with 283 additions and 186 deletions

View File

@@ -1,6 +1,6 @@
# CUDA-free CI builder image for Foxhunt service compilation
# ~2-3GB vs ~8GB for the CUDA devel variant (Dockerfile.ci-builder)
# Contains: Rust 1.89, protoc, sccache, rclone, git, OpenSSL, clang, mold, lld, make
# Contains: Rust 1.89, protoc, sccache, rclone, OpenTofu, Terragrunt, git, OpenSSL, clang, mold, lld, make
# Base: Ubuntu 24.04 (glibc 2.39) — matches local dev and training runtime
# Build: docker build -f infra/docker/Dockerfile.ci-builder-cpu -t foxhunt-ci-builder-cpu .
# Push: docker tag foxhunt-ci-builder-cpu gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/ci-builder-cpu:latest
@@ -55,5 +55,17 @@ RUN curl -fsSL https://downloads.rclone.org/v1.69.1/rclone-v1.69.1-linux-amd64.z
# clippy pre-installed
RUN rustup component add clippy
# OpenTofu 1.9.0 (Terraform-compatible IaC, used by Terragrunt for infra CI)
RUN curl -fsSL https://github.com/opentofu/opentofu/releases/download/v1.9.0/tofu_1.9.0_linux_amd64.zip -o /tmp/tofu.zip \
&& echo "638dd3fb9ecfa6fd9f54a0024b195b12b407c51ccee6f83b18a75a8be79f8214 /tmp/tofu.zip" | sha256sum -c - \
&& unzip -o /tmp/tofu.zip -d /usr/local/bin tofu \
&& rm /tmp/tofu.zip \
&& chmod +x /usr/local/bin/tofu
# Terragrunt 0.77.12 (DRY Terraform/OpenTofu wrapper)
RUN curl -fsSL https://github.com/gruntwork-io/terragrunt/releases/download/v0.77.12/terragrunt_linux_amd64 -o /usr/local/bin/terragrunt \
&& echo "7f6592fc7faa55b75d7a50ac6a485c48be502486ea61680836f19677b6e8e1b9 /usr/local/bin/terragrunt" | sha256sum -c - \
&& chmod +x /usr/local/bin/terragrunt
# Verify
RUN rustc --version && cargo --version && git --version && protoc --version && sccache --version && make --version && mold --version && rclone --version
RUN rustc --version && cargo --version && git --version && protoc --version && sccache --version && make --version && mold --version && rclone --version && tofu --version && terragrunt --version

View File

@@ -172,6 +172,15 @@ spec:
value: foxhunt-training-runtime
when: "{{tasks.detect-changes.outputs.parameters.docker-images}} == true"
- name: terragrunt-apply
dependencies: [detect-changes]
template: terragrunt-apply
arguments:
parameters:
- name: commit-sha
value: "{{workflow.parameters.commit-sha}}"
when: "{{tasks.detect-changes.outputs.parameters.needs-infra}} == true"
# ── detect-changes: granular per-binary change detection ──
- name: detect-changes
nodeSelector:
@@ -262,6 +271,7 @@ spec:
DOCKER_IMAGES=$(check_paths "infra/docker/")
NEEDS_DASHBOARD=$(check_paths "web-dashboard/")
INFRA_CI=$(check_paths "infra/k8s/argo/ci-pipeline infra/k8s/services/")
NEEDS_INFRA=$(check_paths "infra/live/ infra/modules/")
# ========================================
# Build SERVICE package list
@@ -407,6 +417,7 @@ spec:
echo "needs-training: $NEEDS_TRAINING"
echo "needs-code: $NEEDS_CODE"
echo "needs-dashboard: $NEEDS_DASHBOARD"
echo "needs-infra: $NEEDS_INFRA"
echo "docker-images: $DOCKER_IMAGES"
echo "service-packages: $SERVICE_PKGS"
echo "training-examples: $TRAINING_EXAMPLES"
@@ -419,6 +430,7 @@ spec:
echo -n "$NEEDS_DASHBOARD" > /tmp/outputs/needs-dashboard
echo -n "$DOCKER_IMAGES" > /tmp/outputs/docker-images
echo -n "$NEEDS_CODE" > /tmp/outputs/needs-code
echo -n "$NEEDS_INFRA" > /tmp/outputs/needs-infra
echo -n "$SERVICE_PKGS" > /tmp/outputs/service-packages
echo -n "$TRAINING_EXAMPLES" > /tmp/outputs/training-examples
echo -n "$DEPLOY_LIST" > /tmp/outputs/deploy-list
@@ -451,6 +463,9 @@ spec:
- name: deploy-list
valueFrom:
path: /tmp/outputs/deploy-list
- name: needs-infra
valueFrom:
path: /tmp/outputs/needs-infra
# ── create-tag: CalVer auto-tag on code changes ──
- name: create-tag
@@ -1051,3 +1066,104 @@ spec:
done
echo "=== Deploy ${TAG} complete ($DEPLOY_LIST) ==="
# ── terragrunt-apply: apply infra changes on main push ──
- name: terragrunt-apply
inputs:
parameters:
- name: commit-sha
nodeSelector:
k8s.scaleway.com/pool-name: platform
serviceAccountName: argo-workflow
container:
image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/ci-builder-cpu:latest
command: ["/bin/sh", "-c"]
env:
- name: GITLAB_TOKEN
valueFrom:
secretKeyRef:
name: gitlab-pat
key: token
- name: TF_HTTP_USERNAME
value: root
- name: TF_HTTP_PASSWORD
valueFrom:
secretKeyRef:
name: gitlab-pat
key: token
- name: SCW_ACCESS_KEY
valueFrom:
secretKeyRef:
name: scaleway-credentials
key: access-key
- name: SCW_SECRET_KEY
valueFrom:
secretKeyRef:
name: scaleway-credentials
key: secret-key
- name: SCW_DEFAULT_PROJECT_ID
valueFrom:
secretKeyRef:
name: scaleway-credentials
key: project-id
- name: GITLAB_TF_STATE_URL
value: "http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181"
resources:
requests:
cpu: 200m
memory: 256Mi
limits:
cpu: "1"
memory: 512Mi
volumeMounts:
- name: git-ssh-key
mountPath: /etc/git-ssh
readOnly: true
args:
- |
set -e
# Clone repo
mkdir -p /root/.ssh
cp /etc/git-ssh/ssh-privatekey /root/.ssh/id_ed25519
chmod 600 /root/.ssh/id_ed25519
printf 'Host *\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n' > /root/.ssh/config
chmod 600 /root/.ssh/config
SHA="{{inputs.parameters.commit-sha}}"
REPO="ssh://git@gitlab-gitlab-shell.foxhunt.svc.cluster.local:2222/root/foxhunt.git"
git clone --no-checkout --filter=blob:none "$REPO" /workspace/src
cd /workspace/src
git checkout "$SHA"
echo "Checked out $(git rev-parse --short HEAD)"
tofu --version
terragrunt --version
# Apply each terragrunt module
for module in block-storage dns public-gateway kapsule; do
MODULE_DIR="infra/live/production/${module}"
if [ -d "$MODULE_DIR" ]; then
echo "=== Terragrunt plan: ${module} ==="
cd "/workspace/src/${MODULE_DIR}"
terragrunt init --non-interactive -reconfigure
OUTPUT=$(terragrunt plan --non-interactive -detailed-exitcode 2>&1) || EXITCODE=$?
EXITCODE=${EXITCODE:-0}
if [ "$EXITCODE" -eq 2 ]; then
echo "=== Terragrunt plan output: ${module} ==="
echo "$OUTPUT"
echo "=== Terragrunt apply: ${module} ==="
terragrunt apply --non-interactive -auto-approve
elif [ "$EXITCODE" -eq 0 ]; then
echo "=== No changes for ${module} ==="
else
echo "=== ERROR planning ${module} ==="
echo "$OUTPUT"
exit 1
fi
cd /workspace/src
fi
done
echo "=== Terragrunt apply complete ==="

View File

@@ -2,12 +2,22 @@
# Rotates the personal access token monthly, storing the new token in a K8s Secret.
# Uses GitLab's atomic rotate API: creates new token + revokes old in one call.
#
# The rotation runs on the 1st of every month. Additionally, a weekly expiry check
# runs every Monday and logs a warning if the token expires within 14 days, giving
# time to investigate rotation failures before CI breaks.
#
# Initial setup:
# kubectl -n foxhunt create secret generic gitlab-pat \
# --from-literal=token=glpat-<initial-token>
#
# Manual trigger:
# Manual rotation:
# kubectl -n foxhunt create job pat-rotate-manual --from=cronjob/gitlab-pat-rotation
#
# Manual expiry check:
# kubectl -n foxhunt create job pat-check-manual --from=cronjob/gitlab-pat-expiry-check
#
# Sync local token:
# export GITLAB_TOKEN=$(kubectl -n foxhunt get secret gitlab-pat -o jsonpath='{.data.token}' | base64 -d)
---
apiVersion: v1
kind: ServiceAccount
@@ -40,6 +50,7 @@ roleRef:
name: gitlab-pat-rotator
apiGroup: rbac.authorization.k8s.io
---
# Monthly rotation — creates new token, revokes old, updates K8s secret
apiVersion: batch/v1
kind: CronJob
metadata:
@@ -48,12 +59,12 @@ metadata:
spec:
schedule: "0 3 1 * *" # 1st of every month at 03:00 UTC
successfulJobsHistoryLimit: 3
failedJobsHistoryLimit: 3
failedJobsHistoryLimit: 5
concurrencyPolicy: Forbid
jobTemplate:
spec:
backoffLimit: 2
activeDeadlineSeconds: 120
backoffLimit: 3
activeDeadlineSeconds: 180
template:
metadata:
labels:
@@ -62,7 +73,7 @@ spec:
serviceAccountName: gitlab-pat-rotator
restartPolicy: Never
nodeSelector:
k8s.scaleway.com/pool-name: infra
k8s.scaleway.com/pool-name: platform
containers:
- name: rotate
image: alpine:3.21
@@ -76,22 +87,25 @@ spec:
CURRENT_PAT=$(cat /etc/gitlab-pat/token)
GITLAB_URL="http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181"
# Verify current token is still valid
# 1. Verify current token is valid
TOKEN_INFO=$(curl -sf "${GITLAB_URL}/api/v4/personal_access_tokens/self" \
-H "PRIVATE-TOKEN: ${CURRENT_PAT}") || {
echo "ERROR: Current PAT is invalid or GitLab unreachable"
echo "CRITICAL: Current PAT is invalid or GitLab unreachable."
echo "Manual recovery required: create a new PAT in GitLab UI and run:"
echo " kubectl -n foxhunt create secret generic gitlab-pat --from-literal=token=<new-pat> --dry-run=client -o yaml | kubectl apply -f -"
exit 1
}
OLD_EXPIRY=$(echo "$TOKEN_INFO" | jq -r '.expires_at')
echo "Current PAT expires: ${OLD_EXPIRY}"
TOKEN_NAME=$(echo "$TOKEN_INFO" | jq -r '.name')
echo "Current PAT '${TOKEN_NAME}' expires: ${OLD_EXPIRY}"
# Rotate: new expiry = 1 year from now
# 2. Rotate: new expiry = 1 year from now
EXPIRES=$(date -d "@$(($(date +%s) + 31536000))" +%Y-%m-%d)
RESPONSE=$(curl -sf -X POST \
"${GITLAB_URL}/api/v4/personal_access_tokens/self/rotate" \
-H "PRIVATE-TOKEN: ${CURRENT_PAT}" \
-d "expires_at=${EXPIRES}") || {
echo "ERROR: Rotation API call failed"
echo "ERROR: Rotation API call failed. Token may still be valid."
exit 1
}
@@ -103,7 +117,7 @@ spec:
echo "PAT rotated successfully. New expiry: ${EXPIRES}"
# Update K8s secret via API (old token is already revoked at this point)
# 3. Update K8s secret (old token is already revoked at this point)
K8S_TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
K8S_CA=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt
K8S_NS=$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace)
@@ -111,8 +125,7 @@ spec:
NEW_PAT_B64=$(printf '%s' "$NEW_PAT" | base64)
# Retry secret update — losing the new token after rotation is catastrophic
for attempt in 1 2 3; do
for attempt in 1 2 3 4 5; do
if curl -sf -X PATCH \
"${K8S_API}/api/v1/namespaces/${K8S_NS}/secrets/gitlab-pat" \
--cacert "$K8S_CA" \
@@ -120,17 +133,27 @@ spec:
-H "Content-Type: application/strategic-merge-patch+json" \
-d "{\"data\":{\"token\":\"${NEW_PAT_B64}\"}}"; then
echo ""
echo "Secret updated on attempt ${attempt}."
echo "K8s secret updated on attempt ${attempt}."
# 4. Verify the new token works
VERIFY=$(curl -sf "${GITLAB_URL}/api/v4/personal_access_tokens/self" \
-H "PRIVATE-TOKEN: ${NEW_PAT}" | jq -r '.expires_at') || true
if [ "$VERIFY" = "$EXPIRES" ]; then
echo "Verification passed: new token valid until ${VERIFY}"
else
echo "WARNING: Verification returned unexpected expiry: ${VERIFY}"
fi
exit 0
fi
echo "Secret update attempt ${attempt} failed, retrying..."
sleep 2
echo "Secret update attempt ${attempt} failed, retrying in 3s..."
sleep 3
done
# If we get here, rotation succeeded but secret update failed.
# Print token so it can be recovered from job logs.
echo "CRITICAL: Secret update failed after 3 attempts!"
echo "CRITICAL: Secret update failed after 5 attempts!"
echo "New PAT (recover manually): ${NEW_PAT}"
echo "Run: kubectl -n foxhunt create secret generic gitlab-pat --from-literal=token=<above> --dry-run=client -o yaml | kubectl apply -f -"
exit 1
volumeMounts:
- name: gitlab-pat
@@ -147,3 +170,84 @@ spec:
- name: gitlab-pat
secret:
secretName: gitlab-pat
---
# Weekly expiry check — warns if token expires within 14 days
apiVersion: batch/v1
kind: CronJob
metadata:
name: gitlab-pat-expiry-check
namespace: foxhunt
spec:
schedule: "0 8 * * 1" # Every Monday at 08:00 UTC
successfulJobsHistoryLimit: 1
failedJobsHistoryLimit: 3
concurrencyPolicy: Forbid
jobTemplate:
spec:
backoffLimit: 1
activeDeadlineSeconds: 60
template:
metadata:
labels:
app: gitlab-pat-expiry-check
spec:
restartPolicy: Never
nodeSelector:
k8s.scaleway.com/pool-name: platform
containers:
- name: check
image: alpine:3.21
command: ["/bin/sh", "-c"]
args:
- |
set -e
apk add --no-cache -q curl jq
CURRENT_PAT=$(cat /etc/gitlab-pat/token)
GITLAB_URL="http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181"
TOKEN_INFO=$(curl -sf "${GITLAB_URL}/api/v4/personal_access_tokens/self" \
-H "PRIVATE-TOKEN: ${CURRENT_PAT}") || {
echo "CRITICAL: PAT is INVALID. CI will fail!"
echo "Create a new PAT in GitLab UI and run:"
echo " kubectl -n foxhunt create secret generic gitlab-pat --from-literal=token=<new-pat> --dry-run=client -o yaml | kubectl apply -f -"
exit 1
}
EXPIRY=$(echo "$TOKEN_INFO" | jq -r '.expires_at')
TOKEN_NAME=$(echo "$TOKEN_INFO" | jq -r '.name')
SCOPES=$(echo "$TOKEN_INFO" | jq -r '.scopes | join(",")')
# Calculate days until expiry
EXPIRY_EPOCH=$(date -d "$EXPIRY" +%s)
NOW_EPOCH=$(date +%s)
DAYS_LEFT=$(( (EXPIRY_EPOCH - NOW_EPOCH) / 86400 ))
echo "PAT '${TOKEN_NAME}': expires=${EXPIRY} scopes=${SCOPES} days_left=${DAYS_LEFT}"
if [ "$DAYS_LEFT" -lt 0 ]; then
echo "CRITICAL: PAT has EXPIRED ${DAYS_LEFT} days ago!"
exit 1
elif [ "$DAYS_LEFT" -lt 14 ]; then
echo "WARNING: PAT expires in ${DAYS_LEFT} days. Rotation may have failed."
echo "Manual rotation: kubectl -n foxhunt create job pat-rotate-manual --from=cronjob/gitlab-pat-rotation"
exit 1
else
echo "OK: PAT valid for ${DAYS_LEFT} more days."
fi
volumeMounts:
- name: gitlab-pat
mountPath: /etc/gitlab-pat
readOnly: true
resources:
requests:
cpu: 10m
memory: 32Mi
limits:
cpu: 100m
memory: 64Mi
volumes:
- name: gitlab-pat
secret:
secretName: gitlab-pat

View File

@@ -10,49 +10,35 @@ inputs = {
cluster_name = "foxhunt"
k8s_version = "1.34"
# Foxhunt app services pool (api-gateway, trading-service, etc.)
# Renamed from "services" — single DEV1-L, dedicated to microservices
foxhunt_type = "DEV1-L"
foxhunt_max_size = 1
# Platform pool — runs everything: app services, databases, GitLab, monitoring
platform_type = "DEV1-L"
platform_max_size = 3
# Platform infrastructure pool (postgres, redis, minio, questdb, monitoring)
enable_platform_pool = true
platform_type = "DEV1-L"
# GitLab CE pool (dedicated for GitLab + Prometheus + Runner manager)
enable_gitlab_pool = true
gitlab_type = "GP1-XS"
# CPU compile pool (POP2-HC-32C-64G for cargo check/build — 32 vCPU, 64GB RAM, high clock, no GPU)
# CPU compile pool (POP2-HC-32C-64G — 32 vCPU, 64GB RAM, high clock)
enable_ci_compile_cpu_pool = true
ci_compile_cpu_type = "POP2-HC-32C-64G"
ci_compile_cpu_max_size = 4
# L40S training pool (48GB VRAM, CUDA CC 89 — supervised hyperopt)
# L40S training pool (48GB VRAM, CUDA CC 89)
enable_ci_training_l40s_pool = true
ci_training_l40s_type = "L40S-1-48G"
ci_training_l40s_max_size = 1
# H100 training pool (80GB VRAM, CUDA CC 90 — RL hyperopt)
# H100 training pool (80GB VRAM, CUDA CC 90)
enable_ci_training_h100_pool = true
ci_training_h100_type = "H100-1-80G"
ci_training_h100_max_size = 1
# H100x2 training pool (2× 80GB — multi-GPU parallel hyperopt)
# H100x2 training pool (2x 80GB — multi-GPU parallel hyperopt)
enable_ci_training_h100x2_pool = true
ci_training_h100x2_type = "H100-2-80G"
ci_training_h100x2_max_size = 1
# H100-SXM training pool (2× 80GB SXM, NVLink — future multi-GPU)
# H100-SXM training pool (2x 80GB SXM, NVLink)
enable_ci_training_h100_sxm_pool = true
ci_training_h100_sxm_type = "H100-SXM-2-80G"
ci_training_h100_sxm_max_size = 1
# H100-SXM-8 training pool (8× 80GB SXM, NVLink — full multi-GPU saturation)
enable_ci_training_h100_sxm8_pool = true
ci_training_h100_sxm8_type = "H100-SXM-8-80G"
ci_training_h100_sxm8_max_size = 1
# NAT gateway handles egress — no public IPs on nodes
public_ip_disabled = true
}

View File

@@ -20,6 +20,8 @@ remote_state {
address = "${get_env("GITLAB_TF_STATE_URL", "https://tf.fxhnt.ai")}/api/v4/projects/1/terraform/state/${path_relative_to_include()}"
lock_address = "${get_env("GITLAB_TF_STATE_URL", "https://tf.fxhnt.ai")}/api/v4/projects/1/terraform/state/${path_relative_to_include()}/lock"
unlock_address = "${get_env("GITLAB_TF_STATE_URL", "https://tf.fxhnt.ai")}/api/v4/projects/1/terraform/state/${path_relative_to_include()}/lock"
lock_method = "POST"
unlock_method = "DELETE"
}
}

View File

@@ -26,16 +26,14 @@ resource "scaleway_k8s_cluster" "foxhunt" {
}
}
# Foxhunt app services pool (api-gateway, trading-service, etc.)
# Renamed from "services" — was 2× DEV1-L shared with databases,
# now 1× DEV1-L dedicated to foxhunt microservices only.
resource "scaleway_k8s_pool" "foxhunt" {
# Platform pool — runs everything: app services, databases, GitLab, monitoring
resource "scaleway_k8s_pool" "platform" {
cluster_id = scaleway_k8s_cluster.foxhunt.id
name = "foxhunt"
node_type = var.foxhunt_type
name = "platform"
node_type = var.platform_type
size = 1
min_size = 1
max_size = var.foxhunt_max_size
max_size = var.platform_max_size
autoscaling = true
autohealing = true
public_ip_disabled = var.public_ip_disabled
@@ -46,38 +44,7 @@ resource "scaleway_k8s_pool" "foxhunt" {
}
}
# Platform infrastructure pool (postgres, redis, minio, questdb, monitoring)
resource "scaleway_k8s_pool" "platform" {
count = var.enable_platform_pool ? 1 : 0
cluster_id = scaleway_k8s_cluster.foxhunt.id
name = "platform"
node_type = var.platform_type
size = 1
min_size = 1
max_size = 1
autoscaling = false
autohealing = true
public_ip_disabled = var.public_ip_disabled
region = var.region
}
# GitLab CE node pool (dedicated GP1-XS for GitLab + Runner manager)
resource "scaleway_k8s_pool" "gitlab" {
count = var.enable_gitlab_pool ? 1 : 0
cluster_id = scaleway_k8s_cluster.foxhunt.id
name = "gitlab"
node_type = var.gitlab_type
size = 1
min_size = 1
max_size = 1
autoscaling = false
autohealing = true
public_ip_disabled = var.public_ip_disabled
region = var.region
}
# L40S training pool (48GB VRAM, CUDA CC 89 — hyperopt + supervised training)
# Nodes auto-labeled: k8s.scaleway.com/pool-name=ci-training-l40s
resource "scaleway_k8s_pool" "ci_training_l40s" {
count = var.enable_ci_training_l40s_pool ? 1 : 0
cluster_id = scaleway_k8s_cluster.foxhunt.id
@@ -96,8 +63,7 @@ resource "scaleway_k8s_pool" "ci_training_l40s" {
}
}
# H100 training pool (80GB VRAM, CUDA CC 90 — RL hyperopt + large model training)
# Nodes auto-labeled: k8s.scaleway.com/pool-name=ci-training-h100
# H100 training pool (80GB VRAM, CUDA CC 90 — RL hyperopt)
resource "scaleway_k8s_pool" "ci_training_h100" {
count = var.enable_ci_training_h100_pool ? 1 : 0
cluster_id = scaleway_k8s_cluster.foxhunt.id
@@ -116,8 +82,7 @@ resource "scaleway_k8s_pool" "ci_training_h100" {
}
}
# H100x2 training pool (2× 80GB VRAM — multi-GPU parallel hyperopt)
# Nodes auto-labeled: k8s.scaleway.com/pool-name=ci-training-h100x2
# H100x2 training pool (2x 80GB VRAM — multi-GPU parallel hyperopt)
resource "scaleway_k8s_pool" "ci_training_h100x2" {
count = var.enable_ci_training_h100x2_pool ? 1 : 0
cluster_id = scaleway_k8s_cluster.foxhunt.id
@@ -136,8 +101,7 @@ resource "scaleway_k8s_pool" "ci_training_h100x2" {
}
}
# H100-SXM training pool (2× 80GB SXM VRAM — NVLink multi-GPU)
# Nodes auto-labeled: k8s.scaleway.com/pool-name=ci-training-h100-sxm
# H100-SXM training pool (2x 80GB SXM VRAM — NVLink multi-GPU)
resource "scaleway_k8s_pool" "ci_training_h100_sxm" {
count = var.enable_ci_training_h100_sxm_pool ? 1 : 0
cluster_id = scaleway_k8s_cluster.foxhunt.id
@@ -156,28 +120,7 @@ resource "scaleway_k8s_pool" "ci_training_h100_sxm" {
}
}
# H100-SXM-8 training pool (8× 80GB SXM VRAM, NVLink — full multi-GPU saturation)
# Nodes auto-labeled: k8s.scaleway.com/pool-name=ci-training-h100-sxm8
resource "scaleway_k8s_pool" "ci_training_h100_sxm8" {
count = var.enable_ci_training_h100_sxm8_pool ? 1 : 0
cluster_id = scaleway_k8s_cluster.foxhunt.id
name = "ci-training-h100-sxm8"
node_type = var.ci_training_h100_sxm8_type
size = 1
min_size = 0
max_size = var.ci_training_h100_sxm8_max_size
autoscaling = true
autohealing = true
public_ip_disabled = var.public_ip_disabled
region = var.region
lifecycle {
ignore_changes = [size]
}
}
# CPU compile pool (POP2 — high CPU, no GPU, for cargo check/build)
# Nodes auto-labeled: k8s.scaleway.com/pool-name=ci-compile-cpu
# CPU compile pool (POP2-HC — high CPU, no GPU, for cargo check/build)
resource "scaleway_k8s_pool" "ci_compile_cpu" {
count = var.enable_ci_compile_cpu_pool ? 1 : 0
cluster_id = scaleway_k8s_cluster.foxhunt.id

View File

@@ -14,23 +14,13 @@ output "apiserver_url" {
value = scaleway_k8s_cluster.foxhunt.apiserver_url
}
output "foxhunt_pool_id" {
description = "ID of the foxhunt app services node pool"
value = scaleway_k8s_pool.foxhunt.id
}
output "platform_pool_id" {
description = "ID of the platform infrastructure node pool"
value = var.enable_platform_pool ? scaleway_k8s_pool.platform[0].id : ""
}
output "gitlab_pool_id" {
description = "ID of the GitLab node pool"
value = var.enable_gitlab_pool ? scaleway_k8s_pool.gitlab[0].id : ""
description = "ID of the platform node pool"
value = scaleway_k8s_pool.platform.id
}
output "ci_compile_cpu_pool_id" {
description = "ID of the CPU compile node pool (POP2-HC)"
description = "ID of the CPU compile node pool"
value = var.enable_ci_compile_cpu_pool ? scaleway_k8s_pool.ci_compile_cpu[0].id : ""
}
@@ -54,15 +44,7 @@ output "ci_training_h100_sxm_pool_id" {
value = var.enable_ci_training_h100_sxm_pool ? scaleway_k8s_pool.ci_training_h100_sxm[0].id : ""
}
output "ci_training_h100_sxm8_pool_id" {
description = "ID of the H100-SXM-8 training node pool (8× 80GB)"
value = var.enable_ci_training_h100_sxm8_pool ? scaleway_k8s_pool.ci_training_h100_sxm8[0].id : ""
}
# Dev pool removed — DevPod runs on platform pool
output "private_network_id" {
description = "ID of the VPC private network the cluster is attached to"
value = scaleway_vpc_private_network.foxhunt.id
}

View File

@@ -6,53 +6,25 @@ variable "region" {
variable "cluster_name" {
description = "Name of the Kubernetes cluster"
type = string
default = "foxhunt"
}
variable "k8s_version" {
description = "Kubernetes version for the Kapsule cluster"
type = string
default = "1.34"
}
variable "foxhunt_type" {
description = "Instance type for the foxhunt app services node pool"
type = string
default = "DEV1-L"
}
variable "foxhunt_max_size" {
description = "Maximum number of nodes in the foxhunt pool (autoscaling)"
type = number
default = 1
}
variable "enable_platform_pool" {
description = "Create dedicated node pool for platform infrastructure (databases, monitoring)"
type = bool
default = false
}
variable "platform_type" {
description = "Instance type for the platform infrastructure node pool"
description = "Instance type for the platform node pool"
type = string
default = "DEV1-L"
}
variable "enable_gitlab_pool" {
description = "Create dedicated node pool for GitLab CE"
type = bool
default = false
}
variable "gitlab_type" {
description = "Instance type for the GitLab node pool"
type = string
default = "DEV1-L"
variable "platform_max_size" {
description = "Maximum number of nodes in the platform pool"
type = number
}
variable "enable_ci_training_l40s_pool" {
description = "Create L40S GPU node pool for training + hyperopt (48GB VRAM, CC 89)"
description = "Create L40S GPU node pool for training + hyperopt"
type = bool
default = false
}
@@ -70,7 +42,7 @@ variable "ci_training_l40s_max_size" {
}
variable "enable_ci_training_h100_pool" {
description = "Create H100 GPU node pool for training + hyperopt (80GB VRAM, CC 90)"
description = "Create H100 GPU node pool for training + hyperopt"
type = bool
default = false
}
@@ -88,7 +60,7 @@ variable "ci_training_h100_max_size" {
}
variable "enable_ci_training_h100x2_pool" {
description = "Create H100x2 GPU node pool for multi-GPU training (2× 80GB VRAM)"
description = "Create H100x2 GPU node pool for multi-GPU training"
type = bool
default = false
}
@@ -106,7 +78,7 @@ variable "ci_training_h100x2_max_size" {
}
variable "enable_ci_training_h100_sxm_pool" {
description = "Create H100-SXM GPU node pool for NVLink multi-GPU training (2× 80GB SXM)"
description = "Create H100-SXM GPU node pool for NVLink multi-GPU training"
type = bool
default = false
}
@@ -123,34 +95,15 @@ variable "ci_training_h100_sxm_max_size" {
default = 1
}
variable "enable_ci_training_h100_sxm8_pool" {
description = "Create H100-SXM-8 GPU node pool for full multi-GPU training (8× 80GB SXM, NVLink)"
type = bool
default = false
}
variable "ci_training_h100_sxm8_type" {
description = "Instance type for the H100-SXM-8 training pool"
type = string
default = "H100-SXM-8-80G"
}
variable "ci_training_h100_sxm8_max_size" {
description = "Maximum number of nodes in the H100-SXM-8 training pool"
type = number
default = 1
}
variable "enable_ci_compile_cpu_pool" {
description = "Create high-CPU node pool for CI compilation (no GPU needed)"
description = "Create high-CPU node pool for CI compilation"
type = bool
default = false
}
variable "ci_compile_cpu_type" {
description = "Instance type for the CPU compile pool (32 vCPU, 64GB RAM, high clock, no GPU)"
description = "Instance type for the CPU compile pool"
type = string
default = "POP2-HC-32C-64G"
}
variable "ci_compile_cpu_max_size" {
@@ -164,4 +117,3 @@ variable "public_ip_disabled" {
type = bool
default = false
}