infra: automate terragrunt via Argo CI, clean up kapsule module, harden PAT rotation
- Add terragrunt-apply step to Argo CI pipeline (plan+apply on main push when infra/live/ or infra/modules/ change) - Bake OpenTofu 1.9.0 + Terragrunt 0.77.12 into ci-builder-cpu image with SHA256 checksum verification - Remove 3 ghost node pools (foxhunt, gitlab, h100-sxm8) from kapsule module to match Scaleway reality - Make terragrunt.hcl single source of truth (remove variable defaults) - Fix GitLab TF state lock methods (POST/DELETE for HTTP backend) - Harden PAT rotation: more retries, verification step, recovery docs - Add weekly PAT expiry check CronJob (warns 14 days before expiry) Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
# CUDA-free CI builder image for Foxhunt service compilation
|
||||
# ~2-3GB vs ~8GB for the CUDA devel variant (Dockerfile.ci-builder)
|
||||
# Contains: Rust 1.89, protoc, sccache, rclone, git, OpenSSL, clang, mold, lld, make
|
||||
# Contains: Rust 1.89, protoc, sccache, rclone, OpenTofu, Terragrunt, git, OpenSSL, clang, mold, lld, make
|
||||
# Base: Ubuntu 24.04 (glibc 2.39) — matches local dev and training runtime
|
||||
# Build: docker build -f infra/docker/Dockerfile.ci-builder-cpu -t foxhunt-ci-builder-cpu .
|
||||
# Push: docker tag foxhunt-ci-builder-cpu gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/ci-builder-cpu:latest
|
||||
@@ -55,5 +55,17 @@ RUN curl -fsSL https://downloads.rclone.org/v1.69.1/rclone-v1.69.1-linux-amd64.z
|
||||
# clippy pre-installed
|
||||
RUN rustup component add clippy
|
||||
|
||||
# OpenTofu 1.9.0 (Terraform-compatible IaC, used by Terragrunt for infra CI)
|
||||
RUN curl -fsSL https://github.com/opentofu/opentofu/releases/download/v1.9.0/tofu_1.9.0_linux_amd64.zip -o /tmp/tofu.zip \
|
||||
&& echo "638dd3fb9ecfa6fd9f54a0024b195b12b407c51ccee6f83b18a75a8be79f8214 /tmp/tofu.zip" | sha256sum -c - \
|
||||
&& unzip -o /tmp/tofu.zip -d /usr/local/bin tofu \
|
||||
&& rm /tmp/tofu.zip \
|
||||
&& chmod +x /usr/local/bin/tofu
|
||||
|
||||
# Terragrunt 0.77.12 (DRY Terraform/OpenTofu wrapper)
|
||||
RUN curl -fsSL https://github.com/gruntwork-io/terragrunt/releases/download/v0.77.12/terragrunt_linux_amd64 -o /usr/local/bin/terragrunt \
|
||||
&& echo "7f6592fc7faa55b75d7a50ac6a485c48be502486ea61680836f19677b6e8e1b9 /usr/local/bin/terragrunt" | sha256sum -c - \
|
||||
&& chmod +x /usr/local/bin/terragrunt
|
||||
|
||||
# Verify
|
||||
RUN rustc --version && cargo --version && git --version && protoc --version && sccache --version && make --version && mold --version && rclone --version
|
||||
RUN rustc --version && cargo --version && git --version && protoc --version && sccache --version && make --version && mold --version && rclone --version && tofu --version && terragrunt --version
|
||||
|
||||
@@ -172,6 +172,15 @@ spec:
|
||||
value: foxhunt-training-runtime
|
||||
when: "{{tasks.detect-changes.outputs.parameters.docker-images}} == true"
|
||||
|
||||
- name: terragrunt-apply
|
||||
dependencies: [detect-changes]
|
||||
template: terragrunt-apply
|
||||
arguments:
|
||||
parameters:
|
||||
- name: commit-sha
|
||||
value: "{{workflow.parameters.commit-sha}}"
|
||||
when: "{{tasks.detect-changes.outputs.parameters.needs-infra}} == true"
|
||||
|
||||
# ── detect-changes: granular per-binary change detection ──
|
||||
- name: detect-changes
|
||||
nodeSelector:
|
||||
@@ -262,6 +271,7 @@ spec:
|
||||
DOCKER_IMAGES=$(check_paths "infra/docker/")
|
||||
NEEDS_DASHBOARD=$(check_paths "web-dashboard/")
|
||||
INFRA_CI=$(check_paths "infra/k8s/argo/ci-pipeline infra/k8s/services/")
|
||||
NEEDS_INFRA=$(check_paths "infra/live/ infra/modules/")
|
||||
|
||||
# ========================================
|
||||
# Build SERVICE package list
|
||||
@@ -407,6 +417,7 @@ spec:
|
||||
echo "needs-training: $NEEDS_TRAINING"
|
||||
echo "needs-code: $NEEDS_CODE"
|
||||
echo "needs-dashboard: $NEEDS_DASHBOARD"
|
||||
echo "needs-infra: $NEEDS_INFRA"
|
||||
echo "docker-images: $DOCKER_IMAGES"
|
||||
echo "service-packages: $SERVICE_PKGS"
|
||||
echo "training-examples: $TRAINING_EXAMPLES"
|
||||
@@ -419,6 +430,7 @@ spec:
|
||||
echo -n "$NEEDS_DASHBOARD" > /tmp/outputs/needs-dashboard
|
||||
echo -n "$DOCKER_IMAGES" > /tmp/outputs/docker-images
|
||||
echo -n "$NEEDS_CODE" > /tmp/outputs/needs-code
|
||||
echo -n "$NEEDS_INFRA" > /tmp/outputs/needs-infra
|
||||
echo -n "$SERVICE_PKGS" > /tmp/outputs/service-packages
|
||||
echo -n "$TRAINING_EXAMPLES" > /tmp/outputs/training-examples
|
||||
echo -n "$DEPLOY_LIST" > /tmp/outputs/deploy-list
|
||||
@@ -451,6 +463,9 @@ spec:
|
||||
- name: deploy-list
|
||||
valueFrom:
|
||||
path: /tmp/outputs/deploy-list
|
||||
- name: needs-infra
|
||||
valueFrom:
|
||||
path: /tmp/outputs/needs-infra
|
||||
|
||||
# ── create-tag: CalVer auto-tag on code changes ──
|
||||
- name: create-tag
|
||||
@@ -1051,3 +1066,104 @@ spec:
|
||||
done
|
||||
|
||||
echo "=== Deploy ${TAG} complete ($DEPLOY_LIST) ==="
|
||||
|
||||
# ── terragrunt-apply: apply infra changes on main push ──
|
||||
- name: terragrunt-apply
|
||||
inputs:
|
||||
parameters:
|
||||
- name: commit-sha
|
||||
nodeSelector:
|
||||
k8s.scaleway.com/pool-name: platform
|
||||
serviceAccountName: argo-workflow
|
||||
container:
|
||||
image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/ci-builder-cpu:latest
|
||||
command: ["/bin/sh", "-c"]
|
||||
env:
|
||||
- name: GITLAB_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: gitlab-pat
|
||||
key: token
|
||||
- name: TF_HTTP_USERNAME
|
||||
value: root
|
||||
- name: TF_HTTP_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: gitlab-pat
|
||||
key: token
|
||||
- name: SCW_ACCESS_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: scaleway-credentials
|
||||
key: access-key
|
||||
- name: SCW_SECRET_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: scaleway-credentials
|
||||
key: secret-key
|
||||
- name: SCW_DEFAULT_PROJECT_ID
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: scaleway-credentials
|
||||
key: project-id
|
||||
- name: GITLAB_TF_STATE_URL
|
||||
value: "http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181"
|
||||
resources:
|
||||
requests:
|
||||
cpu: 200m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: "1"
|
||||
memory: 512Mi
|
||||
volumeMounts:
|
||||
- name: git-ssh-key
|
||||
mountPath: /etc/git-ssh
|
||||
readOnly: true
|
||||
args:
|
||||
- |
|
||||
set -e
|
||||
|
||||
# Clone repo
|
||||
mkdir -p /root/.ssh
|
||||
cp /etc/git-ssh/ssh-privatekey /root/.ssh/id_ed25519
|
||||
chmod 600 /root/.ssh/id_ed25519
|
||||
printf 'Host *\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n' > /root/.ssh/config
|
||||
chmod 600 /root/.ssh/config
|
||||
|
||||
SHA="{{inputs.parameters.commit-sha}}"
|
||||
REPO="ssh://git@gitlab-gitlab-shell.foxhunt.svc.cluster.local:2222/root/foxhunt.git"
|
||||
git clone --no-checkout --filter=blob:none "$REPO" /workspace/src
|
||||
cd /workspace/src
|
||||
git checkout "$SHA"
|
||||
echo "Checked out $(git rev-parse --short HEAD)"
|
||||
|
||||
tofu --version
|
||||
terragrunt --version
|
||||
|
||||
# Apply each terragrunt module
|
||||
for module in block-storage dns public-gateway kapsule; do
|
||||
MODULE_DIR="infra/live/production/${module}"
|
||||
if [ -d "$MODULE_DIR" ]; then
|
||||
echo "=== Terragrunt plan: ${module} ==="
|
||||
cd "/workspace/src/${MODULE_DIR}"
|
||||
terragrunt init --non-interactive -reconfigure
|
||||
OUTPUT=$(terragrunt plan --non-interactive -detailed-exitcode 2>&1) || EXITCODE=$?
|
||||
EXITCODE=${EXITCODE:-0}
|
||||
|
||||
if [ "$EXITCODE" -eq 2 ]; then
|
||||
echo "=== Terragrunt plan output: ${module} ==="
|
||||
echo "$OUTPUT"
|
||||
echo "=== Terragrunt apply: ${module} ==="
|
||||
terragrunt apply --non-interactive -auto-approve
|
||||
elif [ "$EXITCODE" -eq 0 ]; then
|
||||
echo "=== No changes for ${module} ==="
|
||||
else
|
||||
echo "=== ERROR planning ${module} ==="
|
||||
echo "$OUTPUT"
|
||||
exit 1
|
||||
fi
|
||||
cd /workspace/src
|
||||
fi
|
||||
done
|
||||
|
||||
echo "=== Terragrunt apply complete ==="
|
||||
|
||||
@@ -2,12 +2,22 @@
|
||||
# Rotates the personal access token monthly, storing the new token in a K8s Secret.
|
||||
# Uses GitLab's atomic rotate API: creates new token + revokes old in one call.
|
||||
#
|
||||
# The rotation runs on the 1st of every month. Additionally, a weekly expiry check
|
||||
# runs every Monday and logs a warning if the token expires within 14 days, giving
|
||||
# time to investigate rotation failures before CI breaks.
|
||||
#
|
||||
# Initial setup:
|
||||
# kubectl -n foxhunt create secret generic gitlab-pat \
|
||||
# --from-literal=token=glpat-<initial-token>
|
||||
#
|
||||
# Manual trigger:
|
||||
# Manual rotation:
|
||||
# kubectl -n foxhunt create job pat-rotate-manual --from=cronjob/gitlab-pat-rotation
|
||||
#
|
||||
# Manual expiry check:
|
||||
# kubectl -n foxhunt create job pat-check-manual --from=cronjob/gitlab-pat-expiry-check
|
||||
#
|
||||
# Sync local token:
|
||||
# export GITLAB_TOKEN=$(kubectl -n foxhunt get secret gitlab-pat -o jsonpath='{.data.token}' | base64 -d)
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
@@ -40,6 +50,7 @@ roleRef:
|
||||
name: gitlab-pat-rotator
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
---
|
||||
# Monthly rotation — creates new token, revokes old, updates K8s secret
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
@@ -48,12 +59,12 @@ metadata:
|
||||
spec:
|
||||
schedule: "0 3 1 * *" # 1st of every month at 03:00 UTC
|
||||
successfulJobsHistoryLimit: 3
|
||||
failedJobsHistoryLimit: 3
|
||||
failedJobsHistoryLimit: 5
|
||||
concurrencyPolicy: Forbid
|
||||
jobTemplate:
|
||||
spec:
|
||||
backoffLimit: 2
|
||||
activeDeadlineSeconds: 120
|
||||
backoffLimit: 3
|
||||
activeDeadlineSeconds: 180
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
@@ -62,7 +73,7 @@ spec:
|
||||
serviceAccountName: gitlab-pat-rotator
|
||||
restartPolicy: Never
|
||||
nodeSelector:
|
||||
k8s.scaleway.com/pool-name: infra
|
||||
k8s.scaleway.com/pool-name: platform
|
||||
containers:
|
||||
- name: rotate
|
||||
image: alpine:3.21
|
||||
@@ -76,22 +87,25 @@ spec:
|
||||
CURRENT_PAT=$(cat /etc/gitlab-pat/token)
|
||||
GITLAB_URL="http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181"
|
||||
|
||||
# Verify current token is still valid
|
||||
# 1. Verify current token is valid
|
||||
TOKEN_INFO=$(curl -sf "${GITLAB_URL}/api/v4/personal_access_tokens/self" \
|
||||
-H "PRIVATE-TOKEN: ${CURRENT_PAT}") || {
|
||||
echo "ERROR: Current PAT is invalid or GitLab unreachable"
|
||||
echo "CRITICAL: Current PAT is invalid or GitLab unreachable."
|
||||
echo "Manual recovery required: create a new PAT in GitLab UI and run:"
|
||||
echo " kubectl -n foxhunt create secret generic gitlab-pat --from-literal=token=<new-pat> --dry-run=client -o yaml | kubectl apply -f -"
|
||||
exit 1
|
||||
}
|
||||
OLD_EXPIRY=$(echo "$TOKEN_INFO" | jq -r '.expires_at')
|
||||
echo "Current PAT expires: ${OLD_EXPIRY}"
|
||||
TOKEN_NAME=$(echo "$TOKEN_INFO" | jq -r '.name')
|
||||
echo "Current PAT '${TOKEN_NAME}' expires: ${OLD_EXPIRY}"
|
||||
|
||||
# Rotate: new expiry = 1 year from now
|
||||
# 2. Rotate: new expiry = 1 year from now
|
||||
EXPIRES=$(date -d "@$(($(date +%s) + 31536000))" +%Y-%m-%d)
|
||||
RESPONSE=$(curl -sf -X POST \
|
||||
"${GITLAB_URL}/api/v4/personal_access_tokens/self/rotate" \
|
||||
-H "PRIVATE-TOKEN: ${CURRENT_PAT}" \
|
||||
-d "expires_at=${EXPIRES}") || {
|
||||
echo "ERROR: Rotation API call failed"
|
||||
echo "ERROR: Rotation API call failed. Token may still be valid."
|
||||
exit 1
|
||||
}
|
||||
|
||||
@@ -103,7 +117,7 @@ spec:
|
||||
|
||||
echo "PAT rotated successfully. New expiry: ${EXPIRES}"
|
||||
|
||||
# Update K8s secret via API (old token is already revoked at this point)
|
||||
# 3. Update K8s secret (old token is already revoked at this point)
|
||||
K8S_TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
|
||||
K8S_CA=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt
|
||||
K8S_NS=$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace)
|
||||
@@ -111,8 +125,7 @@ spec:
|
||||
|
||||
NEW_PAT_B64=$(printf '%s' "$NEW_PAT" | base64)
|
||||
|
||||
# Retry secret update — losing the new token after rotation is catastrophic
|
||||
for attempt in 1 2 3; do
|
||||
for attempt in 1 2 3 4 5; do
|
||||
if curl -sf -X PATCH \
|
||||
"${K8S_API}/api/v1/namespaces/${K8S_NS}/secrets/gitlab-pat" \
|
||||
--cacert "$K8S_CA" \
|
||||
@@ -120,17 +133,27 @@ spec:
|
||||
-H "Content-Type: application/strategic-merge-patch+json" \
|
||||
-d "{\"data\":{\"token\":\"${NEW_PAT_B64}\"}}"; then
|
||||
echo ""
|
||||
echo "Secret updated on attempt ${attempt}."
|
||||
echo "K8s secret updated on attempt ${attempt}."
|
||||
|
||||
# 4. Verify the new token works
|
||||
VERIFY=$(curl -sf "${GITLAB_URL}/api/v4/personal_access_tokens/self" \
|
||||
-H "PRIVATE-TOKEN: ${NEW_PAT}" | jq -r '.expires_at') || true
|
||||
if [ "$VERIFY" = "$EXPIRES" ]; then
|
||||
echo "Verification passed: new token valid until ${VERIFY}"
|
||||
else
|
||||
echo "WARNING: Verification returned unexpected expiry: ${VERIFY}"
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
echo "Secret update attempt ${attempt} failed, retrying..."
|
||||
sleep 2
|
||||
echo "Secret update attempt ${attempt} failed, retrying in 3s..."
|
||||
sleep 3
|
||||
done
|
||||
|
||||
# If we get here, rotation succeeded but secret update failed.
|
||||
# Print token so it can be recovered from job logs.
|
||||
echo "CRITICAL: Secret update failed after 3 attempts!"
|
||||
echo "CRITICAL: Secret update failed after 5 attempts!"
|
||||
echo "New PAT (recover manually): ${NEW_PAT}"
|
||||
echo "Run: kubectl -n foxhunt create secret generic gitlab-pat --from-literal=token=<above> --dry-run=client -o yaml | kubectl apply -f -"
|
||||
exit 1
|
||||
volumeMounts:
|
||||
- name: gitlab-pat
|
||||
@@ -147,3 +170,84 @@ spec:
|
||||
- name: gitlab-pat
|
||||
secret:
|
||||
secretName: gitlab-pat
|
||||
---
|
||||
# Weekly expiry check — warns if token expires within 14 days
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: gitlab-pat-expiry-check
|
||||
namespace: foxhunt
|
||||
spec:
|
||||
schedule: "0 8 * * 1" # Every Monday at 08:00 UTC
|
||||
successfulJobsHistoryLimit: 1
|
||||
failedJobsHistoryLimit: 3
|
||||
concurrencyPolicy: Forbid
|
||||
jobTemplate:
|
||||
spec:
|
||||
backoffLimit: 1
|
||||
activeDeadlineSeconds: 60
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: gitlab-pat-expiry-check
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
nodeSelector:
|
||||
k8s.scaleway.com/pool-name: platform
|
||||
containers:
|
||||
- name: check
|
||||
image: alpine:3.21
|
||||
command: ["/bin/sh", "-c"]
|
||||
args:
|
||||
- |
|
||||
set -e
|
||||
|
||||
apk add --no-cache -q curl jq
|
||||
|
||||
CURRENT_PAT=$(cat /etc/gitlab-pat/token)
|
||||
GITLAB_URL="http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181"
|
||||
|
||||
TOKEN_INFO=$(curl -sf "${GITLAB_URL}/api/v4/personal_access_tokens/self" \
|
||||
-H "PRIVATE-TOKEN: ${CURRENT_PAT}") || {
|
||||
echo "CRITICAL: PAT is INVALID. CI will fail!"
|
||||
echo "Create a new PAT in GitLab UI and run:"
|
||||
echo " kubectl -n foxhunt create secret generic gitlab-pat --from-literal=token=<new-pat> --dry-run=client -o yaml | kubectl apply -f -"
|
||||
exit 1
|
||||
}
|
||||
|
||||
EXPIRY=$(echo "$TOKEN_INFO" | jq -r '.expires_at')
|
||||
TOKEN_NAME=$(echo "$TOKEN_INFO" | jq -r '.name')
|
||||
SCOPES=$(echo "$TOKEN_INFO" | jq -r '.scopes | join(",")')
|
||||
|
||||
# Calculate days until expiry
|
||||
EXPIRY_EPOCH=$(date -d "$EXPIRY" +%s)
|
||||
NOW_EPOCH=$(date +%s)
|
||||
DAYS_LEFT=$(( (EXPIRY_EPOCH - NOW_EPOCH) / 86400 ))
|
||||
|
||||
echo "PAT '${TOKEN_NAME}': expires=${EXPIRY} scopes=${SCOPES} days_left=${DAYS_LEFT}"
|
||||
|
||||
if [ "$DAYS_LEFT" -lt 0 ]; then
|
||||
echo "CRITICAL: PAT has EXPIRED ${DAYS_LEFT} days ago!"
|
||||
exit 1
|
||||
elif [ "$DAYS_LEFT" -lt 14 ]; then
|
||||
echo "WARNING: PAT expires in ${DAYS_LEFT} days. Rotation may have failed."
|
||||
echo "Manual rotation: kubectl -n foxhunt create job pat-rotate-manual --from=cronjob/gitlab-pat-rotation"
|
||||
exit 1
|
||||
else
|
||||
echo "OK: PAT valid for ${DAYS_LEFT} more days."
|
||||
fi
|
||||
volumeMounts:
|
||||
- name: gitlab-pat
|
||||
mountPath: /etc/gitlab-pat
|
||||
readOnly: true
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 32Mi
|
||||
limits:
|
||||
cpu: 100m
|
||||
memory: 64Mi
|
||||
volumes:
|
||||
- name: gitlab-pat
|
||||
secret:
|
||||
secretName: gitlab-pat
|
||||
|
||||
@@ -10,49 +10,35 @@ inputs = {
|
||||
cluster_name = "foxhunt"
|
||||
k8s_version = "1.34"
|
||||
|
||||
# Foxhunt app services pool (api-gateway, trading-service, etc.)
|
||||
# Renamed from "services" — single DEV1-L, dedicated to microservices
|
||||
foxhunt_type = "DEV1-L"
|
||||
foxhunt_max_size = 1
|
||||
# Platform pool — runs everything: app services, databases, GitLab, monitoring
|
||||
platform_type = "DEV1-L"
|
||||
platform_max_size = 3
|
||||
|
||||
# Platform infrastructure pool (postgres, redis, minio, questdb, monitoring)
|
||||
enable_platform_pool = true
|
||||
platform_type = "DEV1-L"
|
||||
|
||||
# GitLab CE pool (dedicated for GitLab + Prometheus + Runner manager)
|
||||
enable_gitlab_pool = true
|
||||
gitlab_type = "GP1-XS"
|
||||
|
||||
# CPU compile pool (POP2-HC-32C-64G for cargo check/build — 32 vCPU, 64GB RAM, high clock, no GPU)
|
||||
# CPU compile pool (POP2-HC-32C-64G — 32 vCPU, 64GB RAM, high clock)
|
||||
enable_ci_compile_cpu_pool = true
|
||||
ci_compile_cpu_type = "POP2-HC-32C-64G"
|
||||
ci_compile_cpu_max_size = 4
|
||||
|
||||
# L40S training pool (48GB VRAM, CUDA CC 89 — supervised hyperopt)
|
||||
# L40S training pool (48GB VRAM, CUDA CC 89)
|
||||
enable_ci_training_l40s_pool = true
|
||||
ci_training_l40s_type = "L40S-1-48G"
|
||||
ci_training_l40s_max_size = 1
|
||||
|
||||
# H100 training pool (80GB VRAM, CUDA CC 90 — RL hyperopt)
|
||||
# H100 training pool (80GB VRAM, CUDA CC 90)
|
||||
enable_ci_training_h100_pool = true
|
||||
ci_training_h100_type = "H100-1-80G"
|
||||
ci_training_h100_max_size = 1
|
||||
|
||||
# H100x2 training pool (2× 80GB — multi-GPU parallel hyperopt)
|
||||
# H100x2 training pool (2x 80GB — multi-GPU parallel hyperopt)
|
||||
enable_ci_training_h100x2_pool = true
|
||||
ci_training_h100x2_type = "H100-2-80G"
|
||||
ci_training_h100x2_max_size = 1
|
||||
|
||||
# H100-SXM training pool (2× 80GB SXM, NVLink — future multi-GPU)
|
||||
# H100-SXM training pool (2x 80GB SXM, NVLink)
|
||||
enable_ci_training_h100_sxm_pool = true
|
||||
ci_training_h100_sxm_type = "H100-SXM-2-80G"
|
||||
ci_training_h100_sxm_max_size = 1
|
||||
|
||||
# H100-SXM-8 training pool (8× 80GB SXM, NVLink — full multi-GPU saturation)
|
||||
enable_ci_training_h100_sxm8_pool = true
|
||||
ci_training_h100_sxm8_type = "H100-SXM-8-80G"
|
||||
ci_training_h100_sxm8_max_size = 1
|
||||
|
||||
# NAT gateway handles egress — no public IPs on nodes
|
||||
public_ip_disabled = true
|
||||
}
|
||||
|
||||
@@ -20,6 +20,8 @@ remote_state {
|
||||
address = "${get_env("GITLAB_TF_STATE_URL", "https://tf.fxhnt.ai")}/api/v4/projects/1/terraform/state/${path_relative_to_include()}"
|
||||
lock_address = "${get_env("GITLAB_TF_STATE_URL", "https://tf.fxhnt.ai")}/api/v4/projects/1/terraform/state/${path_relative_to_include()}/lock"
|
||||
unlock_address = "${get_env("GITLAB_TF_STATE_URL", "https://tf.fxhnt.ai")}/api/v4/projects/1/terraform/state/${path_relative_to_include()}/lock"
|
||||
lock_method = "POST"
|
||||
unlock_method = "DELETE"
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -26,16 +26,14 @@ resource "scaleway_k8s_cluster" "foxhunt" {
|
||||
}
|
||||
}
|
||||
|
||||
# Foxhunt app services pool (api-gateway, trading-service, etc.)
|
||||
# Renamed from "services" — was 2× DEV1-L shared with databases,
|
||||
# now 1× DEV1-L dedicated to foxhunt microservices only.
|
||||
resource "scaleway_k8s_pool" "foxhunt" {
|
||||
# Platform pool — runs everything: app services, databases, GitLab, monitoring
|
||||
resource "scaleway_k8s_pool" "platform" {
|
||||
cluster_id = scaleway_k8s_cluster.foxhunt.id
|
||||
name = "foxhunt"
|
||||
node_type = var.foxhunt_type
|
||||
name = "platform"
|
||||
node_type = var.platform_type
|
||||
size = 1
|
||||
min_size = 1
|
||||
max_size = var.foxhunt_max_size
|
||||
max_size = var.platform_max_size
|
||||
autoscaling = true
|
||||
autohealing = true
|
||||
public_ip_disabled = var.public_ip_disabled
|
||||
@@ -46,38 +44,7 @@ resource "scaleway_k8s_pool" "foxhunt" {
|
||||
}
|
||||
}
|
||||
|
||||
# Platform infrastructure pool (postgres, redis, minio, questdb, monitoring)
|
||||
resource "scaleway_k8s_pool" "platform" {
|
||||
count = var.enable_platform_pool ? 1 : 0
|
||||
cluster_id = scaleway_k8s_cluster.foxhunt.id
|
||||
name = "platform"
|
||||
node_type = var.platform_type
|
||||
size = 1
|
||||
min_size = 1
|
||||
max_size = 1
|
||||
autoscaling = false
|
||||
autohealing = true
|
||||
public_ip_disabled = var.public_ip_disabled
|
||||
region = var.region
|
||||
}
|
||||
|
||||
# GitLab CE node pool (dedicated GP1-XS for GitLab + Runner manager)
|
||||
resource "scaleway_k8s_pool" "gitlab" {
|
||||
count = var.enable_gitlab_pool ? 1 : 0
|
||||
cluster_id = scaleway_k8s_cluster.foxhunt.id
|
||||
name = "gitlab"
|
||||
node_type = var.gitlab_type
|
||||
size = 1
|
||||
min_size = 1
|
||||
max_size = 1
|
||||
autoscaling = false
|
||||
autohealing = true
|
||||
public_ip_disabled = var.public_ip_disabled
|
||||
region = var.region
|
||||
}
|
||||
|
||||
# L40S training pool (48GB VRAM, CUDA CC 89 — hyperopt + supervised training)
|
||||
# Nodes auto-labeled: k8s.scaleway.com/pool-name=ci-training-l40s
|
||||
resource "scaleway_k8s_pool" "ci_training_l40s" {
|
||||
count = var.enable_ci_training_l40s_pool ? 1 : 0
|
||||
cluster_id = scaleway_k8s_cluster.foxhunt.id
|
||||
@@ -96,8 +63,7 @@ resource "scaleway_k8s_pool" "ci_training_l40s" {
|
||||
}
|
||||
}
|
||||
|
||||
# H100 training pool (80GB VRAM, CUDA CC 90 — RL hyperopt + large model training)
|
||||
# Nodes auto-labeled: k8s.scaleway.com/pool-name=ci-training-h100
|
||||
# H100 training pool (80GB VRAM, CUDA CC 90 — RL hyperopt)
|
||||
resource "scaleway_k8s_pool" "ci_training_h100" {
|
||||
count = var.enable_ci_training_h100_pool ? 1 : 0
|
||||
cluster_id = scaleway_k8s_cluster.foxhunt.id
|
||||
@@ -116,8 +82,7 @@ resource "scaleway_k8s_pool" "ci_training_h100" {
|
||||
}
|
||||
}
|
||||
|
||||
# H100x2 training pool (2× 80GB VRAM — multi-GPU parallel hyperopt)
|
||||
# Nodes auto-labeled: k8s.scaleway.com/pool-name=ci-training-h100x2
|
||||
# H100x2 training pool (2x 80GB VRAM — multi-GPU parallel hyperopt)
|
||||
resource "scaleway_k8s_pool" "ci_training_h100x2" {
|
||||
count = var.enable_ci_training_h100x2_pool ? 1 : 0
|
||||
cluster_id = scaleway_k8s_cluster.foxhunt.id
|
||||
@@ -136,8 +101,7 @@ resource "scaleway_k8s_pool" "ci_training_h100x2" {
|
||||
}
|
||||
}
|
||||
|
||||
# H100-SXM training pool (2× 80GB SXM VRAM — NVLink multi-GPU)
|
||||
# Nodes auto-labeled: k8s.scaleway.com/pool-name=ci-training-h100-sxm
|
||||
# H100-SXM training pool (2x 80GB SXM VRAM — NVLink multi-GPU)
|
||||
resource "scaleway_k8s_pool" "ci_training_h100_sxm" {
|
||||
count = var.enable_ci_training_h100_sxm_pool ? 1 : 0
|
||||
cluster_id = scaleway_k8s_cluster.foxhunt.id
|
||||
@@ -156,28 +120,7 @@ resource "scaleway_k8s_pool" "ci_training_h100_sxm" {
|
||||
}
|
||||
}
|
||||
|
||||
# H100-SXM-8 training pool (8× 80GB SXM VRAM, NVLink — full multi-GPU saturation)
|
||||
# Nodes auto-labeled: k8s.scaleway.com/pool-name=ci-training-h100-sxm8
|
||||
resource "scaleway_k8s_pool" "ci_training_h100_sxm8" {
|
||||
count = var.enable_ci_training_h100_sxm8_pool ? 1 : 0
|
||||
cluster_id = scaleway_k8s_cluster.foxhunt.id
|
||||
name = "ci-training-h100-sxm8"
|
||||
node_type = var.ci_training_h100_sxm8_type
|
||||
size = 1
|
||||
min_size = 0
|
||||
max_size = var.ci_training_h100_sxm8_max_size
|
||||
autoscaling = true
|
||||
autohealing = true
|
||||
public_ip_disabled = var.public_ip_disabled
|
||||
region = var.region
|
||||
|
||||
lifecycle {
|
||||
ignore_changes = [size]
|
||||
}
|
||||
}
|
||||
|
||||
# CPU compile pool (POP2 — high CPU, no GPU, for cargo check/build)
|
||||
# Nodes auto-labeled: k8s.scaleway.com/pool-name=ci-compile-cpu
|
||||
# CPU compile pool (POP2-HC — high CPU, no GPU, for cargo check/build)
|
||||
resource "scaleway_k8s_pool" "ci_compile_cpu" {
|
||||
count = var.enable_ci_compile_cpu_pool ? 1 : 0
|
||||
cluster_id = scaleway_k8s_cluster.foxhunt.id
|
||||
|
||||
@@ -14,23 +14,13 @@ output "apiserver_url" {
|
||||
value = scaleway_k8s_cluster.foxhunt.apiserver_url
|
||||
}
|
||||
|
||||
output "foxhunt_pool_id" {
|
||||
description = "ID of the foxhunt app services node pool"
|
||||
value = scaleway_k8s_pool.foxhunt.id
|
||||
}
|
||||
|
||||
output "platform_pool_id" {
|
||||
description = "ID of the platform infrastructure node pool"
|
||||
value = var.enable_platform_pool ? scaleway_k8s_pool.platform[0].id : ""
|
||||
}
|
||||
|
||||
output "gitlab_pool_id" {
|
||||
description = "ID of the GitLab node pool"
|
||||
value = var.enable_gitlab_pool ? scaleway_k8s_pool.gitlab[0].id : ""
|
||||
description = "ID of the platform node pool"
|
||||
value = scaleway_k8s_pool.platform.id
|
||||
}
|
||||
|
||||
output "ci_compile_cpu_pool_id" {
|
||||
description = "ID of the CPU compile node pool (POP2-HC)"
|
||||
description = "ID of the CPU compile node pool"
|
||||
value = var.enable_ci_compile_cpu_pool ? scaleway_k8s_pool.ci_compile_cpu[0].id : ""
|
||||
}
|
||||
|
||||
@@ -54,15 +44,7 @@ output "ci_training_h100_sxm_pool_id" {
|
||||
value = var.enable_ci_training_h100_sxm_pool ? scaleway_k8s_pool.ci_training_h100_sxm[0].id : ""
|
||||
}
|
||||
|
||||
output "ci_training_h100_sxm8_pool_id" {
|
||||
description = "ID of the H100-SXM-8 training node pool (8× 80GB)"
|
||||
value = var.enable_ci_training_h100_sxm8_pool ? scaleway_k8s_pool.ci_training_h100_sxm8[0].id : ""
|
||||
}
|
||||
|
||||
# Dev pool removed — DevPod runs on platform pool
|
||||
|
||||
output "private_network_id" {
|
||||
description = "ID of the VPC private network the cluster is attached to"
|
||||
value = scaleway_vpc_private_network.foxhunt.id
|
||||
}
|
||||
|
||||
|
||||
@@ -6,53 +6,25 @@ variable "region" {
|
||||
variable "cluster_name" {
|
||||
description = "Name of the Kubernetes cluster"
|
||||
type = string
|
||||
default = "foxhunt"
|
||||
}
|
||||
|
||||
variable "k8s_version" {
|
||||
description = "Kubernetes version for the Kapsule cluster"
|
||||
type = string
|
||||
default = "1.34"
|
||||
}
|
||||
|
||||
variable "foxhunt_type" {
|
||||
description = "Instance type for the foxhunt app services node pool"
|
||||
type = string
|
||||
default = "DEV1-L"
|
||||
}
|
||||
|
||||
variable "foxhunt_max_size" {
|
||||
description = "Maximum number of nodes in the foxhunt pool (autoscaling)"
|
||||
type = number
|
||||
default = 1
|
||||
}
|
||||
|
||||
variable "enable_platform_pool" {
|
||||
description = "Create dedicated node pool for platform infrastructure (databases, monitoring)"
|
||||
type = bool
|
||||
default = false
|
||||
}
|
||||
|
||||
variable "platform_type" {
|
||||
description = "Instance type for the platform infrastructure node pool"
|
||||
description = "Instance type for the platform node pool"
|
||||
type = string
|
||||
default = "DEV1-L"
|
||||
}
|
||||
|
||||
variable "enable_gitlab_pool" {
|
||||
description = "Create dedicated node pool for GitLab CE"
|
||||
type = bool
|
||||
default = false
|
||||
}
|
||||
|
||||
variable "gitlab_type" {
|
||||
description = "Instance type for the GitLab node pool"
|
||||
type = string
|
||||
default = "DEV1-L"
|
||||
variable "platform_max_size" {
|
||||
description = "Maximum number of nodes in the platform pool"
|
||||
type = number
|
||||
}
|
||||
|
||||
variable "enable_ci_training_l40s_pool" {
|
||||
description = "Create L40S GPU node pool for training + hyperopt (48GB VRAM, CC 89)"
|
||||
description = "Create L40S GPU node pool for training + hyperopt"
|
||||
type = bool
|
||||
default = false
|
||||
}
|
||||
@@ -70,7 +42,7 @@ variable "ci_training_l40s_max_size" {
|
||||
}
|
||||
|
||||
variable "enable_ci_training_h100_pool" {
|
||||
description = "Create H100 GPU node pool for training + hyperopt (80GB VRAM, CC 90)"
|
||||
description = "Create H100 GPU node pool for training + hyperopt"
|
||||
type = bool
|
||||
default = false
|
||||
}
|
||||
@@ -88,7 +60,7 @@ variable "ci_training_h100_max_size" {
|
||||
}
|
||||
|
||||
variable "enable_ci_training_h100x2_pool" {
|
||||
description = "Create H100x2 GPU node pool for multi-GPU training (2× 80GB VRAM)"
|
||||
description = "Create H100x2 GPU node pool for multi-GPU training"
|
||||
type = bool
|
||||
default = false
|
||||
}
|
||||
@@ -106,7 +78,7 @@ variable "ci_training_h100x2_max_size" {
|
||||
}
|
||||
|
||||
variable "enable_ci_training_h100_sxm_pool" {
|
||||
description = "Create H100-SXM GPU node pool for NVLink multi-GPU training (2× 80GB SXM)"
|
||||
description = "Create H100-SXM GPU node pool for NVLink multi-GPU training"
|
||||
type = bool
|
||||
default = false
|
||||
}
|
||||
@@ -123,34 +95,15 @@ variable "ci_training_h100_sxm_max_size" {
|
||||
default = 1
|
||||
}
|
||||
|
||||
variable "enable_ci_training_h100_sxm8_pool" {
|
||||
description = "Create H100-SXM-8 GPU node pool for full multi-GPU training (8× 80GB SXM, NVLink)"
|
||||
type = bool
|
||||
default = false
|
||||
}
|
||||
|
||||
variable "ci_training_h100_sxm8_type" {
|
||||
description = "Instance type for the H100-SXM-8 training pool"
|
||||
type = string
|
||||
default = "H100-SXM-8-80G"
|
||||
}
|
||||
|
||||
variable "ci_training_h100_sxm8_max_size" {
|
||||
description = "Maximum number of nodes in the H100-SXM-8 training pool"
|
||||
type = number
|
||||
default = 1
|
||||
}
|
||||
|
||||
variable "enable_ci_compile_cpu_pool" {
|
||||
description = "Create high-CPU node pool for CI compilation (no GPU needed)"
|
||||
description = "Create high-CPU node pool for CI compilation"
|
||||
type = bool
|
||||
default = false
|
||||
}
|
||||
|
||||
variable "ci_compile_cpu_type" {
|
||||
description = "Instance type for the CPU compile pool (32 vCPU, 64GB RAM, high clock, no GPU)"
|
||||
description = "Instance type for the CPU compile pool"
|
||||
type = string
|
||||
default = "POP2-HC-32C-64G"
|
||||
}
|
||||
|
||||
variable "ci_compile_cpu_max_size" {
|
||||
@@ -164,4 +117,3 @@ variable "public_ip_disabled" {
|
||||
type = bool
|
||||
default = false
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user