10d81b0703
Remove orphaned `ci` pool (GP1-XS, 4 vCPU wasted — nothing scheduled to it). Remove `ci-build` pool (GP1-M) — builds now run on gpu-training (H100-1-80G: 24 vCPU, 240GB, real CUDA). This eliminates the need for CUDA stubs, separate test-gpu jobs, and ml crate exclusions. Pool layout after: always-on DEV1-M (core services, always on) gitlab GP1-XS (GitLab CE + runner manager, always on) gpu-training H100-1-80G (CI builds + ML training, scale-to-zero) gpu-inference L4-1-24G (trading inference, scale-to-zero) Build pod limits bumped to 16 vCPU / 64GB (from 6/12GB) to use H100 capacity. Runner now has `gpu` tag — all tests including ml crate run in single job. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
131 lines
4.5 KiB
Bash
Executable File
131 lines
4.5 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# harden.sh - Post-provisioning security hardening for Foxhunt infra
|
|
# ---------------------------------------------------------------------------
|
|
# Run after terragrunt apply to lock down API access, verify bucket ACLs,
|
|
# and validate no services are publicly exposed.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
CLUSTER_ID="${SCW_KAPSULE_CLUSTER_ID:?Set SCW_KAPSULE_CLUSTER_ID}"
|
|
SCW_REGION="${SCW_REGION:-fr-par}"
|
|
ADMIN_IPV4="${ADMIN_IPV4:-}"
|
|
ADMIN_IPV6="${ADMIN_IPV6:-}"
|
|
|
|
echo "=== Foxhunt Security Hardening ==="
|
|
echo "Cluster: ${CLUSTER_ID}"
|
|
echo "Region: ${SCW_REGION}"
|
|
echo ""
|
|
|
|
# --- 1. K8s API ACL (no TF resource, must use CLI) -----------------------
|
|
echo "--- K8s API ACL ---"
|
|
|
|
ACLS="acls.0.ip=100.64.0.0/10 acls.0.description=tailscale-cgnat"
|
|
IDX=1
|
|
|
|
if [[ -n "$ADMIN_IPV4" ]]; then
|
|
ACLS="$ACLS acls.${IDX}.ip=${ADMIN_IPV4}/32 acls.${IDX}.description=admin-v4"
|
|
(( IDX++ ))
|
|
fi
|
|
|
|
if [[ -n "$ADMIN_IPV6" ]]; then
|
|
ACLS="$ACLS acls.${IDX}.ip=${ADMIN_IPV6}/128 acls.${IDX}.description=admin-v6"
|
|
(( IDX++ ))
|
|
fi
|
|
|
|
# shellcheck disable=SC2086
|
|
scw k8s acl set "cluster-id=${CLUSTER_ID}" "region=${SCW_REGION}" $ACLS
|
|
echo " K8s API restricted to Tailscale + admin IPs"
|
|
|
|
# --- 2. Verify no LoadBalancer/NodePort services -------------------------
|
|
echo ""
|
|
echo "--- Exposed Services Check ---"
|
|
EXPOSED=$(KUBECONFIG="${KUBECONFIG}" kubectl get svc -A -o json 2>/dev/null \
|
|
| python3 -c "
|
|
import sys, json
|
|
data = json.load(sys.stdin)
|
|
for svc in data['items']:
|
|
t = svc['spec']['type']
|
|
if t in ('LoadBalancer', 'NodePort'):
|
|
print(f\" {svc['metadata']['namespace']}/{svc['metadata']['name']}: {t}\")
|
|
" 2>/dev/null || true)
|
|
|
|
if [[ -z "$EXPOSED" ]]; then
|
|
echo " No LoadBalancer or NodePort services found"
|
|
else
|
|
echo " WARNING: Exposed services found:"
|
|
echo "$EXPOSED"
|
|
fi
|
|
|
|
# --- 3. Verify no Ingress resources --------------------------------------
|
|
INGRESS=$(KUBECONFIG="${KUBECONFIG}" kubectl get ingress -A --no-headers 2>/dev/null || true)
|
|
if [[ -z "$INGRESS" ]]; then
|
|
echo " No Ingress resources found"
|
|
else
|
|
echo " WARNING: Ingress resources found:"
|
|
echo "$INGRESS"
|
|
fi
|
|
|
|
# --- 4. Verify container registry is private ----------------------------
|
|
echo ""
|
|
echo "--- Container Registry ---"
|
|
REG_PUBLIC=$(scw registry namespace list "region=${SCW_REGION}" -o json 2>/dev/null \
|
|
| python3 -c "
|
|
import sys, json
|
|
for ns in json.load(sys.stdin):
|
|
if ns.get('name') == 'foxhunt':
|
|
print('public' if ns.get('is_public') else 'private')
|
|
" 2>/dev/null || echo "unknown")
|
|
echo " foxhunt registry: ${REG_PUBLIC}"
|
|
|
|
# --- 5. Verify S3 buckets deny anonymous access -------------------------
|
|
echo ""
|
|
echo "--- S3 Bucket Anonymous Access ---"
|
|
for bucket_url in \
|
|
"https://foxhunt-artifacts.s3.fr-par.scw.cloud/" \
|
|
"https://foxhunt-tfstate.s3.nl-ams.scw.cloud/"; do
|
|
HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" "$bucket_url" 2>/dev/null || echo "000")
|
|
BUCKET=$(echo "$bucket_url" | sed 's|https://\(.*\)\.s3\..*|\1|')
|
|
if [[ "$HTTP_CODE" == "403" ]]; then
|
|
echo " ${BUCKET}: denied (HTTP ${HTTP_CODE})"
|
|
else
|
|
echo " WARNING: ${BUCKET}: HTTP ${HTTP_CODE} (expected 403)"
|
|
fi
|
|
done
|
|
|
|
# --- 6. Scale CI/GPU pools to 0 if idle ---------------------------------
|
|
echo ""
|
|
echo "--- Pool Scale-Down ---"
|
|
for pool_name in gpu-training gpu-inference; do
|
|
POOL_ID=$(scw k8s pool list "cluster-id=${CLUSTER_ID}" "region=${SCW_REGION}" -o json 2>/dev/null \
|
|
| python3 -c "
|
|
import sys, json
|
|
for p in json.load(sys.stdin):
|
|
if p.get('name') == '${pool_name}':
|
|
print(p['id'])
|
|
" 2>/dev/null || true)
|
|
|
|
if [[ -n "$POOL_ID" ]]; then
|
|
POOL_SIZE=$(scw k8s pool get "$POOL_ID" "region=${SCW_REGION}" -o json 2>/dev/null \
|
|
| python3 -c "import sys,json; print(json.load(sys.stdin).get('size',0))" 2>/dev/null || echo "?")
|
|
echo " ${pool_name} pool: size=${POOL_SIZE}"
|
|
if [[ "$POOL_SIZE" != "0" ]]; then
|
|
KUBECONFIG="${KUBECONFIG}" kubectl get pods -A --field-selector="spec.nodeName" -o json 2>/dev/null \
|
|
| python3 -c "
|
|
import sys, json
|
|
pods = json.load(sys.stdin)['items']
|
|
pool_pods = [p for p in pods if '${pool_name}' in p.get('spec',{}).get('nodeName','')]
|
|
user_pods = [p for p in pool_pods if p['metadata']['namespace'] not in ('kube-system','cilium-secrets')]
|
|
if not user_pods:
|
|
print(' No user pods — safe to scale down')
|
|
else:
|
|
print(f' {len(user_pods)} user pods running — keeping nodes')
|
|
" 2>/dev/null || true
|
|
fi
|
|
fi
|
|
done
|
|
|
|
echo ""
|
|
echo "=== Hardening complete ==="
|