infra: security hardening and autoscale-to-zero for CI/GPU pools
- K8s API ACL restricted to Tailscale CGNAT (100.64.0.0/10) + admin IPs - CI and GPU pools: min_size=0 with lifecycle ignore_changes on size so autoscaler manages actual node count without Terraform drift - Add harden.sh script for post-provisioning security lockdown (ACL setup, exposed service check, bucket ACL verify, registry visibility) - Update smoke-test.sh default region to fr-par Security audit results: - All services ClusterIP only (no LoadBalancer/NodePort) - No Ingress resources - Container registry: private - S3 buckets: HTTP 403 on anonymous access - Gitea: all ports closed on public IP (Tailscale-only via security group) - Kapsule nodes: no open ports on external IPs Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
@@ -43,24 +43,30 @@ resource "scaleway_k8s_pool" "ci" {
|
||||
name = "ci"
|
||||
node_type = var.ci_type
|
||||
size = 1
|
||||
min_size = 1
|
||||
min_size = 0
|
||||
max_size = var.ci_max_size
|
||||
autoscaling = true
|
||||
autohealing = true
|
||||
region = var.region
|
||||
|
||||
lifecycle {
|
||||
ignore_changes = [size]
|
||||
}
|
||||
}
|
||||
|
||||
# GPU pool requires fr-par-2 (no GPU types in nl-ams).
|
||||
# Provision separately via kapsule-gpu module in fr-par-2.
|
||||
resource "scaleway_k8s_pool" "gpu" {
|
||||
count = var.enable_gpu_pool ? 1 : 0
|
||||
cluster_id = scaleway_k8s_cluster.foxhunt.id
|
||||
name = "gpu"
|
||||
node_type = var.gpu_type
|
||||
size = 1
|
||||
min_size = 1
|
||||
min_size = 0
|
||||
max_size = var.gpu_max_size
|
||||
autoscaling = true
|
||||
autohealing = true
|
||||
region = var.region
|
||||
}
|
||||
|
||||
lifecycle {
|
||||
ignore_changes = [size]
|
||||
}
|
||||
}
|
||||
130
infra/scripts/harden.sh
Executable file
130
infra/scripts/harden.sh
Executable file
@@ -0,0 +1,130 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# harden.sh - Post-provisioning security hardening for Foxhunt infra
|
||||
# ---------------------------------------------------------------------------
|
||||
# Run after terragrunt apply to lock down API access, verify bucket ACLs,
|
||||
# and validate no services are publicly exposed.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
CLUSTER_ID="${SCW_KAPSULE_CLUSTER_ID:?Set SCW_KAPSULE_CLUSTER_ID}"
|
||||
SCW_REGION="${SCW_REGION:-fr-par}"
|
||||
ADMIN_IPV4="${ADMIN_IPV4:-}"
|
||||
ADMIN_IPV6="${ADMIN_IPV6:-}"
|
||||
|
||||
echo "=== Foxhunt Security Hardening ==="
|
||||
echo "Cluster: ${CLUSTER_ID}"
|
||||
echo "Region: ${SCW_REGION}"
|
||||
echo ""
|
||||
|
||||
# --- 1. K8s API ACL (no TF resource, must use CLI) -----------------------
|
||||
echo "--- K8s API ACL ---"
|
||||
|
||||
ACLS="acls.0.ip=100.64.0.0/10 acls.0.description=tailscale-cgnat"
|
||||
IDX=1
|
||||
|
||||
if [[ -n "$ADMIN_IPV4" ]]; then
|
||||
ACLS="$ACLS acls.${IDX}.ip=${ADMIN_IPV4}/32 acls.${IDX}.description=admin-v4"
|
||||
(( IDX++ ))
|
||||
fi
|
||||
|
||||
if [[ -n "$ADMIN_IPV6" ]]; then
|
||||
ACLS="$ACLS acls.${IDX}.ip=${ADMIN_IPV6}/128 acls.${IDX}.description=admin-v6"
|
||||
(( IDX++ ))
|
||||
fi
|
||||
|
||||
# shellcheck disable=SC2086
|
||||
scw k8s acl set "cluster-id=${CLUSTER_ID}" "region=${SCW_REGION}" $ACLS
|
||||
echo " K8s API restricted to Tailscale + admin IPs"
|
||||
|
||||
# --- 2. Verify no LoadBalancer/NodePort services -------------------------
|
||||
echo ""
|
||||
echo "--- Exposed Services Check ---"
|
||||
EXPOSED=$(KUBECONFIG="${KUBECONFIG}" kubectl get svc -A -o json 2>/dev/null \
|
||||
| python3 -c "
|
||||
import sys, json
|
||||
data = json.load(sys.stdin)
|
||||
for svc in data['items']:
|
||||
t = svc['spec']['type']
|
||||
if t in ('LoadBalancer', 'NodePort'):
|
||||
print(f\" {svc['metadata']['namespace']}/{svc['metadata']['name']}: {t}\")
|
||||
" 2>/dev/null || true)
|
||||
|
||||
if [[ -z "$EXPOSED" ]]; then
|
||||
echo " No LoadBalancer or NodePort services found"
|
||||
else
|
||||
echo " WARNING: Exposed services found:"
|
||||
echo "$EXPOSED"
|
||||
fi
|
||||
|
||||
# --- 3. Verify no Ingress resources --------------------------------------
|
||||
INGRESS=$(KUBECONFIG="${KUBECONFIG}" kubectl get ingress -A --no-headers 2>/dev/null || true)
|
||||
if [[ -z "$INGRESS" ]]; then
|
||||
echo " No Ingress resources found"
|
||||
else
|
||||
echo " WARNING: Ingress resources found:"
|
||||
echo "$INGRESS"
|
||||
fi
|
||||
|
||||
# --- 4. Verify container registry is private ----------------------------
|
||||
echo ""
|
||||
echo "--- Container Registry ---"
|
||||
REG_PUBLIC=$(scw registry namespace list "region=${SCW_REGION}" -o json 2>/dev/null \
|
||||
| python3 -c "
|
||||
import sys, json
|
||||
for ns in json.load(sys.stdin):
|
||||
if ns.get('name') == 'foxhunt':
|
||||
print('public' if ns.get('is_public') else 'private')
|
||||
" 2>/dev/null || echo "unknown")
|
||||
echo " foxhunt registry: ${REG_PUBLIC}"
|
||||
|
||||
# --- 5. Verify S3 buckets deny anonymous access -------------------------
|
||||
echo ""
|
||||
echo "--- S3 Bucket Anonymous Access ---"
|
||||
for bucket_url in \
|
||||
"https://foxhunt-artifacts.s3.fr-par.scw.cloud/" \
|
||||
"https://foxhunt-tfstate.s3.nl-ams.scw.cloud/"; do
|
||||
HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" "$bucket_url" 2>/dev/null || echo "000")
|
||||
BUCKET=$(echo "$bucket_url" | sed 's|https://\(.*\)\.s3\..*|\1|')
|
||||
if [[ "$HTTP_CODE" == "403" ]]; then
|
||||
echo " ${BUCKET}: denied (HTTP ${HTTP_CODE})"
|
||||
else
|
||||
echo " WARNING: ${BUCKET}: HTTP ${HTTP_CODE} (expected 403)"
|
||||
fi
|
||||
done
|
||||
|
||||
# --- 6. Scale CI/GPU pools to 0 if idle ---------------------------------
|
||||
echo ""
|
||||
echo "--- Pool Scale-Down ---"
|
||||
for pool_name in ci gpu; do
|
||||
POOL_ID=$(scw k8s pool list "cluster-id=${CLUSTER_ID}" "region=${SCW_REGION}" -o json 2>/dev/null \
|
||||
| python3 -c "
|
||||
import sys, json
|
||||
for p in json.load(sys.stdin):
|
||||
if p.get('name') == '${pool_name}':
|
||||
print(p['id'])
|
||||
" 2>/dev/null || true)
|
||||
|
||||
if [[ -n "$POOL_ID" ]]; then
|
||||
POOL_SIZE=$(scw k8s pool get "$POOL_ID" "region=${SCW_REGION}" -o json 2>/dev/null \
|
||||
| python3 -c "import sys,json; print(json.load(sys.stdin).get('size',0))" 2>/dev/null || echo "?")
|
||||
echo " ${pool_name} pool: size=${POOL_SIZE}"
|
||||
if [[ "$POOL_SIZE" != "0" ]]; then
|
||||
KUBECONFIG="${KUBECONFIG}" kubectl get pods -A --field-selector="spec.nodeName" -o json 2>/dev/null \
|
||||
| python3 -c "
|
||||
import sys, json
|
||||
pods = json.load(sys.stdin)['items']
|
||||
pool_pods = [p for p in pods if '${pool_name}' in p.get('spec',{}).get('nodeName','')]
|
||||
user_pods = [p for p in pool_pods if p['metadata']['namespace'] not in ('kube-system','cilium-secrets')]
|
||||
if not user_pods:
|
||||
print(' No user pods — safe to scale down')
|
||||
else:
|
||||
print(f' {len(user_pods)} user pods running — keeping nodes')
|
||||
" 2>/dev/null || true
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "=== Hardening complete ==="
|
||||
@@ -12,7 +12,7 @@ NAMESPACE="foxhunt"
|
||||
TAILSCALE_NS="tailscale"
|
||||
KAPSULE_NODE_NAME="foxhunt-kapsule"
|
||||
CLUSTER_ID="${SCW_KAPSULE_CLUSTER_ID:-}"
|
||||
SCW_REGION="${SCW_REGION:-nl-ams}"
|
||||
SCW_REGION="${SCW_REGION:-fr-par}"
|
||||
|
||||
GRPC_SERVICES=(
|
||||
trading-engine
|
||||
|
||||
Reference in New Issue
Block a user