1441f5fbbb
- K8s API ACL restricted to Tailscale CGNAT (100.64.0.0/10) + admin IPs - CI and GPU pools: min_size=0 with lifecycle ignore_changes on size so autoscaler manages actual node count without Terraform drift - Add harden.sh script for post-provisioning security lockdown (ACL setup, exposed service check, bucket ACL verify, registry visibility) - Update smoke-test.sh default region to fr-par Security audit results: - All services ClusterIP only (no LoadBalancer/NodePort) - No Ingress resources - Container registry: private - S3 buckets: HTTP 403 on anonymous access - Gitea: all ports closed on public IP (Tailscale-only via security group) - Kapsule nodes: no open ports on external IPs Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
148 lines
4.7 KiB
Bash
Executable File
148 lines
4.7 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# smoke-test.sh - End-to-end smoke test for the Foxhunt Kapsule cluster
|
|
# ---------------------------------------------------------------------------
|
|
# Runs a series of checks and prints OK/FAIL for each.
|
|
# Exits 0 if everything passes, 1 with failure count otherwise.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
NAMESPACE="foxhunt"
|
|
TAILSCALE_NS="tailscale"
|
|
KAPSULE_NODE_NAME="foxhunt-kapsule"
|
|
CLUSTER_ID="${SCW_KAPSULE_CLUSTER_ID:-}"
|
|
SCW_REGION="${SCW_REGION:-fr-par}"
|
|
|
|
GRPC_SERVICES=(
|
|
trading-engine
|
|
trading-agent
|
|
ml-training
|
|
risk-manager
|
|
data-acquisition
|
|
broker-gateway
|
|
)
|
|
|
|
PASS=0
|
|
FAIL=0
|
|
FAILURES=()
|
|
|
|
# --- Helpers --------------------------------------------------------------
|
|
|
|
check() {
|
|
local description="$1"
|
|
shift
|
|
if "$@" >/dev/null 2>&1; then
|
|
printf " [OK] %s\n" "$description"
|
|
(( PASS++ ))
|
|
else
|
|
printf " [FAIL] %s\n" "$description"
|
|
(( FAIL++ ))
|
|
FAILURES+=("$description")
|
|
fi
|
|
}
|
|
|
|
section() {
|
|
echo ""
|
|
echo "=== $1 ==="
|
|
}
|
|
|
|
# --- 1. Cluster -----------------------------------------------------------
|
|
section "Cluster"
|
|
|
|
check "kubectl can reach nodes" \
|
|
kubectl get nodes --no-headers
|
|
|
|
check "Namespace '${NAMESPACE}' exists" \
|
|
kubectl get namespace "$NAMESPACE"
|
|
|
|
# --- 2. Tailscale ---------------------------------------------------------
|
|
section "Tailscale"
|
|
|
|
check "Tailscale pod running in '${TAILSCALE_NS}' namespace" \
|
|
bash -c "kubectl -n ${TAILSCALE_NS} get pods --field-selector=status.phase=Running --no-headers 2>/dev/null | grep -q ."
|
|
|
|
check "Tailscale status shows '${KAPSULE_NODE_NAME}'" \
|
|
bash -c "tailscale status 2>/dev/null | grep -q '${KAPSULE_NODE_NAME}'"
|
|
|
|
# --- 3. Databases ---------------------------------------------------------
|
|
section "Databases"
|
|
|
|
check "PostgreSQL accepts connections" \
|
|
bash -c "
|
|
PG_POD=\$(kubectl -n ${NAMESPACE} get pod -l app=postgresql -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)
|
|
kubectl -n ${NAMESPACE} exec \"\$PG_POD\" -- pg_isready -U foxhunt
|
|
"
|
|
|
|
check "Redis responds to PING" \
|
|
bash -c "
|
|
REDIS_POD=\$(kubectl -n ${NAMESPACE} get pod -l app=redis -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)
|
|
kubectl -n ${NAMESPACE} exec \"\$REDIS_POD\" -- redis-cli ping | grep -q PONG
|
|
"
|
|
|
|
check "QuestDB HTTP health" \
|
|
bash -c "
|
|
QUEST_POD=\$(kubectl -n ${NAMESPACE} get pod -l app=questdb -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)
|
|
kubectl -n ${NAMESPACE} exec \"\$QUEST_POD\" -- curl -sf http://localhost:9000/
|
|
"
|
|
|
|
# --- 4. Services ----------------------------------------------------------
|
|
section "Services (gRPC + web-gateway)"
|
|
|
|
for svc in "${GRPC_SERVICES[@]}"; do
|
|
check "Pod '${svc}' is Running" \
|
|
bash -c "kubectl -n ${NAMESPACE} get pods -l app=${svc} --field-selector=status.phase=Running --no-headers 2>/dev/null | grep -q ."
|
|
done
|
|
|
|
check "Pod 'web-gateway' is Running" \
|
|
bash -c "kubectl -n ${NAMESPACE} get pods -l app=web-gateway --field-selector=status.phase=Running --no-headers 2>/dev/null | grep -q ."
|
|
|
|
# --- 5. Web gateway health ------------------------------------------------
|
|
section "Web Gateway Health"
|
|
|
|
check "web-gateway /health returns 200" \
|
|
bash -c "
|
|
CLUSTER_IP=\$(kubectl -n ${NAMESPACE} get svc web-gateway -o jsonpath='{.spec.clusterIP}' 2>/dev/null)
|
|
PORT=\$(kubectl -n ${NAMESPACE} get svc web-gateway -o jsonpath='{.spec.ports[0].port}' 2>/dev/null)
|
|
kubectl -n ${NAMESPACE} run smoke-curl --rm -i --restart=Never --image=curlimages/curl -- \
|
|
curl -sf http://\${CLUSTER_IP}:\${PORT}/health
|
|
"
|
|
|
|
# --- 6. Node pools -------------------------------------------------------
|
|
section "Node Pools"
|
|
|
|
check "Always-on node is Ready" \
|
|
bash -c "kubectl get nodes -l k8s.scaleway.com/pool-name=always-on --no-headers 2>/dev/null | grep -q ' Ready'"
|
|
|
|
check "CI node pool exists" \
|
|
bash -c "
|
|
[[ -z '${CLUSTER_ID}' ]] && exit 1
|
|
scw k8s pool list cluster-id=${CLUSTER_ID} region=${SCW_REGION} -o json 2>/dev/null | grep -q '\"ci\"'
|
|
"
|
|
|
|
check "GPU node pool exists" \
|
|
bash -c "
|
|
[[ -z '${CLUSTER_ID}' ]] && exit 1
|
|
scw k8s pool list cluster-id=${CLUSTER_ID} region=${SCW_REGION} -o json 2>/dev/null | grep -q '\"gpu\"'
|
|
"
|
|
|
|
# --- Summary --------------------------------------------------------------
|
|
echo ""
|
|
echo "======================================================================"
|
|
printf " Results: %d passed, %d failed\n" "$PASS" "$FAIL"
|
|
echo "======================================================================"
|
|
|
|
if [[ $FAIL -gt 0 ]]; then
|
|
echo ""
|
|
echo " Failed checks:"
|
|
for f in "${FAILURES[@]}"; do
|
|
echo " - ${f}"
|
|
done
|
|
echo ""
|
|
exit 1
|
|
fi
|
|
|
|
echo ""
|
|
echo " All checks passed."
|
|
exit 0
|