gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

chore(infra): decommission GitLab compute — keep data as cold backup (Phase 2C)

Browse Source 
helm uninstall gitlab + foxhunt-agent (~5.5GiB RAM / 12 pods freed). Removed :5050
registry block from the (now Gitea) tailscale proxy; deleted dead Argo events
(gitlab-push ES, ci-pipeline-trigger sensor) + dead gitlab-pat cronjobs; removed dead
GitLab manifests (values.yaml, pat-rotation.yaml, postgres-init.yaml).

RETAINED (cold backup): gitaly PVC (50Gi), gitlab MinIO+Scaleway buckets, 25 gitlab-* secrets.
KEPT: tailscale-proxy.yaml (serves Gitea + all *.fxhnt.ai), grafana-values.yaml (Phase 2E).
Also fixed minio.fxhnt.ai (proxy admitted to minio ingress netpol).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-06-21 18:06:55 +02:00
parent e86384ff99
commit c6d0d2ab38
4 changed files with 0 additions and 549 deletions
Download Patch File Download Diff File Expand all files Collapse all files

253
infra/k8s/gitlab/pat-rotation.yaml
View File

@@ -1,253 +0,0 @@
# GitLab PAT auto-rotation CronJob
# Rotates the personal access token monthly, storing the new token in a K8s Secret.
# Uses GitLab's atomic rotate API: creates new token + revokes old in one call.
#
# The rotation runs on the 1st of every month. Additionally, a weekly expiry check
# runs every Monday and logs a warning if the token expires within 14 days, giving
# time to investigate rotation failures before CI breaks.
#
# Initial setup:
# kubectl -n foxhunt create secret generic gitlab-pat \
# --from-literal=token=glpat-<initial-token>
#
# Manual rotation:
# kubectl -n foxhunt create job pat-rotate-manual --from=cronjob/gitlab-pat-rotation
#
# Manual expiry check:
# kubectl -n foxhunt create job pat-check-manual --from=cronjob/gitlab-pat-expiry-check
#
# Sync local token:
# export GITLAB_TOKEN=$(kubectl -n foxhunt get secret gitlab-pat -o jsonpath='{.data.token}' | base64 -d)
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: gitlab-pat-rotator
namespace: foxhunt
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: gitlab-pat-rotator
namespace: foxhunt
rules:
- apiGroups: [""]
resources: ["secrets"]
resourceNames: ["gitlab-pat"]
verbs: ["get", "patch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: gitlab-pat-rotator
namespace: foxhunt
subjects:
- kind: ServiceAccount
name: gitlab-pat-rotator
namespace: foxhunt
roleRef:
kind: Role
name: gitlab-pat-rotator
apiGroup: rbac.authorization.k8s.io
---
# Monthly rotation — creates new token, revokes old, updates K8s secret
apiVersion: batch/v1
kind: CronJob
metadata:
name: gitlab-pat-rotation
namespace: foxhunt
spec:
schedule: "0 3 1 * *" # 1st of every month at 03:00 UTC
successfulJobsHistoryLimit: 3
failedJobsHistoryLimit: 5
concurrencyPolicy: Forbid
jobTemplate:
spec:
backoffLimit: 3
activeDeadlineSeconds: 180
template:
metadata:
labels:
app: gitlab-pat-rotation
spec:
serviceAccountName: gitlab-pat-rotator
restartPolicy: Never
nodeSelector:
k8s.scaleway.com/pool-name: platform
containers:
- name: rotate
image: alpine:3.21
command: ["/bin/sh", "-c"]
args:
- |
set -e
apk add --no-cache -q curl jq
CURRENT_PAT=$(cat /etc/gitlab-pat/token)
GITLAB_URL="http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181"
# 1. Verify current token is valid
TOKEN_INFO=$(curl -sf "${GITLAB_URL}/api/v4/personal_access_tokens/self" \
-H "PRIVATE-TOKEN: ${CURRENT_PAT}") || {
echo "CRITICAL: Current PAT is invalid or GitLab unreachable."
echo "Manual recovery required: create a new PAT in GitLab UI and run:"
echo " kubectl -n foxhunt create secret generic gitlab-pat --from-literal=token=<new-pat> --dry-run=client -o yaml | kubectl apply -f -"
exit 1
}
OLD_EXPIRY=$(echo "$TOKEN_INFO" | jq -r '.expires_at')
TOKEN_NAME=$(echo "$TOKEN_INFO" | jq -r '.name')
echo "Current PAT '${TOKEN_NAME}' expires: ${OLD_EXPIRY}"
# 2. Rotate: new expiry = 1 year from now
EXPIRES=$(date -d "@$(($(date +%s) + 31536000))" +%Y-%m-%d)
RESPONSE=$(curl -sf -X POST \
"${GITLAB_URL}/api/v4/personal_access_tokens/self/rotate" \
-H "PRIVATE-TOKEN: ${CURRENT_PAT}" \
-d "expires_at=${EXPIRES}") || {
echo "ERROR: Rotation API call failed. Token may still be valid."
exit 1
}
NEW_PAT=$(echo "$RESPONSE" | jq -r '.token')
if [ "$NEW_PAT" = "null" ] || [ -z "$NEW_PAT" ]; then
echo "ERROR: No token in rotation response: $RESPONSE"
exit 1
fi
echo "PAT rotated successfully. New expiry: ${EXPIRES}"
# 3. Update K8s secret (old token is already revoked at this point)
K8S_TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
K8S_CA=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt
K8S_NS=$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace)
K8S_API="https://kubernetes.default.svc"
NEW_PAT_B64=$(printf '%s' "$NEW_PAT" | base64)
for attempt in 1 2 3 4 5; do
if curl -sf -X PATCH \
"${K8S_API}/api/v1/namespaces/${K8S_NS}/secrets/gitlab-pat" \
--cacert "$K8S_CA" \
-H "Authorization: Bearer ${K8S_TOKEN}" \
-H "Content-Type: application/strategic-merge-patch+json" \
-d "{\"data\":{\"token\":\"${NEW_PAT_B64}\"}}"; then
echo ""
echo "K8s secret updated on attempt ${attempt}."
# 4. Verify the new token works
VERIFY=$(curl -sf "${GITLAB_URL}/api/v4/personal_access_tokens/self" \
-H "PRIVATE-TOKEN: ${NEW_PAT}" | jq -r '.expires_at') || true
if [ "$VERIFY" = "$EXPIRES" ]; then
echo "Verification passed: new token valid until ${VERIFY}"
else
echo "WARNING: Verification returned unexpected expiry: ${VERIFY}"
fi
exit 0
fi
echo "Secret update attempt ${attempt} failed, retrying in 3s..."
sleep 3
done
# If we get here, rotation succeeded but secret update failed.
# Print token so it can be recovered from job logs.
echo "CRITICAL: Secret update failed after 5 attempts!"
echo "New PAT (recover manually): ${NEW_PAT}"
echo "Run: kubectl -n foxhunt create secret generic gitlab-pat --from-literal=token=<above> --dry-run=client -o yaml | kubectl apply -f -"
exit 1
volumeMounts:
- name: gitlab-pat
mountPath: /etc/gitlab-pat
readOnly: true
resources:
requests:
cpu: 50m
memory: 32Mi
limits:
cpu: 200m
memory: 64Mi
volumes:
- name: gitlab-pat
secret:
secretName: gitlab-pat
---
# Weekly expiry check — warns if token expires within 14 days
apiVersion: batch/v1
kind: CronJob
metadata:
name: gitlab-pat-expiry-check
namespace: foxhunt
spec:
schedule: "0 8 * * 1" # Every Monday at 08:00 UTC
successfulJobsHistoryLimit: 1
failedJobsHistoryLimit: 3
concurrencyPolicy: Forbid
jobTemplate:
spec:
backoffLimit: 1
activeDeadlineSeconds: 60
template:
metadata:
labels:
app: gitlab-pat-expiry-check
spec:
restartPolicy: Never
nodeSelector:
k8s.scaleway.com/pool-name: platform
containers:
- name: check
image: alpine:3.21
command: ["/bin/sh", "-c"]
args:
- |
set -e
apk add --no-cache -q curl jq
CURRENT_PAT=$(cat /etc/gitlab-pat/token)
GITLAB_URL="http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181"
TOKEN_INFO=$(curl -sf "${GITLAB_URL}/api/v4/personal_access_tokens/self" \
-H "PRIVATE-TOKEN: ${CURRENT_PAT}") || {
echo "CRITICAL: PAT is INVALID. CI will fail!"
echo "Create a new PAT in GitLab UI and run:"
echo " kubectl -n foxhunt create secret generic gitlab-pat --from-literal=token=<new-pat> --dry-run=client -o yaml | kubectl apply -f -"
exit 1
}
EXPIRY=$(echo "$TOKEN_INFO" | jq -r '.expires_at')
TOKEN_NAME=$(echo "$TOKEN_INFO" | jq -r '.name')
SCOPES=$(echo "$TOKEN_INFO" | jq -r '.scopes | join(",")')
# Calculate days until expiry
EXPIRY_EPOCH=$(date -d "$EXPIRY" +%s)
NOW_EPOCH=$(date +%s)
DAYS_LEFT=$(( (EXPIRY_EPOCH - NOW_EPOCH) / 86400 ))
echo "PAT '${TOKEN_NAME}': expires=${EXPIRY} scopes=${SCOPES} days_left=${DAYS_LEFT}"
if [ "$DAYS_LEFT" -lt 0 ]; then
echo "CRITICAL: PAT has EXPIRED ${DAYS_LEFT} days ago!"
exit 1
elif [ "$DAYS_LEFT" -lt 14 ]; then
echo "WARNING: PAT expires in ${DAYS_LEFT} days. Rotation may have failed."
echo "Manual rotation: kubectl -n foxhunt create job pat-rotate-manual --from=cronjob/gitlab-pat-rotation"
exit 1
else
echo "OK: PAT valid for ${DAYS_LEFT} more days."
fi
volumeMounts:
- name: gitlab-pat
mountPath: /etc/gitlab-pat
readOnly: true
resources:
requests:
cpu: 10m
memory: 32Mi
limits:
cpu: 100m
memory: 64Mi
volumes:
- name: gitlab-pat
secret:
secretName: gitlab-pat

63
infra/k8s/gitlab/postgres-init.yaml
View File

@@ -1,63 +0,0 @@
apiVersion: batch/v1
kind: Job
metadata:
name: gitlab-postgres-init
namespace: foxhunt
labels:
app.kubernetes.io/name: gitlab-postgres-init
app.kubernetes.io/part-of: foxhunt
spec:
ttlSecondsAfterFinished: 300
backoffLimit: 3
template:
spec:
nodeSelector:
k8s.scaleway.com/pool-name: infra
restartPolicy: Never
containers:
- name: init
image: timescale/timescaledb:latest-pg16
command:
- bash
- -c
- |
set -euo pipefail
export PGPASSWORD="$POSTGRES_PASSWORD"
until pg_isready -h postgres -U foxhunt; do
echo "Waiting for postgres..."
sleep 2
done
echo "Creating gitlab user..."
psql -h postgres -U foxhunt -d foxhunt -tc \
"SELECT 1 FROM pg_roles WHERE rolname='gitlab'" | grep -q 1 || \
psql -h postgres -U foxhunt -d foxhunt -c \
"CREATE ROLE gitlab WITH LOGIN PASSWORD '${GITLAB_DB_PASSWORD}'"
echo "Creating gitlab database..."
psql -h postgres -U foxhunt -tc \
"SELECT 1 FROM pg_database WHERE datname='gitlab'" | grep -q 1 || \
psql -h postgres -U foxhunt -c \
"CREATE DATABASE gitlab OWNER gitlab"
echo "Creating extensions..."
psql -h postgres -U foxhunt -d gitlab -c "CREATE EXTENSION IF NOT EXISTS pg_trgm;"
psql -h postgres -U foxhunt -d gitlab -c "CREATE EXTENSION IF NOT EXISTS btree_gist;"
echo "Granting permissions..."
psql -h postgres -U foxhunt -d gitlab -c "GRANT ALL PRIVILEGES ON DATABASE gitlab TO gitlab;"
psql -h postgres -U foxhunt -d gitlab -c "GRANT ALL ON SCHEMA public TO gitlab;"
echo "GitLab database initialized successfully."
env:
- name: POSTGRES_PASSWORD
valueFrom:
secretKeyRef:
name: db-credentials
key: password
- name: GITLAB_DB_PASSWORD
valueFrom:
secretKeyRef:
name: gitlab-secrets
key: db-password

27
infra/k8s/gitlab/tailscale-proxy.yaml
View File

@@ -58,7 +58,6 @@ spec:
ports:
- containerPort: 80
- containerPort: 443
- containerPort: 5050
volumeMounts:
- name: nginx-conf
mountPath: /etc/nginx/conf.d
@@ -179,32 +178,6 @@ data:
}
}
# Container Registry — git.fxhnt.ai:5050
server {
listen 5050 ssl;
server_name git.fxhnt.ai;
ssl_certificate /etc/nginx/certs/tls.crt;
ssl_certificate_key /etc/nginx/certs/tls.key;
ssl_protocols TLSv1.2 TLSv1.3;
client_max_body_size 0;
location / {
proxy_pass http://gitlab-registry.foxhunt.svc.cluster.local:5000;
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto https;
proxy_buffering off;
proxy_request_buffering off;
proxy_http_version 1.1;
proxy_read_timeout 3600s;
proxy_send_timeout 3600s;
}
}
# Grafana — grafana.fxhnt.ai
server {
listen 443 ssl;

206
infra/k8s/gitlab/values.yaml
View File

@@ -1,206 +0,0 @@
# GitLab CE - Foxhunt deployment
# Runs on 'infra' node pool alongside platform services
# External Postgres + Redis on same pool
global:
edition: ce
kas:
enabled: false
hosts:
domain: fxhnt.ai
gitlab:
name: git.fxhnt.ai
https: true
registry:
name: git.fxhnt.ai
https: true
ingress:
enabled: false
configureCertmanager: false
shell:
port: 2222
psql:
host: postgres.foxhunt.svc.cluster.local
port: 5432
database: gitlab
username: gitlab
password:
secret: gitlab-secrets
key: db-password
redis:
host: redis.foxhunt.svc.cluster.local
port: 6379
auth:
enabled: false
minio:
enabled: false
registry:
bucket: foxhunt-gitlab-registry
appConfig:
lfs:
bucket: foxhunt-gitlab-artifacts
connection:
secret: gitlab-s3-credentials
key: connection
artifacts:
bucket: foxhunt-gitlab-artifacts
connection:
secret: gitlab-s3-credentials
key: connection
uploads:
bucket: foxhunt-gitlab-artifacts
connection:
secret: gitlab-s3-credentials
key: connection
packages:
bucket: foxhunt-gitlab-artifacts
connection:
secret: gitlab-s3-credentials
key: connection
# Terraform/OpenTofu state object storage. MUST be present — without it the
# Terraform::StateUploader has no object store and every TF-state API call 403s
# ("Object Storage is not enabled for Terraform::StateUploader"). State files already
# live in foxhunt-gitlab-artifacts (6b/86/<sha256(project_id)>/<state>/<ver>.tfstate).
terraformState:
enabled: true
bucket: foxhunt-gitlab-artifacts
connection:
secret: gitlab-s3-credentials
key: connection
gitlab_kas:
enabled: false
# Disable bundled databases — use external
postgresql:
install: false
redis:
install: false
# Disable components we don't need
installCertmanager: false
certmanager:
installCRDs: false
nginx-ingress:
enabled: false
prometheus:
install: true
rbac:
create: true
alertmanager:
enabled: false
server:
nodeSelector:
k8s.scaleway.com/pool-name: infra
persistentVolume:
enabled: false
resources:
requests:
cpu: 100m
memory: 512Mi
limits:
cpu: 500m
memory: 1Gi
gitlab-runner:
install: false
# Registry — S3-backed, NodePort for kubelet image pulls
# Token auth disabled (cluster-internal only, secured by network policies)
# Pods pull via gitlab-registry.foxhunt.svc.cluster.local:5000 (internal DNS)
registry:
replicaCount: 1
hpa:
minReplicas: 1
maxReplicas: 1
service:
type: NodePort
nodePort: 30500
storage:
secret: gitlab-s3-credentials
key: registry
redirect:
disable: true
nodeSelector:
k8s.scaleway.com/pool-name: infra
# GitLab core components — all on infra node pool
gitlab:
webservice:
replicaCount: 2
hpa:
minReplicas: 2
maxReplicas: 2
workerProcesses: 2
workhorse:
extraArgs: "-apiLimit 0 -apiQueueLimit 0"
nodeSelector:
k8s.scaleway.com/pool-name: infra
resources:
requests:
cpu: 500m
memory: 2Gi
limits:
cpu: 2000m
memory: 4Gi
extraEnv:
REDIS_URL: "redis://redis.foxhunt.svc.cluster.local:6379/8"
sidekiq:
replicas: 1
hpa:
minReplicas: 1
maxReplicas: 1
concurrency: 10
nodeSelector:
k8s.scaleway.com/pool-name: infra
resources:
requests:
cpu: 250m
memory: 1Gi
limits:
cpu: 1000m
memory: 2Gi
extraEnv:
REDIS_URL: "redis://redis.foxhunt.svc.cluster.local:6379/8"
gitaly:
persistence:
enabled: true
size: 50Gi
nodeSelector:
k8s.scaleway.com/pool-name: infra
resources:
requests:
cpu: 200m
memory: 512Mi
limits:
cpu: 1000m
memory: 2Gi
gitlab-shell:
replicaCount: 1
hpa:
minReplicas: 1
maxReplicas: 1
nodeSelector:
k8s.scaleway.com/pool-name: infra
service:
type: NodePort
nodePort: 32222
toolbox:
backups:
objectStorage:
config:
secret: gitlab-s3-credentials
key: connection
nodeSelector:
k8s.scaleway.com/pool-name: infra
kas:
enabled: false
hpa:
minReplicas: 0
maxReplicas: 0
gitlab-exporter:
enabled: false