From c6d0d2ab3840cba192e494c31b703d47f2c8cc49 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sun, 21 Jun 2026 18:06:55 +0200 Subject: [PATCH] =?UTF-8?q?chore(infra):=20decommission=20GitLab=20compute?= =?UTF-8?q?=20=E2=80=94=20keep=20data=20as=20cold=20backup=20(Phase=202C)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit helm uninstall gitlab + foxhunt-agent (~5.5GiB RAM / 12 pods freed). Removed :5050 registry block from the (now Gitea) tailscale proxy; deleted dead Argo events (gitlab-push ES, ci-pipeline-trigger sensor) + dead gitlab-pat cronjobs; removed dead GitLab manifests (values.yaml, pat-rotation.yaml, postgres-init.yaml). RETAINED (cold backup): gitaly PVC (50Gi), gitlab MinIO+Scaleway buckets, 25 gitlab-* secrets. KEPT: tailscale-proxy.yaml (serves Gitea + all *.fxhnt.ai), grafana-values.yaml (Phase 2E). Also fixed minio.fxhnt.ai (proxy admitted to minio ingress netpol). Co-Authored-By: Claude Opus 4.8 --- infra/k8s/gitlab/pat-rotation.yaml | 253 -------------------------- infra/k8s/gitlab/postgres-init.yaml | 63 ------- infra/k8s/gitlab/tailscale-proxy.yaml | 27 --- infra/k8s/gitlab/values.yaml | 206 --------------------- 4 files changed, 549 deletions(-) delete mode 100644 infra/k8s/gitlab/pat-rotation.yaml delete mode 100644 infra/k8s/gitlab/postgres-init.yaml delete mode 100644 infra/k8s/gitlab/values.yaml diff --git a/infra/k8s/gitlab/pat-rotation.yaml b/infra/k8s/gitlab/pat-rotation.yaml deleted file mode 100644 index 6cee96944..000000000 --- a/infra/k8s/gitlab/pat-rotation.yaml +++ /dev/null @@ -1,253 +0,0 @@ -# GitLab PAT auto-rotation CronJob -# Rotates the personal access token monthly, storing the new token in a K8s Secret. -# Uses GitLab's atomic rotate API: creates new token + revokes old in one call. -# -# The rotation runs on the 1st of every month. Additionally, a weekly expiry check -# runs every Monday and logs a warning if the token expires within 14 days, giving -# time to investigate rotation failures before CI breaks. -# -# Initial setup: -# kubectl -n foxhunt create secret generic gitlab-pat \ -# --from-literal=token=glpat- -# -# Manual rotation: -# kubectl -n foxhunt create job pat-rotate-manual --from=cronjob/gitlab-pat-rotation -# -# Manual expiry check: -# kubectl -n foxhunt create job pat-check-manual --from=cronjob/gitlab-pat-expiry-check -# -# Sync local token: -# export GITLAB_TOKEN=$(kubectl -n foxhunt get secret gitlab-pat -o jsonpath='{.data.token}' | base64 -d) ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: gitlab-pat-rotator - namespace: foxhunt ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: gitlab-pat-rotator - namespace: foxhunt -rules: - - apiGroups: [""] - resources: ["secrets"] - resourceNames: ["gitlab-pat"] - verbs: ["get", "patch"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: gitlab-pat-rotator - namespace: foxhunt -subjects: - - kind: ServiceAccount - name: gitlab-pat-rotator - namespace: foxhunt -roleRef: - kind: Role - name: gitlab-pat-rotator - apiGroup: rbac.authorization.k8s.io ---- -# Monthly rotation — creates new token, revokes old, updates K8s secret -apiVersion: batch/v1 -kind: CronJob -metadata: - name: gitlab-pat-rotation - namespace: foxhunt -spec: - schedule: "0 3 1 * *" # 1st of every month at 03:00 UTC - successfulJobsHistoryLimit: 3 - failedJobsHistoryLimit: 5 - concurrencyPolicy: Forbid - jobTemplate: - spec: - backoffLimit: 3 - activeDeadlineSeconds: 180 - template: - metadata: - labels: - app: gitlab-pat-rotation - spec: - serviceAccountName: gitlab-pat-rotator - restartPolicy: Never - nodeSelector: - k8s.scaleway.com/pool-name: platform - containers: - - name: rotate - image: alpine:3.21 - command: ["/bin/sh", "-c"] - args: - - | - set -e - - apk add --no-cache -q curl jq - - CURRENT_PAT=$(cat /etc/gitlab-pat/token) - GITLAB_URL="http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181" - - # 1. Verify current token is valid - TOKEN_INFO=$(curl -sf "${GITLAB_URL}/api/v4/personal_access_tokens/self" \ - -H "PRIVATE-TOKEN: ${CURRENT_PAT}") || { - echo "CRITICAL: Current PAT is invalid or GitLab unreachable." - echo "Manual recovery required: create a new PAT in GitLab UI and run:" - echo " kubectl -n foxhunt create secret generic gitlab-pat --from-literal=token= --dry-run=client -o yaml | kubectl apply -f -" - exit 1 - } - OLD_EXPIRY=$(echo "$TOKEN_INFO" | jq -r '.expires_at') - TOKEN_NAME=$(echo "$TOKEN_INFO" | jq -r '.name') - echo "Current PAT '${TOKEN_NAME}' expires: ${OLD_EXPIRY}" - - # 2. Rotate: new expiry = 1 year from now - EXPIRES=$(date -d "@$(($(date +%s) + 31536000))" +%Y-%m-%d) - RESPONSE=$(curl -sf -X POST \ - "${GITLAB_URL}/api/v4/personal_access_tokens/self/rotate" \ - -H "PRIVATE-TOKEN: ${CURRENT_PAT}" \ - -d "expires_at=${EXPIRES}") || { - echo "ERROR: Rotation API call failed. Token may still be valid." - exit 1 - } - - NEW_PAT=$(echo "$RESPONSE" | jq -r '.token') - if [ "$NEW_PAT" = "null" ] || [ -z "$NEW_PAT" ]; then - echo "ERROR: No token in rotation response: $RESPONSE" - exit 1 - fi - - echo "PAT rotated successfully. New expiry: ${EXPIRES}" - - # 3. Update K8s secret (old token is already revoked at this point) - K8S_TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token) - K8S_CA=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt - K8S_NS=$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace) - K8S_API="https://kubernetes.default.svc" - - NEW_PAT_B64=$(printf '%s' "$NEW_PAT" | base64) - - for attempt in 1 2 3 4 5; do - if curl -sf -X PATCH \ - "${K8S_API}/api/v1/namespaces/${K8S_NS}/secrets/gitlab-pat" \ - --cacert "$K8S_CA" \ - -H "Authorization: Bearer ${K8S_TOKEN}" \ - -H "Content-Type: application/strategic-merge-patch+json" \ - -d "{\"data\":{\"token\":\"${NEW_PAT_B64}\"}}"; then - echo "" - echo "K8s secret updated on attempt ${attempt}." - - # 4. Verify the new token works - VERIFY=$(curl -sf "${GITLAB_URL}/api/v4/personal_access_tokens/self" \ - -H "PRIVATE-TOKEN: ${NEW_PAT}" | jq -r '.expires_at') || true - if [ "$VERIFY" = "$EXPIRES" ]; then - echo "Verification passed: new token valid until ${VERIFY}" - else - echo "WARNING: Verification returned unexpected expiry: ${VERIFY}" - fi - exit 0 - fi - echo "Secret update attempt ${attempt} failed, retrying in 3s..." - sleep 3 - done - - # If we get here, rotation succeeded but secret update failed. - # Print token so it can be recovered from job logs. - echo "CRITICAL: Secret update failed after 5 attempts!" - echo "New PAT (recover manually): ${NEW_PAT}" - echo "Run: kubectl -n foxhunt create secret generic gitlab-pat --from-literal=token= --dry-run=client -o yaml | kubectl apply -f -" - exit 1 - volumeMounts: - - name: gitlab-pat - mountPath: /etc/gitlab-pat - readOnly: true - resources: - requests: - cpu: 50m - memory: 32Mi - limits: - cpu: 200m - memory: 64Mi - volumes: - - name: gitlab-pat - secret: - secretName: gitlab-pat ---- -# Weekly expiry check — warns if token expires within 14 days -apiVersion: batch/v1 -kind: CronJob -metadata: - name: gitlab-pat-expiry-check - namespace: foxhunt -spec: - schedule: "0 8 * * 1" # Every Monday at 08:00 UTC - successfulJobsHistoryLimit: 1 - failedJobsHistoryLimit: 3 - concurrencyPolicy: Forbid - jobTemplate: - spec: - backoffLimit: 1 - activeDeadlineSeconds: 60 - template: - metadata: - labels: - app: gitlab-pat-expiry-check - spec: - restartPolicy: Never - nodeSelector: - k8s.scaleway.com/pool-name: platform - containers: - - name: check - image: alpine:3.21 - command: ["/bin/sh", "-c"] - args: - - | - set -e - - apk add --no-cache -q curl jq - - CURRENT_PAT=$(cat /etc/gitlab-pat/token) - GITLAB_URL="http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181" - - TOKEN_INFO=$(curl -sf "${GITLAB_URL}/api/v4/personal_access_tokens/self" \ - -H "PRIVATE-TOKEN: ${CURRENT_PAT}") || { - echo "CRITICAL: PAT is INVALID. CI will fail!" - echo "Create a new PAT in GitLab UI and run:" - echo " kubectl -n foxhunt create secret generic gitlab-pat --from-literal=token= --dry-run=client -o yaml | kubectl apply -f -" - exit 1 - } - - EXPIRY=$(echo "$TOKEN_INFO" | jq -r '.expires_at') - TOKEN_NAME=$(echo "$TOKEN_INFO" | jq -r '.name') - SCOPES=$(echo "$TOKEN_INFO" | jq -r '.scopes | join(",")') - - # Calculate days until expiry - EXPIRY_EPOCH=$(date -d "$EXPIRY" +%s) - NOW_EPOCH=$(date +%s) - DAYS_LEFT=$(( (EXPIRY_EPOCH - NOW_EPOCH) / 86400 )) - - echo "PAT '${TOKEN_NAME}': expires=${EXPIRY} scopes=${SCOPES} days_left=${DAYS_LEFT}" - - if [ "$DAYS_LEFT" -lt 0 ]; then - echo "CRITICAL: PAT has EXPIRED ${DAYS_LEFT} days ago!" - exit 1 - elif [ "$DAYS_LEFT" -lt 14 ]; then - echo "WARNING: PAT expires in ${DAYS_LEFT} days. Rotation may have failed." - echo "Manual rotation: kubectl -n foxhunt create job pat-rotate-manual --from=cronjob/gitlab-pat-rotation" - exit 1 - else - echo "OK: PAT valid for ${DAYS_LEFT} more days." - fi - volumeMounts: - - name: gitlab-pat - mountPath: /etc/gitlab-pat - readOnly: true - resources: - requests: - cpu: 10m - memory: 32Mi - limits: - cpu: 100m - memory: 64Mi - volumes: - - name: gitlab-pat - secret: - secretName: gitlab-pat diff --git a/infra/k8s/gitlab/postgres-init.yaml b/infra/k8s/gitlab/postgres-init.yaml deleted file mode 100644 index 2532fc060..000000000 --- a/infra/k8s/gitlab/postgres-init.yaml +++ /dev/null @@ -1,63 +0,0 @@ -apiVersion: batch/v1 -kind: Job -metadata: - name: gitlab-postgres-init - namespace: foxhunt - labels: - app.kubernetes.io/name: gitlab-postgres-init - app.kubernetes.io/part-of: foxhunt -spec: - ttlSecondsAfterFinished: 300 - backoffLimit: 3 - template: - spec: - nodeSelector: - k8s.scaleway.com/pool-name: infra - restartPolicy: Never - containers: - - name: init - image: timescale/timescaledb:latest-pg16 - command: - - bash - - -c - - | - set -euo pipefail - export PGPASSWORD="$POSTGRES_PASSWORD" - - until pg_isready -h postgres -U foxhunt; do - echo "Waiting for postgres..." - sleep 2 - done - - echo "Creating gitlab user..." - psql -h postgres -U foxhunt -d foxhunt -tc \ - "SELECT 1 FROM pg_roles WHERE rolname='gitlab'" | grep -q 1 || \ - psql -h postgres -U foxhunt -d foxhunt -c \ - "CREATE ROLE gitlab WITH LOGIN PASSWORD '${GITLAB_DB_PASSWORD}'" - - echo "Creating gitlab database..." - psql -h postgres -U foxhunt -tc \ - "SELECT 1 FROM pg_database WHERE datname='gitlab'" | grep -q 1 || \ - psql -h postgres -U foxhunt -c \ - "CREATE DATABASE gitlab OWNER gitlab" - - echo "Creating extensions..." - psql -h postgres -U foxhunt -d gitlab -c "CREATE EXTENSION IF NOT EXISTS pg_trgm;" - psql -h postgres -U foxhunt -d gitlab -c "CREATE EXTENSION IF NOT EXISTS btree_gist;" - - echo "Granting permissions..." - psql -h postgres -U foxhunt -d gitlab -c "GRANT ALL PRIVILEGES ON DATABASE gitlab TO gitlab;" - psql -h postgres -U foxhunt -d gitlab -c "GRANT ALL ON SCHEMA public TO gitlab;" - - echo "GitLab database initialized successfully." - env: - - name: POSTGRES_PASSWORD - valueFrom: - secretKeyRef: - name: db-credentials - key: password - - name: GITLAB_DB_PASSWORD - valueFrom: - secretKeyRef: - name: gitlab-secrets - key: db-password diff --git a/infra/k8s/gitlab/tailscale-proxy.yaml b/infra/k8s/gitlab/tailscale-proxy.yaml index 2a202425d..ed9c4bb3b 100644 --- a/infra/k8s/gitlab/tailscale-proxy.yaml +++ b/infra/k8s/gitlab/tailscale-proxy.yaml @@ -58,7 +58,6 @@ spec: ports: - containerPort: 80 - containerPort: 443 - - containerPort: 5050 volumeMounts: - name: nginx-conf mountPath: /etc/nginx/conf.d @@ -179,32 +178,6 @@ data: } } - # Container Registry — git.fxhnt.ai:5050 - server { - listen 5050 ssl; - server_name git.fxhnt.ai; - - ssl_certificate /etc/nginx/certs/tls.crt; - ssl_certificate_key /etc/nginx/certs/tls.key; - ssl_protocols TLSv1.2 TLSv1.3; - - client_max_body_size 0; - - location / { - proxy_pass http://gitlab-registry.foxhunt.svc.cluster.local:5000; - proxy_set_header Host $http_host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto https; - - proxy_buffering off; - proxy_request_buffering off; - proxy_http_version 1.1; - proxy_read_timeout 3600s; - proxy_send_timeout 3600s; - } - } - # Grafana — grafana.fxhnt.ai server { listen 443 ssl; diff --git a/infra/k8s/gitlab/values.yaml b/infra/k8s/gitlab/values.yaml deleted file mode 100644 index e10db55c8..000000000 --- a/infra/k8s/gitlab/values.yaml +++ /dev/null @@ -1,206 +0,0 @@ -# GitLab CE - Foxhunt deployment -# Runs on 'infra' node pool alongside platform services -# External Postgres + Redis on same pool - -global: - edition: ce - kas: - enabled: false - hosts: - domain: fxhnt.ai - gitlab: - name: git.fxhnt.ai - https: true - registry: - name: git.fxhnt.ai - https: true - ingress: - enabled: false - configureCertmanager: false - shell: - port: 2222 - psql: - host: postgres.foxhunt.svc.cluster.local - port: 5432 - database: gitlab - username: gitlab - password: - secret: gitlab-secrets - key: db-password - redis: - host: redis.foxhunt.svc.cluster.local - port: 6379 - auth: - enabled: false - minio: - enabled: false - registry: - bucket: foxhunt-gitlab-registry - appConfig: - lfs: - bucket: foxhunt-gitlab-artifacts - connection: - secret: gitlab-s3-credentials - key: connection - artifacts: - bucket: foxhunt-gitlab-artifacts - connection: - secret: gitlab-s3-credentials - key: connection - uploads: - bucket: foxhunt-gitlab-artifacts - connection: - secret: gitlab-s3-credentials - key: connection - packages: - bucket: foxhunt-gitlab-artifacts - connection: - secret: gitlab-s3-credentials - key: connection - # Terraform/OpenTofu state object storage. MUST be present — without it the - # Terraform::StateUploader has no object store and every TF-state API call 403s - # ("Object Storage is not enabled for Terraform::StateUploader"). State files already - # live in foxhunt-gitlab-artifacts (6b/86///.tfstate). - terraformState: - enabled: true - bucket: foxhunt-gitlab-artifacts - connection: - secret: gitlab-s3-credentials - key: connection - gitlab_kas: - enabled: false - -# Disable bundled databases — use external -postgresql: - install: false -redis: - install: false - -# Disable components we don't need -installCertmanager: false -certmanager: - installCRDs: false -nginx-ingress: - enabled: false -prometheus: - install: true - rbac: - create: true - alertmanager: - enabled: false - server: - nodeSelector: - k8s.scaleway.com/pool-name: infra - persistentVolume: - enabled: false - resources: - requests: - cpu: 100m - memory: 512Mi - limits: - cpu: 500m - memory: 1Gi -gitlab-runner: - install: false - -# Registry — S3-backed, NodePort for kubelet image pulls -# Token auth disabled (cluster-internal only, secured by network policies) -# Pods pull via gitlab-registry.foxhunt.svc.cluster.local:5000 (internal DNS) -registry: - replicaCount: 1 - hpa: - minReplicas: 1 - maxReplicas: 1 - service: - type: NodePort - nodePort: 30500 - storage: - secret: gitlab-s3-credentials - key: registry - redirect: - disable: true - nodeSelector: - k8s.scaleway.com/pool-name: infra - -# GitLab core components — all on infra node pool -gitlab: - webservice: - replicaCount: 2 - hpa: - minReplicas: 2 - maxReplicas: 2 - workerProcesses: 2 - workhorse: - extraArgs: "-apiLimit 0 -apiQueueLimit 0" - nodeSelector: - k8s.scaleway.com/pool-name: infra - resources: - requests: - cpu: 500m - memory: 2Gi - limits: - cpu: 2000m - memory: 4Gi - extraEnv: - REDIS_URL: "redis://redis.foxhunt.svc.cluster.local:6379/8" - - sidekiq: - replicas: 1 - hpa: - minReplicas: 1 - maxReplicas: 1 - concurrency: 10 - nodeSelector: - k8s.scaleway.com/pool-name: infra - resources: - requests: - cpu: 250m - memory: 1Gi - limits: - cpu: 1000m - memory: 2Gi - extraEnv: - REDIS_URL: "redis://redis.foxhunt.svc.cluster.local:6379/8" - - gitaly: - persistence: - enabled: true - size: 50Gi - nodeSelector: - k8s.scaleway.com/pool-name: infra - resources: - requests: - cpu: 200m - memory: 512Mi - limits: - cpu: 1000m - memory: 2Gi - - gitlab-shell: - replicaCount: 1 - hpa: - minReplicas: 1 - maxReplicas: 1 - nodeSelector: - k8s.scaleway.com/pool-name: infra - service: - type: NodePort - nodePort: 32222 - - toolbox: - backups: - objectStorage: - config: - secret: gitlab-s3-credentials - key: connection - nodeSelector: - k8s.scaleway.com/pool-name: infra - - kas: - enabled: false - hpa: - minReplicas: 0 - maxReplicas: 0 - - gitlab-exporter: - enabled: false