feat(infra): auto-rotate GitLab PAT monthly via K8s CronJob

Adds a CronJob that runs on the 1st of every month at 03:00 UTC:
1. Reads current PAT from K8s Secret (gitlab-pat)
2. Calls GitLab rotate API (atomic: creates new, revokes old)
3. Updates K8s Secret with new token (3 retries)
4. Prints token to logs as recovery fallback if update fails

RBAC scoped to only get/patch the gitlab-pat secret.
Runs on gitlab node pool (zero GPU cost).

Manual trigger: kubectl -n foxhunt create job <name> --from=cronjob/gitlab-pat-rotation

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

infra/k8s/gitlab/pat-rotation.yaml

@@ -0,0 +1,153 @@
+# GitLab PAT auto-rotation CronJob
+# Rotates the personal access token monthly, storing the new token in a K8s Secret.
+# Uses GitLab's atomic rotate API: creates new token + revokes old in one call.
+#
+# Initial setup:
+#   kubectl -n foxhunt create secret generic gitlab-pat \
+#     --from-literal=token=glpat-<initial-token>
+#
+# Manual trigger:
+#   kubectl -n foxhunt create job pat-rotate-manual --from=cronjob/gitlab-pat-rotation
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: gitlab-pat-rotator
+  namespace: foxhunt
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: gitlab-pat-rotator
+  namespace: foxhunt
+rules:
+  - apiGroups: [""]
+    resources: ["secrets"]
+    resourceNames: ["gitlab-pat"]
+    verbs: ["get", "patch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: gitlab-pat-rotator
+  namespace: foxhunt
+subjects:
+  - kind: ServiceAccount
+    name: gitlab-pat-rotator
+    namespace: foxhunt
+roleRef:
+  kind: Role
+  name: gitlab-pat-rotator
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: gitlab-pat-rotation
+  namespace: foxhunt
+spec:
+  schedule: "0 3 1 * *"  # 1st of every month at 03:00 UTC
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+  concurrencyPolicy: Forbid
+  jobTemplate:
+    spec:
+      backoffLimit: 2
+      activeDeadlineSeconds: 120
+      template:
+        metadata:
+          labels:
+            app: gitlab-pat-rotation
+        spec:
+          serviceAccountName: gitlab-pat-rotator
+          restartPolicy: Never
+          nodeSelector:
+            k8s.scaleway.com/pool-name: gitlab
+          tolerations:
+            - key: node.kubernetes.io/not-ready
+              operator: Exists
+              effect: NoSchedule
+          containers:
+            - name: rotate
+              image: alpine:3.21
+              command: ["/bin/sh", "-c"]
+              args:
+                - |
+                  set -e
+
+                  apk add --no-cache -q curl jq
+
+                  CURRENT_PAT=$(cat /etc/gitlab-pat/token)
+                  GITLAB_URL="http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181"
+
+                  # Verify current token is still valid
+                  TOKEN_INFO=$(curl -sf "${GITLAB_URL}/api/v4/personal_access_tokens/self" \
+                    -H "PRIVATE-TOKEN: ${CURRENT_PAT}") || {
+                    echo "ERROR: Current PAT is invalid or GitLab unreachable"
+                    exit 1
+                  }
+                  OLD_EXPIRY=$(echo "$TOKEN_INFO" | jq -r '.expires_at')
+                  echo "Current PAT expires: ${OLD_EXPIRY}"
+
+                  # Rotate: new expiry = 1 year from now
+                  EXPIRES=$(date -d "@$(($(date +%s) + 31536000))" +%Y-%m-%d)
+                  RESPONSE=$(curl -sf -X POST \
+                    "${GITLAB_URL}/api/v4/personal_access_tokens/self/rotate" \
+                    -H "PRIVATE-TOKEN: ${CURRENT_PAT}" \
+                    -d "expires_at=${EXPIRES}") || {
+                    echo "ERROR: Rotation API call failed"
+                    exit 1
+                  }
+
+                  NEW_PAT=$(echo "$RESPONSE" | jq -r '.token')
+                  if [ "$NEW_PAT" = "null" ] || [ -z "$NEW_PAT" ]; then
+                    echo "ERROR: No token in rotation response: $RESPONSE"
+                    exit 1
+                  fi
+
+                  echo "PAT rotated successfully. New expiry: ${EXPIRES}"
+
+                  # Update K8s secret via API (old token is already revoked at this point)
+                  K8S_TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
+                  K8S_CA=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+                  K8S_NS=$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace)
+                  K8S_API="https://kubernetes.default.svc"
+
+                  NEW_PAT_B64=$(printf '%s' "$NEW_PAT" | base64)
+
+                  # Retry secret update — losing the new token after rotation is catastrophic
+                  for attempt in 1 2 3; do
+                    if curl -sf -X PATCH \
+                      "${K8S_API}/api/v1/namespaces/${K8S_NS}/secrets/gitlab-pat" \
+                      --cacert "$K8S_CA" \
+                      -H "Authorization: Bearer ${K8S_TOKEN}" \
+                      -H "Content-Type: application/strategic-merge-patch+json" \
+                      -d "{\"data\":{\"token\":\"${NEW_PAT_B64}\"}}"; then
+                      echo ""
+                      echo "Secret updated on attempt ${attempt}."
+                      exit 0
+                    fi
+                    echo "Secret update attempt ${attempt} failed, retrying..."
+                    sleep 2
+                  done
+
+                  # If we get here, rotation succeeded but secret update failed.
+                  # Print token so it can be recovered from job logs.
+                  echo "CRITICAL: Secret update failed after 3 attempts!"
+                  echo "New PAT (recover manually): ${NEW_PAT}"
+                  exit 1
+              volumeMounts:
+                - name: gitlab-pat
+                  mountPath: /etc/gitlab-pat
+                  readOnly: true
+              resources:
+                requests:
+                  cpu: 50m
+                  memory: 32Mi
+                limits:
+                  cpu: 200m
+                  memory: 64Mi
+          volumes:
+            - name: gitlab-pat
+              secret:
+                secretName: gitlab-pat