# GitLab PAT auto-rotation CronJob # Rotates the personal access token monthly, storing the new token in a K8s Secret. # Uses GitLab's atomic rotate API: creates new token + revokes old in one call. # # The rotation runs on the 1st of every month. Additionally, a weekly expiry check # runs every Monday and logs a warning if the token expires within 14 days, giving # time to investigate rotation failures before CI breaks. # # Initial setup: # kubectl -n foxhunt create secret generic gitlab-pat \ # --from-literal=token=glpat- # # Manual rotation: # kubectl -n foxhunt create job pat-rotate-manual --from=cronjob/gitlab-pat-rotation # # Manual expiry check: # kubectl -n foxhunt create job pat-check-manual --from=cronjob/gitlab-pat-expiry-check # # Sync local token: # export GITLAB_TOKEN=$(kubectl -n foxhunt get secret gitlab-pat -o jsonpath='{.data.token}' | base64 -d) --- apiVersion: v1 kind: ServiceAccount metadata: name: gitlab-pat-rotator namespace: foxhunt --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: gitlab-pat-rotator namespace: foxhunt rules: - apiGroups: [""] resources: ["secrets"] resourceNames: ["gitlab-pat"] verbs: ["get", "patch"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: gitlab-pat-rotator namespace: foxhunt subjects: - kind: ServiceAccount name: gitlab-pat-rotator namespace: foxhunt roleRef: kind: Role name: gitlab-pat-rotator apiGroup: rbac.authorization.k8s.io --- # Monthly rotation — creates new token, revokes old, updates K8s secret apiVersion: batch/v1 kind: CronJob metadata: name: gitlab-pat-rotation namespace: foxhunt spec: schedule: "0 3 1 * *" # 1st of every month at 03:00 UTC successfulJobsHistoryLimit: 3 failedJobsHistoryLimit: 5 concurrencyPolicy: Forbid jobTemplate: spec: backoffLimit: 3 activeDeadlineSeconds: 180 template: metadata: labels: app: gitlab-pat-rotation spec: serviceAccountName: gitlab-pat-rotator restartPolicy: Never nodeSelector: k8s.scaleway.com/pool-name: platform containers: - name: rotate image: alpine:3.21 command: ["/bin/sh", "-c"] args: - | set -e apk add --no-cache -q curl jq CURRENT_PAT=$(cat /etc/gitlab-pat/token) GITLAB_URL="http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181" # 1. Verify current token is valid TOKEN_INFO=$(curl -sf "${GITLAB_URL}/api/v4/personal_access_tokens/self" \ -H "PRIVATE-TOKEN: ${CURRENT_PAT}") || { echo "CRITICAL: Current PAT is invalid or GitLab unreachable." echo "Manual recovery required: create a new PAT in GitLab UI and run:" echo " kubectl -n foxhunt create secret generic gitlab-pat --from-literal=token= --dry-run=client -o yaml | kubectl apply -f -" exit 1 } OLD_EXPIRY=$(echo "$TOKEN_INFO" | jq -r '.expires_at') TOKEN_NAME=$(echo "$TOKEN_INFO" | jq -r '.name') echo "Current PAT '${TOKEN_NAME}' expires: ${OLD_EXPIRY}" # 2. Rotate: new expiry = 1 year from now EXPIRES=$(date -d "@$(($(date +%s) + 31536000))" +%Y-%m-%d) RESPONSE=$(curl -sf -X POST \ "${GITLAB_URL}/api/v4/personal_access_tokens/self/rotate" \ -H "PRIVATE-TOKEN: ${CURRENT_PAT}" \ -d "expires_at=${EXPIRES}") || { echo "ERROR: Rotation API call failed. Token may still be valid." exit 1 } NEW_PAT=$(echo "$RESPONSE" | jq -r '.token') if [ "$NEW_PAT" = "null" ] || [ -z "$NEW_PAT" ]; then echo "ERROR: No token in rotation response: $RESPONSE" exit 1 fi echo "PAT rotated successfully. New expiry: ${EXPIRES}" # 3. Update K8s secret (old token is already revoked at this point) K8S_TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token) K8S_CA=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt K8S_NS=$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace) K8S_API="https://kubernetes.default.svc" NEW_PAT_B64=$(printf '%s' "$NEW_PAT" | base64) for attempt in 1 2 3 4 5; do if curl -sf -X PATCH \ "${K8S_API}/api/v1/namespaces/${K8S_NS}/secrets/gitlab-pat" \ --cacert "$K8S_CA" \ -H "Authorization: Bearer ${K8S_TOKEN}" \ -H "Content-Type: application/strategic-merge-patch+json" \ -d "{\"data\":{\"token\":\"${NEW_PAT_B64}\"}}"; then echo "" echo "K8s secret updated on attempt ${attempt}." # 4. Verify the new token works VERIFY=$(curl -sf "${GITLAB_URL}/api/v4/personal_access_tokens/self" \ -H "PRIVATE-TOKEN: ${NEW_PAT}" | jq -r '.expires_at') || true if [ "$VERIFY" = "$EXPIRES" ]; then echo "Verification passed: new token valid until ${VERIFY}" else echo "WARNING: Verification returned unexpected expiry: ${VERIFY}" fi exit 0 fi echo "Secret update attempt ${attempt} failed, retrying in 3s..." sleep 3 done # If we get here, rotation succeeded but secret update failed. # Print token so it can be recovered from job logs. echo "CRITICAL: Secret update failed after 5 attempts!" echo "New PAT (recover manually): ${NEW_PAT}" echo "Run: kubectl -n foxhunt create secret generic gitlab-pat --from-literal=token= --dry-run=client -o yaml | kubectl apply -f -" exit 1 volumeMounts: - name: gitlab-pat mountPath: /etc/gitlab-pat readOnly: true resources: requests: cpu: 50m memory: 32Mi limits: cpu: 200m memory: 64Mi volumes: - name: gitlab-pat secret: secretName: gitlab-pat --- # Weekly expiry check — warns if token expires within 14 days apiVersion: batch/v1 kind: CronJob metadata: name: gitlab-pat-expiry-check namespace: foxhunt spec: schedule: "0 8 * * 1" # Every Monday at 08:00 UTC successfulJobsHistoryLimit: 1 failedJobsHistoryLimit: 3 concurrencyPolicy: Forbid jobTemplate: spec: backoffLimit: 1 activeDeadlineSeconds: 60 template: metadata: labels: app: gitlab-pat-expiry-check spec: restartPolicy: Never nodeSelector: k8s.scaleway.com/pool-name: platform containers: - name: check image: alpine:3.21 command: ["/bin/sh", "-c"] args: - | set -e apk add --no-cache -q curl jq CURRENT_PAT=$(cat /etc/gitlab-pat/token) GITLAB_URL="http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181" TOKEN_INFO=$(curl -sf "${GITLAB_URL}/api/v4/personal_access_tokens/self" \ -H "PRIVATE-TOKEN: ${CURRENT_PAT}") || { echo "CRITICAL: PAT is INVALID. CI will fail!" echo "Create a new PAT in GitLab UI and run:" echo " kubectl -n foxhunt create secret generic gitlab-pat --from-literal=token= --dry-run=client -o yaml | kubectl apply -f -" exit 1 } EXPIRY=$(echo "$TOKEN_INFO" | jq -r '.expires_at') TOKEN_NAME=$(echo "$TOKEN_INFO" | jq -r '.name') SCOPES=$(echo "$TOKEN_INFO" | jq -r '.scopes | join(",")') # Calculate days until expiry EXPIRY_EPOCH=$(date -d "$EXPIRY" +%s) NOW_EPOCH=$(date +%s) DAYS_LEFT=$(( (EXPIRY_EPOCH - NOW_EPOCH) / 86400 )) echo "PAT '${TOKEN_NAME}': expires=${EXPIRY} scopes=${SCOPES} days_left=${DAYS_LEFT}" if [ "$DAYS_LEFT" -lt 0 ]; then echo "CRITICAL: PAT has EXPIRED ${DAYS_LEFT} days ago!" exit 1 elif [ "$DAYS_LEFT" -lt 14 ]; then echo "WARNING: PAT expires in ${DAYS_LEFT} days. Rotation may have failed." echo "Manual rotation: kubectl -n foxhunt create job pat-rotate-manual --from=cronjob/gitlab-pat-rotation" exit 1 else echo "OK: PAT valid for ${DAYS_LEFT} more days." fi volumeMounts: - name: gitlab-pat mountPath: /etc/gitlab-pat readOnly: true resources: requests: cpu: 10m memory: 32Mi limits: cpu: 100m memory: 64Mi volumes: - name: gitlab-pat secret: secretName: gitlab-pat