foxhunt/infra/k8s/gitlab/pat-rotation.yaml

# GitLab PAT auto-rotation CronJob
# Rotates the personal access token monthly, storing the new token in a K8s Secret.
# Uses GitLab's atomic rotate API: creates new token + revokes old in one call.
#
# The rotation runs on the 1st of every month. Additionally, a weekly expiry check
# runs every Monday and logs a warning if the token expires within 14 days, giving
# time to investigate rotation failures before CI breaks.
#
# Initial setup:
#   kubectl -n foxhunt create secret generic gitlab-pat \
#     --from-literal=token=glpat-<initial-token>
#
# Manual rotation:
#   kubectl -n foxhunt create job pat-rotate-manual --from=cronjob/gitlab-pat-rotation
#
# Manual expiry check:
#   kubectl -n foxhunt create job pat-check-manual --from=cronjob/gitlab-pat-expiry-check
#
# Sync local token:
#   export GITLAB_TOKEN=$(kubectl -n foxhunt get secret gitlab-pat -o jsonpath='{.data.token}' | base64 -d)
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: gitlab-pat-rotator
  namespace: foxhunt
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: gitlab-pat-rotator
  namespace: foxhunt
rules:
  - apiGroups: [""]
    resources: ["secrets"]
    resourceNames: ["gitlab-pat"]
    verbs: ["get", "patch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: gitlab-pat-rotator
  namespace: foxhunt
subjects:
  - kind: ServiceAccount
    name: gitlab-pat-rotator
    namespace: foxhunt
roleRef:
  kind: Role
  name: gitlab-pat-rotator
  apiGroup: rbac.authorization.k8s.io
---
# Monthly rotation — creates new token, revokes old, updates K8s secret
apiVersion: batch/v1
kind: CronJob
metadata:
  name: gitlab-pat-rotation
  namespace: foxhunt
spec:
  schedule: "0 3 1 * *"  # 1st of every month at 03:00 UTC
  successfulJobsHistoryLimit: 3
  failedJobsHistoryLimit: 5
  concurrencyPolicy: Forbid
  jobTemplate:
    spec:
      backoffLimit: 3
      activeDeadlineSeconds: 180
      template:
        metadata:
          labels:
            app: gitlab-pat-rotation
        spec:
          serviceAccountName: gitlab-pat-rotator
          restartPolicy: Never
          nodeSelector:
            k8s.scaleway.com/pool-name: platform
          containers:
            - name: rotate
              image: alpine:3.21
              command: ["/bin/sh", "-c"]
              args:
                - |
                  set -e

                  apk add --no-cache -q curl jq

                  CURRENT_PAT=$(cat /etc/gitlab-pat/token)
                  GITLAB_URL="http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181"

                  # 1. Verify current token is valid
                  TOKEN_INFO=$(curl -sf "${GITLAB_URL}/api/v4/personal_access_tokens/self" \
                    -H "PRIVATE-TOKEN: ${CURRENT_PAT}") || {
                    echo "CRITICAL: Current PAT is invalid or GitLab unreachable."
                    echo "Manual recovery required: create a new PAT in GitLab UI and run:"
                    echo "  kubectl -n foxhunt create secret generic gitlab-pat --from-literal=token=<new-pat> --dry-run=client -o yaml | kubectl apply -f -"
                    exit 1
                  }
                  OLD_EXPIRY=$(echo "$TOKEN_INFO" | jq -r '.expires_at')
                  TOKEN_NAME=$(echo "$TOKEN_INFO" | jq -r '.name')
                  echo "Current PAT '${TOKEN_NAME}' expires: ${OLD_EXPIRY}"

                  # 2. Rotate: new expiry = 1 year from now
                  EXPIRES=$(date -d "@$(($(date +%s) + 31536000))" +%Y-%m-%d)
                  RESPONSE=$(curl -sf -X POST \
                    "${GITLAB_URL}/api/v4/personal_access_tokens/self/rotate" \
                    -H "PRIVATE-TOKEN: ${CURRENT_PAT}" \
                    -d "expires_at=${EXPIRES}") || {
                    echo "ERROR: Rotation API call failed. Token may still be valid."
                    exit 1
                  }

                  NEW_PAT=$(echo "$RESPONSE" | jq -r '.token')
                  if [ "$NEW_PAT" = "null" ] || [ -z "$NEW_PAT" ]; then
                    echo "ERROR: No token in rotation response: $RESPONSE"
                    exit 1
                  fi

                  echo "PAT rotated successfully. New expiry: ${EXPIRES}"

                  # 3. Update K8s secret (old token is already revoked at this point)
                  K8S_TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
                  K8S_CA=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt
                  K8S_NS=$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace)
                  K8S_API="https://kubernetes.default.svc"

                  NEW_PAT_B64=$(printf '%s' "$NEW_PAT" | base64)

                  for attempt in 1 2 3 4 5; do
                    if curl -sf -X PATCH \
                      "${K8S_API}/api/v1/namespaces/${K8S_NS}/secrets/gitlab-pat" \
                      --cacert "$K8S_CA" \
                      -H "Authorization: Bearer ${K8S_TOKEN}" \
                      -H "Content-Type: application/strategic-merge-patch+json" \
                      -d "{\"data\":{\"token\":\"${NEW_PAT_B64}\"}}"; then
                      echo ""
                      echo "K8s secret updated on attempt ${attempt}."

                      # 4. Verify the new token works
                      VERIFY=$(curl -sf "${GITLAB_URL}/api/v4/personal_access_tokens/self" \
                        -H "PRIVATE-TOKEN: ${NEW_PAT}" | jq -r '.expires_at') || true
                      if [ "$VERIFY" = "$EXPIRES" ]; then
                        echo "Verification passed: new token valid until ${VERIFY}"
                      else
                        echo "WARNING: Verification returned unexpected expiry: ${VERIFY}"
                      fi
                      exit 0
                    fi
                    echo "Secret update attempt ${attempt} failed, retrying in 3s..."
                    sleep 3
                  done

                  # If we get here, rotation succeeded but secret update failed.
                  # Print token so it can be recovered from job logs.
                  echo "CRITICAL: Secret update failed after 5 attempts!"
                  echo "New PAT (recover manually): ${NEW_PAT}"
                  echo "Run: kubectl -n foxhunt create secret generic gitlab-pat --from-literal=token=<above> --dry-run=client -o yaml | kubectl apply -f -"
                  exit 1
              volumeMounts:
                - name: gitlab-pat
                  mountPath: /etc/gitlab-pat
                  readOnly: true
              resources:
                requests:
                  cpu: 50m
                  memory: 32Mi
                limits:
                  cpu: 200m
                  memory: 64Mi
          volumes:
            - name: gitlab-pat
              secret:
                secretName: gitlab-pat
---
# Weekly expiry check — warns if token expires within 14 days
apiVersion: batch/v1
kind: CronJob
metadata:
  name: gitlab-pat-expiry-check
  namespace: foxhunt
spec:
  schedule: "0 8 * * 1"  # Every Monday at 08:00 UTC
  successfulJobsHistoryLimit: 1
  failedJobsHistoryLimit: 3
  concurrencyPolicy: Forbid
  jobTemplate:
    spec:
      backoffLimit: 1
      activeDeadlineSeconds: 60
      template:
        metadata:
          labels:
            app: gitlab-pat-expiry-check
        spec:
          restartPolicy: Never
          nodeSelector:
            k8s.scaleway.com/pool-name: platform
          containers:
            - name: check
              image: alpine:3.21
              command: ["/bin/sh", "-c"]
              args:
                - |
                  set -e

                  apk add --no-cache -q curl jq

                  CURRENT_PAT=$(cat /etc/gitlab-pat/token)
                  GITLAB_URL="http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181"

                  TOKEN_INFO=$(curl -sf "${GITLAB_URL}/api/v4/personal_access_tokens/self" \
                    -H "PRIVATE-TOKEN: ${CURRENT_PAT}") || {
                    echo "CRITICAL: PAT is INVALID. CI will fail!"
                    echo "Create a new PAT in GitLab UI and run:"
                    echo "  kubectl -n foxhunt create secret generic gitlab-pat --from-literal=token=<new-pat> --dry-run=client -o yaml | kubectl apply -f -"
                    exit 1
                  }

                  EXPIRY=$(echo "$TOKEN_INFO" | jq -r '.expires_at')
                  TOKEN_NAME=$(echo "$TOKEN_INFO" | jq -r '.name')
                  SCOPES=$(echo "$TOKEN_INFO" | jq -r '.scopes | join(",")')

                  # Calculate days until expiry
                  EXPIRY_EPOCH=$(date -d "$EXPIRY" +%s)
                  NOW_EPOCH=$(date +%s)
                  DAYS_LEFT=$(( (EXPIRY_EPOCH - NOW_EPOCH) / 86400 ))

                  echo "PAT '${TOKEN_NAME}': expires=${EXPIRY} scopes=${SCOPES} days_left=${DAYS_LEFT}"

                  if [ "$DAYS_LEFT" -lt 0 ]; then
                    echo "CRITICAL: PAT has EXPIRED ${DAYS_LEFT} days ago!"
                    exit 1
                  elif [ "$DAYS_LEFT" -lt 14 ]; then
                    echo "WARNING: PAT expires in ${DAYS_LEFT} days. Rotation may have failed."
                    echo "Manual rotation: kubectl -n foxhunt create job pat-rotate-manual --from=cronjob/gitlab-pat-rotation"
                    exit 1
                  else
                    echo "OK: PAT valid for ${DAYS_LEFT} more days."
                  fi
              volumeMounts:
                - name: gitlab-pat
                  mountPath: /etc/gitlab-pat
                  readOnly: true
              resources:
                requests:
                  cpu: 10m
                  memory: 32Mi
                limits:
                  cpu: 100m
                  memory: 64Mi
          volumes:
            - name: gitlab-pat
              secret:
                secretName: gitlab-pat