foxhunt/infra/k8s/databases/postgres-backup.yaml

# Postgres backup — daily pg_dump to MinIO S3
# Uses initContainer (postgres:16-alpine) for pg_dump, then main container
# (python:3.12-alpine, already cached) uploads to MinIO via S3 API.
# Retains 7 daily + 4 weekly backups.
# Apply: kubectl apply -f postgres-backup.yaml
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: postgres-backup-script
  namespace: foxhunt
  labels:
    app.kubernetes.io/name: postgres-backup
    app.kubernetes.io/part-of: foxhunt
data:
  upload.py: |
    #!/usr/bin/env python3
    """Upload pg_dump backup to MinIO, prune old backups."""
    import hashlib, hmac, os, sys, glob, urllib.request, urllib.error
    from datetime import datetime, timezone

    ENDPOINT = os.environ["MINIO_ENDPOINT"]  # http://minio.foxhunt...:9000
    ACCESS_KEY = os.environ["MINIO_ACCESS_KEY"]
    SECRET_KEY = os.environ["MINIO_SECRET_KEY"]
    BUCKET = "foxhunt-backups"
    BACKUP_DIR = "/backup"

    def s3_sign(method, path, headers, payload_hash):
        """AWS Signature V4 for MinIO."""
        now = datetime.now(timezone.utc)
        date_stamp = now.strftime("%Y%m%d")
        amz_date = now.strftime("%Y%m%dT%H%M%SZ")
        region = "us-east-1"
        service = "s3"
        scope = f"{date_stamp}/{region}/{service}/aws4_request"

        headers["x-amz-date"] = amz_date
        headers["x-amz-content-sha256"] = payload_hash

        signed_headers = ";".join(sorted(headers.keys()))
        canonical_headers = "".join(f"{k}:{headers[k]}\n" for k in sorted(headers.keys()))
        canonical_request = f"{method}\n{path}\n\n{canonical_headers}\n{signed_headers}\n{payload_hash}"

        string_to_sign = f"AWS4-HMAC-SHA256\n{amz_date}\n{scope}\n{hashlib.sha256(canonical_request.encode()).hexdigest()}"

        def sign(key, msg):
            return hmac.new(key, msg.encode(), hashlib.sha256).digest()

        signing_key = sign(sign(sign(sign(
            f"AWS4{SECRET_KEY}".encode(), date_stamp), region), service), "aws4_request")
        signature = hmac.new(signing_key, string_to_sign.encode(), hashlib.sha256).hexdigest()

        headers["Authorization"] = (
            f"AWS4-HMAC-SHA256 Credential={ACCESS_KEY}/{scope}, "
            f"SignedHeaders={signed_headers}, Signature={signature}"
        )
        return headers

    def s3_put(key, data):
        path = f"/{BUCKET}/{key}"
        url = f"{ENDPOINT}{path}"
        payload_hash = hashlib.sha256(data).hexdigest()
        host = ENDPOINT.split("//")[1]
        headers = {"host": host, "content-length": str(len(data))}
        headers = s3_sign("PUT", path, headers, payload_hash)
        req = urllib.request.Request(url, data=data, method="PUT", headers=headers)
        urllib.request.urlopen(req, timeout=120)

    def s3_list(prefix):
        path = f"/{BUCKET}?prefix={prefix}&list-type=2"
        url = f"{ENDPOINT}{path}"
        host = ENDPOINT.split("//")[1]
        headers = {"host": host}
        headers = s3_sign("GET", f"/{BUCKET}", headers, hashlib.sha256(b"").hexdigest())
        req = urllib.request.Request(f"{url}", method="GET", headers=headers)
        try:
            resp = urllib.request.urlopen(req, timeout=30).read().decode()
            import re
            return sorted(re.findall(r"<Key>([^<]+)</Key>", resp))
        except Exception:
            return []

    def s3_delete(key):
        path = f"/{BUCKET}/{key}"
        url = f"{ENDPOINT}{path}"
        host = ENDPOINT.split("//")[1]
        headers = {"host": host}
        headers = s3_sign("DELETE", path, headers, hashlib.sha256(b"").hexdigest())
        req = urllib.request.Request(url, method="DELETE", headers=headers)
        urllib.request.urlopen(req, timeout=30)

    def s3_ensure_bucket():
        path = f"/{BUCKET}"
        url = f"{ENDPOINT}{path}"
        host = ENDPOINT.split("//")[1]
        headers = {"host": host}
        headers = s3_sign("PUT", path, headers, hashlib.sha256(b"").hexdigest())
        req = urllib.request.Request(url, method="PUT", headers=headers)
        try:
            urllib.request.urlopen(req, timeout=30)
        except urllib.error.HTTPError as e:
            if e.code not in (409, 200):  # 409 = bucket exists
                raise

    # Find the backup file
    files = glob.glob(f"{BACKUP_DIR}/*.sql.gz")
    if not files:
        print("ERROR: No backup file found in /backup/")
        sys.exit(1)

    backup_file = files[0]
    filename = os.path.basename(backup_file)
    size_mb = os.path.getsize(backup_file) / (1024 * 1024)
    print(f"Uploading {filename} ({size_mb:.1f} MB)")

    s3_ensure_bucket()

    with open(backup_file, "rb") as f:
        data = f.read()

    # Upload daily
    s3_put(f"daily/{filename}", data)
    print(f"  Uploaded daily/{filename}")

    # On Sundays, also upload weekly
    if datetime.now(timezone.utc).isoweekday() == 7:
        s3_put(f"weekly/{filename}", data)
        print(f"  Uploaded weekly/{filename}")

    # Prune old dailies (keep 7)
    dailies = s3_list("daily/")
    if len(dailies) > 7:
        for old in dailies[:len(dailies) - 7]:
            s3_delete(old)
            print(f"  Pruned {old}")

    # Prune old weeklies (keep 4)
    weeklies = s3_list("weekly/")
    if len(weeklies) > 4:
        for old in weeklies[:len(weeklies) - 4]:
            s3_delete(old)
            print(f"  Pruned {old}")

    print("=== Backup upload complete ===")
---
apiVersion: batch/v1
kind: CronJob
metadata:
  name: postgres-backup
  namespace: foxhunt
  labels:
    app.kubernetes.io/name: postgres-backup
    app.kubernetes.io/part-of: foxhunt
spec:
  schedule: "30 2 * * *"
  successfulJobsHistoryLimit: 1
  failedJobsHistoryLimit: 3
  jobTemplate:
    spec:
      backoffLimit: 2
      activeDeadlineSeconds: 600
      template:
        metadata:
          labels:
            app: postgres-backup
            app.kubernetes.io/part-of: foxhunt
        spec:
          nodeSelector:
            k8s.scaleway.com/pool-name: platform
          restartPolicy: OnFailure
          # Step 1: pg_dump in initContainer → shared /backup volume
          initContainers:
            - name: pg-dump
              image: postgres:16-alpine
              command:
                - sh
                - -c
                - |
                  TIMESTAMP=$(date -u +%Y%m%d-%H%M%S)
                  echo "=== Postgres Backup ${TIMESTAMP} ==="
                  PGPASSWORD="${POSTGRES_PASSWORD}" pg_dumpall \
                    -h "${POSTGRES_HOST}" \
                    -p 5432 \
                    -U "${POSTGRES_USER}" \
                    --clean --if-exists \
                    | gzip > "/backup/foxhunt-${TIMESTAMP}.sql.gz"
                  ls -lh /backup/
                  echo "Dump complete"
              env:
                - name: POSTGRES_HOST
                  value: "postgres.foxhunt.svc.cluster.local"
                - name: POSTGRES_USER
                  value: "foxhunt"
                - name: POSTGRES_PASSWORD
                  valueFrom:
                    secretKeyRef:
                      name: db-credentials
                      key: password
              volumeMounts:
                - name: backup
                  mountPath: /backup
              resources:
                requests:
                  cpu: 50m
                  memory: 128Mi
                limits:
                  cpu: 500m
                  memory: 512Mi
          # Step 2: Upload to MinIO in main container (python cached on nodes)
          containers:
            - name: upload
              image: python:3.12-alpine
              command: ["python3", "-u", "/scripts/upload.py"]
              env:
                - name: MINIO_ENDPOINT
                  value: "http://minio.foxhunt.svc.cluster.local:9000"
                - name: MINIO_ACCESS_KEY
                  valueFrom:
                    secretKeyRef:
                      name: minio-credentials
                      key: access-key
                - name: MINIO_SECRET_KEY
                  valueFrom:
                    secretKeyRef:
                      name: minio-credentials
                      key: secret-key
              volumeMounts:
                - name: backup
                  mountPath: /backup
                - name: script
                  mountPath: /scripts
              resources:
                requests:
                  cpu: 10m
                  memory: 64Mi
                limits:
                  cpu: 100m
                  memory: 128Mi
          volumes:
            - name: backup
              emptyDir:
                sizeLimit: 2Gi
            - name: script
              configMap:
                name: postgres-backup-script
                items:
                  - key: upload.py
                    path: upload.py
                defaultMode: 0755