foxhunt/infra/k8s/monitoring/pvc-autoscaler.yaml

# PVC Auto-Expand — expands PVCs automatically when usage exceeds 85%
# Uses K8s API to list PVCs and exec df inside pods to check usage.
# Runs every 15 minutes. Apply: kubectl apply -f pvc-autoscaler.yaml
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: pvc-autoscaler
  namespace: foxhunt
  labels:
    app.kubernetes.io/name: pvc-autoscaler
    app.kubernetes.io/part-of: foxhunt
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: pvc-autoscaler
  namespace: foxhunt
  labels:
    app.kubernetes.io/name: pvc-autoscaler
    app.kubernetes.io/part-of: foxhunt
rules:
  - apiGroups: [""]
    resources: ["persistentvolumeclaims"]
    verbs: ["get", "list", "patch"]
  - apiGroups: [""]
    resources: ["pods"]
    verbs: ["get", "list"]
  - apiGroups: ["apps"]
    resources: ["deployments", "statefulsets"]
    verbs: ["get", "list"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: pvc-autoscaler-nodes
  labels:
    app.kubernetes.io/name: pvc-autoscaler
    app.kubernetes.io/part-of: foxhunt
rules:
  - apiGroups: [""]
    resources: ["nodes"]
    verbs: ["list"]
  - apiGroups: [""]
    resources: ["nodes/proxy"]
    verbs: ["get"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: pvc-autoscaler
  namespace: foxhunt
  labels:
    app.kubernetes.io/name: pvc-autoscaler
    app.kubernetes.io/part-of: foxhunt
subjects:
  - kind: ServiceAccount
    name: pvc-autoscaler
    namespace: foxhunt
roleRef:
  kind: Role
  name: pvc-autoscaler
  apiGroup: rbac.authorization.k8s.io
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: pvc-autoscaler-nodes
  labels:
    app.kubernetes.io/name: pvc-autoscaler
    app.kubernetes.io/part-of: foxhunt
subjects:
  - kind: ServiceAccount
    name: pvc-autoscaler
    namespace: foxhunt
roleRef:
  kind: ClusterRole
  name: pvc-autoscaler-nodes
  apiGroup: rbac.authorization.k8s.io
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: pvc-autoscaler-script
  namespace: foxhunt
  labels:
    app.kubernetes.io/name: pvc-autoscaler
    app.kubernetes.io/part-of: foxhunt
data:
  autoscale.py: |
    #!/usr/bin/env python3
    """PVC auto-expand via kubelet stats API (no Prometheus dependency)."""
    import json, os, sys, ssl, urllib.request
    from datetime import datetime, timezone

    THRESHOLD = int(os.environ.get("PVC_USAGE_THRESHOLD", "85"))
    GROW_PERCENT = int(os.environ.get("PVC_GROW_PERCENT", "50"))
    MAX_SIZE_GI = int(os.environ.get("PVC_MAX_SIZE_GI", "100"))
    NAMESPACE = os.environ.get("NAMESPACE", "foxhunt")

    TOKEN_PATH = "/var/run/secrets/kubernetes.io/serviceaccount/token"
    CA_PATH = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
    K8S_API = "https://kubernetes.default.svc"

    def k8s(path, method="GET", data=None):
        token = open(TOKEN_PATH).read().strip()
        ctx = ssl.create_default_context(cafile=CA_PATH)
        body = json.dumps(data).encode() if data else None
        req = urllib.request.Request(f"{K8S_API}{path}", data=body, method=method)
        req.add_header("Authorization", f"Bearer {token}")
        if data:
            req.add_header("Content-Type", "application/merge-patch+json")
        return json.loads(urllib.request.urlopen(req, context=ctx, timeout=30).read())

    now = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
    print(f"=== PVC Auto-Expand {now} ===")
    print(f"Threshold: {THRESHOLD}% | Grow: +{GROW_PERCENT}% | Max: {MAX_SIZE_GI}Gi")
    sys.stdout.flush()

    # Step 1: Get all nodes and their kubelet stats via K8s proxy API
    nodes = k8s("/api/v1/nodes")
    pvc_usage = {}  # pvc_name -> (used_bytes, capacity_bytes)

    for node in nodes["items"]:
        node_name = node["metadata"]["name"]
        try:
            stats = k8s(f"/api/v1/nodes/{node_name}/proxy/stats/summary")
            for pod in stats.get("pods", []):
                if pod.get("podRef", {}).get("namespace") != NAMESPACE:
                    continue
                for vol in pod.get("volume", []):
                    pvc_ref = vol.get("pvcRef")
                    if not pvc_ref:
                        continue
                    pvc_name = pvc_ref["name"]
                    used = vol.get("usedBytes", 0)
                    cap = vol.get("capacityBytes", 1)
                    pvc_usage[pvc_name] = (used, cap)
        except Exception as e:
            print(f"  WARN: could not get stats from node {node_name}: {e}")
    sys.stdout.flush()

    if not pvc_usage:
        print("  No PVC stats found")
        sys.exit(0)

    # Step 2: Check each PVC and expand if needed
    expanded = 0
    for pvc_name, (used, cap) in sorted(pvc_usage.items()):
        pct = 100 * used / cap if cap > 0 else 0
        used_gi = used / (1024**3)
        cap_gi = cap / (1024**3)
        print(f"  {pvc_name}: {pct:.1f}% ({used_gi:.1f}G / {cap_gi:.1f}G)")

        if pct >= THRESHOLD:
            print(f"  >> {pvc_name} exceeds {THRESHOLD}%")
            try:
                pvc_obj = k8s(f"/api/v1/namespaces/{NAMESPACE}/persistentvolumeclaims/{pvc_name}")
                sc = pvc_obj.get("spec", {}).get("storageClassName", "")
                # Only expand if StorageClass supports it
                if sc:
                    try:
                        sc_obj = k8s(f"/apis/storage.k8s.io/v1/storageclasses/{sc}")
                        if not sc_obj.get("allowVolumeExpansion"):
                            print(f"     SKIP: StorageClass {sc} does not allow expansion")
                            continue
                    except Exception:
                        pass

                status_storage = pvc_obj.get("status", {}).get("capacity", {}).get("storage", "0Gi")
                current_gi = int(status_storage.replace("Gi", ""))
                spec_storage = pvc_obj["spec"]["resources"]["requests"]["storage"]
                spec_gi = int(spec_storage.replace("Gi", ""))

                new_gi = current_gi + max(current_gi * GROW_PERCENT // 100, 1)
                if new_gi > MAX_SIZE_GI:
                    print(f"     SKIP: target {new_gi}Gi > max {MAX_SIZE_GI}Gi")
                    continue
                if spec_gi >= new_gi:
                    print(f"     SKIP: spec={spec_gi}Gi >= target={new_gi}Gi (resize pending)")
                    continue

                patch = {"spec": {"resources": {"requests": {"storage": f"{new_gi}Gi"}}}}
                k8s(f"/api/v1/namespaces/{NAMESPACE}/persistentvolumeclaims/{pvc_name}",
                    method="PATCH", data=patch)
                print(f"     EXPANDED: {current_gi}Gi -> {new_gi}Gi")
                expanded += 1
            except Exception as e:
                print(f"     FAILED: {e}", file=sys.stderr)
        sys.stdout.flush()

    print(f"=== Done ({expanded} expanded) ===")
---
apiVersion: batch/v1
kind: CronJob
metadata:
  name: pvc-autoscaler
  namespace: foxhunt
  labels:
    app.kubernetes.io/name: pvc-autoscaler
    app.kubernetes.io/part-of: foxhunt
spec:
  schedule: "*/15 * * * *"
  successfulJobsHistoryLimit: 1
  failedJobsHistoryLimit: 3
  jobTemplate:
    spec:
      backoffLimit: 2
      activeDeadlineSeconds: 120
      template:
        metadata:
          labels:
            app: pvc-autoscaler
            app.kubernetes.io/part-of: foxhunt
        spec:
          nodeSelector:
            k8s.scaleway.com/pool-name: platform
          serviceAccountName: pvc-autoscaler
          restartPolicy: OnFailure
          containers:
            - name: autoscaler
              image: python:3.12-alpine
              command: ["python3", "-u", "/scripts/autoscale.py"]
              env:
                - name: PVC_USAGE_THRESHOLD
                  value: "85"
                - name: PVC_GROW_PERCENT
                  value: "50"
                - name: PVC_MAX_SIZE_GI
                  value: "100"
                - name: NAMESPACE
                  value: "foxhunt"
              volumeMounts:
                - name: script
                  mountPath: /scripts
              resources:
                requests:
                  cpu: 10m
                  memory: 64Mi
                limits:
                  cpu: 100m
                  memory: 128Mi
          volumes:
            - name: script
              configMap:
                name: pvc-autoscaler-script
                items:
                  - key: autoscale.py
                    path: autoscale.py
                defaultMode: 0755