Files
foxhunt/infra/k8s/minio/minio.yaml
jgrusewski 99f54e3f94 fix(infra): resolve postgres disk-full cascade, add SPOF mitigations
Postgres PVC hit 100% → crash-looped → GitLab webservice couldn't
verify SSH keys → all git operations failed. Root cause: 10Gi PVC
outgrown by GitLab database.

Fixes applied:
- Expand postgres PVC 10Gi → 20Gi
- Expand minio PVC 100Gi → 150Gi (was 81.8% full)
- Expand prometheus PVC 2Gi → 10Gi, retentionSize 1500MB → 8GB
- Scale gitlab-webservice to 2 replicas for HA
- Add PVC autoscaler CronJob (every 15min, auto-expand at 85%)
- Add daily postgres backup CronJob (pg_dump → MinIO, 7d + 4w retention)
- Add PrometheusRule storage alerts (75% warning, 90% critical)
- Add network policies for maintenance job egress

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-03-28 18:55:25 +01:00

185 lines
4.9 KiB
YAML

# MinIO — in-cluster S3 for compiled binaries and trained models
# Replaces Scaleway public S3 to keep proprietary code/IP in-cluster
# Plain HTTP only — internal cluster traffic, no TLS needed
apiVersion: v1
kind: Secret
metadata:
name: minio-credentials
namespace: foxhunt
labels:
app.kubernetes.io/name: minio
app.kubernetes.io/part-of: foxhunt
type: Opaque
stringData:
root-user: foxhunt-minio
root-password: CHANGE_ME_BEFORE_DEPLOY
# rclone-compatible keys (same values, different key names for initContainers)
access-key: foxhunt-minio
secret-key: CHANGE_ME_BEFORE_DEPLOY
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: minio-data-new
namespace: foxhunt
labels:
app.kubernetes.io/name: minio
app.kubernetes.io/part-of: foxhunt
spec:
accessModes:
- ReadWriteOnce
storageClassName: scw-bssd
resources:
requests:
storage: 150Gi
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: minio
namespace: foxhunt
labels:
app.kubernetes.io/name: minio
app.kubernetes.io/part-of: foxhunt
spec:
replicas: 1
strategy:
type: Recreate # single-node MinIO — no rolling update
selector:
matchLabels:
app.kubernetes.io/name: minio
template:
metadata:
labels:
app.kubernetes.io/name: minio
app.kubernetes.io/part-of: foxhunt
spec:
nodeSelector:
k8s.scaleway.com/pool-name: platform
containers:
- name: minio
image: minio/minio:latest
args: ["server", "/data", "--console-address", ":9001"]
ports:
- containerPort: 9000
name: api
- containerPort: 9001
name: console
env:
- name: MINIO_ROOT_USER
valueFrom:
secretKeyRef:
name: minio-credentials
key: root-user
- name: MINIO_ROOT_PASSWORD
valueFrom:
secretKeyRef:
name: minio-credentials
key: root-password
volumeMounts:
- name: data
mountPath: /data
readinessProbe:
httpGet:
path: /minio/health/live
port: 9000
initialDelaySeconds: 5
periodSeconds: 10
livenessProbe:
httpGet:
path: /minio/health/live
port: 9000
initialDelaySeconds: 10
periodSeconds: 15
failureThreshold: 5
resources:
requests:
cpu: 200m
memory: 512Mi
limits:
cpu: "1"
memory: 2Gi
volumes:
- name: data
persistentVolumeClaim:
claimName: minio-data-new
---
apiVersion: v1
kind: Service
metadata:
name: minio
namespace: foxhunt
labels:
app.kubernetes.io/name: minio
app.kubernetes.io/part-of: foxhunt
spec:
selector:
app.kubernetes.io/name: minio
ports:
- port: 9000
targetPort: 9000
name: api
- port: 9001
targetPort: 9001
name: console
---
# Create buckets on first deploy
apiVersion: batch/v1
kind: Job
metadata:
name: minio-init-buckets
namespace: foxhunt
labels:
app.kubernetes.io/name: minio
app.kubernetes.io/part-of: foxhunt
spec:
backoffLimit: 5
ttlSecondsAfterFinished: 300
template:
metadata:
labels:
app.kubernetes.io/name: minio-init
spec:
nodeSelector:
k8s.scaleway.com/pool-name: platform
restartPolicy: OnFailure
containers:
- name: mc
image: minio/mc:latest
command: ["/bin/sh", "-c"]
args:
- |
set -e
echo "Waiting for MinIO to be ready..."
until mc alias set foxhunt http://minio.foxhunt.svc.cluster.local:9000 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" 2>/dev/null; do
echo "MinIO not ready, retrying in 3s..."
sleep 3
done
echo "Creating buckets..."
mc mb --ignore-existing foxhunt/foxhunt-binaries
mc mb --ignore-existing foxhunt/foxhunt-models
mc mb --ignore-existing foxhunt/foxhunt-training-results
mc mb --ignore-existing foxhunt/argo-logs
mc mb --ignore-existing foxhunt/foxhunt-sccache
mc mb --ignore-existing foxhunt/foxhunt-training-data
echo "Buckets created:"
mc ls foxhunt/
env:
- name: MINIO_ROOT_USER
valueFrom:
secretKeyRef:
name: minio-credentials
key: root-user
- name: MINIO_ROOT_PASSWORD
valueFrom:
secretKeyRef:
name: minio-credentials
key: root-password
resources:
requests:
cpu: 50m
memory: 32Mi
limits:
cpu: 200m
memory: 64Mi