Postgres PVC hit 100% → crash-looped → GitLab webservice couldn't verify SSH keys → all git operations failed. Root cause: 10Gi PVC outgrown by GitLab database. Fixes applied: - Expand postgres PVC 10Gi → 20Gi - Expand minio PVC 100Gi → 150Gi (was 81.8% full) - Expand prometheus PVC 2Gi → 10Gi, retentionSize 1500MB → 8GB - Scale gitlab-webservice to 2 replicas for HA - Add PVC autoscaler CronJob (every 15min, auto-expand at 85%) - Add daily postgres backup CronJob (pg_dump → MinIO, 7d + 4w retention) - Add PrometheusRule storage alerts (75% warning, 90% critical) - Add network policies for maintenance job egress Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
185 lines
4.9 KiB
YAML
185 lines
4.9 KiB
YAML
# MinIO — in-cluster S3 for compiled binaries and trained models
|
|
# Replaces Scaleway public S3 to keep proprietary code/IP in-cluster
|
|
# Plain HTTP only — internal cluster traffic, no TLS needed
|
|
apiVersion: v1
|
|
kind: Secret
|
|
metadata:
|
|
name: minio-credentials
|
|
namespace: foxhunt
|
|
labels:
|
|
app.kubernetes.io/name: minio
|
|
app.kubernetes.io/part-of: foxhunt
|
|
type: Opaque
|
|
stringData:
|
|
root-user: foxhunt-minio
|
|
root-password: CHANGE_ME_BEFORE_DEPLOY
|
|
# rclone-compatible keys (same values, different key names for initContainers)
|
|
access-key: foxhunt-minio
|
|
secret-key: CHANGE_ME_BEFORE_DEPLOY
|
|
---
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata:
|
|
name: minio-data-new
|
|
namespace: foxhunt
|
|
labels:
|
|
app.kubernetes.io/name: minio
|
|
app.kubernetes.io/part-of: foxhunt
|
|
spec:
|
|
accessModes:
|
|
- ReadWriteOnce
|
|
storageClassName: scw-bssd
|
|
resources:
|
|
requests:
|
|
storage: 150Gi
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: minio
|
|
namespace: foxhunt
|
|
labels:
|
|
app.kubernetes.io/name: minio
|
|
app.kubernetes.io/part-of: foxhunt
|
|
spec:
|
|
replicas: 1
|
|
strategy:
|
|
type: Recreate # single-node MinIO — no rolling update
|
|
selector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: minio
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app.kubernetes.io/name: minio
|
|
app.kubernetes.io/part-of: foxhunt
|
|
spec:
|
|
nodeSelector:
|
|
k8s.scaleway.com/pool-name: platform
|
|
containers:
|
|
- name: minio
|
|
image: minio/minio:latest
|
|
args: ["server", "/data", "--console-address", ":9001"]
|
|
ports:
|
|
- containerPort: 9000
|
|
name: api
|
|
- containerPort: 9001
|
|
name: console
|
|
env:
|
|
- name: MINIO_ROOT_USER
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: minio-credentials
|
|
key: root-user
|
|
- name: MINIO_ROOT_PASSWORD
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: minio-credentials
|
|
key: root-password
|
|
volumeMounts:
|
|
- name: data
|
|
mountPath: /data
|
|
readinessProbe:
|
|
httpGet:
|
|
path: /minio/health/live
|
|
port: 9000
|
|
initialDelaySeconds: 5
|
|
periodSeconds: 10
|
|
livenessProbe:
|
|
httpGet:
|
|
path: /minio/health/live
|
|
port: 9000
|
|
initialDelaySeconds: 10
|
|
periodSeconds: 15
|
|
failureThreshold: 5
|
|
resources:
|
|
requests:
|
|
cpu: 200m
|
|
memory: 512Mi
|
|
limits:
|
|
cpu: "1"
|
|
memory: 2Gi
|
|
volumes:
|
|
- name: data
|
|
persistentVolumeClaim:
|
|
claimName: minio-data-new
|
|
---
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: minio
|
|
namespace: foxhunt
|
|
labels:
|
|
app.kubernetes.io/name: minio
|
|
app.kubernetes.io/part-of: foxhunt
|
|
spec:
|
|
selector:
|
|
app.kubernetes.io/name: minio
|
|
ports:
|
|
- port: 9000
|
|
targetPort: 9000
|
|
name: api
|
|
- port: 9001
|
|
targetPort: 9001
|
|
name: console
|
|
---
|
|
# Create buckets on first deploy
|
|
apiVersion: batch/v1
|
|
kind: Job
|
|
metadata:
|
|
name: minio-init-buckets
|
|
namespace: foxhunt
|
|
labels:
|
|
app.kubernetes.io/name: minio
|
|
app.kubernetes.io/part-of: foxhunt
|
|
spec:
|
|
backoffLimit: 5
|
|
ttlSecondsAfterFinished: 300
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app.kubernetes.io/name: minio-init
|
|
spec:
|
|
nodeSelector:
|
|
k8s.scaleway.com/pool-name: platform
|
|
restartPolicy: OnFailure
|
|
containers:
|
|
- name: mc
|
|
image: minio/mc:latest
|
|
command: ["/bin/sh", "-c"]
|
|
args:
|
|
- |
|
|
set -e
|
|
echo "Waiting for MinIO to be ready..."
|
|
until mc alias set foxhunt http://minio.foxhunt.svc.cluster.local:9000 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" 2>/dev/null; do
|
|
echo "MinIO not ready, retrying in 3s..."
|
|
sleep 3
|
|
done
|
|
echo "Creating buckets..."
|
|
mc mb --ignore-existing foxhunt/foxhunt-binaries
|
|
mc mb --ignore-existing foxhunt/foxhunt-models
|
|
mc mb --ignore-existing foxhunt/foxhunt-training-results
|
|
mc mb --ignore-existing foxhunt/argo-logs
|
|
mc mb --ignore-existing foxhunt/foxhunt-sccache
|
|
mc mb --ignore-existing foxhunt/foxhunt-training-data
|
|
echo "Buckets created:"
|
|
mc ls foxhunt/
|
|
env:
|
|
- name: MINIO_ROOT_USER
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: minio-credentials
|
|
key: root-user
|
|
- name: MINIO_ROOT_PASSWORD
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: minio-credentials
|
|
key: root-password
|
|
resources:
|
|
requests:
|
|
cpu: 50m
|
|
memory: 32Mi
|
|
limits:
|
|
cpu: 200m
|
|
memory: 64Mi
|