gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
ffc83da6439ab76da19f2769a8ad9d34ee5f66a7
foxhunt/infra/k8s/monitoring/prometheus-stack-values.yaml
jgrusewski 99f54e3f94 fix(infra): resolve postgres disk-full cascade, add SPOF mitigations 
Postgres PVC hit 100% → crash-looped → GitLab webservice couldn't
verify SSH keys → all git operations failed. Root cause: 10Gi PVC
outgrown by GitLab database.

Fixes applied:
- Expand postgres PVC 10Gi → 20Gi
- Expand minio PVC 100Gi → 150Gi (was 81.8% full)
- Expand prometheus PVC 2Gi → 10Gi, retentionSize 1500MB → 8GB
- Scale gitlab-webservice to 2 replicas for HA
- Add PVC autoscaler CronJob (every 15min, auto-expand at 85%)
- Add daily postgres backup CronJob (pg_dump → MinIO, 7d + 4w retention)
- Add PrometheusRule storage alerts (75% warning, 90% critical)
- Add network policies for maintenance job egress

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-03-28 18:55:25 +01:00

401 lines
14 KiB
YAML
Raw Blame History

# kube-prometheus-stack Helm values — Foxhunt production monitoring
#
# Install:
# helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
# helm repo update
# helm install prometheus-stack prometheus-community/kube-prometheus-stack \
# --namespace foxhunt \
# -f infra/k8s/monitoring/prometheus-stack-values.yaml
#
# Upgrade:
# helm upgrade prometheus-stack prometheus-community/kube-prometheus-stack \
# --namespace foxhunt \
# -f infra/k8s/monitoring/prometheus-stack-values.yaml
#
# Prerequisites:
# kubectl -n foxhunt create secret generic alertmanager-slack \
# --from-literal=slack-webhook-url='https://hooks.slack.com/services/...'
#
# After install, delete old standalone resources:
# kubectl -n foxhunt delete deploy prometheus
# kubectl -n foxhunt delete svc prometheus
# kubectl -n foxhunt delete cm prometheus-config
# kubectl -n foxhunt delete pvc prometheus-data
# kubectl -n foxhunt delete sa prometheus
# kubectl delete clusterrole foxhunt-prometheus
# kubectl delete clusterrolebinding foxhunt-prometheus
# kubectl -n foxhunt delete deploy node-exporter
# kubectl -n foxhunt delete ds node-exporter # if DaemonSet
# kubectl -n foxhunt delete deploy kube-state-metrics
# ─────────────────────────────────────────────
# Prometheus Operator
# ─────────────────────────────────────────────
prometheusOperator:
enabled: true
nodeSelector:
k8s.scaleway.com/pool-name: platform
resources:
requests:
cpu: 50m
memory: 64Mi
limits:
cpu: 200m
memory: 128Mi
# ─────────────────────────────────────────────
# Prometheus
# ─────────────────────────────────────────────
prometheus:
prometheusSpec:
nodeSelector:
k8s.scaleway.com/pool-name: platform
podMetadata:
labels:
app.kubernetes.io/part-of: foxhunt
retention: 14d
retentionSize: 8GB
replicas: 1
resources:
requests:
cpu: 100m
memory: 256Mi
limits:
cpu: 500m
memory: 512Mi
storageSpec:
volumeClaimTemplate:
spec:
storageClassName: scw-bssd
accessModes: ["ReadWriteOnce"]
resources:
requests:
storage: 10Gi
# Discover ALL ServiceMonitors/PodMonitors/PrometheusRules in the cluster,
# not just those matching Helm release labels.
serviceMonitorSelectorNilUsesHelmValues: false
podMonitorSelectorNilUsesHelmValues: false
ruleSelectorNilUsesHelmValues: false
# Scrape configs for targets that don't fit ServiceMonitor/PodMonitor CRDs
additionalScrapeConfigs:
# Argo Workflows controller (HTTPS with self-signed cert)
- job_name: argo-workflows
scheme: https
tls_config:
insecure_skip_verify: true
static_configs:
- targets: ['argo-workflows-workflow-controller-metrics.foxhunt.svc.cluster.local:9090']
relabel_configs:
- target_label: service
replacement: argo-workflows
# Training workflow pods (ephemeral Argo pods on port 9094, Running only)
- job_name: training-pods
kubernetes_sd_configs:
- role: pod
namespaces:
names: [foxhunt]
relabel_configs:
- source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_component]
action: keep
regex: training-workflow|compile-and-train
- source_labels: [__meta_kubernetes_pod_phase]
action: keep
regex: Running
- source_labels: [__meta_kubernetes_pod_ip]
target_label: __address__
replacement: ${1}:9094
- source_labels: [__meta_kubernetes_pod_name]
target_label: pod
- source_labels: [__meta_kubernetes_pod_label_workflows_argoproj_io_workflow]
target_label: workflow
# QuestDB (ML metrics store)
- job_name: questdb
metrics_path: /metrics
static_configs:
- targets: ['questdb.foxhunt.svc.cluster.local:9003']
relabel_configs:
- target_label: service
replacement: questdb
# Pushgateway (training job push metrics)
- job_name: pushgateway
honor_labels: true
static_configs:
- targets: ['pushgateway.foxhunt.svc.cluster.local:9091']
# Loki metrics
- job_name: loki
static_configs:
- targets: ['loki.foxhunt.svc.cluster.local:3100']
# Tempo metrics
- job_name: tempo
static_configs:
- targets: ['tempo.foxhunt.svc.cluster.local:3200']
# ─────────────────────────────────────────────
# AlertManager
# ─────────────────────────────────────────────
alertmanager:
alertmanagerSpec:
nodeSelector:
k8s.scaleway.com/pool-name: platform
replicas: 1
resources:
requests:
cpu: 25m
memory: 32Mi
limits:
cpu: 100m
memory: 64Mi
# Mount K8s Secrets into AlertManager at /etc/alertmanager/secrets/<name>/
secrets:
- alertmanager-mattermost
config:
global:
# Mattermost incoming webhook (Slack-compatible API).
# Reads URL from K8s Secret: alertmanager-mattermost/webhook-url
# Create with:
# kubectl -n foxhunt create secret generic alertmanager-mattermost \
# --from-literal=webhook-url='http://mattermost.foxhunt.svc.cluster.local:8065/hooks/<HOOK_ID>'
slack_api_url_file: /etc/alertmanager/secrets/alertmanager-mattermost/webhook-url
route:
receiver: default
group_by: ['alertname', 'severity', 'service']
group_wait: 5s
group_interval: 30s
repeat_interval: 4h
routes:
# CRITICAL trading/risk → immediate Slack + PagerDuty
- matchers:
- severity="critical"
- component=~"trading_engine|risk_management|trading|risk"
receiver: critical-trading
group_wait: 0s
group_interval: 5s
repeat_interval: 5m
continue: true
# CRITICAL infrastructure → Slack critical channel
- matchers:
- severity="critical"
receiver: critical-infra
group_wait: 0s
group_interval: 10s
repeat_interval: 15m
# HIGH priority → Slack alerts channel
- matchers:
- severity="high"
receiver: high-priority
group_wait: 10s
group_interval: 1m
repeat_interval: 30m
# WARNING / MEDIUM → Slack digest
- matchers:
- severity=~"warning|WARNING|medium"
receiver: warning-alerts
group_wait: 30s
group_interval: 5m
repeat_interval: 4h
receivers:
- name: default
slack_configs:
- channel: '#foxhunt-alerts'
title: '{{ .GroupLabels.alertname }}'
text: >-
{{ range .Alerts }}*{{ .Labels.severity | toUpper }}*: {{ .Annotations.summary }}
{{ end }}
send_resolved: true
- name: critical-trading
slack_configs:
- channel: '#foxhunt-critical'
title: 'CRITICAL: {{ .GroupLabels.alertname }}'
text: >-
{{ range .Alerts }}
*Alert*: {{ .Annotations.summary }}
*Impact*: {{ .Annotations.impact }}
*Action*: {{ .Annotations.action }}
{{ end }}
color: 'danger'
send_resolved: true
- name: critical-infra
slack_configs:
- channel: '#foxhunt-critical'
title: 'INFRA CRITICAL: {{ .GroupLabels.alertname }}'
text: >-
{{ range .Alerts }}{{ .Annotations.summary }}
{{ end }}
color: 'danger'
send_resolved: true
- name: high-priority
slack_configs:
- channel: '#foxhunt-alerts'
title: 'HIGH: {{ .GroupLabels.alertname }}'
text: >-
{{ range .Alerts }}{{ .Annotations.summary }}
{{ end }}
color: 'warning'
send_resolved: true
- name: warning-alerts
slack_configs:
- channel: '#foxhunt-alerts'
title: 'WARNING: {{ .GroupLabels.alertname }}'
text: >-
{{ range .Alerts }}{{ .Annotations.summary }}
{{ end }}
color: '#ff9500'
send_resolved: true
inhibit_rules:
# Trading engine down → suppress all trading alerts
- source_matchers:
- alertname="TradingEngineDown"
target_matchers:
- component=~"trading_engine|trading"
equal: ['service']
# Node down → suppress service-level alerts on that node
- source_matchers:
- alertname=~"NodeDown|KubeNodeNotReady"
target_matchers:
- severity=~"high|warning|medium"
equal: ['instance']
# PostgreSQL down → suppress database alerts
- source_matchers:
- alertname="PostgreSQLDown"
target_matchers:
- component="database"
# ─────────────────────────────────────────────
# ServiceMonitors
# ─────────────────────────────────────────────
additionalServiceMonitors:
# All foxhunt microservices (7 services, all expose named port "metrics")
- name: foxhunt-services
selector:
matchLabels:
app.kubernetes.io/part-of: foxhunt
namespaceSelector:
matchNames: [foxhunt]
endpoints:
- port: metrics
interval: 15s
# DCGM Exporter (GPU metrics from kube-system)
- name: dcgm-exporter
selector:
matchExpressions:
- key: app.kubernetes.io/name
operator: In
values: [dcgm-exporter]
namespaceSelector:
matchNames: [kube-system, monitoring, foxhunt]
endpoints:
- port: metrics
interval: 15s
# ─────────────────────────────────────────────
# Grafana (disabled — managed separately via grafana Helm chart)
# ─────────────────────────────────────────────
grafana:
enabled: false
# ─────────────────────────────────────────────
# Node Exporter (replaces standalone node-exporter.yaml)
# ─────────────────────────────────────────────
prometheus-node-exporter:
tolerations:
- effect: NoSchedule
operator: Exists
resources:
requests:
cpu: 10m
memory: 16Mi
limits:
cpu: 50m
memory: 32Mi
# ─────────────────────────────────────────────
# kube-state-metrics (replaces standalone kube-state-metrics.yaml)
# ─────────────────────────────────────────────
kube-state-metrics:
nodeSelector:
k8s.scaleway.com/pool-name: platform
resources:
requests:
cpu: 10m
memory: 32Mi
limits:
cpu: 50m
memory: 64Mi
# ─────────────────────────────────────────────
# Default rules (Kubernetes cluster health alerts included out-of-box)
# ─────────────────────────────────────────────
defaultRules:
create: true
rules:
alertmanager: true
etcd: false # managed by Scaleway Kapsule
configReloaders: true
general: true
k8sContainerCpuUsageSecondsTotal: true
k8sContainerMemoryCache: true
k8sContainerMemoryRss: true
k8sContainerMemorySwap: true
k8sContainerMemoryWorkingSetBytes: true
k8sPodOwner: true
kubeApiserverAvailability: false # managed by Scaleway
kubeApiserverBurnrate: false
kubeApiserverHistogram: false
kubeApiserverSlos: false
kubeControllerManager: false # managed by Scaleway
kubelet: false # kubelet metrics not accessible on Kapsule
kubeProxy: false # managed by Scaleway
kubePrometheusGeneral: true
kubePrometheusNodeRecording: true
kubernetesAbsent: true
kubernetesApps: true
kubernetesResources: true
kubernetesStorage: true
kubernetesSystem: true
kubeScheduler: false # managed by Scaleway
kubeStateMetrics: true
network: true
node: true
nodeExporterAlerting: true
nodeExporterRecording: true
prometheus: true
prometheusOperator: true
# ─────────────────────────────────────────────
# Disable components managed by Scaleway Kapsule
# ─────────────────────────────────────────────
kubeApiServer:
enabled: false
kubeControllerManager:
enabled: false
kubeScheduler:
enabled: false
kubeProxy:
enabled: false
kubelet:
enabled: false # kubelet metrics not accessible on Kapsule
kubeEtcd:
enabled: false
Reference in New Issue View Git Blame Copy Permalink