f3012611f0
Replace standalone Prometheus/node-exporter/kube-state-metrics with kube-prometheus-stack Helm chart. Add ServiceMonitor CRDs, PrometheusRule CRDs (HFT + broker alerts), comprehensive network policies, and wire AlertManager to Mattermost via incoming webhook. - Delete old standalone prometheus.yaml, node-exporter.yaml, kube-state-metrics.yaml - Add prometheus-stack-values.yaml (Helm values with Scaleway Kapsule tuning) - Add service-monitors.yaml (foxhunt-services + dcgm-exporter ServiceMonitors) - Add prometheus-rules-hft.yaml and prometheus-rules-broker.yaml (PrometheusRule CRDs) - Rewrite prometheus network policy for operator-managed pods (DNS, kube-state-metrics, operator, alertmanager sidecar, monitoring namespace for DCGM) - Add alertmanager network policy (Mattermost egress) - Upgrade Grafana: AlertManager datasource, unified alerting, fix nodeSelector - Fix Tempo: tolerations for GPU nodes, resource tuning Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
121 lines
2.6 KiB
YAML
121 lines
2.6 KiB
YAML
# Grafana — Foxhunt monitoring dashboard
|
|
# Accessed via Tailscale proxy on port 3000
|
|
|
|
replicas: 1
|
|
|
|
nodeSelector:
|
|
k8s.scaleway.com/pool-name: platform
|
|
|
|
resources:
|
|
requests:
|
|
cpu: 50m
|
|
memory: 128Mi
|
|
limits:
|
|
cpu: 200m
|
|
memory: 256Mi
|
|
|
|
persistence:
|
|
enabled: false
|
|
|
|
admin:
|
|
existingSecret: grafana-admin
|
|
userKey: admin-user
|
|
passwordKey: admin-password
|
|
|
|
envFromSecrets:
|
|
- name: db-credentials
|
|
|
|
grafana.ini:
|
|
server:
|
|
root_url: https://grafana.fxhnt.ai
|
|
unified_alerting:
|
|
enabled: true
|
|
alerting:
|
|
enabled: false
|
|
database:
|
|
type: postgres
|
|
host: postgres:5432
|
|
name: grafana
|
|
user: foxhunt
|
|
password: $__env{password}
|
|
ssl_mode: disable
|
|
|
|
datasources:
|
|
datasources.yaml:
|
|
apiVersion: 1
|
|
datasources:
|
|
- name: Prometheus
|
|
type: prometheus
|
|
url: http://prometheus-stack-kube-prom-prometheus.foxhunt.svc.cluster.local:9090
|
|
access: proxy
|
|
isDefault: true
|
|
- name: Alertmanager
|
|
type: alertmanager
|
|
uid: alertmanager
|
|
access: proxy
|
|
url: http://prometheus-stack-kube-prom-alertmanager.foxhunt.svc.cluster.local:9093
|
|
isDefault: false
|
|
jsonData:
|
|
implementation: prometheus
|
|
handleGrafanaManagedAlerts: true
|
|
- name: Loki
|
|
type: loki
|
|
access: proxy
|
|
url: http://loki.foxhunt.svc.cluster.local:3100
|
|
isDefault: false
|
|
jsonData:
|
|
derivedFields:
|
|
- datasourceUid: tempo
|
|
matcherRegex: '"trace_id":"(\w+)"'
|
|
name: TraceID
|
|
url: "$${__value.raw}"
|
|
- name: Tempo
|
|
type: tempo
|
|
uid: tempo
|
|
access: proxy
|
|
url: http://tempo.foxhunt.svc.cluster.local:3200
|
|
isDefault: false
|
|
jsonData:
|
|
tracesToLogsV2:
|
|
datasourceUid: loki
|
|
filterByTraceID: true
|
|
nodeGraph:
|
|
enabled: true
|
|
serviceMap:
|
|
datasourceUid: prometheus
|
|
|
|
sidecar:
|
|
dashboards:
|
|
enabled: true
|
|
folder: /tmp/dashboards
|
|
folderAnnotation: grafana_folder
|
|
label: grafana_dashboard
|
|
labelValue: "1"
|
|
searchNamespace: foxhunt
|
|
provider:
|
|
foldersFromFilesStructure: true
|
|
datasources:
|
|
enabled: false
|
|
label: grafana_datasource
|
|
labelValue: ""
|
|
|
|
dashboardProviders:
|
|
dashboardproviders.yaml:
|
|
apiVersion: 1
|
|
providers:
|
|
- name: default
|
|
orgId: 1
|
|
folder: Foxhunt
|
|
type: file
|
|
disableDeletion: false
|
|
editable: true
|
|
options:
|
|
path: /var/lib/grafana/dashboards/default
|
|
|
|
dashboards:
|
|
default:
|
|
node-exporter:
|
|
gnetId: 1860
|
|
revision: 37
|
|
datasource: Prometheus
|