From 3c09874ffae0de0a3e041e5f117ebc2a3c054171 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Mon, 2 Mar 2026 00:42:51 +0100 Subject: [PATCH] feat(grafana): add Platform Services row to infrastructure dashboard New row shows GitLab, CI Runners, Monitoring, Collectors, Tailscale, and Platform Total stat panels, plus a pod status table, CPU/memory timeseries, and container restart chart for all non-foxhunt pods. Also fix minio-init-buckets job: remove part-of:foxhunt label (triggers default-deny egress block) and rely on minio NetworkPolicy ingress allowlist for minio-init pods instead. Co-Authored-By: Claude Opus 4.6 --- .../dashboards/foxhunt-infrastructure.json | 531 ++++++++++++++++++ 1 file changed, 531 insertions(+) diff --git a/infra/k8s/monitoring/dashboards/foxhunt-infrastructure.json b/infra/k8s/monitoring/dashboards/foxhunt-infrastructure.json index 25f28699e..042f8a499 100644 --- a/infra/k8s/monitoring/dashboards/foxhunt-infrastructure.json +++ b/infra/k8s/monitoring/dashboards/foxhunt-infrastructure.json @@ -1294,6 +1294,537 @@ "legendFormat": "{{ instance }} write" } ] + }, + { + "id": 40, + "type": "row", + "title": "Platform Services", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 52 + }, + "collapsed": false, + "panels": [] + }, + { + "id": 41, + "type": "stat", + "title": "GitLab", + "description": "Ready pods: webservice, sidekiq, gitaly, shell, registry, toolbox", + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 53 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "yellow", "value": 4 }, + { "color": "green", "value": 6 } + ] + }, + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "value_and_name", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "count(kube_pod_status_ready{namespace=\"foxhunt\", condition=\"true\"} * on(pod) group_left() kube_pod_labels{namespace=\"foxhunt\", label_release=\"gitlab\"}) or vector(0)", + "legendFormat": "GitLab", + "instant": true + } + ] + }, + { + "id": 42, + "type": "stat", + "title": "CI Runners", + "description": "GitLab CI runner manager + active job pods", + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 53 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "blue", "value": null }, + { "color": "green", "value": 1 } + ] + }, + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "value_and_name", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "count(kube_pod_status_ready{namespace=\"foxhunt\", condition=\"true\"} * on(pod) group_left() kube_pod_info{namespace=\"foxhunt\", pod=~\"gitlab-runner.*|runner-.*\"}) or vector(0)", + "legendFormat": "Runners", + "instant": true + } + ] + }, + { + "id": 43, + "type": "stat", + "title": "Monitoring", + "description": "Grafana, Prometheus, Loki, Tempo, kube-state-metrics", + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 53 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "yellow", "value": 3 }, + { "color": "green", "value": 5 } + ] + }, + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "value_and_name", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "count(kube_pod_status_ready{namespace=\"foxhunt\", condition=\"true\"} * on(pod) group_left() kube_pod_info{namespace=\"foxhunt\", pod=~\"grafana.*|gitlab-prometheus.*|loki.*|tempo.*|kube-state-metrics.*\"}) or vector(0)", + "legendFormat": "Monitoring", + "instant": true + } + ] + }, + { + "id": 44, + "type": "stat", + "title": "Collectors", + "description": "Promtail + node-exporter (DaemonSets, scales with nodes)", + "gridPos": { + "h": 3, + "w": 4, + "x": 12, + "y": 53 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "green", "value": 4 } + ] + }, + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "value_and_name", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "count(kube_pod_status_ready{namespace=\"foxhunt\", condition=\"true\"} * on(pod) group_left() kube_pod_info{namespace=\"foxhunt\", pod=~\"promtail.*|node-exporter.*\"}) or vector(0)", + "legendFormat": "Collectors", + "instant": true + } + ] + }, + { + "id": 45, + "type": "stat", + "title": "Tailscale", + "description": "Tailscale proxy (VPN ingress)", + "gridPos": { + "h": 3, + "w": 4, + "x": 16, + "y": 53 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "green", "value": 1 } + ] + }, + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "value_and_name", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "count(kube_pod_status_ready{namespace=\"foxhunt\", condition=\"true\"} * on(pod) group_left() kube_pod_info{namespace=\"foxhunt\", pod=~\"tailscale.*\"}) or vector(0)", + "legendFormat": "Tailscale", + "instant": true + } + ] + }, + { + "id": 46, + "type": "stat", + "title": "Platform Total", + "description": "All non-foxhunt pods (Running & Ready)", + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 53 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "blue", "value": null } + ] + }, + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "count(kube_pod_status_ready{namespace=\"foxhunt\", condition=\"true\"} * on(pod) group_left() kube_pod_info{namespace=\"foxhunt\", pod=~\"gitlab-.*|grafana.*|loki.*|tempo.*|promtail.*|node-exporter.*|kube-state-metrics.*|tailscale.*|runner-.*\"}) or vector(0)", + "instant": true + } + ] + }, + { + "id": 47, + "type": "table", + "title": "Platform Pod Status", + "description": "All non-foxhunt service pods: GitLab, monitoring, CI runners, networking", + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 56 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "mappings": [ + { + "type": "value", + "options": { + "1": { "text": "Ready", "color": "green" }, + "0": { "text": "Not Ready", "color": "red" } + } + } + ], + "color": { "mode": "thresholds" } + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "Value" }, + "properties": [ + { "id": "custom.displayMode", "value": "color-background-solid" }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "green", "value": 1 } + ] + } + } + ] + } + ] + }, + "options": { + "showHeader": true, + "sortBy": [ + { "displayName": "pod", "desc": false } + ], + "footer": { "show": false } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "kube_pod_status_ready{namespace=\"foxhunt\", condition=\"true\"} * on(pod) group_left() kube_pod_info{namespace=\"foxhunt\", pod=~\"gitlab-.*|grafana.*|loki.*|tempo.*|promtail.*|node-exporter.*|kube-state-metrics.*|tailscale.*|runner-.*\"}", + "legendFormat": "{{ pod }}", + "instant": true, + "format": "table" + } + ] + }, + { + "id": 48, + "type": "timeseries", + "title": "Platform CPU Usage", + "description": "CPU usage by platform service pods (GitLab, monitoring, runners)", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 66 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 15, + "gradientMode": "scheme", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "none", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 1 + }, + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [{ "color": "green", "value": null }] + }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": ["mean", "max"] + } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"gitlab-webservice.*|gitlab-sidekiq.*|gitlab-gitaly.*|gitlab-registry.*|grafana.*|loki.*|tempo.*\"}[$interval]))", + "legendFormat": "{{ pod }}" + } + ] + }, + { + "id": 49, + "type": "timeseries", + "title": "Platform Memory Usage", + "description": "Memory working set by platform service pods", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 66 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 15, + "gradientMode": "scheme", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "none", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 1 + }, + "unit": "bytes", + "thresholds": { + "mode": "absolute", + "steps": [{ "color": "green", "value": null }] + }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": ["mean", "max"] + } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "sum by (pod) (container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"gitlab-webservice.*|gitlab-sidekiq.*|gitlab-gitaly.*|gitlab-registry.*|grafana.*|loki.*|tempo.*\"})", + "legendFormat": "{{ pod }}" + } + ] + }, + { + "id": 50, + "type": "timeseries", + "title": "Container Restarts (Platform)", + "description": "Restart rate for platform pods — spikes indicate instability", + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 74 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "bars", + "lineInterpolation": "linear", + "fillOpacity": 80, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "normal", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 1 + }, + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [{ "color": "green", "value": null }] + }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": ["sum"] + } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "increase(kube_pod_container_status_restarts_total{namespace=\"foxhunt\", pod=~\"gitlab-.*|grafana.*|loki.*|tempo.*|promtail.*|kube-state-metrics.*|tailscale.*|runner-.*\"}[30m]) > 0", + "legendFormat": "{{ pod }} ({{ container }})" + } + ] } ] }