feat(grafana): add Platform Services row to infrastructure dashboard
New row shows GitLab, CI Runners, Monitoring, Collectors, Tailscale, and Platform Total stat panels, plus a pod status table, CPU/memory timeseries, and container restart chart for all non-foxhunt pods. Also fix minio-init-buckets job: remove part-of:foxhunt label (triggers default-deny egress block) and rely on minio NetworkPolicy ingress allowlist for minio-init pods instead. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
@@ -1294,6 +1294,537 @@
|
||||
"legendFormat": "{{ instance }} write"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 40,
|
||||
"type": "row",
|
||||
"title": "Platform Services",
|
||||
"gridPos": {
|
||||
"h": 1,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 52
|
||||
},
|
||||
"collapsed": false,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"id": 41,
|
||||
"type": "stat",
|
||||
"title": "GitLab",
|
||||
"description": "Ready pods: webservice, sidekiq, gitaly, shell, registry, toolbox",
|
||||
"gridPos": {
|
||||
"h": 3,
|
||||
"w": 4,
|
||||
"x": 0,
|
||||
"y": 53
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "red", "value": null },
|
||||
{ "color": "yellow", "value": 4 },
|
||||
{ "color": "green", "value": 6 }
|
||||
]
|
||||
},
|
||||
"mappings": [],
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "value_and_name",
|
||||
"colorMode": "background",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "center"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"expr": "count(kube_pod_status_ready{namespace=\"foxhunt\", condition=\"true\"} * on(pod) group_left() kube_pod_labels{namespace=\"foxhunt\", label_release=\"gitlab\"}) or vector(0)",
|
||||
"legendFormat": "GitLab",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 42,
|
||||
"type": "stat",
|
||||
"title": "CI Runners",
|
||||
"description": "GitLab CI runner manager + active job pods",
|
||||
"gridPos": {
|
||||
"h": 3,
|
||||
"w": 4,
|
||||
"x": 4,
|
||||
"y": 53
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "blue", "value": null },
|
||||
{ "color": "green", "value": 1 }
|
||||
]
|
||||
},
|
||||
"mappings": [],
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "value_and_name",
|
||||
"colorMode": "background",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "center"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"expr": "count(kube_pod_status_ready{namespace=\"foxhunt\", condition=\"true\"} * on(pod) group_left() kube_pod_info{namespace=\"foxhunt\", pod=~\"gitlab-runner.*|runner-.*\"}) or vector(0)",
|
||||
"legendFormat": "Runners",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 43,
|
||||
"type": "stat",
|
||||
"title": "Monitoring",
|
||||
"description": "Grafana, Prometheus, Loki, Tempo, kube-state-metrics",
|
||||
"gridPos": {
|
||||
"h": 3,
|
||||
"w": 4,
|
||||
"x": 8,
|
||||
"y": 53
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "red", "value": null },
|
||||
{ "color": "yellow", "value": 3 },
|
||||
{ "color": "green", "value": 5 }
|
||||
]
|
||||
},
|
||||
"mappings": [],
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "value_and_name",
|
||||
"colorMode": "background",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "center"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"expr": "count(kube_pod_status_ready{namespace=\"foxhunt\", condition=\"true\"} * on(pod) group_left() kube_pod_info{namespace=\"foxhunt\", pod=~\"grafana.*|gitlab-prometheus.*|loki.*|tempo.*|kube-state-metrics.*\"}) or vector(0)",
|
||||
"legendFormat": "Monitoring",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 44,
|
||||
"type": "stat",
|
||||
"title": "Collectors",
|
||||
"description": "Promtail + node-exporter (DaemonSets, scales with nodes)",
|
||||
"gridPos": {
|
||||
"h": 3,
|
||||
"w": 4,
|
||||
"x": 12,
|
||||
"y": 53
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "red", "value": null },
|
||||
{ "color": "green", "value": 4 }
|
||||
]
|
||||
},
|
||||
"mappings": [],
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "value_and_name",
|
||||
"colorMode": "background",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "center"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"expr": "count(kube_pod_status_ready{namespace=\"foxhunt\", condition=\"true\"} * on(pod) group_left() kube_pod_info{namespace=\"foxhunt\", pod=~\"promtail.*|node-exporter.*\"}) or vector(0)",
|
||||
"legendFormat": "Collectors",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 45,
|
||||
"type": "stat",
|
||||
"title": "Tailscale",
|
||||
"description": "Tailscale proxy (VPN ingress)",
|
||||
"gridPos": {
|
||||
"h": 3,
|
||||
"w": 4,
|
||||
"x": 16,
|
||||
"y": 53
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "red", "value": null },
|
||||
{ "color": "green", "value": 1 }
|
||||
]
|
||||
},
|
||||
"mappings": [],
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "value_and_name",
|
||||
"colorMode": "background",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "center"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"expr": "count(kube_pod_status_ready{namespace=\"foxhunt\", condition=\"true\"} * on(pod) group_left() kube_pod_info{namespace=\"foxhunt\", pod=~\"tailscale.*\"}) or vector(0)",
|
||||
"legendFormat": "Tailscale",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 46,
|
||||
"type": "stat",
|
||||
"title": "Platform Total",
|
||||
"description": "All non-foxhunt pods (Running & Ready)",
|
||||
"gridPos": {
|
||||
"h": 3,
|
||||
"w": 4,
|
||||
"x": 20,
|
||||
"y": 53
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "blue", "value": null }
|
||||
]
|
||||
},
|
||||
"mappings": [],
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "center"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"expr": "count(kube_pod_status_ready{namespace=\"foxhunt\", condition=\"true\"} * on(pod) group_left() kube_pod_info{namespace=\"foxhunt\", pod=~\"gitlab-.*|grafana.*|loki.*|tempo.*|promtail.*|node-exporter.*|kube-state-metrics.*|tailscale.*|runner-.*\"}) or vector(0)",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 47,
|
||||
"type": "table",
|
||||
"title": "Platform Pod Status",
|
||||
"description": "All non-foxhunt service pods: GitLab, monitoring, CI runners, networking",
|
||||
"gridPos": {
|
||||
"h": 10,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 56
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null }
|
||||
]
|
||||
},
|
||||
"mappings": [
|
||||
{
|
||||
"type": "value",
|
||||
"options": {
|
||||
"1": { "text": "Ready", "color": "green" },
|
||||
"0": { "text": "Not Ready", "color": "red" }
|
||||
}
|
||||
}
|
||||
],
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": { "id": "byName", "options": "Value" },
|
||||
"properties": [
|
||||
{ "id": "custom.displayMode", "value": "color-background-solid" },
|
||||
{
|
||||
"id": "thresholds",
|
||||
"value": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "red", "value": null },
|
||||
{ "color": "green", "value": 1 }
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"showHeader": true,
|
||||
"sortBy": [
|
||||
{ "displayName": "pod", "desc": false }
|
||||
],
|
||||
"footer": { "show": false }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"expr": "kube_pod_status_ready{namespace=\"foxhunt\", condition=\"true\"} * on(pod) group_left() kube_pod_info{namespace=\"foxhunt\", pod=~\"gitlab-.*|grafana.*|loki.*|tempo.*|promtail.*|node-exporter.*|kube-state-metrics.*|tailscale.*|runner-.*\"}",
|
||||
"legendFormat": "{{ pod }}",
|
||||
"instant": true,
|
||||
"format": "table"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 48,
|
||||
"type": "timeseries",
|
||||
"title": "Platform CPU Usage",
|
||||
"description": "CPU usage by platform service pods (GitLab, monitoring, runners)",
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 66
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"fillOpacity": 15,
|
||||
"gradientMode": "scheme",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"lineWidth": 1
|
||||
},
|
||||
"unit": "short",
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [{ "color": "green", "value": null }]
|
||||
},
|
||||
"color": { "mode": "palette-classic" },
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"calcs": ["mean", "max"]
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"gitlab-webservice.*|gitlab-sidekiq.*|gitlab-gitaly.*|gitlab-registry.*|grafana.*|loki.*|tempo.*\"}[$interval]))",
|
||||
"legendFormat": "{{ pod }}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 49,
|
||||
"type": "timeseries",
|
||||
"title": "Platform Memory Usage",
|
||||
"description": "Memory working set by platform service pods",
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 66
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"fillOpacity": 15,
|
||||
"gradientMode": "scheme",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"lineWidth": 1
|
||||
},
|
||||
"unit": "bytes",
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [{ "color": "green", "value": null }]
|
||||
},
|
||||
"color": { "mode": "palette-classic" },
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"calcs": ["mean", "max"]
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"expr": "sum by (pod) (container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"gitlab-webservice.*|gitlab-sidekiq.*|gitlab-gitaly.*|gitlab-registry.*|grafana.*|loki.*|tempo.*\"})",
|
||||
"legendFormat": "{{ pod }}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 50,
|
||||
"type": "timeseries",
|
||||
"title": "Container Restarts (Platform)",
|
||||
"description": "Restart rate for platform pods — spikes indicate instability",
|
||||
"gridPos": {
|
||||
"h": 7,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 74
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "bars",
|
||||
"lineInterpolation": "linear",
|
||||
"fillOpacity": 80,
|
||||
"gradientMode": "none",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"stacking": { "mode": "normal", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"lineWidth": 1
|
||||
},
|
||||
"unit": "short",
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [{ "color": "green", "value": null }]
|
||||
},
|
||||
"color": { "mode": "palette-classic" },
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"calcs": ["sum"]
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
||||
"expr": "increase(kube_pod_container_status_restarts_total{namespace=\"foxhunt\", pod=~\"gitlab-.*|grafana.*|loki.*|tempo.*|promtail.*|kube-state-metrics.*|tailscale.*|runner-.*\"}[30m]) > 0",
|
||||
"legendFormat": "{{ pod }} ({{ container }})"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user