From 4c54fe0e377a1b5ac2a2bc5ee9d85e270c3365c4 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Fri, 27 Feb 2026 01:22:24 +0100 Subject: [PATCH] feat(infra): add Cockpit Grafana dashboards for K8s, GPU, and GitLab - cluster-overview: nodes, CPU, memory, disk, network per node/namespace - gpu-overview: DCGM utilization, VRAM, temperature, power, per-pod - gitlab-services: puma, sidekiq, gitaly + foxhunt service resource usage - import.sh: one-shot script to push all dashboards via Grafana API Co-Authored-By: Claude Opus 4.6 --- .../dashboards/cluster-overview.json | 692 ++++++++++++++++++ .../dashboards/gitlab-services.json | 584 +++++++++++++++ .../monitoring/dashboards/gpu-overview.json | 495 +++++++++++++ infra/k8s/monitoring/dashboards/import.sh | 76 ++ 4 files changed, 1847 insertions(+) create mode 100644 infra/k8s/monitoring/dashboards/cluster-overview.json create mode 100644 infra/k8s/monitoring/dashboards/gitlab-services.json create mode 100644 infra/k8s/monitoring/dashboards/gpu-overview.json create mode 100755 infra/k8s/monitoring/dashboards/import.sh diff --git a/infra/k8s/monitoring/dashboards/cluster-overview.json b/infra/k8s/monitoring/dashboards/cluster-overview.json new file mode 100644 index 000000000..24b74b30c --- /dev/null +++ b/infra/k8s/monitoring/dashboards/cluster-overview.json @@ -0,0 +1,692 @@ +{ + "id": null, + "uid": "foxhunt-cluster-overview", + "title": "K8s Cluster Overview", + "description": "Kubernetes cluster health: nodes, pods, CPU, memory, disk, and network", + "tags": ["kubernetes", "foxhunt", "cluster"], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "refresh": "30s", + "time": { + "from": "now-1h", + "to": "now" + }, + "fiscalYearStartMonth": 0, + "liveNow": false, + "weekStart": "", + "templating": { + "list": [ + { + "name": "namespace", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "cfegaydmscvswe" + }, + "query": "label_values(kube_pod_info, namespace)", + "refresh": 2, + "includeAll": true, + "multi": true, + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "sort": 1 + }, + { + "name": "node", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "cfegaydmscvswe" + }, + "query": "label_values(node_uname_info, nodename)", + "refresh": 2, + "includeAll": true, + "multi": true, + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "sort": 1 + } + ] + }, + "panels": [ + { + "id": 1, + "type": "row", + "title": "Cluster Summary", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "collapsed": false, + "panels": [] + }, + { + "id": 2, + "type": "stat", + "title": "Total Nodes", + "gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "green", "value": 1 } + ] + }, + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "count(kube_node_info)", + "legendFormat": "Nodes", + "instant": true + } + ] + }, + { + "id": 3, + "type": "stat", + "title": "Pods Running", + "gridPos": { "h": 4, "w": 4, "x": 4, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "sum(kube_pod_status_phase{namespace=~\"$namespace\", phase=\"Running\"})", + "legendFormat": "Running", + "instant": true + } + ] + }, + { + "id": 4, + "type": "stat", + "title": "Pods Pending", + "gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 1 }, + { "color": "red", "value": 5 } + ] + }, + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "sum(kube_pod_status_phase{namespace=~\"$namespace\", phase=\"Pending\"}) or vector(0)", + "legendFormat": "Pending", + "instant": true + } + ] + }, + { + "id": 5, + "type": "stat", + "title": "Pods Failed", + "gridPos": { "h": 4, "w": 4, "x": 12, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "red", "value": 1 } + ] + }, + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "sum(kube_pod_status_phase{namespace=~\"$namespace\", phase=\"Failed\"}) or vector(0)", + "legendFormat": "Failed", + "instant": true + } + ] + }, + { + "id": 6, + "type": "gauge", + "title": "Cluster CPU Usage %", + "gridPos": { "h": 4, "w": 4, "x": 16, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 70 }, + { "color": "red", "value": 90 } + ] + }, + "min": 0, + "max": 100, + "unit": "percent", + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "orientation": "auto" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "100 - (avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m])) * 100)", + "legendFormat": "CPU %", + "instant": true + } + ] + }, + { + "id": 7, + "type": "gauge", + "title": "Cluster Memory Usage %", + "gridPos": { "h": 4, "w": 4, "x": 20, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 70 }, + { "color": "red", "value": 90 } + ] + }, + "min": 0, + "max": 100, + "unit": "percent", + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "orientation": "auto" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "(1 - (sum(node_memory_MemAvailable_bytes) / sum(node_memory_MemTotal_bytes))) * 100", + "legendFormat": "Memory %", + "instant": true + } + ] + }, + { + "id": 10, + "type": "row", + "title": "CPU", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, + "collapsed": false, + "panels": [] + }, + { + "id": 11, + "type": "timeseries", + "title": "Per-Node CPU Usage", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 }, + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 10, + "gradientMode": "scheme", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "none", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 1 + }, + "unit": "percentunit", + "min": 0, + "max": 1, + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))", + "legendFormat": "{{ instance }}" + } + ] + }, + { + "id": 12, + "type": "timeseries", + "title": "CPU Requests vs Limits vs Usage", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 }, + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 10, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "none", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 1 + }, + "unit": "short", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "sum(kube_pod_container_resource_requests{namespace=~\"$namespace\", resource=\"cpu\"})", + "legendFormat": "Requests" + }, + { + "refId": "B", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "sum(kube_pod_container_resource_limits{namespace=~\"$namespace\", resource=\"cpu\"})", + "legendFormat": "Limits" + }, + { + "refId": "C", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\", image!=\"\"}[5m]))", + "legendFormat": "Usage" + } + ] + }, + { + "id": 20, + "type": "row", + "title": "Memory", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 }, + "collapsed": false, + "panels": [] + }, + { + "id": 21, + "type": "timeseries", + "title": "Per-Node Memory Usage", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 15 }, + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 10, + "gradientMode": "scheme", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "none", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 1 + }, + "unit": "bytes", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes", + "legendFormat": "{{ instance }}" + } + ] + }, + { + "id": 22, + "type": "timeseries", + "title": "Memory Requests vs Limits vs Usage", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 15 }, + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 10, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "none", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 1 + }, + "unit": "bytes", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "sum(kube_pod_container_resource_requests{namespace=~\"$namespace\", resource=\"memory\"})", + "legendFormat": "Requests" + }, + { + "refId": "B", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "sum(kube_pod_container_resource_limits{namespace=~\"$namespace\", resource=\"memory\"})", + "legendFormat": "Limits" + }, + { + "refId": "C", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "sum(container_memory_working_set_bytes{namespace=~\"$namespace\", image!=\"\"})", + "legendFormat": "Usage" + } + ] + }, + { + "id": 30, + "type": "row", + "title": "Disk", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 }, + "collapsed": false, + "panels": [] + }, + { + "id": 31, + "type": "timeseries", + "title": "Node Filesystem Usage", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 }, + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 10, + "gradientMode": "scheme", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "none", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 1 + }, + "unit": "percentunit", + "min": 0, + "max": 1, + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "1 - (node_filesystem_avail_bytes{mountpoint=\"/\", fstype!=\"tmpfs\"} / node_filesystem_size_bytes{mountpoint=\"/\", fstype!=\"tmpfs\"})", + "legendFormat": "{{ instance }}" + } + ] + }, + { + "id": 32, + "type": "timeseries", + "title": "Disk I/O Throughput", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 }, + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 10, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "none", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 1 + }, + "unit": "Bps", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "sum by (instance) (rate(node_disk_read_bytes_total[5m]))", + "legendFormat": "{{ instance }} read" + }, + { + "refId": "B", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "sum by (instance) (rate(node_disk_written_bytes_total[5m]))", + "legendFormat": "{{ instance }} write" + } + ] + }, + { + "id": 40, + "type": "row", + "title": "Network", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 32 }, + "collapsed": false, + "panels": [] + }, + { + "id": 41, + "type": "timeseries", + "title": "Node Network Receive / Transmit", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 33 }, + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 10, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "none", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 1 + }, + "unit": "Bps", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "sum by (instance) (rate(node_network_receive_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[5m]))", + "legendFormat": "{{ instance }} rx" + }, + { + "refId": "B", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "sum by (instance) (rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[5m]))", + "legendFormat": "{{ instance }} tx" + } + ] + }, + { + "id": 42, + "type": "timeseries", + "title": "Pod Network I/O", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 33 }, + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 10, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "normal", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 1 + }, + "unit": "Bps", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "sum by (namespace) (rate(container_network_receive_bytes_total{namespace=~\"$namespace\", image!=\"\"}[5m]))", + "legendFormat": "{{ namespace }} rx" + }, + { + "refId": "B", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "sum by (namespace) (rate(container_network_transmit_bytes_total{namespace=~\"$namespace\", image!=\"\"}[5m]))", + "legendFormat": "{{ namespace }} tx" + } + ] + } + ] +} \ No newline at end of file diff --git a/infra/k8s/monitoring/dashboards/gitlab-services.json b/infra/k8s/monitoring/dashboards/gitlab-services.json new file mode 100644 index 000000000..95aa33a72 --- /dev/null +++ b/infra/k8s/monitoring/dashboards/gitlab-services.json @@ -0,0 +1,584 @@ +{ + "id": null, + "uid": "foxhunt-gitlab-services", + "title": "GitLab + Foxhunt Services", + "description": "GitLab internals (Puma, Sidekiq, Gitaly) and Foxhunt microservice health", + "tags": ["gitlab", "foxhunt", "services", "kubernetes"], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "refresh": "30s", + "time": { + "from": "now-1h", + "to": "now" + }, + "fiscalYearStartMonth": 0, + "liveNow": false, + "weekStart": "", + "templating": { + "list": [ + { + "name": "namespace", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "cfegaydmscvswe" + }, + "query": "label_values(kube_pod_info, namespace)", + "refresh": 2, + "includeAll": true, + "multi": true, + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "sort": 1 + } + ] + }, + "panels": [ + { + "id": 1, + "type": "row", + "title": "GitLab - Puma", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "collapsed": false, + "panels": [] + }, + { + "id": 2, + "type": "timeseries", + "title": "Puma HTTP Request Rate", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 15, + "gradientMode": "scheme", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "none", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 1 + }, + "unit": "reqps", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "sum(rate(http_requests_total{job=~\".*puma.*\"}[5m])) by (method)", + "legendFormat": "{{ method }}" + } + ] + }, + { + "id": 3, + "type": "timeseries", + "title": "Puma Request Duration (p95)", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 10, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "none", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 1 + }, + "unit": "s", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket{job=~\".*puma.*\"}[5m])) by (le))", + "legendFormat": "p95" + }, + { + "refId": "B", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "histogram_quantile(0.50, sum(rate(http_request_duration_seconds_bucket{job=~\".*puma.*\"}[5m])) by (le))", + "legendFormat": "p50" + } + ] + }, + { + "id": 10, + "type": "row", + "title": "GitLab - Sidekiq", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 9 }, + "collapsed": false, + "panels": [] + }, + { + "id": 11, + "type": "timeseries", + "title": "Sidekiq Job Processing Rate", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 10 }, + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 15, + "gradientMode": "scheme", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "none", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 1 + }, + "unit": "ops", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "sum(rate(sidekiq_jobs_processed_total[5m]))", + "legendFormat": "Processed" + }, + { + "refId": "B", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "sum(rate(sidekiq_jobs_failed_total[5m]))", + "legendFormat": "Failed" + } + ] + }, + { + "id": 12, + "type": "stat", + "title": "Sidekiq Queue Size", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 10 }, + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 100 }, + { "color": "red", "value": 500 } + ] + }, + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "sum(sidekiq_queue_size)", + "legendFormat": "Queue Size", + "instant": true + } + ] + }, + { + "id": 20, + "type": "row", + "title": "GitLab - Gitaly", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 18 }, + "collapsed": false, + "panels": [] + }, + { + "id": 21, + "type": "timeseries", + "title": "Gitaly RPC Latency (p95)", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 19 }, + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 10, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "none", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 1 + }, + "unit": "s", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "histogram_quantile(0.95, sum(rate(grpc_server_handling_seconds_bucket{job=~\".*gitaly.*\"}[5m])) by (le, grpc_method))", + "legendFormat": "{{ grpc_method }}" + } + ] + }, + { + "id": 22, + "type": "timeseries", + "title": "Gitaly RPC Rate", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 19 }, + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 15, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "normal", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 1 + }, + "unit": "reqps", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "sum(rate(grpc_server_handled_total{job=~\".*gitaly.*\"}[5m])) by (grpc_code)", + "legendFormat": "{{ grpc_code }}" + } + ] + }, + { + "id": 30, + "type": "row", + "title": "Foxhunt Services - Resource Usage", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 27 }, + "collapsed": false, + "panels": [] + }, + { + "id": 31, + "type": "timeseries", + "title": "Container CPU Usage by Service", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 28 }, + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 15, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "none", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 1 + }, + "unit": "short", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "sum by (container) (rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\", container=~\"web-gateway|api-gateway|trading-service|ml-training-service|backtesting-service|data-acquisition-service|broker-gateway-service|trading-agent-service\", image!=\"\"}[5m]))", + "legendFormat": "{{ container }}" + } + ] + }, + { + "id": 32, + "type": "timeseries", + "title": "Container Memory Usage by Service", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 28 }, + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 15, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "none", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 1 + }, + "unit": "bytes", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "sum by (container) (container_memory_working_set_bytes{namespace=~\"$namespace\", container=~\"web-gateway|api-gateway|trading-service|ml-training-service|backtesting-service|data-acquisition-service|broker-gateway-service|trading-agent-service\", image!=\"\"})", + "legendFormat": "{{ container }}" + } + ] + }, + { + "id": 40, + "type": "row", + "title": "Foxhunt Services - Pod Health", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 36 }, + "collapsed": false, + "panels": [] + }, + { + "id": 41, + "type": "timeseries", + "title": "Pod Restarts (by Container)", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 37 }, + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "bars", + "lineInterpolation": "linear", + "fillOpacity": 80, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "normal", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 1 + }, + "unit": "short", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["sum", "max"] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "increase(kube_pod_container_status_restarts_total{namespace=~\"$namespace\"}[1h])", + "legendFormat": "{{ namespace }}/{{ pod }} ({{ container }})" + } + ] + }, + { + "id": 42, + "type": "table", + "title": "Pod Status by Namespace", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 37 }, + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "fieldConfig": { + "defaults": { + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "Failed" }, + "properties": [ + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "red", "value": 1 } + ] + } + }, + { "id": "custom.displayMode", "value": "color-background" } + ] + }, + { + "matcher": { "id": "byName", "options": "Pending" }, + "properties": [ + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 1 } + ] + } + }, + { "id": "custom.displayMode", "value": "color-background" } + ] + } + ] + }, + "options": { + "showHeader": true, + "sortBy": [{ "displayName": "namespace", "desc": false }], + "footer": { "show": true, "reducer": ["sum"], "fields": "" } + }, + "transformations": [ + { + "id": "seriesToColumns", + "options": { "byField": "namespace" } + } + ], + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "sum by (namespace) (kube_pod_status_phase{namespace=~\"$namespace\", phase=\"Running\"})", + "legendFormat": "Running", + "instant": true, + "format": "table" + }, + { + "refId": "B", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "sum by (namespace) (kube_pod_status_phase{namespace=~\"$namespace\", phase=\"Pending\"})", + "legendFormat": "Pending", + "instant": true, + "format": "table" + }, + { + "refId": "C", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "sum by (namespace) (kube_pod_status_phase{namespace=~\"$namespace\", phase=\"Failed\"})", + "legendFormat": "Failed", + "instant": true, + "format": "table" + } + ] + }, + { + "id": 50, + "type": "row", + "title": "Kubernetes - Deployments", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 45 }, + "collapsed": false, + "panels": [] + }, + { + "id": 51, + "type": "timeseries", + "title": "Deployment Replicas: Desired vs Available", + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 46 }, + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "stepAfter", + "fillOpacity": 0, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "none", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 2 + }, + "unit": "short", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["last"] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "kube_deployment_spec_replicas{namespace=~\"$namespace\"}", + "legendFormat": "{{ deployment }} desired" + }, + { + "refId": "B", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "kube_deployment_status_replicas_available{namespace=~\"$namespace\"}", + "legendFormat": "{{ deployment }} available" + } + ] + } + ] +} \ No newline at end of file diff --git a/infra/k8s/monitoring/dashboards/gpu-overview.json b/infra/k8s/monitoring/dashboards/gpu-overview.json new file mode 100644 index 000000000..8d51efd5c --- /dev/null +++ b/infra/k8s/monitoring/dashboards/gpu-overview.json @@ -0,0 +1,495 @@ +{ + "id": null, + "uid": "foxhunt-gpu-overview", + "title": "GPU Overview", + "description": "NVIDIA GPU metrics from DCGM exporter: utilization, memory, temperature, power", + "tags": ["gpu", "nvidia", "dcgm", "foxhunt"], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "refresh": "15s", + "time": { + "from": "now-1h", + "to": "now" + }, + "fiscalYearStartMonth": 0, + "liveNow": false, + "weekStart": "", + "templating": { + "list": [ + { + "name": "gpu", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "cfegaydmscvswe" + }, + "query": "label_values(DCGM_FI_DEV_GPU_UTIL, gpu)", + "refresh": 2, + "includeAll": true, + "multi": true, + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "sort": 1 + }, + { + "name": "instance", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "cfegaydmscvswe" + }, + "query": "label_values(DCGM_FI_DEV_GPU_UTIL, instance)", + "refresh": 2, + "includeAll": true, + "multi": true, + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "sort": 1 + }, + { + "name": "namespace", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "cfegaydmscvswe" + }, + "query": "label_values(DCGM_FI_DEV_GPU_UTIL, exported_namespace)", + "refresh": 2, + "includeAll": true, + "multi": true, + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "sort": 1 + } + ] + }, + "panels": [ + { + "id": 1, + "type": "row", + "title": "GPU Utilization", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "collapsed": false, + "panels": [] + }, + { + "id": 2, + "type": "timeseries", + "title": "GPU Utilization %", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 20, + "gradientMode": "scheme", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "none", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 2 + }, + "unit": "percent", + "min": 0, + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 80 }, + { "color": "red", "value": 95 } + ] + }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "DCGM_FI_DEV_GPU_UTIL{gpu=~\"$gpu\", instance=~\"$instance\"}", + "legendFormat": "GPU {{ gpu }} ({{ instance }})" + } + ] + }, + { + "id": 3, + "type": "gauge", + "title": "Current GPU Utilization", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "blue", "value": null }, + { "color": "green", "value": 20 }, + { "color": "yellow", "value": 80 }, + { "color": "red", "value": 95 } + ] + }, + "min": 0, + "max": 100, + "unit": "percent", + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "orientation": "auto" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "DCGM_FI_DEV_GPU_UTIL{gpu=~\"$gpu\", instance=~\"$instance\"}", + "legendFormat": "GPU {{ gpu }}", + "instant": true + } + ] + }, + { + "id": 10, + "type": "row", + "title": "GPU Memory", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 9 }, + "collapsed": false, + "panels": [] + }, + { + "id": 11, + "type": "timeseries", + "title": "GPU Memory Used", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 10 }, + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 20, + "gradientMode": "scheme", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "none", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 2 + }, + "unit": "decmbytes", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "DCGM_FI_DEV_FB_USED{gpu=~\"$gpu\", instance=~\"$instance\"}", + "legendFormat": "GPU {{ gpu }} used" + }, + { + "refId": "B", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "DCGM_FI_DEV_FB_TOTAL{gpu=~\"$gpu\", instance=~\"$instance\"}", + "legendFormat": "GPU {{ gpu }} total" + } + ] + }, + { + "id": 12, + "type": "gauge", + "title": "GPU Memory Usage %", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 10 }, + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 70 }, + { "color": "red", "value": 90 } + ] + }, + "min": 0, + "max": 100, + "unit": "percent", + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "orientation": "auto" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "(DCGM_FI_DEV_FB_USED{gpu=~\"$gpu\", instance=~\"$instance\"} / DCGM_FI_DEV_FB_TOTAL{gpu=~\"$gpu\", instance=~\"$instance\"}) * 100", + "legendFormat": "GPU {{ gpu }}", + "instant": true + } + ] + }, + { + "id": 20, + "type": "row", + "title": "GPU Temperature & Power", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 18 }, + "collapsed": false, + "panels": [] + }, + { + "id": 21, + "type": "timeseries", + "title": "GPU Temperature", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 19 }, + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 10, + "gradientMode": "scheme", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "none", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 2 + }, + "unit": "celsius", + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "blue", "value": null }, + { "color": "green", "value": 40 }, + { "color": "yellow", "value": 75 }, + { "color": "red", "value": 85 } + ] + }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "DCGM_FI_DEV_GPU_TEMP{gpu=~\"$gpu\", instance=~\"$instance\"}", + "legendFormat": "GPU {{ gpu }} ({{ instance }})" + } + ] + }, + { + "id": 22, + "type": "timeseries", + "title": "GPU Power Usage", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 19 }, + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 10, + "gradientMode": "scheme", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "none", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 2 + }, + "unit": "watt", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "DCGM_FI_DEV_POWER_USAGE{gpu=~\"$gpu\", instance=~\"$instance\"}", + "legendFormat": "GPU {{ gpu }} ({{ instance }})" + } + ] + }, + { + "id": 30, + "type": "row", + "title": "GPU Clocks & Per-Pod Usage", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 27 }, + "collapsed": false, + "panels": [] + }, + { + "id": 31, + "type": "timeseries", + "title": "GPU Memory Clock", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 28 }, + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 10, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "none", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 2 + }, + "unit": "hertz", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" }, + "mappings": [], + "custom.transform": "negative-Y" + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "DCGM_FI_DEV_MEM_CLK{gpu=~\"$gpu\", instance=~\"$instance\"} * 1000000", + "legendFormat": "GPU {{ gpu }} mem clock" + } + ] + }, + { + "id": 32, + "type": "timeseries", + "title": "Per-Pod GPU Utilization", + "description": "GPU utilization broken down by pod. Requires DCGM exporter pod-level labels (exported_pod, exported_namespace).", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 28 }, + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 15, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "none", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 2 + }, + "unit": "percent", + "min": 0, + "max": 100, + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "DCGM_FI_DEV_GPU_UTIL{exported_namespace=~\"$namespace\", gpu=~\"$gpu\", instance=~\"$instance\"}", + "legendFormat": "{{ exported_namespace }}/{{ exported_pod }} GPU {{ gpu }}" + } + ] + }, + { + "id": 33, + "type": "timeseries", + "title": "Per-Pod GPU Memory Used", + "description": "GPU framebuffer memory used per pod. Requires DCGM exporter pod-level labels.", + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 36 }, + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 15, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "none", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 2 + }, + "unit": "decmbytes", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "DCGM_FI_DEV_FB_USED{exported_namespace=~\"$namespace\", gpu=~\"$gpu\", instance=~\"$instance\"}", + "legendFormat": "{{ exported_namespace }}/{{ exported_pod }} GPU {{ gpu }}" + } + ] + } + ] +} \ No newline at end of file diff --git a/infra/k8s/monitoring/dashboards/import.sh b/infra/k8s/monitoring/dashboards/import.sh new file mode 100755 index 000000000..b7d041f3c --- /dev/null +++ b/infra/k8s/monitoring/dashboards/import.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Import Grafana dashboards to Scaleway Cockpit via the Grafana API. +# +# Required environment variables: +# GRAFANA_URL - Grafana base URL (e.g. https://xxxx.grafana.scw.cloud) +# GRAFANA_PASS - Password for foxhunt-admin user +# +# Usage: +# GRAFANA_URL="https://..." GRAFANA_PASS="..." ./import.sh + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +GRAFANA_USER="${GRAFANA_USER:-foxhunt-admin}" + +if [[ -z "${GRAFANA_URL:-}" ]]; then + echo "ERROR: GRAFANA_URL is not set" >&2 + exit 1 +fi + +if [[ -z "${GRAFANA_PASS:-}" ]]; then + echo "ERROR: GRAFANA_PASS is not set" >&2 + exit 1 +fi + +# Strip trailing slash from URL +GRAFANA_URL="${GRAFANA_URL%/}" + +import_dashboard() { + local file="$1" + local name + name="$(basename "$file" .json)" + + echo "Importing ${name}..." + + local response + response=$(curl -s -w "\n%{http_code}" \ + -u "${GRAFANA_USER}:${GRAFANA_PASS}" \ + -H "Content-Type: application/json" \ + -d "{\"dashboard\": $(cat "$file"), \"overwrite\": true}" \ + "${GRAFANA_URL}/api/dashboards/db") + + local http_code + http_code=$(echo "$response" | tail -1) + local body + body=$(echo "$response" | sed '$d') + + if [[ "$http_code" -ge 200 && "$http_code" -lt 300 ]]; then + echo " OK (${http_code}): ${body}" + else + echo " FAILED (${http_code}): ${body}" >&2 + return 1 + fi +} + +failed=0 + +for dashboard in "${SCRIPT_DIR}/cluster-overview.json" \ + "${SCRIPT_DIR}/gpu-overview.json" \ + "${SCRIPT_DIR}/gitlab-services.json"; do + if [[ -f "$dashboard" ]]; then + if ! import_dashboard "$dashboard"; then + failed=$((failed + 1)) + fi + else + echo "WARNING: ${dashboard} not found, skipping" >&2 + fi +done + +echo "" +if [[ $failed -eq 0 ]]; then + echo "All dashboards imported successfully." +else + echo "ERROR: ${failed} dashboard(s) failed to import." >&2 + exit 1 +fi