From d87c1cf21f254f245bcb4a5bde9afb5b31cbcc01 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Mon, 2 Mar 2026 02:04:24 +0100 Subject: [PATCH] feat(grafana): replace GPU gauges with timeseries, add CPU/Memory side-by-side MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Infrastructure Cockpit: - GPU gauges → side-by-side timeseries (GPU Utilization + GPU VRAM with total line) - Added Memory by Service timeseries next to existing CPU by Service - Cluster resource gauges shrunk to h=4 for tighter layout Training Cockpit: - GPU Utilization/Memory gauges → side-by-side timeseries matching infra cockpit - GPU Temperature/Power widened from w=6 to w=12 in paired row Co-Authored-By: Claude Opus 4.6 --- .../dashboards/foxhunt-cockpit.json | 358 +++++++++++++----- .../dashboards/foxhunt-training-cockpit.json | 228 ++++++----- 2 files changed, 392 insertions(+), 194 deletions(-) diff --git a/infra/k8s/monitoring/dashboards/foxhunt-cockpit.json b/infra/k8s/monitoring/dashboards/foxhunt-cockpit.json index 267f125f1..f76d73086 100644 --- a/infra/k8s/monitoring/dashboards/foxhunt-cockpit.json +++ b/infra/k8s/monitoring/dashboards/foxhunt-cockpit.json @@ -2054,7 +2054,7 @@ "type": "gauge", "title": "Cluster CPU %", "gridPos": { - "h": 5, + "h": 4, "w": 4, "x": 0, "y": 16 @@ -2120,7 +2120,7 @@ "type": "gauge", "title": "Cluster Memory %", "gridPos": { - "h": 5, + "h": 4, "w": 4, "x": 4, "y": 16 @@ -2186,7 +2186,7 @@ "type": "gauge", "title": "Cluster Disk %", "gridPos": { - "h": 5, + "h": 4, "w": 4, "x": 8, "y": 16 @@ -2252,10 +2252,10 @@ "type": "timeseries", "title": "CPU by Service", "gridPos": { - "h": 5, + "h": 6, "w": 12, - "x": 12, - "y": 16 + "x": 0, + "y": 20 }, "datasource": { "type": "prometheus", @@ -2318,27 +2318,14 @@ ] }, { - "id": 22, - "type": "row", - "title": "GPU & Training", + "id": 38, + "type": "timeseries", + "title": "Memory by Service", "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 21 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 23, - "type": "gauge", - "title": "GPU Utilization %", - "gridPos": { - "h": 5, - "w": 6, - "x": 0, - "y": 22 + "h": 6, + "w": 12, + "x": 12, + "y": 20 }, "datasource": { "type": "prometheus", @@ -2346,16 +2333,126 @@ }, "fieldConfig": { "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 1, + "fillOpacity": 20, + "gradientMode": "opacity", + "spanNulls": false, + "showPoints": "never", + "stacking": { + "mode": "normal", + "group": "A" + }, + "axisPlacement": "auto", + "scaleDistribution": { + "type": "linear" + } + }, + "unit": "bytes", "thresholds": { "mode": "absolute", "steps": [ { - "color": "blue", + "color": "green", "value": null - }, + } + ] + }, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "list", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + } + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (pod) (container_memory_working_set_bytes{namespace=\"foxhunt\",container!=\"POD\",container!=\"\",pod=~\"(api-gateway|trading-service|ml-training|backtesting|trading-agent|broker-gateway|data-acquisition|web-gateway).*\"})", + "legendFormat": "{{pod}}", + "range": true + } + ] + }, + { + "id": 22, + "type": "row", + "title": "GPU & Training", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, + "collapsed": false, + "panels": [] + }, + { + "id": 23, + "type": "timeseries", + "title": "GPU Utilization", + "description": "SM compute utilization and memory copy utilization over time", + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 27 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 20, + "gradientMode": "opacity", + "spanNulls": false, + "showPoints": "never", + "stacking": { + "mode": "none", + "group": "A" + }, + "axisPlacement": "auto", + "axisLabel": "", + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "unit": "percent", + "min": 0, + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ { "color": "green", - "value": 30 + "value": null }, { "color": "yellow", @@ -2367,34 +2464,27 @@ } ] }, - "unit": "percent", - "min": 0, - "max": 100, - "mappings": [], "color": { - "mode": "thresholds" - } + "mode": "palette-classic" + }, + "noValue": "GPU offline" }, "overrides": [] }, "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false + "tooltip": { + "mode": "multi", + "sort": "desc" }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "links": [ - { - "title": "GPU & Training Detail", - "url": "/d/foxhunt-gpu-training", - "targetBlank": false + "legend": { + "displayMode": "list", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] } - ], + }, "targets": [ { "refId": "A", @@ -2402,20 +2492,32 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "avg(dcgm_gpu_utilization) or vector(0)", - "instant": true + "expr": "dcgm_gpu_utilization{gpu=\"0\"}", + "legendFormat": "SM Utilization", + "range": true + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "dcgm_mem_copy_utilization{gpu=\"0\"}", + "legendFormat": "Memory Copy", + "range": true } ] }, { "id": 24, - "type": "gauge", - "title": "GPU Memory %", + "type": "timeseries", + "title": "GPU VRAM", + "description": "Framebuffer memory used vs total (NVIDIA L40S 48GB)", "gridPos": { - "h": 5, - "w": 6, - "x": 6, - "y": 22 + "h": 6, + "w": 12, + "x": 12, + "y": 27 }, "datasource": { "type": "prometheus", @@ -2423,51 +2525,86 @@ }, "fieldConfig": { "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 20, + "gradientMode": "opacity", + "spanNulls": false, + "showPoints": "never", + "stacking": { + "mode": "none", + "group": "A" + }, + "axisPlacement": "auto", + "axisLabel": "", + "scaleDistribution": { + "type": "linear" + } + }, + "unit": "decmbytes", + "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null - }, - { - "color": "yellow", - "value": 70 - }, - { - "color": "red", - "value": 90 } ] }, - "unit": "percent", - "min": 0, - "max": 100, - "mappings": [], "color": { - "mode": "thresholds" - } + "mode": "palette-classic" + }, + "noValue": "GPU offline" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 10, + 10 + ] + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "white" + } + } + ] + } + ] }, "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false + "tooltip": { + "mode": "multi", + "sort": "desc" }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "links": [ - { - "title": "GPU & Training Detail", - "url": "/d/foxhunt-gpu-training", - "targetBlank": false + "legend": { + "displayMode": "list", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] } - ], + }, "targets": [ { "refId": "A", @@ -2475,8 +2612,19 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "100 * avg(dcgm_fb_used / (dcgm_fb_used + dcgm_fb_free)) or vector(0)", - "instant": true + "expr": "dcgm_fb_used{gpu=\"0\"}", + "legendFormat": "Used", + "range": true + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "dcgm_fb_used{gpu=\"0\"} + dcgm_fb_free{gpu=\"0\"}", + "legendFormat": "Total", + "range": true } ] }, @@ -2486,9 +2634,9 @@ "title": "Training Jobs Timeline", "gridPos": { "h": 5, - "w": 12, - "x": 12, - "y": 22 + "w": 24, + "x": 0, + "y": 33 }, "datasource": { "type": "prometheus", @@ -2622,7 +2770,7 @@ "h": 1, "w": 24, "x": 0, - "y": 27 + "y": 32 }, "collapsed": false, "panels": [] @@ -2635,7 +2783,7 @@ "h": 5, "w": 12, "x": 0, - "y": 28 + "y": 34 }, "datasource": { "type": "prometheus", @@ -2709,7 +2857,7 @@ "h": 5, "w": 12, "x": 12, - "y": 28 + "y": 34 }, "datasource": { "type": "prometheus", @@ -2788,7 +2936,7 @@ "h": 1, "w": 24, "x": 0, - "y": 33 + "y": 39 }, "collapsed": false, "panels": [] @@ -2801,7 +2949,7 @@ "h": 5, "w": 12, "x": 0, - "y": 34 + "y": 40 }, "datasource": { "type": "loki", @@ -2871,7 +3019,7 @@ "h": 5, "w": 12, "x": 12, - "y": 34 + "y": 40 }, "datasource": { "type": "loki", @@ -2928,7 +3076,7 @@ "h": 1, "w": 24, "x": 0, - "y": 39 + "y": 45 }, "collapsed": false, "panels": [] @@ -2942,7 +3090,7 @@ "h": 7, "w": 24, "x": 0, - "y": 40 + "y": 46 }, "datasource": { "type": "loki", @@ -2978,7 +3126,7 @@ "h": 1, "w": 24, "x": 0, - "y": 47 + "y": 53 }, "collapsed": false, "panels": [] @@ -2991,7 +3139,7 @@ "h": 4, "w": 6, "x": 0, - "y": 48 + "y": 54 }, "datasource": { "type": "prometheus", @@ -3049,7 +3197,7 @@ "h": 4, "w": 6, "x": 6, - "y": 48 + "y": 54 }, "datasource": { "type": "prometheus", @@ -3116,7 +3264,7 @@ "h": 4, "w": 12, "x": 12, - "y": 48 + "y": 54 }, "datasource": { "type": "prometheus", diff --git a/infra/k8s/monitoring/dashboards/foxhunt-training-cockpit.json b/infra/k8s/monitoring/dashboards/foxhunt-training-cockpit.json index 237e0ad7d..46e6422ff 100644 --- a/infra/k8s/monitoring/dashboards/foxhunt-training-cockpit.json +++ b/infra/k8s/monitoring/dashboards/foxhunt-training-cockpit.json @@ -729,11 +729,11 @@ }, { "id": 14, - "type": "gauge", + "type": "timeseries", "title": "GPU Utilization", "gridPos": { "h": 6, - "w": 6, + "w": 12, "x": 0, "y": 14 }, @@ -743,26 +743,26 @@ }, "fieldConfig": { "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 20, + "gradientMode": "scheme", + "spanNulls": false, + "showPoints": "never", + "axisCenteredZero": false, + "stacking": { + "mode": "none" + } + }, "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 80 - }, - { - "color": "red", - "value": 95 - } - ] + "mode": "off", + "steps": [] }, "mappings": [], "color": { - "mode": "thresholds" + "mode": "palette-classic" }, "min": 0, "max": 100, @@ -771,16 +771,14 @@ "overrides": [] }, "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false + "tooltip": { + "mode": "multi", + "sort": "desc" }, - "orientation": "auto", - "showThresholdLabels": false, - "showThresholdMarkers": true + "legend": { + "displayMode": "list", + "placement": "bottom" + } }, "targets": [ { @@ -789,19 +787,32 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "dcgm_gpu_utilization", - "instant": true + "expr": "dcgm_gpu_utilization{gpu=\"0\"}", + "legendFormat": "GPU Core", + "instant": false, + "range": true + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "dcgm_mem_copy_utilization{gpu=\"0\"}", + "legendFormat": "Memory Copy", + "instant": false, + "range": true } ] }, { "id": 15, - "type": "gauge", - "title": "GPU Memory", + "type": "timeseries", + "title": "GPU VRAM", "gridPos": { "h": 6, - "w": 6, - "x": 6, + "w": 12, + "x": 12, "y": 14 }, "datasource": { @@ -810,44 +821,70 @@ }, "fieldConfig": { "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 20, + "gradientMode": "scheme", + "spanNulls": false, + "showPoints": "never", + "axisCenteredZero": false, + "stacking": { + "mode": "none" + } + }, "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 80 - }, - { - "color": "red", - "value": 95 - } - ] + "mode": "off", + "steps": [] }, "mappings": [], "color": { - "mode": "thresholds" + "mode": "palette-classic" }, - "min": 0, - "max": 100, - "unit": "percent" + "unit": "decmbytes" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 10, + 10 + ] + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "color", + "value": { + "fixedColor": "white", + "mode": "fixed" + } + } + ] + } + ] }, "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false + "tooltip": { + "mode": "multi", + "sort": "desc" }, - "orientation": "auto", - "showThresholdLabels": false, - "showThresholdMarkers": true + "legend": { + "displayMode": "list", + "placement": "bottom" + } }, "targets": [ { @@ -856,8 +893,21 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "dcgm_fb_used", - "instant": true + "expr": "dcgm_fb_used{gpu=\"0\"}", + "legendFormat": "Used", + "instant": false, + "range": true + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "dcgm_fb_used{gpu=\"0\"} + dcgm_fb_free{gpu=\"0\"}", + "legendFormat": "Total", + "instant": false, + "range": true } ] }, @@ -867,9 +917,9 @@ "title": "GPU Temperature", "gridPos": { "h": 6, - "w": 6, - "x": 12, - "y": 14 + "w": 12, + "x": 0, + "y": 20 }, "datasource": { "type": "prometheus", @@ -942,9 +992,9 @@ "title": "GPU Power", "gridPos": { "h": 6, - "w": 6, - "x": 18, - "y": 14 + "w": 12, + "x": 12, + "y": 20 }, "datasource": { "type": "prometheus", @@ -1007,7 +1057,7 @@ "h": 1, "w": 24, "x": 0, - "y": 20 + "y": 26 }, "collapsed": false, "panels": [] @@ -1021,7 +1071,7 @@ "h": 6, "w": 12, "x": 0, - "y": 21 + "y": 27 }, "datasource": { "type": "prometheus", @@ -1089,7 +1139,7 @@ "h": 6, "w": 12, "x": 12, - "y": 21 + "y": 27 }, "datasource": { "type": "prometheus", @@ -1178,7 +1228,7 @@ "h": 1, "w": 24, "x": 0, - "y": 27 + "y": 33 }, "collapsed": false, "panels": [] @@ -1191,7 +1241,7 @@ "h": 4, "w": 6, "x": 0, - "y": 28 + "y": 34 }, "datasource": { "type": "prometheus", @@ -1249,7 +1299,7 @@ "h": 4, "w": 6, "x": 6, - "y": 28 + "y": 34 }, "datasource": { "type": "prometheus", @@ -1311,7 +1361,7 @@ "h": 4, "w": 6, "x": 12, - "y": 28 + "y": 34 }, "datasource": { "type": "prometheus", @@ -1378,7 +1428,7 @@ "h": 4, "w": 6, "x": 18, - "y": 28 + "y": 34 }, "datasource": { "type": "prometheus", @@ -1437,7 +1487,7 @@ "h": 1, "w": 24, "x": 0, - "y": 32 + "y": 38 }, "collapsed": false, "panels": [] @@ -1451,7 +1501,7 @@ "h": 6, "w": 8, "x": 0, - "y": 33 + "y": 39 }, "datasource": { "type": "prometheus", @@ -1515,7 +1565,7 @@ "h": 6, "w": 8, "x": 8, - "y": 33 + "y": 39 }, "datasource": { "type": "prometheus", @@ -1578,7 +1628,7 @@ "h": 6, "w": 8, "x": 16, - "y": 33 + "y": 39 }, "datasource": { "type": "prometheus", @@ -1640,7 +1690,7 @@ "h": 1, "w": 24, "x": 0, - "y": 39 + "y": 45 }, "collapsed": false, "panels": [] @@ -1653,7 +1703,7 @@ "h": 4, "w": 6, "x": 0, - "y": 40 + "y": 46 }, "datasource": { "type": "prometheus", @@ -1715,7 +1765,7 @@ "h": 4, "w": 6, "x": 6, - "y": 40 + "y": 46 }, "datasource": { "type": "prometheus", @@ -1778,7 +1828,7 @@ "h": 6, "w": 12, "x": 12, - "y": 40 + "y": 46 }, "datasource": { "type": "prometheus", @@ -1846,7 +1896,7 @@ "h": 1, "w": 24, "x": 0, - "y": 46 + "y": 52 }, "collapsed": false, "panels": [] @@ -1860,7 +1910,7 @@ "h": 6, "w": 12, "x": 0, - "y": 47 + "y": 53 }, "datasource": { "type": "prometheus", @@ -1927,7 +1977,7 @@ "h": 6, "w": 12, "x": 12, - "y": 47 + "y": 53 }, "datasource": { "type": "prometheus",