diff --git a/infra/k8s/monitoring/dashboards/foxhunt-cockpit.json b/infra/k8s/monitoring/dashboards/foxhunt-cockpit.json index 77ff77a0b..285703313 100644 --- a/infra/k8s/monitoring/dashboards/foxhunt-cockpit.json +++ b/infra/k8s/monitoring/dashboards/foxhunt-cockpit.json @@ -1490,7 +1490,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "avg(DCGM_FI_DEV_GPU_UTIL) or vector(0)", + "expr": "avg(dcgm_gpu_utilization) or vector(0)", "instant": true } ] @@ -1563,7 +1563,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "100 * avg(DCGM_FI_DEV_FB_USED / (DCGM_FI_DEV_FB_USED + DCGM_FI_DEV_FB_FREE)) or vector(0)", + "expr": "100 * avg(dcgm_fb_used / (dcgm_fb_used + dcgm_fb_free)) or vector(0)", "instant": true } ] diff --git a/infra/k8s/monitoring/dashboards/foxhunt-gpu-training.json b/infra/k8s/monitoring/dashboards/foxhunt-gpu-training.json index b29b687aa..7873f8768 100644 --- a/infra/k8s/monitoring/dashboards/foxhunt-gpu-training.json +++ b/infra/k8s/monitoring/dashboards/foxhunt-gpu-training.json @@ -44,7 +44,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "query": "label_values(DCGM_FI_DEV_GPU_UTIL, gpu)", + "query": "label_values(dcgm_gpu_utilization, gpu)", "refresh": 2, "includeAll": true, "multi": true, @@ -546,7 +546,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "DCGM_FI_DEV_GPU_UTIL{gpu=~\"$gpu\"}", + "expr": "dcgm_gpu_utilization{gpu=~\"$gpu\"}", "legendFormat": "GPU {{ gpu }} ({{ instance }})" } ] @@ -620,7 +620,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "DCGM_FI_DEV_FB_USED{gpu=~\"$gpu\"}", + "expr": "dcgm_fb_used{gpu=~\"$gpu\"}", "legendFormat": "GPU {{ gpu }} used" }, { @@ -629,7 +629,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "DCGM_FI_DEV_FB_TOTAL{gpu=~\"$gpu\"}", + "expr": "(dcgm_fb_used{gpu=~\"$gpu\"} + dcgm_fb_free{gpu=~\"$gpu\"})", "legendFormat": "GPU {{ gpu }} total" } ] @@ -711,7 +711,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "DCGM_FI_DEV_GPU_TEMP{gpu=~\"$gpu\"}", + "expr": "dcgm_gpu_temp{gpu=~\"$gpu\"}", "legendFormat": "GPU {{ gpu }}" } ] @@ -781,7 +781,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "DCGM_FI_DEV_POWER_USAGE{gpu=~\"$gpu\"}", + "expr": "dcgm_power_usage{gpu=~\"$gpu\"}", "legendFormat": "GPU {{ gpu }}" } ] @@ -1298,4 +1298,4 @@ ] } ] -} +} \ No newline at end of file diff --git a/infra/k8s/monitoring/dashboards/foxhunt-training-cockpit.json b/infra/k8s/monitoring/dashboards/foxhunt-training-cockpit.json index 50b9e938a..62f1dd11c 100644 --- a/infra/k8s/monitoring/dashboards/foxhunt-training-cockpit.json +++ b/infra/k8s/monitoring/dashboards/foxhunt-training-cockpit.json @@ -789,7 +789,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "DCGM_FI_DEV_GPU_UTIL", + "expr": "dcgm_gpu_utilization", "instant": true } ] @@ -856,7 +856,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "DCGM_FI_DEV_FB_USED", + "expr": "dcgm_fb_used", "instant": true } ] @@ -929,7 +929,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "DCGM_FI_DEV_GPU_TEMP", + "expr": "dcgm_gpu_temp", "legendFormat": "GPU Temp", "instant": false, "range": true @@ -992,7 +992,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "DCGM_FI_DEV_POWER_USAGE", + "expr": "dcgm_power_usage", "legendFormat": "GPU Power", "instant": false, "range": true diff --git a/infra/k8s/monitoring/dashboards/gpu-overview.json b/infra/k8s/monitoring/dashboards/gpu-overview.json index d3d40eaef..c63aa267a 100644 --- a/infra/k8s/monitoring/dashboards/gpu-overview.json +++ b/infra/k8s/monitoring/dashboards/gpu-overview.json @@ -1,12 +1,23 @@ { "__inputs": [ - { "name": "DS_PROMETHEUS", "label": "Prometheus", "description": "", "type": "datasource", "pluginId": "prometheus" } + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus" + } ], "id": null, "uid": "foxhunt-gpu-overview", "title": "GPU Overview", "description": "NVIDIA GPU metrics from DCGM exporter: utilization, memory, temperature, power", - "tags": ["gpu", "nvidia", "dcgm", "foxhunt"], + "tags": [ + "gpu", + "nvidia", + "dcgm", + "foxhunt" + ], "timezone": "browser", "schemaVersion": 39, "version": 1, @@ -27,7 +38,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "query": "label_values(DCGM_FI_DEV_GPU_UTIL, gpu)", + "query": "label_values(dcgm_gpu_utilization, gpu)", "refresh": 2, "includeAll": true, "multi": true, @@ -46,7 +57,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "query": "label_values(DCGM_FI_DEV_GPU_UTIL, instance)", + "query": "label_values(dcgm_gpu_utilization, instance)", "refresh": 2, "includeAll": true, "multi": true, @@ -65,7 +76,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "query": "label_values(DCGM_FI_DEV_GPU_UTIL, exported_namespace)", + "query": "label_values(dcgm_gpu_utilization, exported_namespace)", "refresh": 2, "includeAll": true, "multi": true, @@ -84,7 +95,12 @@ "id": 1, "type": "row", "title": "GPU Utilization", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, "collapsed": false, "panels": [] }, @@ -92,8 +108,16 @@ "id": 2, "type": "timeseries", "title": "GPU Utilization %", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": { @@ -103,7 +127,10 @@ "gradientMode": "scheme", "spanNulls": false, "showPoints": "never", - "stacking": { "mode": "none", "group": "A" }, + "stacking": { + "mode": "none", + "group": "A" + }, "axisPlacement": "auto", "lineWidth": 2 }, @@ -113,25 +140,50 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 80 }, - { "color": "red", "value": 95 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 80 + }, + { + "color": "red", + "value": 95 + } ] }, - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "mappings": [] }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max", + "last" + ] + } }, "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "DCGM_FI_DEV_GPU_UTIL{gpu=~\"$gpu\", instance=~\"$instance\"}", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "dcgm_gpu_utilization{gpu=~\"$gpu\", instance=~\"$instance\"}", "legendFormat": "GPU {{ gpu }} ({{ instance }})" } ] @@ -140,29 +192,57 @@ "id": 3, "type": "gauge", "title": "Current GPU Utilization", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ - { "color": "blue", "value": null }, - { "color": "green", "value": 20 }, - { "color": "yellow", "value": 80 }, - { "color": "red", "value": 95 } + { + "color": "blue", + "value": null + }, + { + "color": "green", + "value": 20 + }, + { + "color": "yellow", + "value": 80 + }, + { + "color": "red", + "value": 95 + } ] }, "min": 0, "max": 100, "unit": "percent", "mappings": [], - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "showThresholdLabels": false, "showThresholdMarkers": true, "orientation": "auto" @@ -170,8 +250,11 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "DCGM_FI_DEV_GPU_UTIL{gpu=~\"$gpu\", instance=~\"$instance\"}", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "dcgm_gpu_utilization{gpu=~\"$gpu\", instance=~\"$instance\"}", "legendFormat": "GPU {{ gpu }}", "instant": true } @@ -181,7 +264,12 @@ "id": 10, "type": "row", "title": "GPU Memory", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 9 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, "collapsed": false, "panels": [] }, @@ -189,8 +277,16 @@ "id": 11, "type": "timeseries", "title": "GPU Memory Used", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 10 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 10 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": { @@ -200,32 +296,62 @@ "gradientMode": "scheme", "spanNulls": false, "showPoints": "never", - "stacking": { "mode": "none", "group": "A" }, + "stacking": { + "mode": "none", + "group": "A" + }, "axisPlacement": "auto", "lineWidth": 2 }, "unit": "decmbytes", - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, - "color": { "mode": "palette-classic" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "color": { + "mode": "palette-classic" + }, "mappings": [] }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max", + "last" + ] + } }, "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "DCGM_FI_DEV_FB_USED{gpu=~\"$gpu\", instance=~\"$instance\"}", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "dcgm_fb_used{gpu=~\"$gpu\", instance=~\"$instance\"}", "legendFormat": "GPU {{ gpu }} used" }, { "refId": "B", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "DCGM_FI_DEV_FB_TOTAL{gpu=~\"$gpu\", instance=~\"$instance\"}", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "(dcgm_fb_used{gpu=~\"$gpu\", instance=~\"$instance\"} + dcgm_fb_free{gpu=~\"$gpu\", instance=~\"$instance\"})", "legendFormat": "GPU {{ gpu }} total" } ] @@ -234,28 +360,53 @@ "id": 12, "type": "gauge", "title": "GPU Memory Usage %", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 10 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 10 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 70 }, - { "color": "red", "value": 90 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 70 + }, + { + "color": "red", + "value": 90 + } ] }, "min": 0, "max": 100, "unit": "percent", "mappings": [], - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "showThresholdLabels": false, "showThresholdMarkers": true, "orientation": "auto" @@ -263,8 +414,11 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "(DCGM_FI_DEV_FB_USED{gpu=~\"$gpu\", instance=~\"$instance\"} / DCGM_FI_DEV_FB_TOTAL{gpu=~\"$gpu\", instance=~\"$instance\"}) * 100", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "(dcgm_fb_used{gpu=~\"$gpu\", instance=~\"$instance\"} / (dcgm_fb_used{gpu=~\"$gpu\", instance=~\"$instance\"} + dcgm_fb_free{gpu=~\"$gpu\", instance=~\"$instance\"})) * 100", "legendFormat": "GPU {{ gpu }}", "instant": true } @@ -274,7 +428,12 @@ "id": 20, "type": "row", "title": "GPU Temperature & Power", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 18 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, "collapsed": false, "panels": [] }, @@ -282,8 +441,16 @@ "id": 21, "type": "timeseries", "title": "GPU Temperature", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 19 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 19 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": { @@ -293,7 +460,10 @@ "gradientMode": "scheme", "spanNulls": false, "showPoints": "never", - "stacking": { "mode": "none", "group": "A" }, + "stacking": { + "mode": "none", + "group": "A" + }, "axisPlacement": "auto", "lineWidth": 2 }, @@ -301,26 +471,54 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "blue", "value": null }, - { "color": "green", "value": 40 }, - { "color": "yellow", "value": 75 }, - { "color": "red", "value": 85 } + { + "color": "blue", + "value": null + }, + { + "color": "green", + "value": 40 + }, + { + "color": "yellow", + "value": 75 + }, + { + "color": "red", + "value": 85 + } ] }, - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "mappings": [] }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max", + "last" + ] + } }, "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "DCGM_FI_DEV_GPU_TEMP{gpu=~\"$gpu\", instance=~\"$instance\"}", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "dcgm_gpu_temp{gpu=~\"$gpu\", instance=~\"$instance\"}", "legendFormat": "GPU {{ gpu }} ({{ instance }})" } ] @@ -329,8 +527,16 @@ "id": 22, "type": "timeseries", "title": "GPU Power Usage", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 19 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 19 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": { @@ -340,26 +546,53 @@ "gradientMode": "scheme", "spanNulls": false, "showPoints": "never", - "stacking": { "mode": "none", "group": "A" }, + "stacking": { + "mode": "none", + "group": "A" + }, "axisPlacement": "auto", "lineWidth": 2 }, "unit": "watt", - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, - "color": { "mode": "palette-classic" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "color": { + "mode": "palette-classic" + }, "mappings": [] }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max", + "last" + ] + } }, "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "DCGM_FI_DEV_POWER_USAGE{gpu=~\"$gpu\", instance=~\"$instance\"}", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "dcgm_power_usage{gpu=~\"$gpu\", instance=~\"$instance\"}", "legendFormat": "GPU {{ gpu }} ({{ instance }})" } ] @@ -368,7 +601,12 @@ "id": 30, "type": "row", "title": "GPU Clocks & Per-Pod Usage", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 27 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 27 + }, "collapsed": false, "panels": [] }, @@ -376,8 +614,16 @@ "id": 31, "type": "timeseries", "title": "GPU Memory Clock", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 28 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 28 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": { @@ -387,27 +633,54 @@ "gradientMode": "none", "spanNulls": false, "showPoints": "never", - "stacking": { "mode": "none", "group": "A" }, + "stacking": { + "mode": "none", + "group": "A" + }, "axisPlacement": "auto", "lineWidth": 2 }, "unit": "hertz", - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, - "color": { "mode": "palette-classic" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "color": { + "mode": "palette-classic" + }, "mappings": [], "custom.transform": "negative-Y" }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max", + "last" + ] + } }, "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "DCGM_FI_DEV_MEM_CLK{gpu=~\"$gpu\", instance=~\"$instance\"} * 1000000", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "dcgm_memory_clock{gpu=~\"$gpu\", instance=~\"$instance\"} * 1000000", "legendFormat": "GPU {{ gpu }} mem clock" } ] @@ -417,8 +690,16 @@ "type": "timeseries", "title": "Per-Pod GPU Utilization", "description": "GPU utilization broken down by pod. Requires DCGM exporter pod-level labels (exported_pod, exported_namespace).", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 28 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 28 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": { @@ -428,28 +709,55 @@ "gradientMode": "none", "spanNulls": false, "showPoints": "never", - "stacking": { "mode": "none", "group": "A" }, + "stacking": { + "mode": "none", + "group": "A" + }, "axisPlacement": "auto", "lineWidth": 2 }, "unit": "percent", "min": 0, "max": 100, - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, - "color": { "mode": "palette-classic" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "color": { + "mode": "palette-classic" + }, "mappings": [] }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max", + "last" + ] + } }, "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "DCGM_FI_DEV_GPU_UTIL{exported_namespace=~\"$namespace\", gpu=~\"$gpu\", instance=~\"$instance\"}", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "dcgm_gpu_utilization{exported_namespace=~\"$namespace\", gpu=~\"$gpu\", instance=~\"$instance\"}", "legendFormat": "{{ exported_namespace }}/{{ exported_pod }} GPU {{ gpu }}" } ] @@ -459,8 +767,16 @@ "type": "timeseries", "title": "Per-Pod GPU Memory Used", "description": "GPU framebuffer memory used per pod. Requires DCGM exporter pod-level labels.", - "gridPos": { "h": 8, "w": 24, "x": 0, "y": 36 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 36 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": { @@ -470,26 +786,53 @@ "gradientMode": "none", "spanNulls": false, "showPoints": "never", - "stacking": { "mode": "none", "group": "A" }, + "stacking": { + "mode": "none", + "group": "A" + }, "axisPlacement": "auto", "lineWidth": 2 }, "unit": "decmbytes", - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, - "color": { "mode": "palette-classic" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "color": { + "mode": "palette-classic" + }, "mappings": [] }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max", + "last" + ] + } }, "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "DCGM_FI_DEV_FB_USED{exported_namespace=~\"$namespace\", gpu=~\"$gpu\", instance=~\"$instance\"}", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "dcgm_fb_used{exported_namespace=~\"$namespace\", gpu=~\"$gpu\", instance=~\"$instance\"}", "legendFormat": "{{ exported_namespace }}/{{ exported_pod }} GPU {{ gpu }}" } ]