gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(dashboards): correct DCGM metric names in all Grafana dashboards

Browse Source 
DCGM exporter v3.x emits lowercase metric names (dcgm_gpu_utilization)
but dashboards used the old uppercase format (DCGM_FI_DEV_GPU_UTIL).
Fixed across 4 dashboard files:
- foxhunt-cockpit.json (Infrastructure Cockpit)
- foxhunt-training-cockpit.json (Training Cockpit)
- foxhunt-gpu-training.json (GPU Training)
- gpu-overview.json (GPU Overview)

Replaced DCGM_FI_DEV_FB_TOTAL with (dcgm_fb_used + dcgm_fb_free)
since DCGM v3 doesn't emit a total metric.

Redeployed all dashboard ConfigMaps and restarted Grafana.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-03-01 14:29:57 +01:00
parent 16db63d71e
commit 30fcb9abe9
4 changed files with 454 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
infra/k8s/monitoring/dashboards/foxhunt-cockpit.json
View File

@@ -1490,7 +1490,7 @@
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "avg(DCGM_FI_DEV_GPU_UTIL) or vector(0)",
"expr": "avg(dcgm_gpu_utilization) or vector(0)",
"instant": true
}
]
@@ -1563,7 +1563,7 @@
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "100 * avg(DCGM_FI_DEV_FB_USED / (DCGM_FI_DEV_FB_USED + DCGM_FI_DEV_FB_FREE)) or vector(0)",
"expr": "100 * avg(dcgm_fb_used / (dcgm_fb_used + dcgm_fb_free)) or vector(0)",
"instant": true
}
]

14
infra/k8s/monitoring/dashboards/foxhunt-gpu-training.json
View File

@@ -44,7 +44,7 @@
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"query": "label_values(DCGM_FI_DEV_GPU_UTIL, gpu)",
"query": "label_values(dcgm_gpu_utilization, gpu)",
"refresh": 2,
"includeAll": true,
"multi": true,
@@ -546,7 +546,7 @@
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "DCGM_FI_DEV_GPU_UTIL{gpu=~\"$gpu\"}",
"expr": "dcgm_gpu_utilization{gpu=~\"$gpu\"}",
"legendFormat": "GPU {{ gpu }} ({{ instance }})"
}
]
@@ -620,7 +620,7 @@
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "DCGM_FI_DEV_FB_USED{gpu=~\"$gpu\"}",
"expr": "dcgm_fb_used{gpu=~\"$gpu\"}",
"legendFormat": "GPU {{ gpu }} used"
},
{
@@ -629,7 +629,7 @@
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "DCGM_FI_DEV_FB_TOTAL{gpu=~\"$gpu\"}",
"expr": "(dcgm_fb_used{gpu=~\"$gpu\"} + dcgm_fb_free{gpu=~\"$gpu\"})",
"legendFormat": "GPU {{ gpu }} total"
}
]
@@ -711,7 +711,7 @@
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "DCGM_FI_DEV_GPU_TEMP{gpu=~\"$gpu\"}",
"expr": "dcgm_gpu_temp{gpu=~\"$gpu\"}",
"legendFormat": "GPU {{ gpu }}"
}
]
@@ -781,7 +781,7 @@
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "DCGM_FI_DEV_POWER_USAGE{gpu=~\"$gpu\"}",
"expr": "dcgm_power_usage{gpu=~\"$gpu\"}",
"legendFormat": "GPU {{ gpu }}"
}
]
@@ -1298,4 +1298,4 @@
]
}
]
}
}

8
infra/k8s/monitoring/dashboards/foxhunt-training-cockpit.json
View File

@@ -789,7 +789,7 @@
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "DCGM_FI_DEV_GPU_UTIL",
"expr": "dcgm_gpu_utilization",
"instant": true
}
]
@@ -856,7 +856,7 @@
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "DCGM_FI_DEV_FB_USED",
"expr": "dcgm_fb_used",
"instant": true
}
]
@@ -929,7 +929,7 @@
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "DCGM_FI_DEV_GPU_TEMP",
"expr": "dcgm_gpu_temp",
"legendFormat": "GPU Temp",
"instant": false,
"range": true
@@ -992,7 +992,7 @@
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "DCGM_FI_DEV_POWER_USAGE",
"expr": "dcgm_power_usage",
"legendFormat": "GPU Power",
"instant": false,
"range": true

539
infra/k8s/monitoring/dashboards/gpu-overview.json
View File

@@ -1,12 +1,23 @@
{
"__inputs": [
{ "name": "DS_PROMETHEUS", "label": "Prometheus", "description": "", "type": "datasource", "pluginId": "prometheus" }
{
"name": "DS_PROMETHEUS",
"label": "Prometheus",
"description": "",
"type": "datasource",
"pluginId": "prometheus"
}
],
"id": null,
"uid": "foxhunt-gpu-overview",
"title": "GPU Overview",
"description": "NVIDIA GPU metrics from DCGM exporter: utilization, memory, temperature, power",
"tags": ["gpu", "nvidia", "dcgm", "foxhunt"],
"tags": [
"gpu",
"nvidia",
"dcgm",
"foxhunt"
],
"timezone": "browser",
"schemaVersion": 39,
"version": 1,
@@ -27,7 +38,7 @@
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"query": "label_values(DCGM_FI_DEV_GPU_UTIL, gpu)",
"query": "label_values(dcgm_gpu_utilization, gpu)",
"refresh": 2,
"includeAll": true,
"multi": true,
@@ -46,7 +57,7 @@
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"query": "label_values(DCGM_FI_DEV_GPU_UTIL, instance)",
"query": "label_values(dcgm_gpu_utilization, instance)",
"refresh": 2,
"includeAll": true,
"multi": true,
@@ -65,7 +76,7 @@
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"query": "label_values(DCGM_FI_DEV_GPU_UTIL, exported_namespace)",
"query": "label_values(dcgm_gpu_utilization, exported_namespace)",
"refresh": 2,
"includeAll": true,
"multi": true,
@@ -84,7 +95,12 @@
"id": 1,
"type": "row",
"title": "GPU Utilization",
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 0
},
"collapsed": false,
"panels": []
},
@@ -92,8 +108,16 @@
"id": 2,
"type": "timeseries",
"title": "GPU Utilization %",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 1 },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 1
},
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {
@@ -103,7 +127,10 @@
"gradientMode": "scheme",
"spanNulls": false,
"showPoints": "never",
"stacking": { "mode": "none", "group": "A" },
"stacking": {
"mode": "none",
"group": "A"
},
"axisPlacement": "auto",
"lineWidth": 2
},
@@ -113,25 +140,50 @@
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "yellow", "value": 80 },
{ "color": "red", "value": 95 }
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 80
},
{
"color": "red",
"value": 95
}
]
},
"color": { "mode": "palette-classic" },
"color": {
"mode": "palette-classic"
},
"mappings": []
},
"overrides": []
},
"options": {
"tooltip": { "mode": "multi", "sort": "desc" },
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] }
"tooltip": {
"mode": "multi",
"sort": "desc"
},
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": [
"mean",
"max",
"last"
]
}
},
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "DCGM_FI_DEV_GPU_UTIL{gpu=~\"$gpu\", instance=~\"$instance\"}",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "dcgm_gpu_utilization{gpu=~\"$gpu\", instance=~\"$instance\"}",
"legendFormat": "GPU {{ gpu }} ({{ instance }})"
}
]
@@ -140,29 +192,57 @@
"id": 3,
"type": "gauge",
"title": "Current GPU Utilization",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 1 },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 1
},
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "blue", "value": null },
{ "color": "green", "value": 20 },
{ "color": "yellow", "value": 80 },
{ "color": "red", "value": 95 }
{
"color": "blue",
"value": null
},
{
"color": "green",
"value": 20
},
{
"color": "yellow",
"value": 80
},
{
"color": "red",
"value": 95
}
]
},
"min": 0,
"max": 100,
"unit": "percent",
"mappings": [],
"color": { "mode": "thresholds" }
"color": {
"mode": "thresholds"
}
},
"overrides": []
},
"options": {
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showThresholdLabels": false,
"showThresholdMarkers": true,
"orientation": "auto"
@@ -170,8 +250,11 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "DCGM_FI_DEV_GPU_UTIL{gpu=~\"$gpu\", instance=~\"$instance\"}",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "dcgm_gpu_utilization{gpu=~\"$gpu\", instance=~\"$instance\"}",
"legendFormat": "GPU {{ gpu }}",
"instant": true
}
@@ -181,7 +264,12 @@
"id": 10,
"type": "row",
"title": "GPU Memory",
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 9 },
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 9
},
"collapsed": false,
"panels": []
},
@@ -189,8 +277,16 @@
"id": 11,
"type": "timeseries",
"title": "GPU Memory Used",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 10 },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 10
},
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {
@@ -200,32 +296,62 @@
"gradientMode": "scheme",
"spanNulls": false,
"showPoints": "never",
"stacking": { "mode": "none", "group": "A" },
"stacking": {
"mode": "none",
"group": "A"
},
"axisPlacement": "auto",
"lineWidth": 2
},
"unit": "decmbytes",
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
"color": { "mode": "palette-classic" },
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"color": {
"mode": "palette-classic"
},
"mappings": []
},
"overrides": []
},
"options": {
"tooltip": { "mode": "multi", "sort": "desc" },
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] }
"tooltip": {
"mode": "multi",
"sort": "desc"
},
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": [
"mean",
"max",
"last"
]
}
},
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "DCGM_FI_DEV_FB_USED{gpu=~\"$gpu\", instance=~\"$instance\"}",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "dcgm_fb_used{gpu=~\"$gpu\", instance=~\"$instance\"}",
"legendFormat": "GPU {{ gpu }} used"
},
{
"refId": "B",
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "DCGM_FI_DEV_FB_TOTAL{gpu=~\"$gpu\", instance=~\"$instance\"}",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "(dcgm_fb_used{gpu=~\"$gpu\", instance=~\"$instance\"} + dcgm_fb_free{gpu=~\"$gpu\", instance=~\"$instance\"})",
"legendFormat": "GPU {{ gpu }} total"
}
]
@@ -234,28 +360,53 @@
"id": 12,
"type": "gauge",
"title": "GPU Memory Usage %",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 10 },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 10
},
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "yellow", "value": 70 },
{ "color": "red", "value": 90 }
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 70
},
{
"color": "red",
"value": 90
}
]
},
"min": 0,
"max": 100,
"unit": "percent",
"mappings": [],
"color": { "mode": "thresholds" }
"color": {
"mode": "thresholds"
}
},
"overrides": []
},
"options": {
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showThresholdLabels": false,
"showThresholdMarkers": true,
"orientation": "auto"
@@ -263,8 +414,11 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "(DCGM_FI_DEV_FB_USED{gpu=~\"$gpu\", instance=~\"$instance\"} / DCGM_FI_DEV_FB_TOTAL{gpu=~\"$gpu\", instance=~\"$instance\"}) * 100",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "(dcgm_fb_used{gpu=~\"$gpu\", instance=~\"$instance\"} / (dcgm_fb_used{gpu=~\"$gpu\", instance=~\"$instance\"} + dcgm_fb_free{gpu=~\"$gpu\", instance=~\"$instance\"})) * 100",
"legendFormat": "GPU {{ gpu }}",
"instant": true
}
@@ -274,7 +428,12 @@
"id": 20,
"type": "row",
"title": "GPU Temperature & Power",
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 18 },
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 18
},
"collapsed": false,
"panels": []
},
@@ -282,8 +441,16 @@
"id": 21,
"type": "timeseries",
"title": "GPU Temperature",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 19 },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 19
},
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {
@@ -293,7 +460,10 @@
"gradientMode": "scheme",
"spanNulls": false,
"showPoints": "never",
"stacking": { "mode": "none", "group": "A" },
"stacking": {
"mode": "none",
"group": "A"
},
"axisPlacement": "auto",
"lineWidth": 2
},
@@ -301,26 +471,54 @@
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "blue", "value": null },
{ "color": "green", "value": 40 },
{ "color": "yellow", "value": 75 },
{ "color": "red", "value": 85 }
{
"color": "blue",
"value": null
},
{
"color": "green",
"value": 40
},
{
"color": "yellow",
"value": 75
},
{
"color": "red",
"value": 85
}
]
},
"color": { "mode": "palette-classic" },
"color": {
"mode": "palette-classic"
},
"mappings": []
},
"overrides": []
},
"options": {
"tooltip": { "mode": "multi", "sort": "desc" },
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] }
"tooltip": {
"mode": "multi",
"sort": "desc"
},
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": [
"mean",
"max",
"last"
]
}
},
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "DCGM_FI_DEV_GPU_TEMP{gpu=~\"$gpu\", instance=~\"$instance\"}",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "dcgm_gpu_temp{gpu=~\"$gpu\", instance=~\"$instance\"}",
"legendFormat": "GPU {{ gpu }} ({{ instance }})"
}
]
@@ -329,8 +527,16 @@
"id": 22,
"type": "timeseries",
"title": "GPU Power Usage",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 19 },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 19
},
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {
@@ -340,26 +546,53 @@
"gradientMode": "scheme",
"spanNulls": false,
"showPoints": "never",
"stacking": { "mode": "none", "group": "A" },
"stacking": {
"mode": "none",
"group": "A"
},
"axisPlacement": "auto",
"lineWidth": 2
},
"unit": "watt",
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
"color": { "mode": "palette-classic" },
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"color": {
"mode": "palette-classic"
},
"mappings": []
},
"overrides": []
},
"options": {
"tooltip": { "mode": "multi", "sort": "desc" },
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] }
"tooltip": {
"mode": "multi",
"sort": "desc"
},
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": [
"mean",
"max",
"last"
]
}
},
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "DCGM_FI_DEV_POWER_USAGE{gpu=~\"$gpu\", instance=~\"$instance\"}",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "dcgm_power_usage{gpu=~\"$gpu\", instance=~\"$instance\"}",
"legendFormat": "GPU {{ gpu }} ({{ instance }})"
}
]
@@ -368,7 +601,12 @@
"id": 30,
"type": "row",
"title": "GPU Clocks & Per-Pod Usage",
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 27 },
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 27
},
"collapsed": false,
"panels": []
},
@@ -376,8 +614,16 @@
"id": 31,
"type": "timeseries",
"title": "GPU Memory Clock",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 28 },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 28
},
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {
@@ -387,27 +633,54 @@
"gradientMode": "none",
"spanNulls": false,
"showPoints": "never",
"stacking": { "mode": "none", "group": "A" },
"stacking": {
"mode": "none",
"group": "A"
},
"axisPlacement": "auto",
"lineWidth": 2
},
"unit": "hertz",
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
"color": { "mode": "palette-classic" },
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"color": {
"mode": "palette-classic"
},
"mappings": [],
"custom.transform": "negative-Y"
},
"overrides": []
},
"options": {
"tooltip": { "mode": "multi", "sort": "desc" },
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] }
"tooltip": {
"mode": "multi",
"sort": "desc"
},
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": [
"mean",
"max",
"last"
]
}
},
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "DCGM_FI_DEV_MEM_CLK{gpu=~\"$gpu\", instance=~\"$instance\"} * 1000000",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "dcgm_memory_clock{gpu=~\"$gpu\", instance=~\"$instance\"} * 1000000",
"legendFormat": "GPU {{ gpu }} mem clock"
}
]
@@ -417,8 +690,16 @@
"type": "timeseries",
"title": "Per-Pod GPU Utilization",
"description": "GPU utilization broken down by pod. Requires DCGM exporter pod-level labels (exported_pod, exported_namespace).",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 28 },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 28
},
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {
@@ -428,28 +709,55 @@
"gradientMode": "none",
"spanNulls": false,
"showPoints": "never",
"stacking": { "mode": "none", "group": "A" },
"stacking": {
"mode": "none",
"group": "A"
},
"axisPlacement": "auto",
"lineWidth": 2
},
"unit": "percent",
"min": 0,
"max": 100,
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
"color": { "mode": "palette-classic" },
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"color": {
"mode": "palette-classic"
},
"mappings": []
},
"overrides": []
},
"options": {
"tooltip": { "mode": "multi", "sort": "desc" },
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] }
"tooltip": {
"mode": "multi",
"sort": "desc"
},
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": [
"mean",
"max",
"last"
]
}
},
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "DCGM_FI_DEV_GPU_UTIL{exported_namespace=~\"$namespace\", gpu=~\"$gpu\", instance=~\"$instance\"}",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "dcgm_gpu_utilization{exported_namespace=~\"$namespace\", gpu=~\"$gpu\", instance=~\"$instance\"}",
"legendFormat": "{{ exported_namespace }}/{{ exported_pod }} GPU {{ gpu }}"
}
]
@@ -459,8 +767,16 @@
"type": "timeseries",
"title": "Per-Pod GPU Memory Used",
"description": "GPU framebuffer memory used per pod. Requires DCGM exporter pod-level labels.",
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 36 },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 36
},
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"custom": {
@@ -470,26 +786,53 @@
"gradientMode": "none",
"spanNulls": false,
"showPoints": "never",
"stacking": { "mode": "none", "group": "A" },
"stacking": {
"mode": "none",
"group": "A"
},
"axisPlacement": "auto",
"lineWidth": 2
},
"unit": "decmbytes",
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
"color": { "mode": "palette-classic" },
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"color": {
"mode": "palette-classic"
},
"mappings": []
},
"overrides": []
},
"options": {
"tooltip": { "mode": "multi", "sort": "desc" },
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] }
"tooltip": {
"mode": "multi",
"sort": "desc"
},
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": [
"mean",
"max",
"last"
]
}
},
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "DCGM_FI_DEV_FB_USED{exported_namespace=~\"$namespace\", gpu=~\"$gpu\", instance=~\"$instance\"}",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "dcgm_fb_used{exported_namespace=~\"$namespace\", gpu=~\"$gpu\", instance=~\"$instance\"}",
"legendFormat": "{{ exported_namespace }}/{{ exported_pod }} GPU {{ gpu }}"
}
]