gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
Files
9b85b9efd596ed8a69a22dbfaaa01446cac577c3
foxhunt/infra/k8s/monitoring/dashboards/gpu-overview.json
jgrusewski 33b4020e06 feat(monitoring): add cockpit dashboard, migrate all dashboards to dynamic datasources 
Add foxhunt-cockpit.json — single-page overview with 7 rows (health bar,
service status, resources, GPU/training, restarts/network, errors, live
logs) pulling critical signals from all domain dashboards.

Migrate all 10 dashboards from hardcoded datasource UIDs to ${DS_PROMETHEUS},
${DS_LOKI}, ${DS_TEMPO} variables with __inputs sections for portable import.
Update import.sh to auto-detect datasource UIDs from Grafana API at deploy
time, using /api/dashboards/import endpoint.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-28 23:04:42 +01:00

498 lines
16 KiB
JSON
Raw Blame History

{
"__inputs": [
{ "name": "DS_PROMETHEUS", "label": "Prometheus", "description": "", "type": "datasource", "pluginId": "prometheus" }
],
"id": null,
"uid": "foxhunt-gpu-overview",
"title": "GPU Overview",
"description": "NVIDIA GPU metrics from DCGM exporter: utilization, memory, temperature, power",
"tags": ["gpu", "nvidia", "dcgm", "foxhunt"],
"timezone": "browser",
"schemaVersion": 39,
"version": 1,
"refresh": "15s",
"time": {
"from": "now-1h",
"to": "now"
},
"fiscalYearStartMonth": 0,
"liveNow": false,
"weekStart": "",
"templating": {
"list": [
{
"name": "gpu",
"type": "query",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"query": "label_values(DCGM_FI_DEV_GPU_UTIL, gpu)",
"refresh": 2,
"includeAll": true,
"multi": true,
"allValue": ".*",
"current": {
"selected": true,
"text": "All",
"value": "$__all"
},
"sort": 1
},
{
"name": "instance",
"type": "query",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"query": "label_values(DCGM_FI_DEV_GPU_UTIL, instance)",
"refresh": 2,
"includeAll": true,
"multi": true,
"allValue": ".*",
"current": {
"selected": true,
"text": "All",
"value": "$__all"
},
"sort": 1
},
{
"name": "namespace",
"type": "query",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"query": "label_values(DCGM_FI_DEV_GPU_UTIL, exported_namespace)",
"refresh": 2,
"includeAll": true,
"multi": true,
"allValue": ".*",
"current": {
"selected": true,
"text": "All",
"value": "$__all"
},
"sort": 1
}
]
},
"panels": [
{
"id": 1,
"type": "row",
"title": "GPU Utilization",
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
"collapsed": false,
"panels": []
},
{
"id": 2,
"type": "timeseries",
"title": "GPU Utilization %",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 1 },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 20,
"gradientMode": "scheme",
"spanNulls": false,
"showPoints": "never",
"stacking": { "mode": "none", "group": "A" },
"axisPlacement": "auto",
"lineWidth": 2
},
"unit": "percent",
"min": 0,
"max": 100,
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "yellow", "value": 80 },
{ "color": "red", "value": 95 }
]
},
"color": { "mode": "palette-classic" },
"mappings": []
},
"overrides": []
},
"options": {
"tooltip": { "mode": "multi", "sort": "desc" },
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] }
},
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "DCGM_FI_DEV_GPU_UTIL{gpu=~\"$gpu\", instance=~\"$instance\"}",
"legendFormat": "GPU {{ gpu }} ({{ instance }})"
}
]
},
{
"id": 3,
"type": "gauge",
"title": "Current GPU Utilization",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 1 },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "blue", "value": null },
{ "color": "green", "value": 20 },
{ "color": "yellow", "value": 80 },
{ "color": "red", "value": 95 }
]
},
"min": 0,
"max": 100,
"unit": "percent",
"mappings": [],
"color": { "mode": "thresholds" }
},
"overrides": []
},
"options": {
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
"showThresholdLabels": false,
"showThresholdMarkers": true,
"orientation": "auto"
},
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "DCGM_FI_DEV_GPU_UTIL{gpu=~\"$gpu\", instance=~\"$instance\"}",
"legendFormat": "GPU {{ gpu }}",
"instant": true
}
]
},
{
"id": 10,
"type": "row",
"title": "GPU Memory",
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 9 },
"collapsed": false,
"panels": []
},
{
"id": 11,
"type": "timeseries",
"title": "GPU Memory Used",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 10 },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 20,
"gradientMode": "scheme",
"spanNulls": false,
"showPoints": "never",
"stacking": { "mode": "none", "group": "A" },
"axisPlacement": "auto",
"lineWidth": 2
},
"unit": "decmbytes",
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
"color": { "mode": "palette-classic" },
"mappings": []
},
"overrides": []
},
"options": {
"tooltip": { "mode": "multi", "sort": "desc" },
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] }
},
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "DCGM_FI_DEV_FB_USED{gpu=~\"$gpu\", instance=~\"$instance\"}",
"legendFormat": "GPU {{ gpu }} used"
},
{
"refId": "B",
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "DCGM_FI_DEV_FB_TOTAL{gpu=~\"$gpu\", instance=~\"$instance\"}",
"legendFormat": "GPU {{ gpu }} total"
}
]
},
{
"id": 12,
"type": "gauge",
"title": "GPU Memory Usage %",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 10 },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "yellow", "value": 70 },
{ "color": "red", "value": 90 }
]
},
"min": 0,
"max": 100,
"unit": "percent",
"mappings": [],
"color": { "mode": "thresholds" }
},
"overrides": []
},
"options": {
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
"showThresholdLabels": false,
"showThresholdMarkers": true,
"orientation": "auto"
},
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "(DCGM_FI_DEV_FB_USED{gpu=~\"$gpu\", instance=~\"$instance\"} / DCGM_FI_DEV_FB_TOTAL{gpu=~\"$gpu\", instance=~\"$instance\"}) * 100",
"legendFormat": "GPU {{ gpu }}",
"instant": true
}
]
},
{
"id": 20,
"type": "row",
"title": "GPU Temperature & Power",
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 18 },
"collapsed": false,
"panels": []
},
{
"id": 21,
"type": "timeseries",
"title": "GPU Temperature",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 19 },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 10,
"gradientMode": "scheme",
"spanNulls": false,
"showPoints": "never",
"stacking": { "mode": "none", "group": "A" },
"axisPlacement": "auto",
"lineWidth": 2
},
"unit": "celsius",
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "blue", "value": null },
{ "color": "green", "value": 40 },
{ "color": "yellow", "value": 75 },
{ "color": "red", "value": 85 }
]
},
"color": { "mode": "palette-classic" },
"mappings": []
},
"overrides": []
},
"options": {
"tooltip": { "mode": "multi", "sort": "desc" },
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] }
},
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "DCGM_FI_DEV_GPU_TEMP{gpu=~\"$gpu\", instance=~\"$instance\"}",
"legendFormat": "GPU {{ gpu }} ({{ instance }})"
}
]
},
{
"id": 22,
"type": "timeseries",
"title": "GPU Power Usage",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 19 },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 10,
"gradientMode": "scheme",
"spanNulls": false,
"showPoints": "never",
"stacking": { "mode": "none", "group": "A" },
"axisPlacement": "auto",
"lineWidth": 2
},
"unit": "watt",
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
"color": { "mode": "palette-classic" },
"mappings": []
},
"overrides": []
},
"options": {
"tooltip": { "mode": "multi", "sort": "desc" },
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] }
},
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "DCGM_FI_DEV_POWER_USAGE{gpu=~\"$gpu\", instance=~\"$instance\"}",
"legendFormat": "GPU {{ gpu }} ({{ instance }})"
}
]
},
{
"id": 30,
"type": "row",
"title": "GPU Clocks & Per-Pod Usage",
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 27 },
"collapsed": false,
"panels": []
},
{
"id": 31,
"type": "timeseries",
"title": "GPU Memory Clock",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 28 },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 10,
"gradientMode": "none",
"spanNulls": false,
"showPoints": "never",
"stacking": { "mode": "none", "group": "A" },
"axisPlacement": "auto",
"lineWidth": 2
},
"unit": "hertz",
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
"color": { "mode": "palette-classic" },
"mappings": [],
"custom.transform": "negative-Y"
},
"overrides": []
},
"options": {
"tooltip": { "mode": "multi", "sort": "desc" },
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] }
},
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "DCGM_FI_DEV_MEM_CLK{gpu=~\"$gpu\", instance=~\"$instance\"} * 1000000",
"legendFormat": "GPU {{ gpu }} mem clock"
}
]
},
{
"id": 32,
"type": "timeseries",
"title": "Per-Pod GPU Utilization",
"description": "GPU utilization broken down by pod. Requires DCGM exporter pod-level labels (exported_pod, exported_namespace).",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 28 },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 15,
"gradientMode": "none",
"spanNulls": false,
"showPoints": "never",
"stacking": { "mode": "none", "group": "A" },
"axisPlacement": "auto",
"lineWidth": 2
},
"unit": "percent",
"min": 0,
"max": 100,
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
"color": { "mode": "palette-classic" },
"mappings": []
},
"overrides": []
},
"options": {
"tooltip": { "mode": "multi", "sort": "desc" },
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] }
},
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "DCGM_FI_DEV_GPU_UTIL{exported_namespace=~\"$namespace\", gpu=~\"$gpu\", instance=~\"$instance\"}",
"legendFormat": "{{ exported_namespace }}/{{ exported_pod }} GPU {{ gpu }}"
}
]
},
{
"id": 33,
"type": "timeseries",
"title": "Per-Pod GPU Memory Used",
"description": "GPU framebuffer memory used per pod. Requires DCGM exporter pod-level labels.",
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 36 },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 15,
"gradientMode": "none",
"spanNulls": false,
"showPoints": "never",
"stacking": { "mode": "none", "group": "A" },
"axisPlacement": "auto",
"lineWidth": 2
},
"unit": "decmbytes",
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
"color": { "mode": "palette-classic" },
"mappings": []
},
"overrides": []
},
"options": {
"tooltip": { "mode": "multi", "sort": "desc" },
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] }
},
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "DCGM_FI_DEV_FB_USED{exported_namespace=~\"$namespace\", gpu=~\"$gpu\", instance=~\"$instance\"}",
"legendFormat": "{{ exported_namespace }}/{{ exported_pod }} GPU {{ gpu }}"
}
]
}
]
}
Reference in New Issue View Git Blame Copy Permalink