gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(infra): add Cockpit Grafana dashboards for K8s, GPU, and GitLab

Browse Source 
- cluster-overview: nodes, CPU, memory, disk, network per node/namespace
- gpu-overview: DCGM utilization, VRAM, temperature, power, per-pod
- gitlab-services: puma, sidekiq, gitaly + foxhunt service resource usage
- import.sh: one-shot script to push all dashboards via Grafana API

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-02-27 01:22:24 +01:00
parent e8e5d409f8
commit 4c54fe0e37
4 changed files with 1847 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

692
infra/k8s/monitoring/dashboards/cluster-overview.json Normal file
View File

@@ -0,0 +1,692 @@
{
"id": null,
"uid": "foxhunt-cluster-overview",
"title": "K8s Cluster Overview",
"description": "Kubernetes cluster health: nodes, pods, CPU, memory, disk, and network",
"tags": ["kubernetes", "foxhunt", "cluster"],
"timezone": "browser",
"schemaVersion": 39,
"version": 1,
"refresh": "30s",
"time": {
"from": "now-1h",
"to": "now"
},
"fiscalYearStartMonth": 0,
"liveNow": false,
"weekStart": "",
"templating": {
"list": [
{
"name": "namespace",
"type": "query",
"datasource": {
"type": "prometheus",
"uid": "cfegaydmscvswe"
},
"query": "label_values(kube_pod_info, namespace)",
"refresh": 2,
"includeAll": true,
"multi": true,
"allValue": ".*",
"current": {
"selected": true,
"text": "All",
"value": "$__all"
},
"sort": 1
},
{
"name": "node",
"type": "query",
"datasource": {
"type": "prometheus",
"uid": "cfegaydmscvswe"
},
"query": "label_values(node_uname_info, nodename)",
"refresh": 2,
"includeAll": true,
"multi": true,
"allValue": ".*",
"current": {
"selected": true,
"text": "All",
"value": "$__all"
},
"sort": 1
}
]
},
"panels": [
{
"id": 1,
"type": "row",
"title": "Cluster Summary",
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
"collapsed": false,
"panels": []
},
{
"id": 2,
"type": "stat",
"title": "Total Nodes",
"gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"fieldConfig": {
"defaults": {
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "red", "value": null },
{ "color": "green", "value": 1 }
]
},
"mappings": [],
"color": { "mode": "thresholds" }
},
"overrides": []
},
"options": {
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
"orientation": "auto",
"textMode": "auto",
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto"
},
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"expr": "count(kube_node_info)",
"legendFormat": "Nodes",
"instant": true
}
]
},
{
"id": 3,
"type": "stat",
"title": "Pods Running",
"gridPos": { "h": 4, "w": 4, "x": 4, "y": 1 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"fieldConfig": {
"defaults": {
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null }
]
},
"mappings": [],
"color": { "mode": "thresholds" }
},
"overrides": []
},
"options": {
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
"orientation": "auto",
"textMode": "auto",
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto"
},
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"expr": "sum(kube_pod_status_phase{namespace=~\"$namespace\", phase=\"Running\"})",
"legendFormat": "Running",
"instant": true
}
]
},
{
"id": 4,
"type": "stat",
"title": "Pods Pending",
"gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"fieldConfig": {
"defaults": {
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "yellow", "value": 1 },
{ "color": "red", "value": 5 }
]
},
"mappings": [],
"color": { "mode": "thresholds" }
},
"overrides": []
},
"options": {
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
"orientation": "auto",
"textMode": "auto",
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto"
},
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"expr": "sum(kube_pod_status_phase{namespace=~\"$namespace\", phase=\"Pending\"}) or vector(0)",
"legendFormat": "Pending",
"instant": true
}
]
},
{
"id": 5,
"type": "stat",
"title": "Pods Failed",
"gridPos": { "h": 4, "w": 4, "x": 12, "y": 1 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"fieldConfig": {
"defaults": {
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "red", "value": 1 }
]
},
"mappings": [],
"color": { "mode": "thresholds" }
},
"overrides": []
},
"options": {
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
"orientation": "auto",
"textMode": "auto",
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto"
},
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"expr": "sum(kube_pod_status_phase{namespace=~\"$namespace\", phase=\"Failed\"}) or vector(0)",
"legendFormat": "Failed",
"instant": true
}
]
},
{
"id": 6,
"type": "gauge",
"title": "Cluster CPU Usage %",
"gridPos": { "h": 4, "w": 4, "x": 16, "y": 1 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"fieldConfig": {
"defaults": {
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "yellow", "value": 70 },
{ "color": "red", "value": 90 }
]
},
"min": 0,
"max": 100,
"unit": "percent",
"mappings": [],
"color": { "mode": "thresholds" }
},
"overrides": []
},
"options": {
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
"showThresholdLabels": false,
"showThresholdMarkers": true,
"orientation": "auto"
},
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"expr": "100 - (avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m])) * 100)",
"legendFormat": "CPU %",
"instant": true
}
]
},
{
"id": 7,
"type": "gauge",
"title": "Cluster Memory Usage %",
"gridPos": { "h": 4, "w": 4, "x": 20, "y": 1 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"fieldConfig": {
"defaults": {
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "yellow", "value": 70 },
{ "color": "red", "value": 90 }
]
},
"min": 0,
"max": 100,
"unit": "percent",
"mappings": [],
"color": { "mode": "thresholds" }
},
"overrides": []
},
"options": {
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
"showThresholdLabels": false,
"showThresholdMarkers": true,
"orientation": "auto"
},
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"expr": "(1 - (sum(node_memory_MemAvailable_bytes) / sum(node_memory_MemTotal_bytes))) * 100",
"legendFormat": "Memory %",
"instant": true
}
]
},
{
"id": 10,
"type": "row",
"title": "CPU",
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 },
"collapsed": false,
"panels": []
},
{
"id": 11,
"type": "timeseries",
"title": "Per-Node CPU Usage",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 10,
"gradientMode": "scheme",
"spanNulls": false,
"showPoints": "never",
"stacking": { "mode": "none", "group": "A" },
"axisPlacement": "auto",
"lineWidth": 1
},
"unit": "percentunit",
"min": 0,
"max": 1,
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
"color": { "mode": "palette-classic" },
"mappings": []
},
"overrides": []
},
"options": {
"tooltip": { "mode": "multi", "sort": "desc" },
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] }
},
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"expr": "1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))",
"legendFormat": "{{ instance }}"
}
]
},
{
"id": 12,
"type": "timeseries",
"title": "CPU Requests vs Limits vs Usage",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 10,
"gradientMode": "none",
"spanNulls": false,
"showPoints": "never",
"stacking": { "mode": "none", "group": "A" },
"axisPlacement": "auto",
"lineWidth": 1
},
"unit": "short",
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
"color": { "mode": "palette-classic" },
"mappings": []
},
"overrides": []
},
"options": {
"tooltip": { "mode": "multi", "sort": "desc" },
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] }
},
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"expr": "sum(kube_pod_container_resource_requests{namespace=~\"$namespace\", resource=\"cpu\"})",
"legendFormat": "Requests"
},
{
"refId": "B",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"expr": "sum(kube_pod_container_resource_limits{namespace=~\"$namespace\", resource=\"cpu\"})",
"legendFormat": "Limits"
},
{
"refId": "C",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"expr": "sum(rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\", image!=\"\"}[5m]))",
"legendFormat": "Usage"
}
]
},
{
"id": 20,
"type": "row",
"title": "Memory",
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 },
"collapsed": false,
"panels": []
},
{
"id": 21,
"type": "timeseries",
"title": "Per-Node Memory Usage",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 15 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 10,
"gradientMode": "scheme",
"spanNulls": false,
"showPoints": "never",
"stacking": { "mode": "none", "group": "A" },
"axisPlacement": "auto",
"lineWidth": 1
},
"unit": "bytes",
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
"color": { "mode": "palette-classic" },
"mappings": []
},
"overrides": []
},
"options": {
"tooltip": { "mode": "multi", "sort": "desc" },
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] }
},
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"expr": "node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes",
"legendFormat": "{{ instance }}"
}
]
},
{
"id": 22,
"type": "timeseries",
"title": "Memory Requests vs Limits vs Usage",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 15 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 10,
"gradientMode": "none",
"spanNulls": false,
"showPoints": "never",
"stacking": { "mode": "none", "group": "A" },
"axisPlacement": "auto",
"lineWidth": 1
},
"unit": "bytes",
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
"color": { "mode": "palette-classic" },
"mappings": []
},
"overrides": []
},
"options": {
"tooltip": { "mode": "multi", "sort": "desc" },
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] }
},
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"expr": "sum(kube_pod_container_resource_requests{namespace=~\"$namespace\", resource=\"memory\"})",
"legendFormat": "Requests"
},
{
"refId": "B",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"expr": "sum(kube_pod_container_resource_limits{namespace=~\"$namespace\", resource=\"memory\"})",
"legendFormat": "Limits"
},
{
"refId": "C",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"expr": "sum(container_memory_working_set_bytes{namespace=~\"$namespace\", image!=\"\"})",
"legendFormat": "Usage"
}
]
},
{
"id": 30,
"type": "row",
"title": "Disk",
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 },
"collapsed": false,
"panels": []
},
{
"id": 31,
"type": "timeseries",
"title": "Node Filesystem Usage",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 10,
"gradientMode": "scheme",
"spanNulls": false,
"showPoints": "never",
"stacking": { "mode": "none", "group": "A" },
"axisPlacement": "auto",
"lineWidth": 1
},
"unit": "percentunit",
"min": 0,
"max": 1,
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
"color": { "mode": "palette-classic" },
"mappings": []
},
"overrides": []
},
"options": {
"tooltip": { "mode": "multi", "sort": "desc" },
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] }
},
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"expr": "1 - (node_filesystem_avail_bytes{mountpoint=\"/\", fstype!=\"tmpfs\"} / node_filesystem_size_bytes{mountpoint=\"/\", fstype!=\"tmpfs\"})",
"legendFormat": "{{ instance }}"
}
]
},
{
"id": 32,
"type": "timeseries",
"title": "Disk I/O Throughput",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 10,
"gradientMode": "none",
"spanNulls": false,
"showPoints": "never",
"stacking": { "mode": "none", "group": "A" },
"axisPlacement": "auto",
"lineWidth": 1
},
"unit": "Bps",
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
"color": { "mode": "palette-classic" },
"mappings": []
},
"overrides": []
},
"options": {
"tooltip": { "mode": "multi", "sort": "desc" },
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] }
},
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"expr": "sum by (instance) (rate(node_disk_read_bytes_total[5m]))",
"legendFormat": "{{ instance }} read"
},
{
"refId": "B",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"expr": "sum by (instance) (rate(node_disk_written_bytes_total[5m]))",
"legendFormat": "{{ instance }} write"
}
]
},
{
"id": 40,
"type": "row",
"title": "Network",
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 32 },
"collapsed": false,
"panels": []
},
{
"id": 41,
"type": "timeseries",
"title": "Node Network Receive / Transmit",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 33 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 10,
"gradientMode": "none",
"spanNulls": false,
"showPoints": "never",
"stacking": { "mode": "none", "group": "A" },
"axisPlacement": "auto",
"lineWidth": 1
},
"unit": "Bps",
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
"color": { "mode": "palette-classic" },
"mappings": []
},
"overrides": []
},
"options": {
"tooltip": { "mode": "multi", "sort": "desc" },
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] }
},
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"expr": "sum by (instance) (rate(node_network_receive_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[5m]))",
"legendFormat": "{{ instance }} rx"
},
{
"refId": "B",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"expr": "sum by (instance) (rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[5m]))",
"legendFormat": "{{ instance }} tx"
}
]
},
{
"id": 42,
"type": "timeseries",
"title": "Pod Network I/O",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 33 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 10,
"gradientMode": "none",
"spanNulls": false,
"showPoints": "never",
"stacking": { "mode": "normal", "group": "A" },
"axisPlacement": "auto",
"lineWidth": 1
},
"unit": "Bps",
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
"color": { "mode": "palette-classic" },
"mappings": []
},
"overrides": []
},
"options": {
"tooltip": { "mode": "multi", "sort": "desc" },
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] }
},
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"expr": "sum by (namespace) (rate(container_network_receive_bytes_total{namespace=~\"$namespace\", image!=\"\"}[5m]))",
"legendFormat": "{{ namespace }} rx"
},
{
"refId": "B",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"expr": "sum by (namespace) (rate(container_network_transmit_bytes_total{namespace=~\"$namespace\", image!=\"\"}[5m]))",
"legendFormat": "{{ namespace }} tx"
}
]
}
]
}

584
infra/k8s/monitoring/dashboards/gitlab-services.json Normal file
View File

@@ -0,0 +1,584 @@
{
"id": null,
"uid": "foxhunt-gitlab-services",
"title": "GitLab + Foxhunt Services",
"description": "GitLab internals (Puma, Sidekiq, Gitaly) and Foxhunt microservice health",
"tags": ["gitlab", "foxhunt", "services", "kubernetes"],
"timezone": "browser",
"schemaVersion": 39,
"version": 1,
"refresh": "30s",
"time": {
"from": "now-1h",
"to": "now"
},
"fiscalYearStartMonth": 0,
"liveNow": false,
"weekStart": "",
"templating": {
"list": [
{
"name": "namespace",
"type": "query",
"datasource": {
"type": "prometheus",
"uid": "cfegaydmscvswe"
},
"query": "label_values(kube_pod_info, namespace)",
"refresh": 2,
"includeAll": true,
"multi": true,
"allValue": ".*",
"current": {
"selected": true,
"text": "All",
"value": "$__all"
},
"sort": 1
}
]
},
"panels": [
{
"id": 1,
"type": "row",
"title": "GitLab - Puma",
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
"collapsed": false,
"panels": []
},
{
"id": 2,
"type": "timeseries",
"title": "Puma HTTP Request Rate",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 1 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 15,
"gradientMode": "scheme",
"spanNulls": false,
"showPoints": "never",
"stacking": { "mode": "none", "group": "A" },
"axisPlacement": "auto",
"lineWidth": 1
},
"unit": "reqps",
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
"color": { "mode": "palette-classic" },
"mappings": []
},
"overrides": []
},
"options": {
"tooltip": { "mode": "multi", "sort": "desc" },
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] }
},
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"expr": "sum(rate(http_requests_total{job=~\".*puma.*\"}[5m])) by (method)",
"legendFormat": "{{ method }}"
}
]
},
{
"id": 3,
"type": "timeseries",
"title": "Puma Request Duration (p95)",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 1 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 10,
"gradientMode": "none",
"spanNulls": false,
"showPoints": "never",
"stacking": { "mode": "none", "group": "A" },
"axisPlacement": "auto",
"lineWidth": 1
},
"unit": "s",
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
"color": { "mode": "palette-classic" },
"mappings": []
},
"overrides": []
},
"options": {
"tooltip": { "mode": "multi", "sort": "desc" },
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] }
},
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"expr": "histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket{job=~\".*puma.*\"}[5m])) by (le))",
"legendFormat": "p95"
},
{
"refId": "B",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"expr": "histogram_quantile(0.50, sum(rate(http_request_duration_seconds_bucket{job=~\".*puma.*\"}[5m])) by (le))",
"legendFormat": "p50"
}
]
},
{
"id": 10,
"type": "row",
"title": "GitLab - Sidekiq",
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 9 },
"collapsed": false,
"panels": []
},
{
"id": 11,
"type": "timeseries",
"title": "Sidekiq Job Processing Rate",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 10 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 15,
"gradientMode": "scheme",
"spanNulls": false,
"showPoints": "never",
"stacking": { "mode": "none", "group": "A" },
"axisPlacement": "auto",
"lineWidth": 1
},
"unit": "ops",
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
"color": { "mode": "palette-classic" },
"mappings": []
},
"overrides": []
},
"options": {
"tooltip": { "mode": "multi", "sort": "desc" },
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] }
},
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"expr": "sum(rate(sidekiq_jobs_processed_total[5m]))",
"legendFormat": "Processed"
},
{
"refId": "B",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"expr": "sum(rate(sidekiq_jobs_failed_total[5m]))",
"legendFormat": "Failed"
}
]
},
{
"id": 12,
"type": "stat",
"title": "Sidekiq Queue Size",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 10 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"fieldConfig": {
"defaults": {
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "yellow", "value": 100 },
{ "color": "red", "value": 500 }
]
},
"mappings": [],
"color": { "mode": "thresholds" }
},
"overrides": []
},
"options": {
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
"orientation": "auto",
"textMode": "auto",
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto"
},
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"expr": "sum(sidekiq_queue_size)",
"legendFormat": "Queue Size",
"instant": true
}
]
},
{
"id": 20,
"type": "row",
"title": "GitLab - Gitaly",
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 18 },
"collapsed": false,
"panels": []
},
{
"id": 21,
"type": "timeseries",
"title": "Gitaly RPC Latency (p95)",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 19 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 10,
"gradientMode": "none",
"spanNulls": false,
"showPoints": "never",
"stacking": { "mode": "none", "group": "A" },
"axisPlacement": "auto",
"lineWidth": 1
},
"unit": "s",
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
"color": { "mode": "palette-classic" },
"mappings": []
},
"overrides": []
},
"options": {
"tooltip": { "mode": "multi", "sort": "desc" },
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] }
},
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"expr": "histogram_quantile(0.95, sum(rate(grpc_server_handling_seconds_bucket{job=~\".*gitaly.*\"}[5m])) by (le, grpc_method))",
"legendFormat": "{{ grpc_method }}"
}
]
},
{
"id": 22,
"type": "timeseries",
"title": "Gitaly RPC Rate",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 19 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 15,
"gradientMode": "none",
"spanNulls": false,
"showPoints": "never",
"stacking": { "mode": "normal", "group": "A" },
"axisPlacement": "auto",
"lineWidth": 1
},
"unit": "reqps",
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
"color": { "mode": "palette-classic" },
"mappings": []
},
"overrides": []
},
"options": {
"tooltip": { "mode": "multi", "sort": "desc" },
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] }
},
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"expr": "sum(rate(grpc_server_handled_total{job=~\".*gitaly.*\"}[5m])) by (grpc_code)",
"legendFormat": "{{ grpc_code }}"
}
]
},
{
"id": 30,
"type": "row",
"title": "Foxhunt Services - Resource Usage",
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 27 },
"collapsed": false,
"panels": []
},
{
"id": 31,
"type": "timeseries",
"title": "Container CPU Usage by Service",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 28 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 15,
"gradientMode": "none",
"spanNulls": false,
"showPoints": "never",
"stacking": { "mode": "none", "group": "A" },
"axisPlacement": "auto",
"lineWidth": 1
},
"unit": "short",
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
"color": { "mode": "palette-classic" },
"mappings": []
},
"overrides": []
},
"options": {
"tooltip": { "mode": "multi", "sort": "desc" },
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] }
},
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"expr": "sum by (container) (rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\", container=~\"web-gateway|api-gateway|trading-service|ml-training-service|backtesting-service|data-acquisition-service|broker-gateway-service|trading-agent-service\", image!=\"\"}[5m]))",
"legendFormat": "{{ container }}"
}
]
},
{
"id": 32,
"type": "timeseries",
"title": "Container Memory Usage by Service",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 28 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 15,
"gradientMode": "none",
"spanNulls": false,
"showPoints": "never",
"stacking": { "mode": "none", "group": "A" },
"axisPlacement": "auto",
"lineWidth": 1
},
"unit": "bytes",
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
"color": { "mode": "palette-classic" },
"mappings": []
},
"overrides": []
},
"options": {
"tooltip": { "mode": "multi", "sort": "desc" },
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] }
},
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"expr": "sum by (container) (container_memory_working_set_bytes{namespace=~\"$namespace\", container=~\"web-gateway|api-gateway|trading-service|ml-training-service|backtesting-service|data-acquisition-service|broker-gateway-service|trading-agent-service\", image!=\"\"})",
"legendFormat": "{{ container }}"
}
]
},
{
"id": 40,
"type": "row",
"title": "Foxhunt Services - Pod Health",
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 36 },
"collapsed": false,
"panels": []
},
{
"id": 41,
"type": "timeseries",
"title": "Pod Restarts (by Container)",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 37 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "bars",
"lineInterpolation": "linear",
"fillOpacity": 80,
"gradientMode": "none",
"spanNulls": false,
"showPoints": "never",
"stacking": { "mode": "normal", "group": "A" },
"axisPlacement": "auto",
"lineWidth": 1
},
"unit": "short",
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
"color": { "mode": "palette-classic" },
"mappings": []
},
"overrides": []
},
"options": {
"tooltip": { "mode": "multi", "sort": "desc" },
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["sum", "max"] }
},
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"expr": "increase(kube_pod_container_status_restarts_total{namespace=~\"$namespace\"}[1h])",
"legendFormat": "{{ namespace }}/{{ pod }} ({{ container }})"
}
]
},
{
"id": 42,
"type": "table",
"title": "Pod Status by Namespace",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 37 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"fieldConfig": {
"defaults": {
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
"mappings": [],
"color": { "mode": "thresholds" }
},
"overrides": [
{
"matcher": { "id": "byName", "options": "Failed" },
"properties": [
{
"id": "thresholds",
"value": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "red", "value": 1 }
]
}
},
{ "id": "custom.displayMode", "value": "color-background" }
]
},
{
"matcher": { "id": "byName", "options": "Pending" },
"properties": [
{
"id": "thresholds",
"value": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "yellow", "value": 1 }
]
}
},
{ "id": "custom.displayMode", "value": "color-background" }
]
}
]
},
"options": {
"showHeader": true,
"sortBy": [{ "displayName": "namespace", "desc": false }],
"footer": { "show": true, "reducer": ["sum"], "fields": "" }
},
"transformations": [
{
"id": "seriesToColumns",
"options": { "byField": "namespace" }
}
],
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"expr": "sum by (namespace) (kube_pod_status_phase{namespace=~\"$namespace\", phase=\"Running\"})",
"legendFormat": "Running",
"instant": true,
"format": "table"
},
{
"refId": "B",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"expr": "sum by (namespace) (kube_pod_status_phase{namespace=~\"$namespace\", phase=\"Pending\"})",
"legendFormat": "Pending",
"instant": true,
"format": "table"
},
{
"refId": "C",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"expr": "sum by (namespace) (kube_pod_status_phase{namespace=~\"$namespace\", phase=\"Failed\"})",
"legendFormat": "Failed",
"instant": true,
"format": "table"
}
]
},
{
"id": 50,
"type": "row",
"title": "Kubernetes - Deployments",
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 45 },
"collapsed": false,
"panels": []
},
{
"id": 51,
"type": "timeseries",
"title": "Deployment Replicas: Desired vs Available",
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 46 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"lineInterpolation": "stepAfter",
"fillOpacity": 0,
"gradientMode": "none",
"spanNulls": false,
"showPoints": "never",
"stacking": { "mode": "none", "group": "A" },
"axisPlacement": "auto",
"lineWidth": 2
},
"unit": "short",
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
"color": { "mode": "palette-classic" },
"mappings": []
},
"overrides": []
},
"options": {
"tooltip": { "mode": "multi", "sort": "desc" },
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["last"] }
},
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"expr": "kube_deployment_spec_replicas{namespace=~\"$namespace\"}",
"legendFormat": "{{ deployment }} desired"
},
{
"refId": "B",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"expr": "kube_deployment_status_replicas_available{namespace=~\"$namespace\"}",
"legendFormat": "{{ deployment }} available"
}
]
}
]
}

495
infra/k8s/monitoring/dashboards/gpu-overview.json Normal file
View File

@@ -0,0 +1,495 @@
{
"id": null,
"uid": "foxhunt-gpu-overview",
"title": "GPU Overview",
"description": "NVIDIA GPU metrics from DCGM exporter: utilization, memory, temperature, power",
"tags": ["gpu", "nvidia", "dcgm", "foxhunt"],
"timezone": "browser",
"schemaVersion": 39,
"version": 1,
"refresh": "15s",
"time": {
"from": "now-1h",
"to": "now"
},
"fiscalYearStartMonth": 0,
"liveNow": false,
"weekStart": "",
"templating": {
"list": [
{
"name": "gpu",
"type": "query",
"datasource": {
"type": "prometheus",
"uid": "cfegaydmscvswe"
},
"query": "label_values(DCGM_FI_DEV_GPU_UTIL, gpu)",
"refresh": 2,
"includeAll": true,
"multi": true,
"allValue": ".*",
"current": {
"selected": true,
"text": "All",
"value": "$__all"
},
"sort": 1
},
{
"name": "instance",
"type": "query",
"datasource": {
"type": "prometheus",
"uid": "cfegaydmscvswe"
},
"query": "label_values(DCGM_FI_DEV_GPU_UTIL, instance)",
"refresh": 2,
"includeAll": true,
"multi": true,
"allValue": ".*",
"current": {
"selected": true,
"text": "All",
"value": "$__all"
},
"sort": 1
},
{
"name": "namespace",
"type": "query",
"datasource": {
"type": "prometheus",
"uid": "cfegaydmscvswe"
},
"query": "label_values(DCGM_FI_DEV_GPU_UTIL, exported_namespace)",
"refresh": 2,
"includeAll": true,
"multi": true,
"allValue": ".*",
"current": {
"selected": true,
"text": "All",
"value": "$__all"
},
"sort": 1
}
]
},
"panels": [
{
"id": 1,
"type": "row",
"title": "GPU Utilization",
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
"collapsed": false,
"panels": []
},
{
"id": 2,
"type": "timeseries",
"title": "GPU Utilization %",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 1 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 20,
"gradientMode": "scheme",
"spanNulls": false,
"showPoints": "never",
"stacking": { "mode": "none", "group": "A" },
"axisPlacement": "auto",
"lineWidth": 2
},
"unit": "percent",
"min": 0,
"max": 100,
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "yellow", "value": 80 },
{ "color": "red", "value": 95 }
]
},
"color": { "mode": "palette-classic" },
"mappings": []
},
"overrides": []
},
"options": {
"tooltip": { "mode": "multi", "sort": "desc" },
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] }
},
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"expr": "DCGM_FI_DEV_GPU_UTIL{gpu=~\"$gpu\", instance=~\"$instance\"}",
"legendFormat": "GPU {{ gpu }} ({{ instance }})"
}
]
},
{
"id": 3,
"type": "gauge",
"title": "Current GPU Utilization",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 1 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"fieldConfig": {
"defaults": {
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "blue", "value": null },
{ "color": "green", "value": 20 },
{ "color": "yellow", "value": 80 },
{ "color": "red", "value": 95 }
]
},
"min": 0,
"max": 100,
"unit": "percent",
"mappings": [],
"color": { "mode": "thresholds" }
},
"overrides": []
},
"options": {
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
"showThresholdLabels": false,
"showThresholdMarkers": true,
"orientation": "auto"
},
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"expr": "DCGM_FI_DEV_GPU_UTIL{gpu=~\"$gpu\", instance=~\"$instance\"}",
"legendFormat": "GPU {{ gpu }}",
"instant": true
}
]
},
{
"id": 10,
"type": "row",
"title": "GPU Memory",
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 9 },
"collapsed": false,
"panels": []
},
{
"id": 11,
"type": "timeseries",
"title": "GPU Memory Used",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 10 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 20,
"gradientMode": "scheme",
"spanNulls": false,
"showPoints": "never",
"stacking": { "mode": "none", "group": "A" },
"axisPlacement": "auto",
"lineWidth": 2
},
"unit": "decmbytes",
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
"color": { "mode": "palette-classic" },
"mappings": []
},
"overrides": []
},
"options": {
"tooltip": { "mode": "multi", "sort": "desc" },
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] }
},
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"expr": "DCGM_FI_DEV_FB_USED{gpu=~\"$gpu\", instance=~\"$instance\"}",
"legendFormat": "GPU {{ gpu }} used"
},
{
"refId": "B",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"expr": "DCGM_FI_DEV_FB_TOTAL{gpu=~\"$gpu\", instance=~\"$instance\"}",
"legendFormat": "GPU {{ gpu }} total"
}
]
},
{
"id": 12,
"type": "gauge",
"title": "GPU Memory Usage %",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 10 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"fieldConfig": {
"defaults": {
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "yellow", "value": 70 },
{ "color": "red", "value": 90 }
]
},
"min": 0,
"max": 100,
"unit": "percent",
"mappings": [],
"color": { "mode": "thresholds" }
},
"overrides": []
},
"options": {
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
"showThresholdLabels": false,
"showThresholdMarkers": true,
"orientation": "auto"
},
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"expr": "(DCGM_FI_DEV_FB_USED{gpu=~\"$gpu\", instance=~\"$instance\"} / DCGM_FI_DEV_FB_TOTAL{gpu=~\"$gpu\", instance=~\"$instance\"}) * 100",
"legendFormat": "GPU {{ gpu }}",
"instant": true
}
]
},
{
"id": 20,
"type": "row",
"title": "GPU Temperature & Power",
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 18 },
"collapsed": false,
"panels": []
},
{
"id": 21,
"type": "timeseries",
"title": "GPU Temperature",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 19 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 10,
"gradientMode": "scheme",
"spanNulls": false,
"showPoints": "never",
"stacking": { "mode": "none", "group": "A" },
"axisPlacement": "auto",
"lineWidth": 2
},
"unit": "celsius",
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "blue", "value": null },
{ "color": "green", "value": 40 },
{ "color": "yellow", "value": 75 },
{ "color": "red", "value": 85 }
]
},
"color": { "mode": "palette-classic" },
"mappings": []
},
"overrides": []
},
"options": {
"tooltip": { "mode": "multi", "sort": "desc" },
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] }
},
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"expr": "DCGM_FI_DEV_GPU_TEMP{gpu=~\"$gpu\", instance=~\"$instance\"}",
"legendFormat": "GPU {{ gpu }} ({{ instance }})"
}
]
},
{
"id": 22,
"type": "timeseries",
"title": "GPU Power Usage",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 19 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 10,
"gradientMode": "scheme",
"spanNulls": false,
"showPoints": "never",
"stacking": { "mode": "none", "group": "A" },
"axisPlacement": "auto",
"lineWidth": 2
},
"unit": "watt",
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
"color": { "mode": "palette-classic" },
"mappings": []
},
"overrides": []
},
"options": {
"tooltip": { "mode": "multi", "sort": "desc" },
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] }
},
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"expr": "DCGM_FI_DEV_POWER_USAGE{gpu=~\"$gpu\", instance=~\"$instance\"}",
"legendFormat": "GPU {{ gpu }} ({{ instance }})"
}
]
},
{
"id": 30,
"type": "row",
"title": "GPU Clocks & Per-Pod Usage",
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 27 },
"collapsed": false,
"panels": []
},
{
"id": 31,
"type": "timeseries",
"title": "GPU Memory Clock",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 28 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 10,
"gradientMode": "none",
"spanNulls": false,
"showPoints": "never",
"stacking": { "mode": "none", "group": "A" },
"axisPlacement": "auto",
"lineWidth": 2
},
"unit": "hertz",
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
"color": { "mode": "palette-classic" },
"mappings": [],
"custom.transform": "negative-Y"
},
"overrides": []
},
"options": {
"tooltip": { "mode": "multi", "sort": "desc" },
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] }
},
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"expr": "DCGM_FI_DEV_MEM_CLK{gpu=~\"$gpu\", instance=~\"$instance\"} * 1000000",
"legendFormat": "GPU {{ gpu }} mem clock"
}
]
},
{
"id": 32,
"type": "timeseries",
"title": "Per-Pod GPU Utilization",
"description": "GPU utilization broken down by pod. Requires DCGM exporter pod-level labels (exported_pod, exported_namespace).",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 28 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 15,
"gradientMode": "none",
"spanNulls": false,
"showPoints": "never",
"stacking": { "mode": "none", "group": "A" },
"axisPlacement": "auto",
"lineWidth": 2
},
"unit": "percent",
"min": 0,
"max": 100,
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
"color": { "mode": "palette-classic" },
"mappings": []
},
"overrides": []
},
"options": {
"tooltip": { "mode": "multi", "sort": "desc" },
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] }
},
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"expr": "DCGM_FI_DEV_GPU_UTIL{exported_namespace=~\"$namespace\", gpu=~\"$gpu\", instance=~\"$instance\"}",
"legendFormat": "{{ exported_namespace }}/{{ exported_pod }} GPU {{ gpu }}"
}
]
},
{
"id": 33,
"type": "timeseries",
"title": "Per-Pod GPU Memory Used",
"description": "GPU framebuffer memory used per pod. Requires DCGM exporter pod-level labels.",
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 36 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 15,
"gradientMode": "none",
"spanNulls": false,
"showPoints": "never",
"stacking": { "mode": "none", "group": "A" },
"axisPlacement": "auto",
"lineWidth": 2
},
"unit": "decmbytes",
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
"color": { "mode": "palette-classic" },
"mappings": []
},
"overrides": []
},
"options": {
"tooltip": { "mode": "multi", "sort": "desc" },
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] }
},
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"expr": "DCGM_FI_DEV_FB_USED{exported_namespace=~\"$namespace\", gpu=~\"$gpu\", instance=~\"$instance\"}",
"legendFormat": "{{ exported_namespace }}/{{ exported_pod }} GPU {{ gpu }}"
}
]
}
]
}

76
infra/k8s/monitoring/dashboards/import.sh Executable file
View File

@@ -0,0 +1,76 @@
#!/usr/bin/env bash
set -euo pipefail
# Import Grafana dashboards to Scaleway Cockpit via the Grafana API.
#
# Required environment variables:
# GRAFANA_URL - Grafana base URL (e.g. https://xxxx.grafana.scw.cloud)
# GRAFANA_PASS - Password for foxhunt-admin user
#
# Usage:
# GRAFANA_URL="https://..." GRAFANA_PASS="..." ./import.sh
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
GRAFANA_USER="${GRAFANA_USER:-foxhunt-admin}"
if [[ -z "${GRAFANA_URL:-}" ]]; then
echo "ERROR: GRAFANA_URL is not set" >&2
exit 1
fi
if [[ -z "${GRAFANA_PASS:-}" ]]; then
echo "ERROR: GRAFANA_PASS is not set" >&2
exit 1
fi
# Strip trailing slash from URL
GRAFANA_URL="${GRAFANA_URL%/}"
import_dashboard() {
local file="$1"
local name
name="$(basename "$file" .json)"
echo "Importing ${name}..."
local response
response=$(curl -s -w "\n%{http_code}" \
-u "${GRAFANA_USER}:${GRAFANA_PASS}" \
-H "Content-Type: application/json" \
-d "{\"dashboard\": $(cat "$file"), \"overwrite\": true}" \
"${GRAFANA_URL}/api/dashboards/db")
local http_code
http_code=$(echo "$response" | tail -1)
local body
body=$(echo "$response" | sed '$d')
if [[ "$http_code" -ge 200 && "$http_code" -lt 300 ]]; then
echo " OK (${http_code}): ${body}"
else
echo " FAILED (${http_code}): ${body}" >&2
return 1
fi
}
failed=0
for dashboard in "${SCRIPT_DIR}/cluster-overview.json" \
"${SCRIPT_DIR}/gpu-overview.json" \
"${SCRIPT_DIR}/gitlab-services.json"; do
if [[ -f "$dashboard" ]]; then
if ! import_dashboard "$dashboard"; then
failed=$((failed + 1))
fi
else
echo "WARNING: ${dashboard} not found, skipping" >&2
fi
done
echo ""
if [[ $failed -eq 0 ]]; then
echo "All dashboards imported successfully."
else
echo "ERROR: ${failed} dashboard(s) failed to import." >&2
exit 1
fi