33b4020e06
Add foxhunt-cockpit.json — single-page overview with 7 rows (health bar,
service status, resources, GPU/training, restarts/network, errors, live
logs) pulling critical signals from all domain dashboards.
Migrate all 10 dashboards from hardcoded datasource UIDs to ${DS_PROMETHEUS},
${DS_LOKI}, ${DS_TEMPO} variables with __inputs sections for portable import.
Update import.sh to auto-detect datasource UIDs from Grafana API at deploy
time, using /api/dashboards/import endpoint.
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
765 lines
24 KiB
JSON
765 lines
24 KiB
JSON
{
|
|
"__inputs": [
|
|
{ "name": "DS_PROMETHEUS", "label": "Prometheus", "description": "", "type": "datasource", "pluginId": "prometheus" },
|
|
{ "name": "DS_LOKI", "label": "Loki", "description": "", "type": "datasource", "pluginId": "loki" }
|
|
],
|
|
"id": null,
|
|
"uid": "foxhunt-gpu-training",
|
|
"title": "Foxhunt - GPU & Training Jobs",
|
|
"description": "GPU node pools, DCGM metrics, K8s training jobs, CI pipelines",
|
|
"tags": ["foxhunt", "production"],
|
|
"timezone": "browser",
|
|
"schemaVersion": 39,
|
|
"version": 1,
|
|
"editable": true,
|
|
"refresh": "30s",
|
|
"time": {
|
|
"from": "now-1h",
|
|
"to": "now"
|
|
},
|
|
"fiscalYearStartMonth": 0,
|
|
"liveNow": false,
|
|
"weekStart": "",
|
|
"templating": {
|
|
"list": [
|
|
{
|
|
"name": "gpu",
|
|
"type": "query",
|
|
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
|
"query": "label_values(DCGM_FI_DEV_GPU_UTIL, gpu)",
|
|
"refresh": 2,
|
|
"includeAll": true,
|
|
"multi": true,
|
|
"allValue": ".*",
|
|
"current": { "selected": true, "text": "All", "value": "$__all" },
|
|
"sort": 1
|
|
},
|
|
{
|
|
"name": "interval",
|
|
"type": "interval",
|
|
"query": "1m,5m,15m,30m,1h",
|
|
"current": { "selected": true, "text": "5m", "value": "5m" },
|
|
"auto": false
|
|
}
|
|
]
|
|
},
|
|
"panels": [
|
|
{
|
|
"id": 1,
|
|
"type": "row",
|
|
"title": "GPU Node Pool Status",
|
|
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
|
|
"collapsed": false,
|
|
"panels": []
|
|
},
|
|
{
|
|
"id": 2,
|
|
"type": "stat",
|
|
"title": "ci-compile Nodes",
|
|
"description": "L4-1-24G GPU nodes (0-1 autoscale)",
|
|
"gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 },
|
|
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{ "color": "blue", "value": null },
|
|
{ "color": "green", "value": 1 }
|
|
]
|
|
},
|
|
"mappings": [],
|
|
"color": { "mode": "thresholds" }
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
|
"orientation": "auto",
|
|
"textMode": "auto",
|
|
"colorMode": "background",
|
|
"graphMode": "none",
|
|
"justifyMode": "center"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
|
"expr": "count(kube_node_labels{label_k8s_scaleway_com_pool_name=\"ci-compile\"}) or vector(0)",
|
|
"instant": true
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 3,
|
|
"type": "stat",
|
|
"title": "ci-training Nodes",
|
|
"description": "L40S-1-48G GPU nodes (0-1 autoscale)",
|
|
"gridPos": { "h": 4, "w": 4, "x": 4, "y": 1 },
|
|
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{ "color": "blue", "value": null },
|
|
{ "color": "green", "value": 1 }
|
|
]
|
|
},
|
|
"mappings": [],
|
|
"color": { "mode": "thresholds" }
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
|
"orientation": "auto",
|
|
"textMode": "auto",
|
|
"colorMode": "background",
|
|
"graphMode": "none",
|
|
"justifyMode": "center"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
|
"expr": "count(kube_node_labels{label_k8s_scaleway_com_pool_name=\"ci-training\"}) or vector(0)",
|
|
"instant": true
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 4,
|
|
"type": "stat",
|
|
"title": "ci-compile-cpu Nodes",
|
|
"description": "POP2-32C-128G CPU nodes (0-1 autoscale)",
|
|
"gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 },
|
|
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{ "color": "blue", "value": null },
|
|
{ "color": "green", "value": 1 }
|
|
]
|
|
},
|
|
"mappings": [],
|
|
"color": { "mode": "thresholds" }
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
|
"orientation": "auto",
|
|
"textMode": "auto",
|
|
"colorMode": "background",
|
|
"graphMode": "none",
|
|
"justifyMode": "center"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
|
"expr": "count(kube_node_labels{label_k8s_scaleway_com_pool_name=\"ci-compile-cpu\"}) or vector(0)",
|
|
"instant": true
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 5,
|
|
"type": "stat",
|
|
"title": "Active Training Jobs",
|
|
"gridPos": { "h": 4, "w": 4, "x": 12, "y": 1 },
|
|
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{ "color": "blue", "value": null },
|
|
{ "color": "green", "value": 1 }
|
|
]
|
|
},
|
|
"mappings": [],
|
|
"color": { "mode": "thresholds" }
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
|
"orientation": "auto",
|
|
"textMode": "auto",
|
|
"colorMode": "background",
|
|
"graphMode": "none",
|
|
"justifyMode": "center"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
|
"expr": "sum(kube_job_status_active{namespace=\"foxhunt\"}) or vector(0)",
|
|
"instant": true
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 6,
|
|
"type": "stat",
|
|
"title": "Jobs Succeeded (24h)",
|
|
"gridPos": { "h": 4, "w": 4, "x": 16, "y": 1 },
|
|
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{ "color": "green", "value": null }
|
|
]
|
|
},
|
|
"mappings": [],
|
|
"color": { "mode": "thresholds" }
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
|
"orientation": "auto",
|
|
"textMode": "auto",
|
|
"colorMode": "background",
|
|
"graphMode": "none",
|
|
"justifyMode": "center"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
|
"expr": "sum(increase(kube_job_status_succeeded{namespace=\"foxhunt\"}[24h])) or vector(0)",
|
|
"instant": true
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 7,
|
|
"type": "stat",
|
|
"title": "Jobs Failed (24h)",
|
|
"gridPos": { "h": 4, "w": 4, "x": 20, "y": 1 },
|
|
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{ "color": "green", "value": null },
|
|
{ "color": "red", "value": 1 }
|
|
]
|
|
},
|
|
"mappings": [],
|
|
"color": { "mode": "thresholds" }
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
|
"orientation": "auto",
|
|
"textMode": "auto",
|
|
"colorMode": "background",
|
|
"graphMode": "none",
|
|
"justifyMode": "center"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
|
"expr": "sum(increase(kube_job_status_failed{namespace=\"foxhunt\"}[24h])) or vector(0)",
|
|
"instant": true
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 10,
|
|
"type": "row",
|
|
"title": "GPU Utilization (DCGM)",
|
|
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 },
|
|
"collapsed": false,
|
|
"panels": []
|
|
},
|
|
{
|
|
"id": 11,
|
|
"type": "timeseries",
|
|
"title": "GPU Utilization %",
|
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 },
|
|
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineInterpolation": "smooth",
|
|
"fillOpacity": 20,
|
|
"gradientMode": "scheme",
|
|
"spanNulls": false,
|
|
"showPoints": "never",
|
|
"stacking": { "mode": "none", "group": "A" },
|
|
"axisPlacement": "auto",
|
|
"lineWidth": 2
|
|
},
|
|
"unit": "percent",
|
|
"min": 0,
|
|
"max": 100,
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{ "color": "green", "value": null },
|
|
{ "color": "yellow", "value": 80 },
|
|
{ "color": "red", "value": 95 }
|
|
]
|
|
},
|
|
"color": { "mode": "palette-classic" },
|
|
"mappings": []
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"tooltip": { "mode": "multi", "sort": "desc" },
|
|
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] }
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
|
"expr": "DCGM_FI_DEV_GPU_UTIL{gpu=~\"$gpu\"}",
|
|
"legendFormat": "GPU {{ gpu }} ({{ instance }})"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 12,
|
|
"type": "timeseries",
|
|
"title": "GPU Memory Used (MB)",
|
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 },
|
|
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineInterpolation": "smooth",
|
|
"fillOpacity": 20,
|
|
"gradientMode": "scheme",
|
|
"spanNulls": false,
|
|
"showPoints": "never",
|
|
"stacking": { "mode": "none", "group": "A" },
|
|
"axisPlacement": "auto",
|
|
"lineWidth": 2
|
|
},
|
|
"unit": "decmbytes",
|
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
|
"color": { "mode": "palette-classic" },
|
|
"mappings": []
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"tooltip": { "mode": "multi", "sort": "desc" },
|
|
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] }
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
|
"expr": "DCGM_FI_DEV_FB_USED{gpu=~\"$gpu\"}",
|
|
"legendFormat": "GPU {{ gpu }} used"
|
|
},
|
|
{
|
|
"refId": "B",
|
|
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
|
"expr": "DCGM_FI_DEV_FB_TOTAL{gpu=~\"$gpu\"}",
|
|
"legendFormat": "GPU {{ gpu }} total"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 13,
|
|
"type": "timeseries",
|
|
"title": "GPU Temperature",
|
|
"gridPos": { "h": 6, "w": 12, "x": 0, "y": 14 },
|
|
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineInterpolation": "smooth",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "scheme",
|
|
"spanNulls": false,
|
|
"showPoints": "never",
|
|
"stacking": { "mode": "none", "group": "A" },
|
|
"axisPlacement": "auto",
|
|
"lineWidth": 2
|
|
},
|
|
"unit": "celsius",
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{ "color": "blue", "value": null },
|
|
{ "color": "green", "value": 40 },
|
|
{ "color": "yellow", "value": 75 },
|
|
{ "color": "red", "value": 85 }
|
|
]
|
|
},
|
|
"color": { "mode": "palette-classic" },
|
|
"mappings": []
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"tooltip": { "mode": "multi", "sort": "desc" },
|
|
"legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
|
"expr": "DCGM_FI_DEV_GPU_TEMP{gpu=~\"$gpu\"}",
|
|
"legendFormat": "GPU {{ gpu }}"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 14,
|
|
"type": "timeseries",
|
|
"title": "GPU Power (Watts)",
|
|
"gridPos": { "h": 6, "w": 12, "x": 12, "y": 14 },
|
|
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineInterpolation": "smooth",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "scheme",
|
|
"spanNulls": false,
|
|
"showPoints": "never",
|
|
"stacking": { "mode": "none", "group": "A" },
|
|
"axisPlacement": "auto",
|
|
"lineWidth": 2
|
|
},
|
|
"unit": "watt",
|
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
|
"color": { "mode": "palette-classic" },
|
|
"mappings": []
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"tooltip": { "mode": "multi", "sort": "desc" },
|
|
"legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
|
"expr": "DCGM_FI_DEV_POWER_USAGE{gpu=~\"$gpu\"}",
|
|
"legendFormat": "GPU {{ gpu }}"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 20,
|
|
"type": "row",
|
|
"title": "Training Jobs",
|
|
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 20 },
|
|
"collapsed": false,
|
|
"panels": []
|
|
},
|
|
{
|
|
"id": 21,
|
|
"type": "timeseries",
|
|
"title": "Active Jobs Over Time",
|
|
"gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 },
|
|
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineInterpolation": "stepAfter",
|
|
"fillOpacity": 30,
|
|
"gradientMode": "none",
|
|
"spanNulls": false,
|
|
"showPoints": "never",
|
|
"stacking": { "mode": "none", "group": "A" },
|
|
"axisPlacement": "auto",
|
|
"lineWidth": 2
|
|
},
|
|
"unit": "short",
|
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
|
"color": { "mode": "palette-classic" },
|
|
"mappings": []
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"tooltip": { "mode": "multi", "sort": "desc" },
|
|
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["last"] }
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
|
"expr": "kube_job_status_active{namespace=\"foxhunt\"}",
|
|
"legendFormat": "{{ job_name }}"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 22,
|
|
"type": "table",
|
|
"title": "Job Status Summary",
|
|
"gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 },
|
|
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
|
"mappings": [],
|
|
"color": { "mode": "thresholds" }
|
|
},
|
|
"overrides": [
|
|
{
|
|
"matcher": { "id": "byName", "options": "Failed" },
|
|
"properties": [
|
|
{
|
|
"id": "thresholds",
|
|
"value": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{ "color": "green", "value": null },
|
|
{ "color": "red", "value": 1 }
|
|
]
|
|
}
|
|
},
|
|
{ "id": "custom.displayMode", "value": "color-background" }
|
|
]
|
|
}
|
|
]
|
|
},
|
|
"options": {
|
|
"showHeader": true,
|
|
"sortBy": [{ "displayName": "job_name", "desc": false }],
|
|
"footer": { "show": false }
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
|
"expr": "kube_job_status_active{namespace=\"foxhunt\"}",
|
|
"legendFormat": "Active",
|
|
"instant": true,
|
|
"format": "table"
|
|
},
|
|
{
|
|
"refId": "B",
|
|
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
|
"expr": "kube_job_status_succeeded{namespace=\"foxhunt\"}",
|
|
"legendFormat": "Succeeded",
|
|
"instant": true,
|
|
"format": "table"
|
|
},
|
|
{
|
|
"refId": "C",
|
|
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
|
"expr": "kube_job_status_failed{namespace=\"foxhunt\"}",
|
|
"legendFormat": "Failed",
|
|
"instant": true,
|
|
"format": "table"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 25,
|
|
"type": "timeseries",
|
|
"title": "GPU Node CPU Usage (Compute Pools)",
|
|
"description": "CPU usage on GPU/compute node pools when no DCGM data is available",
|
|
"gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 },
|
|
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineInterpolation": "smooth",
|
|
"fillOpacity": 15,
|
|
"gradientMode": "scheme",
|
|
"spanNulls": false,
|
|
"showPoints": "never",
|
|
"stacking": { "mode": "none", "group": "A" },
|
|
"axisPlacement": "auto",
|
|
"lineWidth": 1
|
|
},
|
|
"unit": "percentunit",
|
|
"min": 0,
|
|
"max": 1,
|
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
|
"color": { "mode": "palette-classic" },
|
|
"mappings": []
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"tooltip": { "mode": "multi", "sort": "desc" },
|
|
"legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
|
"expr": "1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[$interval]))",
|
|
"legendFormat": "{{ instance }}"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 26,
|
|
"type": "timeseries",
|
|
"title": "GPU Node Memory Usage (Compute Pools)",
|
|
"gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 },
|
|
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineInterpolation": "smooth",
|
|
"fillOpacity": 15,
|
|
"gradientMode": "scheme",
|
|
"spanNulls": false,
|
|
"showPoints": "never",
|
|
"stacking": { "mode": "none", "group": "A" },
|
|
"axisPlacement": "auto",
|
|
"lineWidth": 1
|
|
},
|
|
"unit": "bytes",
|
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
|
"color": { "mode": "palette-classic" },
|
|
"mappings": []
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"tooltip": { "mode": "multi", "sort": "desc" },
|
|
"legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
|
"expr": "node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes",
|
|
"legendFormat": "{{ instance }}"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 30,
|
|
"type": "row",
|
|
"title": "Training & CI Logs",
|
|
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 35 },
|
|
"collapsed": false,
|
|
"panels": []
|
|
},
|
|
{
|
|
"id": 31,
|
|
"type": "logs",
|
|
"title": "Training Job Logs",
|
|
"gridPos": { "h": 10, "w": 12, "x": 0, "y": 36 },
|
|
"datasource": { "type": "loki", "uid": "${DS_LOKI}" },
|
|
"options": {
|
|
"showTime": true,
|
|
"showLabels": true,
|
|
"showCommonLabels": false,
|
|
"wrapLogMessage": true,
|
|
"prettifyLogMessage": false,
|
|
"enableLogDetails": true,
|
|
"sortOrder": "Descending",
|
|
"dedupStrategy": "none"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": { "type": "loki", "uid": "${DS_LOKI}" },
|
|
"expr": "{namespace=\"foxhunt\", container=~\".*train.*\"}",
|
|
"maxLines": 200
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 32,
|
|
"type": "logs",
|
|
"title": "CI / GitLab Runner Logs",
|
|
"gridPos": { "h": 10, "w": 12, "x": 12, "y": 36 },
|
|
"datasource": { "type": "loki", "uid": "${DS_LOKI}" },
|
|
"options": {
|
|
"showTime": true,
|
|
"showLabels": true,
|
|
"showCommonLabels": false,
|
|
"wrapLogMessage": true,
|
|
"prettifyLogMessage": false,
|
|
"enableLogDetails": true,
|
|
"sortOrder": "Descending",
|
|
"dedupStrategy": "none"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": { "type": "loki", "uid": "${DS_LOKI}" },
|
|
"expr": "{namespace=\"foxhunt\", app=~\".*runner.*|.*gitlab.*\"}",
|
|
"maxLines": 200
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 40,
|
|
"type": "row",
|
|
"title": "Node Autoscale Events",
|
|
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 46 },
|
|
"collapsed": false,
|
|
"panels": []
|
|
},
|
|
{
|
|
"id": 41,
|
|
"type": "timeseries",
|
|
"title": "Node Count Over Time",
|
|
"description": "Tracks autoscaling of ci-compile and ci-training pools",
|
|
"gridPos": { "h": 7, "w": 24, "x": 0, "y": 47 },
|
|
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineInterpolation": "stepAfter",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"spanNulls": false,
|
|
"showPoints": "auto",
|
|
"stacking": { "mode": "none", "group": "A" },
|
|
"axisPlacement": "auto",
|
|
"lineWidth": 2
|
|
},
|
|
"unit": "short",
|
|
"decimals": 0,
|
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
|
"color": { "mode": "palette-classic" },
|
|
"mappings": []
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"tooltip": { "mode": "multi", "sort": "desc" },
|
|
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["min", "max", "last"] }
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
|
"expr": "count by (label_k8s_scaleway_com_pool_name) (kube_node_labels)",
|
|
"legendFormat": "{{ label_k8s_scaleway_com_pool_name }}"
|
|
}
|
|
]
|
|
}
|
|
]
|
|
} |