{ "annotations": { "list": [ { "builtIn": 1, "datasource": { "type": "grafana", "uid": "-- Grafana --" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard" } ] }, "description": "K8s cluster health, node resources, GPU, storage, and Prometheus health", "editable": true, "fiscalYearStartMonth": 0, "links": [], "liveNow": false, "panels": [ { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 1, "panels": [], "title": "Cluster Overview", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "green", "value": 1 } ] } }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 }, "id": 2, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "targets": [ { "expr": "count(kube_node_info)", "legendFormat": "", "refId": "A" } ], "title": "Total Nodes", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "green", "value": 1 } ] } }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 4, "y": 1 }, "id": 3, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "targets": [ { "expr": "sum(kube_node_status_condition{condition=\"Ready\", status=\"true\"})", "legendFormat": "", "refId": "A" } ], "title": "Nodes Ready", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "green", "value": 1 } ] } }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 }, "id": 4, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "targets": [ { "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", phase=\"Running\"})", "legendFormat": "", "refId": "A" } ], "title": "Pods Running", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 12, "y": 1 }, "id": 5, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": { "titleSize": 12 }, "textMode": "auto" }, "targets": [ { "expr": "sum(kube_node_status_capacity{resource=\"cpu\"})", "legendFormat": "", "refId": "A" } ], "title": "Total CPU Capacity", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 16, "y": 1 }, "id": 6, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "targets": [ { "expr": "sum(kube_node_status_capacity{resource=\"memory\"})", "legendFormat": "", "refId": "A" } ], "title": "Total Memory Capacity", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "yellow", "value": null }, { "color": "green", "value": 1 } ] } }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 20, "y": 1 }, "id": 7, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "targets": [ { "expr": "sum(kube_node_status_capacity{resource=\"nvidia_com_gpu\"}) or vector(0)", "legendFormat": "", "refId": "A" } ], "title": "GPUs Available", "type": "stat" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, "id": 10, "panels": [], "title": "Node Resources", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisSoftMax": 1, "axisSoftMin": 0, "drawStyle": "line", "fillOpacity": 30, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" } }, "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 }, "id": 11, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "expr": "(1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[$__rate_interval]))) * on(instance) group_left(nodename) node_uname_info", "legendFormat": "{{ nodename }}", "refId": "A" } ], "title": "CPU Usage by Node (stacked)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "fillOpacity": 20, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "mode": "none" } }, "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 0.8 }, { "color": "red", "value": 0.95 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 }, "id": 12, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "expr": "(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * on(instance) group_left(nodename) node_uname_info", "legendFormat": "{{ nodename }}", "refId": "A" } ], "title": "Memory Usage by Node", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "fillOpacity": 15, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "mode": "none" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "Bps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 14 }, "id": 13, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "expr": "sum by (instance) (rate(node_disk_read_bytes_total[$__rate_interval])) * on(instance) group_left(nodename) node_uname_info", "legendFormat": "{{ nodename }} read", "refId": "A" }, { "expr": "sum by (instance) (rate(node_disk_written_bytes_total[$__rate_interval])) * on(instance) group_left(nodename) node_uname_info", "legendFormat": "{{ nodename }} write", "refId": "B" } ], "title": "Disk I/O (Read / Write)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "fillOpacity": 15, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "mode": "none" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "Bps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 14 }, "id": 14, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "expr": "sum by (instance) (rate(node_network_receive_bytes_total{device!~\"lo|veth.*|cali.*|flannel.*\"}[$__rate_interval])) * on(instance) group_left(nodename) node_uname_info", "legendFormat": "{{ nodename }} rx", "refId": "A" }, { "expr": "sum by (instance) (rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|cali.*|flannel.*\"}[$__rate_interval])) * on(instance) group_left(nodename) node_uname_info", "legendFormat": "{{ nodename }} tx", "refId": "B" } ], "title": "Network I/O (Receive / Transmit)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "fillOpacity": 0, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "mode": "none" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 14 }, "id": 15, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "expr": "node_load1 * on(instance) group_left(nodename) node_uname_info", "legendFormat": "{{ nodename }} load1", "refId": "A" }, { "expr": "node_load5 * on(instance) group_left(nodename) node_uname_info", "legendFormat": "{{ nodename }} load5", "refId": "B" }, { "expr": "node_load15 * on(instance) group_left(nodename) node_uname_info", "legendFormat": "{{ nodename }} load15", "refId": "C" } ], "title": "Load Average (1m / 5m / 15m)", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 22 }, "id": 20, "panels": [], "title": "Kubernetes Workloads", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "fillOpacity": 0, "lineInterpolation": "stepAfter", "lineWidth": 2, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "mode": "none" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [ { "matcher": { "id": "byRegexp", "options": ".*desired" }, "properties": [ { "id": "custom.lineStyle", "value": { "dash": [ 10, 10 ], "fill": "dash" } } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 23 }, "id": 21, "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "expr": "kube_deployment_spec_replicas{namespace=\"foxhunt\"}", "legendFormat": "{{ deployment }} desired", "refId": "A" }, { "expr": "kube_deployment_status_replicas_available{namespace=\"foxhunt\"}", "legendFormat": "{{ deployment }} available", "refId": "B" } ], "title": "Deployment Replicas (Desired vs Available)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "Running" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Pending" }, "properties": [ { "id": "color", "value": { "fixedColor": "yellow", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Failed" }, "properties": [ { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "Succeeded" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 8, "w": 6, "x": 12, "y": 23 }, "id": 22, "options": { "legend": { "displayMode": "list", "placement": "right" }, "pieType": "donut", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "tooltip": { "mode": "single" } }, "targets": [ { "expr": "sum by (phase) (kube_pod_status_phase{namespace=\"foxhunt\"})", "legendFormat": "{{ phase }}", "refId": "A" } ], "title": "Pod Phase Distribution", "type": "piechart" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "bars", "fillOpacity": 80, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "normal" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 6, "x": 18, "y": 23 }, "id": 23, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "expr": "increase(kube_pod_container_status_restarts_total{namespace=\"foxhunt\"}[$__rate_interval])", "legendFormat": "{{ pod }} / {{ container }}", "refId": "A" } ], "title": "Container Restarts", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "Time" }, "properties": [ { "id": "custom.hidden", "value": true } ] }, { "matcher": { "id": "byName", "options": "__name__" }, "properties": [ { "id": "custom.hidden", "value": true } ] }, { "matcher": { "id": "byName", "options": "job" }, "properties": [ { "id": "custom.hidden", "value": true } ] }, { "matcher": { "id": "byName", "options": "instance" }, "properties": [ { "id": "custom.hidden", "value": true } ] } ] }, "gridPos": { "h": 6, "w": 24, "x": 0, "y": 31 }, "id": 24, "options": { "footer": { "show": false }, "showHeader": true, "sortBy": [ { "desc": true, "displayName": "Value" } ] }, "targets": [ { "expr": "kube_pod_status_phase{namespace=\"foxhunt\", phase=~\"Pending|Failed\"} > 0", "format": "table", "instant": true, "legendFormat": "", "refId": "A" } ], "title": "Pending / Failed Pods", "transformations": [ { "id": "organize", "options": { "excludeByName": { "Time": true, "__name__": true, "instance": true, "job": true }, "indexByName": {}, "renameByName": { "Value": "Count", "namespace": "Namespace", "phase": "Phase", "pod": "Pod" } } } ], "type": "table" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 37 }, "id": 30, "panels": [], "title": "Storage", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 0.7 }, { "color": "orange", "value": 0.85 }, { "color": "red", "value": 0.95 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 38 }, "id": 31, "options": { "displayMode": "gradient", "minVizHeight": 10, "minVizWidth": 0, "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showUnfilled": true }, "targets": [ { "expr": "(1 - (node_filesystem_avail_bytes{fstype=~\"ext4|xfs|btrfs\"} / node_filesystem_size_bytes{fstype=~\"ext4|xfs|btrfs\"})) * on(instance) group_left(nodename) node_uname_info", "instant": true, "legendFormat": "{{ nodename }} {{ mountpoint }}", "refId": "A" } ], "title": "Disk Usage by Mountpoint", "type": "bargauge" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "Storage" }, "properties": [ { "id": "unit", "value": "bytes" } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 38 }, "id": 32, "options": { "footer": { "show": false }, "showHeader": true }, "targets": [ { "expr": "kube_persistentvolumeclaim_status_phase{namespace=\"foxhunt\"}", "format": "table", "instant": true, "legendFormat": "", "refId": "A" }, { "expr": "kube_persistentvolumeclaim_resource_requests_storage_bytes{namespace=\"foxhunt\"}", "format": "table", "instant": true, "legendFormat": "", "refId": "B" } ], "title": "PVC Status & Requested Storage", "transformations": [ { "id": "seriesToColumns", "options": { "byField": "persistentvolumeclaim" } }, { "id": "organize", "options": { "excludeByName": { "Time 1": true, "Time 2": true, "__name__ 1": true, "__name__ 2": true, "instance 1": true, "instance 2": true, "job 1": true, "job 2": true, "namespace 1": true, "namespace 2": true, "phase": false }, "renameByName": { "Value 1": "Phase Value", "Value 2": "Storage", "persistentvolumeclaim": "PVC", "phase": "Phase" } } } ], "type": "table" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 46 }, "id": 40, "panels": [], "title": "GPU Resources", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "fillOpacity": 25, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "line" } }, "max": 100, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 80 }, { "color": "red", "value": 95 } ] }, "unit": "percent" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 47 }, "id": 41, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "expr": "DCGM_FI_DEV_GPU_UTIL", "legendFormat": "GPU {{ gpu }} ({{ instance }})", "refId": "A" } ], "title": "GPU Utilization %", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "fillOpacity": 20, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "mode": "none" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "decmbytes" }, "overrides": [ { "matcher": { "id": "byRegexp", "options": ".*free" }, "properties": [ { "id": "custom.fillOpacity", "value": 5 }, { "id": "custom.lineStyle", "value": { "dash": [ 10, 10 ], "fill": "dash" } } ] } ] }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 47 }, "id": 42, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "expr": "DCGM_FI_DEV_FB_USED", "legendFormat": "GPU {{ gpu }} used", "refId": "A" }, { "expr": "DCGM_FI_DEV_FB_FREE", "legendFormat": "GPU {{ gpu }} free", "refId": "B" } ], "title": "VRAM Usage", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "fillOpacity": 0, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "line" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 75 }, { "color": "red", "value": 90 } ] }, "unit": "celsius" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 47 }, "id": 43, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "expr": "DCGM_FI_DEV_GPU_TEMP", "legendFormat": "GPU {{ gpu }} core", "refId": "A" }, { "expr": "DCGM_FI_DEV_MEMORY_TEMP", "legendFormat": "GPU {{ gpu }} mem", "refId": "B" } ], "title": "GPU Temperature", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "fillOpacity": 15, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "mode": "none" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "watt" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 55 }, "id": 44, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "expr": "DCGM_FI_DEV_POWER_USAGE", "legendFormat": "GPU {{ gpu }}", "refId": "A" } ], "title": "GPU Power Draw", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "fillOpacity": 15, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "mode": "none" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "Bps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 55 }, "id": 45, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "expr": "DCGM_FI_PROF_PCIE_RX_BYTES", "legendFormat": "GPU {{ gpu }} rx", "refId": "A" }, { "expr": "DCGM_FI_PROF_PCIE_TX_BYTES", "legendFormat": "GPU {{ gpu }} tx", "refId": "B" } ], "title": "GPU PCIe Throughput", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 63 }, "id": 50, "panels": [], "title": "Prometheus Health", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 500000 }, { "color": "red", "value": 1000000 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 4, "w": 6, "x": 0, "y": 64 }, "id": 51, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "targets": [ { "expr": "prometheus_tsdb_head_series", "legendFormat": "", "refId": "A" } ], "title": "Active Time Series", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 5368709120 }, { "color": "red", "value": 10737418240 } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 4, "w": 6, "x": 6, "y": 64 }, "id": 52, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "targets": [ { "expr": "prometheus_tsdb_storage_blocks_bytes", "legendFormat": "", "refId": "A" } ], "title": "TSDB Storage Size", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "down" }, "properties": [ { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 4, "w": 6, "x": 12, "y": 64 }, "id": 53, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "value_and_name" }, "targets": [ { "expr": "count(up == 1)", "legendFormat": "up", "refId": "A" }, { "expr": "count(up == 0)", "legendFormat": "down", "refId": "B" } ], "title": "Scrape Targets Up", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "fillOpacity": 10, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "mode": "none" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 5 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 68 }, "id": 54, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "expr": "scrape_duration_seconds", "legendFormat": "{{ job }} / {{ instance }}", "refId": "A" } ], "title": "Scrape Duration by Job", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [ { "options": { "0": { "color": "red", "text": "DOWN" }, "1": { "color": "green", "text": "UP" } }, "type": "value" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "green", "value": 1 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "Time" }, "properties": [ { "id": "custom.hidden", "value": true } ] }, { "matcher": { "id": "byName", "options": "__name__" }, "properties": [ { "id": "custom.hidden", "value": true } ] }, { "matcher": { "id": "byName", "options": "Value" }, "properties": [ { "id": "custom.cellOptions", "value": { "type": "color-background" } } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 68 }, "id": 55, "options": { "footer": { "show": false }, "showHeader": true, "sortBy": [ { "desc": false, "displayName": "Value" } ] }, "targets": [ { "expr": "up", "format": "table", "instant": true, "legendFormat": "", "refId": "A" } ], "title": "Scrape Targets Status", "transformations": [ { "id": "organize", "options": { "excludeByName": { "Time": true, "__name__": true }, "renameByName": { "Value": "Status", "instance": "Instance", "job": "Job" } } } ], "type": "table" } ], "refresh": "30s", "schemaVersion": 39, "style": "dark", "tags": [ "foxhunt", "infrastructure", "k8s" ], "templating": { "list": [ { "auto": true, "auto_count": 30, "auto_min": "1m", "current": { "selected": true, "text": "auto", "value": "$__auto_interval_interval" }, "hide": 0, "name": "interval", "query": "1m,5m,15m,30m,1h", "refresh": 2, "type": "interval" }, { "allValue": ".*", "current": { "selected": true, "text": "All", "value": "$__all" }, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "hide": 0, "includeAll": true, "multi": true, "name": "node", "query": "label_values(node_uname_info, nodename)", "refresh": 2, "sort": 1, "type": "query" } ] }, "time": { "from": "now-3h", "to": "now" }, "timezone": "browser", "title": "Infrastructure & Cluster", "uid": "foxhunt-infrastructure", "weekStart": "", "__inputs": [ { "name": "DS_PROMETHEUS", "type": "datasource", "pluginId": "prometheus" }, { "name": "DS_LOKI", "type": "datasource", "pluginId": "loki" }, { "name": "DS_TEMPO", "type": "datasource", "pluginId": "${DS_TEMPO}" } ] }