From b3c85d4f7ca97b36ded4337d22e8f72f859cc2cb Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Thu, 5 Mar 2026 17:43:25 +0100 Subject: [PATCH] =?UTF-8?q?fix(infra):=20resolve=20dashboard=20datasource?= =?UTF-8?q?=20UIDs,=20fix=20Grafana=E2=86=92Prometheus=20netpol?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove duplicate annotated-pods and gitlab-annotated-pods scrape jobs (all targets already covered by dedicated jobs) - Drop Argo controller from foxhunt-services job (dedicated HTTPS job exists) - Resolve datasource variable placeholders to actual UIDs in dashboard JSON - Add Ingress rule to Prometheus netpol allowing Grafana on port 9090 Targets: 49 → 18 (zero duplicates) Co-Authored-By: Claude Opus 4.6 --- .../dashboards/foxhunt-cockpit.json | 1005 ++++++++++--- .../dashboards/foxhunt-infrastructure.json | 1071 +++++++++++--- .../dashboards/foxhunt-observability.json | 741 ++++++++-- .../dashboards/foxhunt-trading.json | 91 +- .../dashboards/foxhunt-training.json | 1243 +++++++++++++---- infra/k8s/monitoring/prometheus.yaml | 55 +- infra/k8s/network-policies/prometheus.yaml | 10 + 7 files changed, 3339 insertions(+), 877 deletions(-) diff --git a/infra/k8s/monitoring/dashboards/foxhunt-cockpit.json b/infra/k8s/monitoring/dashboards/foxhunt-cockpit.json index 709f0beb8..bef3b5b92 100644 --- a/infra/k8s/monitoring/dashboards/foxhunt-cockpit.json +++ b/infra/k8s/monitoring/dashboards/foxhunt-cockpit.json @@ -1,13 +1,4 @@ { - "__inputs": [ - { - "name": "DS_PROMETHEUS", - "label": "Prometheus", - "description": "", - "type": "datasource", - "pluginId": "prometheus" - } - ], "id": null, "uid": "foxhunt-cockpit", "title": "Foxhunt Operations Cockpit", @@ -51,7 +42,12 @@ "id": 1, "type": "row", "title": "Health Status Bar", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, "collapsed": false, "panels": [] }, @@ -59,8 +55,16 @@ "id": 2, "type": "stat", "title": "Services Running", - "gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 1 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "count(up{job=~\"foxhunt.*|pushgateway|argo-workflows\"} == 1)", @@ -78,9 +82,18 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "red", "value": null }, - { "color": "orange", "value": 3 }, - { "color": "green", "value": 6 } + { + "color": "red", + "value": null + }, + { + "color": "orange", + "value": 3 + }, + { + "color": "green", + "value": 6 + } ] }, "unit": "none", @@ -89,7 +102,11 @@ "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, "colorMode": "background", "graphMode": "none", "textMode": "auto", @@ -101,8 +118,16 @@ "id": 3, "type": "stat", "title": "Total Request Rate", - "gridPos": { "h": 4, "w": 4, "x": 4, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 1 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "sum(rate(api_gateway_http_requests_total[5m]))", @@ -115,7 +140,10 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "blue", "value": null } + { + "color": "blue", + "value": null + } ] }, "unit": "reqps", @@ -125,7 +153,11 @@ "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, "colorMode": "background", "graphMode": "none", "textMode": "auto", @@ -136,8 +168,16 @@ "id": 4, "type": "stat", "title": "Error Rate", - "gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 1 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "sum(rate(api_gateway_http_requests_total{status=~\"5..\"}[5m])) / sum(rate(api_gateway_http_requests_total[5m])) * 100", @@ -150,9 +190,18 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 1 }, - { "color": "red", "value": 5 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 5 + } ] }, "unit": "percent", @@ -164,7 +213,11 @@ "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, "colorMode": "background", "graphMode": "none", "textMode": "auto", @@ -175,8 +228,16 @@ "id": 5, "type": "stat", "title": "Active Workflows", - "gridPos": { "h": 4, "w": 4, "x": 12, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 1 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "argo_workflows_gauge{status=\"Running\"}", @@ -189,8 +250,14 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "blue", "value": null }, - { "color": "green", "value": 1 } + { + "color": "blue", + "value": null + }, + { + "color": "green", + "value": 1 + } ] }, "unit": "none", @@ -199,7 +266,11 @@ "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, "colorMode": "background", "graphMode": "none", "textMode": "auto", @@ -210,8 +281,16 @@ "id": 6, "type": "stat", "title": "GPU Utilization", - "gridPos": { "h": 4, "w": 4, "x": 16, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 1 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "avg(DCGM_FI_DEV_GPU_UTIL)", @@ -224,9 +303,18 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 70 }, - { "color": "red", "value": 90 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 70 + }, + { + "color": "red", + "value": 90 + } ] }, "unit": "percent", @@ -238,7 +326,11 @@ "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, "colorMode": "background", "graphMode": "none", "textMode": "auto", @@ -249,8 +341,16 @@ "id": 7, "type": "stat", "title": "Training Active", - "gridPos": { "h": 4, "w": 4, "x": 20, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 1 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "ml_training_active_workers or count(foxhunt_training_current_epoch > 0)", @@ -263,8 +363,14 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "semi-dark-blue", "value": null }, - { "color": "green", "value": 1 } + { + "color": "semi-dark-blue", + "value": null + }, + { + "color": "green", + "value": 1 + } ] }, "unit": "none", @@ -273,7 +379,10 @@ "type": "special", "options": { "match": "null", - "result": { "text": "0", "color": "text" } + "result": { + "text": "0", + "color": "text" + } } } ] @@ -281,7 +390,11 @@ "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, "colorMode": "background", "graphMode": "none", "textMode": "auto", @@ -292,7 +405,12 @@ "id": 8, "type": "row", "title": "Service Health Matrix", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, "collapsed": false, "panels": [] }, @@ -300,8 +418,16 @@ "id": 9, "type": "table", "title": "Service Status Table", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 6 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "up{job=~\"foxhunt.*\"}", @@ -344,81 +470,177 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null } + { + "color": "green", + "value": null + } ] }, "mappings": [] }, "overrides": [ { - "matcher": { "id": "byName", "options": "Value #A" }, + "matcher": { + "id": "byName", + "options": "Value #A" + }, "properties": [ - { "id": "displayName", "value": "Status" }, + { + "id": "displayName", + "value": "Status" + }, { "id": "mappings", "value": [ - { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } } + { + "type": "value", + "options": { + "0": { + "text": "DOWN", + "color": "red" + }, + "1": { + "text": "UP", + "color": "green" + } + } + } ] }, - { "id": "custom.cellOptions", "value": { "type": "color-background" } } + { + "id": "custom.cellOptions", + "value": { + "type": "color-background" + } + } ] }, { - "matcher": { "id": "byName", "options": "Value #B" }, + "matcher": { + "id": "byName", + "options": "Value #B" + }, "properties": [ - { "id": "displayName", "value": "Uptime" }, - { "id": "unit", "value": "s" } + { + "id": "displayName", + "value": "Uptime" + }, + { + "id": "unit", + "value": "s" + } ] }, { - "matcher": { "id": "byName", "options": "Value #C" }, + "matcher": { + "id": "byName", + "options": "Value #C" + }, "properties": [ - { "id": "displayName", "value": "Request Rate" }, - { "id": "unit", "value": "reqps" }, - { "id": "decimals", "value": 2 } + { + "id": "displayName", + "value": "Request Rate" + }, + { + "id": "unit", + "value": "reqps" + }, + { + "id": "decimals", + "value": 2 + } ] }, { - "matcher": { "id": "byName", "options": "Value #D" }, + "matcher": { + "id": "byName", + "options": "Value #D" + }, "properties": [ - { "id": "displayName", "value": "Error Rate %" }, - { "id": "unit", "value": "percent" }, - { "id": "decimals", "value": 2 }, + { + "id": "displayName", + "value": "Error Rate %" + }, + { + "id": "unit", + "value": "percent" + }, + { + "id": "decimals", + "value": 2 + }, { "id": "thresholds", "value": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 1 }, - { "color": "red", "value": 5 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 5 + } ] } }, - { "id": "custom.cellOptions", "value": { "type": "color-background" } } + { + "id": "custom.cellOptions", + "value": { + "type": "color-background" + } + } ] }, { - "matcher": { "id": "byName", "options": "Value #E" }, + "matcher": { + "id": "byName", + "options": "Value #E" + }, "properties": [ - { "id": "displayName", "value": "P99 Latency" }, - { "id": "unit", "value": "s" }, - { "id": "decimals", "value": 3 } + { + "id": "displayName", + "value": "P99 Latency" + }, + { + "id": "unit", + "value": "s" + }, + { + "id": "decimals", + "value": 3 + } ] }, { - "matcher": { "id": "byName", "options": "job" }, + "matcher": { + "id": "byName", + "options": "job" + }, "properties": [ - { "id": "displayName", "value": "Service" } + { + "id": "displayName", + "value": "Service" + } ] } ] }, "options": { "showHeader": true, - "footer": { "show": false }, + "footer": { + "show": false + }, "sortBy": [ - { "displayName": "Service", "desc": false } + { + "displayName": "Service", + "desc": false + } ] }, "transformations": [ @@ -442,8 +664,16 @@ "id": 10, "type": "timeseries", "title": "Service Request Rate", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 6 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "sum by (service) (rate(api_gateway_http_requests_total[$interval]))", @@ -456,7 +686,10 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null } + { + "color": "green", + "value": null + } ] }, "unit": "reqps", @@ -464,7 +697,10 @@ "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 30, - "stacking": { "mode": "normal", "group": "A" }, + "stacking": { + "mode": "normal", + "group": "A" + }, "lineWidth": 1, "pointSize": 5, "showPoints": "never", @@ -476,15 +712,31 @@ "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "lastNotNull"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + } } }, { "id": 11, "type": "row", "title": "Argo CI/CD Pipeline", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, "collapsed": false, "panels": [] }, @@ -492,8 +744,16 @@ "id": 12, "type": "stat", "title": "Workflow Status", - "gridPos": { "h": 8, "w": 6, "x": 0, "y": 15 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 15 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "argo_workflows_gauge{status=\"Running\"}", @@ -521,7 +781,10 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "text", "value": null } + { + "color": "text", + "value": null + } ] }, "unit": "none", @@ -529,33 +792,73 @@ }, "overrides": [ { - "matcher": { "id": "byName", "options": "Running" }, + "matcher": { + "id": "byName", + "options": "Running" + }, "properties": [ - { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } ] }, { - "matcher": { "id": "byName", "options": "Succeeded" }, + "matcher": { + "id": "byName", + "options": "Succeeded" + }, "properties": [ - { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } ] }, { - "matcher": { "id": "byName", "options": "Failed" }, + "matcher": { + "id": "byName", + "options": "Failed" + }, "properties": [ - { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } ] }, { - "matcher": { "id": "byName", "options": "Pending" }, + "matcher": { + "id": "byName", + "options": "Pending" + }, "properties": [ - { "id": "color", "value": { "fixedColor": "yellow", "mode": "fixed" } } + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } ] } ] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, "colorMode": "background", "graphMode": "none", "textMode": "auto", @@ -567,8 +870,16 @@ "id": 13, "type": "timeseries", "title": "Workflow Duration", - "gridPos": { "h": 8, "w": 9, "x": 6, "y": 15 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 8, + "w": 9, + "x": 6, + "y": 15 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "histogram_quantile(0.50, sum by (le) (rate(argo_workflows_operation_duration_seconds_bucket[5m])))", @@ -586,7 +897,10 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null } + { + "color": "green", + "value": null + } ] }, "unit": "s", @@ -604,30 +918,67 @@ }, "overrides": [ { - "matcher": { "id": "byName", "options": "p95" }, + "matcher": { + "id": "byName", + "options": "p95" + }, "properties": [ - { "id": "color", "value": { "fixedColor": "orange", "mode": "fixed" } } + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } ] }, { - "matcher": { "id": "byName", "options": "p50" }, + "matcher": { + "id": "byName", + "options": "p50" + }, "properties": [ - { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } ] } ] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "lastNotNull"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + } } }, { "id": 14, "type": "timeseries", "title": "Workflow Errors", - "gridPos": { "h": 8, "w": 9, "x": 15, "y": 15 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 8, + "w": 9, + "x": 15, + "y": 15 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "rate(argo_workflows_error_count[5m])", @@ -640,7 +991,10 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null } + { + "color": "green", + "value": null + } ] }, "unit": "ops", @@ -648,28 +1002,48 @@ "drawStyle": "bars", "lineInterpolation": "linear", "fillOpacity": 80, - "stacking": { "mode": "normal", "group": "A" }, + "stacking": { + "mode": "normal", + "group": "A" + }, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "gradientMode": "none", "axisPlacement": "auto" }, - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "mappings": [] }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["sum", "lastNotNull"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "sum", + "lastNotNull" + ] + } } }, { "id": 15, "type": "row", "title": "Resource Overview", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, "collapsed": false, "panels": [] }, @@ -677,8 +1051,16 @@ "id": 16, "type": "gauge", "title": "CPU Usage", - "gridPos": { "h": 8, "w": 6, "x": 0, "y": 24 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 24 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "avg(1 - rate(node_cpu_seconds_total{mode=\"idle\"}[5m])) * 100", @@ -691,9 +1073,18 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 60 }, - { "color": "red", "value": 80 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 60 + }, + { + "color": "red", + "value": 80 + } ] }, "unit": "percent", @@ -705,7 +1096,11 @@ "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, "showThresholdLabels": false, "showThresholdMarkers": true, "orientation": "auto" @@ -715,8 +1110,16 @@ "id": 17, "type": "gauge", "title": "Memory Usage", - "gridPos": { "h": 8, "w": 6, "x": 6, "y": 24 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 24 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "(1 - (node_memory_AvailableBytes / node_memory_MemoryTotal_bytes)) * 100", @@ -729,9 +1132,18 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 60 }, - { "color": "red", "value": 80 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 60 + }, + { + "color": "red", + "value": 80 + } ] }, "unit": "percent", @@ -743,7 +1155,11 @@ "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, "showThresholdLabels": false, "showThresholdMarkers": true, "orientation": "auto" @@ -753,8 +1169,16 @@ "id": 18, "type": "gauge", "title": "Disk Usage", - "gridPos": { "h": 8, "w": 6, "x": 12, "y": 24 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 24 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "(1 - (node_filesystem_avail_bytes{mountpoint=\"/\"} / node_filesystem_size_bytes{mountpoint=\"/\"})) * 100", @@ -767,9 +1191,18 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 60 }, - { "color": "red", "value": 80 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 60 + }, + { + "color": "red", + "value": 80 + } ] }, "unit": "percent", @@ -781,7 +1214,11 @@ "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, "showThresholdLabels": false, "showThresholdMarkers": true, "orientation": "auto" @@ -791,8 +1228,16 @@ "id": 19, "type": "timeseries", "title": "Network IO", - "gridPos": { "h": 8, "w": 6, "x": 18, "y": 24 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 24 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "rate(node_network_receive_bytes_total{device!=\"lo\"}[5m])", @@ -810,7 +1255,10 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null } + { + "color": "green", + "value": null + } ] }, "unit": "Bps", @@ -828,23 +1276,44 @@ }, "overrides": [ { - "matcher": { "id": "byRegexp", "options": "/Transmit.*/" }, + "matcher": { + "id": "byRegexp", + "options": "/Transmit.*/" + }, "properties": [ - { "id": "custom.transform", "value": "negative-Y" } + { + "id": "custom.transform", + "value": "negative-Y" + } ] } ] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + } } }, { "id": 20, "type": "row", "title": "gRPC & Backend", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 32 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 32 + }, "collapsed": false, "panels": [] }, @@ -852,8 +1321,16 @@ "id": 21, "type": "timeseries", "title": "gRPC Request Rate", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 33 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 33 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "sum by (grpc_service, grpc_method) (rate(grpc_server_handled_total[5m]))", @@ -866,7 +1343,10 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null } + { + "color": "green", + "value": null + } ] }, "unit": "ops", @@ -874,29 +1354,53 @@ "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 20, - "stacking": { "mode": "normal", "group": "A" }, + "stacking": { + "mode": "normal", + "group": "A" + }, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "gradientMode": "opacity", "axisPlacement": "auto" }, - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "mappings": [] }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "lastNotNull"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + } } }, { "id": 22, "type": "timeseries", "title": "gRPC Latency", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 33 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 33 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "histogram_quantile(0.50, sum by (le, grpc_service) (rate(grpc_server_handling_seconds_bucket[5m])))", @@ -919,7 +1423,10 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null } + { + "color": "green", + "value": null + } ] }, "unit": "s", @@ -933,36 +1440,80 @@ "gradientMode": "none", "axisPlacement": "auto" }, - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "mappings": [] }, "overrides": [ { - "matcher": { "id": "byRegexp", "options": "/p99.*/" }, + "matcher": { + "id": "byRegexp", + "options": "/p99.*/" + }, "properties": [ - { "id": "custom.lineStyle", "value": { "fill": "solid" } }, - { "id": "custom.lineWidth", "value": 2 } + { + "id": "custom.lineStyle", + "value": { + "fill": "solid" + } + }, + { + "id": "custom.lineWidth", + "value": 2 + } ] }, { - "matcher": { "id": "byRegexp", "options": "/p50.*/" }, + "matcher": { + "id": "byRegexp", + "options": "/p50.*/" + }, "properties": [ - { "id": "custom.lineStyle", "value": { "fill": "dash", "dash": [10, 10] } }, - { "id": "custom.lineWidth", "value": 1 } + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 10, + 10 + ] + } + }, + { + "id": "custom.lineWidth", + "value": 1 + } ] } ] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "lastNotNull"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + } } }, { "id": 23, "type": "row", "title": "Rate Limiting & Auth", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 41 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 41 + }, "collapsed": false, "panels": [] }, @@ -970,8 +1521,16 @@ "id": 24, "type": "timeseries", "title": "Rate Limit Rejections", - "gridPos": { "h": 8, "w": 8, "x": 0, "y": 42 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 42 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "rate(api_gateway_rate_limit_rejected_total[5m])", @@ -984,7 +1543,10 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null } + { + "color": "green", + "value": null + } ] }, "unit": "ops", @@ -998,22 +1560,43 @@ "gradientMode": "opacity", "axisPlacement": "auto" }, - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "mappings": [] }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["sum", "mean", "lastNotNull"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "sum", + "mean", + "lastNotNull" + ] + } } }, { "id": 25, "type": "timeseries", "title": "Auth Success/Fail", - "gridPos": { "h": 8, "w": 8, "x": 8, "y": 42 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 42 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "rate(api_gateway_auth_attempts_total{result=\"success\"}[5m])", @@ -1031,7 +1614,10 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null } + { + "color": "green", + "value": null + } ] }, "unit": "ops", @@ -1049,30 +1635,67 @@ }, "overrides": [ { - "matcher": { "id": "byName", "options": "Success" }, + "matcher": { + "id": "byName", + "options": "Success" + }, "properties": [ - { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } ] }, { - "matcher": { "id": "byName", "options": "Failure" }, + "matcher": { + "id": "byName", + "options": "Failure" + }, "properties": [ - { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } ] } ] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["sum", "mean", "lastNotNull"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "sum", + "mean", + "lastNotNull" + ] + } } }, { "id": 26, "type": "stat", "title": "JWT Cache Hit Rate", - "gridPos": { "h": 8, "w": 8, "x": 16, "y": 42 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 42 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "api_gateway_jwt_cache_hit_total / (api_gateway_jwt_cache_hit_total + api_gateway_jwt_cache_miss_total) * 100", @@ -1085,9 +1708,18 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "red", "value": null }, - { "color": "yellow", "value": 50 }, - { "color": "green", "value": 80 } + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 50 + }, + { + "color": "green", + "value": 80 + } ] }, "unit": "percent", @@ -1099,14 +1731,20 @@ "type": "special", "options": { "match": "null", - "result": { "text": "N/A", "color": "text" } + "result": { + "text": "N/A", + "color": "text" + } } }, { "type": "special", "options": { "match": "nan", - "result": { "text": "N/A", "color": "text" } + "result": { + "text": "N/A", + "color": "text" + } } } ] @@ -1114,7 +1752,11 @@ "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, "colorMode": "background", "graphMode": "none", "textMode": "auto", @@ -1126,7 +1768,10 @@ "list": [ { "builtIn": 1, - "datasource": { "type": "grafana", "uid": "-- Grafana --" }, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", diff --git a/infra/k8s/monitoring/dashboards/foxhunt-infrastructure.json b/infra/k8s/monitoring/dashboards/foxhunt-infrastructure.json index 027a97f16..925b6fe5f 100644 --- a/infra/k8s/monitoring/dashboards/foxhunt-infrastructure.json +++ b/infra/k8s/monitoring/dashboards/foxhunt-infrastructure.json @@ -1,13 +1,4 @@ { - "__inputs": [ - { - "name": "DS_PROMETHEUS", - "label": "Prometheus", - "description": "", - "type": "datasource", - "pluginId": "prometheus" - } - ], "id": null, "uid": "foxhunt-infrastructure", "title": "Infrastructure & Cluster", @@ -51,7 +42,7 @@ "type": "query", "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "PBFA97CFB590B2093" }, "query": "label_values(node_uname_info, nodename)", "refresh": 2, @@ -73,7 +64,12 @@ "id": 100, "type": "row", "title": "Cluster Health", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, "collapsed": false, "panels": [] }, @@ -81,8 +77,16 @@ "id": 1, "type": "stat", "title": "Nodes Ready", - "gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 1 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "sum(kube_node_status_condition{condition=\"Ready\",status=\"true\"})", @@ -96,8 +100,14 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "red", "value": null }, - { "color": "green", "value": 1 } + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } ] }, "unit": "none", @@ -106,7 +116,11 @@ "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, "colorMode": "background", "graphMode": "none", "textMode": "auto", @@ -117,8 +131,16 @@ "id": 2, "type": "stat", "title": "Total Pods", - "gridPos": { "h": 4, "w": 4, "x": 4, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 1 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\",phase=\"Running\"})", @@ -132,8 +154,14 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "red", "value": null }, - { "color": "green", "value": 1 } + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } ] }, "unit": "none", @@ -142,7 +170,11 @@ "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, "colorMode": "background", "graphMode": "none", "textMode": "auto", @@ -153,8 +185,16 @@ "id": 3, "type": "stat", "title": "Pending Pods", - "gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 1 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\",phase=\"Pending\"})", @@ -168,8 +208,14 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "red", "value": 1 } + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } ] }, "unit": "none", @@ -178,7 +224,11 @@ "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, "colorMode": "background", "graphMode": "none", "textMode": "auto", @@ -189,8 +239,16 @@ "id": 4, "type": "stat", "title": "Restarts (1h)", - "gridPos": { "h": 4, "w": 4, "x": 12, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 1 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "sum(increase(kube_pod_container_status_restarts_total{namespace=\"foxhunt\"}[1h]))", @@ -204,9 +262,18 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "orange", "value": 3 }, - { "color": "red", "value": 5 } + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 3 + }, + { + "color": "red", + "value": 5 + } ] }, "unit": "none", @@ -216,7 +283,11 @@ "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, "colorMode": "background", "graphMode": "none", "textMode": "auto", @@ -227,8 +298,16 @@ "id": 5, "type": "stat", "title": "CPU Request %", - "gridPos": { "h": 4, "w": 4, "x": 16, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 1 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "sum(kube_pod_container_resource_requests{namespace=\"foxhunt\",resource=\"cpu\"}) / sum(kube_node_status_allocatable{resource=\"cpu\"}) * 100", @@ -242,9 +321,18 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "orange", "value": 70 }, - { "color": "red", "value": 90 } + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 70 + }, + { + "color": "red", + "value": 90 + } ] }, "unit": "percent", @@ -254,7 +342,11 @@ "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, "colorMode": "background", "graphMode": "none", "textMode": "auto", @@ -265,8 +357,16 @@ "id": 6, "type": "stat", "title": "Memory Request %", - "gridPos": { "h": 4, "w": 4, "x": 20, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 1 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "sum(kube_pod_container_resource_requests{namespace=\"foxhunt\",resource=\"memory\"}) / sum(kube_node_status_allocatable{resource=\"memory\"}) * 100", @@ -280,9 +380,18 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "orange", "value": 70 }, - { "color": "red", "value": 90 } + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 70 + }, + { + "color": "red", + "value": 90 + } ] }, "unit": "percent", @@ -292,7 +401,11 @@ "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, "colorMode": "background", "graphMode": "none", "textMode": "auto", @@ -303,7 +416,12 @@ "id": 101, "type": "row", "title": "Node CPU & Memory", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, "collapsed": false, "panels": [] }, @@ -311,8 +429,16 @@ "id": 7, "type": "timeseries", "title": "CPU Usage by Node", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 6 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[$interval]))", @@ -325,9 +451,18 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "orange", "value": 0.7 }, - { "color": "red", "value": 0.9 } + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.7 + }, + { + "color": "red", + "value": 0.9 + } ] }, "unit": "percentunit", @@ -343,23 +478,45 @@ "spanNulls": true, "axisCenteredZero": false, "axisColorMode": "text", - "scaleDistribution": { "type": "linear" }, - "thresholdsStyle": { "mode": "off" } + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + } } }, { "id": 8, "type": "timeseries", "title": "Memory Usage by Node", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 6 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "1 - (node_memory_AvailableBytes / node_memory_MemoryTotal_bytes)", @@ -372,9 +529,18 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "orange", "value": 0.7 }, - { "color": "red", "value": 0.9 } + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.7 + }, + { + "color": "red", + "value": 0.9 + } ] }, "unit": "percentunit", @@ -390,22 +556,41 @@ "spanNulls": true, "axisCenteredZero": false, "axisColorMode": "text", - "scaleDistribution": { "type": "linear" }, - "thresholdsStyle": { "mode": "off" } + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + } } }, { "id": 102, "type": "row", "title": "Pod Resources", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, "collapsed": false, "panels": [] }, @@ -413,8 +598,16 @@ "id": 9, "type": "timeseries", "title": "Pod CPU Usage", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 15 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 15 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "topk(10, sum by (pod) (rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\",container!=\"POD\",container!=\"\"}[$interval])))", @@ -427,7 +620,10 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null } + { + "color": "green", + "value": null + } ] }, "unit": "short", @@ -442,23 +638,45 @@ "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "CPU cores", - "scaleDistribution": { "type": "linear" }, - "thresholdsStyle": { "mode": "off" } + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + } } }, { "id": 10, "type": "timeseries", "title": "Pod Memory Usage", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 15 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 15 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "topk(10, sum by (pod) (container_memory_working_set_bytes{namespace=\"foxhunt\",container!=\"POD\",container!=\"\"}))", @@ -471,7 +689,10 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null } + { + "color": "green", + "value": null + } ] }, "unit": "bytes", @@ -485,22 +706,41 @@ "spanNulls": true, "axisCenteredZero": false, "axisColorMode": "text", - "scaleDistribution": { "type": "linear" }, - "thresholdsStyle": { "mode": "off" } + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + } } }, { "id": 103, "type": "row", "title": "Storage", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, "collapsed": false, "panels": [] }, @@ -508,8 +748,16 @@ "id": 11, "type": "table", "title": "PVC Usage", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "kubelet_volume_stats_used_bytes{namespace=\"foxhunt\"}", @@ -538,57 +786,138 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "orange", "value": 70 }, - { "color": "red", "value": 90 } + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 70 + }, + { + "color": "red", + "value": 90 + } ] }, "custom": { "align": "auto", - "cellOptions": { "type": "auto" }, + "cellOptions": { + "type": "auto" + }, "filterable": true } }, "overrides": [ { - "matcher": { "id": "byName", "options": "Value #A" }, + "matcher": { + "id": "byName", + "options": "Value #A" + }, "properties": [ - { "id": "displayName", "value": "Used" }, - { "id": "unit", "value": "bytes" } + { + "id": "displayName", + "value": "Used" + }, + { + "id": "unit", + "value": "bytes" + } ] }, { - "matcher": { "id": "byName", "options": "Value #B" }, + "matcher": { + "id": "byName", + "options": "Value #B" + }, "properties": [ - { "id": "displayName", "value": "Total" }, - { "id": "unit", "value": "bytes" } + { + "id": "displayName", + "value": "Total" + }, + { + "id": "unit", + "value": "bytes" + } ] }, { - "matcher": { "id": "byName", "options": "Value #C" }, + "matcher": { + "id": "byName", + "options": "Value #C" + }, "properties": [ - { "id": "displayName", "value": "Used %" }, - { "id": "unit", "value": "percent" }, - { "id": "custom.cellOptions", "value": { "mode": "gradient", "type": "gauge" } }, - { "id": "thresholds", "value": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "orange", "value": 70 }, { "color": "red", "value": 90 }] } } + { + "id": "displayName", + "value": "Used %" + }, + { + "id": "unit", + "value": "percent" + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "gauge" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 70 + }, + { + "color": "red", + "value": 90 + } + ] + } + } ] }, { - "matcher": { "id": "byName", "options": "persistentvolumeclaim" }, + "matcher": { + "id": "byName", + "options": "persistentvolumeclaim" + }, "properties": [ - { "id": "displayName", "value": "PVC" } + { + "id": "displayName", + "value": "PVC" + } ] }, { - "matcher": { "id": "byName", "options": "Time" }, + "matcher": { + "id": "byName", + "options": "Time" + }, "properties": [ - { "id": "custom.hidden", "value": true } + { + "id": "custom.hidden", + "value": true + } ] }, { - "matcher": { "id": "byName", "options": "__name__" }, + "matcher": { + "id": "byName", + "options": "__name__" + }, "properties": [ - { "id": "custom.hidden", "value": true } + { + "id": "custom.hidden", + "value": true + } ] } ] @@ -596,9 +925,14 @@ "options": { "showHeader": true, "sortBy": [ - { "displayName": "Used %", "desc": true } + { + "displayName": "Used %", + "desc": true + } ], - "footer": { "show": false } + "footer": { + "show": false + } }, "transformations": [ { @@ -631,8 +965,16 @@ "id": 12, "type": "timeseries", "title": "Disk I/O", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "rate(node_disk_read_bytes_total{device=~\"sd.*|nvme.*\"}[$interval])", @@ -650,7 +992,10 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null } + { + "color": "green", + "value": null + } ] }, "unit": "Bps", @@ -664,29 +1009,54 @@ "spanNulls": true, "axisCenteredZero": false, "axisColorMode": "text", - "scaleDistribution": { "type": "linear" }, - "thresholdsStyle": { "mode": "off" } + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } } }, "overrides": [ { - "matcher": { "id": "byRegexp", "options": ".*write.*" }, + "matcher": { + "id": "byRegexp", + "options": ".*write.*" + }, "properties": [ - { "id": "custom.transform", "value": "negative-Y" } + { + "id": "custom.transform", + "value": "negative-Y" + } ] } ] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + } } }, { "id": 104, "type": "row", "title": "Network", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 32 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 32 + }, "collapsed": false, "panels": [] }, @@ -694,8 +1064,16 @@ "id": 13, "type": "timeseries", "title": "Network Receive", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 33 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 33 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "sum by (instance) (rate(node_network_receive_bytes_total{device!=\"lo\"}[$interval]))", @@ -708,7 +1086,10 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null } + { + "color": "green", + "value": null + } ] }, "unit": "Bps", @@ -722,23 +1103,45 @@ "spanNulls": true, "axisCenteredZero": false, "axisColorMode": "text", - "scaleDistribution": { "type": "linear" }, - "thresholdsStyle": { "mode": "off" } + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + } } }, { "id": 14, "type": "timeseries", "title": "Network Transmit", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 33 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 33 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "sum by (instance) (rate(node_network_transmit_bytes_total{device!=\"lo\"}[$interval]))", @@ -751,7 +1154,10 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null } + { + "color": "green", + "value": null + } ] }, "unit": "Bps", @@ -765,22 +1171,41 @@ "spanNulls": true, "axisCenteredZero": false, "axisColorMode": "text", - "scaleDistribution": { "type": "linear" }, - "thresholdsStyle": { "mode": "off" } + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + } } }, { "id": 105, "type": "row", "title": "Pod Status Detail", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 41 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 41 + }, "collapsed": false, "panels": [] }, @@ -788,8 +1213,16 @@ "id": 15, "type": "timeseries", "title": "Pod Restarts", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 42 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 42 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "increase(kube_pod_container_status_restarts_total{namespace=\"foxhunt\"}[$interval])", @@ -802,9 +1235,18 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "orange", "value": 1 }, - { "color": "red", "value": 5 } + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 1 + }, + { + "color": "red", + "value": 5 + } ] }, "unit": "none", @@ -819,24 +1261,48 @@ "spanNulls": true, "axisCenteredZero": false, "axisColorMode": "text", - "scaleDistribution": { "type": "linear" }, - "thresholdsStyle": { "mode": "off" }, - "stacking": { "mode": "normal", "group": "A" } + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + }, + "stacking": { + "mode": "normal", + "group": "A" + } } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["sum"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "sum" + ] + } } }, { "id": 16, "type": "table", "title": "Container Status", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 42 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 42 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "kube_pod_container_status_running{namespace=\"foxhunt\"}", @@ -865,48 +1331,125 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null } + { + "color": "green", + "value": null + } ] }, "custom": { "align": "auto", - "cellOptions": { "type": "auto" }, + "cellOptions": { + "type": "auto" + }, "filterable": true } }, "overrides": [ { - "matcher": { "id": "byName", "options": "Value #A" }, + "matcher": { + "id": "byName", + "options": "Value #A" + }, "properties": [ - { "id": "displayName", "value": "Running" } + { + "id": "displayName", + "value": "Running" + } ] }, { - "matcher": { "id": "byName", "options": "Value #B" }, + "matcher": { + "id": "byName", + "options": "Value #B" + }, "properties": [ - { "id": "displayName", "value": "Waiting" }, - { "id": "custom.cellOptions", "value": { "mode": "basic", "type": "color-background" } }, - { "id": "thresholds", "value": { "mode": "absolute", "steps": [{ "color": "transparent", "value": null }, { "color": "orange", "value": 1 }] } } + { + "id": "displayName", + "value": "Waiting" + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "basic", + "type": "color-background" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "orange", + "value": 1 + } + ] + } + } ] }, { - "matcher": { "id": "byName", "options": "Value #C" }, + "matcher": { + "id": "byName", + "options": "Value #C" + }, "properties": [ - { "id": "displayName", "value": "Terminated" }, - { "id": "custom.cellOptions", "value": { "mode": "basic", "type": "color-background" } }, - { "id": "thresholds", "value": { "mode": "absolute", "steps": [{ "color": "transparent", "value": null }, { "color": "red", "value": 1 }] } } + { + "id": "displayName", + "value": "Terminated" + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "basic", + "type": "color-background" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + } ] }, { - "matcher": { "id": "byName", "options": "Time" }, + "matcher": { + "id": "byName", + "options": "Time" + }, "properties": [ - { "id": "custom.hidden", "value": true } + { + "id": "custom.hidden", + "value": true + } ] }, { - "matcher": { "id": "byName", "options": "__name__" }, + "matcher": { + "id": "byName", + "options": "__name__" + }, "properties": [ - { "id": "custom.hidden", "value": true } + { + "id": "custom.hidden", + "value": true + } ] } ] @@ -914,9 +1457,14 @@ "options": { "showHeader": true, "sortBy": [ - { "displayName": "Waiting", "desc": true } + { + "displayName": "Waiting", + "desc": true + } ], - "footer": { "show": false } + "footer": { + "show": false + } }, "transformations": [ { @@ -951,7 +1499,12 @@ "id": 106, "type": "row", "title": "kube-state-metrics", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 50 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 50 + }, "collapsed": false, "panels": [] }, @@ -959,8 +1512,16 @@ "id": 17, "type": "table", "title": "Deployment Replicas", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 51 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 51 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "kube_deployment_spec_replicas{namespace=\"foxhunt\"}", @@ -989,46 +1550,102 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null } + { + "color": "green", + "value": null + } ] }, "custom": { "align": "auto", - "cellOptions": { "type": "auto" }, + "cellOptions": { + "type": "auto" + }, "filterable": true } }, "overrides": [ { - "matcher": { "id": "byName", "options": "Value #A" }, + "matcher": { + "id": "byName", + "options": "Value #A" + }, "properties": [ - { "id": "displayName", "value": "Desired" } + { + "id": "displayName", + "value": "Desired" + } ] }, { - "matcher": { "id": "byName", "options": "Value #B" }, + "matcher": { + "id": "byName", + "options": "Value #B" + }, "properties": [ - { "id": "displayName", "value": "Available" } + { + "id": "displayName", + "value": "Available" + } ] }, { - "matcher": { "id": "byName", "options": "Value #C" }, + "matcher": { + "id": "byName", + "options": "Value #C" + }, "properties": [ - { "id": "displayName", "value": "Unavailable" }, - { "id": "custom.cellOptions", "value": { "mode": "basic", "type": "color-background" } }, - { "id": "thresholds", "value": { "mode": "absolute", "steps": [{ "color": "transparent", "value": null }, { "color": "red", "value": 1 }] } } + { + "id": "displayName", + "value": "Unavailable" + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "basic", + "type": "color-background" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + } ] }, { - "matcher": { "id": "byName", "options": "Time" }, + "matcher": { + "id": "byName", + "options": "Time" + }, "properties": [ - { "id": "custom.hidden", "value": true } + { + "id": "custom.hidden", + "value": true + } ] }, { - "matcher": { "id": "byName", "options": "__name__" }, + "matcher": { + "id": "byName", + "options": "__name__" + }, "properties": [ - { "id": "custom.hidden", "value": true } + { + "id": "custom.hidden", + "value": true + } ] } ] @@ -1036,9 +1653,14 @@ "options": { "showHeader": true, "sortBy": [ - { "displayName": "Unavailable", "desc": true } + { + "displayName": "Unavailable", + "desc": true + } ], - "footer": { "show": false } + "footer": { + "show": false + } }, "transformations": [ { @@ -1072,8 +1694,16 @@ "id": 18, "type": "timeseries", "title": "HPA Status", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 51 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 51 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "kube_horizontalpodautoscaler_status_current_replicas{namespace=\"foxhunt\"}", @@ -1101,7 +1731,10 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null } + { + "color": "green", + "value": null + } ] }, "unit": "none", @@ -1116,30 +1749,79 @@ "spanNulls": true, "axisCenteredZero": false, "axisColorMode": "text", - "scaleDistribution": { "type": "linear" }, - "thresholdsStyle": { "mode": "off" } + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } } }, "overrides": [ { - "matcher": { "id": "byRegexp", "options": ".*min.*" }, + "matcher": { + "id": "byRegexp", + "options": ".*min.*" + }, "properties": [ - { "id": "custom.lineStyle", "value": { "fill": "dash", "dash": [10, 10] } }, - { "id": "color", "value": { "mode": "fixed", "fixedColor": "orange" } } + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 10, + 10 + ] + } + }, + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "orange" + } + } ] }, { - "matcher": { "id": "byRegexp", "options": ".*max.*" }, + "matcher": { + "id": "byRegexp", + "options": ".*max.*" + }, "properties": [ - { "id": "custom.lineStyle", "value": { "fill": "dash", "dash": [10, 10] } }, - { "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } } + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 10, + 10 + ] + } + }, + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "red" + } + } ] } ] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + } } } ], @@ -1147,7 +1829,10 @@ "list": [ { "builtIn": 1, - "datasource": { "type": "grafana", "uid": "-- Grafana --" }, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", @@ -1158,4 +1843,4 @@ }, "links": [], "style": "dark" -} \ No newline at end of file +} diff --git a/infra/k8s/monitoring/dashboards/foxhunt-observability.json b/infra/k8s/monitoring/dashboards/foxhunt-observability.json index 0ba731e7b..7d288b579 100644 --- a/infra/k8s/monitoring/dashboards/foxhunt-observability.json +++ b/infra/k8s/monitoring/dashboards/foxhunt-observability.json @@ -1,24 +1,4 @@ { - "__inputs": [ - { - "name": "DS_PROMETHEUS", - "label": "Prometheus", - "type": "datasource", - "pluginId": "prometheus" - }, - { - "name": "DS_LOKI", - "label": "Loki", - "type": "datasource", - "pluginId": "loki" - }, - { - "name": "DS_TEMPO", - "label": "Tempo", - "type": "datasource", - "pluginId": "tempo" - } - ], "annotations": { "list": [] }, @@ -30,30 +10,52 @@ "panels": [ { "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, "id": 100, "title": "API Gateway Overview", "type": "row" }, { - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null } + { + "color": "green", + "value": null + } ] }, "unit": "reqps" }, "overrides": [] }, - "gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 1 + }, "id": 1, "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, "colorMode": "background", "graphMode": "none" }, @@ -68,26 +70,49 @@ ] }, { - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 1 }, - { "color": "red", "value": 5 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 5 + } ] }, "unit": "percent" }, "overrides": [] }, - "gridPos": { "h": 4, "w": 4, "x": 4, "y": 1 }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 1 + }, "id": 2, "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, "colorMode": "background", "graphMode": "none" }, @@ -102,26 +127,49 @@ ] }, { - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 0.5 }, - { "color": "red", "value": 1 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.5 + }, + { + "color": "red", + "value": 1 + } ] }, "unit": "s" }, "overrides": [] }, - "gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 1 + }, "id": 3, "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, "colorMode": "background", "graphMode": "none" }, @@ -136,24 +184,41 @@ ] }, { - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "thresholds": { "mode": "absolute", "steps": [ - { "color": "blue", "value": null } + { + "color": "blue", + "value": null + } ] }, "unit": "short" }, "overrides": [] }, - "gridPos": { "h": 4, "w": 4, "x": 12, "y": 1 }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 1 + }, "id": 4, "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, "colorMode": "background", "graphMode": "none" }, @@ -168,26 +233,49 @@ ] }, { - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "orange", "value": 1 }, - { "color": "red", "value": 10 } + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 1 + }, + { + "color": "red", + "value": 10 + } ] }, "unit": "reqps" }, "overrides": [] }, - "gridPos": { "h": 4, "w": 4, "x": 16, "y": 1 }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 1 + }, "id": 5, "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, "colorMode": "background", "graphMode": "none" }, @@ -202,26 +290,49 @@ ] }, { - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "orange", "value": 0.5 }, - { "color": "red", "value": 2 } + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.5 + }, + { + "color": "red", + "value": 2 + } ] }, "unit": "reqps" }, "overrides": [] }, - "gridPos": { "h": 4, "w": 4, "x": 20, "y": 1 }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 1 + }, "id": 6, "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, "colorMode": "background", "graphMode": "none" }, @@ -237,16 +348,26 @@ }, { "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, "id": 101, "title": "Request Patterns", "type": "row" }, { - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -256,17 +377,29 @@ "lineWidth": 1, "pointSize": 5, "showPoints": "auto", - "stacking": { "mode": "none" } + "stacking": { + "mode": "none" + } }, "unit": "reqps" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 6 + }, "id": 7, "options": { - "tooltip": { "mode": "multi" }, - "legend": { "displayMode": "table", "placement": "bottom" } + "tooltip": { + "mode": "multi" + }, + "legend": { + "displayMode": "table", + "placement": "bottom" + } }, "title": "Request Rate by Handler", "type": "timeseries", @@ -279,10 +412,15 @@ ] }, { - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -292,17 +430,29 @@ "lineWidth": 1, "pointSize": 5, "showPoints": "auto", - "stacking": { "mode": "none" } + "stacking": { + "mode": "none" + } }, "unit": "s" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 6 + }, "id": 8, "options": { - "tooltip": { "mode": "multi" }, - "legend": { "displayMode": "table", "placement": "bottom" } + "tooltip": { + "mode": "multi" + }, + "legend": { + "displayMode": "table", + "placement": "bottom" + } }, "title": "Latency by Handler", "type": "timeseries", @@ -321,16 +471,26 @@ }, { "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, "id": 102, "title": "Auth & Security", "type": "row" }, { - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -340,17 +500,29 @@ "lineWidth": 1, "pointSize": 5, "showPoints": "auto", - "stacking": { "mode": "none" } + "stacking": { + "mode": "none" + } }, "unit": "s" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 8, "x": 0, "y": 15 }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 15 + }, "id": 9, "options": { - "tooltip": { "mode": "multi" }, - "legend": { "displayMode": "table", "placement": "bottom" } + "tooltip": { + "mode": "multi" + }, + "legend": { + "displayMode": "table", + "placement": "bottom" + } }, "title": "Auth Timing", "type": "timeseries", @@ -373,10 +545,15 @@ ] }, { - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -386,17 +563,29 @@ "lineWidth": 1, "pointSize": 5, "showPoints": "auto", - "stacking": { "mode": "none" } + "stacking": { + "mode": "none" + } }, "unit": "ops" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 8, "x": 8, "y": 15 }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 15 + }, "id": 10, "options": { - "tooltip": { "mode": "multi" }, - "legend": { "displayMode": "table", "placement": "bottom" } + "tooltip": { + "mode": "multi" + }, + "legend": { + "displayMode": "table", + "placement": "bottom" + } }, "title": "RBAC Checks", "type": "timeseries", @@ -409,10 +598,15 @@ ] }, { - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -422,17 +616,29 @@ "lineWidth": 1, "pointSize": 5, "showPoints": "never", - "stacking": { "mode": "normal" } + "stacking": { + "mode": "normal" + } }, "unit": "reqps" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 8, "x": 16, "y": 15 }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 15 + }, "id": 11, "options": { - "tooltip": { "mode": "multi" }, - "legend": { "displayMode": "table", "placement": "bottom" } + "tooltip": { + "mode": "multi" + }, + "legend": { + "displayMode": "table", + "placement": "bottom" + } }, "title": "Rate Limiting by Tier", "type": "timeseries", @@ -451,16 +657,26 @@ }, { "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, "id": 103, "title": "Log Volume", "type": "row" }, { - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -470,34 +686,90 @@ "lineWidth": 1, "pointSize": 5, "showPoints": "never", - "stacking": { "mode": "normal" } + "stacking": { + "mode": "normal" + } }, "unit": "short" }, "overrides": [ { - "matcher": { "id": "byName", "options": "error" }, - "properties": [{ "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } }] + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] }, { - "matcher": { "id": "byName", "options": "warn" }, - "properties": [{ "id": "color", "value": { "fixedColor": "yellow", "mode": "fixed" } }] + "matcher": { + "id": "byName", + "options": "warn" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] }, { - "matcher": { "id": "byName", "options": "info" }, - "properties": [{ "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } }] + "matcher": { + "id": "byName", + "options": "info" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] }, { - "matcher": { "id": "byName", "options": "debug" }, - "properties": [{ "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } }] + "matcher": { + "id": "byName", + "options": "debug" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] } ] }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, "id": 12, "options": { - "tooltip": { "mode": "multi" }, - "legend": { "displayMode": "table", "placement": "bottom" } + "tooltip": { + "mode": "multi" + }, + "legend": { + "displayMode": "table", + "placement": "bottom" + } }, "title": "Log Rate by Level", "type": "timeseries", @@ -510,8 +782,16 @@ ] }, { - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 }, + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, "id": 13, "options": { "showTime": true, @@ -531,16 +811,26 @@ }, { "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 32 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 32 + }, "id": 104, "title": "Log Detail", "type": "row" }, { - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -550,17 +840,29 @@ "lineWidth": 1, "pointSize": 5, "showPoints": "never", - "stacking": { "mode": "normal" } + "stacking": { + "mode": "normal" + } }, "unit": "short" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 33 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 33 + }, "id": 14, "options": { - "tooltip": { "mode": "multi" }, - "legend": { "displayMode": "table", "placement": "bottom" } + "tooltip": { + "mode": "multi" + }, + "legend": { + "displayMode": "table", + "placement": "bottom" + } }, "title": "Logs by Service", "type": "timeseries", @@ -573,8 +875,16 @@ ] }, { - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 33 }, + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 33 + }, "id": 15, "options": { "showTime": true, @@ -594,14 +904,27 @@ }, { "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 41 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 41 + }, "id": 105, "title": "Traces", "type": "row" }, { - "datasource": { "type": "tempo", "uid": "${DS_TEMPO}" }, - "gridPos": { "h": 8, "w": 24, "x": 0, "y": 42 }, + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 42 + }, "id": 16, "options": {}, "title": "Trace Search", @@ -615,16 +938,26 @@ }, { "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 50 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 50 + }, "id": 106, "title": "Prometheus Self-Monitoring", "type": "row" }, { - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -634,17 +967,29 @@ "lineWidth": 1, "pointSize": 5, "showPoints": "auto", - "stacking": { "mode": "none" } + "stacking": { + "mode": "none" + } }, "unit": "s" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 51 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 51 + }, "id": 17, "options": { - "tooltip": { "mode": "multi" }, - "legend": { "displayMode": "table", "placement": "bottom" } + "tooltip": { + "mode": "multi" + }, + "legend": { + "displayMode": "table", + "placement": "bottom" + } }, "title": "Prometheus Scrape Duration", "type": "timeseries", @@ -657,10 +1002,15 @@ ] }, { - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -670,10 +1020,24 @@ "lineWidth": 2, "pointSize": 5, "showPoints": "never", - "stacking": { "mode": "none" } + "stacking": { + "mode": "none" + } }, "mappings": [ - { "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } }, "type": "value" } + { + "options": { + "0": { + "text": "DOWN", + "color": "red" + }, + "1": { + "text": "UP", + "color": "green" + } + }, + "type": "value" + } ], "unit": "short", "min": 0, @@ -681,11 +1045,21 @@ }, "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 51 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 51 + }, "id": 18, "options": { - "tooltip": { "mode": "multi" }, - "legend": { "displayMode": "table", "placement": "bottom" } + "tooltip": { + "mode": "multi" + }, + "legend": { + "displayMode": "table", + "placement": "bottom" + } }, "title": "Targets Up/Down", "type": "timeseries", @@ -700,31 +1074,71 @@ ], "refresh": "15s", "schemaVersion": 39, - "tags": ["foxhunt", "observability", "logs", "traces"], + "tags": [ + "foxhunt", + "observability", + "logs", + "traces" + ], "templating": { "list": [ { "auto": true, "auto_count": 30, "auto_min": "10s", - "current": { "selected": false, "text": "auto", "value": "$__auto_interval_interval" }, + "current": { + "selected": false, + "text": "auto", + "value": "$__auto_interval_interval" + }, "hide": 0, "label": "Interval", "name": "interval", "options": [ - { "selected": false, "text": "1m", "value": "1m" }, - { "selected": false, "text": "5m", "value": "5m" }, - { "selected": false, "text": "15m", "value": "15m" }, - { "selected": false, "text": "30m", "value": "30m" }, - { "selected": false, "text": "1h", "value": "1h" } + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "15m", + "value": "15m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } ], "query": "1m,5m,15m,30m,1h", "type": "interval" }, { "allValue": ".*", - "current": { "selected": true, "text": ["All"], "value": ["$__all"] }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "definition": "label_values(up{job=~\"foxhunt.*\"}, job)", "hide": 0, "includeAll": true, @@ -741,27 +1155,54 @@ }, { "allValue": ".*", - "current": { "selected": true, "text": ["All"], "value": ["$__all"] }, + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, "hide": 0, "includeAll": true, "label": "Level", "multi": true, "name": "level", "options": [ - { "selected": false, "text": "error", "value": "error" }, - { "selected": false, "text": "warn", "value": "warn" }, - { "selected": false, "text": "info", "value": "info" }, - { "selected": false, "text": "debug", "value": "debug" } + { + "selected": false, + "text": "error", + "value": "error" + }, + { + "selected": false, + "text": "warn", + "value": "warn" + }, + { + "selected": false, + "text": "info", + "value": "info" + }, + { + "selected": false, + "text": "debug", + "value": "debug" + } ], "query": "error,warn,info,debug", "type": "custom" } ] }, - "time": { "from": "now-1h", "to": "now" }, + "time": { + "from": "now-1h", + "to": "now" + }, "timepicker": {}, "timezone": "browser", "title": "Observability & API Gateway", "uid": "foxhunt-observability", "version": 1 -} \ No newline at end of file +} diff --git a/infra/k8s/monitoring/dashboards/foxhunt-trading.json b/infra/k8s/monitoring/dashboards/foxhunt-trading.json index 54b1b6a9b..882911f99 100644 --- a/infra/k8s/monitoring/dashboards/foxhunt-trading.json +++ b/infra/k8s/monitoring/dashboards/foxhunt-trading.json @@ -1,13 +1,4 @@ { - "__inputs": [ - { - "name": "DS_PROMETHEUS", - "label": "Prometheus", - "description": "", - "type": "datasource", - "pluginId": "prometheus" - } - ], "id": null, "uid": "foxhunt-trading", "title": "Trading & ML Intelligence", @@ -48,7 +39,7 @@ "query": "label_values(foxhunt_predictions_confidence_bucket, model)", "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "PBFA97CFB590B2093" }, "refresh": 2, "includeAll": true, @@ -64,7 +55,7 @@ "query": "label_values(foxhunt_predictions_confidence_bucket, symbol)", "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "PBFA97CFB590B2093" }, "refresh": 2, "includeAll": true, @@ -102,7 +93,7 @@ }, "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { @@ -150,7 +141,7 @@ "refId": "A", "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "PBFA97CFB590B2093" }, "expr": "foxhunt_ensemble_agreement_rate{symbol=~\"$symbol\"}", "instant": true, @@ -170,7 +161,7 @@ }, "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { @@ -218,7 +209,7 @@ "refId": "A", "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "PBFA97CFB590B2093" }, "expr": "histogram_quantile(0.5, rate(foxhunt_predictions_confidence_bucket{model=~\"$model\"}[5m]))", "instant": true, @@ -238,7 +229,7 @@ }, "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { @@ -286,7 +277,7 @@ "refId": "A", "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "PBFA97CFB590B2093" }, "expr": "sum(rate(foxhunt_predictions_total{model=~\"$model\"}[5m]))", "instant": true, @@ -306,7 +297,7 @@ }, "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { @@ -354,7 +345,7 @@ "refId": "A", "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "PBFA97CFB590B2093" }, "expr": "count(foxhunt_model_loaded{model=~\"$model\"} == 1)", "instant": true, @@ -387,7 +378,7 @@ }, "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { @@ -432,7 +423,7 @@ "refId": "A", "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "PBFA97CFB590B2093" }, "expr": "histogram_quantile(0.1, rate(foxhunt_predictions_confidence_bucket{model=~\"$model\"}[$interval]))", "legendFormat": "p10" @@ -441,7 +432,7 @@ "refId": "B", "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "PBFA97CFB590B2093" }, "expr": "histogram_quantile(0.5, rate(foxhunt_predictions_confidence_bucket{model=~\"$model\"}[$interval]))", "legendFormat": "p50" @@ -450,7 +441,7 @@ "refId": "C", "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "PBFA97CFB590B2093" }, "expr": "histogram_quantile(0.9, rate(foxhunt_predictions_confidence_bucket{model=~\"$model\"}[$interval]))", "legendFormat": "p90" @@ -469,7 +460,7 @@ }, "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { @@ -515,7 +506,7 @@ "refId": "A", "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "PBFA97CFB590B2093" }, "expr": "sum by (model) (rate(foxhunt_predictions_total{model=~\"$model\"}[$interval]))", "legendFormat": "{{model}}" @@ -547,7 +538,7 @@ }, "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { @@ -599,7 +590,7 @@ "refId": "A", "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "PBFA97CFB590B2093" }, "expr": "sum(rate(trading_orders_total[5m]))", "instant": true, @@ -619,7 +610,7 @@ }, "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { @@ -667,7 +658,7 @@ "refId": "A", "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "PBFA97CFB590B2093" }, "expr": "histogram_quantile(0.99, rate(trading_order_processing_seconds_bucket[5m]))", "instant": true, @@ -687,7 +678,7 @@ }, "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { @@ -735,7 +726,7 @@ "refId": "A", "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "PBFA97CFB590B2093" }, "expr": "histogram_quantile(0.99, rate(trading_risk_check_seconds_bucket[5m]))", "instant": true, @@ -755,7 +746,7 @@ }, "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { @@ -803,7 +794,7 @@ "refId": "A", "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "PBFA97CFB590B2093" }, "expr": "histogram_quantile(0.99, rate(trading_total_latency_seconds_bucket[5m]))", "instant": true, @@ -836,7 +827,7 @@ }, "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { @@ -882,7 +873,7 @@ "refId": "A", "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "PBFA97CFB590B2093" }, "expr": "histogram_quantile(0.5, rate(trading_order_processing_seconds_bucket[$interval]))", "legendFormat": "p50" @@ -891,7 +882,7 @@ "refId": "B", "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "PBFA97CFB590B2093" }, "expr": "histogram_quantile(0.95, rate(trading_order_processing_seconds_bucket[$interval]))", "legendFormat": "p95" @@ -900,7 +891,7 @@ "refId": "C", "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "PBFA97CFB590B2093" }, "expr": "histogram_quantile(0.99, rate(trading_order_processing_seconds_bucket[$interval]))", "legendFormat": "p99" @@ -919,7 +910,7 @@ }, "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { @@ -965,7 +956,7 @@ "refId": "A", "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "PBFA97CFB590B2093" }, "expr": "histogram_quantile(0.5, rate(trading_risk_check_seconds_bucket[$interval]))", "legendFormat": "p50" @@ -974,7 +965,7 @@ "refId": "B", "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "PBFA97CFB590B2093" }, "expr": "histogram_quantile(0.95, rate(trading_risk_check_seconds_bucket[$interval]))", "legendFormat": "p95" @@ -983,7 +974,7 @@ "refId": "C", "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "PBFA97CFB590B2093" }, "expr": "histogram_quantile(0.99, rate(trading_risk_check_seconds_bucket[$interval]))", "legendFormat": "p99" @@ -1015,7 +1006,7 @@ }, "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { @@ -1061,7 +1052,7 @@ "refId": "A", "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "PBFA97CFB590B2093" }, "expr": "histogram_quantile(0.99, rate(api_gateway_request_duration_seconds_bucket{handler=~\".*trade.*|.*order.*\"}[$interval]))", "legendFormat": "{{handler}} p99" @@ -1080,7 +1071,7 @@ }, "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { @@ -1156,7 +1147,7 @@ "refId": "A", "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "PBFA97CFB590B2093" }, "expr": "rate(api_gateway_rate_limit_rejected_total{tier=\"trading\"}[$interval])", "legendFormat": "rejected" @@ -1165,7 +1156,7 @@ "refId": "B", "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "PBFA97CFB590B2093" }, "expr": "rate(api_gateway_rate_limit_allowed_total{tier=\"trading\"}[$interval])", "legendFormat": "allowed" @@ -1197,7 +1188,7 @@ }, "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { @@ -1242,7 +1233,7 @@ "refId": "A", "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "PBFA97CFB590B2093" }, "expr": "sum by (grpc_method) (rate(grpc_server_handled_total{grpc_service=~\".*trading.*|.*ml.*\"}[$interval]))", "legendFormat": "{{grpc_method}}" @@ -1261,7 +1252,7 @@ }, "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { @@ -1306,7 +1297,7 @@ "refId": "A", "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "uid": "PBFA97CFB590B2093" }, "expr": "sum by (grpc_service, grpc_code) (rate(grpc_server_handled_total{grpc_code!=\"OK\"}[$interval]))", "legendFormat": "{{grpc_service}} {{grpc_code}}" diff --git a/infra/k8s/monitoring/dashboards/foxhunt-training.json b/infra/k8s/monitoring/dashboards/foxhunt-training.json index 771e2edf1..167147171 100644 --- a/infra/k8s/monitoring/dashboards/foxhunt-training.json +++ b/infra/k8s/monitoring/dashboards/foxhunt-training.json @@ -1,50 +1,13 @@ { - "__inputs": [ - { - "name": "DS_PROMETHEUS", - "label": "Prometheus", - "description": "Prometheus datasource for foxhunt metrics", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - } - ], - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "10.0.0" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "1.0.0" - }, - { - "type": "panel", - "id": "stat", - "name": "Stat", - "version": "" - }, - { - "type": "panel", - "id": "timeseries", - "name": "Time series", - "version": "" - }, - { - "type": "panel", - "id": "gauge", - "name": "Gauge", - "version": "" - } - ], "uid": "foxhunt-training", "title": "Training & CI/CD", "description": "ML model training, hyperopt, GPU utilization, and Argo CI pipelines for the foxhunt HFT platform", - "tags": ["foxhunt", "training", "gpu", "ci"], + "tags": [ + "foxhunt", + "training", + "gpu", + "ci" + ], "editable": true, "graphTooltip": 1, "refresh": "15s", @@ -70,11 +33,31 @@ "value": "$__auto_interval_interval" }, "options": [ - { "selected": false, "text": "1m", "value": "1m" }, - { "selected": false, "text": "5m", "value": "5m" }, - { "selected": false, "text": "15m", "value": "15m" }, - { "selected": false, "text": "30m", "value": "30m" }, - { "selected": false, "text": "1h", "value": "1h" } + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "15m", + "value": "15m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } ], "query": "1m,5m,15m,30m,1h" } @@ -84,7 +67,12 @@ { "type": "row", "title": "Training Status", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, "collapsed": false, "panels": [] }, @@ -92,8 +80,16 @@ "id": 1, "type": "stat", "title": "Active Workers", - "gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 1 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "ml_training_active_workers", @@ -106,8 +102,14 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "blue", "value": null }, - { "color": "green", "value": 1 } + { + "color": "blue", + "value": null + }, + { + "color": "green", + "value": 1 + } ] }, "mappings": [], @@ -116,7 +118,11 @@ "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, "colorMode": "background", "graphMode": "none" } @@ -125,8 +131,16 @@ "id": 2, "type": "stat", "title": "Jobs Started", - "gridPos": { "h": 4, "w": 4, "x": 4, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 1 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "ml_training_jobs_started_total", @@ -139,7 +153,10 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "blue", "value": null } + { + "color": "blue", + "value": null + } ] }, "mappings": [], @@ -148,7 +165,11 @@ "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, "colorMode": "background", "graphMode": "none" } @@ -157,8 +178,16 @@ "id": 3, "type": "stat", "title": "Jobs Completed", - "gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 1 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "ml_training_jobs_completed_total", @@ -171,7 +200,10 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null } + { + "color": "green", + "value": null + } ] }, "mappings": [], @@ -180,7 +212,11 @@ "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, "colorMode": "background", "graphMode": "none" } @@ -189,8 +225,16 @@ "id": 4, "type": "stat", "title": "Training Errors", - "gridPos": { "h": 4, "w": 4, "x": 12, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 1 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "ml_training_errors_total", @@ -203,8 +247,14 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "red", "value": 1 } + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } ] }, "mappings": [], @@ -213,7 +263,11 @@ "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, "colorMode": "background", "graphMode": "none" } @@ -222,8 +276,16 @@ "id": 5, "type": "stat", "title": "Current Epoch", - "gridPos": { "h": 4, "w": 4, "x": 16, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 1 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "foxhunt_training_current_epoch", @@ -236,7 +298,10 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "purple", "value": null } + { + "color": "purple", + "value": null + } ] }, "mappings": [], @@ -245,7 +310,11 @@ "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, "colorMode": "background", "graphMode": "none" } @@ -254,8 +323,16 @@ "id": 6, "type": "stat", "title": "Hyperopt Mode", - "gridPos": { "h": 4, "w": 4, "x": 20, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 1 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "foxhunt_hyperopt_mode", @@ -268,18 +345,41 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "orange", "value": null } + { + "color": "orange", + "value": null + } ] }, "mappings": [ - { "type": "value", "options": { "0": { "text": "Idle", "color": "text" }, "1": { "text": "PSO", "color": "orange" }, "2": { "text": "Grid", "color": "blue" } } } + { + "type": "value", + "options": { + "0": { + "text": "Idle", + "color": "text" + }, + "1": { + "text": "PSO", + "color": "orange" + }, + "2": { + "text": "Grid", + "color": "blue" + } + } + } ], "unit": "none" }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, "colorMode": "background", "graphMode": "none", "textMode": "value_and_name" @@ -288,7 +388,12 @@ { "type": "row", "title": "Training Curves", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, "collapsed": false, "panels": [] }, @@ -296,8 +401,16 @@ "id": 7, "type": "timeseries", "title": "Epoch Loss", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 6 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "foxhunt_training_epoch_loss", @@ -317,30 +430,56 @@ "pointSize": 5, "showPoints": "auto", "axisLabel": "Loss", - "scaleDistribution": { "type": "linear" } + "scaleDistribution": { + "type": "linear" + } }, "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null } + { + "color": "green", + "value": null + } ] }, "unit": "none", - "color": { "mode": "palette-classic" } + "color": { + "mode": "palette-classic" + } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["min", "max", "last"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "min", + "max", + "last" + ] + } } }, { "id": 8, "type": "timeseries", "title": "Epoch Sharpe Ratio", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 6 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "foxhunt_training_epoch_sharpe", @@ -360,30 +499,58 @@ "pointSize": 5, "showPoints": "auto", "axisLabel": "Sharpe", - "scaleDistribution": { "type": "linear" }, - "thresholdsStyle": { "mode": "line" } + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "line" + } }, "thresholds": { "mode": "absolute", "steps": [ - { "color": "red", "value": null }, - { "color": "green", "value": 0 } + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 0 + } ] }, "unit": "none", - "color": { "mode": "palette-classic" } + "color": { + "mode": "palette-classic" + } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["min", "max", "last"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "min", + "max", + "last" + ] + } } }, { "type": "row", "title": "Training Quality", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 13 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 13 + }, "collapsed": false, "panels": [] }, @@ -391,8 +558,16 @@ "id": 9, "type": "timeseries", "title": "Win Rate", - "gridPos": { "h": 8, "w": 8, "x": 0, "y": 14 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 14 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "foxhunt_training_epoch_win_rate", @@ -412,34 +587,64 @@ "pointSize": 5, "showPoints": "auto", "axisLabel": "Win Rate", - "scaleDistribution": { "type": "linear" } + "scaleDistribution": { + "type": "linear" + } }, "thresholds": { "mode": "absolute", "steps": [ - { "color": "red", "value": null }, - { "color": "yellow", "value": 40 }, - { "color": "green", "value": 55 } + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 40 + }, + { + "color": "green", + "value": 55 + } ] }, "unit": "percent", "min": 0, "max": 100, - "color": { "mode": "palette-classic" } + "color": { + "mode": "palette-classic" + } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["last"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "last" + ] + } } }, { "id": 10, "type": "timeseries", "title": "Learning Rate", - "gridPos": { "h": 8, "w": 8, "x": 8, "y": 14 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 14 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "foxhunt_training_learning_rate", @@ -458,30 +663,55 @@ "pointSize": 5, "showPoints": "auto", "axisLabel": "LR", - "scaleDistribution": { "type": "log", "log": 10 } + "scaleDistribution": { + "type": "log", + "log": 10 + } }, "thresholds": { "mode": "absolute", "steps": [ - { "color": "blue", "value": null } + { + "color": "blue", + "value": null + } ] }, "unit": "none", - "color": { "mode": "palette-classic" } + "color": { + "mode": "palette-classic" + } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["last"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "last" + ] + } } }, { "id": 11, "type": "timeseries", "title": "Gradient Norm", - "gridPos": { "h": 8, "w": 8, "x": 16, "y": 14 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 14 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "foxhunt_training_gradient_norm", @@ -500,31 +730,61 @@ "pointSize": 5, "showPoints": "auto", "axisLabel": "Grad Norm", - "scaleDistribution": { "type": "linear" }, - "thresholdsStyle": { "mode": "line+area" } + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "line+area" + } }, "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 10 }, - { "color": "red", "value": 100 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 10 + }, + { + "color": "red", + "value": 100 + } ] }, "unit": "none", - "color": { "mode": "palette-classic" } + "color": { + "mode": "palette-classic" + } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["max", "last"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "max", + "last" + ] + } } }, { "type": "row", "title": "Hyperopt Progress", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 21 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, "collapsed": false, "panels": [] }, @@ -532,8 +792,16 @@ "id": 12, "type": "gauge", "title": "Trial Progress", - "gridPos": { "h": 8, "w": 8, "x": 0, "y": 22 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 22 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "foxhunt_hyperopt_trial_current / foxhunt_hyperopt_trial_total * 100", @@ -546,11 +814,26 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "red", "value": null }, - { "color": "orange", "value": 25 }, - { "color": "yellow", "value": 50 }, - { "color": "light-green", "value": 75 }, - { "color": "green", "value": 95 } + { + "color": "red", + "value": null + }, + { + "color": "orange", + "value": 25 + }, + { + "color": "yellow", + "value": 50 + }, + { + "color": "light-green", + "value": 75 + }, + { + "color": "green", + "value": 95 + } ] }, "unit": "percent", @@ -560,7 +843,11 @@ "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, "showThresholdLabels": false, "showThresholdMarkers": true } @@ -569,8 +856,16 @@ "id": 13, "type": "timeseries", "title": "Best Objective", - "gridPos": { "h": 8, "w": 16, "x": 8, "y": 22 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 8, + "w": 16, + "x": 8, + "y": 22 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "foxhunt_training_best_objective", @@ -590,28 +885,51 @@ "pointSize": 5, "showPoints": "auto", "axisLabel": "Objective", - "scaleDistribution": { "type": "linear" } + "scaleDistribution": { + "type": "linear" + } }, "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null } + { + "color": "green", + "value": null + } ] }, "unit": "none", - "color": { "mode": "palette-classic" } + "color": { + "mode": "palette-classic" + } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["min", "max", "last"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "min", + "max", + "last" + ] + } } }, { "type": "row", "title": "GPU Metrics", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 29 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 29 + }, "collapsed": false, "panels": [] }, @@ -619,8 +937,16 @@ "id": 14, "type": "timeseries", "title": "GPU Utilization", - "gridPos": { "h": 8, "w": 8, "x": 0, "y": 30 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 30 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "DCGM_FI_DEV_GPU_UTIL", @@ -640,34 +966,66 @@ "pointSize": 5, "showPoints": "never", "axisLabel": "Utilization", - "scaleDistribution": { "type": "linear" } + "scaleDistribution": { + "type": "linear" + } }, "thresholds": { "mode": "absolute", "steps": [ - { "color": "red", "value": null }, - { "color": "yellow", "value": 50 }, - { "color": "green", "value": 80 } + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 50 + }, + { + "color": "green", + "value": 80 + } ] }, "unit": "percent", "min": 0, "max": 100, - "color": { "mode": "palette-classic" } + "color": { + "mode": "palette-classic" + } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max", + "last" + ] + } } }, { "id": 15, "type": "timeseries", "title": "GPU Memory Used", - "gridPos": { "h": 8, "w": 8, "x": 8, "y": 30 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 30 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "DCGM_FI_DEV_FB_USED / (DCGM_FI_DEV_FB_USED + DCGM_FI_DEV_FB_FREE) * 100", @@ -687,34 +1045,66 @@ "pointSize": 5, "showPoints": "never", "axisLabel": "Memory %", - "scaleDistribution": { "type": "linear" } + "scaleDistribution": { + "type": "linear" + } }, "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 75 }, - { "color": "red", "value": 90 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 75 + }, + { + "color": "red", + "value": 90 + } ] }, "unit": "percent", "min": 0, "max": 100, - "color": { "mode": "palette-classic" } + "color": { + "mode": "palette-classic" + } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max", + "last" + ] + } } }, { "id": 16, "type": "timeseries", "title": "GPU Temperature", - "gridPos": { "h": 8, "w": 8, "x": 16, "y": 30 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 30 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "DCGM_FI_DEV_GPU_TEMP", @@ -733,31 +1123,62 @@ "pointSize": 5, "showPoints": "never", "axisLabel": "Temp", - "scaleDistribution": { "type": "linear" }, - "thresholdsStyle": { "mode": "line+area" } + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "line+area" + } }, "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "orange", "value": 80 }, - { "color": "red", "value": 90 } + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 80 + }, + { + "color": "red", + "value": 90 + } ] }, "unit": "celsius", - "color": { "mode": "palette-classic" } + "color": { + "mode": "palette-classic" + } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max", + "last" + ] + } } }, { "type": "row", "title": "GPU Detail", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 37 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 37 + }, "collapsed": false, "panels": [] }, @@ -765,8 +1186,16 @@ "id": 17, "type": "timeseries", "title": "GPU Power Draw", - "gridPos": { "h": 8, "w": 8, "x": 0, "y": 38 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 38 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "DCGM_FI_DEV_POWER_USAGE", @@ -786,30 +1215,56 @@ "pointSize": 5, "showPoints": "never", "axisLabel": "Power", - "scaleDistribution": { "type": "linear" } + "scaleDistribution": { + "type": "linear" + } }, "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null } + { + "color": "green", + "value": null + } ] }, "unit": "watt", - "color": { "mode": "palette-classic" } + "color": { + "mode": "palette-classic" + } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max", + "last" + ] + } } }, { "id": 18, "type": "timeseries", "title": "SM Clock", - "gridPos": { "h": 8, "w": 8, "x": 8, "y": 38 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 38 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "DCGM_FI_DEV_SM_CLOCK", @@ -828,30 +1283,55 @@ "pointSize": 5, "showPoints": "never", "axisLabel": "Clock", - "scaleDistribution": { "type": "linear" } + "scaleDistribution": { + "type": "linear" + } }, "thresholds": { "mode": "absolute", "steps": [ - { "color": "blue", "value": null } + { + "color": "blue", + "value": null + } ] }, "unit": "clockHz", - "color": { "mode": "palette-classic" } + "color": { + "mode": "palette-classic" + } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "last"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "last" + ] + } } }, { "id": 19, "type": "timeseries", "title": "PCIe TX/RX", - "gridPos": { "h": 8, "w": 8, "x": 16, "y": 38 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 38 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "DCGM_FI_DEV_PCIE_TX_THROUGHPUT", @@ -875,28 +1355,51 @@ "pointSize": 5, "showPoints": "never", "axisLabel": "Throughput", - "scaleDistribution": { "type": "linear" } + "scaleDistribution": { + "type": "linear" + } }, "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null } + { + "color": "green", + "value": null + } ] }, "unit": "Bps", - "color": { "mode": "palette-classic" } + "color": { + "mode": "palette-classic" + } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max", + "last" + ] + } } }, { "type": "row", "title": "Argo CI Pipelines", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 45 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 45 + }, "collapsed": false, "panels": [] }, @@ -904,8 +1407,16 @@ "id": 20, "type": "stat", "title": "Active Workflows", - "gridPos": { "h": 4, "w": 6, "x": 0, "y": 46 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 46 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "argo_workflows_gauge{status=\"Running\"}", @@ -918,8 +1429,14 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "blue", "value": null }, - { "color": "green", "value": 1 } + { + "color": "blue", + "value": null + }, + { + "color": "green", + "value": 1 + } ] }, "mappings": [], @@ -928,7 +1445,11 @@ "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, "colorMode": "background", "graphMode": "none" } @@ -937,8 +1458,16 @@ "id": 21, "type": "stat", "title": "Succeeded", - "gridPos": { "h": 4, "w": 6, "x": 6, "y": 46 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 46 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "argo_workflows_gauge{status=\"Succeeded\"}", @@ -951,7 +1480,10 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null } + { + "color": "green", + "value": null + } ] }, "mappings": [], @@ -960,7 +1492,11 @@ "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, "colorMode": "background", "graphMode": "none" } @@ -969,8 +1505,16 @@ "id": 22, "type": "stat", "title": "Failed", - "gridPos": { "h": 4, "w": 6, "x": 12, "y": 46 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 46 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "argo_workflows_gauge{status=\"Failed\"}", @@ -983,8 +1527,14 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "red", "value": 1 } + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } ] }, "mappings": [], @@ -993,7 +1543,11 @@ "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, "colorMode": "background", "graphMode": "none" } @@ -1002,8 +1556,16 @@ "id": 23, "type": "stat", "title": "Pending", - "gridPos": { "h": 4, "w": 6, "x": 18, "y": 46 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 46 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "argo_workflows_gauge{status=\"Pending\"}", @@ -1016,8 +1578,14 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 1 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + } ] }, "mappings": [], @@ -1026,7 +1594,11 @@ "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, "colorMode": "background", "graphMode": "none" } @@ -1034,7 +1606,12 @@ { "type": "row", "title": "Argo Performance", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 50 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 50 + }, "collapsed": false, "panels": [] }, @@ -1042,8 +1619,16 @@ "id": 24, "type": "timeseries", "title": "Operation Duration", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 51 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 51 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "histogram_quantile(0.50, rate(argo_workflows_operation_duration_seconds_bucket[$interval]))", @@ -1068,39 +1653,83 @@ "pointSize": 5, "showPoints": "auto", "axisLabel": "Duration", - "scaleDistribution": { "type": "linear" } + "scaleDistribution": { + "type": "linear" + } }, "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 30 }, - { "color": "red", "value": 60 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 30 + }, + { + "color": "red", + "value": 60 + } ] }, "unit": "s", - "color": { "mode": "palette-classic" } + "color": { + "mode": "palette-classic" + } }, "overrides": [ { - "matcher": { "id": "byName", "options": "p95" }, + "matcher": { + "id": "byName", + "options": "p95" + }, "properties": [ - { "id": "custom.lineStyle", "value": { "fill": "dash", "dash": [10, 10] } } + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 10, + 10 + ] + } + } ] } ] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max", + "last" + ] + } } }, { "id": 25, "type": "timeseries", "title": "K8s API Requests", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 51 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 51 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "topk(10, sum by (verb, kind) (rate(argo_workflows_k8s_request_total[$interval])))", @@ -1120,28 +1749,51 @@ "pointSize": 5, "showPoints": "never", "axisLabel": "req/s", - "scaleDistribution": { "type": "linear" } + "scaleDistribution": { + "type": "linear" + } }, "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null } + { + "color": "green", + "value": null + } ] }, "unit": "reqps", - "color": { "mode": "palette-classic" } + "color": { + "mode": "palette-classic" + } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max", + "last" + ] + } } }, { "type": "row", "title": "Argo Internals", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 58 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 58 + }, "collapsed": false, "panels": [] }, @@ -1149,8 +1801,16 @@ "id": 26, "type": "timeseries", "title": "Queue Depth", - "gridPos": { "h": 8, "w": 8, "x": 0, "y": 59 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 59 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "argo_workflows_queue_depth_gauge", @@ -1170,33 +1830,65 @@ "pointSize": 5, "showPoints": "never", "axisLabel": "Depth", - "scaleDistribution": { "type": "linear" } + "scaleDistribution": { + "type": "linear" + } }, "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 50 }, - { "color": "red", "value": 100 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 50 + }, + { + "color": "red", + "value": 100 + } ] }, "unit": "none", "min": 0, - "color": { "mode": "palette-classic" } + "color": { + "mode": "palette-classic" + } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max", + "last" + ] + } } }, { "id": 27, "type": "timeseries", "title": "Worker Busy", - "gridPos": { "h": 8, "w": 8, "x": 8, "y": 59 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 59 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "argo_workflows_workers_busy_count", @@ -1216,31 +1908,57 @@ "pointSize": 5, "showPoints": "never", "axisLabel": "Busy", - "scaleDistribution": { "type": "linear" } + "scaleDistribution": { + "type": "linear" + } }, "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null } + { + "color": "green", + "value": null + } ] }, "unit": "none", "min": 0, - "color": { "mode": "palette-classic" } + "color": { + "mode": "palette-classic" + } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max", + "last" + ] + } } }, { "id": 28, "type": "timeseries", "title": "Error Count", - "gridPos": { "h": 8, "w": 8, "x": 16, "y": 59 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 59 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "rate(argo_workflows_error_count[$interval])", @@ -1260,25 +1978,46 @@ "pointSize": 5, "showPoints": "never", "axisLabel": "Errors/s", - "scaleDistribution": { "type": "linear" } + "scaleDistribution": { + "type": "linear" + } }, "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "red", "value": 0.1 } + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.1 + } ] }, "unit": "none", "min": 0, - "color": { "mode": "palette-classic" } + "color": { + "mode": "palette-classic" + } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["sum", "max", "last"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "sum", + "max", + "last" + ] + } } } ] -} \ No newline at end of file +} diff --git a/infra/k8s/monitoring/prometheus.yaml b/infra/k8s/monitoring/prometheus.yaml index 08080e455..7de4685b6 100644 --- a/infra/k8s/monitoring/prometheus.yaml +++ b/infra/k8s/monitoring/prometheus.yaml @@ -63,6 +63,9 @@ data: - source_labels: [__meta_kubernetes_service_label_app_kubernetes_io_part_of] action: keep regex: foxhunt + - source_labels: [__meta_kubernetes_service_name] + action: drop + regex: argo-workflows-workflow-controller-metrics - source_labels: [__meta_kubernetes_endpoint_port_name] action: keep regex: metrics @@ -71,58 +74,6 @@ data: - source_labels: [__meta_kubernetes_namespace] target_label: namespace - # Pod annotation-based scraping (prometheus.io/scrape: "true") - - job_name: annotated-pods - kubernetes_sd_configs: - - role: pod - namespaces: - names: [foxhunt] - relabel_configs: - - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] - action: keep - regex: "true" - - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] - action: replace - regex: (.+) - target_label: __metrics_path__ - - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] - action: replace - regex: ([^:]+)(?::\d+)?;(\d+) - replacement: $1:$2 - target_label: __address__ - - action: labelmap - regex: __meta_kubernetes_pod_label_(.+) - - source_labels: [__meta_kubernetes_namespace] - target_label: namespace - - source_labels: [__meta_kubernetes_pod_name] - target_label: pod - - # GitLab annotation-based scraping (gitlab.com/prometheus_scrape: "true") - - job_name: gitlab-annotated-pods - kubernetes_sd_configs: - - role: pod - namespaces: - names: [foxhunt] - relabel_configs: - - source_labels: [__meta_kubernetes_pod_annotation_gitlab_com_prometheus_scrape] - action: keep - regex: "true" - - source_labels: [__meta_kubernetes_pod_annotation_gitlab_com_prometheus_path] - action: replace - regex: (.+) - target_label: __metrics_path__ - - source_labels: [__address__, __meta_kubernetes_pod_annotation_gitlab_com_prometheus_port] - action: replace - regex: ([^:]+)(?::\d+)?;(\d+) - replacement: $1:$2 - target_label: __address__ - - action: labelmap - regex: __meta_kubernetes_pod_label_(.+) - - source_labels: [__meta_kubernetes_namespace] - target_label: namespace - - source_labels: [__meta_kubernetes_pod_name] - target_label: kubernetes_pod_name - # Node-exporter (host metrics) - job_name: node-exporter kubernetes_sd_configs: diff --git a/infra/k8s/network-policies/prometheus.yaml b/infra/k8s/network-policies/prometheus.yaml index 36c11feaa..44d835342 100644 --- a/infra/k8s/network-policies/prometheus.yaml +++ b/infra/k8s/network-policies/prometheus.yaml @@ -13,7 +13,17 @@ spec: app.kubernetes.io/name: prometheus app.kubernetes.io/part-of: foxhunt policyTypes: + - Ingress - Egress + ingress: + # Grafana queries Prometheus on port 9090 + - from: + - podSelector: + matchLabels: + app.kubernetes.io/name: grafana + ports: + - protocol: TCP + port: 9090 egress: # Kubernetes API server (required for kubernetes_sd_configs service discovery) # ClusterIP 10.32.0.1:443 NATs to node IP 172.16.0.0/16:6443