diff --git a/infra/k8s/monitoring/dashboards/foxhunt-ci-logs.json b/infra/k8s/monitoring/dashboards/foxhunt-ci-logs.json index f573c7db0..71c901efd 100644 --- a/infra/k8s/monitoring/dashboards/foxhunt-ci-logs.json +++ b/infra/k8s/monitoring/dashboards/foxhunt-ci-logs.json @@ -1,183 +1,212 @@ { - "uid": "foxhunt-ci-logs", - "title": "Foxhunt CI Pipeline Logs", - "description": "Loki-powered CI pipeline log viewer — test gate, clippy, builds, errors", - "tags": ["foxhunt", "ci", "loki", "logs"], + "description": "Loki-powered CI pipeline log viewer \u2014 test gate, clippy, builds, errors", "editable": true, "graphTooltip": 2, - "refresh": "30s", - "schemaVersion": 39, - "time": { "from": "now-6h", "to": "now" }, - "timepicker": {}, - "timezone": "browser", - "__inputs": [ - { "name": "DS_LOKI", "label": "Loki", "type": "datasource", "pluginId": "loki" } - ], - "templating": { - "list": [ - { - "name": "pipeline", - "label": "Pipeline", - "type": "custom", - "query": "ci-pipeline : ci-pipeline, Compile+Deploy : compile-and-deploy, Compile+Train : compile-and-train, GPU Test : gpu-test, Build Image : build-ci-image", - "current": { "selected": true, "text": "ci-pipeline", "value": "ci-pipeline" }, - "options": [ - { "selected": true, "text": "ci-pipeline", "value": "ci-pipeline" }, - { "selected": false, "text": "Compile+Deploy", "value": "compile-and-deploy" }, - { "selected": false, "text": "Compile+Train", "value": "compile-and-train" }, - { "selected": false, "text": "GPU Test", "value": "gpu-test" }, - { "selected": false, "text": "Build Image", "value": "build-ci-image" } - ] - }, - { - "name": "level", - "label": "Level", - "type": "custom", - "query": "ERROR,WARN,INFO,DEBUG,TRACE", - "current": { "selected": true, "text": "All", "value": "$__all" }, - "includeAll": true, - "allValue": ".*", - "multi": true, - "options": [ - { "selected": true, "text": "All", "value": "$__all" }, - { "selected": false, "text": "ERROR", "value": "ERROR" }, - { "selected": false, "text": "WARN", "value": "WARN" }, - { "selected": false, "text": "INFO", "value": "INFO" }, - { "selected": false, "text": "DEBUG", "value": "DEBUG" }, - { "selected": false, "text": "TRACE", "value": "TRACE" } - ] - }, - { - "name": "search", - "label": "Search", - "type": "textbox", - "current": { "selected": false, "text": "", "value": "" } - } - ] - }, "panels": [ { "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, "id": 1, "title": "Pipeline Overview", "type": "row" }, { - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, "fieldConfig": { "defaults": { - "color": { "fixedColor": "red", "mode": "fixed" }, + "color": { + "fixedColor": "red", + "mode": "fixed" + }, "mappings": [], + "noValue": "0", "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "red", "value": 1 } + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } ] - }, - "noValue": "0" + } }, "overrides": [] }, - "gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 1 + }, "id": 2, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "center", "orientation": "auto", - "reduceOptions": { "calcs": ["sum"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, "textMode": "auto" }, - "title": "Errors", - "type": "stat", "targets": [ { - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, "editorMode": "code", "expr": "sum(count_over_time({namespace=\"foxhunt\", pod=~\"$pipeline-.*\", level=\"ERROR\"} [$__range]))", "queryType": "instant", "refId": "A" } - ] + ], + "title": "Errors", + "type": "stat" }, { - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, "fieldConfig": { "defaults": { - "color": { "fixedColor": "yellow", "mode": "fixed" }, + "color": { + "fixedColor": "yellow", + "mode": "fixed" + }, "mappings": [], + "noValue": "0", "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 1 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + } ] - }, - "noValue": "0" + } }, "overrides": [] }, - "gridPos": { "h": 4, "w": 4, "x": 4, "y": 1 }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 1 + }, "id": 3, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "center", "orientation": "auto", - "reduceOptions": { "calcs": ["sum"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, "textMode": "auto" }, - "title": "Warnings", - "type": "stat", "targets": [ { - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, "editorMode": "code", "expr": "sum(count_over_time({namespace=\"foxhunt\", pod=~\"$pipeline-.*\", level=\"WARN\"} [$__range]))", "queryType": "instant", "refId": "A" } - ] + ], + "title": "Warnings", + "type": "stat" }, { - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, "fieldConfig": { "defaults": { - "color": { "fixedColor": "blue", "mode": "fixed" }, + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, "mappings": [], "noValue": "0" }, "overrides": [] }, - "gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 1 + }, "id": 4, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "center", "orientation": "auto", - "reduceOptions": { "calcs": ["sum"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, "textMode": "auto" }, - "title": "Total Log Lines", - "type": "stat", "targets": [ { - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, "editorMode": "code", "expr": "sum(count_over_time({namespace=\"foxhunt\", pod=~\"$pipeline-.*\"} [$__range]))", "queryType": "instant", "refId": "A" } - ] + ], + "title": "Total Log Lines", + "type": "stat" }, { - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -187,297 +216,594 @@ "lineWidth": 0, "pointSize": 5, "showPoints": "never", - "stacking": { "mode": "normal" } + "stacking": { + "mode": "normal" + } } }, "overrides": [ { - "matcher": { "id": "byName", "options": "ERROR" }, - "properties": [{ "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } }] + "matcher": { + "id": "byName", + "options": "ERROR" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] }, { - "matcher": { "id": "byName", "options": "WARN" }, - "properties": [{ "id": "color", "value": { "fixedColor": "yellow", "mode": "fixed" } }] + "matcher": { + "id": "byName", + "options": "WARN" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] }, { - "matcher": { "id": "byName", "options": "INFO" }, - "properties": [{ "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } }] + "matcher": { + "id": "byName", + "options": "INFO" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] }, { - "matcher": { "id": "byName", "options": "DEBUG" }, - "properties": [{ "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } }] + "matcher": { + "id": "byName", + "options": "DEBUG" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] } ] }, - "gridPos": { "h": 6, "w": 12, "x": 12, "y": 1 }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 1 + }, "id": 5, - "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi" } }, - "title": "Log Volume by Level", - "type": "timeseries", + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, "targets": [ { - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, "editorMode": "code", "expr": "sum by (level) (count_over_time({namespace=\"foxhunt\", pod=~\"$pipeline-.*\"} [$__interval]))", "legendFormat": "{{level}}", "queryType": "range", "refId": "A" } - ] + ], + "title": "Log Volume by Level", + "type": "timeseries" }, { - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "drawStyle": "bars", "fillOpacity": 60, "lineWidth": 0, "showPoints": "never", - "stacking": { "mode": "normal" } + "stacking": { + "mode": "normal" + } } }, "overrides": [] }, - "gridPos": { "h": 4, "w": 12, "x": 0, "y": 5 }, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 5 + }, "id": 6, - "options": { "legend": { "displayMode": "list", "placement": "right" }, "tooltip": { "mode": "multi" } }, - "title": "Log Volume by Container", - "type": "timeseries", + "options": { + "legend": { + "displayMode": "list", + "placement": "right" + }, + "tooltip": { + "mode": "multi" + } + }, "targets": [ { - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, "editorMode": "code", "expr": "sum by (container) (count_over_time({namespace=\"foxhunt\", pod=~\"$pipeline-.*\"} [$__interval]))", "legendFormat": "{{container}}", "queryType": "range", "refId": "A" } - ] + ], + "title": "Log Volume by Container", + "type": "timeseries" }, { "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 9 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, "id": 10, "title": "Full Pipeline Log Stream", "type": "row" }, { - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, - "gridPos": { "h": 14, "w": 24, "x": 0, "y": 10 }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 14, + "w": 24, + "x": 0, + "y": 10 + }, "id": 11, "options": { - "showTime": true, - "showLabels": true, - "showCommonLabels": false, - "wrapLogMessage": true, - "prettifyLogMessage": false, - "enableLogDetails": true, "dedupStrategy": "none", - "sortOrder": "Descending" + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true }, - "title": "Pipeline Logs — $pipeline", - "type": "logs", "targets": [ { - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, "editorMode": "code", "expr": "{namespace=\"foxhunt\", pod=~\"$pipeline-.*\", level=~\"$level\"} ${search:pipe}", "queryType": "range", "refId": "A" } - ] + ], + "title": "Pipeline Logs \u2014 $pipeline", + "type": "logs" }, { "collapsed": true, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 24 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 24 + }, "id": 20, - "title": "Test Gate (clippy + cargo test)", - "type": "row", "panels": [ { - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, - "gridPos": { "h": 12, "w": 24, "x": 0, "y": 25 }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 25 + }, "id": 21, "options": { - "showTime": true, - "showLabels": false, - "wrapLogMessage": true, - "enableLogDetails": true, "dedupStrategy": "none", - "sortOrder": "Descending" + "enableLogDetails": true, + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true }, - "title": "Clippy + Test Output", - "type": "logs", "targets": [ { - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, "editorMode": "code", "expr": "{namespace=\"foxhunt\", pod=~\"ci-pipeline-.*\", container=\"main\"} |~ \"(clippy|cargo test|warning\\\\[|error\\\\[|FAILED|test result|running \\\\d+ test|failures:)\"", "queryType": "range", "refId": "A" } - ] + ], + "title": "Clippy + Test Output", + "type": "logs" } - ] + ], + "title": "Test Gate (clippy + cargo test)", + "type": "row" }, { "collapsed": true, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 25 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 25 + }, "id": 30, - "title": "Redis Sidecar", - "type": "row", "panels": [ { - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, - "gridPos": { "h": 8, "w": 24, "x": 0, "y": 26 }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 26 + }, "id": 31, "options": { - "showTime": true, - "showLabels": false, - "wrapLogMessage": true, - "enableLogDetails": true, "dedupStrategy": "none", - "sortOrder": "Descending" + "enableLogDetails": true, + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true }, - "title": "Redis Sidecar Logs", - "type": "logs", "targets": [ { - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, "editorMode": "code", "expr": "{namespace=\"foxhunt\", pod=~\"ci-pipeline-.*\", container=\"redis\"}", "queryType": "range", "refId": "A" } - ] + ], + "title": "Redis Sidecar Logs", + "type": "logs" } - ] + ], + "title": "Redis Sidecar", + "type": "row" }, { "collapsed": true, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 26 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, "id": 40, - "title": "Errors & Failures", - "type": "row", "panels": [ { - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, - "gridPos": { "h": 12, "w": 24, "x": 0, "y": 27 }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 27 + }, "id": 41, "options": { - "showTime": true, - "showLabels": true, - "wrapLogMessage": true, - "enableLogDetails": true, "dedupStrategy": "none", - "sortOrder": "Descending" + "enableLogDetails": true, + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true }, - "title": "All Errors & Failures", - "type": "logs", "targets": [ { - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, "editorMode": "code", "expr": "{namespace=\"foxhunt\", pod=~\"$pipeline-.*\"} |~ \"(?i)(error|failed|failure|panic|fatal|SIGABRT|SIGSEGV|thread .* panicked)\"", "queryType": "range", "refId": "A" } - ] + ], + "title": "All Errors & Failures", + "type": "logs" } - ] + ], + "title": "Errors & Failures", + "type": "row" }, { "collapsed": true, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 27 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 27 + }, "id": 50, - "title": "Build & Deploy", - "type": "row", "panels": [ { - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, - "gridPos": { "h": 10, "w": 12, "x": 0, "y": 28 }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 28 + }, "id": 51, "options": { - "showTime": true, - "showLabels": false, - "wrapLogMessage": true, - "enableLogDetails": true, "dedupStrategy": "none", - "sortOrder": "Descending" + "enableLogDetails": true, + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true }, - "title": "Compile Logs", - "type": "logs", "targets": [ { - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, "editorMode": "code", "expr": "{namespace=\"foxhunt\", pod=~\"(ci-pipeline|compile-and-deploy|compile-and-train)-.*\", container=\"main\"} |~ \"(Compiling|Finished|warning|error|Downloading|Building|cargo build|rclone)\"", "queryType": "range", "refId": "A" } - ] + ], + "title": "Compile Logs", + "type": "logs" }, { - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, - "gridPos": { "h": 10, "w": 12, "x": 12, "y": 28 }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 28 + }, "id": 52, "options": { - "showTime": true, - "showLabels": false, - "wrapLogMessage": true, - "enableLogDetails": true, "dedupStrategy": "none", - "sortOrder": "Descending" + "enableLogDetails": true, + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true }, - "title": "Deploy & Argo Template Logs", - "type": "logs", "targets": [ { - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, "editorMode": "code", "expr": "{namespace=\"foxhunt\", pod=~\"(ci-pipeline|compile-and-deploy)-.*\", container=\"main\"} |~ \"(kubectl|apply|rollout|deploy|configured|unchanged|created|image|kaniko)\"", "queryType": "range", "refId": "A" } - ] + ], + "title": "Deploy & Argo Template Logs", + "type": "logs" } - ] + ], + "title": "Build & Deploy", + "type": "row" }, { "collapsed": true, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 28 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 28 + }, "id": 60, - "title": "GPU Test Pipeline", - "type": "row", "panels": [ { - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, - "gridPos": { "h": 12, "w": 24, "x": 0, "y": 29 }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 29 + }, "id": 61, "options": { - "showTime": true, - "showLabels": true, - "wrapLogMessage": true, - "enableLogDetails": true, "dedupStrategy": "none", - "sortOrder": "Descending" + "enableLogDetails": true, + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true }, - "title": "GPU Test Logs (DQN/PPO/TFT + Supervised)", - "type": "logs", "targets": [ { - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, "editorMode": "code", "expr": "{namespace=\"foxhunt\", pod=~\"gpu-test-.*\"}", "queryType": "range", "refId": "A" } - ] + ], + "title": "GPU Test Logs (DQN/PPO/TFT + Supervised)", + "type": "logs" } - ] + ], + "title": "GPU Test Pipeline", + "type": "row" + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": [ + "foxhunt", + "ci", + "loki", + "logs" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": "ci-pipeline", + "value": "ci-pipeline" + }, + "label": "Pipeline", + "name": "pipeline", + "options": [], + "query": "ci-pipeline,compile-and-deploy,compile-and-train,gpu-test,gpu-diag,gpu-walltime,build-ci-image,build-training-runtime,evaluate", + "type": "custom" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "includeAll": true, + "label": "Level", + "multi": true, + "name": "level", + "options": [ + { + "selected": true, + "text": "All", + "value": "$__all" + }, + { + "selected": false, + "text": "ERROR", + "value": "ERROR" + }, + { + "selected": false, + "text": "WARN", + "value": "WARN" + }, + { + "selected": false, + "text": "INFO", + "value": "INFO" + }, + { + "selected": false, + "text": "DEBUG", + "value": "DEBUG" + }, + { + "selected": false, + "text": "TRACE", + "value": "TRACE" + } + ], + "query": "ERROR,WARN,INFO,DEBUG,TRACE", + "type": "custom" + }, + { + "current": { + "selected": false, + "text": "", + "value": "" + }, + "label": "Search", + "name": "search", + "type": "textbox" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "CI Pipeline Logs", + "uid": "foxhunt-ci-logs", + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "type": "datasource", + "pluginId": "prometheus" + }, + { + "name": "DS_LOKI", + "type": "datasource", + "pluginId": "loki" + }, + { + "name": "DS_TEMPO", + "type": "datasource", + "pluginId": "${DS_TEMPO}" } ] } diff --git a/infra/k8s/monitoring/dashboards/foxhunt-cockpit.json b/infra/k8s/monitoring/dashboards/foxhunt-cockpit.json index 0f270abb1..38fd68b15 100644 --- a/infra/k8s/monitoring/dashboards/foxhunt-cockpit.json +++ b/infra/k8s/monitoring/dashboards/foxhunt-cockpit.json @@ -1,80 +1,51 @@ { - "id": null, - "uid": "foxhunt-cockpit", - "title": "Foxhunt Operations Cockpit", - "description": "Operations overview: service health, gRPC traffic, CI/CD pipelines, API gateway, trading metrics, and metrics pipeline health.", - "tags": [ - "foxhunt", - "cockpit" - ], - "timezone": "browser", - "schemaVersion": 39, - "version": 2, - "editable": true, - "refresh": "30s", - "time": { - "from": "now-6h", - "to": "now" - }, - "fiscalYearStartMonth": 0, - "liveNow": false, - "weekStart": "", - "templating": { + "annotations": { "list": [ { - "name": "interval", - "type": "interval", - "query": "1m,5m,15m,30m,1h", - "current": { - "selected": true, - "text": "auto", - "value": "$__auto_interval_interval" + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" }, - "auto": true, - "auto_count": 30, - "auto_min": "1m", - "label": "Interval" + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" } ] }, + "description": "Operations overview: service health, gRPC traffic, CI/CD pipelines, API gateway, trading metrics, and metrics pipeline health.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "liveNow": false, "panels": [ { - "id": 1, - "type": "row", - "title": "Service Health", + "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, - "collapsed": false, - "panels": [] + "id": 1, + "panels": [], + "title": "Service Health", + "type": "row" }, { - "id": 2, - "type": "stat", - "title": "Services Up", - "description": "Number of foxhunt service targets currently up", - "gridPos": { - "h": 4, - "w": 4, - "x": 0, - "y": 1 - }, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, - "targets": [ - { - "expr": "count(up{job=\"foxhunt-services\"} == 1)", - "legendFormat": "up", - "refId": "A" - } - ], + "description": "Number of foxhunt service targets currently up", "fieldConfig": { "defaults": { + "color": { + "mode": "thresholds" + }, "thresholds": { "mode": "absolute", "steps": [ @@ -92,14 +63,22 @@ } ] }, - "color": { - "mode": "thresholds" - }, "unit": "none" }, "overrides": [] }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 2, "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" @@ -107,37 +86,29 @@ "fields": "", "values": false }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto" - } - }, - { - "id": 3, - "type": "stat", - "title": "Services Down", - "description": "Number of foxhunt service targets currently down", - "gridPos": { - "h": 4, - "w": 4, - "x": 4, - "y": 1 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "textMode": "auto" }, "targets": [ { - "expr": "count(up{job=\"foxhunt-services\"} == 0) OR on() vector(0)", - "legendFormat": "down", + "expr": "count(up{job=\"foxhunt-services\"} == 1)", + "legendFormat": "up", "refId": "A" } ], + "title": "Services Up", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of foxhunt service targets currently down", "fieldConfig": { "defaults": { + "color": { + "mode": "thresholds" + }, "thresholds": { "mode": "absolute", "steps": [ @@ -155,14 +126,22 @@ } ] }, - "color": { - "mode": "thresholds" - }, "unit": "none" }, "overrides": [] }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 1 + }, + "id": 3, "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" @@ -170,26 +149,69 @@ "fields": "", "values": false }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto" - } + "textMode": "auto" + }, + "targets": [ + { + "expr": "count(up{job=\"foxhunt-services\"} == 0) OR on() vector(0)", + "legendFormat": "down", + "refId": "A" + } + ], + "title": "Services Down", + "type": "stat" }, { - "id": 4, - "type": "stat", - "title": "Trading Service Uptime", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 300 + }, + { + "color": "green", + "value": 3600 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, "gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" }, "targets": [ { @@ -198,8 +220,19 @@ "refId": "A" } ], + "title": "Trading Service Uptime", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { + "color": { + "mode": "thresholds" + }, "thresholds": { "mode": "absolute", "steps": [ @@ -217,14 +250,22 @@ } ] }, - "color": { - "mode": "thresholds" - }, "unit": "s" }, "overrides": [] }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 1 + }, + "id": 5, "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" @@ -232,26 +273,7 @@ "fields": "", "values": false }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto" - } - }, - { - "id": 5, - "type": "stat", - "title": "Backtesting Service Uptime", - "gridPos": { - "h": 4, - "w": 4, - "x": 12, - "y": 1 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "textMode": "auto" }, "targets": [ { @@ -260,8 +282,19 @@ "refId": "A" } ], + "title": "Backtesting Service Uptime", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { + "color": { + "mode": "thresholds" + }, "thresholds": { "mode": "absolute", "steps": [ @@ -279,14 +312,22 @@ } ] }, - "color": { - "mode": "thresholds" - }, "unit": "s" }, "overrides": [] }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 1 + }, + "id": 6, "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" @@ -294,26 +335,7 @@ "fields": "", "values": false }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto" - } - }, - { - "id": 6, - "type": "stat", - "title": "ML Training Service Uptime", - "gridPos": { - "h": 4, - "w": 4, - "x": 16, - "y": 1 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "textMode": "auto" }, "targets": [ { @@ -322,8 +344,19 @@ "refId": "A" } ], + "title": "ML Training Service Uptime", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { + "color": { + "mode": "thresholds" + }, "thresholds": { "mode": "absolute", "steps": [ @@ -341,14 +374,22 @@ } ] }, - "color": { - "mode": "thresholds" - }, "unit": "s" }, "overrides": [] }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 5 + }, + "id": 7, "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" @@ -356,26 +397,7 @@ "fields": "", "values": false }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto" - } - }, - { - "id": 7, - "type": "stat", - "title": "Data Acquisition Uptime", - "gridPos": { - "h": 4, - "w": 4, - "x": 0, - "y": 5 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "textMode": "auto" }, "targets": [ { @@ -384,71 +406,20 @@ "refId": "A" } ], + "title": "Data Acquisition Uptime", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "yellow", - "value": 300 - }, - { - "color": "green", - "value": 3600 - } - ] - }, "color": { "mode": "thresholds" }, - "unit": "s" - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto" - } - }, - { - "id": 8, - "type": "stat", - "title": "API Gateway Uptime", - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 4, - "y": 5 - }, - "targets": [ - { - "expr": "api_uptime_seconds", - "legendFormat": "api", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "unit": "s", + "mappings": [], "thresholds": { "mode": "absolute", "steps": [ @@ -466,14 +437,22 @@ } ] }, - "mappings": [], - "color": { - "mode": "thresholds" - } + "unit": "s" }, "overrides": [] }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 5 + }, + "id": 8, "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" @@ -481,37 +460,29 @@ "fields": "", "values": false }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto" - } - }, - { - "id": 9, - "type": "stat", - "title": "API Auth Rate", - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 8, - "y": 5 + "textMode": "auto" }, "targets": [ { - "expr": "sum(rate(api_auth_requests_total[$__rate_interval]))", - "legendFormat": "req/s", + "expr": "max(time() - process_start_time_seconds{job=\"foxhunt-services\", instance=~\".*:9091\"})", + "legendFormat": "api", "refId": "A" } ], + "title": "API Gateway Uptime", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "unit": "reqps", + "color": { + "mode": "thresholds" + }, + "mappings": [], "thresholds": { "mode": "absolute", "steps": [ @@ -521,14 +492,22 @@ } ] }, - "mappings": [], - "color": { - "mode": "thresholds" - } + "unit": "reqps" }, "overrides": [] }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 5 + }, + "id": 9, "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" @@ -536,37 +515,24 @@ "fields": "", "values": false }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto" - } - }, - { - "id": 10, - "type": "table", - "title": "Per-Service Status", - "description": "Current up/down status for each service target", - "gridPos": { - "h": 4, - "w": 12, - "x": 12, - "y": 5 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "textMode": "auto" }, "targets": [ { - "expr": "up{job=\"foxhunt-services\"}", - "legendFormat": "{{service}}", - "refId": "A", - "format": "table", - "instant": true + "expr": "sum(rate(api_auth_requests_total[$__rate_interval]))", + "legendFormat": "req/s", + "refId": "A" } ], + "title": "API Auth Rate", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Current up/down status for each service target", "fieldConfig": { "defaults": { "custom": { @@ -585,22 +551,22 @@ "id": "mappings", "value": [ { - "type": "value", "options": { "1": { - "text": "UP", - "color": "green" + "color": "green", + "text": "UP" } - } + }, + "type": "value" }, { - "type": "value", "options": { "0": { - "text": "DOWN", - "color": "red" + "color": "red", + "text": "DOWN" } - } + }, + "type": "value" } ] }, @@ -648,15 +614,32 @@ } ] }, + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 5 + }, + "id": 10, "options": { "showHeader": true, "sortBy": [ { - "displayName": "service", - "desc": false + "desc": false, + "displayName": "service" } ] }, + "targets": [ + { + "expr": "up{job=\"foxhunt-services\"}", + "format": "table", + "instant": true, + "legendFormat": "{{service}}", + "refId": "A" + } + ], + "title": "Per-Service Status", "transformations": [ { "id": "organize", @@ -667,40 +650,73 @@ "job": true }, "renameByName": { - "service": "Service", + "Value": "Status", "instance": "Instance", - "Value": "Status" + "service": "Service" } } } - ] + ], + "type": "table" }, { - "id": 20, - "type": "row", - "title": "gRPC Overview", + "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 9 }, - "collapsed": true, + "id": 20, "panels": [ { - "id": 21, - "type": "timeseries", - "title": "gRPC Request Rate (by service)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "description": "Rate of gRPC calls started, grouped by exported_service", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 15, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "mode": "none" + } + }, + "unit": "reqps" + }, + "overrides": [] + }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 10 }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "id": 21, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, "targets": [ { @@ -709,83 +725,32 @@ "refId": "A" } ], - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 15, - "pointSize": 5, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "mode": "none" - }, - "axisPlacement": "auto", - "lineWidth": 2 - }, - "unit": "reqps", - "color": { - "mode": "palette-classic" - } - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "max" - ] - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - } + "title": "gRPC Request Rate (by service)", + "type": "timeseries" }, { - "id": 22, - "type": "timeseries", - "title": "gRPC Error Rate (by service)", - "description": "Rate of gRPC calls that returned a non-OK code", - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 10 - }, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, - "targets": [ - { - "expr": "sum by (exported_service) (rate(grpc_server_handled_total{grpc_code!=\"0\"}[$__rate_interval]))", - "legendFormat": "{{exported_service}}", - "refId": "A" - } - ], + "description": "Rate of gRPC calls that returned a non-OK code", "fieldConfig": { "defaults": { + "color": { + "mode": "palette-classic" + }, "custom": { + "axisPlacement": "auto", "drawStyle": "line", - "lineInterpolation": "smooth", "fillOpacity": 15, + "lineInterpolation": "smooth", + "lineWidth": 2, "pointSize": 5, "showPoints": "auto", "spanNulls": false, "stacking": { "mode": "none" - }, - "axisPlacement": "auto", - "lineWidth": 2 - }, - "unit": "reqps", - "color": { - "mode": "palette-classic" + } }, "thresholds": { "mode": "absolute", @@ -803,65 +768,65 @@ "value": 5 } ] - } + }, + "unit": "reqps" }, "overrides": [] }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 10 + }, + "id": 22, "options": { "legend": { - "displayMode": "table", - "placement": "bottom", "calcs": [ "mean", "max" - ] + ], + "displayMode": "table", + "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } - } - }, - { - "id": 23, - "type": "timeseries", - "title": "gRPC p99 Latency (by service)", - "description": "99th percentile gRPC server handling duration", - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 18 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" }, "targets": [ { - "expr": "histogram_quantile(0.99, sum by (exported_service, le) (rate(grpc_server_handling_seconds_bucket[$__rate_interval])))", - "legendFormat": "{{exported_service}} p99", + "expr": "sum by (exported_service) (rate(grpc_server_handled_total{grpc_code!=\"0\"}[$__rate_interval]))", + "legendFormat": "{{exported_service}}", "refId": "A" } ], + "title": "gRPC Error Rate (by service)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "99th percentile gRPC server handling duration", "fieldConfig": { "defaults": { + "color": { + "mode": "palette-classic" + }, "custom": { + "axisPlacement": "auto", "drawStyle": "line", - "lineInterpolation": "smooth", "fillOpacity": 10, + "lineInterpolation": "smooth", + "lineWidth": 2, "pointSize": 5, "showPoints": "auto", "spanNulls": false, "stacking": { "mode": "none" - }, - "axisPlacement": "auto", - "lineWidth": 2 - }, - "unit": "s", - "color": { - "mode": "palette-classic" + } }, "thresholds": { "mode": "absolute", @@ -879,49 +844,55 @@ "value": 2 } ] - } + }, + "unit": "s" }, "overrides": [] }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 23, "options": { "legend": { - "displayMode": "table", - "placement": "bottom", "calcs": [ "mean", "max" - ] + ], + "displayMode": "table", + "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } - } - }, - { - "id": 24, - "type": "stat", - "title": "gRPC Error Ratio", - "description": "Percentage of gRPC calls returning non-OK status", - "gridPos": { - "h": 8, - "w": 6, - "x": 12, - "y": 18 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" }, "targets": [ { - "expr": "sum(rate(grpc_server_handled_total{grpc_code!=\"0\"}[$__rate_interval])) / sum(rate(grpc_server_handled_total[$__rate_interval]))", - "legendFormat": "error ratio", + "expr": "histogram_quantile(0.99, sum by (exported_service, le) (rate(grpc_server_handling_seconds_bucket[$__rate_interval])))", + "legendFormat": "{{exported_service}} p99", "refId": "A" } ], + "title": "gRPC p99 Latency (by service)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Percentage of gRPC calls returning non-OK status", "fieldConfig": { "defaults": { + "color": { + "mode": "thresholds" + }, + "max": 1, + "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -939,16 +910,22 @@ } ] }, - "color": { - "mode": "thresholds" - }, - "unit": "percentunit", - "min": 0, - "max": 1 + "unit": "percentunit" }, "overrides": [] }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 18 + }, + "id": 24, "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" @@ -956,37 +933,29 @@ "fields": "", "values": false }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "area", - "justifyMode": "auto" - } - }, - { - "id": 25, - "type": "stat", - "title": "gRPC Total RPS", - "description": "Total gRPC requests per second across all services", - "gridPos": { - "h": 8, - "w": 6, - "x": 18, - "y": 18 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "textMode": "auto" }, "targets": [ { - "expr": "sum(rate(grpc_server_started_total[$__rate_interval]))", - "legendFormat": "total rps", + "expr": "sum(rate(grpc_server_handled_total{grpc_code!=\"0\"}[$__rate_interval])) / sum(rate(grpc_server_handled_total[$__rate_interval]))", + "legendFormat": "error ratio", "refId": "A" } ], + "title": "gRPC Error Ratio", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Total gRPC requests per second across all services", "fieldConfig": { "defaults": { + "color": { + "mode": "thresholds" + }, "thresholds": { "mode": "absolute", "steps": [ @@ -1008,14 +977,22 @@ } ] }, - "color": { - "mode": "thresholds" - }, "unit": "reqps" }, "overrides": [] }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 18 + }, + "id": 25, "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" @@ -1023,50 +1000,44 @@ "fields": "", "values": false }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto" - } + "textMode": "auto" + }, + "targets": [ + { + "expr": "sum(rate(grpc_server_started_total[$__rate_interval]))", + "legendFormat": "total rps", + "refId": "A" + } + ], + "title": "gRPC Total RPS", + "type": "stat" } - ] + ], + "title": "gRPC Overview", + "type": "row" }, { - "id": 30, - "type": "row", - "title": "Argo CI/CD", + "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 10 }, - "collapsed": true, + "id": 30, "panels": [ { - "id": 31, - "type": "gauge", - "title": "Running Workflows", - "gridPos": { - "h": 6, - "w": 4, - "x": 0, - "y": 11 - }, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, - "targets": [ - { - "expr": "argo_workflows_gauge{status=\"Running\"}", - "legendFormat": "running", - "refId": "A" - } - ], "fieldConfig": { "defaults": { + "color": { + "mode": "thresholds" + }, + "max": 15, + "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -1084,16 +1055,19 @@ } ] }, - "color": { - "mode": "thresholds" - }, - "unit": "none", - "min": 0, - "max": 15 + "unit": "none" }, "overrides": [] }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 11 + }, + "id": 31, "options": { + "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" @@ -1102,33 +1076,30 @@ "values": false }, "showThresholdLabels": false, - "showThresholdMarkers": true, - "orientation": "auto" - } - }, - { - "id": 32, - "type": "gauge", - "title": "Pending Workflows", - "gridPos": { - "h": 6, - "w": 4, - "x": 4, - "y": 11 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "showThresholdMarkers": true }, "targets": [ { - "expr": "argo_workflows_gauge{status=\"Pending\"}", - "legendFormat": "pending", + "expr": "argo_workflows_gauge{status=\"Running\"}", + "legendFormat": "running", "refId": "A" } ], + "title": "Running Workflows", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { + "color": { + "mode": "thresholds" + }, + "max": 15, + "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -1146,16 +1117,19 @@ } ] }, - "color": { - "mode": "thresholds" - }, - "unit": "none", - "min": 0, - "max": 15 + "unit": "none" }, "overrides": [] }, + "gridPos": { + "h": 6, + "w": 4, + "x": 4, + "y": 11 + }, + "id": 32, "options": { + "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" @@ -1164,33 +1138,28 @@ "values": false }, "showThresholdLabels": false, - "showThresholdMarkers": true, - "orientation": "auto" - } - }, - { - "id": 33, - "type": "stat", - "title": "Failed Workflows", - "gridPos": { - "h": 6, - "w": 4, - "x": 8, - "y": 11 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "showThresholdMarkers": true }, "targets": [ { - "expr": "argo_workflows_gauge{status=\"Failed\"}", - "legendFormat": "failed", + "expr": "argo_workflows_gauge{status=\"Pending\"}", + "legendFormat": "pending", "refId": "A" } ], + "title": "Pending Workflows", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { + "color": { + "mode": "thresholds" + }, "thresholds": { "mode": "absolute", "steps": [ @@ -1208,14 +1177,22 @@ } ] }, - "color": { - "mode": "thresholds" - }, "unit": "none" }, "overrides": [] }, + "gridPos": { + "h": 6, + "w": 4, + "x": 8, + "y": 11 + }, + "id": 33, "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" @@ -1223,36 +1200,28 @@ "fields": "", "values": false }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto" - } - }, - { - "id": 34, - "type": "stat", - "title": "Succeeded Workflows", - "gridPos": { - "h": 6, - "w": 4, - "x": 12, - "y": 11 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "textMode": "auto" }, "targets": [ { - "expr": "argo_workflows_gauge{status=\"Succeeded\"}", - "legendFormat": "succeeded", + "expr": "argo_workflows_gauge{status=\"Failed\"}", + "legendFormat": "failed", "refId": "A" } ], + "title": "Failed Workflows", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { + "color": { + "mode": "thresholds" + }, "thresholds": { "mode": "absolute", "steps": [ @@ -1270,14 +1239,22 @@ } ] }, - "color": { - "mode": "thresholds" - }, "unit": "none" }, "overrides": [] }, + "gridPos": { + "h": 6, + "w": 4, + "x": 12, + "y": 11 + }, + "id": 34, "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" @@ -1285,36 +1262,28 @@ "fields": "", "values": false }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto" - } - }, - { - "id": 35, - "type": "stat", - "title": "Workflow Pods", - "gridPos": { - "h": 6, - "w": 4, - "x": 16, - "y": 11 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "textMode": "auto" }, "targets": [ { - "expr": "argo_workflows_pods_gauge", - "legendFormat": "pods", + "expr": "argo_workflows_gauge{status=\"Succeeded\"}", + "legendFormat": "succeeded", "refId": "A" } ], + "title": "Succeeded Workflows", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { + "color": { + "mode": "thresholds" + }, "thresholds": { "mode": "absolute", "steps": [ @@ -1336,14 +1305,22 @@ } ] }, - "color": { - "mode": "thresholds" - }, "unit": "none" }, "overrides": [] }, + "gridPos": { + "h": 6, + "w": 4, + "x": 16, + "y": 11 + }, + "id": 35, "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" @@ -1351,57 +1328,45 @@ "fields": "", "values": false }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto" - } - }, - { - "id": 36, - "type": "timeseries", - "title": "Argo Error Rate", - "description": "Rate of errors from Argo Workflows controller", - "gridPos": { - "h": 6, - "w": 4, - "x": 20, - "y": 11 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "textMode": "auto" }, "targets": [ { - "expr": "rate(argo_workflows_error_count[$__rate_interval])", - "legendFormat": "errors/s", + "expr": "argo_workflows_pods_gauge", + "legendFormat": "pods", "refId": "A" } ], + "title": "Workflow Pods", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Rate of errors from Argo Workflows controller", "fieldConfig": { "defaults": { + "color": { + "mode": "palette-classic" + }, "custom": { + "axisPlacement": "auto", "drawStyle": "line", - "lineInterpolation": "smooth", "fillOpacity": 20, + "lineInterpolation": "smooth", + "lineWidth": 2, "pointSize": 5, "showPoints": "auto", "spanNulls": false, "stacking": { "mode": "none" }, - "axisPlacement": "auto", - "lineWidth": 2, "thresholdsStyle": { "mode": "area" } }, - "unit": "ops", - "color": { - "mode": "palette-classic" - }, "thresholds": { "mode": "absolute", "steps": [ @@ -1418,10 +1383,18 @@ "value": 1 } ] - } + }, + "unit": "ops" }, "overrides": [] }, + "gridPos": { + "h": 6, + "w": 4, + "x": 20, + "y": 11 + }, + "id": 36, "options": { "legend": { "displayMode": "list", @@ -1430,53 +1403,45 @@ "tooltip": { "mode": "single" } - } - }, - { - "id": 37, - "type": "timeseries", - "title": "Argo Operation Duration p95", - "description": "95th percentile Argo operation duration", - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 17 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" }, "targets": [ { - "expr": "histogram_quantile(0.95, sum by (le) (rate(argo_workflows_operation_duration_seconds_bucket[$__rate_interval])))", - "legendFormat": "p95", + "expr": "rate(argo_workflows_error_count[$__rate_interval])", + "legendFormat": "errors/s", "refId": "A" } ], + "title": "Argo Error Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "95th percentile Argo operation duration", "fieldConfig": { "defaults": { + "color": { + "fixedColor": "purple", + "mode": "fixed" + }, "custom": { + "axisPlacement": "auto", "drawStyle": "line", - "lineInterpolation": "smooth", "fillOpacity": 10, + "lineInterpolation": "smooth", + "lineWidth": 2, "pointSize": 5, "showPoints": "auto", "spanNulls": false, "stacking": { "mode": "none" }, - "axisPlacement": "auto", - "lineWidth": 2, "thresholdsStyle": { "mode": "line" } }, - "unit": "s", - "color": { - "fixedColor": "purple", - "mode": "fixed" - }, "thresholds": { "mode": "absolute", "steps": [ @@ -1493,10 +1458,18 @@ "value": 120 } ] - } + }, + "unit": "s" }, "overrides": [] }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 17 + }, + "id": 37, "options": { "legend": { "displayMode": "list", @@ -1505,22 +1478,61 @@ "tooltip": { "mode": "single" } - } + }, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum by (le) (rate(argo_workflows_operation_duration_seconds_bucket[$__rate_interval])))", + "legendFormat": "p95", + "refId": "A" + } + ], + "title": "Argo Operation Duration p95", + "type": "timeseries" }, { - "id": 38, - "type": "timeseries", - "title": "Argo K8s API Requests", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "description": "Rate of Kubernetes API requests from Argo controller", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "orange", + "mode": "fixed" + }, + "custom": { + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "mode": "none" + } + }, + "unit": "reqps" + }, + "overrides": [] + }, "gridPos": { "h": 6, "w": 12, "x": 12, "y": 17 }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "id": 38, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } }, "targets": [ { @@ -1529,104 +1541,48 @@ "refId": "A" } ], - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 10, - "pointSize": 5, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "mode": "none" - }, - "axisPlacement": "auto", - "lineWidth": 2 - }, - "unit": "reqps", - "color": { - "fixedColor": "orange", - "mode": "fixed" - } - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - } + "title": "Argo K8s API Requests", + "type": "timeseries" } - ] + ], + "title": "Argo CI/CD", + "type": "row" }, { - "id": 40, - "type": "row", - "title": "API Gateway", + "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 11 }, - "collapsed": true, + "id": 40, "panels": [ { - "id": 41, - "type": "timeseries", - "title": "Auth Request Rate", - "description": "Rate of authentication requests (total, success, failure)", - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 12 - }, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, - "targets": [ - { - "expr": "rate(api_auth_requests_total[$__rate_interval])", - "legendFormat": "total", - "refId": "A" - }, - { - "expr": "rate(api_auth_requests_success[$__rate_interval])", - "legendFormat": "success", - "refId": "B" - }, - { - "expr": "rate(api_auth_requests_failure[$__rate_interval])", - "legendFormat": "failure", - "refId": "C" - } - ], + "description": "Rate of authentication requests (total, success, failure)", "fieldConfig": { "defaults": { + "color": { + "mode": "palette-classic" + }, "custom": { + "axisPlacement": "auto", "drawStyle": "line", - "lineInterpolation": "smooth", "fillOpacity": 10, + "lineInterpolation": "smooth", + "lineWidth": 2, "pointSize": 5, "showPoints": "auto", "spanNulls": false, "stacking": { "mode": "none" - }, - "axisPlacement": "auto", - "lineWidth": 2 + } }, - "unit": "reqps", - "color": { - "mode": "palette-classic" - } + "unit": "reqps" }, "overrides": [ { @@ -1661,6 +1617,13 @@ } ] }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 12 + }, + "id": 41, "options": { "legend": { "displayMode": "list", @@ -1670,32 +1633,38 @@ "mode": "multi", "sort": "desc" } - } - }, - { - "id": 42, - "type": "stat", - "title": "Rate Limiter Entries", - "description": "Current number of entries in the API gateway rate limiter", - "gridPos": { - "h": 4, - "w": 4, - "x": 8, - "y": 12 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" }, "targets": [ { - "expr": "api_rate_limiter_entries", - "legendFormat": "entries", + "expr": "rate(api_auth_requests_total[$__rate_interval])", + "legendFormat": "total", "refId": "A" + }, + { + "expr": "rate(api_auth_requests_success[$__rate_interval])", + "legendFormat": "success", + "refId": "B" + }, + { + "expr": "rate(api_auth_requests_failure[$__rate_interval])", + "legendFormat": "failure", + "refId": "C" } ], + "title": "Auth Request Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Current number of entries in the API gateway rate limiter", "fieldConfig": { "defaults": { + "color": { + "mode": "thresholds" + }, "thresholds": { "mode": "absolute", "steps": [ @@ -1713,14 +1682,22 @@ } ] }, - "color": { - "mode": "thresholds" - }, "unit": "none" }, "overrides": [] }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 12 + }, + "id": 42, "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" @@ -1728,37 +1705,31 @@ "fields": "", "values": false }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "area", - "justifyMode": "auto" - } - }, - { - "id": 43, - "type": "stat", - "title": "JWT Cache Hit Ratio", - "description": "Ratio of JWT cache hits to total lookups", - "gridPos": { - "h": 4, - "w": 4, - "x": 12, - "y": 12 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "textMode": "auto" }, "targets": [ { - "expr": "api_jwt_cache_hits / (api_jwt_cache_hits + api_jwt_cache_misses)", - "legendFormat": "hit ratio", + "expr": "api_rate_limiter_entries", + "legendFormat": "entries", "refId": "A" } ], + "title": "Rate Limiter Entries", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Ratio of JWT cache hits to total lookups", "fieldConfig": { "defaults": { + "color": { + "mode": "thresholds" + }, + "max": 1, + "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -1776,16 +1747,22 @@ } ] }, - "color": { - "mode": "thresholds" - }, - "unit": "percentunit", - "min": 0, - "max": 1 + "unit": "percentunit" }, "overrides": [] }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 12 + }, + "id": 43, "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" @@ -1793,27 +1770,79 @@ "fields": "", "values": false }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "area", - "justifyMode": "auto" - } + "textMode": "auto" + }, + "targets": [ + { + "expr": "api_jwt_cache_hits / (api_jwt_cache_hits + api_jwt_cache_misses)", + "legendFormat": "hit ratio", + "refId": "A" + } + ], + "title": "JWT Cache Hit Ratio", + "type": "stat" }, { - "id": 44, - "type": "timeseries", - "title": "Routing Latency p50 / p95 / p99", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "description": "API gateway routing duration percentiles", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 5, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 5000 + }, + { + "color": "red", + "value": 50000 + } + ] + }, + "unit": "us" + }, + "overrides": [] + }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 12 }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "id": 44, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, "targets": [ { @@ -1832,80 +1861,22 @@ "refId": "C" } ], - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 5, - "pointSize": 5, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "mode": "none" - }, - "axisPlacement": "auto", - "lineWidth": 2 - }, - "unit": "us", - "color": { - "mode": "palette-classic" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 5000 - }, - { - "color": "red", - "value": 50000 - } - ] - } - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - } + "title": "Routing Latency p50 / p95 / p99", + "type": "timeseries" }, { - "id": 45, - "type": "stat", - "title": "Auth Failure Rate", - "description": "Percentage of auth requests that fail", - "gridPos": { - "h": 4, - "w": 4, - "x": 8, - "y": 16 - }, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, - "targets": [ - { - "expr": "rate(api_auth_requests_failure[$__rate_interval]) / rate(api_auth_requests_total[$__rate_interval])", - "legendFormat": "failure rate", - "refId": "A" - } - ], + "description": "Percentage of auth requests that fail", "fieldConfig": { "defaults": { + "color": { + "mode": "thresholds" + }, + "max": 1, + "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -1923,16 +1894,22 @@ } ] }, - "color": { - "mode": "thresholds" - }, - "unit": "percentunit", - "min": 0, - "max": 1 + "unit": "percentunit" }, "overrides": [] }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 16 + }, + "id": 45, "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" @@ -1940,37 +1917,29 @@ "fields": "", "values": false }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto" - } - }, - { - "id": 46, - "type": "stat", - "title": "JWT Cache Misses/s", - "description": "Rate of JWT cache misses", - "gridPos": { - "h": 4, - "w": 4, - "x": 12, - "y": 16 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "textMode": "auto" }, "targets": [ { - "expr": "rate(api_jwt_cache_misses[$__rate_interval])", - "legendFormat": "misses/s", + "expr": "rate(api_auth_requests_failure[$__rate_interval]) / rate(api_auth_requests_total[$__rate_interval])", + "legendFormat": "failure rate", "refId": "A" } ], + "title": "Auth Failure Rate", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Rate of JWT cache misses", "fieldConfig": { "defaults": { + "color": { + "mode": "thresholds" + }, "thresholds": { "mode": "absolute", "steps": [ @@ -1988,14 +1957,22 @@ } ] }, - "color": { - "mode": "thresholds" - }, "unit": "ops" }, "overrides": [] }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 16 + }, + "id": 46, "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" @@ -2003,51 +1980,44 @@ "fields": "", "values": false }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto" - } + "textMode": "auto" + }, + "targets": [ + { + "expr": "rate(api_jwt_cache_misses[$__rate_interval])", + "legendFormat": "misses/s", + "refId": "A" + } + ], + "title": "JWT Cache Misses/s", + "type": "stat" } - ] + ], + "title": "API Gateway", + "type": "row" }, { - "id": 50, - "type": "row", - "title": "Trading", + "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 12 }, - "collapsed": true, + "id": 50, "panels": [ { - "id": 51, - "type": "stat", - "title": "Portfolio Value", - "description": "Current portfolio value in USD", - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 13 - }, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, - "targets": [ - { - "expr": "trading_agent_portfolio_value_usd", - "legendFormat": "portfolio", - "refId": "A" - } - ], + "description": "Current portfolio value in USD", "fieldConfig": { "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, "thresholds": { "mode": "absolute", "steps": [ @@ -2065,15 +2035,22 @@ } ] }, - "color": { - "mode": "thresholds" - }, - "unit": "currencyUSD", - "decimals": 2 + "unit": "currencyUSD" }, "overrides": [] }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 13 + }, + "id": 51, "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" @@ -2081,37 +2058,29 @@ "fields": "", "values": false }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto" - } - }, - { - "id": 52, - "type": "stat", - "title": "Assets Selected", - "description": "Number of assets currently selected for trading", - "gridPos": { - "h": 6, - "w": 3, - "x": 6, - "y": 13 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "textMode": "auto" }, "targets": [ { - "expr": "trading_agent_assets_selected", - "legendFormat": "selected", + "expr": "trading_agent_portfolio_value_usd", + "legendFormat": "portfolio", "refId": "A" } ], + "title": "Portfolio Value", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of assets currently selected for trading", "fieldConfig": { "defaults": { + "color": { + "mode": "thresholds" + }, "thresholds": { "mode": "absolute", "steps": [ @@ -2125,14 +2094,22 @@ } ] }, - "color": { - "mode": "thresholds" - }, "unit": "none" }, "overrides": [] }, + "gridPos": { + "h": 6, + "w": 3, + "x": 6, + "y": 13 + }, + "id": 52, "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" @@ -2140,37 +2117,29 @@ "fields": "", "values": false }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto" - } - }, - { - "id": 53, - "type": "stat", - "title": "Universe Instruments", - "description": "Total instruments in the trading universe", - "gridPos": { - "h": 6, - "w": 3, - "x": 9, - "y": 13 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "textMode": "auto" }, "targets": [ { - "expr": "trading_agent_universe_instruments", - "legendFormat": "instruments", + "expr": "trading_agent_assets_selected", + "legendFormat": "selected", "refId": "A" } ], + "title": "Assets Selected", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Total instruments in the trading universe", "fieldConfig": { "defaults": { + "color": { + "mode": "thresholds" + }, "thresholds": { "mode": "absolute", "steps": [ @@ -2180,14 +2149,22 @@ } ] }, - "color": { - "mode": "thresholds" - }, "unit": "none" }, "overrides": [] }, + "gridPos": { + "h": 6, + "w": 3, + "x": 9, + "y": 13 + }, + "id": 53, "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" @@ -2195,58 +2172,54 @@ "fields": "", "values": false }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto" - } + "textMode": "auto" + }, + "targets": [ + { + "expr": "trading_agent_universe_instruments", + "legendFormat": "instruments", + "refId": "A" + } + ], + "title": "Universe Instruments", + "type": "stat" }, { - "id": 54, - "type": "timeseries", - "title": "Trading Measurements Rate", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "description": "Rate of trading measurements emitted", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "custom": { + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 15, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "mode": "none" + } + }, + "unit": "ops" + }, + "overrides": [] + }, "gridPos": { "h": 6, "w": 6, "x": 12, "y": 13 }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "targets": [ - { - "expr": "rate(trading_measurements_total[$__rate_interval])", - "legendFormat": "measurements/s", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 15, - "pointSize": 5, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "mode": "none" - }, - "axisPlacement": "auto", - "lineWidth": 2 - }, - "unit": "ops", - "color": { - "fixedColor": "blue", - "mode": "fixed" - } - }, - "overrides": [] - }, + "id": 54, "options": { "legend": { "displayMode": "list", @@ -2255,53 +2228,45 @@ "tooltip": { "mode": "single" } - } - }, - { - "id": 55, - "type": "timeseries", - "title": "Total Trading Latency", - "description": "End-to-end trading latency", - "gridPos": { - "h": 6, - "w": 6, - "x": 18, - "y": 13 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" }, "targets": [ { - "expr": "trading_total_latency_seconds", - "legendFormat": "total latency", + "expr": "rate(trading_measurements_total[$__rate_interval])", + "legendFormat": "measurements/s", "refId": "A" } ], + "title": "Trading Measurements Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "End-to-end trading latency", "fieldConfig": { "defaults": { + "color": { + "fixedColor": "orange", + "mode": "fixed" + }, "custom": { + "axisPlacement": "auto", "drawStyle": "line", - "lineInterpolation": "smooth", "fillOpacity": 10, + "lineInterpolation": "smooth", + "lineWidth": 2, "pointSize": 5, "showPoints": "auto", "spanNulls": false, "stacking": { "mode": "none" }, - "axisPlacement": "auto", - "lineWidth": 2, "thresholdsStyle": { "mode": "area" } }, - "unit": "s", - "color": { - "fixedColor": "orange", - "mode": "fixed" - }, "thresholds": { "mode": "absolute", "steps": [ @@ -2318,10 +2283,18 @@ "value": 0.1 } ] - } + }, + "unit": "s" }, "overrides": [] }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 13 + }, + "id": 55, "options": { "legend": { "displayMode": "list", @@ -2330,59 +2303,42 @@ "tooltip": { "mode": "single" } - } - }, - { - "id": 56, - "type": "timeseries", - "title": "Order Processing vs Risk Check vs Market Data", - "description": "Breakdown of trading pipeline latency components", - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 19 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" }, "targets": [ { - "expr": "trading_order_processing_seconds", - "legendFormat": "order processing", + "expr": "trading_total_latency_seconds", + "legendFormat": "total latency", "refId": "A" - }, - { - "expr": "trading_risk_check_seconds", - "legendFormat": "risk check", - "refId": "B" - }, - { - "expr": "trading_market_data_seconds", - "legendFormat": "market data", - "refId": "C" } ], + "title": "Total Trading Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Breakdown of trading pipeline latency components", "fieldConfig": { "defaults": { + "color": { + "mode": "palette-classic" + }, "custom": { + "axisPlacement": "auto", "drawStyle": "line", - "lineInterpolation": "smooth", "fillOpacity": 10, + "lineInterpolation": "smooth", + "lineWidth": 2, "pointSize": 5, "showPoints": "auto", "spanNulls": false, "stacking": { "mode": "none" - }, - "axisPlacement": "auto", - "lineWidth": 2 + } }, - "unit": "s", - "color": { - "mode": "palette-classic" - } + "unit": "s" }, "overrides": [ { @@ -2432,68 +2388,85 @@ } ] }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 19 + }, + "id": 56, "options": { "legend": { - "displayMode": "table", - "placement": "bottom", "calcs": [ "mean", "max", "lastNotNull" - ] + ], + "displayMode": "table", + "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } - } + }, + "targets": [ + { + "expr": "trading_order_processing_seconds", + "legendFormat": "order processing", + "refId": "A" + }, + { + "expr": "trading_risk_check_seconds", + "legendFormat": "risk check", + "refId": "B" + }, + { + "expr": "trading_market_data_seconds", + "legendFormat": "market data", + "refId": "C" + } + ], + "title": "Order Processing vs Risk Check vs Market Data", + "type": "timeseries" }, { - "id": 57, - "type": "timeseries", - "title": "Portfolio Value Over Time", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "description": "Historical portfolio value trend", + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "custom": { + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "scheme", + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none" + } + }, + "decimals": 2, + "unit": "currencyUSD" + }, + "overrides": [] + }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 19 }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "targets": [ - { - "expr": "trading_agent_portfolio_value_usd", - "legendFormat": "portfolio USD", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 20, - "pointSize": 5, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "mode": "none" - }, - "axisPlacement": "auto", - "lineWidth": 2, - "gradientMode": "scheme" - }, - "unit": "currencyUSD", - "color": { - "mode": "continuous-GrYlRd" - }, - "decimals": 2 - }, - "overrides": [] - }, + "id": 57, "options": { "legend": { "displayMode": "list", @@ -2502,46 +2475,44 @@ "tooltip": { "mode": "single" } - } + }, + "targets": [ + { + "expr": "trading_agent_portfolio_value_usd", + "legendFormat": "portfolio USD", + "refId": "A" + } + ], + "title": "Portfolio Value Over Time", + "type": "timeseries" } - ] + ], + "title": "Trading", + "type": "row" }, { - "id": 60, - "type": "row", - "title": "Metrics Health", + "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 13 }, - "collapsed": true, + "id": 60, "panels": [ { - "id": 61, - "type": "gauge", - "title": "Metrics Buffer Utilization", - "description": "Percentage of the metrics buffer currently in use", - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 14 - }, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, - "targets": [ - { - "expr": "metrics_buffer_utilization_percent", - "legendFormat": "buffer %", - "refId": "A" - } - ], + "description": "Percentage of the metrics buffer currently in use", "fieldConfig": { "defaults": { + "color": { + "mode": "thresholds" + }, + "max": 100, + "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -2563,16 +2534,19 @@ } ] }, - "color": { - "mode": "thresholds" - }, - "unit": "percent", - "min": 0, - "max": 100 + "unit": "percent" }, "overrides": [] }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 14 + }, + "id": 61, "options": { + "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" @@ -2581,55 +2555,46 @@ "values": false }, "showThresholdLabels": false, - "showThresholdMarkers": true, - "orientation": "auto" - } - }, - { - "id": 62, - "type": "timeseries", - "title": "Metrics Dropped Rate", - "description": "Rate at which metrics are being dropped", - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 14 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "showThresholdMarkers": true }, "targets": [ { - "expr": "rate(metrics_dropped_total[$__rate_interval])", - "legendFormat": "dropped/s", + "expr": "metrics_buffer_utilization_percent", + "legendFormat": "buffer %", "refId": "A" } ], + "title": "Metrics Buffer Utilization", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Rate at which metrics are being dropped", "fieldConfig": { "defaults": { + "color": { + "fixedColor": "red", + "mode": "fixed" + }, "custom": { + "axisPlacement": "auto", "drawStyle": "line", - "lineInterpolation": "smooth", "fillOpacity": 20, + "lineInterpolation": "smooth", + "lineWidth": 2, "pointSize": 5, "showPoints": "auto", "spanNulls": false, "stacking": { "mode": "none" }, - "axisPlacement": "auto", - "lineWidth": 2, "thresholdsStyle": { "mode": "area" } }, - "unit": "ops", - "color": { - "fixedColor": "red", - "mode": "fixed" - }, "thresholds": { "mode": "absolute", "steps": [ @@ -2646,10 +2611,18 @@ "value": 10 } ] - } + }, + "unit": "ops" }, "overrides": [] }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 14 + }, + "id": 62, "options": { "legend": { "displayMode": "list", @@ -2658,32 +2631,28 @@ "tooltip": { "mode": "single" } - } - }, - { - "id": 63, - "type": "stat", - "title": "Total Metrics Dropped", - "description": "Total count of metrics dropped since service start", - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 14 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" }, "targets": [ { - "expr": "metrics_dropped_total", - "legendFormat": "total dropped", + "expr": "rate(metrics_dropped_total[$__rate_interval])", + "legendFormat": "dropped/s", "refId": "A" } ], + "title": "Metrics Dropped Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Total count of metrics dropped since service start", "fieldConfig": { "defaults": { + "color": { + "mode": "thresholds" + }, "thresholds": { "mode": "absolute", "steps": [ @@ -2701,14 +2670,22 @@ } ] }, - "color": { - "mode": "thresholds" - }, "unit": "none" }, "overrides": [] }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 14 + }, + "id": 63, "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" @@ -2716,32 +2693,70 @@ "fields": "", "values": false }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "area", - "justifyMode": "auto" - } + "textMode": "auto" + }, + "targets": [ + { + "expr": "metrics_dropped_total", + "legendFormat": "total dropped", + "refId": "A" + } + ], + "title": "Total Metrics Dropped", + "type": "stat" } - ] + ], + "title": "Metrics Health", + "type": "row" } ], - "annotations": { + "refresh": "30s", + "schemaVersion": 39, + "tags": [ + "foxhunt", + "cockpit" + ], + "templating": { "list": [ { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" + "auto": true, + "auto_count": 30, + "auto_min": "1m", + "current": { + "selected": true, + "text": "auto", + "value": "$__auto_interval_interval" }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" + "label": "Interval", + "name": "interval", + "query": "1m,5m,15m,30m,1h", + "type": "interval" } ] }, - "links": [], - "graphTooltip": 1 + "time": { + "from": "now-6h", + "to": "now" + }, + "timezone": "browser", + "title": "Operations Cockpit", + "uid": "foxhunt-cockpit", + "weekStart": "", + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "type": "datasource", + "pluginId": "prometheus" + }, + { + "name": "DS_LOKI", + "type": "datasource", + "pluginId": "loki" + }, + { + "name": "DS_TEMPO", + "type": "datasource", + "pluginId": "${DS_TEMPO}" + } + ] } diff --git a/infra/k8s/monitoring/dashboards/foxhunt-infrastructure.json b/infra/k8s/monitoring/dashboards/foxhunt-infrastructure.json index 9784b801c..418f33696 100644 --- a/infra/k8s/monitoring/dashboards/foxhunt-infrastructure.json +++ b/infra/k8s/monitoring/dashboards/foxhunt-infrastructure.json @@ -1,61 +1,87 @@ { - "id": null, - "uid": "foxhunt-infrastructure", - "title": "Foxhunt Infrastructure & Cluster", - "description": "K8s cluster health, node resources, GPU, storage, and Prometheus health", - "tags": ["foxhunt", "infrastructure", "k8s"], - "timezone": "browser", - "schemaVersion": 39, - "version": 1, - "editable": true, - "refresh": "30s", - "time": { "from": "now-3h", "to": "now" }, - "fiscalYearStartMonth": 0, - "liveNow": false, - "weekStart": "", - "templating": { + "annotations": { "list": [ { - "name": "interval", - "type": "interval", - "query": "1m,5m,15m,30m,1h", - "auto": true, - "auto_count": 30, - "auto_min": "1m", - "current": { "selected": true, "text": "auto", "value": "$__auto_interval_interval" }, - "hide": 0, - "refresh": 2 - }, - { - "name": "node", - "type": "query", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "query": "label_values(node_uname_info, nodename)", - "refresh": 2, - "includeAll": true, - "multi": true, - "allValue": ".*", - "current": { "selected": true, "text": "All", "value": "$__all" }, - "sort": 1, - "hide": 0 + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" } ] }, + "description": "K8s cluster health, node resources, GPU, storage, and Prometheus health", + "editable": true, + "fiscalYearStartMonth": 0, + "links": [], + "liveNow": false, "panels": [ { - "id": 1, - "type": "row", - "title": "Cluster Overview", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "collapsed": false, - "panels": [] + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "Cluster Overview", + "type": "row" }, { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 1 + }, "id": 2, - "type": "stat", - "title": "Total Nodes", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 }, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, "targets": [ { "expr": "count(kube_node_info)", @@ -63,35 +89,57 @@ "refId": "A" } ], + "title": "Total Nodes", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], "thresholds": { "mode": "absolute", "steps": [ - { "color": "red", "value": null }, - { "color": "green", "value": 1 } + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } ] - }, - "mappings": [], - "color": { "mode": "thresholds" } + } }, "overrides": [] }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 1 + }, + "id": 3, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, - "orientation": "auto", - "textMode": "auto", "colorMode": "value", "graphMode": "none", - "justifyMode": "auto" - } - }, - { - "id": 3, - "type": "stat", - "title": "Nodes Ready", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "gridPos": { "h": 4, "w": 4, "x": 4, "y": 1 }, + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, "targets": [ { "expr": "sum(kube_node_status_condition{condition=\"Ready\", status=\"true\"})", @@ -99,35 +147,57 @@ "refId": "A" } ], + "title": "Nodes Ready", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], "thresholds": { "mode": "absolute", "steps": [ - { "color": "red", "value": null }, - { "color": "green", "value": 1 } + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } ] - }, - "mappings": [], - "color": { "mode": "thresholds" } + } }, "overrides": [] }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 1 + }, + "id": 4, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, - "orientation": "auto", - "textMode": "auto", "colorMode": "value", "graphMode": "none", - "justifyMode": "auto" - } - }, - { - "id": 4, - "type": "stat", - "title": "Pods Running", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 }, + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, "targets": [ { "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", phase=\"Running\"})", @@ -135,35 +205,57 @@ "refId": "A" } ], + "title": "Pods Running", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], "thresholds": { "mode": "absolute", "steps": [ - { "color": "red", "value": null }, - { "color": "green", "value": 1 } + { + "color": "green", + "value": null + } ] }, - "mappings": [], - "color": { "mode": "thresholds" } + "unit": "short" }, "overrides": [] }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 1 + }, + "id": 5, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, - "orientation": "auto", - "textMode": "auto", "colorMode": "value", "graphMode": "none", - "justifyMode": "auto" - } - }, - { - "id": 5, - "type": "stat", - "title": "Total CPU Capacity", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "gridPos": { "h": 4, "w": 4, "x": 12, "y": 1 }, + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "titleSize": 12 + }, + "textMode": "auto" + }, "targets": [ { "expr": "sum(kube_node_status_capacity{resource=\"cpu\"})", @@ -171,36 +263,54 @@ "refId": "A" } ], + "title": "Total CPU Capacity", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "unit": "short", + "color": { + "mode": "thresholds" + }, + "mappings": [], "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null } + { + "color": "green", + "value": null + } ] }, - "mappings": [], - "color": { "mode": "thresholds" } + "unit": "bytes" }, "overrides": [] }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 1 + }, + "id": 6, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, - "orientation": "auto", - "textMode": "auto", "colorMode": "value", "graphMode": "none", "justifyMode": "auto", - "text": { "titleSize": 12 } - } - }, - { - "id": 6, - "type": "stat", - "title": "Total Memory Capacity", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "gridPos": { "h": 4, "w": 4, "x": 16, "y": 1 }, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, "targets": [ { "expr": "sum(kube_node_status_capacity{resource=\"memory\"})", @@ -208,35 +318,57 @@ "refId": "A" } ], + "title": "Total Memory Capacity", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "unit": "bytes", + "color": { + "mode": "thresholds" + }, + "mappings": [], "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null } + { + "color": "yellow", + "value": null + }, + { + "color": "green", + "value": 1 + } ] - }, - "mappings": [], - "color": { "mode": "thresholds" } + } }, "overrides": [] }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 1 + }, + "id": 7, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, - "orientation": "auto", - "textMode": "auto", "colorMode": "value", "graphMode": "none", - "justifyMode": "auto" - } - }, - { - "id": 7, - "type": "stat", - "title": "GPUs Available", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "gridPos": { "h": 4, "w": 4, "x": 20, "y": 1 }, + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, "targets": [ { "expr": "sum(kube_node_status_capacity{resource=\"nvidia_com_gpu\"}) or vector(0)", @@ -244,43 +376,83 @@ "refId": "A" } ], + "title": "GPUs Available", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 10, + "panels": [], + "title": "Node Resources", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisSoftMax": 1, + "axisSoftMin": 0, + "drawStyle": "line", + "fillOpacity": 30, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "max": 1, + "min": 0, "thresholds": { "mode": "absolute", "steps": [ - { "color": "yellow", "value": null }, - { "color": "green", "value": 1 } + { + "color": "green", + "value": null + } ] }, - "mappings": [], - "color": { "mode": "thresholds" } + "unit": "percentunit" }, "overrides": [] }, - "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto" - } - }, - { - "id": 10, - "type": "row", - "title": "Node Resources", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, - "collapsed": false, - "panels": [] - }, - { + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 6 + }, "id": 11, - "type": "timeseries", - "title": "CPU Usage by Node (stacked)", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, "targets": [ { "expr": "(1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[$__rate_interval]))) * on(instance) group_left(nodename) node_uname_info", @@ -288,39 +460,75 @@ "refId": "A" } ], + "title": "CPU Usage by Node (stacked)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "unit": "percentunit", - "min": 0, - "max": 1, + "color": { + "mode": "palette-classic" + }, "custom": { "drawStyle": "line", + "fillOpacity": 20, "lineInterpolation": "smooth", - "fillOpacity": 30, - "stacking": { "mode": "normal", "group": "A" }, "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, - "axisSoftMin": 0, - "axisSoftMax": 1 + "stacking": { + "mode": "none" + } }, - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, - "color": { "mode": "palette-classic" } + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.8 + }, + { + "color": "red", + "value": 0.95 + } + ] + }, + "unit": "percentunit" }, "overrides": [] }, - "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } - } - }, - { + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 6 + }, "id": 12, - "type": "timeseries", - "title": "Memory Usage by Node", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, "targets": [ { "expr": "(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * on(instance) group_left(nodename) node_uname_info", @@ -328,37 +536,61 @@ "refId": "A" } ], + "title": "Memory Usage by Node", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "unit": "percentunit", - "min": 0, - "max": 1, + "color": { + "mode": "palette-classic" + }, "custom": { "drawStyle": "line", + "fillOpacity": 15, "lineInterpolation": "smooth", - "fillOpacity": 20, - "stacking": { "mode": "none" }, "lineWidth": 1, "pointSize": 5, "showPoints": "never", - "spanNulls": false + "spanNulls": false, + "stacking": { + "mode": "none" + } }, - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "orange", "value": 0.8 }, { "color": "red", "value": 0.95 }] }, - "color": { "mode": "palette-classic" } + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" }, "overrides": [] }, - "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } - } - }, - { + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 14 + }, "id": 13, - "type": "timeseries", - "title": "Disk I/O (Read / Write)", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "gridPos": { "h": 8, "w": 8, "x": 0, "y": 14 }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, "targets": [ { "expr": "sum by (instance) (rate(node_disk_read_bytes_total[$__rate_interval])) * on(instance) group_left(nodename) node_uname_info", @@ -371,35 +603,61 @@ "refId": "B" } ], + "title": "Disk I/O (Read / Write)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "unit": "Bps", + "color": { + "mode": "palette-classic" + }, "custom": { "drawStyle": "line", - "lineInterpolation": "smooth", "fillOpacity": 15, - "stacking": { "mode": "none" }, + "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "showPoints": "never", - "spanNulls": false + "spanNulls": false, + "stacking": { + "mode": "none" + } }, - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, - "color": { "mode": "palette-classic" } + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" }, "overrides": [] }, - "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "list", "placement": "bottom" } - } - }, - { + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 14 + }, "id": 14, - "type": "timeseries", - "title": "Network I/O (Receive / Transmit)", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "gridPos": { "h": 8, "w": 8, "x": 8, "y": 14 }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, "targets": [ { "expr": "sum by (instance) (rate(node_network_receive_bytes_total{device!~\"lo|veth.*|cali.*|flannel.*\"}[$__rate_interval])) * on(instance) group_left(nodename) node_uname_info", @@ -412,35 +670,61 @@ "refId": "B" } ], + "title": "Network I/O (Receive / Transmit)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "unit": "Bps", + "color": { + "mode": "palette-classic" + }, "custom": { "drawStyle": "line", + "fillOpacity": 0, "lineInterpolation": "smooth", - "fillOpacity": 15, - "stacking": { "mode": "none" }, "lineWidth": 1, "pointSize": 5, "showPoints": "never", - "spanNulls": false + "spanNulls": false, + "stacking": { + "mode": "none" + } }, - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, - "color": { "mode": "palette-classic" } + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" }, "overrides": [] }, - "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "list", "placement": "bottom" } - } - }, - { + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 14 + }, "id": 15, - "type": "timeseries", - "title": "Load Average (1m / 5m / 15m)", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "gridPos": { "h": 8, "w": 8, "x": 16, "y": 14 }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, "targets": [ { "expr": "node_load1 * on(instance) group_left(nodename) node_uname_info", @@ -458,43 +742,96 @@ "refId": "C" } ], + "title": "Load Average (1m / 5m / 15m)", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 20, + "panels": [], + "title": "Kubernetes Workloads", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "unit": "short", + "color": { + "mode": "palette-classic" + }, "custom": { "drawStyle": "line", - "lineInterpolation": "smooth", "fillOpacity": 0, - "stacking": { "mode": "none" }, - "lineWidth": 1, + "lineInterpolation": "stepAfter", + "lineWidth": 2, "pointSize": 5, "showPoints": "never", - "spanNulls": false + "spanNulls": false, + "stacking": { + "mode": "none" + } }, - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, - "color": { "mode": "palette-classic" } + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*desired" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 23 }, - "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "list", "placement": "bottom" } - } - }, - { - "id": 20, - "type": "row", - "title": "Kubernetes Workloads", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 22 }, - "collapsed": false, - "panels": [] - }, - { "id": 21, - "type": "timeseries", - "title": "Deployment Replicas (Desired vs Available)", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 23 }, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, "targets": [ { "expr": "kube_deployment_spec_replicas{namespace=\"foxhunt\"}", @@ -507,40 +844,116 @@ "refId": "B" } ], + "title": "Deployment Replicas (Desired vs Available)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "unit": "short", - "custom": { - "drawStyle": "line", - "lineInterpolation": "stepAfter", - "fillOpacity": 0, - "stacking": { "mode": "none" }, - "lineWidth": 2, - "pointSize": 5, - "showPoints": "never", - "spanNulls": false + "color": { + "mode": "palette-classic" }, - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, - "color": { "mode": "palette-classic" } + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } }, "overrides": [ { - "matcher": { "id": "byRegexp", "options": ".*desired" }, - "properties": [{ "id": "custom.lineStyle", "value": { "fill": "dash", "dash": [10, 10] } }] + "matcher": { + "id": "byName", + "options": "Running" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Pending" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Failed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Succeeded" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] } ] }, - "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull"] } - } - }, - { + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 23 + }, "id": 22, - "type": "piechart", - "title": "Pod Phase Distribution", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "gridPos": { "h": 8, "w": 6, "x": 12, "y": 23 }, + "options": { + "legend": { + "displayMode": "list", + "placement": "right" + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single" + } + }, "targets": [ { "expr": "sum by (phase) (kube_pod_status_phase{namespace=\"foxhunt\"})", @@ -548,31 +961,66 @@ "refId": "A" } ], - "fieldConfig": { - "defaults": { - "color": { "mode": "palette-classic" }, - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } - }, - "overrides": [ - { "matcher": { "id": "byName", "options": "Running" }, "properties": [{ "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } }] }, - { "matcher": { "id": "byName", "options": "Pending" }, "properties": [{ "id": "color", "value": { "fixedColor": "yellow", "mode": "fixed" } }] }, - { "matcher": { "id": "byName", "options": "Failed" }, "properties": [{ "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } }] }, - { "matcher": { "id": "byName", "options": "Succeeded" }, "properties": [{ "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } }] } - ] - }, - "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, - "pieType": "donut", - "tooltip": { "mode": "single" }, - "legend": { "displayMode": "list", "placement": "right" } - } + "title": "Pod Phase Distribution", + "type": "piechart" }, { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "bars", + "fillOpacity": 80, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 23 + }, "id": 23, - "type": "timeseries", - "title": "Container Restarts", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "gridPos": { "h": 8, "w": 6, "x": 18, "y": 23 }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, "targets": [ { "expr": "increase(kube_pod_container_status_restarts_total{namespace=\"foxhunt\"}[$__rate_interval])", @@ -580,177 +1028,390 @@ "refId": "A" } ], + "title": "Container Restarts", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "unit": "short", - "custom": { - "drawStyle": "bars", - "lineInterpolation": "linear", - "fillOpacity": 80, - "stacking": { "mode": "normal", "group": "A" }, - "lineWidth": 1, - "pointSize": 5, - "showPoints": "never", - "spanNulls": false + "color": { + "mode": "thresholds" }, - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "red", "value": 1 }] }, - "color": { "mode": "palette-classic" } - }, - "overrides": [] - }, - "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "list", "placement": "bottom" } - } - }, - { - "id": 24, - "type": "table", - "title": "Pending / Failed Pods", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "gridPos": { "h": 6, "w": 24, "x": 0, "y": 31 }, - "targets": [ - { - "expr": "kube_pod_status_phase{namespace=\"foxhunt\", phase=~\"Pending|Failed\"} > 0", - "legendFormat": "", - "refId": "A", - "format": "table", - "instant": true - } - ], - "fieldConfig": { - "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, - "color": { "mode": "thresholds" } - }, - "overrides": [ - { "matcher": { "id": "byName", "options": "Time" }, "properties": [{ "id": "custom.hidden", "value": true }] }, - { "matcher": { "id": "byName", "options": "__name__" }, "properties": [{ "id": "custom.hidden", "value": true }] }, - { "matcher": { "id": "byName", "options": "job" }, "properties": [{ "id": "custom.hidden", "value": true }] }, - { "matcher": { "id": "byName", "options": "instance" }, "properties": [{ "id": "custom.hidden", "value": true }] } - ] - }, - "options": { - "showHeader": true, - "sortBy": [{ "desc": true, "displayName": "Value" }], - "footer": { "show": false } - }, - "transformations": [ - { "id": "organize", "options": { "excludeByName": { "Time": true, "__name__": true, "job": true, "instance": true }, "indexByName": {}, "renameByName": { "pod": "Pod", "phase": "Phase", "namespace": "Namespace", "Value": "Count" } } } - ] - }, - { - "id": 30, - "type": "row", - "title": "Storage", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 37 }, - "collapsed": false, - "panels": [] - }, - { - "id": 31, - "type": "bargauge", - "title": "Disk Usage by Mountpoint", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 38 }, - "targets": [ - { - "expr": "(1 - (node_filesystem_avail_bytes{fstype=~\"ext4|xfs|btrfs\"} / node_filesystem_size_bytes{fstype=~\"ext4|xfs|btrfs\"})) * on(instance) group_left(nodename) node_uname_info", - "legendFormat": "{{ nodename }} {{ mountpoint }}", - "refId": "A", - "instant": true - } - ], - "fieldConfig": { - "defaults": { - "unit": "percentunit", - "min": 0, - "max": 1, "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 0.7 }, - { "color": "orange", "value": 0.85 }, - { "color": "red", "value": 0.95 } + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } ] }, - "color": { "mode": "thresholds" } + { + "matcher": { + "id": "byName", + "options": "__name__" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "job" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "instance" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 24, + "options": { + "footer": { + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Value" + } + ] + }, + "targets": [ + { + "expr": "kube_pod_status_phase{namespace=\"foxhunt\", phase=~\"Pending|Failed\"} > 0", + "format": "table", + "instant": true, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Pending / Failed Pods", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "__name__": true, + "instance": true, + "job": true + }, + "indexByName": {}, + "renameByName": { + "Value": "Count", + "namespace": "Namespace", + "phase": "Phase", + "pod": "Pod" + } + } + } + ], + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 37 + }, + "id": 30, + "panels": [], + "title": "Storage", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.7 + }, + { + "color": "orange", + "value": 0.85 + }, + { + "color": "red", + "value": 0.95 + } + ] + }, + "unit": "percentunit" }, "overrides": [] }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 38 + }, + "id": 31, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, - "orientation": "horizontal", "displayMode": "gradient", - "showUnfilled": true, + "minVizHeight": 10, "minVizWidth": 0, - "minVizHeight": 10 - } + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "targets": [ + { + "expr": "(1 - (node_filesystem_avail_bytes{fstype=~\"ext4|xfs|btrfs\"} / node_filesystem_size_bytes{fstype=~\"ext4|xfs|btrfs\"})) * on(instance) group_left(nodename) node_uname_info", + "instant": true, + "legendFormat": "{{ nodename }} {{ mountpoint }}", + "refId": "A" + } + ], + "title": "Disk Usage by Mountpoint", + "type": "bargauge" }, { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Storage" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 38 + }, "id": 32, - "type": "table", - "title": "PVC Status & Requested Storage", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 38 }, + "options": { + "footer": { + "show": false + }, + "showHeader": true + }, "targets": [ { "expr": "kube_persistentvolumeclaim_status_phase{namespace=\"foxhunt\"}", - "legendFormat": "", - "refId": "A", "format": "table", - "instant": true + "instant": true, + "legendFormat": "", + "refId": "A" }, { "expr": "kube_persistentvolumeclaim_resource_requests_storage_bytes{namespace=\"foxhunt\"}", - "legendFormat": "", - "refId": "B", "format": "table", - "instant": true + "instant": true, + "legendFormat": "", + "refId": "B" } ], - "fieldConfig": { - "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, - "color": { "mode": "thresholds" } - }, - "overrides": [ - { "matcher": { "id": "byName", "options": "Storage" }, "properties": [{ "id": "unit", "value": "bytes" }] } - ] - }, - "options": { - "showHeader": true, - "footer": { "show": false } - }, + "title": "PVC Status & Requested Storage", "transformations": [ { "id": "seriesToColumns", - "options": { "byField": "persistentvolumeclaim" } + "options": { + "byField": "persistentvolumeclaim" + } }, { "id": "organize", "options": { - "excludeByName": { "Time 1": true, "Time 2": true, "__name__ 1": true, "__name__ 2": true, "job 1": true, "job 2": true, "instance 1": true, "instance 2": true, "namespace 1": true, "namespace 2": true, "phase": false }, - "renameByName": { "persistentvolumeclaim": "PVC", "phase": "Phase", "Value 1": "Phase Value", "Value 2": "Storage" } + "excludeByName": { + "Time 1": true, + "Time 2": true, + "__name__ 1": true, + "__name__ 2": true, + "instance 1": true, + "instance 2": true, + "job 1": true, + "job 2": true, + "namespace 1": true, + "namespace 2": true, + "phase": false + }, + "renameByName": { + "Value 1": "Phase Value", + "Value 2": "Storage", + "persistentvolumeclaim": "PVC", + "phase": "Phase" + } } } - ] + ], + "type": "table" }, { - "id": 40, - "type": "row", - "title": "GPU Resources", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 46 }, "collapsed": false, - "panels": [] + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 46 + }, + "id": 40, + "panels": [], + "title": "GPU Resources", + "type": "row" }, { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 25, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 80 + }, + { + "color": "red", + "value": 95 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 47 + }, "id": 41, - "type": "timeseries", - "title": "GPU Utilization %", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "gridPos": { "h": 8, "w": 8, "x": 0, "y": 47 }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, "targets": [ { "expr": "DCGM_FI_DEV_GPU_UTIL", @@ -758,45 +1419,84 @@ "refId": "A" } ], + "title": "GPU Utilization %", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "unit": "percent", - "min": 0, - "max": 100, + "color": { + "mode": "palette-classic" + }, "custom": { "drawStyle": "line", + "fillOpacity": 20, "lineInterpolation": "smooth", - "fillOpacity": 25, - "stacking": { "mode": "none" }, - "lineWidth": 2, + "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false, - "thresholdsStyle": { "mode": "line" } + "stacking": { + "mode": "none" + } }, "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 80 }, - { "color": "red", "value": 95 } + { + "color": "green", + "value": null + } ] }, - "color": { "mode": "palette-classic" } + "unit": "decmbytes" }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*free" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 5 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 47 }, - "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "list", "placement": "bottom" } - } - }, - { "id": 42, - "type": "timeseries", - "title": "VRAM Usage", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "gridPos": { "h": 8, "w": 8, "x": 8, "y": 47 }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, "targets": [ { "expr": "DCGM_FI_DEV_FB_USED", @@ -809,37 +1509,72 @@ "refId": "B" } ], - "fieldConfig": { - "defaults": { - "unit": "decmbytes", - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 20, - "stacking": { "mode": "none" }, - "lineWidth": 1, - "pointSize": 5, - "showPoints": "never", - "spanNulls": false - }, - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, - "color": { "mode": "palette-classic" } - }, - "overrides": [ - { "matcher": { "id": "byRegexp", "options": ".*free" }, "properties": [{ "id": "custom.fillOpacity", "value": 5 }, { "id": "custom.lineStyle", "value": { "fill": "dash", "dash": [10, 10] } }] } - ] - }, - "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "list", "placement": "bottom" } - } + "title": "VRAM Usage", + "type": "timeseries" }, { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 0, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 75 + }, + { + "color": "red", + "value": 90 + } + ] + }, + "unit": "celsius" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 47 + }, "id": 43, - "type": "timeseries", - "title": "GPU Temperature", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "gridPos": { "h": 8, "w": 8, "x": 16, "y": 47 }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, "targets": [ { "expr": "DCGM_FI_DEV_GPU_TEMP", @@ -852,43 +1587,61 @@ "refId": "B" } ], + "title": "GPU Temperature", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "unit": "celsius", + "color": { + "mode": "palette-classic" + }, "custom": { "drawStyle": "line", + "fillOpacity": 15, "lineInterpolation": "smooth", - "fillOpacity": 0, - "stacking": { "mode": "none" }, "lineWidth": 2, "pointSize": 5, "showPoints": "never", "spanNulls": false, - "thresholdsStyle": { "mode": "line" } + "stacking": { + "mode": "none" + } }, "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "orange", "value": 75 }, - { "color": "red", "value": 90 } + { + "color": "green", + "value": null + } ] }, - "color": { "mode": "palette-classic" } + "unit": "watt" }, "overrides": [] }, - "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "list", "placement": "bottom" } - } - }, - { + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 55 + }, "id": 44, - "type": "timeseries", - "title": "GPU Power Draw", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 55 }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, "targets": [ { "expr": "DCGM_FI_DEV_POWER_USAGE", @@ -896,35 +1649,61 @@ "refId": "A" } ], + "title": "GPU Power Draw", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "unit": "watt", + "color": { + "mode": "palette-classic" + }, "custom": { "drawStyle": "line", - "lineInterpolation": "smooth", "fillOpacity": 15, - "stacking": { "mode": "none" }, - "lineWidth": 2, + "lineInterpolation": "smooth", + "lineWidth": 1, "pointSize": 5, "showPoints": "never", - "spanNulls": false + "spanNulls": false, + "stacking": { + "mode": "none" + } }, - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, - "color": { "mode": "palette-classic" } + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" }, "overrides": [] }, - "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "list", "placement": "bottom" } - } - }, - { + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 55 + }, "id": 45, - "type": "timeseries", - "title": "GPU PCIe Throughput", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 55 }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, "targets": [ { "expr": "DCGM_FI_PROF_PCIE_RX_BYTES", @@ -937,43 +1716,75 @@ "refId": "B" } ], + "title": "GPU PCIe Throughput", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 63 + }, + "id": 50, + "panels": [], + "title": "Prometheus Health", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "unit": "Bps", - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 15, - "stacking": { "mode": "none" }, - "lineWidth": 1, - "pointSize": 5, - "showPoints": "never", - "spanNulls": false + "color": { + "mode": "thresholds" }, - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, - "color": { "mode": "palette-classic" } + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 500000 + }, + { + "color": "red", + "value": 1000000 + } + ] + }, + "unit": "short" }, "overrides": [] }, - "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "list", "placement": "bottom" } - } - }, - { - "id": 50, - "type": "row", - "title": "Prometheus Health", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 63 }, - "collapsed": false, - "panels": [] - }, - { + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 64 + }, "id": 51, - "type": "stat", - "title": "Active Time Series", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "gridPos": { "h": 4, "w": 6, "x": 0, "y": 64 }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, "targets": [ { "expr": "prometheus_tsdb_head_series", @@ -981,37 +1792,62 @@ "refId": "A" } ], + "title": "Active Time Series", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "unit": "short", + "color": { + "mode": "thresholds" + }, + "mappings": [], "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 500000 }, - { "color": "red", "value": 1000000 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 5368709120 + }, + { + "color": "red", + "value": 10737418240 + } ] }, - "mappings": [], - "color": { "mode": "thresholds" } + "unit": "bytes" }, "overrides": [] }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 64 + }, + "id": 52, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, - "orientation": "auto", - "textMode": "auto", "colorMode": "value", "graphMode": "area", - "justifyMode": "auto" - } - }, - { - "id": 52, - "type": "stat", - "title": "TSDB Storage Size", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "gridPos": { "h": 4, "w": 6, "x": 6, "y": 64 }, + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, "targets": [ { "expr": "prometheus_tsdb_storage_blocks_bytes", @@ -1019,37 +1855,69 @@ "refId": "A" } ], + "title": "TSDB Storage Size", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "unit": "bytes", + "color": { + "mode": "thresholds" + }, + "mappings": [], "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 5368709120 }, - { "color": "red", "value": 10737418240 } + { + "color": "green", + "value": null + } ] - }, - "mappings": [], - "color": { "mode": "thresholds" } + } }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "down" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 64 }, - "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto" - } - }, - { "id": 53, - "type": "stat", - "title": "Scrape Targets Up", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "gridPos": { "h": 4, "w": 6, "x": 12, "y": 64 }, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, "targets": [ { "expr": "count(up == 1)", @@ -1062,36 +1930,69 @@ "refId": "B" } ], + "title": "Scrape Targets Up", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "mode": "none" + } + }, "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null } + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 5 + } ] }, - "mappings": [], - "color": { "mode": "thresholds" } + "unit": "s" }, - "overrides": [ - { "matcher": { "id": "byName", "options": "down" }, "properties": [{ "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } }] } - ] + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 68 }, - "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, - "orientation": "auto", - "textMode": "value_and_name", - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto" - } - }, - { "id": 54, - "type": "timeseries", - "title": "Scrape Duration by Job", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 68 }, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, "targets": [ { "expr": "scrape_duration_seconds", @@ -1099,81 +2000,207 @@ "refId": "A" } ], - "fieldConfig": { - "defaults": { - "unit": "s", - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 10, - "stacking": { "mode": "none" }, - "lineWidth": 1, - "pointSize": 5, - "showPoints": "never", - "spanNulls": false - }, - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "red", "value": 5 }] }, - "color": { "mode": "palette-classic" } - }, - "overrides": [] - }, - "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } - } + "title": "Scrape Duration by Job", + "type": "timeseries" }, { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "text": "DOWN" + }, + "1": { + "color": "green", + "text": "UP" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "__name__" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-background" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 68 + }, "id": 55, - "type": "table", - "title": "Scrape Targets Status", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 68 }, + "options": { + "footer": { + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "Value" + } + ] + }, "targets": [ { "expr": "up", - "legendFormat": "", - "refId": "A", "format": "table", - "instant": true + "instant": true, + "legendFormat": "", + "refId": "A" } ], - "fieldConfig": { - "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] }, - "color": { "mode": "thresholds" }, - "mappings": [ - { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } } - ] - }, - "overrides": [ - { "matcher": { "id": "byName", "options": "Time" }, "properties": [{ "id": "custom.hidden", "value": true }] }, - { "matcher": { "id": "byName", "options": "__name__" }, "properties": [{ "id": "custom.hidden", "value": true }] }, - { "matcher": { "id": "byName", "options": "Value" }, "properties": [{ "id": "custom.cellOptions", "value": { "type": "color-background" } }] } - ] - }, - "options": { - "showHeader": true, - "sortBy": [{ "desc": false, "displayName": "Value" }], - "footer": { "show": false } - }, + "title": "Scrape Targets Status", "transformations": [ - { "id": "organize", "options": { "excludeByName": { "Time": true, "__name__": true }, "renameByName": { "job": "Job", "instance": "Instance", "Value": "Status" } } } - ] + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "__name__": true + }, + "renameByName": { + "Value": "Status", + "instance": "Instance", + "job": "Job" + } + } + } + ], + "type": "table" } ], - "annotations": { + "refresh": "30s", + "schemaVersion": 39, + "style": "dark", + "tags": [ + "foxhunt", + "infrastructure", + "k8s" + ], + "templating": { "list": [ { - "builtIn": 1, - "datasource": { "type": "grafana", "uid": "-- Grafana --" }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" + "auto": true, + "auto_count": 30, + "auto_min": "1m", + "current": { + "selected": true, + "text": "auto", + "value": "$__auto_interval_interval" + }, + "hide": 0, + "name": "interval", + "query": "1m,5m,15m,30m,1h", + "refresh": 2, + "type": "interval" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "hide": 0, + "includeAll": true, + "multi": true, + "name": "node", + "query": "label_values(node_uname_info, nodename)", + "refresh": 2, + "sort": 1, + "type": "query" } ] }, - "links": [], - "style": "dark" + "time": { + "from": "now-3h", + "to": "now" + }, + "timezone": "browser", + "title": "Infrastructure & Cluster", + "uid": "foxhunt-infrastructure", + "weekStart": "", + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "type": "datasource", + "pluginId": "prometheus" + }, + { + "name": "DS_LOKI", + "type": "datasource", + "pluginId": "loki" + }, + { + "name": "DS_TEMPO", + "type": "datasource", + "pluginId": "${DS_TEMPO}" + } + ] } diff --git a/infra/k8s/monitoring/dashboards/foxhunt-observability.json b/infra/k8s/monitoring/dashboards/foxhunt-observability.json index cb342dc14..c960124b6 100644 --- a/infra/k8s/monitoring/dashboards/foxhunt-observability.json +++ b/infra/k8s/monitoring/dashboards/foxhunt-observability.json @@ -5,21 +5,30 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, - "id": null, "links": [], "panels": [ { "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, "id": 100, "title": "API Gateway Auth", "type": "row" }, { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -29,33 +38,61 @@ "lineWidth": 1, "pointSize": 5, "showPoints": "auto", - "stacking": { "mode": "none" } + "stacking": { + "mode": "none" + } }, "unit": "ops" }, "overrides": [ { - "matcher": { "id": "byName", "options": "failure" }, + "matcher": { + "id": "byName", + "options": "failure" + }, "properties": [ - { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } ] }, { - "matcher": { "id": "byName", "options": "success" }, + "matcher": { + "id": "byName", + "options": "success" + }, "properties": [ - { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } ] } ] }, - "gridPos": { "h": 8, "w": 8, "x": 0, "y": 1 }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 1 + }, "id": 1, "options": { - "tooltip": { "mode": "multi" }, - "legend": { "displayMode": "table", "placement": "bottom" } + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } }, - "title": "Auth Request Rate (Success / Failure)", - "type": "timeseries", "targets": [ { "expr": "rate(api_auth_requests_success[$interval])", @@ -72,13 +109,20 @@ "legendFormat": "total", "refId": "C" } - ] + ], + "title": "Auth Request Rate (Success / Failure)", + "type": "timeseries" }, { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -88,20 +132,30 @@ "lineWidth": 1, "pointSize": 5, "showPoints": "never", - "stacking": { "mode": "normal" } + "stacking": { + "mode": "normal" + } }, "unit": "ops" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 8, "x": 8, "y": 1 }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 1 + }, "id": 2, "options": { - "tooltip": { "mode": "multi" }, - "legend": { "displayMode": "table", "placement": "bottom" } + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } }, - "title": "Auth Error Breakdown", - "type": "timeseries", "targets": [ { "expr": "rate(api_auth_errors_expired_jwt[$interval])", @@ -128,13 +182,20 @@ "legendFormat": "rate_limited", "refId": "E" } - ] + ], + "title": "Auth Error Breakdown", + "type": "timeseries" }, { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -144,20 +205,30 @@ "lineWidth": 1, "pointSize": 5, "showPoints": "auto", - "stacking": { "mode": "none" } + "stacking": { + "mode": "none" + } }, "unit": "\u00b5s" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 4, "x": 16, "y": 1 }, + "gridPos": { + "h": 8, + "w": 4, + "x": 16, + "y": 1 + }, "id": 3, "options": { - "tooltip": { "mode": "multi" }, - "legend": { "displayMode": "list", "placement": "bottom" } + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } }, - "title": "JWT Validation Latency", - "type": "timeseries", "targets": [ { "expr": "histogram_quantile(0.50, rate(api_jwt_validation_duration_microseconds_bucket[$interval]))", @@ -174,86 +245,138 @@ "legendFormat": "p99", "refId": "C" } - ] + ], + "title": "JWT Validation Latency", + "type": "timeseries" }, { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "thresholds": { "mode": "absolute", "steps": [ - { "color": "blue", "value": null } + { + "color": "blue", + "value": null + } ] }, "unit": "short" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 4, "x": 20, "y": 1 }, + "gridPos": { + "h": 8, + "w": 4, + "x": 20, + "y": 1 + }, "id": 4, "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "background", - "graphMode": "area" + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } }, - "title": "Active JWT Tokens", - "type": "stat", "targets": [ { "expr": "api_active_jwt_tokens", "legendFormat": "", "refId": "A" } - ] + ], + "title": "Active JWT Tokens", + "type": "stat" }, { "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 9 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, "id": 101, "title": "API Gateway Security", "type": "row" }, { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "orange", "value": 500 }, - { "color": "red", "value": 1000 } + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 500 + }, + { + "color": "red", + "value": 1000 + } ] }, "unit": "short" }, "overrides": [] }, - "gridPos": { "h": 6, "w": 6, "x": 0, "y": 10 }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 10 + }, "id": 5, "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "background", - "graphMode": "area" + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } }, - "title": "Rate Limiter Entries", - "type": "stat", "targets": [ { "expr": "api_rate_limiter_entries", "legendFormat": "", "refId": "A" } - ] + ], + "title": "Rate Limiter Entries", + "type": "stat" }, { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -263,69 +386,109 @@ "lineWidth": 1, "pointSize": 5, "showPoints": "auto", - "stacking": { "mode": "none" } + "stacking": { + "mode": "none" + } }, - "unit": "percentunit", + "max": 1, "min": 0, - "max": 1 + "unit": "percentunit" }, "overrides": [] }, - "gridPos": { "h": 6, "w": 6, "x": 6, "y": 10 }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 10 + }, "id": 6, "options": { - "tooltip": { "mode": "multi" }, - "legend": { "displayMode": "list", "placement": "bottom" } + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } }, - "title": "RBAC Cache Hit Ratio", - "type": "timeseries", "targets": [ { "expr": "api_rbac_cache_hits / (api_rbac_cache_hits + api_rbac_cache_misses)", "legendFormat": "hit ratio", "refId": "A" } - ] + ], + "title": "RBAC Cache Hit Ratio", + "type": "timeseries" }, { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "orange", "value": 10 }, - { "color": "red", "value": 50 } + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 10 + }, + { + "color": "red", + "value": 50 + } ] }, "unit": "short" }, "overrides": [] }, - "gridPos": { "h": 6, "w": 6, "x": 12, "y": 10 }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 10 + }, "id": 7, "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "background", - "graphMode": "area" + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } }, - "title": "Revoked Tokens Cached", - "type": "stat", "targets": [ { "expr": "api_revoked_tokens_cached", "legendFormat": "", "refId": "A" } - ] + ], + "title": "Revoked Tokens Cached", + "type": "stat" }, { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -335,20 +498,30 @@ "lineWidth": 1, "pointSize": 5, "showPoints": "auto", - "stacking": { "mode": "none" } + "stacking": { + "mode": "none" + } }, "unit": "\u00b5s" }, "overrides": [] }, - "gridPos": { "h": 6, "w": 6, "x": 18, "y": 10 }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 10 + }, "id": 8, "options": { - "tooltip": { "mode": "multi" }, - "legend": { "displayMode": "list", "placement": "bottom" } + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } }, - "title": "MFA Verification Latency", - "type": "timeseries", "targets": [ { "expr": "histogram_quantile(0.50, rate(api_mfa_verification_duration_microseconds_bucket[$interval]))", @@ -365,20 +538,32 @@ "legendFormat": "p99", "refId": "C" } - ] + ], + "title": "MFA Verification Latency", + "type": "timeseries" }, { "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 16 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, "id": 102, "title": "API Gateway Routing", "type": "row" }, { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -388,20 +573,30 @@ "lineWidth": 1, "pointSize": 5, "showPoints": "auto", - "stacking": { "mode": "none" } + "stacking": { + "mode": "none" + } }, "unit": "\u00b5s" }, "overrides": [] }, - "gridPos": { "h": 7, "w": 8, "x": 0, "y": 17 }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 17 + }, "id": 9, "options": { - "tooltip": { "mode": "multi" }, - "legend": { "displayMode": "list", "placement": "bottom" } + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } }, - "title": "Routing Duration p50/p95/p99", - "type": "timeseries", "targets": [ { "expr": "histogram_quantile(0.50, rate(api_routing_duration_microseconds_bucket[$interval]))", @@ -418,13 +613,20 @@ "legendFormat": "p99", "refId": "C" } - ] + ], + "title": "Routing Duration p50/p95/p99", + "type": "timeseries" }, { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -434,20 +636,30 @@ "lineWidth": 1, "pointSize": 5, "showPoints": "never", - "stacking": { "mode": "normal" } + "stacking": { + "mode": "normal" + } }, "unit": "ops" }, "overrides": [] }, - "gridPos": { "h": 7, "w": 8, "x": 8, "y": 17 }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 17 + }, "id": 10, "options": { - "tooltip": { "mode": "multi" }, - "legend": { "displayMode": "table", "placement": "bottom" } + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } }, - "title": "Config Updates Rate", - "type": "timeseries", "targets": [ { "expr": "rate(api_config_updates_auth[$interval])", @@ -469,13 +681,20 @@ "legendFormat": "routing", "refId": "D" } - ] + ], + "title": "Config Updates Rate", + "type": "timeseries" }, { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -485,22 +704,32 @@ "lineWidth": 1, "pointSize": 5, "showPoints": "auto", - "stacking": { "mode": "none" } + "stacking": { + "mode": "none" + } }, - "unit": "percentunit", + "max": 1, "min": 0, - "max": 1 + "unit": "percentunit" }, "overrides": [] }, - "gridPos": { "h": 7, "w": 8, "x": 16, "y": 17 }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 17 + }, "id": 11, "options": { - "tooltip": { "mode": "multi" }, - "legend": { "displayMode": "list", "placement": "bottom" } + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } }, - "title": "Config Cache Hit Ratio", - "type": "timeseries", "targets": [ { "expr": "api_config_cache_hits / (api_config_cache_hits + api_config_cache_misses)", @@ -512,20 +741,32 @@ "legendFormat": "jwt hit ratio", "refId": "B" } - ] + ], + "title": "Config Cache Hit Ratio", + "type": "timeseries" }, { "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 24 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 24 + }, "id": 103, "title": "API Gateway Events", "type": "row" }, { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -535,33 +776,61 @@ "lineWidth": 1, "pointSize": 5, "showPoints": "auto", - "stacking": { "mode": "none" } + "stacking": { + "mode": "none" + } }, "unit": "ops" }, "overrides": [ { - "matcher": { "id": "byName", "options": "success" }, + "matcher": { + "id": "byName", + "options": "success" + }, "properties": [ - { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } ] }, { - "matcher": { "id": "byName", "options": "failure" }, + "matcher": { + "id": "byName", + "options": "failure" + }, "properties": [ - { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } ] } ] }, - "gridPos": { "h": 7, "w": 8, "x": 0, "y": 25 }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 25 + }, "id": 12, "options": { - "tooltip": { "mode": "multi" }, - "legend": { "displayMode": "table", "placement": "bottom" } + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } }, - "title": "Notify Events Rate", - "type": "timeseries", "targets": [ { "expr": "rate(api_notify_events_total[$interval])", @@ -578,45 +847,69 @@ "legendFormat": "failure", "refId": "C" } - ] + ], + "title": "Notify Events Rate", + "type": "timeseries" }, { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null } + { + "color": "green", + "value": null + } ] }, "unit": "short" }, "overrides": [] }, - "gridPos": { "h": 7, "w": 8, "x": 8, "y": 25 }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 25 + }, "id": 13, "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "background", - "graphMode": "area" + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } }, - "title": "Notify Listener Connected", - "type": "stat", "targets": [ { "expr": "api_notify_listener_connected", "legendFormat": "", "refId": "A" } - ] + ], + "title": "Notify Listener Connected", + "type": "stat" }, { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -626,33 +919,61 @@ "lineWidth": 1, "pointSize": 5, "showPoints": "auto", - "stacking": { "mode": "none" } + "stacking": { + "mode": "none" + } }, "unit": "short" }, "overrides": [ { - "matcher": { "id": "byName", "options": "errors" }, + "matcher": { + "id": "byName", + "options": "errors" + }, "properties": [ - { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } ] }, { - "matcher": { "id": "byName", "options": "reconnections" }, + "matcher": { + "id": "byName", + "options": "reconnections" + }, "properties": [ - { "id": "color", "value": { "fixedColor": "orange", "mode": "fixed" } } + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } ] } ] }, - "gridPos": { "h": 7, "w": 8, "x": 16, "y": 25 }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 25 + }, "id": 14, "options": { - "tooltip": { "mode": "multi" }, - "legend": { "displayMode": "list", "placement": "bottom" } + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } }, - "title": "Listener Errors & Reconnections", - "type": "timeseries", "targets": [ { "expr": "rate(api_notify_listener_errors[$interval])", @@ -664,20 +985,32 @@ "legendFormat": "reconnections", "refId": "B" } - ] + ], + "title": "Listener Errors & Reconnections", + "type": "timeseries" }, { "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 32 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 32 + }, "id": 104, "title": "Logs (Loki)", "type": "row" }, { - "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -687,58 +1020,111 @@ "lineWidth": 1, "pointSize": 5, "showPoints": "never", - "stacking": { "mode": "normal" } + "stacking": { + "mode": "normal" + } }, "unit": "short" }, "overrides": [ { - "matcher": { "id": "byName", "options": "error" }, + "matcher": { + "id": "byName", + "options": "error" + }, "properties": [ - { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } ] }, { - "matcher": { "id": "byName", "options": "warn" }, + "matcher": { + "id": "byName", + "options": "warn" + }, "properties": [ - { "id": "color", "value": { "fixedColor": "yellow", "mode": "fixed" } } + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } ] }, { - "matcher": { "id": "byName", "options": "info" }, + "matcher": { + "id": "byName", + "options": "info" + }, "properties": [ - { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } ] }, { - "matcher": { "id": "byName", "options": "debug" }, + "matcher": { + "id": "byName", + "options": "debug" + }, "properties": [ - { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } ] } ] }, - "gridPos": { "h": 8, "w": 8, "x": 0, "y": 33 }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 33 + }, "id": 15, "options": { - "tooltip": { "mode": "multi" }, - "legend": { "displayMode": "table", "placement": "bottom" } + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } }, - "title": "Log Volume by Level", - "type": "timeseries", "targets": [ { "expr": "sum by (level) (count_over_time({namespace=\"foxhunt\"} | json [$__interval]))", "legendFormat": "{{level}}", "refId": "A" } - ] + ], + "title": "Log Volume by Level", + "type": "timeseries" }, { - "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -748,142 +1134,224 @@ "lineWidth": 2, "pointSize": 5, "showPoints": "never", - "stacking": { "mode": "none" } + "stacking": { + "mode": "none" + } }, "unit": "short" }, "overrides": [ { - "matcher": { "id": "byFrameRefID", "options": "A" }, + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, "properties": [ - { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } ] } ] }, - "gridPos": { "h": 8, "w": 8, "x": 8, "y": 33 }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 33 + }, "id": 16, "options": { - "tooltip": { "mode": "multi" }, - "legend": { "displayMode": "list", "placement": "bottom" } + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } }, - "title": "Error Log Rate by App", - "type": "timeseries", "targets": [ { "expr": "sum by (app) (count_over_time({namespace=\"foxhunt\"} |= \"error\" [$__interval]))", "legendFormat": "{{app}}", "refId": "A" } - ] + ], + "title": "Error Log Rate by App", + "type": "timeseries" }, { - "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, - "gridPos": { "h": 8, "w": 8, "x": 16, "y": 33 }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 33 + }, "id": 17, "options": { - "showTime": true, - "showLabels": true, + "enableLogDetails": true, "showCommonLabels": false, - "wrapLogMessage": true, - "enableLogDetails": true + "showLabels": true, + "showTime": true, + "wrapLogMessage": true }, - "title": "Recent Errors", - "type": "logs", "targets": [ { "expr": "{namespace=\"foxhunt\"} |= \"error\"", "refId": "A" } - ] + ], + "title": "Recent Errors", + "type": "logs" }, { "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 41 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 41 + }, "id": 105, "title": "Traces (Tempo)", "type": "row" }, { - "datasource": { "type": "tempo", "uid": "tempo" }, - "gridPos": { "h": 8, "w": 24, "x": 0, "y": 42 }, + "datasource": { + "type": "${DS_TEMPO}", + "uid": "${DS_TEMPO}" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 42 + }, "id": 18, "options": {}, - "title": "Trace Search", - "type": "traces", "targets": [ { "queryType": "traceqlSearch", "refId": "A" } - ] + ], + "title": "Trace Search", + "type": "traces" }, { "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 50 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 50 + }, "id": 106, "title": "Prometheus Health", "type": "row" }, { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "color": { "mode": "fixed", "fixedColor": "text" }, + "color": { + "fixedColor": "text", + "mode": "fixed" + }, "custom": { "align": "auto", - "cellOptions": { "type": "color-text" }, + "cellOptions": { + "type": "color-text" + }, "inspect": false }, - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "red", "value": null }, - { "color": "green", "value": 1 } - ] - }, "mappings": [ { "options": { - "0": { "text": "DOWN", "color": "red" }, - "1": { "text": "UP", "color": "green" } + "0": { + "color": "red", + "text": "DOWN" + }, + "1": { + "color": "green", + "text": "UP" + } }, "type": "value" } - ] + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } }, "overrides": [ { - "matcher": { "id": "byName", "options": "Value" }, + "matcher": { + "id": "byName", + "options": "Value" + }, "properties": [ { "id": "custom.cellOptions", - "value": { "type": "color-background" } + "value": { + "type": "color-background" + } }, - { "id": "custom.width", "value": 80 } + { + "id": "custom.width", + "value": 80 + } ] } ] }, - "gridPos": { "h": 8, "w": 6, "x": 0, "y": 51 }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 51 + }, "id": 19, "options": { "showHeader": true, "sortBy": [ - { "desc": false, "displayName": "Value" } + { + "desc": false, + "displayName": "Value" + } ] }, - "title": "Scrape Targets Up/Down", - "type": "table", "targets": [ { "expr": "up", - "legendFormat": "{{job}}", - "refId": "A", + "format": "table", "instant": true, - "format": "table" + "legendFormat": "{{job}}", + "refId": "A" } ], + "title": "Scrape Targets Up/Down", "transformations": [ { "id": "organize", @@ -893,19 +1361,25 @@ "__name__": true }, "renameByName": { - "job": "Job", + "Value": "Status", "instance": "Instance", - "Value": "Status" + "job": "Job" } } } - ] + ], + "type": "table" }, { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -915,95 +1389,153 @@ "lineWidth": 1, "pointSize": 5, "showPoints": "auto", - "stacking": { "mode": "none" } + "stacking": { + "mode": "none" + } }, "unit": "s" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 6, "x": 6, "y": 51 }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 51 + }, "id": 20, "options": { - "tooltip": { "mode": "multi" }, - "legend": { "displayMode": "table", "placement": "bottom" } + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } }, - "title": "Scrape Duration by Job", - "type": "timeseries", "targets": [ { "expr": "scrape_duration_seconds", "legendFormat": "{{job}}", "refId": "A" } - ] + ], + "title": "Scrape Duration by Job", + "type": "timeseries" }, { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 50000 }, - { "color": "red", "value": 100000 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 50000 + }, + { + "color": "red", + "value": 100000 + } ] }, "unit": "short" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 6, "x": 12, "y": 51 }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 51 + }, "id": 21, "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "background", - "graphMode": "area" + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } }, - "title": "TSDB Head Series", - "type": "stat", "targets": [ { "expr": "prometheus_tsdb_head_series", "legendFormat": "", "refId": "A" } - ] + ], + "title": "TSDB Head Series", + "type": "stat" }, { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 1073741824 }, - { "color": "red", "value": 5368709120 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1073741824 + }, + { + "color": "red", + "value": 5368709120 + } ] }, "unit": "decbytes" }, "overrides": [] }, - "gridPos": { "h": 8, "w": 6, "x": 18, "y": 51 }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 51 + }, "id": 22, "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "background", - "graphMode": "area" + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } }, - "title": "TSDB Storage Bytes", - "type": "stat", "targets": [ { "expr": "prometheus_tsdb_storage_blocks_bytes", "legendFormat": "", "refId": "A" } - ] + ], + "title": "TSDB Storage Bytes", + "type": "stat" } ], "refresh": "15s", @@ -1030,11 +1562,31 @@ "label": "Interval", "name": "interval", "options": [ - { "selected": false, "text": "1m", "value": "1m" }, - { "selected": false, "text": "5m", "value": "5m" }, - { "selected": false, "text": "15m", "value": "15m" }, - { "selected": false, "text": "30m", "value": "30m" }, - { "selected": false, "text": "1h", "value": "1h" } + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "15m", + "value": "15m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } ], "query": "1m,5m,15m,30m,1h", "type": "interval" @@ -1047,7 +1599,23 @@ }, "timepicker": {}, "timezone": "browser", - "title": "Foxhunt Observability & API Gateway", + "title": "Observability & API Gateway", "uid": "foxhunt-observability", - "version": 1 + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "type": "datasource", + "pluginId": "prometheus" + }, + { + "name": "DS_LOKI", + "type": "datasource", + "pluginId": "loki" + }, + { + "name": "DS_TEMPO", + "type": "datasource", + "pluginId": "${DS_TEMPO}" + } + ] } diff --git a/infra/k8s/monitoring/dashboards/foxhunt-trading.json b/infra/k8s/monitoring/dashboards/foxhunt-trading.json index 38ef2abef..4acb6900f 100644 --- a/infra/k8s/monitoring/dashboards/foxhunt-trading.json +++ b/infra/k8s/monitoring/dashboards/foxhunt-trading.json @@ -1,1272 +1,4 @@ { - "id": null, - "uid": "foxhunt-trading", - "title": "Foxhunt Trading & ML Intelligence", - "description": "Trading overview, execution latency, gRPC pipeline health, data acquisition, and backtesting for the foxhunt HFT platform", - "tags": [ - "foxhunt", - "trading", - "ml" - ], - "timezone": "browser", - "schemaVersion": 39, - "version": 2, - "editable": true, - "refresh": "10s", - "time": { - "from": "now-1h", - "to": "now" - }, - "fiscalYearStartMonth": 0, - "liveNow": false, - "weekStart": "", - "templating": { - "list": [ - { - "name": "interval", - "type": "interval", - "query": "1m,5m,15m,30m,1h", - "current": { - "selected": true, - "text": "auto", - "value": "$__auto_interval_interval" - }, - "auto": true, - "auto_count": 30, - "auto_min": "10s" - } - ] - }, - "panels": [ - { - "id": 1, - "type": "row", - "title": "Trading Overview", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, - "collapsed": false, - "panels": [] - }, - { - "id": 2, - "type": "stat", - "title": "Portfolio Value", - "description": "Current portfolio value in USD from the trading agent", - "gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "red", "value": null }, - { "color": "yellow", "value": 10000 }, - { "color": "green", "value": 100000 } - ] - }, - "unit": "currencyUSD", - "color": { "mode": "thresholds" }, - "noValue": "N/A" - }, - "overrides": [] - }, - "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "area", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "expr": "trading_agent_portfolio_value_usd", - "instant": true, - "legendFormat": "portfolio" - } - ] - }, - { - "id": 3, - "type": "stat", - "title": "Assets Selected", - "description": "Number of assets currently selected by the trading agent", - "gridPos": { "h": 4, "w": 6, "x": 6, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "blue", "value": null }, - { "color": "green", "value": 1 }, - { "color": "yellow", "value": 10 } - ] - }, - "unit": "none", - "color": { "mode": "thresholds" }, - "noValue": "0" - }, - "overrides": [] - }, - "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "expr": "trading_agent_assets_selected", - "instant": true, - "legendFormat": "selected" - } - ] - }, - { - "id": 4, - "type": "stat", - "title": "Universe Instruments", - "description": "Total instruments in the trading universe", - "gridPos": { "h": 4, "w": 6, "x": 12, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "blue", "value": null }, - { "color": "green", "value": 1 } - ] - }, - "unit": "none", - "color": { "mode": "thresholds" }, - "noValue": "0" - }, - "overrides": [] - }, - "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "expr": "trading_agent_universe_instruments", - "instant": true, - "legendFormat": "instruments" - } - ] - }, - { - "id": 5, - "type": "stat", - "title": "Trading Service Uptime", - "description": "How long the trading service has been running", - "gridPos": { "h": 4, "w": 6, "x": 18, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "red", "value": null }, - { "color": "yellow", "value": 3600 }, - { "color": "green", "value": 86400 } - ] - }, - "unit": "s", - "color": { "mode": "thresholds" }, - "noValue": "Down" - }, - "overrides": [] - }, - "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "expr": "trading_service_uptime_seconds", - "instant": true, - "legendFormat": "uptime" - } - ] - }, - { - "id": 6, - "type": "timeseries", - "title": "Trading Measurements Rate", - "description": "Rate of trading measurements over time", - "gridPos": { "h": 6, "w": 12, "x": 0, "y": 5 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "fieldConfig": { - "defaults": { - "unit": "ops", - "color": { "mode": "palette-classic" }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 15, - "gradientMode": "scheme", - "showPoints": "never", - "spanNulls": false, - "axisCenteredZero": false, - "axisLabel": "", - "scaleDistribution": { "type": "linear" } - } - }, - "overrides": [] - }, - "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "list", "placement": "bottom", "calcs": ["lastNotNull", "mean"] } - }, - "targets": [ - { - "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "expr": "rate(trading_measurements_total[$__rate_interval])", - "legendFormat": "measurements/s" - } - ] - }, - { - "id": 7, - "type": "timeseries", - "title": "Portfolio Value Over Time", - "description": "Trading agent portfolio value trend", - "gridPos": { "h": 6, "w": 12, "x": 12, "y": 5 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "fieldConfig": { - "defaults": { - "unit": "currencyUSD", - "color": { "mode": "palette-classic" }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 10, - "gradientMode": "scheme", - "showPoints": "never", - "spanNulls": false, - "axisCenteredZero": false, - "axisLabel": "", - "scaleDistribution": { "type": "linear" } - } - }, - "overrides": [] - }, - "options": { - "tooltip": { "mode": "single", "sort": "none" }, - "legend": { "displayMode": "list", "placement": "bottom", "calcs": ["lastNotNull", "min", "max"] } - }, - "targets": [ - { - "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "expr": "trading_agent_portfolio_value_usd", - "legendFormat": "portfolio value" - } - ] - }, - { - "id": 10, - "type": "row", - "title": "Trading Latency", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 11 }, - "collapsed": false, - "panels": [] - }, - { - "id": 11, - "type": "timeseries", - "title": "Total Trading Latency (p50 / p95 / p99)", - "description": "End-to-end trading latency percentiles from trading_total_latency_seconds histogram", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "fieldConfig": { - "defaults": { - "unit": "s", - "color": { "mode": "palette-classic" }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 10, - "gradientMode": "none", - "showPoints": "auto", - "pointSize": 5, - "spanNulls": false, - "axisCenteredZero": false, - "axisLabel": "", - "scaleDistribution": { "type": "linear" } - } - }, - "overrides": [ - { - "matcher": { "id": "byName", "options": "p99" }, - "properties": [ - { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } - ] - }, - { - "matcher": { "id": "byName", "options": "p95" }, - "properties": [ - { "id": "color", "value": { "fixedColor": "yellow", "mode": "fixed" } } - ] - }, - { - "matcher": { "id": "byName", "options": "p50" }, - "properties": [ - { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } - ] - } - ] - }, - "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "mean", "max"] } - }, - "targets": [ - { - "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "expr": "histogram_quantile(0.5, rate(trading_total_latency_seconds_bucket[$__rate_interval]))", - "legendFormat": "p50" - }, - { - "refId": "B", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "expr": "histogram_quantile(0.95, rate(trading_total_latency_seconds_bucket[$__rate_interval]))", - "legendFormat": "p95" - }, - { - "refId": "C", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "expr": "histogram_quantile(0.99, rate(trading_total_latency_seconds_bucket[$__rate_interval]))", - "legendFormat": "p99" - } - ] - }, - { - "id": 12, - "type": "timeseries", - "title": "Order Processing Latency (p50 / p95 / p99)", - "description": "Order processing latency percentiles from trading_order_processing_seconds histogram", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "fieldConfig": { - "defaults": { - "unit": "s", - "color": { "mode": "palette-classic" }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 10, - "gradientMode": "none", - "showPoints": "auto", - "pointSize": 5, - "spanNulls": false, - "axisCenteredZero": false, - "axisLabel": "", - "scaleDistribution": { "type": "linear" } - } - }, - "overrides": [ - { - "matcher": { "id": "byName", "options": "p99" }, - "properties": [ - { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } - ] - }, - { - "matcher": { "id": "byName", "options": "p95" }, - "properties": [ - { "id": "color", "value": { "fixedColor": "yellow", "mode": "fixed" } } - ] - }, - { - "matcher": { "id": "byName", "options": "p50" }, - "properties": [ - { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } - ] - } - ] - }, - "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "mean", "max"] } - }, - "targets": [ - { - "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "expr": "histogram_quantile(0.5, rate(trading_order_processing_seconds_bucket[$__rate_interval]))", - "legendFormat": "p50" - }, - { - "refId": "B", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "expr": "histogram_quantile(0.95, rate(trading_order_processing_seconds_bucket[$__rate_interval]))", - "legendFormat": "p95" - }, - { - "refId": "C", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "expr": "histogram_quantile(0.99, rate(trading_order_processing_seconds_bucket[$__rate_interval]))", - "legendFormat": "p99" - } - ] - }, - { - "id": 13, - "type": "timeseries", - "title": "Risk Check Latency (p50 / p95 / p99)", - "description": "Risk check latency percentiles from trading_risk_check_seconds histogram", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 20 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "fieldConfig": { - "defaults": { - "unit": "s", - "color": { "mode": "palette-classic" }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 10, - "gradientMode": "none", - "showPoints": "auto", - "pointSize": 5, - "spanNulls": false, - "axisCenteredZero": false, - "axisLabel": "", - "scaleDistribution": { "type": "linear" } - } - }, - "overrides": [ - { - "matcher": { "id": "byName", "options": "p99" }, - "properties": [ - { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } - ] - }, - { - "matcher": { "id": "byName", "options": "p95" }, - "properties": [ - { "id": "color", "value": { "fixedColor": "yellow", "mode": "fixed" } } - ] - }, - { - "matcher": { "id": "byName", "options": "p50" }, - "properties": [ - { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } - ] - } - ] - }, - "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "mean", "max"] } - }, - "targets": [ - { - "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "expr": "histogram_quantile(0.5, rate(trading_risk_check_seconds_bucket[$__rate_interval]))", - "legendFormat": "p50" - }, - { - "refId": "B", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "expr": "histogram_quantile(0.95, rate(trading_risk_check_seconds_bucket[$__rate_interval]))", - "legendFormat": "p95" - }, - { - "refId": "C", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "expr": "histogram_quantile(0.99, rate(trading_risk_check_seconds_bucket[$__rate_interval]))", - "legendFormat": "p99" - } - ] - }, - { - "id": 14, - "type": "timeseries", - "title": "Market Data Latency (p50 / p95 / p99)", - "description": "Market data fetch latency percentiles from trading_market_data_seconds histogram", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 20 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "fieldConfig": { - "defaults": { - "unit": "s", - "color": { "mode": "palette-classic" }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 10, - "gradientMode": "none", - "showPoints": "auto", - "pointSize": 5, - "spanNulls": false, - "axisCenteredZero": false, - "axisLabel": "", - "scaleDistribution": { "type": "linear" } - } - }, - "overrides": [ - { - "matcher": { "id": "byName", "options": "p99" }, - "properties": [ - { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } - ] - }, - { - "matcher": { "id": "byName", "options": "p95" }, - "properties": [ - { "id": "color", "value": { "fixedColor": "yellow", "mode": "fixed" } } - ] - }, - { - "matcher": { "id": "byName", "options": "p50" }, - "properties": [ - { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } - ] - } - ] - }, - "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "mean", "max"] } - }, - "targets": [ - { - "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "expr": "histogram_quantile(0.5, rate(trading_market_data_seconds_bucket[$__rate_interval]))", - "legendFormat": "p50" - }, - { - "refId": "B", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "expr": "histogram_quantile(0.95, rate(trading_market_data_seconds_bucket[$__rate_interval]))", - "legendFormat": "p95" - }, - { - "refId": "C", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "expr": "histogram_quantile(0.99, rate(trading_market_data_seconds_bucket[$__rate_interval]))", - "legendFormat": "p99" - } - ] - }, - { - "id": 20, - "type": "row", - "title": "gRPC Trading Pipeline", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 28 }, - "collapsed": false, - "panels": [] - }, - { - "id": 21, - "type": "timeseries", - "title": "gRPC Request Rate by Service", - "description": "Rate of gRPC calls handled per service (trading-service, trading-agent-service, broker-gateway)", - "gridPos": { "h": 8, "w": 8, "x": 0, "y": 29 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "fieldConfig": { - "defaults": { - "unit": "reqps", - "color": { "mode": "palette-classic" }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 15, - "gradientMode": "none", - "showPoints": "never", - "spanNulls": false, - "stacking": { "mode": "none", "group": "A" }, - "axisCenteredZero": false, - "axisLabel": "", - "scaleDistribution": { "type": "linear" } - } - }, - "overrides": [] - }, - "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "mean"] } - }, - "targets": [ - { - "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "expr": "sum by (exported_service) (rate(grpc_server_handled_total{exported_service=~\"trading-agent-service|broker-gateway|backtesting-service|ml-training-service|data-acquisition-service\"}[$__rate_interval]))", - "legendFormat": "{{exported_service}}" - } - ] - }, - { - "id": 22, - "type": "timeseries", - "title": "gRPC Error Rate by Service", - "description": "Rate of non-OK gRPC responses across trading services", - "gridPos": { "h": 8, "w": 8, "x": 8, "y": 29 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "fieldConfig": { - "defaults": { - "unit": "reqps", - "color": { "mode": "palette-classic" }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 15, - "gradientMode": "none", - "showPoints": "never", - "spanNulls": false, - "axisCenteredZero": false, - "axisLabel": "", - "scaleDistribution": { "type": "linear" } - } - }, - "overrides": [] - }, - "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "sum"] } - }, - "targets": [ - { - "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "expr": "sum by (exported_service, grpc_code) (rate(grpc_server_handled_total{exported_service=~\"trading-agent-service|broker-gateway|backtesting-service|ml-training-service|data-acquisition-service\", grpc_code!=\"0\"}[$__rate_interval]))", - "legendFormat": "{{exported_service}} {{grpc_code}}" - } - ] - }, - { - "id": 23, - "type": "timeseries", - "title": "gRPC p99 Latency by Service", - "description": "99th percentile gRPC handling latency per trading service", - "gridPos": { "h": 8, "w": 8, "x": 16, "y": 29 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "fieldConfig": { - "defaults": { - "unit": "s", - "color": { "mode": "palette-classic" }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 10, - "gradientMode": "none", - "showPoints": "auto", - "pointSize": 5, - "spanNulls": false, - "axisCenteredZero": false, - "axisLabel": "", - "scaleDistribution": { "type": "linear" } - } - }, - "overrides": [] - }, - "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "mean", "max"] } - }, - "targets": [ - { - "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "expr": "histogram_quantile(0.99, sum by (exported_service, le) (rate(grpc_server_handling_seconds_bucket{exported_service=~\"trading-agent-service|broker-gateway|backtesting-service|ml-training-service|data-acquisition-service\"}[$__rate_interval])))", - "legendFormat": "{{exported_service}} p99" - } - ] - }, - { - "id": 24, - "type": "stat", - "title": "gRPC Total Requests Started", - "description": "Total gRPC requests started across all trading services", - "gridPos": { "h": 4, "w": 8, "x": 0, "y": 37 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "blue", "value": null }, - { "color": "green", "value": 1 } - ] - }, - "unit": "short", - "color": { "mode": "thresholds" }, - "noValue": "0" - }, - "overrides": [] - }, - "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "area", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "expr": "sum(grpc_server_started_total{exported_service=~\"trading-agent-service|broker-gateway|backtesting-service|ml-training-service|data-acquisition-service\"})", - "instant": true, - "legendFormat": "total started" - } - ] - }, - { - "id": 25, - "type": "stat", - "title": "gRPC Error Ratio", - "description": "Ratio of non-OK to total handled gRPC requests across trading services", - "gridPos": { "h": 4, "w": 8, "x": 8, "y": 37 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 0.01 }, - { "color": "red", "value": 0.05 } - ] - }, - "unit": "percentunit", - "color": { "mode": "thresholds" }, - "noValue": "0%", - "min": 0, - "max": 1 - }, - "overrides": [] - }, - "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "expr": "sum(rate(grpc_server_handled_total{exported_service=~\"trading-agent-service|broker-gateway|backtesting-service|ml-training-service|data-acquisition-service\", grpc_code!=\"0\"}[$__rate_interval])) / sum(rate(grpc_server_handled_total{exported_service=~\"trading-agent-service|broker-gateway|backtesting-service|ml-training-service|data-acquisition-service\"}[$__rate_interval]))", - "instant": true, - "legendFormat": "error ratio" - } - ] - }, - { - "id": 26, - "type": "stat", - "title": "Broker Gateway Requests", - "description": "Total handled requests on broker-gateway (highest volume service)", - "gridPos": { "h": 4, "w": 8, "x": 16, "y": 37 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "blue", "value": null }, - { "color": "green", "value": 100 } - ] - }, - "unit": "short", - "color": { "mode": "thresholds" }, - "noValue": "0" - }, - "overrides": [] - }, - "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "area", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "expr": "sum(grpc_server_handled_total{exported_service=\"broker-gateway\"})", - "instant": true, - "legendFormat": "broker-gateway total" - } - ] - }, - { - "id": 30, - "type": "row", - "title": "Data Acquisition", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 41 }, - "collapsed": false, - "panels": [] - }, - { - "id": 31, - "type": "stat", - "title": "Active Feeds", - "description": "Number of currently active data acquisition feeds", - "gridPos": { "h": 5, "w": 8, "x": 0, "y": 42 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "red", "value": null }, - { "color": "yellow", "value": 1 }, - { "color": "green", "value": 2 } - ] - }, - "unit": "none", - "color": { "mode": "thresholds" }, - "noValue": "0" - }, - "overrides": [] - }, - "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "area", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "expr": "foxhunt_data_acquisition_feeds_active", - "instant": true, - "legendFormat": "active feeds" - } - ] - }, - { - "id": 32, - "type": "timeseries", - "title": "Data Acquisition Error Rate", - "description": "Rate of data acquisition errors over time", - "gridPos": { "h": 5, "w": 8, "x": 8, "y": 42 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "fieldConfig": { - "defaults": { - "unit": "ops", - "color": { "mode": "fixed", "fixedColor": "red" }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 20, - "gradientMode": "none", - "showPoints": "never", - "spanNulls": false, - "axisCenteredZero": false, - "axisLabel": "", - "scaleDistribution": { "type": "linear" } - } - }, - "overrides": [] - }, - "options": { - "tooltip": { "mode": "single", "sort": "none" }, - "legend": { "displayMode": "list", "placement": "bottom", "calcs": ["lastNotNull", "mean"] } - }, - "targets": [ - { - "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "expr": "rate(foxhunt_data_acquisition_errors_total[$__rate_interval])", - "legendFormat": "errors/s" - } - ] - }, - { - "id": 33, - "type": "stat", - "title": "Data Acquisition Uptime", - "description": "How long the data acquisition service has been running", - "gridPos": { "h": 5, "w": 8, "x": 16, "y": 42 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "red", "value": null }, - { "color": "yellow", "value": 3600 }, - { "color": "green", "value": 86400 } - ] - }, - "unit": "s", - "color": { "mode": "thresholds" }, - "noValue": "Down" - }, - "overrides": [] - }, - "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "expr": "foxhunt_data_acquisition_uptime_seconds", - "instant": true, - "legendFormat": "uptime" - } - ] - }, - { - "id": 40, - "type": "row", - "title": "Backtesting", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 47 }, - "collapsed": false, - "panels": [] - }, - { - "id": 41, - "type": "stat", - "title": "Backtests Completed", - "description": "Total backtests completed", - "gridPos": { "h": 5, "w": 6, "x": 0, "y": 48 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "blue", "value": null }, - { "color": "green", "value": 1 } - ] - }, - "unit": "short", - "color": { "mode": "thresholds" }, - "noValue": "0" - }, - "overrides": [] - }, - "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "area", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "expr": "backtesting_backtests_completed_total", - "instant": true, - "legendFormat": "completed" - } - ] - }, - { - "id": 42, - "type": "stat", - "title": "Backtests Started", - "description": "Total backtests started", - "gridPos": { "h": 5, "w": 6, "x": 6, "y": 48 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "blue", "value": null }, - { "color": "green", "value": 1 } - ] - }, - "unit": "short", - "color": { "mode": "thresholds" }, - "noValue": "0" - }, - "overrides": [] - }, - "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "area", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "expr": "backtesting_backtests_started_total", - "instant": true, - "legendFormat": "started" - } - ] - }, - { - "id": 43, - "type": "gauge", - "title": "Backtest Completion Ratio", - "description": "Ratio of completed to started backtests (1.0 = all finished)", - "gridPos": { "h": 5, "w": 6, "x": 12, "y": 48 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "red", "value": null }, - { "color": "yellow", "value": 0.5 }, - { "color": "green", "value": 0.9 } - ] - }, - "unit": "percentunit", - "color": { "mode": "thresholds" }, - "min": 0, - "max": 1, - "noValue": "N/A" - }, - "overrides": [] - }, - "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "orientation": "auto" - }, - "targets": [ - { - "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "expr": "backtesting_backtests_completed_total / backtesting_backtests_started_total", - "instant": true, - "legendFormat": "completion ratio" - } - ] - }, - { - "id": 44, - "type": "timeseries", - "title": "Backtest Error Rate", - "description": "Rate of backtesting errors over time", - "gridPos": { "h": 5, "w": 6, "x": 18, "y": 48 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "fieldConfig": { - "defaults": { - "unit": "ops", - "color": { "mode": "fixed", "fixedColor": "red" }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 20, - "gradientMode": "none", - "showPoints": "never", - "spanNulls": false, - "axisCenteredZero": false, - "axisLabel": "", - "scaleDistribution": { "type": "linear" } - } - }, - "overrides": [] - }, - "options": { - "tooltip": { "mode": "single", "sort": "none" }, - "legend": { "displayMode": "list", "placement": "bottom", "calcs": ["lastNotNull", "mean"] } - }, - "targets": [ - { - "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "expr": "rate(backtesting_errors_total[$__rate_interval])", - "legendFormat": "errors/s" - } - ] - }, - { - "id": 45, - "type": "timeseries", - "title": "Backtests Started vs Completed Over Time", - "description": "Rate of backtests started and completed, showing active backtest throughput", - "gridPos": { "h": 6, "w": 12, "x": 0, "y": 53 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "fieldConfig": { - "defaults": { - "unit": "ops", - "color": { "mode": "palette-classic" }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 15, - "gradientMode": "none", - "showPoints": "never", - "spanNulls": false, - "axisCenteredZero": false, - "axisLabel": "", - "scaleDistribution": { "type": "linear" } - } - }, - "overrides": [ - { - "matcher": { "id": "byName", "options": "started" }, - "properties": [ - { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } - ] - }, - { - "matcher": { "id": "byName", "options": "completed" }, - "properties": [ - { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } - ] - } - ] - }, - "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "mean"] } - }, - "targets": [ - { - "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "expr": "rate(backtesting_backtests_started_total[$__rate_interval])", - "legendFormat": "started" - }, - { - "refId": "B", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "expr": "rate(backtesting_backtests_completed_total[$__rate_interval])", - "legendFormat": "completed" - } - ] - }, - { - "id": 46, - "type": "stat", - "title": "Active Backtests (est.)", - "description": "Estimated active backtests: started minus completed", - "gridPos": { "h": 6, "w": 6, "x": 12, "y": 53 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 5 }, - { "color": "red", "value": 10 } - ] - }, - "unit": "none", - "color": { "mode": "thresholds" }, - "noValue": "0" - }, - "overrides": [] - }, - "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "area", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "expr": "backtesting_backtests_started_total - backtesting_backtests_completed_total", - "instant": true, - "legendFormat": "active" - } - ] - }, - { - "id": 47, - "type": "stat", - "title": "Backtesting Uptime", - "description": "How long the backtesting service has been running", - "gridPos": { "h": 6, "w": 6, "x": 18, "y": 53 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "red", "value": null }, - { "color": "yellow", "value": 3600 }, - { "color": "green", "value": 86400 } - ] - }, - "unit": "s", - "color": { "mode": "thresholds" }, - "noValue": "Down" - }, - "overrides": [] - }, - "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "expr": "backtesting_service_uptime_seconds", - "instant": true, - "legendFormat": "uptime" - } - ] - } - ], "annotations": { "list": [ { @@ -1282,5 +14,2154 @@ "type": "dashboard" } ] - } + }, + "description": "Trading overview, execution latency, gRPC pipeline health, data acquisition, and backtesting for the foxhunt HFT platform", + "editable": true, + "fiscalYearStartMonth": 0, + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "Trading Overview", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Current portfolio value in USD from the trading agent", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "noValue": "N/A", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 10000 + }, + { + "color": "green", + "value": 100000 + } + ] + }, + "unit": "currencyUSD" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "trading_agent_portfolio_value_usd", + "instant": true, + "legendFormat": "portfolio", + "refId": "A" + } + ], + "title": "Portfolio Value", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of assets currently selected by the trading agent", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + }, + { + "color": "green", + "value": 1 + }, + { + "color": "yellow", + "value": 10 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "trading_agent_assets_selected", + "instant": true, + "legendFormat": "selected", + "refId": "A" + } + ], + "title": "Assets Selected", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Total instruments in the trading universe", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 4, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "trading_agent_universe_instruments", + "instant": true, + "legendFormat": "instruments", + "refId": "A" + } + ], + "title": "Universe Instruments", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "How long the trading service has been running", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "noValue": "Down", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 3600 + }, + { + "color": "green", + "value": 86400 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 5, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "trading_service_uptime_seconds", + "instant": true, + "legendFormat": "uptime", + "refId": "A" + } + ], + "title": "Trading Service Uptime", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Rate of trading measurements over time", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "", + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "scheme", + "lineInterpolation": "smooth", + "lineWidth": 2, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 5 + }, + "id": 6, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "mean" + ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(trading_measurements_total[$__rate_interval])", + "legendFormat": "measurements/s", + "refId": "A" + } + ], + "title": "Trading Measurements Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Trading agent portfolio value trend", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "", + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "scheme", + "lineInterpolation": "smooth", + "lineWidth": 2, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false + }, + "unit": "currencyUSD" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 5 + }, + "id": 7, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "min", + "max" + ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "trading_agent_portfolio_value_usd", + "legendFormat": "portfolio value", + "refId": "A" + } + ], + "title": "Portfolio Value Over Time", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 10, + "panels": [], + "title": "Trading Latency", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "End-to-end trading latency percentiles from trading_total_latency_seconds histogram", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "", + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "p99" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p95" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p50" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 11, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.5, rate(trading_total_latency_seconds_bucket[$__rate_interval]))", + "legendFormat": "p50", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.95, rate(trading_total_latency_seconds_bucket[$__rate_interval]))", + "legendFormat": "p95", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.99, rate(trading_total_latency_seconds_bucket[$__rate_interval]))", + "legendFormat": "p99", + "refId": "C" + } + ], + "title": "Total Trading Latency (p50 / p95 / p99)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Order processing latency percentiles from trading_order_processing_seconds histogram", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "", + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "p99" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p95" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p50" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 12, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.5, rate(trading_order_processing_seconds_bucket[$__rate_interval]))", + "legendFormat": "p50", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.95, rate(trading_order_processing_seconds_bucket[$__rate_interval]))", + "legendFormat": "p95", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.99, rate(trading_order_processing_seconds_bucket[$__rate_interval]))", + "legendFormat": "p99", + "refId": "C" + } + ], + "title": "Order Processing Latency (p50 / p95 / p99)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Risk check latency percentiles from trading_risk_check_seconds histogram", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "", + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "p99" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p95" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p50" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 20 + }, + "id": 13, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.5, rate(trading_risk_check_seconds_bucket[$__rate_interval]))", + "legendFormat": "p50", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.95, rate(trading_risk_check_seconds_bucket[$__rate_interval]))", + "legendFormat": "p95", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.99, rate(trading_risk_check_seconds_bucket[$__rate_interval]))", + "legendFormat": "p99", + "refId": "C" + } + ], + "title": "Risk Check Latency (p50 / p95 / p99)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Market data fetch latency percentiles from trading_market_data_seconds histogram", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "", + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "p99" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p95" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p50" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 20 + }, + "id": 14, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.5, rate(trading_market_data_seconds_bucket[$__rate_interval]))", + "legendFormat": "p50", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.95, rate(trading_market_data_seconds_bucket[$__rate_interval]))", + "legendFormat": "p95", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.99, rate(trading_market_data_seconds_bucket[$__rate_interval]))", + "legendFormat": "p99", + "refId": "C" + } + ], + "title": "Market Data Latency (p50 / p95 / p99)", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 28 + }, + "id": 20, + "panels": [], + "title": "gRPC Trading Pipeline", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Rate of gRPC calls handled per service (trading-service, trading-agent-service, broker-gateway)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "", + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "none", + "lineInterpolation": "smooth", + "lineWidth": 2, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 29 + }, + "id": 21, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "mean" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (exported_service) (rate(grpc_server_handled_total{exported_service=~\"trading-agent-service|broker-gateway|backtesting-service|ml-training-service|data-acquisition-service\"}[$__rate_interval]))", + "legendFormat": "{{exported_service}}", + "refId": "A" + } + ], + "title": "gRPC Request Rate by Service", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Rate of non-OK gRPC responses across trading services", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "", + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "none", + "lineInterpolation": "smooth", + "lineWidth": 2, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 29 + }, + "id": 22, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "sum" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (exported_service, grpc_code) (rate(grpc_server_handled_total{exported_service=~\"trading-agent-service|broker-gateway|backtesting-service|ml-training-service|data-acquisition-service\", grpc_code!=\"0\"}[$__rate_interval]))", + "legendFormat": "{{exported_service}} {{grpc_code}}", + "refId": "A" + } + ], + "title": "gRPC Error Rate by Service", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "99th percentile gRPC handling latency per trading service", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "", + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 29 + }, + "id": 23, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.99, sum by (exported_service, le) (rate(grpc_server_handling_seconds_bucket{exported_service=~\"trading-agent-service|broker-gateway|backtesting-service|ml-training-service|data-acquisition-service\"}[$__rate_interval])))", + "legendFormat": "{{exported_service}} p99", + "refId": "A" + } + ], + "title": "gRPC p99 Latency by Service", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Total gRPC requests started across all trading services", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 0, + "y": 37 + }, + "id": 24, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(grpc_server_started_total{exported_service=~\"trading-agent-service|broker-gateway|backtesting-service|ml-training-service|data-acquisition-service\"})", + "instant": true, + "legendFormat": "total started", + "refId": "A" + } + ], + "title": "gRPC Total Requests Started", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Ratio of non-OK to total handled gRPC requests across trading services", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "max": 1, + "min": 0, + "noValue": "0%", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.01 + }, + { + "color": "red", + "value": 0.05 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 8, + "y": 37 + }, + "id": 25, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(grpc_server_handled_total{exported_service=~\"trading-agent-service|broker-gateway|backtesting-service|ml-training-service|data-acquisition-service\", grpc_code!=\"0\"}[$__rate_interval])) / sum(rate(grpc_server_handled_total{exported_service=~\"trading-agent-service|broker-gateway|backtesting-service|ml-training-service|data-acquisition-service\"}[$__rate_interval]))", + "instant": true, + "legendFormat": "error ratio", + "refId": "A" + } + ], + "title": "gRPC Error Ratio", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Total handled requests on broker-gateway (highest volume service)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + }, + { + "color": "green", + "value": 100 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 16, + "y": 37 + }, + "id": 26, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(grpc_server_handled_total{exported_service=\"broker-gateway\"})", + "instant": true, + "legendFormat": "broker-gateway total", + "refId": "A" + } + ], + "title": "Broker Gateway Requests", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 41 + }, + "id": 30, + "panels": [], + "title": "Data Acquisition", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of currently active data acquisition feeds", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "green", + "value": 2 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 42 + }, + "id": 31, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "foxhunt_data_acquisition_feeds_active", + "instant": true, + "legendFormat": "active feeds", + "refId": "A" + } + ], + "title": "Active Feeds", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Rate of data acquisition errors over time", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "red", + "mode": "fixed" + }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "", + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "lineInterpolation": "smooth", + "lineWidth": 2, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 42 + }, + "id": 32, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "mean" + ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(foxhunt_data_acquisition_errors_total[$__rate_interval])", + "legendFormat": "errors/s", + "refId": "A" + } + ], + "title": "Data Acquisition Error Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "How long the data acquisition service has been running", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "noValue": "Down", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 3600 + }, + { + "color": "green", + "value": 86400 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 42 + }, + "id": 33, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "foxhunt_data_acquisition_uptime_seconds", + "instant": true, + "legendFormat": "uptime", + "refId": "A" + } + ], + "title": "Data Acquisition Uptime", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 47 + }, + "id": 40, + "panels": [], + "title": "Backtesting", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Total backtests completed", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 48 + }, + "id": 41, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "backtesting_backtests_completed_total", + "instant": true, + "legendFormat": "completed", + "refId": "A" + } + ], + "title": "Backtests Completed", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Total backtests started", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 48 + }, + "id": 42, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "backtesting_backtests_started_total", + "instant": true, + "legendFormat": "started", + "refId": "A" + } + ], + "title": "Backtests Started", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Ratio of completed to started backtests (1.0 = all finished)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "max": 1, + "min": 0, + "noValue": "N/A", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 0.5 + }, + { + "color": "green", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 48 + }, + "id": 43, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "backtesting_backtests_completed_total / backtesting_backtests_started_total", + "instant": true, + "legendFormat": "completion ratio", + "refId": "A" + } + ], + "title": "Backtest Completion Ratio", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Rate of backtesting errors over time", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "red", + "mode": "fixed" + }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "", + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "lineInterpolation": "smooth", + "lineWidth": 2, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 48 + }, + "id": 44, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "mean" + ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(backtesting_errors_total[$__rate_interval])", + "legendFormat": "errors/s", + "refId": "A" + } + ], + "title": "Backtest Error Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Rate of backtests started and completed, showing active backtest throughput", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "", + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "none", + "lineInterpolation": "smooth", + "lineWidth": 2, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false + }, + "unit": "ops" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "started" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "completed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 53 + }, + "id": 45, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "mean" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(backtesting_backtests_started_total[$__rate_interval])", + "legendFormat": "started", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(backtesting_backtests_completed_total[$__rate_interval])", + "legendFormat": "completed", + "refId": "B" + } + ], + "title": "Backtests Started vs Completed Over Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Estimated active backtests: started minus completed", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 5 + }, + { + "color": "red", + "value": 10 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 53 + }, + "id": 46, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "backtesting_backtests_started_total - backtesting_backtests_completed_total", + "instant": true, + "legendFormat": "active", + "refId": "A" + } + ], + "title": "Active Backtests (est.)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "How long the backtesting service has been running", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "noValue": "Down", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 3600 + }, + { + "color": "green", + "value": 86400 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 53 + }, + "id": 47, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "backtesting_service_uptime_seconds", + "instant": true, + "legendFormat": "uptime", + "refId": "A" + } + ], + "title": "Backtesting Uptime", + "type": "stat" + } + ], + "refresh": "10s", + "schemaVersion": 39, + "tags": [ + "foxhunt", + "trading", + "ml" + ], + "templating": { + "list": [ + { + "auto": true, + "auto_count": 30, + "auto_min": "10s", + "current": { + "selected": true, + "text": "auto", + "value": "$__auto_interval_interval" + }, + "name": "interval", + "query": "1m,5m,15m,30m,1h", + "type": "interval" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "browser", + "title": "Trading & ML Intelligence", + "uid": "foxhunt-trading", + "weekStart": "", + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "type": "datasource", + "pluginId": "prometheus" + }, + { + "name": "DS_LOKI", + "type": "datasource", + "pluginId": "loki" + }, + { + "name": "DS_TEMPO", + "type": "datasource", + "pluginId": "${DS_TEMPO}" + } + ] } diff --git a/infra/k8s/monitoring/dashboards/foxhunt-training-deep-dive.json b/infra/k8s/monitoring/dashboards/foxhunt-training-deep-dive.json index 6985018be..23425d785 100644 --- a/infra/k8s/monitoring/dashboards/foxhunt-training-deep-dive.json +++ b/infra/k8s/monitoring/dashboards/foxhunt-training-deep-dive.json @@ -1,95 +1,4 @@ { - "id": null, - "uid": "foxhunt-training-deep-dive", - "title": "Foxhunt Training Deep-Dive", - "description": "Unified training pipeline monitor \u2014 target a running pod to see training curves, GPU health, eval metrics, hyperopt progress, container resources, and live logs in one place.", - "tags": [ - "foxhunt", - "training", - "gpu", - "loki", - "deep-dive" - ], - "timezone": "browser", - "refresh": "10s", - "time": { - "from": "now-3h", - "to": "now" - }, - "fiscalYearStartMonth": 0, - "liveNow": true, - "schemaVersion": 39, - "version": 1, - "editable": true, - "__inputs": [ - { - "name": "DS_PROMETHEUS", - "type": "datasource", - "pluginId": "prometheus" - }, - { - "name": "DS_LOKI", - "type": "datasource", - "pluginId": "loki" - } - ], - "templating": { - "list": [ - { - "name": "pod", - "type": "query", - "label": "Pod", - "query": "label_values({namespace=\"foxhunt\", pod=~\"(ci-pipeline|compile-and-train|compile-and-deploy|train|hyperopt|gpu-test|evaluate|build-training).*\"}, pod)", - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "refresh": 2, - "includeAll": true, - "multi": true, - "regex": "", - "sort": 2, - "current": {} - }, - { - "name": "prom_job", - "type": "query", - "label": "Prometheus Job", - "query": "label_values(up{namespace=\"foxhunt\"}, job)", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "refresh": 2, - "includeAll": true, - "multi": false, - "regex": "/(train|hyperopt|evaluate|pushgateway|training-pods|dcgm)/", - "sort": 1, - "current": {} - }, - { - "name": "level", - "type": "custom", - "label": "Log Level", - "query": "ERROR,WARN,INFO,DEBUG,TRACE", - "includeAll": true, - "multi": true, - "current": { - "text": "All", - "value": "$__all" - } - }, - { - "name": "search", - "type": "textbox", - "label": "Search", - "current": { - "text": "", - "value": "" - } - } - ] - }, "annotations": { "list": [ { @@ -98,9 +7,9 @@ "uid": "${DS_LOKI}" }, "enable": true, - "name": "Errors", "expr": "{namespace=\"foxhunt\", pod=~\"$pod\", level=\"ERROR\"}", "iconColor": "red", + "name": "Errors", "tagKeys": "level", "titleFormat": "Error" }, @@ -110,39 +19,64 @@ "uid": "${DS_PROMETHEUS}" }, "enable": true, - "name": "Checkpoints", "expr": "changes(foxhunt_training_checkpoint_saves_total{pod=~\"$pod\", namespace=\"foxhunt\"}[1m]) > 0", "iconColor": "blue", + "name": "Checkpoints", "titleFormat": "Checkpoint Saved" } ] }, - "panels": [ + "description": "Unified training pipeline monitor \u2014 target a running pod to see training curves, GPU health, eval metrics, hyperopt progress, container resources, and live logs in one place.", + "editable": true, + "fiscalYearStartMonth": 0, + "links": [ { - "type": "row", - "title": "Job Overview", - "collapsed": false, - "id": 1, - "gridPos": { - "x": 0, - "y": 0, - "w": 24, - "h": 1 - }, - "panels": [] + "title": "CI Pipeline Logs", + "type": "link", + "url": "/d/foxhunt-ci-logs/foxhunt-ci-pipeline-logs" + }, + { + "title": "Training Logs", + "type": "link", + "url": "/d/foxhunt-training-logs/foxhunt-training-pipeline-logs" + }, + { + "title": "Training & CI/CD", + "type": "link", + "url": "/d/foxhunt-training/foxhunt-training-and-ci-cd" + }, + { + "title": "Operations Cockpit", + "type": "link", + "url": "/d/foxhunt-cockpit/foxhunt-operations-cockpit" + } + ], + "liveNow": true, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "Job Overview", + "type": "row" }, { - "type": "stat", - "title": "Epoch", - "description": "", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "id": 2, + "description": "", "fieldConfig": { "defaults": { - "unit": "none", + "color": { + "mode": "thresholds" + }, "thresholds": { "mode": "absolute", "steps": [ @@ -152,20 +86,25 @@ } ] }, - "color": { - "mode": "thresholds" - } + "unit": "none" }, "overrides": [] }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 1 + }, + "id": 2, "options": { + "colorMode": "background", + "graphMode": "area", "reduceOptions": { "calcs": [ "lastNotNull" ] }, - "colorMode": "background", - "graphMode": "area", "textMode": "auto" }, "targets": [ @@ -175,25 +114,20 @@ "refId": "A" } ], - "gridPos": { - "x": 0, - "y": 1, - "w": 3, - "h": 4 - } + "title": "Epoch", + "type": "stat" }, { - "type": "stat", - "title": "Step", - "description": "", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "id": 3, + "description": "", "fieldConfig": { "defaults": { - "unit": "none", + "color": { + "mode": "thresholds" + }, "thresholds": { "mode": "absolute", "steps": [ @@ -203,20 +137,25 @@ } ] }, - "color": { - "mode": "thresholds" - } + "unit": "none" }, "overrides": [] }, + "gridPos": { + "h": 4, + "w": 3, + "x": 3, + "y": 1 + }, + "id": 3, "options": { + "colorMode": "background", + "graphMode": "area", "reduceOptions": { "calcs": [ "lastNotNull" ] }, - "colorMode": "background", - "graphMode": "area", "textMode": "auto" }, "targets": [ @@ -226,25 +165,20 @@ "refId": "A" } ], - "gridPos": { - "x": 3, - "y": 1, - "w": 3, - "h": 4 - } + "title": "Step", + "type": "stat" }, { - "type": "stat", - "title": "Loss", - "description": "", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "id": 4, + "description": "", "fieldConfig": { "defaults": { - "unit": "none", + "color": { + "mode": "thresholds" + }, "thresholds": { "mode": "absolute", "steps": [ @@ -258,24 +192,29 @@ }, { "color": "red", - "value": 1.0 + "value": 1 } ] }, - "color": { - "mode": "thresholds" - } + "unit": "none" }, "overrides": [] }, + "gridPos": { + "h": 4, + "w": 3, + "x": 6, + "y": 1 + }, + "id": 4, "options": { + "colorMode": "background", + "graphMode": "area", "reduceOptions": { "calcs": [ "lastNotNull" ] }, - "colorMode": "background", - "graphMode": "area", "textMode": "auto" }, "targets": [ @@ -285,25 +224,20 @@ "refId": "A" } ], - "gridPos": { - "x": 6, - "y": 1, - "w": 3, - "h": 4 - } + "title": "Loss", + "type": "stat" }, { - "type": "stat", - "title": "Val Loss", - "description": "", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "id": 5, + "description": "", "fieldConfig": { "defaults": { - "unit": "none", + "color": { + "mode": "thresholds" + }, "thresholds": { "mode": "absolute", "steps": [ @@ -317,24 +251,29 @@ }, { "color": "red", - "value": 1.0 + "value": 1 } ] }, - "color": { - "mode": "thresholds" - } + "unit": "none" }, "overrides": [] }, + "gridPos": { + "h": 4, + "w": 3, + "x": 9, + "y": 1 + }, + "id": 5, "options": { + "colorMode": "background", + "graphMode": "area", "reduceOptions": { "calcs": [ "lastNotNull" ] }, - "colorMode": "background", - "graphMode": "area", "textMode": "auto" }, "targets": [ @@ -344,25 +283,20 @@ "refId": "A" } ], - "gridPos": { - "x": 9, - "y": 1, - "w": 3, - "h": 4 - } + "title": "Val Loss", + "type": "stat" }, { - "type": "stat", - "title": "Sharpe", - "description": "", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "id": 6, + "description": "", "fieldConfig": { "defaults": { - "unit": "none", + "color": { + "mode": "thresholds" + }, "thresholds": { "mode": "absolute", "steps": [ @@ -376,24 +310,29 @@ }, { "color": "green", - "value": 1.0 + "value": 1 } ] }, - "color": { - "mode": "thresholds" - } + "unit": "none" }, "overrides": [] }, + "gridPos": { + "h": 4, + "w": 3, + "x": 12, + "y": 1 + }, + "id": 6, "options": { + "colorMode": "background", + "graphMode": "area", "reduceOptions": { "calcs": [ "lastNotNull" ] }, - "colorMode": "background", - "graphMode": "area", "textMode": "auto" }, "targets": [ @@ -403,25 +342,20 @@ "refId": "A" } ], - "gridPos": { - "x": 12, - "y": 1, - "w": 3, - "h": 4 - } + "title": "Sharpe", + "type": "stat" }, { - "type": "stat", - "title": "MaxDD", - "description": "", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "id": 7, + "description": "", "fieldConfig": { "defaults": { - "unit": "percentunit", + "color": { + "mode": "thresholds" + }, "thresholds": { "mode": "absolute", "steps": [ @@ -439,20 +373,25 @@ } ] }, - "color": { - "mode": "thresholds" - } + "unit": "percentunit" }, "overrides": [] }, + "gridPos": { + "h": 4, + "w": 3, + "x": 15, + "y": 1 + }, + "id": 7, "options": { + "colorMode": "background", + "graphMode": "area", "reduceOptions": { "calcs": [ "lastNotNull" ] }, - "colorMode": "background", - "graphMode": "area", "textMode": "auto" }, "targets": [ @@ -462,26 +401,21 @@ "refId": "A" } ], - "gridPos": { - "x": 15, - "y": 1, - "w": 3, - "h": 4 - } + "title": "MaxDD", + "type": "stat" }, { - "type": "gauge", - "title": "GPU Util", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "id": 8, "fieldConfig": { "defaults": { - "unit": "percent", - "min": 0, + "color": { + "mode": "thresholds" + }, "max": 100, + "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -499,12 +433,17 @@ } ] }, - "color": { - "mode": "thresholds" - } + "unit": "percent" }, "overrides": [] }, + "gridPos": { + "h": 4, + "w": 3, + "x": 18, + "y": 1 + }, + "id": 8, "options": { "reduceOptions": { "calcs": [ @@ -521,26 +460,21 @@ "refId": "A" } ], - "gridPos": { - "x": 18, - "y": 1, - "w": 3, - "h": 4 - } + "title": "GPU Util", + "type": "gauge" }, { - "type": "gauge", - "title": "VRAM Used", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "id": 9, "fieldConfig": { "defaults": { - "unit": "percent", - "min": 0, + "color": { + "mode": "thresholds" + }, "max": 100, + "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -558,12 +492,17 @@ } ] }, - "color": { - "mode": "thresholds" - } + "unit": "percent" }, "overrides": [] }, + "gridPos": { + "h": 4, + "w": 3, + "x": 21, + "y": 1 + }, + "id": 9, "options": { "reduceOptions": { "calcs": [ @@ -580,38 +519,33 @@ "refId": "A" } ], - "gridPos": { - "x": 21, - "y": 1, - "w": 3, - "h": 4 - } + "title": "VRAM Used", + "type": "gauge" }, { - "type": "row", - "title": "CI Pipeline & Argo Workflows", "collapsed": false, - "id": 10, "gridPos": { - "x": 0, - "y": 5, + "h": 1, "w": 24, - "h": 1 + "x": 0, + "y": 5 }, - "panels": [] + "id": 10, + "panels": [], + "title": "CI Pipeline & Argo Workflows", + "type": "row" }, { - "type": "stat", - "title": "Active Workflows", - "description": "Currently running Argo workflows", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "id": 11, + "description": "Currently running Argo workflows", "fieldConfig": { "defaults": { - "unit": "none", + "color": { + "mode": "thresholds" + }, "thresholds": { "mode": "absolute", "steps": [ @@ -621,20 +555,25 @@ } ] }, - "color": { - "mode": "thresholds" - } + "unit": "none" }, "overrides": [] }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 6 + }, + "id": 11, "options": { + "colorMode": "background", + "graphMode": "area", "reduceOptions": { "calcs": [ "lastNotNull" ] }, - "colorMode": "background", - "graphMode": "area", "textMode": "auto" }, "targets": [ @@ -644,25 +583,20 @@ "refId": "A" } ], - "gridPos": { - "x": 0, - "y": 6, - "w": 3, - "h": 4 - } + "title": "Active Workflows", + "type": "stat" }, { - "type": "stat", - "title": "Pending Pods", - "description": "", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "id": 12, + "description": "", "fieldConfig": { "defaults": { - "unit": "none", + "color": { + "mode": "thresholds" + }, "thresholds": { "mode": "absolute", "steps": [ @@ -680,20 +614,25 @@ } ] }, - "color": { - "mode": "thresholds" - } + "unit": "none" }, "overrides": [] }, + "gridPos": { + "h": 4, + "w": 3, + "x": 3, + "y": 6 + }, + "id": 12, "options": { + "colorMode": "background", + "graphMode": "area", "reduceOptions": { "calcs": [ "lastNotNull" ] }, - "colorMode": "background", - "graphMode": "area", "textMode": "auto" }, "targets": [ @@ -703,25 +642,20 @@ "refId": "A" } ], - "gridPos": { - "x": 3, - "y": 6, - "w": 3, - "h": 4 - } + "title": "Pending Pods", + "type": "stat" }, { - "type": "stat", - "title": "Total Completed", - "description": "", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "id": 13, + "description": "", "fieldConfig": { "defaults": { - "unit": "none", + "color": { + "mode": "thresholds" + }, "thresholds": { "mode": "absolute", "steps": [ @@ -731,20 +665,25 @@ } ] }, - "color": { - "mode": "thresholds" - } + "unit": "none" }, "overrides": [] }, + "gridPos": { + "h": 4, + "w": 3, + "x": 6, + "y": 6 + }, + "id": 13, "options": { + "colorMode": "background", + "graphMode": "area", "reduceOptions": { "calcs": [ "lastNotNull" ] }, - "colorMode": "background", - "graphMode": "area", "textMode": "auto" }, "targets": [ @@ -754,25 +693,20 @@ "refId": "A" } ], - "gridPos": { - "x": 6, - "y": 6, - "w": 3, - "h": 4 - } + "title": "Total Completed", + "type": "stat" }, { - "type": "stat", - "title": "Errors", - "description": "", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "id": 14, + "description": "", "fieldConfig": { "defaults": { - "unit": "none", + "color": { + "mode": "thresholds" + }, "thresholds": { "mode": "absolute", "steps": [ @@ -786,20 +720,25 @@ } ] }, - "color": { - "mode": "thresholds" - } + "unit": "none" }, "overrides": [] }, + "gridPos": { + "h": 4, + "w": 3, + "x": 9, + "y": 6 + }, + "id": 14, "options": { + "colorMode": "background", + "graphMode": "area", "reduceOptions": { "calcs": [ "lastNotNull" ] }, - "colorMode": "background", - "graphMode": "area", "textMode": "auto" }, "targets": [ @@ -809,36 +748,36 @@ "refId": "A" } ], - "gridPos": { - "x": 9, - "y": 6, - "w": 3, - "h": 4 - } + "title": "Errors", + "type": "stat" }, { - "type": "timeseries", - "title": "Workflow Duration", - "description": "Average workflow operation duration", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "id": 15, + "description": "Average workflow operation duration", "fieldConfig": { "defaults": { - "unit": "s", "custom": { "fillOpacity": 0, + "lineWidth": 1, + "spanNulls": true, "stacking": { "mode": "none" - }, - "lineWidth": 1, - "spanNulls": true - } + } + }, + "unit": "s" }, "overrides": [] }, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 6 + }, + "id": 15, "options": { "legend": { "displayMode": "list", @@ -856,36 +795,36 @@ "refId": "A" } ], - "gridPos": { - "x": 12, - "y": 6, - "w": 6, - "h": 7 - } + "title": "Workflow Duration", + "type": "timeseries" }, { - "type": "timeseries", - "title": "K8s API Request Rate", - "description": "Argo controller K8s API calls \u2014 high rates may indicate thrashing", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "id": 16, + "description": "Argo controller K8s API calls \u2014 high rates may indicate thrashing", "fieldConfig": { "defaults": { - "unit": "none", "custom": { "fillOpacity": 0, + "lineWidth": 1, + "spanNulls": true, "stacking": { "mode": "none" - }, - "lineWidth": 1, - "spanNulls": true - } + } + }, + "unit": "none" }, "overrides": [] }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 6 + }, + "id": 16, "options": { "legend": { "displayMode": "list", @@ -903,36 +842,36 @@ "refId": "A" } ], - "gridPos": { - "x": 18, - "y": 6, - "w": 6, - "h": 7 - } + "title": "K8s API Request Rate", + "type": "timeseries" }, { - "type": "timeseries", - "title": "CI Pod CPU", - "description": "CPU usage of selected CI/training pods", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "id": 17, + "description": "CPU usage of selected CI/training pods", "fieldConfig": { "defaults": { - "unit": "short", "custom": { "fillOpacity": 0, + "lineWidth": 1, + "spanNulls": true, "stacking": { "mode": "none" - }, - "lineWidth": 1, - "spanNulls": true - } + } + }, + "unit": "short" }, "overrides": [] }, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 10 + }, + "id": 17, "options": { "legend": { "displayMode": "list", @@ -950,36 +889,36 @@ "refId": "A" } ], - "gridPos": { - "x": 0, - "y": 10, - "w": 6, - "h": 7 - } + "title": "CI Pod CPU", + "type": "timeseries" }, { - "type": "timeseries", - "title": "CI Pod Memory", - "description": "RSS memory of selected CI/training pods", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "id": 18, + "description": "RSS memory of selected CI/training pods", "fieldConfig": { "defaults": { - "unit": "bytes", "custom": { "fillOpacity": 0, + "lineWidth": 1, + "spanNulls": true, "stacking": { "mode": "none" - }, - "lineWidth": 1, - "spanNulls": true - } + } + }, + "unit": "bytes" }, "overrides": [] }, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 10 + }, + "id": 18, "options": { "legend": { "displayMode": "list", @@ -997,44 +936,44 @@ "refId": "A" } ], - "gridPos": { - "x": 6, - "y": 10, - "w": 6, - "h": 7 - } + "title": "CI Pod Memory", + "type": "timeseries" }, { - "type": "row", - "title": "CI Logs", "collapsed": false, - "id": 19, "gridPos": { - "x": 0, - "y": 17, + "h": 1, "w": 24, - "h": 1 + "x": 0, + "y": 17 }, - "panels": [] + "id": 19, + "panels": [], + "title": "CI Logs", + "type": "row" }, { - "type": "logs", - "title": "CI Full Log Stream", - "description": "All logs from selected CI/training pods", "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "description": "All logs from selected CI/training pods", + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 18 + }, "id": 20, "options": { - "showTime": true, - "showLabels": false, - "showCommonLabels": false, - "wrapLogMessage": true, - "prettifyLogMessage": false, + "dedupStrategy": "none", "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": true, "sortOrder": "Descending", - "dedupStrategy": "none" + "wrapLogMessage": true }, "targets": [ { @@ -1042,31 +981,31 @@ "refId": "A" } ], - "gridPos": { - "x": 0, - "y": 18, - "w": 12, - "h": 9 - } + "title": "CI Full Log Stream", + "type": "logs" }, { - "type": "logs", - "title": "Compile & Test Results", - "description": "Rust compile output, test results, clippy warnings", "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "description": "Rust compile output, test results, clippy warnings", + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 18 + }, "id": 21, "options": { - "showTime": true, - "showLabels": false, - "showCommonLabels": false, - "wrapLogMessage": true, - "prettifyLogMessage": false, + "dedupStrategy": "none", "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": true, "sortOrder": "Descending", - "dedupStrategy": "none" + "wrapLogMessage": true }, "targets": [ { @@ -1074,31 +1013,31 @@ "refId": "A" } ], - "gridPos": { - "x": 12, - "y": 18, - "w": 12, - "h": 9 - } + "title": "Compile & Test Results", + "type": "logs" }, { - "type": "logs", - "title": "Build Artifacts & Deploy", - "description": "S3 binary uploads, kubectl deploys, image builds", "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "description": "S3 binary uploads, kubectl deploys, image builds", + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 27 + }, "id": 22, "options": { - "showTime": true, - "showLabels": false, - "showCommonLabels": false, - "wrapLogMessage": true, - "prettifyLogMessage": false, + "dedupStrategy": "none", "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": true, "sortOrder": "Descending", - "dedupStrategy": "none" + "wrapLogMessage": true }, "targets": [ { @@ -1106,31 +1045,31 @@ "refId": "A" } ], - "gridPos": { - "x": 0, - "y": 27, - "w": 12, - "h": 9 - } + "title": "Build Artifacts & Deploy", + "type": "logs" }, { - "type": "logs", - "title": "Redis Sidecar & Data", - "description": "Data fetching, redis cache, dbn file operations", "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "description": "Data fetching, redis cache, dbn file operations", + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 27 + }, "id": 23, "options": { - "showTime": true, - "showLabels": false, - "showCommonLabels": false, - "wrapLogMessage": true, - "prettifyLogMessage": false, + "dedupStrategy": "none", "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": true, "sortOrder": "Descending", - "dedupStrategy": "none" + "wrapLogMessage": true }, "targets": [ { @@ -1138,49 +1077,49 @@ "refId": "A" } ], - "gridPos": { - "x": 12, - "y": 27, - "w": 12, - "h": 9 - } + "title": "Redis Sidecar & Data", + "type": "logs" }, { - "type": "row", - "title": "Training Curves", "collapsed": false, - "id": 24, "gridPos": { - "x": 0, - "y": 36, + "h": 1, "w": 24, - "h": 1 + "x": 0, + "y": 36 }, - "panels": [] + "id": 24, + "panels": [], + "title": "Training Curves", + "type": "row" }, { - "type": "timeseries", - "title": "Loss & Validation Loss", - "description": "Training and validation loss over time", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "id": 25, + "description": "Training and validation loss over time", "fieldConfig": { "defaults": { - "unit": "none", "custom": { "fillOpacity": 0, + "lineWidth": 1, + "spanNulls": true, "stacking": { "mode": "none" - }, - "lineWidth": 1, - "spanNulls": true - } + } + }, + "unit": "none" }, "overrides": [] }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 37 + }, + "id": 25, "options": { "legend": { "displayMode": "list", @@ -1203,36 +1142,36 @@ "refId": "B" } ], - "gridPos": { - "x": 0, - "y": 37, - "w": 8, - "h": 7 - } + "title": "Loss & Validation Loss", + "type": "timeseries" }, { - "type": "timeseries", - "title": "Learning Rate", - "description": "Learning rate schedule", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "id": 26, + "description": "Learning rate schedule", "fieldConfig": { "defaults": { - "unit": "none", "custom": { "fillOpacity": 0, + "lineWidth": 1, + "spanNulls": true, "stacking": { "mode": "none" - }, - "lineWidth": 1, - "spanNulls": true - } + } + }, + "unit": "none" }, "overrides": [] }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 37 + }, + "id": 26, "options": { "legend": { "displayMode": "list", @@ -1250,36 +1189,36 @@ "refId": "A" } ], - "gridPos": { - "x": 8, - "y": 37, - "w": 8, - "h": 7 - } + "title": "Learning Rate", + "type": "timeseries" }, { - "type": "timeseries", - "title": "Gradient Norm", - "description": "Gradient L2 norm \u2014 spikes indicate instability", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "id": 27, + "description": "Gradient L2 norm \u2014 spikes indicate instability", "fieldConfig": { "defaults": { - "unit": "none", "custom": { "fillOpacity": 0, + "lineWidth": 1, + "spanNulls": true, "stacking": { "mode": "none" - }, - "lineWidth": 1, - "spanNulls": true - } + } + }, + "unit": "none" }, "overrides": [] }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 37 + }, + "id": 27, "options": { "legend": { "displayMode": "list", @@ -1297,49 +1236,49 @@ "refId": "A" } ], - "gridPos": { - "x": 16, - "y": 37, - "w": 8, - "h": 7 - } + "title": "Gradient Norm", + "type": "timeseries" }, { - "type": "row", - "title": "Trading Performance", "collapsed": false, - "id": 28, "gridPos": { - "x": 0, - "y": 44, + "h": 1, "w": 24, - "h": 1 + "x": 0, + "y": 44 }, - "panels": [] + "id": 28, + "panels": [], + "title": "Trading Performance", + "type": "row" }, { - "type": "timeseries", - "title": "Sharpe / Sortino / MaxDD", - "description": "Risk-adjusted return metrics per epoch", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "id": 29, + "description": "Risk-adjusted return metrics per epoch", "fieldConfig": { "defaults": { - "unit": "none", "custom": { "fillOpacity": 0, + "lineWidth": 1, + "spanNulls": true, "stacking": { "mode": "none" - }, - "lineWidth": 1, - "spanNulls": true - } + } + }, + "unit": "none" }, "overrides": [] }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 45 + }, + "id": 29, "options": { "legend": { "displayMode": "list", @@ -1367,36 +1306,36 @@ "refId": "C" } ], - "gridPos": { - "x": 0, - "y": 45, - "w": 8, - "h": 7 - } + "title": "Sharpe / Sortino / MaxDD", + "type": "timeseries" }, { - "type": "timeseries", - "title": "Win Rate & Profit Factor", - "description": "", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "id": 30, + "description": "", "fieldConfig": { "defaults": { - "unit": "none", "custom": { "fillOpacity": 0, + "lineWidth": 1, + "spanNulls": true, "stacking": { "mode": "none" - }, - "lineWidth": 1, - "spanNulls": true - } + } + }, + "unit": "none" }, "overrides": [] }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 45 + }, + "id": 30, "options": { "legend": { "displayMode": "list", @@ -1419,36 +1358,36 @@ "refId": "B" } ], - "gridPos": { - "x": 8, - "y": 45, - "w": 8, - "h": 7 - } + "title": "Win Rate & Profit Factor", + "type": "timeseries" }, { - "type": "timeseries", - "title": "Total Return & Avg Return", - "description": "", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "id": 31, + "description": "", "fieldConfig": { "defaults": { - "unit": "none", "custom": { "fillOpacity": 0, + "lineWidth": 1, + "spanNulls": true, "stacking": { "mode": "none" - }, - "lineWidth": 1, - "spanNulls": true - } + } + }, + "unit": "none" }, "overrides": [] }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 45 + }, + "id": 31, "options": { "legend": { "displayMode": "list", @@ -1471,38 +1410,38 @@ "refId": "B" } ], - "gridPos": { - "x": 16, - "y": 45, - "w": 8, - "h": 7 - } + "title": "Total Return & Avg Return", + "type": "timeseries" }, { - "type": "timeseries", - "title": "Action Distribution", - "description": "Action distribution per epoch \u2014 watch for collapse to single action", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "id": 32, + "description": "Action distribution per epoch \u2014 watch for collapse to single action", "fieldConfig": { "defaults": { - "unit": "percentunit", "custom": { "fillOpacity": 30, + "lineWidth": 1, + "spanNulls": true, "stacking": { "mode": "normal" - }, - "lineWidth": 1, - "spanNulls": true + } }, + "max": 1, "min": 0, - "max": 1 + "unit": "percentunit" }, "overrides": [] }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 52 + }, + "id": 32, "options": { "legend": { "displayMode": "list", @@ -1530,36 +1469,36 @@ "refId": "C" } ], - "gridPos": { - "x": 0, - "y": 52, - "w": 8, - "h": 7 - } + "title": "Action Distribution", + "type": "timeseries" }, { - "type": "timeseries", - "title": "Action Entropy & Diversity", - "description": "Higher entropy = more exploration. Collapse \u2192 policy degeneration", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "id": 33, + "description": "Higher entropy = more exploration. Collapse \u2192 policy degeneration", "fieldConfig": { "defaults": { - "unit": "none", "custom": { "fillOpacity": 0, + "lineWidth": 1, + "spanNulls": true, "stacking": { "mode": "none" - }, - "lineWidth": 1, - "spanNulls": true - } + } + }, + "unit": "none" }, "overrides": [] }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 52 + }, + "id": 33, "options": { "legend": { "displayMode": "list", @@ -1582,36 +1521,36 @@ "refId": "B" } ], - "gridPos": { - "x": 8, - "y": 52, - "w": 8, - "h": 7 - } + "title": "Action Entropy & Diversity", + "type": "timeseries" }, { - "type": "timeseries", - "title": "Total Trades", - "description": "", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "id": 34, + "description": "", "fieldConfig": { "defaults": { - "unit": "none", "custom": { "fillOpacity": 0, + "lineWidth": 1, + "spanNulls": true, "stacking": { "mode": "none" - }, - "lineWidth": 1, - "spanNulls": true - } + } + }, + "unit": "none" }, "overrides": [] }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 52 + }, + "id": 34, "options": { "legend": { "displayMode": "list", @@ -1629,51 +1568,51 @@ "refId": "A" } ], - "gridPos": { - "x": 16, - "y": 52, - "w": 8, - "h": 7 - } + "title": "Total Trades", + "type": "timeseries" }, { - "type": "row", - "title": "Evaluation Metrics", "collapsed": false, - "id": 35, "gridPos": { - "x": 0, - "y": 59, + "h": 1, "w": 24, - "h": 1 + "x": 0, + "y": 59 }, - "panels": [] + "id": 35, + "panels": [], + "title": "Evaluation Metrics", + "type": "row" }, { - "type": "timeseries", - "title": "Eval Accuracy", - "description": "", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "id": 36, + "description": "", "fieldConfig": { "defaults": { - "unit": "percentunit", "custom": { "fillOpacity": 0, + "lineWidth": 1, + "spanNulls": true, "stacking": { "mode": "none" - }, - "lineWidth": 1, - "spanNulls": true + } }, + "max": 1, "min": 0, - "max": 1 + "unit": "percentunit" }, "overrides": [] }, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 60 + }, + "id": 36, "options": { "legend": { "displayMode": "list", @@ -1691,38 +1630,38 @@ "refId": "A" } ], - "gridPos": { - "x": 0, - "y": 60, - "w": 6, - "h": 7 - } + "title": "Eval Accuracy", + "type": "timeseries" }, { - "type": "timeseries", - "title": "Precision / Recall / F1", - "description": "Classification quality of directional predictions", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "id": 37, + "description": "Classification quality of directional predictions", "fieldConfig": { "defaults": { - "unit": "percentunit", "custom": { "fillOpacity": 0, + "lineWidth": 1, + "spanNulls": true, "stacking": { "mode": "none" - }, - "lineWidth": 1, - "spanNulls": true + } }, + "max": 1, "min": 0, - "max": 1 + "unit": "percentunit" }, "overrides": [] }, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 60 + }, + "id": 37, "options": { "legend": { "displayMode": "list", @@ -1750,36 +1689,36 @@ "refId": "C" } ], - "gridPos": { - "x": 6, - "y": 60, - "w": 6, - "h": 7 - } + "title": "Precision / Recall / F1", + "type": "timeseries" }, { - "type": "timeseries", - "title": "Q-Values (DQN)", - "description": "DQN Q-value estimates \u2014 divergence from max indicates overestimation", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "id": 38, + "description": "DQN Q-value estimates \u2014 divergence from max indicates overestimation", "fieldConfig": { "defaults": { - "unit": "none", "custom": { "fillOpacity": 0, + "lineWidth": 1, + "spanNulls": true, "stacking": { "mode": "none" - }, - "lineWidth": 1, - "spanNulls": true - } + } + }, + "unit": "none" }, "overrides": [] }, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 60 + }, + "id": 38, "options": { "legend": { "displayMode": "list", @@ -1802,36 +1741,36 @@ "refId": "B" } ], - "gridPos": { - "x": 12, - "y": 60, - "w": 6, - "h": 7 - } + "title": "Q-Values (DQN)", + "type": "timeseries" }, { - "type": "timeseries", - "title": "Q-Overestimation & Replay Buffer", - "description": "Ratio > 1 = Q-values overestimate true returns", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "id": 39, + "description": "Ratio > 1 = Q-values overestimate true returns", "fieldConfig": { "defaults": { - "unit": "none", "custom": { "fillOpacity": 0, + "lineWidth": 1, + "spanNulls": true, "stacking": { "mode": "none" - }, - "lineWidth": 1, - "spanNulls": true - } + } + }, + "unit": "none" }, "overrides": [] }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 60 + }, + "id": 39, "options": { "legend": { "displayMode": "list", @@ -1849,51 +1788,51 @@ "refId": "A" } ], - "gridPos": { - "x": 18, - "y": 60, - "w": 6, - "h": 7 - } + "title": "Q-Overestimation & Replay Buffer", + "type": "timeseries" }, { - "type": "row", - "title": "GPU & Hardware", "collapsed": false, - "id": 40, "gridPos": { - "x": 0, - "y": 67, + "h": 1, "w": 24, - "h": 1 + "x": 0, + "y": 67 }, - "panels": [] + "id": 40, + "panels": [], + "title": "GPU & Hardware", + "type": "row" }, { - "type": "timeseries", - "title": "GPU Utilization", - "description": "Overall GPU and graphics engine utilization", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "id": 41, + "description": "Overall GPU and graphics engine utilization", "fieldConfig": { "defaults": { - "unit": "percent", "custom": { "fillOpacity": 10, + "lineWidth": 1, + "spanNulls": true, "stacking": { "mode": "none" - }, - "lineWidth": 1, - "spanNulls": true + } }, + "max": 100, "min": 0, - "max": 100 + "unit": "percent" }, "overrides": [] }, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 68 + }, + "id": 41, "options": { "legend": { "displayMode": "list", @@ -1916,38 +1855,38 @@ "refId": "B" } ], - "gridPos": { - "x": 0, - "y": 68, - "w": 6, - "h": 7 - } + "title": "GPU Utilization", + "type": "timeseries" }, { - "type": "timeseries", - "title": "Tensor Core Activity", - "description": "Tensor core and DRAM bandwidth utilization \u2014 key H100 saturation indicators", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "id": 42, + "description": "Tensor core and DRAM bandwidth utilization \u2014 key H100 saturation indicators", "fieldConfig": { "defaults": { - "unit": "percent", "custom": { "fillOpacity": 10, + "lineWidth": 1, + "spanNulls": true, "stacking": { "mode": "none" - }, - "lineWidth": 1, - "spanNulls": true + } }, + "max": 100, "min": 0, - "max": 100 + "unit": "percent" }, "overrides": [] }, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 68 + }, + "id": 42, "options": { "legend": { "displayMode": "list", @@ -1970,36 +1909,36 @@ "refId": "B" } ], - "gridPos": { - "x": 6, - "y": 68, - "w": 6, - "h": 7 - } + "title": "Tensor Core Activity", + "type": "timeseries" }, { - "type": "timeseries", - "title": "VRAM (Framebuffer)", - "description": "GPU framebuffer memory usage", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "id": 43, + "description": "GPU framebuffer memory usage", "fieldConfig": { "defaults": { - "unit": "decmbytes", "custom": { "fillOpacity": 20, + "lineWidth": 1, + "spanNulls": true, "stacking": { "mode": "normal" - }, - "lineWidth": 1, - "spanNulls": true - } + } + }, + "unit": "decmbytes" }, "overrides": [] }, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 68 + }, + "id": 43, "options": { "legend": { "displayMode": "list", @@ -2022,36 +1961,36 @@ "refId": "B" } ], - "gridPos": { - "x": 12, - "y": 68, - "w": 6, - "h": 7 - } + "title": "VRAM (Framebuffer)", + "type": "timeseries" }, { - "type": "timeseries", - "title": "GPU Temperature", - "description": "Thermal throttling starts at ~83\u00b0C on H100", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "id": 44, + "description": "Thermal throttling starts at ~83\u00b0C on H100", "fieldConfig": { "defaults": { - "unit": "celsius", "custom": { "fillOpacity": 0, + "lineWidth": 1, + "spanNulls": true, "stacking": { "mode": "none" - }, - "lineWidth": 1, - "spanNulls": true - } + } + }, + "unit": "celsius" }, "overrides": [] }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 68 + }, + "id": 44, "options": { "legend": { "displayMode": "list", @@ -2074,36 +2013,36 @@ "refId": "B" } ], - "gridPos": { - "x": 18, - "y": 68, - "w": 6, - "h": 7 - } + "title": "GPU Temperature", + "type": "timeseries" }, { - "type": "timeseries", - "title": "Power Usage", - "description": "GPU power draw \u2014 H100 TDP is 700W", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "id": 45, + "description": "GPU power draw \u2014 H100 TDP is 700W", "fieldConfig": { "defaults": { - "unit": "watt", "custom": { "fillOpacity": 0, + "lineWidth": 1, + "spanNulls": true, "stacking": { "mode": "none" - }, - "lineWidth": 1, - "spanNulls": true - } + } + }, + "unit": "watt" }, "overrides": [] }, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 75 + }, + "id": 45, "options": { "legend": { "displayMode": "list", @@ -2121,36 +2060,36 @@ "refId": "A" } ], - "gridPos": { - "x": 0, - "y": 75, - "w": 6, - "h": 7 - } + "title": "Power Usage", + "type": "timeseries" }, { - "type": "timeseries", - "title": "PCIe Throughput", - "description": "PCIe bandwidth \u2014 high RX during data loading, high TX during checkpoint saves", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "id": 46, + "description": "PCIe bandwidth \u2014 high RX during data loading, high TX during checkpoint saves", "fieldConfig": { "defaults": { - "unit": "Bps", "custom": { "fillOpacity": 0, + "lineWidth": 1, + "spanNulls": true, "stacking": { "mode": "none" - }, - "lineWidth": 1, - "spanNulls": true - } + } + }, + "unit": "Bps" }, "overrides": [] }, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 75 + }, + "id": 46, "options": { "legend": { "displayMode": "list", @@ -2173,36 +2112,36 @@ "refId": "B" } ], - "gridPos": { - "x": 6, - "y": 75, - "w": 6, - "h": 7 - } + "title": "PCIe Throughput", + "type": "timeseries" }, { - "type": "timeseries", - "title": "SM & Memory Clock", - "description": "Clock throttling indicates thermal or power issues", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "id": 47, + "description": "Clock throttling indicates thermal or power issues", "fieldConfig": { "defaults": { - "unit": "clockhertz", "custom": { "fillOpacity": 0, + "lineWidth": 1, + "spanNulls": true, "stacking": { "mode": "none" - }, - "lineWidth": 1, - "spanNulls": true - } + } + }, + "unit": "clockhertz" }, "overrides": [] }, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 75 + }, + "id": 47, "options": { "legend": { "displayMode": "list", @@ -2225,38 +2164,38 @@ "refId": "B" } ], - "gridPos": { - "x": 12, - "y": 75, - "w": 6, - "h": 7 - } + "title": "SM & Memory Clock", + "type": "timeseries" }, { - "type": "timeseries", - "title": "Memory Copy Utilization", - "description": "", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "id": 48, + "description": "", "fieldConfig": { "defaults": { - "unit": "percent", "custom": { "fillOpacity": 10, + "lineWidth": 1, + "spanNulls": true, "stacking": { "mode": "none" - }, - "lineWidth": 1, - "spanNulls": true + } }, + "max": 100, "min": 0, - "max": 100 + "unit": "percent" }, "overrides": [] }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 75 + }, + "id": 48, "options": { "legend": { "displayMode": "list", @@ -2274,49 +2213,49 @@ "refId": "A" } ], - "gridPos": { - "x": 18, - "y": 75, - "w": 6, - "h": 7 - } + "title": "Memory Copy Utilization", + "type": "timeseries" }, { - "type": "row", - "title": "Throughput & Performance", "collapsed": false, - "id": 49, "gridPos": { - "x": 0, - "y": 82, + "h": 1, "w": 24, - "h": 1 + "x": 0, + "y": 82 }, - "panels": [] + "id": 49, + "panels": [], + "title": "Throughput & Performance", + "type": "row" }, { - "type": "timeseries", - "title": "Batches / Second", - "description": "Training throughput \u2014 higher is better", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "id": 50, + "description": "Training throughput \u2014 higher is better", "fieldConfig": { "defaults": { - "unit": "none", "custom": { "fillOpacity": 0, + "lineWidth": 1, + "spanNulls": true, "stacking": { "mode": "none" - }, - "lineWidth": 1, - "spanNulls": true - } + } + }, + "unit": "none" }, "overrides": [] }, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 83 + }, + "id": 50, "options": { "legend": { "displayMode": "list", @@ -2334,36 +2273,36 @@ "refId": "A" } ], - "gridPos": { - "x": 0, - "y": 83, - "w": 6, - "h": 7 - } + "title": "Batches / Second", + "type": "timeseries" }, { - "type": "timeseries", - "title": "Epoch Duration", - "description": "Wall-clock time per epoch", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "id": 51, + "description": "Wall-clock time per epoch", "fieldConfig": { "defaults": { - "unit": "s", "custom": { "fillOpacity": 0, + "lineWidth": 1, + "spanNulls": true, "stacking": { "mode": "none" - }, - "lineWidth": 1, - "spanNulls": true - } + } + }, + "unit": "s" }, "overrides": [] }, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 83 + }, + "id": 51, "options": { "legend": { "displayMode": "list", @@ -2381,36 +2320,36 @@ "refId": "A" } ], - "gridPos": { - "x": 6, - "y": 83, - "w": 6, - "h": 7 - } + "title": "Epoch Duration", + "type": "timeseries" }, { - "type": "timeseries", - "title": "Iteration Time", - "description": "Per-step training iteration time", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "id": 52, + "description": "Per-step training iteration time", "fieldConfig": { "defaults": { - "unit": "s", "custom": { "fillOpacity": 0, + "lineWidth": 1, + "spanNulls": true, "stacking": { "mode": "none" - }, - "lineWidth": 1, - "spanNulls": true - } + } + }, + "unit": "s" }, "overrides": [] }, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 83 + }, + "id": 52, "options": { "legend": { "displayMode": "list", @@ -2428,74 +2367,74 @@ "refId": "A" } ], - "gridPos": { - "x": 12, - "y": 83, - "w": 6, - "h": 7 - } + "title": "Iteration Time", + "type": "timeseries" }, { - "type": "heatmap", - "title": "Data Load Latency", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "id": 53, "fieldConfig": { "defaults": {}, "overrides": [] }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 83 + }, + "id": 53, "options": { "calculate": false, + "cellGap": 1, + "color": { + "mode": "scheme", + "scheme": "Oranges" + }, "yAxis": { "unit": "s" - }, - "color": { - "scheme": "Oranges", - "mode": "scheme" - }, - "cellGap": 1 + } }, "targets": [ { "expr": "rate(foxhunt_training_data_load_seconds_bucket{pod=~\"$pod\", namespace=\"foxhunt\"}[$__rate_interval])", + "format": "heatmap", "legendFormat": "{{le}}", - "refId": "A", - "format": "heatmap" + "refId": "A" } ], - "gridPos": { - "x": 18, - "y": 83, - "w": 6, - "h": 7 - } + "title": "Data Load Latency", + "type": "heatmap" }, { - "type": "timeseries", - "title": "Checkpoint Saves", - "description": "Checkpoint save count over time", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "id": 54, + "description": "Checkpoint save count over time", "fieldConfig": { "defaults": { - "unit": "none", "custom": { "fillOpacity": 0, + "lineWidth": 1, + "spanNulls": true, "stacking": { "mode": "none" - }, - "lineWidth": 1, - "spanNulls": true - } + } + }, + "unit": "none" }, "overrides": [] }, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 90 + }, + "id": 54, "options": { "legend": { "displayMode": "list", @@ -2513,36 +2452,36 @@ "refId": "A" } ], - "gridPos": { - "x": 0, - "y": 90, - "w": 6, - "h": 7 - } + "title": "Checkpoint Saves", + "type": "timeseries" }, { - "type": "timeseries", - "title": "Checkpoint Size & Duration", - "description": "Checkpoint file size", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "id": 55, + "description": "Checkpoint file size", "fieldConfig": { "defaults": { - "unit": "bytes", "custom": { "fillOpacity": 0, + "lineWidth": 1, + "spanNulls": true, "stacking": { "mode": "none" - }, - "lineWidth": 1, - "spanNulls": true - } + } + }, + "unit": "bytes" }, "overrides": [] }, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 90 + }, + "id": 55, "options": { "legend": { "displayMode": "list", @@ -2560,36 +2499,36 @@ "refId": "A" } ], - "gridPos": { - "x": 6, - "y": 90, - "w": 6, - "h": 7 - } + "title": "Checkpoint Size & Duration", + "type": "timeseries" }, { - "type": "timeseries", - "title": "Replay Buffer Size", - "description": "DQN/PER replay buffer population", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "id": 56, + "description": "DQN/PER replay buffer population", "fieldConfig": { "defaults": { - "unit": "none", "custom": { "fillOpacity": 0, + "lineWidth": 1, + "spanNulls": true, "stacking": { "mode": "none" - }, - "lineWidth": 1, - "spanNulls": true - } + } + }, + "unit": "none" }, "overrides": [] }, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 90 + }, + "id": 56, "options": { "legend": { "displayMode": "list", @@ -2607,36 +2546,36 @@ "refId": "A" } ], - "gridPos": { - "x": 12, - "y": 90, - "w": 6, - "h": 7 - } + "title": "Replay Buffer Size", + "type": "timeseries" }, { - "type": "timeseries", - "title": "Active Workers", - "description": "Parallel training workers", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "id": 57, + "description": "Parallel training workers", "fieldConfig": { "defaults": { - "unit": "none", "custom": { "fillOpacity": 0, + "lineWidth": 1, + "spanNulls": true, "stacking": { "mode": "none" - }, - "lineWidth": 1, - "spanNulls": true - } + } + }, + "unit": "none" }, "overrides": [] }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 90 + }, + "id": 57, "options": { "legend": { "displayMode": "list", @@ -2654,38 +2593,33 @@ "refId": "A" } ], - "gridPos": { - "x": 18, - "y": 90, - "w": 6, - "h": 7 - } + "title": "Active Workers", + "type": "timeseries" }, { - "type": "row", - "title": "Hyperopt Trials", "collapsed": false, - "id": 58, "gridPos": { - "x": 0, - "y": 97, + "h": 1, "w": 24, - "h": 1 + "x": 0, + "y": 97 }, - "panels": [] + "id": 58, + "panels": [], + "title": "Hyperopt Trials", + "type": "row" }, { - "type": "stat", - "title": "Current Trial", - "description": "", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "id": 59, + "description": "", "fieldConfig": { "defaults": { - "unit": "none", + "color": { + "mode": "thresholds" + }, "thresholds": { "mode": "absolute", "steps": [ @@ -2695,20 +2629,25 @@ } ] }, - "color": { - "mode": "thresholds" - } + "unit": "none" }, "overrides": [] }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 98 + }, + "id": 59, "options": { + "colorMode": "background", + "graphMode": "area", "reduceOptions": { "calcs": [ "lastNotNull" ] }, - "colorMode": "background", - "graphMode": "area", "textMode": "auto" }, "targets": [ @@ -2718,25 +2657,20 @@ "refId": "A" } ], - "gridPos": { - "x": 0, - "y": 98, - "w": 3, - "h": 4 - } + "title": "Current Trial", + "type": "stat" }, { - "type": "stat", - "title": "Total Trials", - "description": "", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "id": 60, + "description": "", "fieldConfig": { "defaults": { - "unit": "none", + "color": { + "mode": "thresholds" + }, "thresholds": { "mode": "absolute", "steps": [ @@ -2746,20 +2680,25 @@ } ] }, - "color": { - "mode": "thresholds" - } + "unit": "none" }, "overrides": [] }, + "gridPos": { + "h": 4, + "w": 3, + "x": 3, + "y": 98 + }, + "id": 60, "options": { + "colorMode": "background", + "graphMode": "area", "reduceOptions": { "calcs": [ "lastNotNull" ] }, - "colorMode": "background", - "graphMode": "area", "textMode": "auto" }, "targets": [ @@ -2769,25 +2708,20 @@ "refId": "A" } ], - "gridPos": { - "x": 3, - "y": 98, - "w": 3, - "h": 4 - } + "title": "Total Trials", + "type": "stat" }, { - "type": "stat", - "title": "Best Loss", - "description": "", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "id": 61, + "description": "", "fieldConfig": { "defaults": { - "unit": "none", + "color": { + "mode": "thresholds" + }, "thresholds": { "mode": "absolute", "steps": [ @@ -2801,20 +2735,25 @@ } ] }, - "color": { - "mode": "thresholds" - } + "unit": "none" }, "overrides": [] }, + "gridPos": { + "h": 4, + "w": 3, + "x": 6, + "y": 98 + }, + "id": 61, "options": { + "colorMode": "background", + "graphMode": "area", "reduceOptions": { "calcs": [ "lastNotNull" ] }, - "colorMode": "background", - "graphMode": "area", "textMode": "auto" }, "targets": [ @@ -2824,25 +2763,20 @@ "refId": "A" } ], - "gridPos": { - "x": 6, - "y": 98, - "w": 3, - "h": 4 - } + "title": "Best Loss", + "type": "stat" }, { - "type": "stat", - "title": "Elapsed", - "description": "", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "id": 62, + "description": "", "fieldConfig": { "defaults": { - "unit": "s", + "color": { + "mode": "thresholds" + }, "thresholds": { "mode": "absolute", "steps": [ @@ -2852,20 +2786,25 @@ } ] }, - "color": { - "mode": "thresholds" - } + "unit": "s" }, "overrides": [] }, + "gridPos": { + "h": 4, + "w": 3, + "x": 9, + "y": 98 + }, + "id": 62, "options": { + "colorMode": "background", + "graphMode": "area", "reduceOptions": { "calcs": [ "lastNotNull" ] }, - "colorMode": "background", - "graphMode": "area", "textMode": "auto" }, "targets": [ @@ -2875,36 +2814,36 @@ "refId": "A" } ], - "gridPos": { - "x": 9, - "y": 98, - "w": 3, - "h": 4 - } + "title": "Elapsed", + "type": "stat" }, { - "type": "timeseries", - "title": "Best Loss Over Trials", - "description": "Convergence curve \u2014 should decrease over trials", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "id": 63, + "description": "Convergence curve \u2014 should decrease over trials", "fieldConfig": { "defaults": { - "unit": "none", "custom": { "fillOpacity": 0, + "lineWidth": 1, + "spanNulls": true, "stacking": { "mode": "none" - }, - "lineWidth": 1, - "spanNulls": true - } + } + }, + "unit": "none" }, "overrides": [] }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 98 + }, + "id": 63, "options": { "legend": { "displayMode": "list", @@ -2927,87 +2866,87 @@ "refId": "B" } ], - "gridPos": { - "x": 12, - "y": 98, - "w": 12, - "h": 7 - } + "title": "Best Loss Over Trials", + "type": "timeseries" }, { - "type": "heatmap", - "title": "Trial Duration Distribution", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "id": 64, "fieldConfig": { "defaults": {}, "overrides": [] }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 102 + }, + "id": 64, "options": { "calculate": false, + "cellGap": 1, + "color": { + "mode": "scheme", + "scheme": "Oranges" + }, "yAxis": { "unit": "s" - }, - "color": { - "scheme": "Oranges", - "mode": "scheme" - }, - "cellGap": 1 + } }, "targets": [ { "expr": "rate(foxhunt_hyperopt_trial_duration_seconds_bucket{pod=~\"$pod\", namespace=\"foxhunt\"}[$__rate_interval])", + "format": "heatmap", "legendFormat": "{{le}}", - "refId": "A", - "format": "heatmap" + "refId": "A" } ], - "gridPos": { - "x": 0, - "y": 102, - "w": 12, - "h": 7 - } + "title": "Trial Duration Distribution", + "type": "heatmap" }, { - "type": "row", - "title": "Container Resources", "collapsed": false, - "id": 65, "gridPos": { - "x": 0, - "y": 109, + "h": 1, "w": 24, - "h": 1 + "x": 0, + "y": 109 }, - "panels": [] + "id": 65, + "panels": [], + "title": "Container Resources", + "type": "row" }, { - "type": "timeseries", - "title": "Container CPU Usage", - "description": "CPU cores consumed by each container in the pod", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "id": 66, + "description": "CPU cores consumed by each container in the pod", "fieldConfig": { "defaults": { - "unit": "short", "custom": { "fillOpacity": 0, + "lineWidth": 1, + "spanNulls": true, "stacking": { "mode": "none" - }, - "lineWidth": 1, - "spanNulls": true - } + } + }, + "unit": "short" }, "overrides": [] }, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 110 + }, + "id": 66, "options": { "legend": { "displayMode": "list", @@ -3025,36 +2964,36 @@ "refId": "A" } ], - "gridPos": { - "x": 0, - "y": 110, - "w": 6, - "h": 7 - } + "title": "Container CPU Usage", + "type": "timeseries" }, { - "type": "timeseries", - "title": "Container Memory (RSS)", - "description": "Resident set size \u2014 actual physical memory used", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "id": 67, + "description": "Resident set size \u2014 actual physical memory used", "fieldConfig": { "defaults": { - "unit": "bytes", "custom": { "fillOpacity": 0, + "lineWidth": 1, + "spanNulls": true, "stacking": { "mode": "none" - }, - "lineWidth": 1, - "spanNulls": true - } + } + }, + "unit": "bytes" }, "overrides": [] }, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 110 + }, + "id": 67, "options": { "legend": { "displayMode": "list", @@ -3072,36 +3011,36 @@ "refId": "A" } ], - "gridPos": { - "x": 6, - "y": 110, - "w": 6, - "h": 7 - } + "title": "Container Memory (RSS)", + "type": "timeseries" }, { - "type": "timeseries", - "title": "Container Network I/O", - "description": "", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "id": 68, + "description": "", "fieldConfig": { "defaults": { - "unit": "Bps", "custom": { "fillOpacity": 0, + "lineWidth": 1, + "spanNulls": true, "stacking": { "mode": "none" - }, - "lineWidth": 1, - "spanNulls": true - } + } + }, + "unit": "Bps" }, "overrides": [] }, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 110 + }, + "id": 68, "options": { "legend": { "displayMode": "list", @@ -3124,36 +3063,36 @@ "refId": "B" } ], - "gridPos": { - "x": 12, - "y": 110, - "w": 6, - "h": 7 - } + "title": "Container Network I/O", + "type": "timeseries" }, { - "type": "timeseries", - "title": "Pod Restarts", - "description": "Non-zero restarts indicate OOM kills or crashes", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "id": 69, + "description": "Non-zero restarts indicate OOM kills or crashes", "fieldConfig": { "defaults": { - "unit": "none", "custom": { "fillOpacity": 0, + "lineWidth": 1, + "spanNulls": true, "stacking": { "mode": "none" - }, - "lineWidth": 1, - "spanNulls": true - } + } + }, + "unit": "none" }, "overrides": [] }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 110 + }, + "id": 69, "options": { "legend": { "displayMode": "list", @@ -3171,44 +3110,44 @@ "refId": "A" } ], - "gridPos": { - "x": 18, - "y": 110, - "w": 6, - "h": 7 - } + "title": "Pod Restarts", + "type": "timeseries" }, { - "type": "row", - "title": "Live Logs", "collapsed": false, - "id": 70, "gridPos": { - "x": 0, - "y": 117, + "h": 1, "w": 24, - "h": 1 + "x": 0, + "y": 117 }, - "panels": [] + "id": 70, + "panels": [], + "title": "Live Logs", + "type": "row" }, { - "type": "logs", - "title": "Full Log Stream", - "description": "All logs from the selected pod, filtered by level and search text", "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "description": "All logs from the selected pod, filtered by level and search text", + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 118 + }, "id": 71, "options": { - "showTime": true, - "showLabels": false, - "showCommonLabels": false, - "wrapLogMessage": true, - "prettifyLogMessage": false, + "dedupStrategy": "none", "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": true, "sortOrder": "Descending", - "dedupStrategy": "none" + "wrapLogMessage": true }, "targets": [ { @@ -3216,31 +3155,31 @@ "refId": "A" } ], - "gridPos": { - "x": 0, - "y": 118, - "w": 12, - "h": 9 - } + "title": "Full Log Stream", + "type": "logs" }, { - "type": "logs", - "title": "Errors & Warnings", - "description": "Only ERROR and WARN level logs", "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "description": "Only ERROR and WARN level logs", + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 118 + }, "id": 72, "options": { - "showTime": true, - "showLabels": false, - "showCommonLabels": false, - "wrapLogMessage": true, - "prettifyLogMessage": false, + "dedupStrategy": "none", "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": true, "sortOrder": "Descending", - "dedupStrategy": "none" + "wrapLogMessage": true }, "targets": [ { @@ -3248,31 +3187,31 @@ "refId": "A" } ], - "gridPos": { - "x": 12, - "y": 118, - "w": 12, - "h": 9 - } + "title": "Errors & Warnings", + "type": "logs" }, { - "type": "logs", - "title": "Training Progress (Epoch/Loss/Sharpe)", - "description": "Log lines mentioning training metrics \u2014 epoch completions, loss values, Sharpe ratios", "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "description": "Log lines mentioning training metrics \u2014 epoch completions, loss values, Sharpe ratios", + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 127 + }, "id": 73, "options": { - "showTime": true, - "showLabels": false, - "showCommonLabels": false, - "wrapLogMessage": true, - "prettifyLogMessage": false, + "dedupStrategy": "none", "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": true, "sortOrder": "Descending", - "dedupStrategy": "none" + "wrapLogMessage": true }, "targets": [ { @@ -3280,31 +3219,31 @@ "refId": "A" } ], - "gridPos": { - "x": 0, - "y": 127, - "w": 12, - "h": 9 - } + "title": "Training Progress (Epoch/Loss/Sharpe)", + "type": "logs" }, { - "type": "logs", - "title": "GPU / CUDA / OOM", - "description": "GPU-related log lines \u2014 CUDA errors, OOM events, NCCL issues", "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "description": "GPU-related log lines \u2014 CUDA errors, OOM events, NCCL issues", + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 127 + }, "id": 74, "options": { - "showTime": true, - "showLabels": false, - "showCommonLabels": false, - "wrapLogMessage": true, - "prettifyLogMessage": false, + "dedupStrategy": "none", "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": true, "sortOrder": "Descending", - "dedupStrategy": "none" + "wrapLogMessage": true }, "targets": [ { @@ -3312,34 +3251,98 @@ "refId": "A" } ], - "gridPos": { - "x": 12, - "y": 127, - "w": 12, - "h": 9 - } + "title": "GPU / CUDA / OOM", + "type": "logs" } ], - "links": [ + "refresh": "10s", + "schemaVersion": 39, + "tags": [ + "foxhunt", + "training", + "gpu", + "loki", + "deep-dive" + ], + "templating": { + "list": [ + { + "current": {}, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "includeAll": true, + "label": "Pod", + "multi": true, + "name": "pod", + "query": "{namespace=\"foxhunt\", pod\\!~\"(grafana|prometheus|loki|promtail|${DS_TEMPO}|redis|postgres|mattermost|tailscale|gitlab|cilium|csi|konnectivity|nvidia-gpu|argo-workflows|questdb|web-gateway|trading-service).*\"}", + "refresh": 2, + "regex": "", + "sort": 2, + "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "label": "Prometheus Job", + "multi": false, + "name": "prom_job", + "query": "label_values(up{namespace=\"foxhunt\"}, job)", + "refresh": 2, + "regex": "/(train|hyperopt|evaluate|pushgateway|training-pods|dcgm)/", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": "All", + "value": "$__all" + }, + "includeAll": true, + "label": "Log Level", + "multi": true, + "name": "level", + "query": "ERROR,WARN,INFO,DEBUG,TRACE", + "type": "custom" + }, + { + "current": { + "text": "", + "value": "" + }, + "label": "Search", + "name": "search", + "type": "textbox" + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timezone": "browser", + "title": "Training Deep-Dive", + "uid": "foxhunt-training-deep-dive", + "__inputs": [ { - "title": "CI Pipeline Logs", - "url": "/d/foxhunt-ci-logs/foxhunt-ci-pipeline-logs", - "type": "link" + "name": "DS_PROMETHEUS", + "type": "datasource", + "pluginId": "prometheus" }, { - "title": "Training Logs", - "url": "/d/foxhunt-training-logs/foxhunt-training-pipeline-logs", - "type": "link" + "name": "DS_LOKI", + "type": "datasource", + "pluginId": "loki" }, { - "title": "Training & CI/CD", - "url": "/d/foxhunt-training/foxhunt-training-and-ci-cd", - "type": "link" - }, - { - "title": "Operations Cockpit", - "url": "/d/foxhunt-cockpit/foxhunt-operations-cockpit", - "type": "link" + "name": "DS_TEMPO", + "type": "datasource", + "pluginId": "${DS_TEMPO}" } ] -} \ No newline at end of file +} diff --git a/infra/k8s/monitoring/dashboards/foxhunt-training-logs.json b/infra/k8s/monitoring/dashboards/foxhunt-training-logs.json index c785ac352..43f5e3ca1 100644 --- a/infra/k8s/monitoring/dashboards/foxhunt-training-logs.json +++ b/infra/k8s/monitoring/dashboards/foxhunt-training-logs.json @@ -1,181 +1,209 @@ { - "uid": "foxhunt-training-logs", - "title": "Foxhunt Training Pipeline Logs", - "description": "Loki-powered training log viewer — hyperopt trials, epoch progress, GPU status, errors", - "tags": ["foxhunt", "training", "loki", "logs", "gpu"], + "description": "Loki-powered training log viewer \u2014 hyperopt trials, epoch progress, GPU status, errors", "editable": true, "graphTooltip": 2, - "refresh": "30s", - "schemaVersion": 39, - "time": { "from": "now-12h", "to": "now" }, - "timepicker": {}, - "timezone": "browser", - "__inputs": [ - { "name": "DS_LOKI", "label": "Loki", "type": "datasource", "pluginId": "loki" } - ], - "templating": { - "list": [ - { - "name": "job", - "label": "Job Type", - "type": "custom", - "query": "All Training : (train|hyperopt|compile-and-train), DQN : train-dqn, PPO : train-ppo, Supervised : train-supervised, Hyperopt : hyperopt, Compile+Train : compile-and-train", - "current": { "selected": true, "text": "All Training", "value": "(train|hyperopt|compile-and-train)" }, - "options": [ - { "selected": true, "text": "All Training", "value": "(train|hyperopt|compile-and-train)" }, - { "selected": false, "text": "DQN", "value": "train-dqn" }, - { "selected": false, "text": "PPO", "value": "train-ppo" }, - { "selected": false, "text": "Supervised", "value": "train-supervised" }, - { "selected": false, "text": "Hyperopt", "value": "hyperopt" }, - { "selected": false, "text": "Compile+Train", "value": "compile-and-train" } - ] - }, - { - "name": "level", - "label": "Level", - "type": "custom", - "query": "ERROR,WARN,INFO,DEBUG,TRACE", - "current": { "selected": true, "text": "All", "value": "$__all" }, - "includeAll": true, - "allValue": ".*", - "multi": true, - "options": [ - { "selected": true, "text": "All", "value": "$__all" }, - { "selected": false, "text": "ERROR", "value": "ERROR" }, - { "selected": false, "text": "WARN", "value": "WARN" }, - { "selected": false, "text": "INFO", "value": "INFO" }, - { "selected": false, "text": "DEBUG", "value": "DEBUG" }, - { "selected": false, "text": "TRACE", "value": "TRACE" } - ] - }, - { - "name": "search", - "label": "Search", - "type": "textbox", - "current": { "selected": false, "text": "", "value": "" } - } - ] - }, "panels": [ { "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, "id": 1, "title": "Training Overview", "type": "row" }, { - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, "fieldConfig": { "defaults": { - "color": { "fixedColor": "red", "mode": "fixed" }, + "color": { + "fixedColor": "red", + "mode": "fixed" + }, "mappings": [], + "noValue": "0", "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "red", "value": 1 } + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } ] - }, - "noValue": "0" + } }, "overrides": [] }, - "gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 1 + }, "id": 2, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "center", - "reduceOptions": { "calcs": ["sum"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, "textMode": "auto" }, - "title": "Errors", - "type": "stat", "targets": [ { - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, "editorMode": "code", "expr": "sum(count_over_time({namespace=\"foxhunt\", pod=~\"$job-.*\", level=\"ERROR\"} [$__range]))", "queryType": "instant", "refId": "A" } - ] + ], + "title": "Errors", + "type": "stat" }, { - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, "fieldConfig": { "defaults": { - "color": { "fixedColor": "yellow", "mode": "fixed" }, + "color": { + "fixedColor": "yellow", + "mode": "fixed" + }, "mappings": [], + "noValue": "0", "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 1 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + } ] - }, - "noValue": "0" + } }, "overrides": [] }, - "gridPos": { "h": 4, "w": 4, "x": 4, "y": 1 }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 1 + }, "id": 3, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "center", - "reduceOptions": { "calcs": ["sum"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, "textMode": "auto" }, - "title": "Warnings", - "type": "stat", "targets": [ { - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, "editorMode": "code", "expr": "sum(count_over_time({namespace=\"foxhunt\", pod=~\"$job-.*\", level=\"WARN\"} [$__range]))", "queryType": "instant", "refId": "A" } - ] + ], + "title": "Warnings", + "type": "stat" }, { - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, "fieldConfig": { "defaults": { - "color": { "fixedColor": "blue", "mode": "fixed" }, + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, "mappings": [], "noValue": "0" }, "overrides": [] }, - "gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 1 + }, "id": 4, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "center", - "reduceOptions": { "calcs": ["sum"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, "textMode": "auto" }, - "title": "Total Log Lines", - "type": "stat", "targets": [ { - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, "editorMode": "code", "expr": "sum(count_over_time({namespace=\"foxhunt\", pod=~\"$job-.*\"} [$__range]))", "queryType": "instant", "refId": "A" } - ] + ], + "title": "Total Log Lines", + "type": "stat" }, { - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "axisLabel": "lines/interval", @@ -183,302 +211,625 @@ "fillOpacity": 80, "lineWidth": 0, "showPoints": "never", - "stacking": { "mode": "normal" } + "stacking": { + "mode": "normal" + } } }, "overrides": [ { - "matcher": { "id": "byName", "options": "ERROR" }, - "properties": [{ "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } }] + "matcher": { + "id": "byName", + "options": "ERROR" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] }, { - "matcher": { "id": "byName", "options": "WARN" }, - "properties": [{ "id": "color", "value": { "fixedColor": "yellow", "mode": "fixed" } }] + "matcher": { + "id": "byName", + "options": "WARN" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] }, { - "matcher": { "id": "byName", "options": "INFO" }, - "properties": [{ "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } }] + "matcher": { + "id": "byName", + "options": "INFO" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] } ] }, - "gridPos": { "h": 6, "w": 12, "x": 12, "y": 1 }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 1 + }, "id": 5, - "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi" } }, - "title": "Log Volume by Level", - "type": "timeseries", + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, "targets": [ { - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, "editorMode": "code", "expr": "sum by (level) (count_over_time({namespace=\"foxhunt\", pod=~\"$job-.*\"} [$__interval]))", "legendFormat": "{{level}}", "queryType": "range", "refId": "A" } - ] + ], + "title": "Log Volume by Level", + "type": "timeseries" }, { - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "drawStyle": "bars", "fillOpacity": 60, "lineWidth": 0, "showPoints": "never", - "stacking": { "mode": "normal" } + "stacking": { + "mode": "normal" + } } }, "overrides": [] }, - "gridPos": { "h": 4, "w": 12, "x": 0, "y": 5 }, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 5 + }, "id": 6, - "options": { "legend": { "displayMode": "list", "placement": "right" }, "tooltip": { "mode": "multi" } }, - "title": "Log Volume by Pod", - "type": "timeseries", + "options": { + "legend": { + "displayMode": "list", + "placement": "right" + }, + "tooltip": { + "mode": "multi" + } + }, "targets": [ { - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, "editorMode": "code", "expr": "sum by (pod) (count_over_time({namespace=\"foxhunt\", pod=~\"$job-.*\"} [$__interval]))", "legendFormat": "{{pod}}", "queryType": "range", "refId": "A" } - ] + ], + "title": "Log Volume by Pod", + "type": "timeseries" }, { "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 9 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, "id": 10, "title": "Full Training Log Stream", "type": "row" }, { - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, - "gridPos": { "h": 14, "w": 24, "x": 0, "y": 10 }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 14, + "w": 24, + "x": 0, + "y": 10 + }, "id": 11, "options": { - "showTime": true, - "showLabels": true, - "showCommonLabels": false, - "wrapLogMessage": true, - "prettifyLogMessage": false, - "enableLogDetails": true, "dedupStrategy": "none", - "sortOrder": "Descending" + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true }, - "title": "Training Logs — $job", - "type": "logs", "targets": [ { - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, "editorMode": "code", "expr": "{namespace=\"foxhunt\", pod=~\"$job-.*\", level=~\"$level\"} ${search:pipe}", "queryType": "range", "refId": "A" } - ] + ], + "title": "Training Logs \u2014 $job", + "type": "logs" }, { "collapsed": true, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 24 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 24 + }, "id": 20, - "title": "Epoch Progress & Loss", - "type": "row", "panels": [ { - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, - "gridPos": { "h": 12, "w": 24, "x": 0, "y": 25 }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 25 + }, "id": 21, "options": { - "showTime": true, - "showLabels": true, - "wrapLogMessage": true, - "enableLogDetails": true, "dedupStrategy": "none", - "sortOrder": "Descending" + "enableLogDetails": true, + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true }, - "title": "Epoch / Loss / Reward / Sharpe", - "type": "logs", "targets": [ { - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, "editorMode": "code", "expr": "{namespace=\"foxhunt\", pod=~\"$job-.*\"} |~ \"(?i)(epoch|loss|reward|sharpe|accuracy|val_loss|train_loss|best_trial|trial|fold|window|learning_rate|lr=|MaxDD|win.?rate|drawdown)\"", "queryType": "range", "refId": "A" } - ] + ], + "title": "Epoch / Loss / Reward / Sharpe", + "type": "logs" } - ] + ], + "title": "Epoch Progress & Loss", + "type": "row" }, { "collapsed": true, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 25 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 25 + }, "id": 30, - "title": "Hyperopt Trials", - "type": "row", "panels": [ { - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, - "gridPos": { "h": 12, "w": 24, "x": 0, "y": 26 }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 26 + }, "id": 31, "options": { - "showTime": true, - "showLabels": true, - "wrapLogMessage": true, - "enableLogDetails": true, "dedupStrategy": "none", - "sortOrder": "Descending" + "enableLogDetails": true, + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true }, - "title": "Hyperopt Trial Results", - "type": "logs", "targets": [ { - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, "editorMode": "code", "expr": "{namespace=\"foxhunt\", pod=~\"(hyperopt|train)-.*\"} |~ \"(?i)(trial|best|cost|param|learning_rate|hidden_dim|batch_size|buffer_size|gamma|epsilon|clip|PSO|iteration|objective|optimal)\"", "queryType": "range", "refId": "A" } - ] + ], + "title": "Hyperopt Trial Results", + "type": "logs" } - ] + ], + "title": "Hyperopt Trials", + "type": "row" }, { "collapsed": true, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 26 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, "id": 40, - "title": "GPU / CUDA / VRAM", - "type": "row", "panels": [ { - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, - "gridPos": { "h": 12, "w": 24, "x": 0, "y": 27 }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 27 + }, "id": 41, "options": { - "showTime": true, - "showLabels": true, - "wrapLogMessage": true, - "enableLogDetails": true, "dedupStrategy": "none", - "sortOrder": "Descending" + "enableLogDetails": true, + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true }, - "title": "GPU / CUDA / VRAM Logs", - "type": "logs", "targets": [ { - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, "editorMode": "code", "expr": "{namespace=\"foxhunt\", pod=~\"$job-.*\"} |~ \"(?i)(GPU|CUDA|VRAM|cuda|gpu|vram|OOM|out.of.memory|nvrtc|tensor.core|BF16|bf16|SM|occupancy|kernel|device|nvidia)\"", "queryType": "range", "refId": "A" } - ] + ], + "title": "GPU / CUDA / VRAM Logs", + "type": "logs" } - ] + ], + "title": "GPU / CUDA / VRAM", + "type": "row" }, { "collapsed": true, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 27 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 27 + }, "id": 50, - "title": "Walk-Forward Windows", - "type": "row", "panels": [ { - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, - "gridPos": { "h": 10, "w": 24, "x": 0, "y": 28 }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 28 + }, "id": 51, "options": { - "showTime": true, - "showLabels": true, - "wrapLogMessage": true, - "enableLogDetails": true, "dedupStrategy": "none", - "sortOrder": "Descending" + "enableLogDetails": true, + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true }, - "title": "Walk-Forward / Fold / Backtest", - "type": "logs", "targets": [ { - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, "editorMode": "code", "expr": "{namespace=\"foxhunt\", pod=~\"$job-.*\"} |~ \"(?i)(walk.?forward|fold|backtest|window|train_start|train_end|val_start|val_end|test_start|test_end|embargo|purge|bars|features)\"", "queryType": "range", "refId": "A" } - ] + ], + "title": "Walk-Forward / Fold / Backtest", + "type": "logs" } - ] + ], + "title": "Walk-Forward Windows", + "type": "row" }, { "collapsed": true, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 28 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 28 + }, "id": 60, - "title": "Errors & Failures", - "type": "row", "panels": [ { - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, - "gridPos": { "h": 12, "w": 24, "x": 0, "y": 29 }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 29 + }, "id": 61, "options": { - "showTime": true, - "showLabels": true, - "wrapLogMessage": true, - "enableLogDetails": true, "dedupStrategy": "none", - "sortOrder": "Descending" + "enableLogDetails": true, + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true }, - "title": "All Training Errors & Failures", - "type": "logs", "targets": [ { - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, "editorMode": "code", "expr": "{namespace=\"foxhunt\", pod=~\"$job-.*\"} |~ \"(?i)(error|failed|failure|panic|fatal|SIGABRT|SIGSEGV|thread .* panicked|OOM|NaN|inf|diverge)\"", "queryType": "range", "refId": "A" } - ] + ], + "title": "All Training Errors & Failures", + "type": "logs" } - ] + ], + "title": "Errors & Failures", + "type": "row" }, { "collapsed": true, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 29 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 29 + }, "id": 70, - "title": "Data Loading & OFI", - "type": "row", "panels": [ { - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, - "gridPos": { "h": 10, "w": 24, "x": 0, "y": 30 }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 30 + }, "id": 71, "options": { - "showTime": true, - "showLabels": true, - "wrapLogMessage": true, - "enableLogDetails": true, "dedupStrategy": "none", - "sortOrder": "Descending" + "enableLogDetails": true, + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true }, - "title": "Data Loading / OFI / DBN / Features", - "type": "logs", "targets": [ { - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, "editorMode": "code", "expr": "{namespace=\"foxhunt\", pod=~\"$job-.*\"} |~ \"(?i)(loading|loaded|dbn|ofi|features|bars|MBP.?10|OHLCV|symbol|normalize|norm_stats|state_dim|rclone|S3|download)\"", "queryType": "range", "refId": "A" } - ] + ], + "title": "Data Loading / OFI / DBN / Features", + "type": "logs" } - ] + ], + "title": "Data Loading & OFI", + "type": "row" + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": [ + "foxhunt", + "training", + "loki", + "logs", + "gpu" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": "All Training", + "value": "(train|hyperopt|compile-and-train)" + }, + "label": "Job Type", + "name": "job", + "options": [ + { + "selected": true, + "text": "All Training", + "value": "(train|hyperopt|compile-and-train)" + }, + { + "selected": false, + "text": "DQN", + "value": "train-dqn" + }, + { + "selected": false, + "text": "PPO", + "value": "train-ppo" + }, + { + "selected": false, + "text": "Supervised", + "value": "train-supervised" + }, + { + "selected": false, + "text": "Hyperopt", + "value": "hyperopt" + }, + { + "selected": false, + "text": "Compile+Train", + "value": "compile-and-train" + } + ], + "query": "All Training : (train|hyperopt|compile-and-train), DQN : train-dqn, PPO : train-ppo, Supervised : train-supervised, Hyperopt : hyperopt, Compile+Train : compile-and-train", + "type": "custom" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "includeAll": true, + "label": "Level", + "multi": true, + "name": "level", + "options": [ + { + "selected": true, + "text": "All", + "value": "$__all" + }, + { + "selected": false, + "text": "ERROR", + "value": "ERROR" + }, + { + "selected": false, + "text": "WARN", + "value": "WARN" + }, + { + "selected": false, + "text": "INFO", + "value": "INFO" + }, + { + "selected": false, + "text": "DEBUG", + "value": "DEBUG" + }, + { + "selected": false, + "text": "TRACE", + "value": "TRACE" + } + ], + "query": "ERROR,WARN,INFO,DEBUG,TRACE", + "type": "custom" + }, + { + "current": { + "selected": false, + "text": "", + "value": "" + }, + "label": "Search", + "name": "search", + "type": "textbox" + } + ] + }, + "time": { + "from": "now-12h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Training Pipeline Logs", + "uid": "foxhunt-training-logs", + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "type": "datasource", + "pluginId": "prometheus" + }, + { + "name": "DS_LOKI", + "type": "datasource", + "pluginId": "loki" + }, + { + "name": "DS_TEMPO", + "type": "datasource", + "pluginId": "${DS_TEMPO}" } ] } diff --git a/infra/k8s/monitoring/dashboards/foxhunt-training.json b/infra/k8s/monitoring/dashboards/foxhunt-training.json index 59ffa51b4..797583064 100644 --- a/infra/k8s/monitoring/dashboards/foxhunt-training.json +++ b/infra/k8s/monitoring/dashboards/foxhunt-training.json @@ -1,166 +1,27 @@ { - "uid": "foxhunt-training", - "title": "Foxhunt Training & CI/CD", "description": "ML model training, hyperopt, GPU utilization, Argo CI pipelines, and full metrics coverage", - "tags": [ - "foxhunt", - "training", - "gpu", - "ci" - ], "editable": true, "graphTooltip": 2, - "refresh": "15s", - "schemaVersion": 39, - "time": { - "from": "now-6h", - "to": "now" - }, - "timepicker": {}, - "timezone": "browser", - "templating": { - "list": [ - { - "name": "interval", - "label": "Interval", - "type": "interval", - "auto": true, - "auto_count": 30, - "auto_min": "1m", - "current": { - "selected": true, - "text": "auto", - "value": "$__auto_interval_interval" - }, - "options": [ - { - "selected": false, - "text": "1m", - "value": "1m" - }, - { - "selected": false, - "text": "5m", - "value": "5m" - }, - { - "selected": false, - "text": "15m", - "value": "15m" - }, - { - "selected": false, - "text": "30m", - "value": "30m" - }, - { - "selected": false, - "text": "1h", - "value": "1h" - } - ], - "query": "1m,5m,15m,30m,1h" - }, - { - "name": "source", - "label": "Source", - "description": "Live = running training pods, CI History = completed CI runs (pushgateway), All = both", - "type": "custom", - "query": "Live : training-pods, CI History : .+_baseline.*, All : training-pods|.+_baseline.*", - "current": { - "selected": true, - "text": "Live", - "value": "training-pods" - }, - "options": [ - { - "selected": true, - "text": "Live", - "value": "training-pods" - }, - { - "selected": false, - "text": "CI History", - "value": ".+_baseline.*" - }, - { - "selected": false, - "text": "All", - "value": "training-pods|.+_baseline.*" - } - ] - }, - { - "name": "model", - "label": "Model", - "type": "query", - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "query": "label_values(foxhunt_training_step{job=~\"$source\"}, model)", - "refresh": 2, - "includeAll": true, - "multi": true, - "allValue": ".*", - "current": { - "selected": true, - "text": "All", - "value": "$__all" - }, - "sort": 1 - }, - { - "name": "fold", - "label": "Fold", - "type": "query", - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "query": "label_values(foxhunt_training_step{job=~\"$source\"}, fold)", - "refresh": 2, - "includeAll": true, - "multi": true, - "allValue": ".*", - "current": { - "selected": true, - "text": "All", - "value": "$__all" - }, - "sort": 1 - } - ] - }, "panels": [ { - "type": "row", - "title": "Training Status", + "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, - "collapsed": false, "id": 1, - "panels": [] + "panels": [], + "title": "Training Status", + "type": "row" }, { - "id": 2, - "type": "stat", - "title": "Active Workers", - "description": "", - "gridPos": { - "h": 5, - "w": 3, - "x": 0, - "y": 1 - }, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "", "fieldConfig": { "defaults": { "color": { @@ -182,7 +43,17 @@ }, "overrides": [] }, + "gridPos": { + "h": 5, + "w": 3, + "x": 0, + "y": 1 + }, + "id": 2, "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", "reduceOptions": { "calcs": [ "lastNotNull" @@ -190,43 +61,34 @@ "fields": "", "values": false }, - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", "textMode": "value" }, "targets": [ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "sum(foxhunt_training_active_workers{job=\"training-pods\"}) or vector(0)", "legendFormat": "Workers", "refId": "A" } - ] + ], + "title": "Active Workers", + "type": "stat" }, { - "id": 3, - "type": "stat", - "title": "Current Epoch", - "description": "Current epoch and step within epoch", - "gridPos": { - "h": 5, - "w": 4, - "x": 7, - "y": 1 - }, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Current epoch and step within epoch", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, + "decimals": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -235,12 +97,21 @@ "value": null } ] - }, - "decimals": 0 + } }, "overrides": [] }, + "gridPos": { + "h": 5, + "w": 4, + "x": 7, + "y": 1 + }, + "id": 3, "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", "reduceOptions": { "calcs": [ "lastNotNull" @@ -248,16 +119,13 @@ "fields": "", "values": false }, - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", "textMode": "value" }, "targets": [ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_current_epoch{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "legendFormat": "{{model}} / fold {{fold}}", @@ -266,34 +134,28 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_step{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "legendFormat": "step {{model}}/{{fold}}", "refId": "B" } - ] + ], + "title": "Current Epoch", + "type": "stat" }, { - "id": 101, - "type": "stat", - "title": "Current Trial", - "description": "", - "gridPos": { - "h": 5, - "w": 4, - "x": 3, - "y": 1 - }, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, + "decimals": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -302,12 +164,21 @@ "value": null } ] - }, - "decimals": 0 + } }, "overrides": [] }, + "gridPos": { + "h": 5, + "w": 4, + "x": 3, + "y": 1 + }, + "id": 101, "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", "reduceOptions": { "calcs": [ "lastNotNull" @@ -315,43 +186,34 @@ "fields": "", "values": false }, - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", "textMode": "value" }, "targets": [ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_hyperopt_trial_current{model=~\"$model\", job=~\"$source\"}", "legendFormat": "{{model}}", "refId": "A" } - ] + ], + "title": "Current Trial", + "type": "stat" }, { - "id": 4, - "type": "stat", - "title": "Best Objective Δ", - "description": "Rate of improvement in best hyperopt objective (negative = improving for loss minimization)", - "gridPos": { - "h": 5, - "w": 3, - "x": 11, - "y": 1 - }, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Rate of improvement in best hyperopt objective (negative = improving for loss minimization)", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, + "decimals": 4, "thresholds": { "mode": "absolute", "steps": [ @@ -364,12 +226,21 @@ "value": 0 } ] - }, - "decimals": 4 + } }, "overrides": [] }, + "gridPos": { + "h": 5, + "w": 3, + "x": 11, + "y": 1 + }, + "id": 4, "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", "reduceOptions": { "calcs": [ "lastNotNull" @@ -377,40 +248,32 @@ "fields": "", "values": false }, - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", "textMode": "value" }, "targets": [ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "deriv(foxhunt_hyperopt_best_objective{model=~\"$model\", job=~\"$source\"}[10m])", - "legendFormat": "Δ objective", + "legendFormat": "\u0394 objective", "refId": "A" } - ] + ], + "title": "Best Objective \u0394", + "type": "stat" }, { - "id": 5, - "type": "gauge", - "title": "Hyperopt Trial Progress", - "gridPos": { - "h": 5, - "w": 3, - "x": 14, - "y": 1 - }, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { - "unit": "percentunit", + "color": { + "mode": "thresholds" + }, "thresholds": { "mode": "absolute", "steps": [ @@ -428,12 +291,17 @@ } ] }, - "color": { - "mode": "thresholds" - } + "unit": "percentunit" }, "overrides": [] }, + "gridPos": { + "h": 5, + "w": 3, + "x": 14, + "y": 1 + }, + "id": 5, "options": { "reduceOptions": { "calcs": [ @@ -449,29 +317,22 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_hyperopt_trial_current{model=~\"$model\", job=~\"$source\"} / foxhunt_hyperopt_trial_total{model=~\"$model\", job=~\"$source\"}", "legendFormat": "{{model}}", "refId": "A" } - ] + ], + "title": "Hyperopt Trial Progress", + "type": "gauge" }, { - "id": 6, - "type": "stat", - "title": "ML Jobs / Errors", - "description": "", - "gridPos": { - "h": 5, - "w": 3, - "x": 17, - "y": 1 - }, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "", "fieldConfig": { "defaults": { "color": { @@ -514,7 +375,17 @@ } ] }, + "gridPos": { + "h": 5, + "w": 3, + "x": 17, + "y": 1 + }, + "id": 6, "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", "reduceOptions": { "calcs": [ "lastNotNull" @@ -522,16 +393,13 @@ "fields": "", "values": false }, - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", "textMode": "value" }, "targets": [ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "ml_training_jobs_started_total", "legendFormat": "Jobs", @@ -540,29 +408,22 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "ml_training_errors_total", "legendFormat": "Errors", "refId": "B" } - ] + ], + "title": "ML Jobs / Errors", + "type": "stat" }, { - "id": 99, - "type": "stat", - "title": "Hyperopt Elapsed", - "description": "", - "gridPos": { - "h": 5, - "w": 4, - "x": 20, - "y": 1 - }, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "", "fieldConfig": { "defaults": { "color": { @@ -581,7 +442,17 @@ }, "overrides": [] }, + "gridPos": { + "h": 5, + "w": 4, + "x": 20, + "y": 1 + }, + "id": 99, "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", "reduceOptions": { "calcs": [ "lastNotNull" @@ -589,26 +460,23 @@ "fields": "", "values": false }, - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", "textMode": "value" }, "targets": [ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_hyperopt_elapsed_seconds{model=~\"$model\", job=~\"$source\"}", "legendFormat": "Elapsed", "refId": "A" } - ] + ], + "title": "Hyperopt Elapsed", + "type": "stat" }, { - "type": "row", - "title": "Run Summary", "collapsed": false, "gridPos": { "h": 1, @@ -617,21 +485,14 @@ "y": 6 }, "id": 66, - "panels": [] + "panels": [], + "title": "Run Summary", + "type": "row" }, { - "type": "timeseries", - "title": "Best Val Loss", - "id": 67, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 6 - }, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -639,22 +500,23 @@ "mode": "palette-classic" }, "custom": { + "axisPlacement": "auto", "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, "fillOpacity": 15, "gradientMode": "opacity", - "spanNulls": true, + "lineInterpolation": "smooth", + "lineWidth": 2, "pointSize": 5, "showPoints": "auto", + "spanNulls": true, "stacking": { "mode": "none" }, - "axisPlacement": "auto", "thresholdsStyle": { "mode": "off" } }, + "decimals": 4, "thresholds": { "mode": "absolute", "steps": [ @@ -672,52 +534,51 @@ } ] }, - "unit": "none", - "decimals": 4 + "unit": "none" }, "overrides": [] }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 6 + }, + "id": 67, "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, "legend": { - "displayMode": "list", - "placement": "bottom", "calcs": [ "lastNotNull", "max", "min" - ] + ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" } }, "targets": [ { - "expr": "clamp_max(min_over_time(foxhunt_training_validation_loss{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}[$__range]), 5)", - "legendFormat": "{{model}}/{{fold}}", - "refId": "A", "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, - "instant": false + "expr": "clamp_max(min_over_time(foxhunt_training_validation_loss{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}[$__range]), 5)", + "instant": false, + "legendFormat": "{{model}}/{{fold}}", + "refId": "A" } - ] + ], + "title": "Best Val Loss", + "type": "timeseries" }, { - "type": "timeseries", - "title": "Best Sharpe", - "id": 68, - "gridPos": { - "h": 6, - "w": 6, - "x": 6, - "y": 6 - }, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -725,22 +586,23 @@ "mode": "palette-classic" }, "custom": { + "axisPlacement": "auto", "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, "fillOpacity": 15, "gradientMode": "opacity", - "spanNulls": true, + "lineInterpolation": "smooth", + "lineWidth": 2, "pointSize": 5, "showPoints": "auto", + "spanNulls": true, "stacking": { "mode": "none" }, - "axisPlacement": "auto", "thresholdsStyle": { "mode": "off" } }, + "decimals": 2, "thresholds": { "mode": "absolute", "steps": [ @@ -758,52 +620,51 @@ } ] }, - "unit": "none", - "decimals": 2 + "unit": "none" }, "overrides": [] }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 6 + }, + "id": 68, "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, "legend": { - "displayMode": "list", - "placement": "bottom", "calcs": [ "lastNotNull", "max", "min" - ] + ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" } }, "targets": [ { - "expr": "clamp_max(max_over_time(foxhunt_training_epoch_sharpe{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}[$__range]), 5)", - "legendFormat": "{{model}}/{{fold}}", - "refId": "A", "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, - "instant": false + "expr": "clamp_max(max_over_time(foxhunt_training_epoch_sharpe{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}[$__range]), 5)", + "instant": false, + "legendFormat": "{{model}}/{{fold}}", + "refId": "A" } - ] + ], + "title": "Best Sharpe", + "type": "timeseries" }, { - "type": "timeseries", - "title": "Best Win Rate", - "id": 69, - "gridPos": { - "h": 6, - "w": 6, - "x": 12, - "y": 6 - }, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -811,22 +672,23 @@ "mode": "palette-classic" }, "custom": { + "axisPlacement": "auto", "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, "fillOpacity": 15, "gradientMode": "opacity", - "spanNulls": true, + "lineInterpolation": "smooth", + "lineWidth": 2, "pointSize": 5, "showPoints": "auto", + "spanNulls": true, "stacking": { "mode": "none" }, - "axisPlacement": "auto", "thresholdsStyle": { "mode": "off" } }, + "decimals": 1, "thresholds": { "mode": "absolute", "steps": [ @@ -844,79 +706,85 @@ } ] }, - "unit": "percentunit", - "decimals": 1 + "unit": "percentunit" }, "overrides": [] }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 6 + }, + "id": 69, "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, "legend": { - "displayMode": "list", - "placement": "bottom", "calcs": [ "lastNotNull", "max", "min" - ] + ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" } }, "targets": [ { - "expr": "max_over_time(foxhunt_training_epoch_win_rate{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}[$__range])", - "legendFormat": "{{model}}/{{fold}}", - "refId": "A", "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, - "instant": false + "expr": "max_over_time(foxhunt_training_epoch_win_rate{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}[$__range])", + "instant": false, + "legendFormat": "{{model}}/{{fold}}", + "refId": "A" } - ] + ], + "title": "Best Win Rate", + "type": "timeseries" }, { - "id": 42, - "type": "timeseries", - "title": "Best Objective Over Time", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "Objective", + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 4, + "showPoints": "auto" + } + }, + "overrides": [] + }, "gridPos": { "h": 6, "w": 6, "x": 18, "y": 6 }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 10, - "showPoints": "auto", - "pointSize": 4, - "axisLabel": "Objective", - "gradientMode": "opacity" - }, - "color": { - "mode": "palette-classic" - } - }, - "overrides": [] - }, + "id": 42, "options": { "legend": { - "displayMode": "table", - "placement": "bottom", "calcs": [ "lastNotNull", "min" - ] + ], + "displayMode": "table", + "placement": "bottom" }, "tooltip": { "mode": "multi", @@ -927,7 +795,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_hyperopt_best_objective{model=~\"$model\", job=~\"$source\"}", "legendFormat": "best {{model}}", @@ -936,27 +804,20 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_hyperopt_trial_best_loss{model=~\"$model\", job=~\"$source\"}", "legendFormat": "trial loss {{model}}", "refId": "B" } - ] + ], + "title": "Best Objective Over Time", + "type": "timeseries" }, { - "type": "timeseries", - "title": "Min Max Drawdown", - "id": 70, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 12 - }, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -964,22 +825,23 @@ "mode": "palette-classic" }, "custom": { + "axisPlacement": "auto", "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, "fillOpacity": 15, "gradientMode": "opacity", - "spanNulls": true, + "lineInterpolation": "smooth", + "lineWidth": 2, "pointSize": 5, "showPoints": "auto", + "spanNulls": true, "stacking": { "mode": "none" }, - "axisPlacement": "auto", "thresholdsStyle": { "mode": "off" } }, + "decimals": 1, "thresholds": { "mode": "absolute", "steps": [ @@ -997,52 +859,51 @@ } ] }, - "unit": "percentunit", - "decimals": 1 + "unit": "percentunit" }, "overrides": [] }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 12 + }, + "id": 70, "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, "legend": { - "displayMode": "list", - "placement": "bottom", "calcs": [ "lastNotNull", "max", "min" - ] + ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" } }, "targets": [ { - "expr": "min_over_time(foxhunt_training_epoch_max_drawdown{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}[$__range])", - "legendFormat": "{{model}}/{{fold}}", - "refId": "A", "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, - "instant": false + "expr": "min_over_time(foxhunt_training_epoch_max_drawdown{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}[$__range])", + "instant": false, + "legendFormat": "{{model}}/{{fold}}", + "refId": "A" } - ] + ], + "title": "Min Max Drawdown", + "type": "timeseries" }, { - "type": "timeseries", - "title": "Total Epochs", - "id": 71, - "gridPos": { - "h": 6, - "w": 6, - "x": 6, - "y": 12 - }, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -1050,22 +911,23 @@ "mode": "palette-classic" }, "custom": { + "axisPlacement": "auto", "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, "fillOpacity": 15, "gradientMode": "opacity", - "spanNulls": true, + "lineInterpolation": "smooth", + "lineWidth": 2, "pointSize": 5, "showPoints": "auto", + "spanNulls": true, "stacking": { "mode": "none" }, - "axisPlacement": "auto", "thresholdsStyle": { "mode": "off" } }, + "decimals": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -1075,52 +937,51 @@ } ] }, - "unit": "none", - "decimals": 0 + "unit": "none" }, "overrides": [] }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 12 + }, + "id": 71, "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, "legend": { - "displayMode": "list", - "placement": "bottom", "calcs": [ "lastNotNull", "max", "min" - ] + ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" } }, "targets": [ { - "expr": "foxhunt_training_current_epoch{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", - "legendFormat": "{{model}}/{{fold}}", - "refId": "A", "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, - "instant": false + "expr": "foxhunt_training_current_epoch{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", + "instant": false, + "legendFormat": "{{model}}/{{fold}}", + "refId": "A" } - ] + ], + "title": "Total Epochs", + "type": "timeseries" }, { - "type": "timeseries", - "title": "Best Profit Factor", - "id": 72, - "gridPos": { - "h": 6, - "w": 6, - "x": 12, - "y": 12 - }, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { @@ -1128,22 +989,23 @@ "mode": "palette-classic" }, "custom": { + "axisPlacement": "auto", "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, "fillOpacity": 15, "gradientMode": "opacity", - "spanNulls": true, + "lineInterpolation": "smooth", + "lineWidth": 2, "pointSize": 5, "showPoints": "auto", + "spanNulls": true, "stacking": { "mode": "none" }, - "axisPlacement": "auto", "thresholdsStyle": { "mode": "off" } }, + "decimals": 2, "thresholds": { "mode": "absolute", "steps": [ @@ -1153,7 +1015,7 @@ }, { "color": "yellow", - "value": 1.0 + "value": 1 }, { "color": "green", @@ -1161,72 +1023,78 @@ } ] }, - "unit": "none", - "decimals": 2 + "unit": "none" }, "overrides": [] }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 12 + }, + "id": 72, "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, "legend": { - "displayMode": "list", - "placement": "bottom", "calcs": [ "lastNotNull", "max", "min" - ] + ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" } }, "targets": [ { - "expr": "clamp_max(max_over_time(foxhunt_training_epoch_profit_factor{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}[$__range]), 20)", - "legendFormat": "{{model}}/{{fold}}", - "refId": "A", "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, - "instant": false + "expr": "clamp_max(max_over_time(foxhunt_training_epoch_profit_factor{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}[$__range]), 20)", + "instant": false, + "legendFormat": "{{model}}/{{fold}}", + "refId": "A" } - ] + ], + "title": "Best Profit Factor", + "type": "timeseries" }, { - "id": 43, - "type": "timeseries", - "title": "Trial Duration (p50/p95/p99)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "Duration", + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 4, + "showPoints": "never" + }, + "unit": "s" + }, + "overrides": [] + }, "gridPos": { "h": 6, "w": 6, "x": 18, "y": 12 }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 10, - "showPoints": "never", - "axisLabel": "Duration", - "gradientMode": "opacity", - "pointSize": 4 - }, - "color": { - "mode": "palette-classic" - }, - "unit": "s" - }, - "overrides": [] - }, + "id": 43, "options": { "legend": { "displayMode": "list", @@ -1241,7 +1109,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.50, rate(foxhunt_hyperopt_trial_duration_seconds_bucket{model=~\"$model\", job=~\"$source\"}[$__rate_interval]))", "legendFormat": "p50 {{model}}", @@ -1250,7 +1118,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.95, rate(foxhunt_hyperopt_trial_duration_seconds_bucket{model=~\"$model\", job=~\"$source\"}[$__rate_interval]))", "legendFormat": "p95 {{model}}", @@ -1259,68 +1127,68 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.99, rate(foxhunt_hyperopt_trial_duration_seconds_bucket{model=~\"$model\", job=~\"$source\"}[$__rate_interval]))", "legendFormat": "p99 {{model}}", "refId": "C" } - ] + ], + "title": "Trial Duration (p50/p95/p99)", + "type": "timeseries" }, { - "type": "row", - "title": "Training Curves", + "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 18 }, - "collapsed": false, "id": 10, - "panels": [] + "panels": [], + "title": "Training Curves", + "type": "row" }, { - "id": 11, - "type": "timeseries", - "title": "Training & Validation Loss", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "Loss", + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never" + } + }, + "overrides": [] + }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 19 }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 10, - "gradientMode": "opacity", - "showPoints": "never", - "pointSize": 5, - "axisCenteredZero": false, - "axisLabel": "Loss" - }, - "color": { - "mode": "palette-classic" - } - }, - "overrides": [] - }, + "id": 11, "options": { "legend": { - "displayMode": "table", - "placement": "bottom", "calcs": [ "lastNotNull", "min" - ] + ], + "displayMode": "table", + "placement": "bottom" }, "tooltip": { "mode": "multi", @@ -1331,7 +1199,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_epoch_loss{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "legendFormat": "train {{model}}/{{fold}}", @@ -1340,58 +1208,58 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_validation_loss{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "legendFormat": "val {{model}}/{{fold}}", "refId": "B" } - ] + ], + "title": "Training & Validation Loss", + "type": "timeseries" }, { - "id": 12, - "type": "timeseries", - "title": "Learning Rate", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "LR", + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "log": 10, + "type": "log" + }, + "showPoints": "never" + }, + "unit": "short" + }, + "overrides": [] + }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 19 }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 15, - "showPoints": "never", - "pointSize": 5, - "scaleDistribution": { - "type": "log", - "log": 10 - }, - "axisLabel": "LR", - "gradientMode": "opacity" - }, - "color": { - "mode": "palette-classic" - }, - "unit": "short" - }, - "overrides": [] - }, + "id": 12, "options": { "legend": { - "displayMode": "table", - "placement": "bottom", "calcs": [ "lastNotNull" - ] + ], + "displayMode": "table", + "placement": "bottom" }, "tooltip": { "mode": "multi", @@ -1402,46 +1270,46 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_learning_rate{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "legendFormat": "{{model}}/{{fold}}", "refId": "A" } - ] + ], + "title": "Learning Rate", + "type": "timeseries" }, { - "id": 13, - "type": "timeseries", - "title": "Gradient Norm", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "Norm", + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 4, + "showPoints": "never" + } + }, + "overrides": [] + }, "gridPos": { "h": 8, "w": 4, "x": 16, "y": 19 }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 10, - "showPoints": "never", - "axisLabel": "Norm", - "gradientMode": "opacity", - "pointSize": 4 - }, - "color": { - "mode": "palette-classic" - } - }, - "overrides": [] - }, + "id": 13, "options": { "legend": { "displayMode": "list", @@ -1456,41 +1324,34 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_gradient_norm{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "legendFormat": "{{model}}/{{fold}}", "refId": "A" } - ] + ], + "title": "Gradient Norm", + "type": "timeseries" }, { - "id": 14, - "type": "timeseries", - "title": "Batches/sec & Epoch Duration", - "gridPos": { - "h": 8, - "w": 4, - "x": 20, - "y": 19 - }, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 10, - "showPoints": "never", - "gradientMode": "opacity", - "pointSize": 4 - }, "color": { "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 4, + "showPoints": "never" } }, "overrides": [ @@ -1512,6 +1373,13 @@ } ] }, + "gridPos": { + "h": 8, + "w": 4, + "x": 20, + "y": 19 + }, + "id": 14, "options": { "legend": { "displayMode": "list", @@ -1526,7 +1394,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_batches_per_second{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "legendFormat": "batch/s {{model}}", @@ -1535,56 +1403,49 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_epoch_duration_seconds{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "legendFormat": "epoch dur {{model}}", "refId": "B" } - ] + ], + "title": "Batches/sec & Epoch Duration", + "type": "timeseries" }, { - "type": "row", - "title": "Training Quality", + "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 27 }, - "collapsed": false, "id": 20, - "panels": [] + "panels": [], + "title": "Training Quality", + "type": "row" }, { - "id": 21, - "type": "timeseries", - "title": "Epoch Sharpe Ratio", - "gridPos": { - "h": 7, - "w": 6, - "x": 0, - "y": 28 - }, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 10, - "gradientMode": "opacity", - "showPoints": "auto", - "pointSize": 4, - "axisLabel": "Sharpe" - }, "color": { "mode": "palette-classic" }, + "custom": { + "axisLabel": "Sharpe", + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 4, + "showPoints": "auto" + }, "thresholds": { "mode": "absolute", "steps": [ @@ -1605,14 +1466,21 @@ }, "overrides": [] }, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 28 + }, + "id": 21, "options": { "legend": { - "displayMode": "table", - "placement": "bottom", "calcs": [ "lastNotNull", "max" - ] + ], + "displayMode": "table", + "placement": "bottom" }, "tooltip": { "mode": "multi", @@ -1623,54 +1491,54 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "clamp_max(foxhunt_training_epoch_sharpe{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}, 5)", "legendFormat": "{{model}}/{{fold}}", "refId": "A" } - ] + ], + "title": "Epoch Sharpe Ratio", + "type": "timeseries" }, { - "id": 22, - "type": "timeseries", - "title": "Epoch Sortino Ratio", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "Sortino", + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 4, + "showPoints": "auto" + } + }, + "overrides": [] + }, "gridPos": { "h": 7, "w": 6, "x": 6, "y": 28 }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 10, - "showPoints": "auto", - "pointSize": 4, - "axisLabel": "Sortino", - "gradientMode": "opacity" - }, - "color": { - "mode": "palette-classic" - } - }, - "overrides": [] - }, + "id": 22, "options": { "legend": { - "displayMode": "table", - "placement": "bottom", "calcs": [ "lastNotNull", "max" - ] + ], + "displayMode": "table", + "placement": "bottom" }, "tooltip": { "mode": "multi", @@ -1681,47 +1549,47 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "clamp_max(foxhunt_training_epoch_sortino{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}, 10)", "legendFormat": "{{model}}/{{fold}}", "refId": "A" } - ] + ], + "title": "Epoch Sortino Ratio", + "type": "timeseries" }, { - "id": 23, - "type": "timeseries", - "title": "Win Rate", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "Win Rate", + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 4, + "showPoints": "auto" + }, + "unit": "percentunit" + }, + "overrides": [] + }, "gridPos": { "h": 7, "w": 4, "x": 12, "y": 28 }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 10, - "showPoints": "auto", - "pointSize": 4, - "axisLabel": "Win Rate", - "gradientMode": "opacity" - }, - "color": { - "mode": "palette-classic" - }, - "unit": "percentunit" - }, - "overrides": [] - }, + "id": 23, "options": { "legend": { "displayMode": "list", @@ -1736,47 +1604,47 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_epoch_win_rate{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "legendFormat": "{{model}}/{{fold}}", "refId": "A" } - ] + ], + "title": "Win Rate", + "type": "timeseries" }, { - "id": 24, - "type": "timeseries", - "title": "Max Drawdown", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "Drawdown", + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 4, + "showPoints": "auto" + }, + "unit": "percentunit" + }, + "overrides": [] + }, "gridPos": { "h": 7, "w": 4, "x": 16, "y": 28 }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 10, - "showPoints": "auto", - "pointSize": 4, - "axisLabel": "Drawdown", - "gradientMode": "opacity" - }, - "color": { - "mode": "palette-classic" - }, - "unit": "percentunit" - }, - "overrides": [] - }, + "id": 24, "options": { "legend": { "displayMode": "list", @@ -1791,41 +1659,34 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_epoch_max_drawdown{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "legendFormat": "{{model}}/{{fold}}", "refId": "A" } - ] + ], + "title": "Max Drawdown", + "type": "timeseries" }, { - "id": 25, - "type": "timeseries", - "title": "Profit Factor & Total Trades", - "gridPos": { - "h": 7, - "w": 4, - "x": 20, - "y": 28 - }, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 15, - "showPoints": "auto", - "pointSize": 4, - "gradientMode": "opacity" - }, "color": { "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 4, + "showPoints": "auto" } }, "overrides": [ @@ -1867,6 +1728,13 @@ } ] }, + "gridPos": { + "h": 7, + "w": 4, + "x": 20, + "y": 28 + }, + "id": 25, "options": { "legend": { "displayMode": "list", @@ -1881,7 +1749,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "clamp_max(foxhunt_training_epoch_profit_factor{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}, 20)", "legendFormat": "PF {{model}}", @@ -1890,68 +1758,68 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_epoch_total_trades{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "legendFormat": "trades {{model}}", "refId": "B" } - ] + ], + "title": "Profit Factor & Total Trades", + "type": "timeseries" }, { - "type": "row", - "title": "RL Diagnostics", + "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 35 }, - "collapsed": false, "id": 30, - "panels": [] + "panels": [], + "title": "RL Diagnostics", + "type": "row" }, { - "id": 31, - "type": "timeseries", - "title": "Q-Value (Mean & Max)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "Q-Value", + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 4, + "showPoints": "never" + } + }, + "overrides": [] + }, "gridPos": { "h": 7, "w": 6, "x": 0, "y": 36 }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 10, - "showPoints": "never", - "axisLabel": "Q-Value", - "gradientMode": "opacity", - "pointSize": 4 - }, - "color": { - "mode": "palette-classic" - } - }, - "overrides": [] - }, + "id": 31, "options": { "legend": { - "displayMode": "table", - "placement": "bottom", "calcs": [ "lastNotNull", "max", "min" - ] + ], + "displayMode": "table", + "placement": "bottom" }, "tooltip": { "mode": "multi", @@ -1962,7 +1830,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_q_value_mean{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "legendFormat": "mean {{model}}/{{fold}}", @@ -1971,47 +1839,47 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_q_value_max{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "legendFormat": "max {{model}}/{{fold}}", "refId": "B" } - ] + ], + "title": "Q-Value (Mean & Max)", + "type": "timeseries" }, { - "id": 32, - "type": "timeseries", - "title": "Replay Buffer Size", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-blues" + }, + "custom": { + "axisLabel": "Transitions", + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "scheme", + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 4, + "showPoints": "never" + }, + "unit": "short" + }, + "overrides": [] + }, "gridPos": { "h": 7, "w": 4, "x": 6, "y": 36 }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 30, - "showPoints": "never", - "axisLabel": "Transitions", - "gradientMode": "scheme", - "pointSize": 4 - }, - "color": { - "mode": "continuous-blues" - }, - "unit": "short" - }, - "overrides": [] - }, + "id": 32, "options": { "legend": { "displayMode": "list", @@ -2026,44 +1894,40 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_replay_buffer_size{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "legendFormat": "{{model}}/{{fold}}", "refId": "A" } - ] + ], + "title": "Replay Buffer Size", + "type": "timeseries" }, { - "id": 33, - "type": "timeseries", - "title": "Action Entropy", - "description": "Normalized entropy of policy actions. 0 = collapsed to single action, 1 = uniform distribution.", - "gridPos": { - "h": 7, - "w": 4, - "x": 10, - "y": 36 - }, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Normalized entropy of policy actions. 0 = collapsed to single action, 1 = uniform distribution.", "fieldConfig": { "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 15, - "showPoints": "never", - "axisLabel": "Entropy", - "gradientMode": "opacity", - "pointSize": 4 - }, "color": { "mode": "palette-classic" }, + "custom": { + "axisLabel": "Entropy", + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 4, + "showPoints": "never" + }, + "custom.thresholdsStyle": { + "mode": "line" + }, "thresholds": { "mode": "absolute", "steps": [ @@ -2080,13 +1944,17 @@ "value": 0.3 } ] - }, - "custom.thresholdsStyle": { - "mode": "line" } }, "overrides": [] }, + "gridPos": { + "h": 7, + "w": 4, + "x": 10, + "y": 36 + }, + "id": 33, "options": { "legend": { "displayMode": "list", @@ -2101,48 +1969,48 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_action_entropy{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "legendFormat": "{{model}}/{{fold}}", "refId": "A" } - ] + ], + "title": "Action Entropy", + "type": "timeseries" }, { - "id": 34, - "type": "timeseries", - "title": "Action Diversity", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "description": "Fraction of 45 factored actions actually used by the policy.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "Diversity", + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 4, + "showPoints": "never" + }, + "unit": "percentunit" + }, + "overrides": [] + }, "gridPos": { "h": 7, "w": 4, "x": 14, "y": 36 }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 15, - "showPoints": "never", - "axisLabel": "Diversity", - "gradientMode": "opacity", - "pointSize": 4 - }, - "color": { - "mode": "palette-classic" - }, - "unit": "percentunit" - }, - "overrides": [] - }, + "id": 34, "options": { "legend": { "displayMode": "list", @@ -2157,47 +2025,40 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_action_diversity{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "legendFormat": "{{model}}/{{fold}}", "refId": "A" } - ] + ], + "title": "Action Diversity", + "type": "timeseries" }, { - "id": 35, - "type": "timeseries", - "title": "Action Distribution (Buy/Sell/Hold %)", - "gridPos": { - "h": 7, - "w": 6, - "x": 18, - "y": 36 - }, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 1, - "fillOpacity": 50, - "showPoints": "never", - "stacking": { - "mode": "normal", - "group": "A" - }, - "axisLabel": "Pct", - "gradientMode": "opacity", - "pointSize": 4 - }, "color": { "mode": "palette-classic" }, + "custom": { + "axisLabel": "Pct", + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 4, + "showPoints": "never", + "stacking": { + "group": "A", + "mode": "normal" + } + }, "unit": "percentunit" }, "overrides": [ @@ -2248,6 +2109,13 @@ } ] }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 36 + }, + "id": 35, "options": { "legend": { "displayMode": "list", @@ -2262,7 +2130,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_epoch_action_buy_pct{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "legendFormat": "buy {{model}}", @@ -2271,7 +2139,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_epoch_action_sell_pct{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "legendFormat": "sell {{model}}", @@ -2280,51 +2148,45 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_epoch_action_hold_pct{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "legendFormat": "hold {{model}}", "refId": "C" } - ] + ], + "title": "Action Distribution (Buy/Sell/Hold %)", + "type": "timeseries" }, { - "id": 102, - "type": "timeseries", - "title": "Q-Overestimation Ratio", - "description": "Q-max / |Q-mean| — spikes above 10 indicate DQN overestimation", - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 43 - }, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Q-max / |Q-mean| \u2014 spikes above 10 indicate DQN overestimation", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisPlacement": "auto", "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, "fillOpacity": 15, "gradientMode": "opacity", - "spanNulls": true, + "lineInterpolation": "smooth", + "lineWidth": 2, "pointSize": 5, "showPoints": "auto", + "spanNulls": true, "stacking": { "mode": "none" }, - "axisPlacement": "auto", "thresholdsStyle": { "mode": "off" } }, + "decimals": 1, "thresholds": { "mode": "absolute", "steps": [ @@ -2342,75 +2204,75 @@ } ] }, - "unit": "none", - "decimals": 1 + "unit": "none" }, "overrides": [] }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 43 + }, + "id": 102, "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, "legend": { - "displayMode": "list", - "placement": "bottom", "calcs": [ "lastNotNull", "max", "min" - ] + ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_q_overestimation_ratio{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "legendFormat": "{{model}}/{{fold}}", "refId": "A" } - ] + ], + "title": "Q-Overestimation Ratio", + "type": "timeseries" }, { - "id": 103, - "type": "timeseries", - "title": "Policy Entropy & KL Divergence (PPO)", - "description": "Policy entropy (exploration health) and KL divergence (policy drift)", - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 43 - }, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Policy entropy (exploration health) and KL divergence (policy drift)", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisPlacement": "auto", "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, "fillOpacity": 15, "gradientMode": "opacity", - "spanNulls": true, + "lineInterpolation": "smooth", + "lineWidth": 2, "pointSize": 5, "showPoints": "auto", + "spanNulls": true, "stacking": { "mode": "none" }, - "axisPlacement": "auto", "thresholdsStyle": { "mode": "off" } }, + "decimals": 4, "thresholds": { "mode": "absolute", "steps": [ @@ -2420,31 +2282,37 @@ } ] }, - "unit": "none", - "decimals": 4 + "unit": "none" }, "overrides": [] }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 43 + }, + "id": 103, "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, "legend": { - "displayMode": "list", - "placement": "bottom", "calcs": [ "lastNotNull", "max", "min" - ] + ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_policy_entropy{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "legendFormat": "entropy {{model}}/{{fold}}", @@ -2453,51 +2321,45 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_kl_divergence{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "legendFormat": "KL {{model}}/{{fold}}", "refId": "B" } - ] + ], + "title": "Policy Entropy & KL Divergence (PPO)", + "type": "timeseries" }, { - "id": 104, - "type": "timeseries", - "title": "Advantage Mean (PPO)", - "description": "Mean advantage estimate — should hover near zero", - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 43 - }, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Mean advantage estimate \u2014 should hover near zero", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisPlacement": "auto", "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, "fillOpacity": 15, "gradientMode": "opacity", - "spanNulls": true, + "lineInterpolation": "smooth", + "lineWidth": 2, "pointSize": 5, "showPoints": "auto", + "spanNulls": true, "stacking": { "mode": "none" }, - "axisPlacement": "auto", "thresholdsStyle": { "mode": "off" } }, + "decimals": 4, "thresholds": { "mode": "absolute", "steps": [ @@ -2507,41 +2369,47 @@ } ] }, - "unit": "none", - "decimals": 4 + "unit": "none" }, "overrides": [] }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 43 + }, + "id": 104, "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, "legend": { - "displayMode": "list", - "placement": "bottom", "calcs": [ "lastNotNull", "max", "min" - ] + ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_advantage_mean{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "legendFormat": "{{model}}/{{fold}}", "refId": "A" } - ] + ], + "title": "Advantage Mean (PPO)", + "type": "timeseries" }, { - "type": "row", - "title": "Training Health", "collapsed": false, "gridPos": { "h": 1, @@ -2550,45 +2418,39 @@ "y": 51 }, "id": 105, - "panels": [] + "panels": [], + "title": "Training Health", + "type": "row" }, { - "id": 106, - "type": "timeseries", - "title": "Training Anomalies", - "description": "NaN loss events, gradient explosions, and feature extraction failures", - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 52 - }, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "NaN loss events, gradient explosions, and feature extraction failures", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisPlacement": "auto", "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, "fillOpacity": 15, "gradientMode": "opacity", - "spanNulls": true, + "lineInterpolation": "smooth", + "lineWidth": 2, "pointSize": 5, "showPoints": "auto", + "spanNulls": true, "stacking": { "mode": "none" }, - "axisPlacement": "auto", "thresholdsStyle": { "mode": "off" } }, + "decimals": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -2602,31 +2464,37 @@ } ] }, - "unit": "none", - "decimals": 0 + "unit": "none" }, "overrides": [] }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 52 + }, + "id": 106, "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, "legend": { - "displayMode": "list", - "placement": "bottom", "calcs": [ "lastNotNull", "max", "min" - ] + ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "increase(foxhunt_training_nan_detected_total{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}[$__rate_interval])", "legendFormat": "NaN {{model}}/{{fold}}", @@ -2635,7 +2503,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "increase(foxhunt_training_gradient_explosion_total{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}[$__rate_interval])", "legendFormat": "grad explosion {{model}}/{{fold}}", @@ -2644,51 +2512,45 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "increase(foxhunt_training_feature_errors_total{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}[$__rate_interval])", "legendFormat": "feature err {{model}}/{{fold}}", "refId": "C" } - ] + ], + "title": "Training Anomalies", + "type": "timeseries" }, { - "id": 107, - "type": "timeseries", - "title": "Checkpoint Saves & Failures", - "description": "Checkpoint save/failure rate", - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 52 - }, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Checkpoint save/failure rate", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisPlacement": "auto", "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, "fillOpacity": 15, "gradientMode": "opacity", - "spanNulls": true, + "lineInterpolation": "smooth", + "lineWidth": 2, "pointSize": 5, "showPoints": "auto", + "spanNulls": true, "stacking": { "mode": "none" }, - "axisPlacement": "auto", "thresholdsStyle": { "mode": "off" } }, + "decimals": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -2698,31 +2560,37 @@ } ] }, - "unit": "none", - "decimals": 0 + "unit": "none" }, "overrides": [] }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 52 + }, + "id": 107, "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, "legend": { - "displayMode": "list", - "placement": "bottom", "calcs": [ "lastNotNull", "max", "min" - ] + ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "increase(foxhunt_training_checkpoint_saves_total{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}[$__rate_interval])", "legendFormat": "saves {{model}}/{{fold}}", @@ -2731,51 +2599,45 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "increase(foxhunt_training_checkpoint_failures_total{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}[$__rate_interval])", "legendFormat": "failures {{model}}/{{fold}}", "refId": "B" } - ] + ], + "title": "Checkpoint Saves & Failures", + "type": "timeseries" }, { - "id": 108, - "type": "timeseries", - "title": "Checkpoint & Data Load Latency", - "description": "p95 checkpoint save and data loading latency", - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 52 - }, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "p95 checkpoint save and data loading latency", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisPlacement": "auto", "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, "fillOpacity": 15, "gradientMode": "opacity", - "spanNulls": true, + "lineInterpolation": "smooth", + "lineWidth": 2, "pointSize": 5, "showPoints": "auto", + "spanNulls": true, "stacking": { "mode": "none" }, - "axisPlacement": "auto", "thresholdsStyle": { "mode": "off" } }, + "decimals": 1, "thresholds": { "mode": "absolute", "steps": [ @@ -2785,31 +2647,37 @@ } ] }, - "unit": "s", - "decimals": 1 + "unit": "s" }, "overrides": [] }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 52 + }, + "id": 108, "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, "legend": { - "displayMode": "list", - "placement": "bottom", "calcs": [ "lastNotNull", "max", "min" - ] + ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.95, rate(foxhunt_training_checkpoint_duration_seconds_bucket{model=~\"$model\", job=~\"$source\"}[$__rate_interval]))", "legendFormat": "ckpt p95 {{model}}", @@ -2818,51 +2686,123 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.95, rate(foxhunt_training_data_load_seconds_bucket{model=~\"$model\", job=~\"$source\"}[$__rate_interval]))", "legendFormat": "data load p95 {{model}}", "refId": "B" } - ] + ], + "title": "Checkpoint & Data Load Latency", + "type": "timeseries" }, { - "id": 109, - "type": "timeseries", - "title": "Checkpoint Size", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "description": "Size of last saved checkpoint", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 60 }, + "id": 109, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "foxhunt_training_checkpoint_size_bytes{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", + "legendFormat": "{{model}}/{{fold}}", + "refId": "A" + } + ], + "title": "Checkpoint Size", + "type": "timeseries" + }, + { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Total batches processed this run", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisPlacement": "auto", "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, "fillOpacity": 15, "gradientMode": "opacity", - "spanNulls": true, + "lineInterpolation": "smooth", + "lineWidth": 2, "pointSize": 5, "showPoints": "auto", + "spanNulls": true, "stacking": { "mode": "none" }, - "axisPlacement": "auto", "thresholdsStyle": { "mode": "off" } }, + "decimals": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -2872,75 +2812,75 @@ } ] }, - "unit": "bytes", - "decimals": 0 + "unit": "none" }, "overrides": [] }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "list", - "placement": "bottom", - "calcs": [ - "lastNotNull", - "max", - "min" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "foxhunt_training_checkpoint_size_bytes{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", - "legendFormat": "{{model}}/{{fold}}", - "refId": "A" - } - ] - }, - { - "id": 110, - "type": "timeseries", - "title": "Batches Processed", - "description": "Total batches processed this run", "gridPos": { "h": 8, "w": 8, "x": 8, "y": 60 }, + "id": 110, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "foxhunt_training_batches_processed{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", + "legendFormat": "{{model}}/{{fold}}", + "refId": "A" + } + ], + "title": "Batches Processed", + "type": "timeseries" + }, + { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Last epoch wall time", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisPlacement": "auto", "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, "fillOpacity": 15, "gradientMode": "opacity", - "spanNulls": true, + "lineInterpolation": "smooth", + "lineWidth": 2, "pointSize": 5, "showPoints": "auto", + "spanNulls": true, "stacking": { "mode": "none" }, - "axisPlacement": "auto", "thresholdsStyle": { "mode": "off" } }, + "decimals": 1, "thresholds": { "mode": "absolute", "steps": [ @@ -2950,119 +2890,47 @@ } ] }, - "unit": "none", - "decimals": 0 + "unit": "s" }, "overrides": [] }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "list", - "placement": "bottom", - "calcs": [ - "lastNotNull", - "max", - "min" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "foxhunt_training_batches_processed{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", - "legendFormat": "{{model}}/{{fold}}", - "refId": "A" - } - ] - }, - { - "id": 111, - "type": "timeseries", - "title": "Iteration Time", - "description": "Last epoch wall time", "gridPos": { "h": 8, "w": 8, "x": 16, "y": 60 }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 15, - "gradientMode": "opacity", - "spanNulls": true, - "pointSize": 5, - "showPoints": "auto", - "stacking": { - "mode": "none" - }, - "axisPlacement": "auto", - "thresholdsStyle": { - "mode": "off" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "s", - "decimals": 1 - }, - "overrides": [] - }, + "id": 111, "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, "legend": { - "displayMode": "list", - "placement": "bottom", "calcs": [ "lastNotNull", "max", "min" - ] + ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_iteration_seconds{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "legendFormat": "{{model}}/{{fold}}", "refId": "A" } - ] + ], + "title": "Iteration Time", + "type": "timeseries" }, { - "type": "row", - "title": "Epoch Returns", "collapsed": false, "gridPos": { "h": 1, @@ -3071,45 +2939,117 @@ "y": 68 }, "id": 112, - "panels": [] + "panels": [], + "title": "Epoch Returns", + "type": "row" }, { - "id": 113, - "type": "timeseries", - "title": "Total Return per Epoch", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "description": "Epoch total return (fractional)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 69 }, + "id": 113, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "min" + ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "foxhunt_training_epoch_total_return{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", + "legendFormat": "{{model}}/{{fold}}", + "refId": "A" + } + ], + "title": "Total Return per Epoch", + "type": "timeseries" + }, + { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Average return per trade", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisPlacement": "auto", "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, "fillOpacity": 15, "gradientMode": "opacity", - "spanNulls": true, + "lineInterpolation": "smooth", + "lineWidth": 2, "pointSize": 5, "showPoints": "auto", + "spanNulls": true, "stacking": { "mode": "none" }, - "axisPlacement": "auto", "thresholdsStyle": { "mode": "off" } }, + "decimals": 4, "thresholds": { "mode": "absolute", "steps": [ @@ -3119,119 +3059,47 @@ } ] }, - "unit": "percentunit", - "decimals": 2 + "unit": "none" }, "overrides": [] }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "list", - "placement": "bottom", - "calcs": [ - "lastNotNull", - "max", - "min" - ] - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "foxhunt_training_epoch_total_return{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", - "legendFormat": "{{model}}/{{fold}}", - "refId": "A" - } - ] - }, - { - "id": 114, - "type": "timeseries", - "title": "Avg Return per Trade", - "description": "Average return per trade", "gridPos": { "h": 8, "w": 12, "x": 12, "y": 69 }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 15, - "gradientMode": "opacity", - "spanNulls": true, - "pointSize": 5, - "showPoints": "auto", - "stacking": { - "mode": "none" - }, - "axisPlacement": "auto", - "thresholdsStyle": { - "mode": "off" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "none", - "decimals": 4 - }, - "overrides": [] - }, + "id": 114, "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, "legend": { - "displayMode": "list", - "placement": "bottom", "calcs": [ "lastNotNull", "max", "min" - ] + ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_epoch_avg_return{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "legendFormat": "{{model}}/{{fold}}", "refId": "A" } - ] + ], + "title": "Avg Return per Trade", + "type": "timeseries" }, { - "type": "row", - "title": "Supervised Model Eval", "collapsed": false, "gridPos": { "h": 1, @@ -3240,45 +3108,39 @@ "y": 77 }, "id": 115, - "panels": [] + "panels": [], + "title": "Supervised Model Eval", + "type": "row" }, { - "id": 116, - "type": "timeseries", - "title": "Accuracy & F1", - "description": "Supervised eval accuracy and F1 score", - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 78 - }, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Supervised eval accuracy and F1 score", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisPlacement": "auto", "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, "fillOpacity": 15, "gradientMode": "opacity", - "spanNulls": true, + "lineInterpolation": "smooth", + "lineWidth": 2, "pointSize": 5, "showPoints": "auto", + "spanNulls": true, "stacking": { "mode": "none" }, - "axisPlacement": "auto", "thresholdsStyle": { "mode": "off" } }, + "decimals": 2, "thresholds": { "mode": "absolute", "steps": [ @@ -3288,31 +3150,37 @@ } ] }, - "unit": "percentunit", - "decimals": 2 + "unit": "percentunit" }, "overrides": [] }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 78 + }, + "id": 116, "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, "legend": { - "displayMode": "list", - "placement": "bottom", "calcs": [ "lastNotNull", "max", "min" - ] + ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_eval_accuracy{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "legendFormat": "accuracy {{model}}/{{fold}}", @@ -3321,51 +3189,45 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_eval_f1{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "legendFormat": "F1 {{model}}/{{fold}}", "refId": "B" } - ] + ], + "title": "Accuracy & F1", + "type": "timeseries" }, { - "id": 117, - "type": "timeseries", - "title": "Precision & Recall", - "description": "Supervised eval precision and recall", - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 78 - }, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Supervised eval precision and recall", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisPlacement": "auto", "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, "fillOpacity": 15, "gradientMode": "opacity", - "spanNulls": true, + "lineInterpolation": "smooth", + "lineWidth": 2, "pointSize": 5, "showPoints": "auto", + "spanNulls": true, "stacking": { "mode": "none" }, - "axisPlacement": "auto", "thresholdsStyle": { "mode": "off" } }, + "decimals": 2, "thresholds": { "mode": "absolute", "steps": [ @@ -3375,31 +3237,37 @@ } ] }, - "unit": "percentunit", - "decimals": 2 + "unit": "percentunit" }, "overrides": [] }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 78 + }, + "id": 117, "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, "legend": { - "displayMode": "list", - "placement": "bottom", "calcs": [ "lastNotNull", "max", "min" - ] + ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_eval_precision{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "legendFormat": "precision {{model}}/{{fold}}", @@ -3408,17 +3276,17 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_eval_recall{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "legendFormat": "recall {{model}}/{{fold}}", "refId": "B" } - ] + ], + "title": "Precision & Recall", + "type": "timeseries" }, { - "type": "row", - "title": "Hyperopt Details", "collapsed": false, "gridPos": { "h": 1, @@ -3427,28 +3295,22 @@ "y": 86 }, "id": 118, - "panels": [] + "panels": [], + "title": "Hyperopt Details", + "type": "row" }, { - "id": 119, - "type": "stat", - "title": "Hyperopt Mode", - "description": "1 = hyperopt active, 0 = training only", - "gridPos": { - "h": 5, - "w": 4, - "x": 0, - "y": 87 - }, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "1 = hyperopt active, 0 = training only", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, + "decimals": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -3462,12 +3324,21 @@ } ] }, - "unit": "none", - "decimals": 0 + "unit": "none" }, "overrides": [] }, + "gridPos": { + "h": 5, + "w": 4, + "x": 0, + "y": 87 + }, + "id": 119, "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", "reduceOptions": { "calcs": [ "lastNotNull" @@ -3475,60 +3346,51 @@ "fields": "", "values": false }, - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", "textMode": "value" }, "targets": [ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_hyperopt_mode{model=~\"$model\", job=~\"$source\"}", "legendFormat": "{{model}}", "refId": "A" } - ] + ], + "title": "Hyperopt Mode", + "type": "stat" }, { - "id": 120, - "type": "timeseries", - "title": "Hyperopt Trial Epoch", - "description": "Current epoch within the active trial", - "gridPos": { - "h": 8, - "w": 10, - "x": 4, - "y": 87 - }, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Current epoch within the active trial", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisPlacement": "auto", "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, "fillOpacity": 15, "gradientMode": "opacity", - "spanNulls": true, + "lineInterpolation": "smooth", + "lineWidth": 2, "pointSize": 5, "showPoints": "auto", + "spanNulls": true, "stacking": { "mode": "none" }, - "axisPlacement": "auto", "thresholdsStyle": { "mode": "off" } }, + "decimals": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -3538,75 +3400,75 @@ } ] }, - "unit": "none", - "decimals": 0 + "unit": "none" }, "overrides": [] }, + "gridPos": { + "h": 8, + "w": 10, + "x": 4, + "y": 87 + }, + "id": 120, "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, "legend": { - "displayMode": "list", - "placement": "bottom", "calcs": [ "lastNotNull", "max", "min" - ] + ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_hyperopt_trial_epoch{model=~\"$model\", job=~\"$source\"}", "legendFormat": "{{model}}", "refId": "A" } - ] + ], + "title": "Hyperopt Trial Epoch", + "type": "timeseries" }, { - "id": 121, - "type": "timeseries", - "title": "Hyperopt Failed Trials", - "description": "Cumulative failed trial count", - "gridPos": { - "h": 8, - "w": 10, - "x": 14, - "y": 87 - }, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, + "description": "Cumulative failed trial count", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisPlacement": "auto", "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, "fillOpacity": 15, "gradientMode": "opacity", - "spanNulls": true, + "lineInterpolation": "smooth", + "lineWidth": 2, "pointSize": 5, "showPoints": "auto", + "spanNulls": true, "stacking": { "mode": "none" }, - "axisPlacement": "auto", "thresholdsStyle": { "mode": "off" } }, + "decimals": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -3620,81 +3482,79 @@ } ] }, - "unit": "none", - "decimals": 0 + "unit": "none" }, "overrides": [] }, + "gridPos": { + "h": 8, + "w": 10, + "x": 14, + "y": 87 + }, + "id": 121, "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, "legend": { - "displayMode": "list", - "placement": "bottom", "calcs": [ "lastNotNull", "max", "min" - ] + ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_hyperopt_trials_failed_total{model=~\"$model\", job=~\"$source\"}", "legendFormat": "{{model}}", "refId": "A" } - ] + ], + "title": "Hyperopt Failed Trials", + "type": "timeseries" }, { - "type": "row", - "title": "GPU Utilization", + "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 95 }, - "collapsed": false, "id": 50, - "panels": [] + "panels": [], + "title": "GPU Utilization", + "type": "row" }, { - "id": 51, - "type": "timeseries", - "title": "GPU Utilization %", - "gridPos": { - "h": 7, - "w": 6, - "x": 0, - "y": 96 - }, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 20, - "gradientMode": "opacity", - "showPoints": "never", - "axisLabel": "Util %", - "pointSize": 4 - }, "color": { "mode": "continuous-GrYlRd" }, - "unit": "percent", + "custom": { + "axisLabel": "Util %", + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 4, + "showPoints": "never" + }, "thresholds": { "mode": "absolute", "steps": [ @@ -3711,18 +3571,26 @@ "value": 80 } ] - } + }, + "unit": "percent" }, "overrides": [] }, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 96 + }, + "id": 51, "options": { "legend": { - "displayMode": "table", - "placement": "bottom", "calcs": [ "mean", "max" - ] + ], + "displayMode": "table", + "placement": "bottom" }, "tooltip": { "mode": "multi", @@ -3733,43 +3601,36 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "DCGM_FI_DEV_GPU_UTIL", "legendFormat": "GPU {{gpu}}", "refId": "A" } - ] + ], + "title": "GPU Utilization %", + "type": "timeseries" }, { - "id": 52, - "type": "timeseries", - "title": "VRAM Used / Free (MiB)", - "gridPos": { - "h": 7, - "w": 6, - "x": 6, - "y": 96 - }, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 15, - "showPoints": "never", - "axisLabel": "MiB", - "gradientMode": "opacity", - "pointSize": 4 - }, "color": { "mode": "palette-classic" }, + "custom": { + "axisLabel": "MiB", + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 4, + "showPoints": "never" + }, "unit": "decmbytes" }, "overrides": [ @@ -3786,24 +3647,31 @@ { "id": "custom.lineStyle", "value": { - "fill": "dash", "dash": [ 10, 5 - ] + ], + "fill": "dash" } } ] } ] }, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 96 + }, + "id": 52, "options": { "legend": { - "displayMode": "table", - "placement": "bottom", "calcs": [ "lastNotNull" - ] + ], + "displayMode": "table", + "placement": "bottom" }, "tooltip": { "mode": "multi", @@ -3814,7 +3682,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "DCGM_FI_DEV_FB_USED", "legendFormat": "used GPU {{gpu}}", @@ -3823,44 +3691,36 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "DCGM_FI_DEV_FB_FREE", "legendFormat": "free GPU {{gpu}}", "refId": "B" } - ] + ], + "title": "VRAM Used / Free (MiB)", + "type": "timeseries" }, { - "id": 53, - "type": "timeseries", - "title": "GPU / Memory Temperature", - "gridPos": { - "h": 7, - "w": 4, - "x": 12, - "y": 96 - }, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 10, - "showPoints": "never", - "axisLabel": "Temp", - "gradientMode": "opacity", - "pointSize": 4 - }, "color": { "mode": "palette-classic" }, - "unit": "celsius", + "custom": { + "axisLabel": "Temp", + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 4, + "showPoints": "never" + }, "thresholds": { "mode": "absolute", "steps": [ @@ -3877,10 +3737,18 @@ "value": 85 } ] - } + }, + "unit": "celsius" }, "overrides": [] }, + "gridPos": { + "h": 7, + "w": 4, + "x": 12, + "y": 96 + }, + "id": 53, "options": { "legend": { "displayMode": "list", @@ -3895,7 +3763,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "DCGM_FI_DEV_GPU_TEMP", "legendFormat": "GPU temp {{gpu}}", @@ -3904,47 +3772,47 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "DCGM_FI_DEV_MEMORY_TEMP", "legendFormat": "mem temp {{gpu}}", "refId": "B" } - ] + ], + "title": "GPU / Memory Temperature", + "type": "timeseries" }, { - "id": 54, - "type": "timeseries", - "title": "Power Usage (W)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "Watts", + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 4, + "showPoints": "never" + }, + "unit": "watt" + }, + "overrides": [] + }, "gridPos": { "h": 7, "w": 4, "x": 16, "y": 96 }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 15, - "showPoints": "never", - "axisLabel": "Watts", - "gradientMode": "opacity", - "pointSize": 4 - }, - "color": { - "mode": "palette-classic" - }, - "unit": "watt" - }, - "overrides": [] - }, + "id": 54, "options": { "legend": { "displayMode": "list", @@ -3959,47 +3827,47 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "DCGM_FI_DEV_POWER_USAGE", "legendFormat": "GPU {{gpu}}", "refId": "A" } - ] + ], + "title": "Power Usage (W)", + "type": "timeseries" }, { - "id": 55, - "type": "timeseries", - "title": "Tensor Core & DRAM Activity", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "Activity", + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 4, + "showPoints": "never" + }, + "unit": "percentunit" + }, + "overrides": [] + }, "gridPos": { "h": 7, "w": 4, "x": 20, "y": 96 }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 15, - "showPoints": "never", - "axisLabel": "Activity", - "gradientMode": "opacity", - "pointSize": 4 - }, - "color": { - "mode": "palette-classic" - }, - "unit": "percentunit" - }, - "overrides": [] - }, + "id": 55, "options": { "legend": { "displayMode": "list", @@ -4014,7 +3882,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "DCGM_FI_PROF_PIPE_TENSOR_ACTIVE", "legendFormat": "tensor {{gpu}}", @@ -4023,7 +3891,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "DCGM_FI_PROF_DRAM_ACTIVE", "legendFormat": "DRAM {{gpu}}", @@ -4032,43 +3900,39 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "DCGM_FI_PROF_GR_ENGINE_ACTIVE", "legendFormat": "engine {{gpu}}", "refId": "C" } - ] + ], + "title": "Tensor Core & DRAM Activity", + "type": "timeseries" }, { - "type": "row", - "title": "Argo CI/CD", + "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 103 }, - "collapsed": false, "id": 60, - "panels": [] + "panels": [], + "title": "Argo CI/CD", + "type": "row" }, { - "id": 61, - "type": "stat", - "title": "Workflows by Status", - "gridPos": { - "h": 5, - "w": 6, - "x": 0, - "y": 104 - }, "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { + "color": { + "mode": "thresholds" + }, "thresholds": { "mode": "absolute", "steps": [ @@ -4077,9 +3941,6 @@ "value": null } ] - }, - "color": { - "mode": "thresholds" } }, "overrides": [ @@ -4145,7 +4006,18 @@ } ] }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 104 + }, + "id": 61, "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" @@ -4153,17 +4025,13 @@ "fields": "", "values": false }, - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "textMode": "auto", - "orientation": "horizontal" + "textMode": "auto" }, "targets": [ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "argo_workflows_gauge{status=\"Running\"}", "legendFormat": "Running", @@ -4172,7 +4040,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "argo_workflows_gauge{status=\"Pending\"}", "legendFormat": "Pending", @@ -4181,7 +4049,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "argo_workflows_gauge{status=\"Failed\"}", "legendFormat": "Failed", @@ -4190,7 +4058,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "argo_workflows_gauge{status=\"Succeeded\"}", "legendFormat": "Succeeded", @@ -4199,47 +4067,47 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "argo_workflows_gauge{status=\"Error\"}", "legendFormat": "Error", "refId": "E" } - ] + ], + "title": "Workflows by Status", + "type": "stat" }, { - "id": 62, - "type": "timeseries", - "title": "Workflow Pods", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "Pods", + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 4, + "showPoints": "never" + }, + "unit": "short" + }, + "overrides": [] + }, "gridPos": { "h": 5, "w": 4, "x": 6, "y": 104 }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 20, - "showPoints": "never", - "axisLabel": "Pods", - "gradientMode": "opacity", - "pointSize": 4 - }, - "color": { - "mode": "palette-classic" - }, - "unit": "short" - }, - "overrides": [] - }, + "id": 62, "options": { "legend": { "displayMode": "list", @@ -4254,48 +4122,48 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "argo_workflows_pods_gauge", "legendFormat": "pods", "refId": "A" } - ] + ], + "title": "Workflow Pods", + "type": "timeseries" }, { - "id": 63, - "type": "timeseries", - "title": "Error Rate", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "red", + "mode": "fixed" + }, + "custom": { + "axisLabel": "Errors/s", + "drawStyle": "bars", + "fillOpacity": 50, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 4, + "showPoints": "never" + }, + "unit": "ops" + }, + "overrides": [] + }, "gridPos": { "h": 5, "w": 4, "x": 10, "y": 104 }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "bars", - "lineWidth": 2, - "fillOpacity": 50, - "showPoints": "never", - "axisLabel": "Errors/s", - "lineInterpolation": "smooth", - "gradientMode": "opacity", - "pointSize": 4 - }, - "color": { - "fixedColor": "red", - "mode": "fixed" - }, - "unit": "ops" - }, - "overrides": [] - }, + "id": 63, "options": { "legend": { "displayMode": "list", @@ -4310,47 +4178,47 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "rate(argo_workflows_error_count[$__rate_interval])", "legendFormat": "errors/s", "refId": "A" } - ] + ], + "title": "Error Rate", + "type": "timeseries" }, { - "id": 64, - "type": "timeseries", - "title": "Operation Duration (p50/p95)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "Duration", + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 4, + "showPoints": "never" + }, + "unit": "s" + }, + "overrides": [] + }, "gridPos": { "h": 5, "w": 5, "x": 14, "y": 104 }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 10, - "showPoints": "never", - "axisLabel": "Duration", - "gradientMode": "opacity", - "pointSize": 4 - }, - "color": { - "mode": "palette-classic" - }, - "unit": "s" - }, - "overrides": [] - }, + "id": 64, "options": { "legend": { "displayMode": "list", @@ -4365,7 +4233,7 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.50, rate(argo_workflows_operation_duration_seconds_bucket[$__rate_interval]))", "legendFormat": "p50", @@ -4374,50 +4242,50 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.95, rate(argo_workflows_operation_duration_seconds_bucket[$__rate_interval]))", "legendFormat": "p95", "refId": "B" } - ] + ], + "title": "Operation Duration (p50/p95)", + "type": "timeseries" }, { - "id": 65, - "type": "timeseries", - "title": "Template Trigger Rate", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "Triggers/s", + "drawStyle": "bars", + "fillOpacity": 40, + "gradientMode": "opacity", + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 4, + "showPoints": "never", + "stacking": { + "mode": "normal" + } + }, + "unit": "ops" + }, + "overrides": [] + }, "gridPos": { "h": 5, "w": 5, "x": 19, "y": 104 }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "bars", - "lineWidth": 2, - "fillOpacity": 40, - "showPoints": "never", - "stacking": { - "mode": "normal" - }, - "axisLabel": "Triggers/s", - "lineInterpolation": "smooth", - "gradientMode": "opacity", - "pointSize": 4 - }, - "color": { - "mode": "palette-classic" - }, - "unit": "ops" - }, - "overrides": [] - }, + "id": 65, "options": { "legend": { "displayMode": "list", @@ -4432,14 +4300,163 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "${DS_PROMETHEUS}" }, "expr": "rate(argo_workflows_workflowtemplate_triggered_total[$__rate_interval])", "legendFormat": "triggers/s", "refId": "A" } - ] + ], + "title": "Template Trigger Rate", + "type": "timeseries" } ], - "style": "dark" -} \ No newline at end of file + "refresh": "15s", + "schemaVersion": 39, + "style": "dark", + "tags": [ + "foxhunt", + "training", + "gpu", + "ci" + ], + "templating": { + "list": [ + { + "auto": true, + "auto_count": 30, + "auto_min": "1m", + "current": { + "selected": true, + "text": "auto", + "value": "$__auto_interval_interval" + }, + "label": "Interval", + "name": "interval", + "options": [ + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "15m", + "value": "15m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "1m,5m,15m,30m,1h", + "type": "interval" + }, + { + "current": { + "selected": true, + "text": "Live", + "value": "training-pods" + }, + "description": "Live = running training pods, CI History = completed CI runs (pushgateway), All = both", + "label": "Source", + "name": "source", + "options": [ + { + "selected": true, + "text": "Live", + "value": "training-pods" + }, + { + "selected": false, + "text": "CI History", + "value": ".+_baseline.*" + }, + { + "selected": false, + "text": "All", + "value": "training-pods|.+_baseline.*" + } + ], + "query": "Live : training-pods, CI History : .+_baseline.*, All : training-pods|.+_baseline.*", + "type": "custom" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "label": "Model", + "multi": true, + "name": "model", + "query": "label_values(foxhunt_training_step{job=~\"$source\"}, model)", + "refresh": 2, + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "includeAll": true, + "label": "Fold", + "multi": true, + "name": "fold", + "query": "label_values(foxhunt_training_step{job=~\"$source\"}, fold)", + "refresh": 2, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Training & CI/CD", + "uid": "foxhunt-training", + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "type": "datasource", + "pluginId": "prometheus" + }, + { + "name": "DS_LOKI", + "type": "datasource", + "pluginId": "loki" + }, + { + "name": "DS_TEMPO", + "type": "datasource", + "pluginId": "${DS_TEMPO}" + } + ] +} diff --git a/infra/k8s/monitoring/dashboards/import.sh b/infra/k8s/monitoring/dashboards/import.sh index bf75b5d2b..54fa1672e 100755 --- a/infra/k8s/monitoring/dashboards/import.sh +++ b/infra/k8s/monitoring/dashboards/import.sh @@ -1,12 +1,11 @@ #!/usr/bin/env bash set -euo pipefail -# Import Grafana dashboards with dynamic datasource resolution. +# Import all Grafana dashboards via API (Postgres-backed, no ConfigMaps). # # Dashboards use ${DS_PROMETHEUS}, ${DS_LOKI}, ${DS_TEMPO} variables. -# This script auto-detects datasource UIDs from the Grafana instance, -# resolves variables, and deploys via ConfigMaps (for sidecar provisioning) -# with API fallback for non-provisioned dashboards. +# This script auto-detects datasource UIDs, resolves variables, and +# imports via the Grafana API. Dashboards are stored in Postgres. # # Required environment variables: # GRAFANA_PASS - Password for admin user @@ -17,11 +16,9 @@ set -euo pipefail # GRAFANA_PROM_UID - Override Prometheus datasource UID # GRAFANA_LOKI_UID - Override Loki datasource UID # GRAFANA_TEMPO_UID - Override Tempo datasource UID -# KUBE_NAMESPACE - K8s namespace (default: foxhunt) -# SKIP_CONFIGMAP - Set to 1 to skip ConfigMap deploy (API-only mode) # # Usage: -# GRAFANA_PASS="Welcome01" ./import.sh +# GRAFANA_PASS="xxx" ./import.sh SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" GRAFANA_USER="${GRAFANA_USER:-admin}" @@ -61,7 +58,7 @@ if [[ -z "$PROM_UID" ]]; then exit 1 fi -# --- Resolve datasource variables and strip __inputs --- +# --- Resolve datasource variables --- resolve_dashboard() { local file="$1" @@ -83,15 +80,49 @@ print(raw) " } -# --- Import a single dashboard --- +# --- Ensure folders exist --- + +ensure_folder() { + local title="$1" + local uid + uid=$(curl -sf -u "${GRAFANA_USER}:${GRAFANA_PASS}" "${GRAFANA_URL}/api/folders" 2>/dev/null | \ + python3 -c " +import sys, json +for f in json.load(sys.stdin): + if f['title'] == '${title}': + print(f['uid']) + break +" 2>/dev/null || true) + + if [[ -z "$uid" ]]; then + uid=$(curl -sf -u "${GRAFANA_USER}:${GRAFANA_PASS}" \ + -H "Content-Type: application/json" \ + -d "{\"title\": \"${title}\"}" \ + "${GRAFANA_URL}/api/folders" 2>/dev/null | \ + python3 -c "import sys,json; print(json.load(sys.stdin)['uid'])" 2>/dev/null) + echo " Created folder: ${title} (${uid})" + fi + echo "$uid" +} + +echo "" +echo "Ensuring folders..." +FOLDER_CI=$(ensure_folder "CI") +FOLDER_INFRA=$(ensure_folder "Infrastructure") +FOLDER_OPS=$(ensure_folder "Operations") +FOLDER_TRADING=$(ensure_folder "Trading") +FOLDER_TRAINING=$(ensure_folder "Training") +FOLDER_SERVICES=$(ensure_folder "Services") +FOLDER_PLATFORM=$(ensure_folder "Platform") + +# --- Import a single dashboard to a folder --- import_dashboard() { local file="$1" + local folder_uid="$2" local name name="$(basename "$file" .json)" - echo "Importing ${name}..." - local resolved resolved=$(resolve_dashboard "$file") @@ -99,160 +130,82 @@ import_dashboard() { response=$(curl -s -w "\n%{http_code}" \ -u "${GRAFANA_USER}:${GRAFANA_PASS}" \ -H "Content-Type: application/json" \ - -d "{\"dashboard\": ${resolved}, \"overwrite\": true}" \ + -d "{\"dashboard\": ${resolved}, \"folderUid\": \"${folder_uid}\", \"overwrite\": true}" \ "${GRAFANA_URL}/api/dashboards/db") local http_code http_code=$(echo "$response" | tail -1) - local body - body=$(echo "$response" | sed '$d') if [[ "$http_code" -ge 200 && "$http_code" -lt 300 ]]; then - echo " OK (${http_code})" + echo " OK ${name}" else - echo " FAILED (${http_code}): ${body}" >&2 + local body + body=$(echo "$response" | sed '$d') + echo " FAIL ${name} (${http_code}): ${body}" >&2 return 1 fi } -# --- ConfigMap deployment --- -# Dashboard JSON files are grouped into ConfigMaps that Grafana's sidecar -# watches (label grafana_dashboard=1). This is the primary deploy method. +# --- Dashboard → Folder mapping --- -KUBE_NAMESPACE="${KUBE_NAMESPACE:-foxhunt}" - -# ConfigMap grouping: name -> list of dashboard files -declare -A CONFIGMAP_GROUPS=( - [grafana-dashboards-cockpit]="foxhunt-cockpit.json" - [grafana-dashboards-trading]="foxhunt-trading.json" - [grafana-dashboards-training]="foxhunt-training.json" - [grafana-dashboards-infrastructure]="foxhunt-infrastructure.json" - [grafana-dashboards-observability]="foxhunt-observability.json" - [grafana-dashboards-ci-logs]="foxhunt-ci-logs.json" - [grafana-dashboards-training-logs]="foxhunt-training-logs.json" - [grafana-dashboards-deep-dive]="foxhunt-training-deep-dive.json" +declare -A DASHBOARD_FOLDERS=( + # Original dashboards + [foxhunt-cockpit]="${FOLDER_OPS}" + [foxhunt-trading]="${FOLDER_TRADING}" + [foxhunt-training]="${FOLDER_TRAINING}" + [foxhunt-infrastructure]="${FOLDER_INFRA}" + [foxhunt-observability]="${FOLDER_INFRA}" + [foxhunt-ci-logs]="${FOLDER_CI}" + [foxhunt-training-logs]="${FOLDER_TRAINING}" + [foxhunt-training-deep-dive]="${FOLDER_TRAINING}" + # Application services + [svc-api-gateway]="${FOLDER_SERVICES}" + [svc-trading-service]="${FOLDER_SERVICES}" + [svc-backtesting]="${FOLDER_SERVICES}" + [svc-data-acquisition]="${FOLDER_SERVICES}" + [svc-ml-training]="${FOLDER_SERVICES}" + [svc-trading-agent]="${FOLDER_SERVICES}" + [svc-web-gateway]="${FOLDER_SERVICES}" + # Platform services + [platform-prometheus]="${FOLDER_PLATFORM}" + [platform-grafana]="${FOLDER_PLATFORM}" + [platform-loki]="${FOLDER_PLATFORM}" + [platform-tempo]="${FOLDER_PLATFORM}" + [platform-redis]="${FOLDER_PLATFORM}" + [platform-postgres]="${FOLDER_PLATFORM}" + [platform-argo]="${FOLDER_PLATFORM}" + [platform-questdb]="${FOLDER_PLATFORM}" + [platform-costs]="${FOLDER_PLATFORM}" ) -# Folder mapping: ConfigMap name -> Grafana folder name (used by sidecar annotation) -declare -A CONFIGMAP_FOLDERS=( - [grafana-dashboards-cockpit]="Foxhunt" - [grafana-dashboards-trading]="Foxhunt" - [grafana-dashboards-training]="Foxhunt" - [grafana-dashboards-infrastructure]="Foxhunt" - [grafana-dashboards-observability]="Foxhunt" - [grafana-dashboards-ci-logs]="Foxhunt" - [grafana-dashboards-training-logs]="Foxhunt" - [grafana-dashboards-deep-dive]="Foxhunt" -) +# --- Deploy all dashboards --- -deploy_configmaps() { - echo "" - echo "Deploying dashboard ConfigMaps..." - - local tmp_dir - tmp_dir=$(mktemp -d) - trap "rm -rf $tmp_dir" RETURN - - # Resolve all dashboard files to temp dir - for f in "${SCRIPT_DIR}"/*.json; do - resolve_dashboard "$f" > "${tmp_dir}/$(basename "$f")" - done - - # Build and apply ConfigMaps - python3 -c " -import json, os - -tmp = '${tmp_dir}' -ns = '${KUBE_NAMESPACE}' -groups = ${CONFIGMAP_GROUPS_JSON} -folders = ${CONFIGMAP_FOLDERS_JSON} - -items = [] -for name, files in groups.items(): - cm = { - 'apiVersion': 'v1', - 'kind': 'ConfigMap', - 'metadata': {'name': name, 'namespace': ns, 'labels': {'grafana_dashboard': '1'}, 'annotations': {'grafana_folder': folders.get(name, '')}}, - 'data': {} - } - for f in files: - path = os.path.join(tmp, f) - if os.path.exists(path): - with open(path) as fh: - cm['data'][f] = fh.read() - items.append(cm) - -print(json.dumps({'apiVersion': 'v1', 'kind': 'List', 'items': items})) -" | kubectl apply -f - -} - -# Export groups as JSON for python -CONFIGMAP_GROUPS_JSON='{' -first=true -for name in "${!CONFIGMAP_GROUPS[@]}"; do - $first || CONFIGMAP_GROUPS_JSON+=',' - files_json="[" - ffirst=true - for f in ${CONFIGMAP_GROUPS[${name}]}; do - $ffirst || files_json+="," - files_json+="\"${f}\"" - ffirst=false - done - files_json+="]" - CONFIGMAP_GROUPS_JSON+="\"${name}\":${files_json}" - first=false -done -CONFIGMAP_GROUPS_JSON+='}' -export CONFIGMAP_GROUPS_JSON - -# Export folders as JSON for python -CONFIGMAP_FOLDERS_JSON='{' -first=true -for name in "${!CONFIGMAP_FOLDERS[@]}"; do - $first || CONFIGMAP_FOLDERS_JSON+=',' - CONFIGMAP_FOLDERS_JSON+="\"${name}\":\"${CONFIGMAP_FOLDERS[${name}]}\"" - first=false -done -CONFIGMAP_FOLDERS_JSON+='}' -export CONFIGMAP_FOLDERS_JSON - -# --- Deploy --- +echo "" +echo "Importing dashboards..." failed=0 +total=0 -if [[ "${SKIP_CONFIGMAP:-0}" != "1" ]] && command -v kubectl &>/dev/null; then - deploy_configmaps - echo "" - echo "Restarting Grafana to reload dashboards..." - kubectl rollout restart deployment grafana -n "${KUBE_NAMESPACE}" 2>/dev/null || true - kubectl rollout status deployment grafana -n "${KUBE_NAMESPACE}" --timeout=60s 2>/dev/null || true -else - echo "" - echo "Skipping ConfigMap deploy (kubectl not available or SKIP_CONFIGMAP=1)." - echo "Falling back to API import..." +for dashboard in "${SCRIPT_DIR}"/*.json; do + [[ -f "$dashboard" ]] || continue + name="$(basename "$dashboard" .json)" + folder_uid="${DASHBOARD_FOLDERS[$name]:-}" - for dashboard in "${SCRIPT_DIR}/foxhunt-cockpit.json" \ - "${SCRIPT_DIR}/foxhunt-trading.json" \ - "${SCRIPT_DIR}/foxhunt-training.json" \ - "${SCRIPT_DIR}/foxhunt-infrastructure.json" \ - "${SCRIPT_DIR}/foxhunt-observability.json" \ - "${SCRIPT_DIR}/foxhunt-ci-logs.json" \ - "${SCRIPT_DIR}/foxhunt-training-logs.json" \ - "${SCRIPT_DIR}/foxhunt-training-deep-dive.json"; do - if [[ -f "$dashboard" ]]; then - if ! import_dashboard "$dashboard"; then - failed=$((failed + 1)) - fi - else - echo "WARNING: ${dashboard} not found, skipping" >&2 - fi - done -fi + if [[ -z "$folder_uid" ]]; then + echo " SKIP ${name} (no folder mapping)" + continue + fi + + total=$((total + 1)) + if ! import_dashboard "$dashboard" "$folder_uid"; then + failed=$((failed + 1)) + fi +done echo "" if [[ $failed -eq 0 ]]; then - echo "All dashboards deployed successfully." + echo "All ${total} dashboards deployed successfully." else - echo "ERROR: ${failed} dashboard(s) failed." >&2 + echo "ERROR: ${failed}/${total} dashboard(s) failed." >&2 exit 1 fi diff --git a/infra/k8s/monitoring/dashboards/platform-argo.json b/infra/k8s/monitoring/dashboards/platform-argo.json new file mode 100644 index 000000000..37e4b6918 --- /dev/null +++ b/infra/k8s/monitoring/dashboards/platform-argo.json @@ -0,0 +1,1632 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "Argo Workflows Health", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"argo-workflows.*\", phase=\"Running\"}", + "instant": true, + "legendFormat": "Running", + "range": false, + "refId": "A" + } + ], + "title": "Pod Status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 3 + }, + { + "color": "red", + "value": 10 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 3, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(kube_pod_container_status_restarts_total{namespace=\"foxhunt\", pod=~\"argo-workflows.*\"})", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Restarts", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.7 + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 6, + "y": 1 + }, + "id": 4, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"argo-workflows.*\", container!=\"\"}[$interval]))", + "legendFormat": "", + "refId": "A" + } + ], + "title": "CPU Usage", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.7 + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 9, + "y": 1 + }, + "id": 5, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"argo-workflows.*\", container!=\"\"}) / sum(kube_pod_container_resource_limits{namespace=\"foxhunt\", pod=~\"argo-workflows.*\", resource=\"memory\"})", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Memory", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 12, + "y": 1 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"argo-workflows.*\", container!=\"\"})", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Memory (bytes)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 15, + "y": 1 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"argo-workflows.*\", container!=\"\"}[5m]))", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "CPU Cores", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 18, + "y": 1 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"argo-workflows.*\"}[5m]))", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Network RX", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 21, + "y": 1 + }, + "id": 9, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"argo-workflows.*\"}[5m]))", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Network TX", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 10, + "panels": [], + "title": "Workflow Metrics", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 0, + "y": 6 + }, + "id": 11, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "argo_workflows_gauge{status=\"Running\"}", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Active Workflows", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 4, + "y": 6 + }, + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "argo_workflows_total_count{status=\"Succeeded\"}", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Succeeded", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 8, + "y": 6 + }, + "id": 13, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "argo_workflows_total_count{status=\"Failed\"}", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Failed", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 12, + "y": 6 + }, + "id": 14, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "argo_workflows_pod_pending_count", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Pending Pods", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 16, + "y": 6 + }, + "id": 15, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "argo_workflows_workers_busy_count", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Workers Busy", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 20, + "y": 6 + }, + "id": 16, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "argo_workflows_queue_depth_gauge", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Queue Depth", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 17, + "panels": [], + "title": "Performance", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 12 + }, + "id": 18, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "argo_workflows_gauge{status=\"Running\"}", + "legendFormat": "Running", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "argo_workflows_gauge{status=\"Pending\"}", + "legendFormat": "Pending", + "refId": "B" + } + ], + "title": "Workflow Count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 12 + }, + "id": 19, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(argo_workflows_k8s_request_total[$interval])", + "legendFormat": "{{verb}} {{kind}}", + "refId": "A" + } + ], + "title": "K8s API Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 12 + }, + "id": 20, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.99, rate(argo_workflows_operation_duration_seconds_bucket[$interval]))", + "legendFormat": "p99", + "refId": "A" + } + ], + "title": "Operation Duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 20 + }, + "id": 21, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.99, rate(argo_workflows_queue_latency_bucket[$interval]))", + "legendFormat": "p99", + "refId": "A" + } + ], + "title": "Queue Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 20 + }, + "id": 22, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(argo_workflows_error_count[$interval])", + "legendFormat": "errors/s", + "refId": "A" + } + ], + "title": "Errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 20 + }, + "id": 23, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "argo_workflows_pods_total_count", + "legendFormat": "{{status}}", + "refId": "A" + } + ], + "title": "Pods Total", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 28 + }, + "id": 24, + "panels": [], + "title": "Go Runtime", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 29 + }, + "id": 25, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "argo_workflows_process.runtime.go.goroutines", + "legendFormat": "goroutines", + "refId": "A" + } + ], + "title": "Goroutines", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 29 + }, + "id": 26, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "argo_workflows_process.runtime.go.mem.heap_inuse", + "legendFormat": "in-use", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "argo_workflows_process.runtime.go.mem.heap_alloc", + "legendFormat": "alloc", + "refId": "B" + } + ], + "title": "Heap Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 29 + }, + "id": 27, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(argo_workflows_process.runtime.go.gc.pause_total_ns[$interval])", + "legendFormat": "ns/s", + "refId": "A" + } + ], + "title": "GC Pauses", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 37 + }, + "id": 28, + "panels": [], + "title": "Resources", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 38 + }, + "id": 29, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"argo-workflows.*\", container!=\"\"}[$interval])", + "legendFormat": "{{container}}", + "refId": "A" + } + ], + "title": "CPU Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 38 + }, + "id": 30, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"argo-workflows.*\", container!=\"\"}", + "legendFormat": "{{container}}", + "refId": "A" + } + ], + "title": "Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 38 + }, + "id": 31, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"argo-workflows.*\"}[$interval])", + "legendFormat": "RX {{pod}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"argo-workflows.*\"}[$interval])", + "legendFormat": "TX {{pod}}", + "refId": "B" + } + ], + "title": "Network I/O", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 46 + }, + "id": 32, + "panels": [], + "title": "Logs", + "type": "row" + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 47 + }, + "id": 33, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "expr": "{namespace=\"foxhunt\", pod=~\"argo-workflows.*\"}", + "refId": "A" + } + ], + "title": "Log Stream", + "type": "logs" + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 57 + }, + "id": 34, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "expr": "{namespace=\"foxhunt\", pod=~\"argo-workflows.*\"} |~ \"(?i)(error|warn|panic|fatal)\"", + "refId": "A" + } + ], + "title": "Errors & Warnings", + "type": "logs" + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": [ + "platform", + "argo", + "ci" + ], + "templating": { + "list": [ + { + "auto": false, + "current": { + "text": "5m", + "value": "5m" + }, + "name": "interval", + "options": [ + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "15m", + "value": "15m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "1m,5m,15m,30m,1h", + "type": "interval" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "browser", + "title": "Argo Workflows", + "uid": "plat-argo" +} \ No newline at end of file diff --git a/infra/k8s/monitoring/dashboards/platform-costs.json b/infra/k8s/monitoring/dashboards/platform-costs.json new file mode 100644 index 000000000..c072ab13b --- /dev/null +++ b/infra/k8s/monitoring/dashboards/platform-costs.json @@ -0,0 +1,1350 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "Cluster Overview", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "count(kube_node_info)", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Total Nodes", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 3, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "count(kube_pod_info{namespace=\"foxhunt\"})", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Total Pods", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 6, + "y": 1 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "count(kube_pod_status_phase{namespace=\"foxhunt\", phase=\"Running\"})", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Running Pods", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 9, + "y": 1 + }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(kube_node_status_capacity{resource=\"cpu\"})", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "CPU Cores (Total)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 12, + "y": 1 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(kube_node_status_capacity{resource=\"memory\"})", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Memory (Total)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 15, + "y": 1 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(kube_pod_container_resource_requests{namespace=\"foxhunt\",resource=\"cpu\"}) / sum(kube_node_status_allocatable{resource=\"cpu\"})", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "CPU Requested %", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 18, + "y": 1 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(kube_pod_container_resource_requests{namespace=\"foxhunt\",resource=\"memory\"}) / sum(kube_node_status_allocatable{resource=\"memory\"})", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Memory Requested %", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 21, + "y": 1 + }, + "id": 9, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "count(kube_persistentvolumeclaim_info{namespace=\"foxhunt\"})", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "PVCs", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 10, + "panels": [], + "title": "Scaleway Cost Estimates", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 0, + "y": 6 + }, + "id": 11, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "count(kube_node_info{node=~\".*platform.*\"})", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Platform Nodes", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "currencyEUR" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 4, + "y": 6 + }, + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "count(kube_node_info{node=~\".*platform.*\"}) * 0.569", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Platform Cost (\u20ac/hr)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "currencyEUR" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 8, + "y": 6 + }, + "id": 13, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "count(kube_node_info{node=~\".*platform.*\"}) * 0.569 * 730", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Platform Cost (\u20ac/month)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 12, + "y": 6 + }, + "id": 14, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "count(kube_node_info{node=~\".*h100.*\"}) or vector(0)", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "GPU Nodes", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "currencyEUR" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 16, + "y": 6 + }, + "id": 15, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "(count(kube_node_info{node=~\".*h100.*\"}) or vector(0)) * 5.08", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "GPU Cost (\u20ac/hr)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 20, + "y": 6 + }, + "id": 16, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "count(kube_node_info{node=~\".*compile.*\"}) or vector(0)", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "CI CPU Nodes", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 17, + "panels": [], + "title": "Total Estimated Costs", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "normal" + } + }, + "unit": "currencyEUR" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 18, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "count(kube_node_info{node=~\".*platform.*\"}) * 0.569", + "legendFormat": "Platform", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "(count(kube_node_info{node=~\".*h100.*\"}) or vector(0)) * 5.08", + "legendFormat": "GPU (H100)", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "(count(kube_node_info{node=~\".*compile.*\"}) or vector(0)) * 0.569", + "legendFormat": "CI Compile", + "refId": "C" + } + ], + "title": "Hourly Compute Cost (\u20ac)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "normal" + } + }, + "unit": "currencyEUR" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 19, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "count(kube_node_info{node=~\".*platform.*\"}) * 0.569 * 730", + "legendFormat": "Platform", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "(count(kube_node_info{node=~\".*h100.*\"}) or vector(0)) * 5.08 * 730", + "legendFormat": "GPU (H100)", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "(count(kube_node_info{node=~\".*compile.*\"}) or vector(0)) * 0.569 * 730", + "legendFormat": "CI Compile", + "refId": "C" + } + ], + "title": "Projected Monthly Cost (\u20ac)", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 20, + "panels": [], + "title": "Node Resource Usage", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 21, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "1 - avg by (node) (rate(node_cpu_seconds_total{mode=\"idle\"}[$interval]))", + "legendFormat": "{{node}}", + "refId": "A" + } + ], + "title": "Node CPU Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 21 + }, + "id": 22, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "Node Memory Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 29 + }, + "id": 23, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "1 - node_filesystem_avail_bytes{mountpoint=\"/\"} / node_filesystem_size_bytes{mountpoint=\"/\"}", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "Node Disk Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 29 + }, + "id": 24, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(node_network_receive_bytes_total{device!=\"lo\"}[$interval])", + "legendFormat": "RX {{instance}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(node_network_transmit_bytes_total{device!=\"lo\"}[$interval])", + "legendFormat": "TX {{instance}}", + "refId": "B" + } + ], + "title": "Node Network", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 37 + }, + "id": 25, + "panels": [], + "title": "Namespace Resource Breakdown", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 38 + }, + "id": 26, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", container!=\"\"}[$interval]))", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "title": "CPU by Service", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 38 + }, + "id": 27, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (pod) (container_memory_working_set_bytes{namespace=\"foxhunt\", container!=\"\"})", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "title": "Memory by Service", + "type": "timeseries" + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": [ + "platform", + "costs", + "scaleway" + ], + "templating": { + "list": [ + { + "auto": false, + "current": { + "text": "5m", + "value": "5m" + }, + "name": "interval", + "options": [ + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "15m", + "value": "15m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "1m,5m,15m,30m,1h", + "type": "interval" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "browser", + "title": "Cluster & Costs", + "uid": "plat-costs" +} \ No newline at end of file diff --git a/infra/k8s/monitoring/dashboards/platform-grafana.json b/infra/k8s/monitoring/dashboards/platform-grafana.json new file mode 100644 index 000000000..6de38cfcb --- /dev/null +++ b/infra/k8s/monitoring/dashboards/platform-grafana.json @@ -0,0 +1,802 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "Grafana Health", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"grafana.*\", phase=\"Running\"}", + "instant": true, + "legendFormat": "Running", + "range": false, + "refId": "A" + } + ], + "title": "Pod Status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 3 + }, + { + "color": "red", + "value": 10 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 3, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(kube_pod_container_status_restarts_total{namespace=\"foxhunt\", pod=~\"grafana.*\"})", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Restarts", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.7 + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 6, + "y": 1 + }, + "id": 4, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"grafana.*\", container!=\"\"}[$interval]))", + "legendFormat": "", + "refId": "A" + } + ], + "title": "CPU Usage", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.7 + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 9, + "y": 1 + }, + "id": 5, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"grafana.*\", container!=\"\"}) / sum(kube_pod_container_resource_limits{namespace=\"foxhunt\", pod=~\"grafana.*\", resource=\"memory\"})", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Memory", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 12, + "y": 1 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"grafana.*\", container!=\"\"})", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Memory (bytes)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 15, + "y": 1 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"grafana.*\", container!=\"\"}[5m]))", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "CPU Cores", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 18, + "y": 1 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"grafana.*\"}[5m]))", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Network RX", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 21, + "y": 1 + }, + "id": 9, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"grafana.*\"}[5m]))", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Network TX", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 10, + "panels": [], + "title": "Resources", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 6 + }, + "id": 11, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"grafana.*\", container!=\"\"}[$interval])", + "legendFormat": "{{container}}", + "refId": "A" + } + ], + "title": "CPU Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 6 + }, + "id": 12, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"grafana.*\", container!=\"\"}", + "legendFormat": "{{container}}", + "refId": "A" + } + ], + "title": "Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 6 + }, + "id": 13, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"grafana.*\"}[$interval])", + "legendFormat": "RX {{pod}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"grafana.*\"}[$interval])", + "legendFormat": "TX {{pod}}", + "refId": "B" + } + ], + "title": "Network I/O", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 14, + "panels": [], + "title": "Logs", + "type": "row" + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 15, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "expr": "{namespace=\"foxhunt\", pod=~\"grafana.*\"}", + "refId": "A" + } + ], + "title": "Log Stream", + "type": "logs" + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 16, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "expr": "{namespace=\"foxhunt\", pod=~\"grafana.*\"} |~ \"(?i)(error|warn|panic|fatal)\"", + "refId": "A" + } + ], + "title": "Errors & Warnings", + "type": "logs" + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": [ + "platform", + "grafana" + ], + "templating": { + "list": [ + { + "auto": false, + "current": { + "text": "5m", + "value": "5m" + }, + "name": "interval", + "options": [ + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "15m", + "value": "15m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "1m,5m,15m,30m,1h", + "type": "interval" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "browser", + "title": "Grafana", + "uid": "plat-grafana" +} \ No newline at end of file diff --git a/infra/k8s/monitoring/dashboards/platform-loki.json b/infra/k8s/monitoring/dashboards/platform-loki.json new file mode 100644 index 000000000..4a536d7bd --- /dev/null +++ b/infra/k8s/monitoring/dashboards/platform-loki.json @@ -0,0 +1,1532 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "Loki Health", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"loki.*\", phase=\"Running\"}", + "instant": true, + "legendFormat": "Running", + "range": false, + "refId": "A" + } + ], + "title": "Pod Status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 3 + }, + { + "color": "red", + "value": 10 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 3, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(kube_pod_container_status_restarts_total{namespace=\"foxhunt\", pod=~\"loki.*\"})", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Restarts", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.7 + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 6, + "y": 1 + }, + "id": 4, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"loki.*\", container!=\"\"}[$interval]))", + "legendFormat": "", + "refId": "A" + } + ], + "title": "CPU Usage", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.7 + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 9, + "y": 1 + }, + "id": 5, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"loki.*\", container!=\"\"}) / sum(kube_pod_container_resource_limits{namespace=\"foxhunt\", pod=~\"loki.*\", resource=\"memory\"})", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Memory", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 12, + "y": 1 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"loki.*\", container!=\"\"})", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Memory (bytes)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 15, + "y": 1 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"loki.*\", container!=\"\"}[5m]))", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "CPU Cores", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 18, + "y": 1 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"loki.*\"}[5m]))", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Network RX", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 21, + "y": 1 + }, + "id": 9, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"loki.*\"}[5m]))", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Network TX", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 10, + "panels": [], + "title": "Ingestion", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 6 + }, + "id": 11, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(loki_distributor_bytes_received_total[$interval])", + "legendFormat": "bytes/s", + "refId": "A" + } + ], + "title": "Log Ingestion Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 6 + }, + "id": 12, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "loki_ingester_streams_created_total - loki_ingester_streams_removed_total", + "legendFormat": "streams", + "refId": "A" + } + ], + "title": "Active Streams", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 6 + }, + "id": 13, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(loki_ingester_chunk_stored_bytes_total[$interval])", + "legendFormat": "bytes/s", + "refId": "A" + } + ], + "title": "Chunk Flushes", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 14, + "panels": [], + "title": "Promtail", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 15 + }, + "id": 15, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"promtail.*\", phase=\"Running\"}", + "instant": true, + "legendFormat": "Running", + "range": false, + "refId": "A" + } + ], + "title": "Pod Status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 3 + }, + { + "color": "red", + "value": 10 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 3, + "y": 15 + }, + "id": 16, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(kube_pod_container_status_restarts_total{namespace=\"foxhunt\", pod=~\"promtail.*\"})", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Restarts", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.7 + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 6, + "y": 15 + }, + "id": 17, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"promtail.*\", container!=\"\"}[$interval]))", + "legendFormat": "", + "refId": "A" + } + ], + "title": "CPU Usage", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.7 + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 9, + "y": 15 + }, + "id": 18, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"promtail.*\", container!=\"\"}) / sum(kube_pod_container_resource_limits{namespace=\"foxhunt\", pod=~\"promtail.*\", resource=\"memory\"})", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Memory", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 12, + "y": 15 + }, + "id": 19, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"promtail.*\", container!=\"\"})", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Memory (bytes)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 15, + "y": 15 + }, + "id": 20, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"promtail.*\", container!=\"\"}[5m]))", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "CPU Cores", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 18, + "y": 15 + }, + "id": 21, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"promtail.*\"}[5m]))", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Network RX", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 21, + "y": 15 + }, + "id": 22, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"promtail.*\"}[5m]))", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Network TX", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 19 + }, + "id": 23, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(promtail_read_bytes_total[$interval])", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "title": "Promtail Read Bytes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 19 + }, + "id": 24, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "promtail_targets_active_total", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "title": "Promtail Targets", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 25, + "panels": [], + "title": "Resources", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 28 + }, + "id": 26, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"loki.*\", container!=\"\"}[$interval])", + "legendFormat": "{{container}}", + "refId": "A" + } + ], + "title": "CPU Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 28 + }, + "id": 27, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"loki.*\", container!=\"\"}", + "legendFormat": "{{container}}", + "refId": "A" + } + ], + "title": "Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 28 + }, + "id": 28, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"loki.*\"}[$interval])", + "legendFormat": "RX {{pod}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"loki.*\"}[$interval])", + "legendFormat": "TX {{pod}}", + "refId": "B" + } + ], + "title": "Network I/O", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 36 + }, + "id": 29, + "panels": [], + "title": "Logs", + "type": "row" + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 37 + }, + "id": 30, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "expr": "{namespace=\"foxhunt\", pod=~\"loki.*\"}", + "refId": "A" + } + ], + "title": "Log Stream", + "type": "logs" + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 47 + }, + "id": 31, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "expr": "{namespace=\"foxhunt\", pod=~\"loki.*\"} |~ \"(?i)(error|warn|panic|fatal)\"", + "refId": "A" + } + ], + "title": "Errors & Warnings", + "type": "logs" + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": [ + "platform", + "loki" + ], + "templating": { + "list": [ + { + "auto": false, + "current": { + "text": "5m", + "value": "5m" + }, + "name": "interval", + "options": [ + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "15m", + "value": "15m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "1m,5m,15m,30m,1h", + "type": "interval" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "browser", + "title": "Loki & Promtail", + "uid": "plat-loki" +} \ No newline at end of file diff --git a/infra/k8s/monitoring/dashboards/platform-postgres.json b/infra/k8s/monitoring/dashboards/platform-postgres.json new file mode 100644 index 000000000..c1284e596 --- /dev/null +++ b/infra/k8s/monitoring/dashboards/platform-postgres.json @@ -0,0 +1,1146 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "PostgreSQL Health", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"postgres.*\", phase=\"Running\"}", + "instant": true, + "legendFormat": "Running", + "range": false, + "refId": "A" + } + ], + "title": "Pod Status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 3 + }, + { + "color": "red", + "value": 10 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 3, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(kube_pod_container_status_restarts_total{namespace=\"foxhunt\", pod=~\"postgres.*\"})", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Restarts", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.7 + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 6, + "y": 1 + }, + "id": 4, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"postgres.*\", container!=\"\"}[$interval]))", + "legendFormat": "", + "refId": "A" + } + ], + "title": "CPU Usage", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.7 + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 9, + "y": 1 + }, + "id": 5, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"postgres.*\", container!=\"\"}) / sum(kube_pod_container_resource_limits{namespace=\"foxhunt\", pod=~\"postgres.*\", resource=\"memory\"})", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Memory", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 12, + "y": 1 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"postgres.*\", container!=\"\"})", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Memory (bytes)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 15, + "y": 1 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"postgres.*\", container!=\"\"}[5m]))", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "CPU Cores", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 18, + "y": 1 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"postgres.*\"}[5m]))", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Network RX", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 21, + "y": 1 + }, + "id": 9, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"postgres.*\"}[5m]))", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Network TX", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 10, + "panels": [], + "title": "Database Metrics", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 0, + "y": 6 + }, + "id": 11, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "pg_stat_activity_count{namespace=\"foxhunt\", state=\"active\"}", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Active Connections", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 4, + "y": 6 + }, + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "pg_database_size_bytes{namespace=\"foxhunt\"}", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "DB Size", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 8, + "y": 6 + }, + "id": 13, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "pg_stat_database_blks_hit{namespace=\"foxhunt\"} / (pg_stat_database_blks_hit{namespace=\"foxhunt\"} + pg_stat_database_blks_read{namespace=\"foxhunt\"})", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Cache Hit Ratio", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 12, + "y": 6 + }, + "id": 14, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(pg_stat_database_xact_commit{namespace=\"foxhunt\"}[5m])", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Transactions/s", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 16, + "y": 6 + }, + "id": 15, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "pg_stat_database_deadlocks{namespace=\"foxhunt\"}", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Deadlocks", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 20, + "y": 6 + }, + "id": 16, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "pg_stat_database_temp_bytes{namespace=\"foxhunt\"}", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Temp Files", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 17, + "panels": [], + "title": "Resources", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 12 + }, + "id": 18, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"postgres.*\", container!=\"\"}[$interval])", + "legendFormat": "{{container}}", + "refId": "A" + } + ], + "title": "CPU Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 12 + }, + "id": 19, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"postgres.*\", container!=\"\"}", + "legendFormat": "{{container}}", + "refId": "A" + } + ], + "title": "Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 12 + }, + "id": 20, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"postgres.*\"}[$interval])", + "legendFormat": "RX {{pod}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"postgres.*\"}[$interval])", + "legendFormat": "TX {{pod}}", + "refId": "B" + } + ], + "title": "Network I/O", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 21, + "panels": [], + "title": "Logs", + "type": "row" + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 22, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "expr": "{namespace=\"foxhunt\", pod=~\"postgres.*\"}", + "refId": "A" + } + ], + "title": "Log Stream", + "type": "logs" + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 23, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "expr": "{namespace=\"foxhunt\", pod=~\"postgres.*\"} |~ \"(?i)(error|warn|panic|fatal)\"", + "refId": "A" + } + ], + "title": "Errors & Warnings", + "type": "logs" + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": [ + "platform", + "postgres" + ], + "templating": { + "list": [ + { + "auto": false, + "current": { + "text": "5m", + "value": "5m" + }, + "name": "interval", + "options": [ + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "15m", + "value": "15m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "1m,5m,15m,30m,1h", + "type": "interval" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "browser", + "title": "PostgreSQL", + "uid": "plat-postgres" +} \ No newline at end of file diff --git a/infra/k8s/monitoring/dashboards/platform-prometheus.json b/infra/k8s/monitoring/dashboards/platform-prometheus.json new file mode 100644 index 000000000..981977072 --- /dev/null +++ b/infra/k8s/monitoring/dashboards/platform-prometheus.json @@ -0,0 +1,946 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "Prometheus Health", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "count(up{job!=\"\"} == 1)", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Targets Up", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 3, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "count(up{job!=\"\"} == 0) or vector(0)", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Targets Down", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 6, + "y": 1 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "prometheus_tsdb_head_series", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Head Series", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 9, + "y": 1 + }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(prometheus_tsdb_head_samples_appended_total[5m])", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Ingestion Rate", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 12, + "y": 1 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "prometheus_tsdb_wal_storage_size_bytes", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "WAL Size", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 15, + "y": 1 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "prometheus_tsdb_head_chunks", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Head Chunks", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 18, + "y": 1 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "prometheus_tsdb_compactions_total", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Compactions", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 21, + "y": 1 + }, + "id": 9, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "prometheus_tsdb_storage_blocks_bytes", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Storage Size", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 10, + "panels": [], + "title": "Resources", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 6 + }, + "id": 11, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"prometheus.*\", container!=\"\"}[$interval])", + "legendFormat": "{{container}}", + "refId": "A" + } + ], + "title": "CPU Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 6 + }, + "id": 12, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"prometheus.*\", container!=\"\"}", + "legendFormat": "{{container}}", + "refId": "A" + } + ], + "title": "Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 6 + }, + "id": 13, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"prometheus.*\"}[$interval])", + "legendFormat": "RX {{pod}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"prometheus.*\"}[$interval])", + "legendFormat": "TX {{pod}}", + "refId": "B" + } + ], + "title": "Network I/O", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 14, + "panels": [], + "title": "Scrape Performance", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 15 + }, + "id": 15, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "scrape_duration_seconds", + "legendFormat": "{{job}}", + "refId": "A" + } + ], + "title": "Scrape Duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 15 + }, + "id": 16, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(prometheus_tsdb_head_samples_appended_total[$interval])", + "legendFormat": "samples/s", + "refId": "A" + } + ], + "title": "Samples Ingested", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 17, + "panels": [], + "title": "Target Status", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 18, + "options": { + "showHeader": true, + "sortBy": [] + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "up", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Target Health", + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 19, + "panels": [], + "title": "Logs", + "type": "row" + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 33 + }, + "id": 20, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "expr": "{namespace=\"foxhunt\", pod=~\"prometheus.*\"}", + "refId": "A" + } + ], + "title": "Log Stream", + "type": "logs" + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 43 + }, + "id": 21, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "expr": "{namespace=\"foxhunt\", pod=~\"prometheus.*\"} |~ \"(?i)(error|warn|panic|fatal)\"", + "refId": "A" + } + ], + "title": "Errors & Warnings", + "type": "logs" + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": [ + "platform", + "prometheus" + ], + "templating": { + "list": [ + { + "auto": false, + "current": { + "text": "5m", + "value": "5m" + }, + "name": "interval", + "options": [ + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "15m", + "value": "15m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "1m,5m,15m,30m,1h", + "type": "interval" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "browser", + "title": "Prometheus", + "uid": "plat-prometheus" +} \ No newline at end of file diff --git a/infra/k8s/monitoring/dashboards/platform-questdb.json b/infra/k8s/monitoring/dashboards/platform-questdb.json new file mode 100644 index 000000000..9d9a0a617 --- /dev/null +++ b/infra/k8s/monitoring/dashboards/platform-questdb.json @@ -0,0 +1,802 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "QuestDB Health", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"questdb.*\", phase=\"Running\"}", + "instant": true, + "legendFormat": "Running", + "range": false, + "refId": "A" + } + ], + "title": "Pod Status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 3 + }, + { + "color": "red", + "value": 10 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 3, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(kube_pod_container_status_restarts_total{namespace=\"foxhunt\", pod=~\"questdb.*\"})", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Restarts", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.7 + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 6, + "y": 1 + }, + "id": 4, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"questdb.*\", container!=\"\"}[$interval]))", + "legendFormat": "", + "refId": "A" + } + ], + "title": "CPU Usage", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.7 + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 9, + "y": 1 + }, + "id": 5, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"questdb.*\", container!=\"\"}) / sum(kube_pod_container_resource_limits{namespace=\"foxhunt\", pod=~\"questdb.*\", resource=\"memory\"})", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Memory", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 12, + "y": 1 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"questdb.*\", container!=\"\"})", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Memory (bytes)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 15, + "y": 1 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"questdb.*\", container!=\"\"}[5m]))", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "CPU Cores", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 18, + "y": 1 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"questdb.*\"}[5m]))", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Network RX", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 21, + "y": 1 + }, + "id": 9, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"questdb.*\"}[5m]))", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Network TX", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 10, + "panels": [], + "title": "Resources", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 6 + }, + "id": 11, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"questdb.*\", container!=\"\"}[$interval])", + "legendFormat": "{{container}}", + "refId": "A" + } + ], + "title": "CPU Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 6 + }, + "id": 12, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"questdb.*\", container!=\"\"}", + "legendFormat": "{{container}}", + "refId": "A" + } + ], + "title": "Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 6 + }, + "id": 13, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"questdb.*\"}[$interval])", + "legendFormat": "RX {{pod}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"questdb.*\"}[$interval])", + "legendFormat": "TX {{pod}}", + "refId": "B" + } + ], + "title": "Network I/O", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 14, + "panels": [], + "title": "Logs", + "type": "row" + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 15, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "expr": "{namespace=\"foxhunt\", pod=~\"questdb.*\"}", + "refId": "A" + } + ], + "title": "Log Stream", + "type": "logs" + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 16, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "expr": "{namespace=\"foxhunt\", pod=~\"questdb.*\"} |~ \"(?i)(error|warn|panic|fatal)\"", + "refId": "A" + } + ], + "title": "Errors & Warnings", + "type": "logs" + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": [ + "platform", + "questdb" + ], + "templating": { + "list": [ + { + "auto": false, + "current": { + "text": "5m", + "value": "5m" + }, + "name": "interval", + "options": [ + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "15m", + "value": "15m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "1m,5m,15m,30m,1h", + "type": "interval" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "browser", + "title": "QuestDB", + "uid": "plat-questdb" +} \ No newline at end of file diff --git a/infra/k8s/monitoring/dashboards/platform-redis.json b/infra/k8s/monitoring/dashboards/platform-redis.json new file mode 100644 index 000000000..a638e7c94 --- /dev/null +++ b/infra/k8s/monitoring/dashboards/platform-redis.json @@ -0,0 +1,1315 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "Redis Health", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"redis.*\", phase=\"Running\"}", + "instant": true, + "legendFormat": "Running", + "range": false, + "refId": "A" + } + ], + "title": "Pod Status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 3 + }, + { + "color": "red", + "value": 10 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 3, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(kube_pod_container_status_restarts_total{namespace=\"foxhunt\", pod=~\"redis.*\"})", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Restarts", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.7 + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 6, + "y": 1 + }, + "id": 4, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"redis.*\", container!=\"\"}[$interval]))", + "legendFormat": "", + "refId": "A" + } + ], + "title": "CPU Usage", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.7 + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 9, + "y": 1 + }, + "id": 5, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"redis.*\", container!=\"\"}) / sum(kube_pod_container_resource_limits{namespace=\"foxhunt\", pod=~\"redis.*\", resource=\"memory\"})", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Memory", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 12, + "y": 1 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"redis.*\", container!=\"\"})", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Memory (bytes)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 15, + "y": 1 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"redis.*\", container!=\"\"}[5m]))", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "CPU Cores", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 18, + "y": 1 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"redis.*\"}[5m]))", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Network RX", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 21, + "y": 1 + }, + "id": 9, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"redis.*\"}[5m]))", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Network TX", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 10, + "panels": [], + "title": "Redis Metrics", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 0, + "y": 6 + }, + "id": 11, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "redis_connected_clients{namespace=\"foxhunt\"}", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Connected Clients", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 4, + "y": 6 + }, + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "redis_memory_used_bytes{namespace=\"foxhunt\"}", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Used Memory", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 8, + "y": 6 + }, + "id": 13, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(redis_commands_processed_total{namespace=\"foxhunt\"}[5m])", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Ops/sec", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 12, + "y": 6 + }, + "id": 14, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "redis_keyspace_hits_total{namespace=\"foxhunt\"} / (redis_keyspace_hits_total{namespace=\"foxhunt\"} + redis_keyspace_misses_total{namespace=\"foxhunt\"})", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Hit Ratio", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 16, + "y": 6 + }, + "id": 15, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "redis_db_keys{namespace=\"foxhunt\"}", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Keys", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 20, + "y": 6 + }, + "id": 16, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "redis_evicted_keys_total{namespace=\"foxhunt\"}", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Evictions", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 17, + "panels": [], + "title": "Performance", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 12 + }, + "id": 18, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "redis_memory_used_bytes{namespace=\"foxhunt\"}", + "legendFormat": "used", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "redis_memory_max_bytes{namespace=\"foxhunt\"}", + "legendFormat": "max", + "refId": "B" + } + ], + "title": "Memory Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 12 + }, + "id": 19, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(redis_commands_processed_total{namespace=\"foxhunt\"}[$interval])", + "legendFormat": "ops/s", + "refId": "A" + } + ], + "title": "Command Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 12 + }, + "id": 20, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "redis_connected_clients{namespace=\"foxhunt\"}", + "legendFormat": "clients", + "refId": "A" + } + ], + "title": "Connections", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 21, + "panels": [], + "title": "Resources", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 21 + }, + "id": 22, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"redis.*\", container!=\"\"}[$interval])", + "legendFormat": "{{container}}", + "refId": "A" + } + ], + "title": "CPU Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 21 + }, + "id": 23, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"redis.*\", container!=\"\"}", + "legendFormat": "{{container}}", + "refId": "A" + } + ], + "title": "Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 21 + }, + "id": 24, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"redis.*\"}[$interval])", + "legendFormat": "RX {{pod}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"redis.*\"}[$interval])", + "legendFormat": "TX {{pod}}", + "refId": "B" + } + ], + "title": "Network I/O", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 25, + "panels": [], + "title": "Logs", + "type": "row" + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 26, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "expr": "{namespace=\"foxhunt\", pod=~\"redis.*\"}", + "refId": "A" + } + ], + "title": "Log Stream", + "type": "logs" + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 27, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "expr": "{namespace=\"foxhunt\", pod=~\"redis.*\"} |~ \"(?i)(error|warn|panic|fatal)\"", + "refId": "A" + } + ], + "title": "Errors & Warnings", + "type": "logs" + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": [ + "platform", + "redis" + ], + "templating": { + "list": [ + { + "auto": false, + "current": { + "text": "5m", + "value": "5m" + }, + "name": "interval", + "options": [ + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "15m", + "value": "15m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "1m,5m,15m,30m,1h", + "type": "interval" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "browser", + "title": "Redis", + "uid": "plat-redis" +} \ No newline at end of file diff --git a/infra/k8s/monitoring/dashboards/platform-tempo.json b/infra/k8s/monitoring/dashboards/platform-tempo.json new file mode 100644 index 000000000..aeaa0b2fb --- /dev/null +++ b/infra/k8s/monitoring/dashboards/platform-tempo.json @@ -0,0 +1,802 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "Tempo Health", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\", phase=\"Running\"}", + "instant": true, + "legendFormat": "Running", + "range": false, + "refId": "A" + } + ], + "title": "Pod Status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 3 + }, + { + "color": "red", + "value": 10 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 3, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(kube_pod_container_status_restarts_total{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\"})", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Restarts", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.7 + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 6, + "y": 1 + }, + "id": 4, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\", container!=\"\"}[$interval]))", + "legendFormat": "", + "refId": "A" + } + ], + "title": "CPU Usage", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.7 + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 9, + "y": 1 + }, + "id": 5, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\", container!=\"\"}) / sum(kube_pod_container_resource_limits{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\", resource=\"memory\"})", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Memory", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 12, + "y": 1 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\", container!=\"\"})", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Memory (bytes)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 15, + "y": 1 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\", container!=\"\"}[5m]))", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "CPU Cores", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 18, + "y": 1 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\"}[5m]))", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Network RX", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 21, + "y": 1 + }, + "id": 9, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\"}[5m]))", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Network TX", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 10, + "panels": [], + "title": "Resources", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 6 + }, + "id": 11, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\", container!=\"\"}[$interval])", + "legendFormat": "{{container}}", + "refId": "A" + } + ], + "title": "CPU Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 6 + }, + "id": 12, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\", container!=\"\"}", + "legendFormat": "{{container}}", + "refId": "A" + } + ], + "title": "Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 6 + }, + "id": 13, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\"}[$interval])", + "legendFormat": "RX {{pod}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\"}[$interval])", + "legendFormat": "TX {{pod}}", + "refId": "B" + } + ], + "title": "Network I/O", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 14, + "panels": [], + "title": "Logs", + "type": "row" + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 15, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "expr": "{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\"}", + "refId": "A" + } + ], + "title": "Log Stream", + "type": "logs" + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 16, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "expr": "{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\"} |~ \"(?i)(error|warn|panic|fatal)\"", + "refId": "A" + } + ], + "title": "Errors & Warnings", + "type": "logs" + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": [ + "platform", + "${DS_TEMPO}" + ], + "templating": { + "list": [ + { + "auto": false, + "current": { + "text": "5m", + "value": "5m" + }, + "name": "interval", + "options": [ + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "15m", + "value": "15m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "1m,5m,15m,30m,1h", + "type": "interval" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "browser", + "title": "Tempo", + "uid": "plat-${DS_TEMPO}" +} \ No newline at end of file diff --git a/infra/k8s/monitoring/dashboards/svc-api-gateway.json b/infra/k8s/monitoring/dashboards/svc-api-gateway.json new file mode 100644 index 000000000..b5fc824c5 --- /dev/null +++ b/infra/k8s/monitoring/dashboards/svc-api-gateway.json @@ -0,0 +1,963 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "Service Health", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"api.*\", phase=\"Running\"}", + "instant": true, + "legendFormat": "Running", + "range": false, + "refId": "A" + } + ], + "title": "Pod Status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 3 + }, + { + "color": "red", + "value": 10 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 3, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(kube_pod_container_status_restarts_total{namespace=\"foxhunt\", pod=~\"api.*\"})", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Restarts", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.7 + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 6, + "y": 1 + }, + "id": 4, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"api.*\", container!=\"\"}[$interval]))", + "legendFormat": "", + "refId": "A" + } + ], + "title": "CPU Usage", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.7 + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 9, + "y": 1 + }, + "id": 5, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"api.*\", container!=\"\"}) / sum(kube_pod_container_resource_limits{namespace=\"foxhunt\", pod=~\"api.*\", resource=\"memory\"})", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Memory", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 12, + "y": 1 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"api.*\", container!=\"\"})", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Memory (bytes)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 15, + "y": 1 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"api.*\", container!=\"\"}[5m]))", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "CPU Cores", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 18, + "y": 1 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"api.*\"}[5m]))", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Network RX", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 21, + "y": 1 + }, + "id": 9, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"api.*\"}[5m]))", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Network TX", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 10, + "panels": [], + "title": "Resources", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 6 + }, + "id": 11, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"api.*\", container!=\"\"}[$interval])", + "legendFormat": "{{container}}", + "refId": "A" + } + ], + "title": "CPU Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 6 + }, + "id": 12, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"api.*\", container!=\"\"}", + "legendFormat": "{{container}}", + "refId": "A" + } + ], + "title": "Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 6 + }, + "id": 13, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"api.*\"}[$interval])", + "legendFormat": "RX {{pod}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"api.*\"}[$interval])", + "legendFormat": "TX {{pod}}", + "refId": "B" + } + ], + "title": "Network I/O", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 14, + "panels": [], + "title": "gRPC Metrics", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 15 + }, + "id": 15, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(grpc_server_handled_total{grpc_service=~\"foxhunt.*\", pod=~\"api.*\"}[$interval])", + "legendFormat": "{{grpc_method}}", + "refId": "A" + } + ], + "title": "gRPC Request Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 15 + }, + "id": 16, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.99, rate(grpc_server_handling_seconds_bucket{pod=~\"api.*\"}[$interval]))", + "legendFormat": "p99", + "refId": "A" + } + ], + "title": "gRPC Latency (p99)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 15 + }, + "id": 17, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(grpc_server_handled_total{grpc_code!=\"OK\", pod=~\"api.*\"}[$interval])", + "legendFormat": "{{grpc_code}}", + "refId": "A" + } + ], + "title": "gRPC Errors", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 18, + "panels": [], + "title": "Logs", + "type": "row" + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 19, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "expr": "{namespace=\"foxhunt\", pod=~\"api.*\"}", + "refId": "A" + } + ], + "title": "Log Stream", + "type": "logs" + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 20, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "expr": "{namespace=\"foxhunt\", pod=~\"api.*\"} |~ \"(?i)(error|warn|panic|fatal)\"", + "refId": "A" + } + ], + "title": "Errors & Warnings", + "type": "logs" + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": [ + "service", + "api" + ], + "templating": { + "list": [ + { + "auto": false, + "current": { + "text": "5m", + "value": "5m" + }, + "name": "interval", + "options": [ + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "15m", + "value": "15m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "1m,5m,15m,30m,1h", + "type": "interval" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "browser", + "title": "API Gateway", + "uid": "svc-api-gateway" +} \ No newline at end of file diff --git a/infra/k8s/monitoring/dashboards/svc-backtesting.json b/infra/k8s/monitoring/dashboards/svc-backtesting.json new file mode 100644 index 000000000..8f6060a2b --- /dev/null +++ b/infra/k8s/monitoring/dashboards/svc-backtesting.json @@ -0,0 +1,1075 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "Service Health", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"backtesting-service.*\", phase=\"Running\"}", + "instant": true, + "legendFormat": "Running", + "range": false, + "refId": "A" + } + ], + "title": "Pod Status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 3 + }, + { + "color": "red", + "value": 10 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 3, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(kube_pod_container_status_restarts_total{namespace=\"foxhunt\", pod=~\"backtesting-service.*\"})", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Restarts", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.7 + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 6, + "y": 1 + }, + "id": 4, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"backtesting-service.*\", container!=\"\"}[$interval]))", + "legendFormat": "", + "refId": "A" + } + ], + "title": "CPU Usage", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.7 + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 9, + "y": 1 + }, + "id": 5, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"backtesting-service.*\", container!=\"\"}) / sum(kube_pod_container_resource_limits{namespace=\"foxhunt\", pod=~\"backtesting-service.*\", resource=\"memory\"})", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Memory", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 12, + "y": 1 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"backtesting-service.*\", container!=\"\"})", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Memory (bytes)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 15, + "y": 1 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"backtesting-service.*\", container!=\"\"}[5m]))", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "CPU Cores", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 18, + "y": 1 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"backtesting-service.*\"}[5m]))", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Network RX", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 21, + "y": 1 + }, + "id": 9, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"backtesting-service.*\"}[5m]))", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Network TX", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 10, + "panels": [], + "title": "Resources", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 6 + }, + "id": 11, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"backtesting-service.*\", container!=\"\"}[$interval])", + "legendFormat": "{{container}}", + "refId": "A" + } + ], + "title": "CPU Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 6 + }, + "id": 12, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"backtesting-service.*\", container!=\"\"}", + "legendFormat": "{{container}}", + "refId": "A" + } + ], + "title": "Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 6 + }, + "id": 13, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"backtesting-service.*\"}[$interval])", + "legendFormat": "RX {{pod}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"backtesting-service.*\"}[$interval])", + "legendFormat": "TX {{pod}}", + "refId": "B" + } + ], + "title": "Network I/O", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 14, + "panels": [], + "title": "Backtesting Metrics", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 15 + }, + "id": 15, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "foxhunt_backtest_duration_seconds{pod=~\"backtesting.*\"}", + "legendFormat": "{{strategy}}", + "refId": "A" + } + ], + "title": "Backtest Duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 15 + }, + "id": 16, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "foxhunt_backtest_sharpe{pod=~\"backtesting.*\"}", + "legendFormat": "{{strategy}}", + "refId": "A" + } + ], + "title": "Backtest Sharpe", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 15 + }, + "id": 17, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "foxhunt_backtest_windows_total{pod=~\"backtesting.*\"}", + "legendFormat": "{{strategy}}", + "refId": "A" + } + ], + "title": "Walk-Forward Windows", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 18, + "panels": [], + "title": "gRPC", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 19, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(grpc_server_handled_total{pod=~\"backtesting.*\"}[$interval])", + "legendFormat": "{{grpc_method}}", + "refId": "A" + } + ], + "title": "gRPC Handled", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 20, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.99, rate(grpc_server_handling_seconds_bucket{pod=~\"backtesting.*\"}[$interval]))", + "legendFormat": "p99", + "refId": "A" + } + ], + "title": "gRPC Latency", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 21, + "panels": [], + "title": "Logs", + "type": "row" + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 33 + }, + "id": 22, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "expr": "{namespace=\"foxhunt\", pod=~\"backtesting.*\"}", + "refId": "A" + } + ], + "title": "Log Stream", + "type": "logs" + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 43 + }, + "id": 23, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "expr": "{namespace=\"foxhunt\", pod=~\"backtesting.*\"} |~ \"(?i)(error|warn|panic|fatal)\"", + "refId": "A" + } + ], + "title": "Errors & Warnings", + "type": "logs" + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": [ + "service", + "backtesting" + ], + "templating": { + "list": [ + { + "auto": false, + "current": { + "text": "5m", + "value": "5m" + }, + "name": "interval", + "options": [ + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "15m", + "value": "15m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "1m,5m,15m,30m,1h", + "type": "interval" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "browser", + "title": "Backtesting Service", + "uid": "svc-backtesting" +} \ No newline at end of file diff --git a/infra/k8s/monitoring/dashboards/svc-data-acquisition.json b/infra/k8s/monitoring/dashboards/svc-data-acquisition.json new file mode 100644 index 000000000..ae615e125 --- /dev/null +++ b/infra/k8s/monitoring/dashboards/svc-data-acquisition.json @@ -0,0 +1,1031 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "Service Health", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"data-acquisition.*\", phase=\"Running\"}", + "instant": true, + "legendFormat": "Running", + "range": false, + "refId": "A" + } + ], + "title": "Pod Status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 3 + }, + { + "color": "red", + "value": 10 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 3, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(kube_pod_container_status_restarts_total{namespace=\"foxhunt\", pod=~\"data-acquisition.*\"})", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Restarts", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.7 + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 6, + "y": 1 + }, + "id": 4, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"data-acquisition.*\", container!=\"\"}[$interval]))", + "legendFormat": "", + "refId": "A" + } + ], + "title": "CPU Usage", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.7 + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 9, + "y": 1 + }, + "id": 5, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"data-acquisition.*\", container!=\"\"}) / sum(kube_pod_container_resource_limits{namespace=\"foxhunt\", pod=~\"data-acquisition.*\", resource=\"memory\"})", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Memory", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 12, + "y": 1 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"data-acquisition.*\", container!=\"\"})", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Memory (bytes)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 15, + "y": 1 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"data-acquisition.*\", container!=\"\"}[5m]))", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "CPU Cores", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 18, + "y": 1 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"data-acquisition.*\"}[5m]))", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Network RX", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 21, + "y": 1 + }, + "id": 9, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"data-acquisition.*\"}[5m]))", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Network TX", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 10, + "panels": [], + "title": "Data Feed Metrics", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 0, + "y": 6 + }, + "id": 11, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "foxhunt_data_acquisition_feeds_active", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Active Feeds", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 4, + "y": 6 + }, + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "foxhunt_data_acquisition_errors_total", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Errors", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 8, + "y": 6 + }, + "id": 13, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "foxhunt_data_acquisition_uptime_seconds", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Uptime", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 14, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "foxhunt_data_acquisition_feeds_active", + "legendFormat": "Active Feeds", + "refId": "A" + } + ], + "title": "Feed Activity", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 15, + "panels": [], + "title": "Resources", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 12 + }, + "id": 16, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"data-acquisition.*\", container!=\"\"}[$interval])", + "legendFormat": "{{container}}", + "refId": "A" + } + ], + "title": "CPU Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 12 + }, + "id": 17, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"data-acquisition.*\", container!=\"\"}", + "legendFormat": "{{container}}", + "refId": "A" + } + ], + "title": "Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 12 + }, + "id": 18, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"data-acquisition.*\"}[$interval])", + "legendFormat": "RX {{pod}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"data-acquisition.*\"}[$interval])", + "legendFormat": "TX {{pod}}", + "refId": "B" + } + ], + "title": "Network I/O", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 19, + "panels": [], + "title": "Logs", + "type": "row" + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 20, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "expr": "{namespace=\"foxhunt\", pod=~\"data-acquisition.*\"}", + "refId": "A" + } + ], + "title": "Log Stream", + "type": "logs" + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 21, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "expr": "{namespace=\"foxhunt\", pod=~\"data-acquisition.*\"} |~ \"(?i)(error|warn|panic|fatal)\"", + "refId": "A" + } + ], + "title": "Errors & Warnings", + "type": "logs" + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": [ + "service", + "data" + ], + "templating": { + "list": [ + { + "auto": false, + "current": { + "text": "5m", + "value": "5m" + }, + "name": "interval", + "options": [ + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "15m", + "value": "15m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "1m,5m,15m,30m,1h", + "type": "interval" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "browser", + "title": "Data Acquisition", + "uid": "svc-data-acq" +} \ No newline at end of file diff --git a/infra/k8s/monitoring/dashboards/svc-ml-training.json b/infra/k8s/monitoring/dashboards/svc-ml-training.json new file mode 100644 index 000000000..41a8d89ba --- /dev/null +++ b/infra/k8s/monitoring/dashboards/svc-ml-training.json @@ -0,0 +1,1026 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "Service Health", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"ml-training.*\", phase=\"Running\"}", + "instant": true, + "legendFormat": "Running", + "range": false, + "refId": "A" + } + ], + "title": "Pod Status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 3 + }, + { + "color": "red", + "value": 10 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 3, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(kube_pod_container_status_restarts_total{namespace=\"foxhunt\", pod=~\"ml-training.*\"})", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Restarts", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.7 + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 6, + "y": 1 + }, + "id": 4, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"ml-training.*\", container!=\"\"}[$interval]))", + "legendFormat": "", + "refId": "A" + } + ], + "title": "CPU Usage", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.7 + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 9, + "y": 1 + }, + "id": 5, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"ml-training.*\", container!=\"\"}) / sum(kube_pod_container_resource_limits{namespace=\"foxhunt\", pod=~\"ml-training.*\", resource=\"memory\"})", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Memory", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 12, + "y": 1 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"ml-training.*\", container!=\"\"})", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Memory (bytes)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 15, + "y": 1 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"ml-training.*\", container!=\"\"}[5m]))", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "CPU Cores", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 18, + "y": 1 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"ml-training.*\"}[5m]))", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Network RX", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 21, + "y": 1 + }, + "id": 9, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"ml-training.*\"}[5m]))", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Network TX", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 10, + "panels": [], + "title": "Resources", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 6 + }, + "id": 11, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"ml-training.*\", container!=\"\"}[$interval])", + "legendFormat": "{{container}}", + "refId": "A" + } + ], + "title": "CPU Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 6 + }, + "id": 12, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"ml-training.*\", container!=\"\"}", + "legendFormat": "{{container}}", + "refId": "A" + } + ], + "title": "Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 6 + }, + "id": 13, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"ml-training.*\"}[$interval])", + "legendFormat": "RX {{pod}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"ml-training.*\"}[$interval])", + "legendFormat": "TX {{pod}}", + "refId": "B" + } + ], + "title": "Network I/O", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 14, + "panels": [], + "title": "Job Management", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 15 + }, + "id": 15, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"training-.*\", phase=\"Running\"}", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "title": "Active Training Jobs", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 15 + }, + "id": 16, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"training-.*\", phase=\"Succeeded\"}", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "title": "Job Completions", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 17, + "panels": [], + "title": "gRPC", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 18, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(grpc_server_handled_total{pod=~\"ml-training.*\"}[$interval])", + "legendFormat": "{{grpc_method}}", + "refId": "A" + } + ], + "title": "gRPC Handled", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 19, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.99, rate(grpc_server_handling_seconds_bucket{pod=~\"ml-training.*\"}[$interval]))", + "legendFormat": "p99", + "refId": "A" + } + ], + "title": "gRPC Latency", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 20, + "panels": [], + "title": "Logs", + "type": "row" + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 33 + }, + "id": 21, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "expr": "{namespace=\"foxhunt\", pod=~\"ml-training.*\"}", + "refId": "A" + } + ], + "title": "Log Stream", + "type": "logs" + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 43 + }, + "id": 22, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "expr": "{namespace=\"foxhunt\", pod=~\"ml-training.*\"} |~ \"(?i)(error|warn|panic|fatal)\"", + "refId": "A" + } + ], + "title": "Errors & Warnings", + "type": "logs" + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": [ + "service", + "ml", + "training" + ], + "templating": { + "list": [ + { + "auto": false, + "current": { + "text": "5m", + "value": "5m" + }, + "name": "interval", + "options": [ + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "15m", + "value": "15m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "1m,5m,15m,30m,1h", + "type": "interval" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "browser", + "title": "ML Training Service", + "uid": "svc-ml-training" +} \ No newline at end of file diff --git a/infra/k8s/monitoring/dashboards/svc-trading-agent.json b/infra/k8s/monitoring/dashboards/svc-trading-agent.json new file mode 100644 index 000000000..30b3a387f --- /dev/null +++ b/infra/k8s/monitoring/dashboards/svc-trading-agent.json @@ -0,0 +1,1075 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "Service Health", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"trading-agent.*\", phase=\"Running\"}", + "instant": true, + "legendFormat": "Running", + "range": false, + "refId": "A" + } + ], + "title": "Pod Status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 3 + }, + { + "color": "red", + "value": 10 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 3, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(kube_pod_container_status_restarts_total{namespace=\"foxhunt\", pod=~\"trading-agent.*\"})", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Restarts", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.7 + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 6, + "y": 1 + }, + "id": 4, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"trading-agent.*\", container!=\"\"}[$interval]))", + "legendFormat": "", + "refId": "A" + } + ], + "title": "CPU Usage", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.7 + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 9, + "y": 1 + }, + "id": 5, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"trading-agent.*\", container!=\"\"}) / sum(kube_pod_container_resource_limits{namespace=\"foxhunt\", pod=~\"trading-agent.*\", resource=\"memory\"})", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Memory", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 12, + "y": 1 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"trading-agent.*\", container!=\"\"})", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Memory (bytes)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 15, + "y": 1 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"trading-agent.*\", container!=\"\"}[5m]))", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "CPU Cores", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 18, + "y": 1 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"trading-agent.*\"}[5m]))", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Network RX", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 21, + "y": 1 + }, + "id": 9, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"trading-agent.*\"}[5m]))", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Network TX", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 10, + "panels": [], + "title": "Resources", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 6 + }, + "id": 11, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"trading-agent.*\", container!=\"\"}[$interval])", + "legendFormat": "{{container}}", + "refId": "A" + } + ], + "title": "CPU Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 6 + }, + "id": 12, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"trading-agent.*\", container!=\"\"}", + "legendFormat": "{{container}}", + "refId": "A" + } + ], + "title": "Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 6 + }, + "id": 13, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"trading-agent.*\"}[$interval])", + "legendFormat": "RX {{pod}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"trading-agent.*\"}[$interval])", + "legendFormat": "TX {{pod}}", + "refId": "B" + } + ], + "title": "Network I/O", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 14, + "panels": [], + "title": "Agent Metrics", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 15 + }, + "id": 15, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "foxhunt_agent_ensemble_confidence{pod=~\"trading-agent.*\"}", + "legendFormat": "{{model}}", + "refId": "A" + } + ], + "title": "Ensemble Confidence", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 15 + }, + "id": 16, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(foxhunt_agent_signals_total{pod=~\"trading-agent.*\"}[$interval])", + "legendFormat": "{{signal_type}}", + "refId": "A" + } + ], + "title": "Signal Generation Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 15 + }, + "id": 17, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.99, rate(foxhunt_agent_decision_seconds_bucket{pod=~\"trading-agent.*\"}[$interval]))", + "legendFormat": "p99", + "refId": "A" + } + ], + "title": "Decision Latency", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 18, + "panels": [], + "title": "gRPC", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 19, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(grpc_server_handled_total{pod=~\"trading-agent.*\"}[$interval])", + "legendFormat": "{{grpc_method}}", + "refId": "A" + } + ], + "title": "gRPC Handled", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 20, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.99, rate(grpc_server_handling_seconds_bucket{pod=~\"trading-agent.*\"}[$interval]))", + "legendFormat": "p99", + "refId": "A" + } + ], + "title": "gRPC Latency", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 21, + "panels": [], + "title": "Logs", + "type": "row" + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 33 + }, + "id": 22, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "expr": "{namespace=\"foxhunt\", pod=~\"trading-agent.*\"}", + "refId": "A" + } + ], + "title": "Log Stream", + "type": "logs" + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 43 + }, + "id": 23, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "expr": "{namespace=\"foxhunt\", pod=~\"trading-agent.*\"} |~ \"(?i)(error|warn|panic|fatal)\"", + "refId": "A" + } + ], + "title": "Errors & Warnings", + "type": "logs" + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": [ + "service", + "agent" + ], + "templating": { + "list": [ + { + "auto": false, + "current": { + "text": "5m", + "value": "5m" + }, + "name": "interval", + "options": [ + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "15m", + "value": "15m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "1m,5m,15m,30m,1h", + "type": "interval" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "browser", + "title": "Trading Agent", + "uid": "svc-trading-agent" +} \ No newline at end of file diff --git a/infra/k8s/monitoring/dashboards/svc-trading-service.json b/infra/k8s/monitoring/dashboards/svc-trading-service.json new file mode 100644 index 000000000..5fcfa3c3f --- /dev/null +++ b/infra/k8s/monitoring/dashboards/svc-trading-service.json @@ -0,0 +1,1074 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "Service Health", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"trading-service.*\", phase=\"Running\"}", + "instant": true, + "legendFormat": "Running", + "range": false, + "refId": "A" + } + ], + "title": "Pod Status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 3 + }, + { + "color": "red", + "value": 10 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 3, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(kube_pod_container_status_restarts_total{namespace=\"foxhunt\", pod=~\"trading-service.*\"})", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Restarts", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.7 + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 6, + "y": 1 + }, + "id": 4, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"trading-service.*\", container!=\"\"}[$interval]))", + "legendFormat": "", + "refId": "A" + } + ], + "title": "CPU Usage", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.7 + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 9, + "y": 1 + }, + "id": 5, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"trading-service.*\", container!=\"\"}) / sum(kube_pod_container_resource_limits{namespace=\"foxhunt\", pod=~\"trading-service.*\", resource=\"memory\"})", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Memory", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 12, + "y": 1 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"trading-service.*\", container!=\"\"})", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Memory (bytes)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 15, + "y": 1 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"trading-service.*\", container!=\"\"}[5m]))", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "CPU Cores", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 18, + "y": 1 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"trading-service.*\"}[5m]))", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Network RX", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 21, + "y": 1 + }, + "id": 9, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"trading-service.*\"}[5m]))", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Network TX", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 10, + "panels": [], + "title": "Resources", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 6 + }, + "id": 11, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"trading-service.*\", container!=\"\"}[$interval])", + "legendFormat": "{{container}}", + "refId": "A" + } + ], + "title": "CPU Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 6 + }, + "id": 12, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"trading-service.*\", container!=\"\"}", + "legendFormat": "{{container}}", + "refId": "A" + } + ], + "title": "Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 6 + }, + "id": 13, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"trading-service.*\"}[$interval])", + "legendFormat": "RX {{pod}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"trading-service.*\"}[$interval])", + "legendFormat": "TX {{pod}}", + "refId": "B" + } + ], + "title": "Network I/O", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 14, + "panels": [], + "title": "Trading Metrics", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 15 + }, + "id": 15, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "foxhunt_trading_sharpe_ratio{pod=~\"trading-service.*\"}", + "legendFormat": "Sharpe", + "refId": "A" + } + ], + "title": "Ensemble Sharpe Ratio", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 15 + }, + "id": 16, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "foxhunt_trading_open_positions{pod=~\"trading-service.*\"}", + "legendFormat": "Open", + "refId": "A" + } + ], + "title": "Position Count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 15 + }, + "id": 17, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(foxhunt_trading_orders_total{pod=~\"trading-service.*\"}[$interval])", + "legendFormat": "{{side}}", + "refId": "A" + } + ], + "title": "Order Rate", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 18, + "panels": [], + "title": "gRPC", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 19, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(grpc_server_handled_total{pod=~\"trading-service.*\"}[$interval])", + "legendFormat": "{{grpc_method}}", + "refId": "A" + } + ], + "title": "gRPC Request Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 20, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.99, rate(grpc_server_handling_seconds_bucket{pod=~\"trading-service.*\"}[$interval]))", + "legendFormat": "p99", + "refId": "A" + } + ], + "title": "gRPC Latency", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 21, + "panels": [], + "title": "Logs", + "type": "row" + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 33 + }, + "id": 22, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "expr": "{namespace=\"foxhunt\", pod=~\"trading-service.*\"}", + "refId": "A" + } + ], + "title": "Log Stream", + "type": "logs" + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 43 + }, + "id": 23, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "expr": "{namespace=\"foxhunt\", pod=~\"trading-service.*\"} |~ \"(?i)(error|warn|panic|fatal)\"", + "refId": "A" + } + ], + "title": "Errors & Warnings", + "type": "logs" + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": [ + "service", + "trading" + ], + "templating": { + "list": [ + { + "auto": false, + "current": { + "text": "5m", + "value": "5m" + }, + "name": "interval", + "options": [ + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "15m", + "value": "15m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "1m,5m,15m,30m,1h", + "type": "interval" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "browser", + "title": "Trading Service", + "uid": "svc-trading" +} \ No newline at end of file diff --git a/infra/k8s/monitoring/dashboards/svc-web-gateway.json b/infra/k8s/monitoring/dashboards/svc-web-gateway.json new file mode 100644 index 000000000..e0279af01 --- /dev/null +++ b/infra/k8s/monitoring/dashboards/svc-web-gateway.json @@ -0,0 +1,1022 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "Service Health", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"web-gateway.*\", phase=\"Running\"}", + "instant": true, + "legendFormat": "Running", + "range": false, + "refId": "A" + } + ], + "title": "Pod Status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 3 + }, + { + "color": "red", + "value": 10 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 3, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(kube_pod_container_status_restarts_total{namespace=\"foxhunt\", pod=~\"web-gateway.*\"})", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Restarts", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.7 + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 6, + "y": 1 + }, + "id": 4, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"web-gateway.*\", container!=\"\"}[$interval]))", + "legendFormat": "", + "refId": "A" + } + ], + "title": "CPU Usage", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.7 + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 9, + "y": 1 + }, + "id": 5, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"web-gateway.*\", container!=\"\"}) / sum(kube_pod_container_resource_limits{namespace=\"foxhunt\", pod=~\"web-gateway.*\", resource=\"memory\"})", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Memory", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 12, + "y": 1 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"web-gateway.*\", container!=\"\"})", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Memory (bytes)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 15, + "y": 1 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"web-gateway.*\", container!=\"\"}[5m]))", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "CPU Cores", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 18, + "y": 1 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"web-gateway.*\"}[5m]))", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Network RX", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 21, + "y": 1 + }, + "id": 9, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"web-gateway.*\"}[5m]))", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Network TX", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 10, + "panels": [], + "title": "Gateway Metrics", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 0, + "y": 6 + }, + "id": 11, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "foxhunt_web_gateway_ws_connections_active", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Active WS Connections", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 4, + "y": 6 + }, + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "foxhunt_web_gateway_uptime_seconds", + "instant": true, + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Uptime", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 6 + }, + "id": 13, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "foxhunt_web_gateway_ws_connections_active", + "legendFormat": "Active", + "refId": "A" + } + ], + "title": "WebSocket Connections", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 6 + }, + "id": 14, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(foxhunt_web_gateway_http_requests_total{pod=~\"web-gateway.*\"}[$interval])", + "legendFormat": "{{method}} {{path}}", + "refId": "A" + } + ], + "title": "HTTP Request Rate", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 15, + "panels": [], + "title": "Resources", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 12 + }, + "id": 16, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"web-gateway.*\", container!=\"\"}[$interval])", + "legendFormat": "{{container}}", + "refId": "A" + } + ], + "title": "CPU Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 12 + }, + "id": 17, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"web-gateway.*\", container!=\"\"}", + "legendFormat": "{{container}}", + "refId": "A" + } + ], + "title": "Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "stacking": { + "mode": "none" + } + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 12 + }, + "id": 18, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"web-gateway.*\"}[$interval])", + "legendFormat": "RX {{pod}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"web-gateway.*\"}[$interval])", + "legendFormat": "TX {{pod}}", + "refId": "B" + } + ], + "title": "Network I/O", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 19, + "panels": [], + "title": "Logs", + "type": "row" + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 20, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "expr": "{namespace=\"foxhunt\", pod=~\"web-gateway.*\"}", + "refId": "A" + } + ], + "title": "Log Stream", + "type": "logs" + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 21, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "expr": "{namespace=\"foxhunt\", pod=~\"web-gateway.*\"} |~ \"(?i)(error|warn|panic|fatal)\"", + "refId": "A" + } + ], + "title": "Errors & Warnings", + "type": "logs" + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": [ + "service", + "web" + ], + "templating": { + "list": [ + { + "auto": false, + "current": { + "text": "5m", + "value": "5m" + }, + "name": "interval", + "options": [ + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "15m", + "value": "15m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "1m,5m,15m,30m,1h", + "type": "interval" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "browser", + "title": "Web Gateway", + "uid": "svc-web-gateway" +} \ No newline at end of file