From 6349e384d29cec9f9e008c231c6422ca2df2435c Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sat, 14 Mar 2026 02:04:24 +0100 Subject: [PATCH] feat(monitoring): add Training Deep-Dive dashboard with unified Prometheus + Loki 74-panel dashboard for in-depth pipeline monitoring. Select a running pod to see training curves, GPU health (DCGM), eval metrics, CI/Argo workflows, container resources, and live logs in one place. Stored in Postgres via API (no ConfigMap restarts needed). Sections: Job Overview, CI Pipeline & Argo Workflows, CI Logs, Training Curves, Trading Performance, Evaluation Metrics, GPU & Hardware, Throughput, Hyperopt Trials, Container Resources, Live Logs. Co-Authored-By: Claude Opus 4.6 --- .../foxhunt-training-deep-dive.json | 3345 +++++++++++++++++ infra/k8s/monitoring/dashboards/import.sh | 5 +- 2 files changed, 3349 insertions(+), 1 deletion(-) create mode 100644 infra/k8s/monitoring/dashboards/foxhunt-training-deep-dive.json diff --git a/infra/k8s/monitoring/dashboards/foxhunt-training-deep-dive.json b/infra/k8s/monitoring/dashboards/foxhunt-training-deep-dive.json new file mode 100644 index 000000000..6985018be --- /dev/null +++ b/infra/k8s/monitoring/dashboards/foxhunt-training-deep-dive.json @@ -0,0 +1,3345 @@ +{ + "id": null, + "uid": "foxhunt-training-deep-dive", + "title": "Foxhunt Training Deep-Dive", + "description": "Unified training pipeline monitor \u2014 target a running pod to see training curves, GPU health, eval metrics, hyperopt progress, container resources, and live logs in one place.", + "tags": [ + "foxhunt", + "training", + "gpu", + "loki", + "deep-dive" + ], + "timezone": "browser", + "refresh": "10s", + "time": { + "from": "now-3h", + "to": "now" + }, + "fiscalYearStartMonth": 0, + "liveNow": true, + "schemaVersion": 39, + "version": 1, + "editable": true, + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "type": "datasource", + "pluginId": "prometheus" + }, + { + "name": "DS_LOKI", + "type": "datasource", + "pluginId": "loki" + } + ], + "templating": { + "list": [ + { + "name": "pod", + "type": "query", + "label": "Pod", + "query": "label_values({namespace=\"foxhunt\", pod=~\"(ci-pipeline|compile-and-train|compile-and-deploy|train|hyperopt|gpu-test|evaluate|build-training).*\"}, pod)", + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "refresh": 2, + "includeAll": true, + "multi": true, + "regex": "", + "sort": 2, + "current": {} + }, + { + "name": "prom_job", + "type": "query", + "label": "Prometheus Job", + "query": "label_values(up{namespace=\"foxhunt\"}, job)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "refresh": 2, + "includeAll": true, + "multi": false, + "regex": "/(train|hyperopt|evaluate|pushgateway|training-pods|dcgm)/", + "sort": 1, + "current": {} + }, + { + "name": "level", + "type": "custom", + "label": "Log Level", + "query": "ERROR,WARN,INFO,DEBUG,TRACE", + "includeAll": true, + "multi": true, + "current": { + "text": "All", + "value": "$__all" + } + }, + { + "name": "search", + "type": "textbox", + "label": "Search", + "current": { + "text": "", + "value": "" + } + } + ] + }, + "annotations": { + "list": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "enable": true, + "name": "Errors", + "expr": "{namespace=\"foxhunt\", pod=~\"$pod\", level=\"ERROR\"}", + "iconColor": "red", + "tagKeys": "level", + "titleFormat": "Error" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "enable": true, + "name": "Checkpoints", + "expr": "changes(foxhunt_training_checkpoint_saves_total{pod=~\"$pod\", namespace=\"foxhunt\"}[1m]) > 0", + "iconColor": "blue", + "titleFormat": "Checkpoint Saved" + } + ] + }, + "panels": [ + { + "type": "row", + "title": "Job Overview", + "collapsed": false, + "id": 1, + "gridPos": { + "x": 0, + "y": 0, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "type": "stat", + "title": "Epoch", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 2, + "fieldConfig": { + "defaults": { + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "background", + "graphMode": "area", + "textMode": "auto" + }, + "targets": [ + { + "expr": "foxhunt_training_current_epoch{pod=~\"$pod\", namespace=\"foxhunt\"}", + "legendFormat": "", + "refId": "A" + } + ], + "gridPos": { + "x": 0, + "y": 1, + "w": 3, + "h": 4 + } + }, + { + "type": "stat", + "title": "Step", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 3, + "fieldConfig": { + "defaults": { + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "background", + "graphMode": "area", + "textMode": "auto" + }, + "targets": [ + { + "expr": "foxhunt_training_step{pod=~\"$pod\", namespace=\"foxhunt\"}", + "legendFormat": "", + "refId": "A" + } + ], + "gridPos": { + "x": 3, + "y": 1, + "w": 3, + "h": 4 + } + }, + { + "type": "stat", + "title": "Loss", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 4, + "fieldConfig": { + "defaults": { + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.5 + }, + { + "color": "red", + "value": 1.0 + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "background", + "graphMode": "area", + "textMode": "auto" + }, + "targets": [ + { + "expr": "foxhunt_training_epoch_loss{pod=~\"$pod\", namespace=\"foxhunt\"}", + "legendFormat": "", + "refId": "A" + } + ], + "gridPos": { + "x": 6, + "y": 1, + "w": 3, + "h": 4 + } + }, + { + "type": "stat", + "title": "Val Loss", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 5, + "fieldConfig": { + "defaults": { + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.5 + }, + { + "color": "red", + "value": 1.0 + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "background", + "graphMode": "area", + "textMode": "auto" + }, + "targets": [ + { + "expr": "foxhunt_training_validation_loss{pod=~\"$pod\", namespace=\"foxhunt\"}", + "legendFormat": "", + "refId": "A" + } + ], + "gridPos": { + "x": 9, + "y": 1, + "w": 3, + "h": 4 + } + }, + { + "type": "stat", + "title": "Sharpe", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 6, + "fieldConfig": { + "defaults": { + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 0.5 + }, + { + "color": "green", + "value": 1.0 + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "background", + "graphMode": "area", + "textMode": "auto" + }, + "targets": [ + { + "expr": "foxhunt_training_epoch_sharpe{pod=~\"$pod\", namespace=\"foxhunt\"}", + "legendFormat": "", + "refId": "A" + } + ], + "gridPos": { + "x": 12, + "y": 1, + "w": 3, + "h": 4 + } + }, + { + "type": "stat", + "title": "MaxDD", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 7, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.15 + }, + { + "color": "red", + "value": 0.3 + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "background", + "graphMode": "area", + "textMode": "auto" + }, + "targets": [ + { + "expr": "foxhunt_training_epoch_max_drawdown{pod=~\"$pod\", namespace=\"foxhunt\"}", + "legendFormat": "", + "refId": "A" + } + ], + "gridPos": { + "x": 15, + "y": 1, + "w": 3, + "h": 4 + } + }, + { + "type": "gauge", + "title": "GPU Util", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 8, + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 70 + }, + { + "color": "red", + "value": 90 + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "targets": [ + { + "expr": "DCGM_FI_DEV_GPU_UTIL{pod=~\"$pod\", namespace=\"foxhunt\"}", + "legendFormat": "", + "refId": "A" + } + ], + "gridPos": { + "x": 18, + "y": 1, + "w": 3, + "h": 4 + } + }, + { + "type": "gauge", + "title": "VRAM Used", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 9, + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 70 + }, + { + "color": "red", + "value": 90 + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "targets": [ + { + "expr": "DCGM_FI_DEV_FB_USED{pod=~\"$pod\", namespace=\"foxhunt\"} / (DCGM_FI_DEV_FB_USED{pod=~\"$pod\", namespace=\"foxhunt\"} + DCGM_FI_DEV_FB_FREE{pod=~\"$pod\", namespace=\"foxhunt\"}) * 100", + "legendFormat": "", + "refId": "A" + } + ], + "gridPos": { + "x": 21, + "y": 1, + "w": 3, + "h": 4 + } + }, + { + "type": "row", + "title": "CI Pipeline & Argo Workflows", + "collapsed": false, + "id": 10, + "gridPos": { + "x": 0, + "y": 5, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "type": "stat", + "title": "Active Workflows", + "description": "Currently running Argo workflows", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 11, + "fieldConfig": { + "defaults": { + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "background", + "graphMode": "area", + "textMode": "auto" + }, + "targets": [ + { + "expr": "argo_workflows_gauge{status=\"Running\"}", + "legendFormat": "", + "refId": "A" + } + ], + "gridPos": { + "x": 0, + "y": 6, + "w": 3, + "h": 4 + } + }, + { + "type": "stat", + "title": "Pending Pods", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 12, + "fieldConfig": { + "defaults": { + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 3 + }, + { + "color": "red", + "value": 10 + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "background", + "graphMode": "area", + "textMode": "auto" + }, + "targets": [ + { + "expr": "argo_workflows_pod_pending_count", + "legendFormat": "", + "refId": "A" + } + ], + "gridPos": { + "x": 3, + "y": 6, + "w": 3, + "h": 4 + } + }, + { + "type": "stat", + "title": "Total Completed", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 13, + "fieldConfig": { + "defaults": { + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "background", + "graphMode": "area", + "textMode": "auto" + }, + "targets": [ + { + "expr": "argo_workflows_total_count{status=\"Succeeded\"}", + "legendFormat": "", + "refId": "A" + } + ], + "gridPos": { + "x": 6, + "y": 6, + "w": 3, + "h": 4 + } + }, + { + "type": "stat", + "title": "Errors", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 14, + "fieldConfig": { + "defaults": { + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "background", + "graphMode": "area", + "textMode": "auto" + }, + "targets": [ + { + "expr": "argo_workflows_error_count", + "legendFormat": "", + "refId": "A" + } + ], + "gridPos": { + "x": 9, + "y": 6, + "w": 3, + "h": 4 + } + }, + { + "type": "timeseries", + "title": "Workflow Duration", + "description": "Average workflow operation duration", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 15, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "fillOpacity": 0, + "stacking": { + "mode": "none" + }, + "lineWidth": 1, + "spanNulls": true + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "rate(argo_workflows_operation_duration_seconds_sum[5m]) / rate(argo_workflows_operation_duration_seconds_count[5m])", + "legendFormat": "Avg Duration", + "refId": "A" + } + ], + "gridPos": { + "x": 12, + "y": 6, + "w": 6, + "h": 7 + } + }, + { + "type": "timeseries", + "title": "K8s API Request Rate", + "description": "Argo controller K8s API calls \u2014 high rates may indicate thrashing", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 16, + "fieldConfig": { + "defaults": { + "unit": "none", + "custom": { + "fillOpacity": 0, + "stacking": { + "mode": "none" + }, + "lineWidth": 1, + "spanNulls": true + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "rate(argo_workflows_k8s_request_total[5m])", + "legendFormat": "{{verb}} {{status_code}}", + "refId": "A" + } + ], + "gridPos": { + "x": 18, + "y": 6, + "w": 6, + "h": 7 + } + }, + { + "type": "timeseries", + "title": "CI Pod CPU", + "description": "CPU usage of selected CI/training pods", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 17, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "fillOpacity": 0, + "stacking": { + "mode": "none" + }, + "lineWidth": 1, + "spanNulls": true + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"$pod\"}[5m])", + "legendFormat": "{{container}}", + "refId": "A" + } + ], + "gridPos": { + "x": 0, + "y": 10, + "w": 6, + "h": 7 + } + }, + { + "type": "timeseries", + "title": "CI Pod Memory", + "description": "RSS memory of selected CI/training pods", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 18, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "fillOpacity": 0, + "stacking": { + "mode": "none" + }, + "lineWidth": 1, + "spanNulls": true + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "container_memory_rss{namespace=\"foxhunt\", pod=~\"$pod\"}", + "legendFormat": "{{container}}", + "refId": "A" + } + ], + "gridPos": { + "x": 6, + "y": 10, + "w": 6, + "h": 7 + } + }, + { + "type": "row", + "title": "CI Logs", + "collapsed": false, + "id": 19, + "gridPos": { + "x": 0, + "y": 17, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "type": "logs", + "title": "CI Full Log Stream", + "description": "All logs from selected CI/training pods", + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "id": 20, + "options": { + "showTime": true, + "showLabels": false, + "showCommonLabels": false, + "wrapLogMessage": true, + "prettifyLogMessage": false, + "enableLogDetails": true, + "sortOrder": "Descending", + "dedupStrategy": "none" + }, + "targets": [ + { + "expr": "{namespace=\"foxhunt\", pod=~\"$pod\"} |~ \"$search\"", + "refId": "A" + } + ], + "gridPos": { + "x": 0, + "y": 18, + "w": 12, + "h": 9 + } + }, + { + "type": "logs", + "title": "Compile & Test Results", + "description": "Rust compile output, test results, clippy warnings", + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "id": 21, + "options": { + "showTime": true, + "showLabels": false, + "showCommonLabels": false, + "wrapLogMessage": true, + "prettifyLogMessage": false, + "enableLogDetails": true, + "sortOrder": "Descending", + "dedupStrategy": "none" + }, + "targets": [ + { + "expr": "{namespace=\"foxhunt\", pod=~\"$pod\"} |~ \"(?i)(cargo (build|test|check|clippy)|Compiling|Finished|running \\\\d+ test|test result|FAILED|passed|warning:)\"", + "refId": "A" + } + ], + "gridPos": { + "x": 12, + "y": 18, + "w": 12, + "h": 9 + } + }, + { + "type": "logs", + "title": "Build Artifacts & Deploy", + "description": "S3 binary uploads, kubectl deploys, image builds", + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "id": 22, + "options": { + "showTime": true, + "showLabels": false, + "showCommonLabels": false, + "wrapLogMessage": true, + "prettifyLogMessage": false, + "enableLogDetails": true, + "sortOrder": "Descending", + "dedupStrategy": "none" + }, + "targets": [ + { + "expr": "{namespace=\"foxhunt\", pod=~\"$pod\"} |~ \"(?i)(rclone|s3://|upload|download|kubectl|rollout|image|docker|kaniko|configmap)\"", + "refId": "A" + } + ], + "gridPos": { + "x": 0, + "y": 27, + "w": 12, + "h": 9 + } + }, + { + "type": "logs", + "title": "Redis Sidecar & Data", + "description": "Data fetching, redis cache, dbn file operations", + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "id": 23, + "options": { + "showTime": true, + "showLabels": false, + "showCommonLabels": false, + "wrapLogMessage": true, + "prettifyLogMessage": false, + "enableLogDetails": true, + "sortOrder": "Descending", + "dedupStrategy": "none" + }, + "targets": [ + { + "expr": "{namespace=\"foxhunt\", pod=~\"$pod\"} |~ \"(?i)(redis|rclone|dbn|data|cache|download|decompress)\"", + "refId": "A" + } + ], + "gridPos": { + "x": 12, + "y": 27, + "w": 12, + "h": 9 + } + }, + { + "type": "row", + "title": "Training Curves", + "collapsed": false, + "id": 24, + "gridPos": { + "x": 0, + "y": 36, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "type": "timeseries", + "title": "Loss & Validation Loss", + "description": "Training and validation loss over time", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 25, + "fieldConfig": { + "defaults": { + "unit": "none", + "custom": { + "fillOpacity": 0, + "stacking": { + "mode": "none" + }, + "lineWidth": 1, + "spanNulls": true + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "foxhunt_training_epoch_loss{pod=~\"$pod\", namespace=\"foxhunt\"}", + "legendFormat": "Train Loss", + "refId": "A" + }, + { + "expr": "foxhunt_training_validation_loss{pod=~\"$pod\", namespace=\"foxhunt\"}", + "legendFormat": "Val Loss", + "refId": "B" + } + ], + "gridPos": { + "x": 0, + "y": 37, + "w": 8, + "h": 7 + } + }, + { + "type": "timeseries", + "title": "Learning Rate", + "description": "Learning rate schedule", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 26, + "fieldConfig": { + "defaults": { + "unit": "none", + "custom": { + "fillOpacity": 0, + "stacking": { + "mode": "none" + }, + "lineWidth": 1, + "spanNulls": true + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "foxhunt_training_learning_rate{pod=~\"$pod\", namespace=\"foxhunt\"}", + "legendFormat": "LR", + "refId": "A" + } + ], + "gridPos": { + "x": 8, + "y": 37, + "w": 8, + "h": 7 + } + }, + { + "type": "timeseries", + "title": "Gradient Norm", + "description": "Gradient L2 norm \u2014 spikes indicate instability", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 27, + "fieldConfig": { + "defaults": { + "unit": "none", + "custom": { + "fillOpacity": 0, + "stacking": { + "mode": "none" + }, + "lineWidth": 1, + "spanNulls": true + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "foxhunt_training_gradient_norm{pod=~\"$pod\", namespace=\"foxhunt\"}", + "legendFormat": "Grad Norm", + "refId": "A" + } + ], + "gridPos": { + "x": 16, + "y": 37, + "w": 8, + "h": 7 + } + }, + { + "type": "row", + "title": "Trading Performance", + "collapsed": false, + "id": 28, + "gridPos": { + "x": 0, + "y": 44, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "type": "timeseries", + "title": "Sharpe / Sortino / MaxDD", + "description": "Risk-adjusted return metrics per epoch", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 29, + "fieldConfig": { + "defaults": { + "unit": "none", + "custom": { + "fillOpacity": 0, + "stacking": { + "mode": "none" + }, + "lineWidth": 1, + "spanNulls": true + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "foxhunt_training_epoch_sharpe{pod=~\"$pod\", namespace=\"foxhunt\"}", + "legendFormat": "Sharpe", + "refId": "A" + }, + { + "expr": "foxhunt_training_epoch_sortino{pod=~\"$pod\", namespace=\"foxhunt\"}", + "legendFormat": "Sortino", + "refId": "B" + }, + { + "expr": "foxhunt_training_epoch_max_drawdown{pod=~\"$pod\", namespace=\"foxhunt\"}", + "legendFormat": "MaxDD", + "refId": "C" + } + ], + "gridPos": { + "x": 0, + "y": 45, + "w": 8, + "h": 7 + } + }, + { + "type": "timeseries", + "title": "Win Rate & Profit Factor", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 30, + "fieldConfig": { + "defaults": { + "unit": "none", + "custom": { + "fillOpacity": 0, + "stacking": { + "mode": "none" + }, + "lineWidth": 1, + "spanNulls": true + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "foxhunt_training_epoch_win_rate{pod=~\"$pod\", namespace=\"foxhunt\"}", + "legendFormat": "Win Rate", + "refId": "A" + }, + { + "expr": "foxhunt_training_epoch_profit_factor{pod=~\"$pod\", namespace=\"foxhunt\"}", + "legendFormat": "Profit Factor", + "refId": "B" + } + ], + "gridPos": { + "x": 8, + "y": 45, + "w": 8, + "h": 7 + } + }, + { + "type": "timeseries", + "title": "Total Return & Avg Return", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 31, + "fieldConfig": { + "defaults": { + "unit": "none", + "custom": { + "fillOpacity": 0, + "stacking": { + "mode": "none" + }, + "lineWidth": 1, + "spanNulls": true + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "foxhunt_training_epoch_total_return{pod=~\"$pod\", namespace=\"foxhunt\"}", + "legendFormat": "Total Return", + "refId": "A" + }, + { + "expr": "foxhunt_training_epoch_avg_return{pod=~\"$pod\", namespace=\"foxhunt\"}", + "legendFormat": "Avg Return", + "refId": "B" + } + ], + "gridPos": { + "x": 16, + "y": 45, + "w": 8, + "h": 7 + } + }, + { + "type": "timeseries", + "title": "Action Distribution", + "description": "Action distribution per epoch \u2014 watch for collapse to single action", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 32, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "custom": { + "fillOpacity": 30, + "stacking": { + "mode": "normal" + }, + "lineWidth": 1, + "spanNulls": true + }, + "min": 0, + "max": 1 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "foxhunt_training_epoch_action_buy_pct{pod=~\"$pod\", namespace=\"foxhunt\"}", + "legendFormat": "Buy %", + "refId": "A" + }, + { + "expr": "foxhunt_training_epoch_action_hold_pct{pod=~\"$pod\", namespace=\"foxhunt\"}", + "legendFormat": "Hold %", + "refId": "B" + }, + { + "expr": "foxhunt_training_epoch_action_sell_pct{pod=~\"$pod\", namespace=\"foxhunt\"}", + "legendFormat": "Sell %", + "refId": "C" + } + ], + "gridPos": { + "x": 0, + "y": 52, + "w": 8, + "h": 7 + } + }, + { + "type": "timeseries", + "title": "Action Entropy & Diversity", + "description": "Higher entropy = more exploration. Collapse \u2192 policy degeneration", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 33, + "fieldConfig": { + "defaults": { + "unit": "none", + "custom": { + "fillOpacity": 0, + "stacking": { + "mode": "none" + }, + "lineWidth": 1, + "spanNulls": true + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "foxhunt_training_action_entropy{pod=~\"$pod\", namespace=\"foxhunt\"}", + "legendFormat": "Entropy", + "refId": "A" + }, + { + "expr": "foxhunt_training_action_diversity{pod=~\"$pod\", namespace=\"foxhunt\"}", + "legendFormat": "Diversity", + "refId": "B" + } + ], + "gridPos": { + "x": 8, + "y": 52, + "w": 8, + "h": 7 + } + }, + { + "type": "timeseries", + "title": "Total Trades", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 34, + "fieldConfig": { + "defaults": { + "unit": "none", + "custom": { + "fillOpacity": 0, + "stacking": { + "mode": "none" + }, + "lineWidth": 1, + "spanNulls": true + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "foxhunt_training_epoch_total_trades{pod=~\"$pod\", namespace=\"foxhunt\"}", + "legendFormat": "Trades/Epoch", + "refId": "A" + } + ], + "gridPos": { + "x": 16, + "y": 52, + "w": 8, + "h": 7 + } + }, + { + "type": "row", + "title": "Evaluation Metrics", + "collapsed": false, + "id": 35, + "gridPos": { + "x": 0, + "y": 59, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "type": "timeseries", + "title": "Eval Accuracy", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 36, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "custom": { + "fillOpacity": 0, + "stacking": { + "mode": "none" + }, + "lineWidth": 1, + "spanNulls": true + }, + "min": 0, + "max": 1 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "foxhunt_training_eval_accuracy{pod=~\"$pod\", namespace=\"foxhunt\"}", + "legendFormat": "Accuracy", + "refId": "A" + } + ], + "gridPos": { + "x": 0, + "y": 60, + "w": 6, + "h": 7 + } + }, + { + "type": "timeseries", + "title": "Precision / Recall / F1", + "description": "Classification quality of directional predictions", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 37, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "custom": { + "fillOpacity": 0, + "stacking": { + "mode": "none" + }, + "lineWidth": 1, + "spanNulls": true + }, + "min": 0, + "max": 1 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "foxhunt_training_eval_precision{pod=~\"$pod\", namespace=\"foxhunt\"}", + "legendFormat": "Precision", + "refId": "A" + }, + { + "expr": "foxhunt_training_eval_recall{pod=~\"$pod\", namespace=\"foxhunt\"}", + "legendFormat": "Recall", + "refId": "B" + }, + { + "expr": "foxhunt_training_eval_f1{pod=~\"$pod\", namespace=\"foxhunt\"}", + "legendFormat": "F1", + "refId": "C" + } + ], + "gridPos": { + "x": 6, + "y": 60, + "w": 6, + "h": 7 + } + }, + { + "type": "timeseries", + "title": "Q-Values (DQN)", + "description": "DQN Q-value estimates \u2014 divergence from max indicates overestimation", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 38, + "fieldConfig": { + "defaults": { + "unit": "none", + "custom": { + "fillOpacity": 0, + "stacking": { + "mode": "none" + }, + "lineWidth": 1, + "spanNulls": true + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "foxhunt_training_q_value_mean{pod=~\"$pod\", namespace=\"foxhunt\"}", + "legendFormat": "Q Mean", + "refId": "A" + }, + { + "expr": "foxhunt_training_q_value_max{pod=~\"$pod\", namespace=\"foxhunt\"}", + "legendFormat": "Q Max", + "refId": "B" + } + ], + "gridPos": { + "x": 12, + "y": 60, + "w": 6, + "h": 7 + } + }, + { + "type": "timeseries", + "title": "Q-Overestimation & Replay Buffer", + "description": "Ratio > 1 = Q-values overestimate true returns", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 39, + "fieldConfig": { + "defaults": { + "unit": "none", + "custom": { + "fillOpacity": 0, + "stacking": { + "mode": "none" + }, + "lineWidth": 1, + "spanNulls": true + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "foxhunt_training_q_overestimation_ratio{pod=~\"$pod\", namespace=\"foxhunt\"}", + "legendFormat": "Overestimation Ratio", + "refId": "A" + } + ], + "gridPos": { + "x": 18, + "y": 60, + "w": 6, + "h": 7 + } + }, + { + "type": "row", + "title": "GPU & Hardware", + "collapsed": false, + "id": 40, + "gridPos": { + "x": 0, + "y": 67, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "type": "timeseries", + "title": "GPU Utilization", + "description": "Overall GPU and graphics engine utilization", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 41, + "fieldConfig": { + "defaults": { + "unit": "percent", + "custom": { + "fillOpacity": 10, + "stacking": { + "mode": "none" + }, + "lineWidth": 1, + "spanNulls": true + }, + "min": 0, + "max": 100 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "DCGM_FI_DEV_GPU_UTIL{pod=~\"$pod\", namespace=\"foxhunt\"}", + "legendFormat": "GPU %", + "refId": "A" + }, + { + "expr": "DCGM_FI_PROF_GR_ENGINE_ACTIVE{pod=~\"$pod\", namespace=\"foxhunt\"} * 100", + "legendFormat": "GR Engine %", + "refId": "B" + } + ], + "gridPos": { + "x": 0, + "y": 68, + "w": 6, + "h": 7 + } + }, + { + "type": "timeseries", + "title": "Tensor Core Activity", + "description": "Tensor core and DRAM bandwidth utilization \u2014 key H100 saturation indicators", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 42, + "fieldConfig": { + "defaults": { + "unit": "percent", + "custom": { + "fillOpacity": 10, + "stacking": { + "mode": "none" + }, + "lineWidth": 1, + "spanNulls": true + }, + "min": 0, + "max": 100 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{pod=~\"$pod\", namespace=\"foxhunt\"} * 100", + "legendFormat": "Tensor Cores %", + "refId": "A" + }, + { + "expr": "DCGM_FI_PROF_DRAM_ACTIVE{pod=~\"$pod\", namespace=\"foxhunt\"} * 100", + "legendFormat": "DRAM Active %", + "refId": "B" + } + ], + "gridPos": { + "x": 6, + "y": 68, + "w": 6, + "h": 7 + } + }, + { + "type": "timeseries", + "title": "VRAM (Framebuffer)", + "description": "GPU framebuffer memory usage", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 43, + "fieldConfig": { + "defaults": { + "unit": "decmbytes", + "custom": { + "fillOpacity": 20, + "stacking": { + "mode": "normal" + }, + "lineWidth": 1, + "spanNulls": true + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "DCGM_FI_DEV_FB_USED{pod=~\"$pod\", namespace=\"foxhunt\"}", + "legendFormat": "Used (MiB)", + "refId": "A" + }, + { + "expr": "DCGM_FI_DEV_FB_FREE{pod=~\"$pod\", namespace=\"foxhunt\"}", + "legendFormat": "Free (MiB)", + "refId": "B" + } + ], + "gridPos": { + "x": 12, + "y": 68, + "w": 6, + "h": 7 + } + }, + { + "type": "timeseries", + "title": "GPU Temperature", + "description": "Thermal throttling starts at ~83\u00b0C on H100", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 44, + "fieldConfig": { + "defaults": { + "unit": "celsius", + "custom": { + "fillOpacity": 0, + "stacking": { + "mode": "none" + }, + "lineWidth": 1, + "spanNulls": true + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "DCGM_FI_DEV_GPU_TEMP{pod=~\"$pod\", namespace=\"foxhunt\"}", + "legendFormat": "GPU Temp", + "refId": "A" + }, + { + "expr": "DCGM_FI_DEV_MEMORY_TEMP{pod=~\"$pod\", namespace=\"foxhunt\"}", + "legendFormat": "Memory Temp", + "refId": "B" + } + ], + "gridPos": { + "x": 18, + "y": 68, + "w": 6, + "h": 7 + } + }, + { + "type": "timeseries", + "title": "Power Usage", + "description": "GPU power draw \u2014 H100 TDP is 700W", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 45, + "fieldConfig": { + "defaults": { + "unit": "watt", + "custom": { + "fillOpacity": 0, + "stacking": { + "mode": "none" + }, + "lineWidth": 1, + "spanNulls": true + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "DCGM_FI_DEV_POWER_USAGE{pod=~\"$pod\", namespace=\"foxhunt\"}", + "legendFormat": "Power (W)", + "refId": "A" + } + ], + "gridPos": { + "x": 0, + "y": 75, + "w": 6, + "h": 7 + } + }, + { + "type": "timeseries", + "title": "PCIe Throughput", + "description": "PCIe bandwidth \u2014 high RX during data loading, high TX during checkpoint saves", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 46, + "fieldConfig": { + "defaults": { + "unit": "Bps", + "custom": { + "fillOpacity": 0, + "stacking": { + "mode": "none" + }, + "lineWidth": 1, + "spanNulls": true + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "rate(DCGM_FI_PROF_PCIE_TX_BYTES{pod=~\"$pod\", namespace=\"foxhunt\"}[1m])", + "legendFormat": "TX", + "refId": "A" + }, + { + "expr": "rate(DCGM_FI_PROF_PCIE_RX_BYTES{pod=~\"$pod\", namespace=\"foxhunt\"}[1m])", + "legendFormat": "RX", + "refId": "B" + } + ], + "gridPos": { + "x": 6, + "y": 75, + "w": 6, + "h": 7 + } + }, + { + "type": "timeseries", + "title": "SM & Memory Clock", + "description": "Clock throttling indicates thermal or power issues", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 47, + "fieldConfig": { + "defaults": { + "unit": "clockhertz", + "custom": { + "fillOpacity": 0, + "stacking": { + "mode": "none" + }, + "lineWidth": 1, + "spanNulls": true + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "DCGM_FI_DEV_SM_CLOCK{pod=~\"$pod\", namespace=\"foxhunt\"}", + "legendFormat": "SM Clock", + "refId": "A" + }, + { + "expr": "DCGM_FI_DEV_MEM_CLOCK{pod=~\"$pod\", namespace=\"foxhunt\"}", + "legendFormat": "Mem Clock", + "refId": "B" + } + ], + "gridPos": { + "x": 12, + "y": 75, + "w": 6, + "h": 7 + } + }, + { + "type": "timeseries", + "title": "Memory Copy Utilization", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 48, + "fieldConfig": { + "defaults": { + "unit": "percent", + "custom": { + "fillOpacity": 10, + "stacking": { + "mode": "none" + }, + "lineWidth": 1, + "spanNulls": true + }, + "min": 0, + "max": 100 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "DCGM_FI_DEV_MEM_COPY_UTIL{pod=~\"$pod\", namespace=\"foxhunt\"}", + "legendFormat": "Mem Copy %", + "refId": "A" + } + ], + "gridPos": { + "x": 18, + "y": 75, + "w": 6, + "h": 7 + } + }, + { + "type": "row", + "title": "Throughput & Performance", + "collapsed": false, + "id": 49, + "gridPos": { + "x": 0, + "y": 82, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "type": "timeseries", + "title": "Batches / Second", + "description": "Training throughput \u2014 higher is better", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 50, + "fieldConfig": { + "defaults": { + "unit": "none", + "custom": { + "fillOpacity": 0, + "stacking": { + "mode": "none" + }, + "lineWidth": 1, + "spanNulls": true + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "foxhunt_training_batches_per_second{pod=~\"$pod\", namespace=\"foxhunt\"}", + "legendFormat": "Batches/s", + "refId": "A" + } + ], + "gridPos": { + "x": 0, + "y": 83, + "w": 6, + "h": 7 + } + }, + { + "type": "timeseries", + "title": "Epoch Duration", + "description": "Wall-clock time per epoch", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 51, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "fillOpacity": 0, + "stacking": { + "mode": "none" + }, + "lineWidth": 1, + "spanNulls": true + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "foxhunt_training_epoch_duration_seconds{pod=~\"$pod\", namespace=\"foxhunt\"}", + "legendFormat": "Epoch Time", + "refId": "A" + } + ], + "gridPos": { + "x": 6, + "y": 83, + "w": 6, + "h": 7 + } + }, + { + "type": "timeseries", + "title": "Iteration Time", + "description": "Per-step training iteration time", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 52, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "fillOpacity": 0, + "stacking": { + "mode": "none" + }, + "lineWidth": 1, + "spanNulls": true + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "foxhunt_training_iteration_seconds{pod=~\"$pod\", namespace=\"foxhunt\"}", + "legendFormat": "Iter Time", + "refId": "A" + } + ], + "gridPos": { + "x": 12, + "y": 83, + "w": 6, + "h": 7 + } + }, + { + "type": "heatmap", + "title": "Data Load Latency", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 53, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": { + "calculate": false, + "yAxis": { + "unit": "s" + }, + "color": { + "scheme": "Oranges", + "mode": "scheme" + }, + "cellGap": 1 + }, + "targets": [ + { + "expr": "rate(foxhunt_training_data_load_seconds_bucket{pod=~\"$pod\", namespace=\"foxhunt\"}[$__rate_interval])", + "legendFormat": "{{le}}", + "refId": "A", + "format": "heatmap" + } + ], + "gridPos": { + "x": 18, + "y": 83, + "w": 6, + "h": 7 + } + }, + { + "type": "timeseries", + "title": "Checkpoint Saves", + "description": "Checkpoint save count over time", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 54, + "fieldConfig": { + "defaults": { + "unit": "none", + "custom": { + "fillOpacity": 0, + "stacking": { + "mode": "none" + }, + "lineWidth": 1, + "spanNulls": true + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "foxhunt_training_checkpoint_saves_total{pod=~\"$pod\", namespace=\"foxhunt\"}", + "legendFormat": "Total Saves", + "refId": "A" + } + ], + "gridPos": { + "x": 0, + "y": 90, + "w": 6, + "h": 7 + } + }, + { + "type": "timeseries", + "title": "Checkpoint Size & Duration", + "description": "Checkpoint file size", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 55, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "fillOpacity": 0, + "stacking": { + "mode": "none" + }, + "lineWidth": 1, + "spanNulls": true + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "foxhunt_training_checkpoint_size_bytes{pod=~\"$pod\", namespace=\"foxhunt\"}", + "legendFormat": "Size (bytes)", + "refId": "A" + } + ], + "gridPos": { + "x": 6, + "y": 90, + "w": 6, + "h": 7 + } + }, + { + "type": "timeseries", + "title": "Replay Buffer Size", + "description": "DQN/PER replay buffer population", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 56, + "fieldConfig": { + "defaults": { + "unit": "none", + "custom": { + "fillOpacity": 0, + "stacking": { + "mode": "none" + }, + "lineWidth": 1, + "spanNulls": true + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "foxhunt_training_replay_buffer_size{pod=~\"$pod\", namespace=\"foxhunt\"}", + "legendFormat": "Buffer Size", + "refId": "A" + } + ], + "gridPos": { + "x": 12, + "y": 90, + "w": 6, + "h": 7 + } + }, + { + "type": "timeseries", + "title": "Active Workers", + "description": "Parallel training workers", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 57, + "fieldConfig": { + "defaults": { + "unit": "none", + "custom": { + "fillOpacity": 0, + "stacking": { + "mode": "none" + }, + "lineWidth": 1, + "spanNulls": true + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "foxhunt_training_active_workers{pod=~\"$pod\", namespace=\"foxhunt\"}", + "legendFormat": "Workers", + "refId": "A" + } + ], + "gridPos": { + "x": 18, + "y": 90, + "w": 6, + "h": 7 + } + }, + { + "type": "row", + "title": "Hyperopt Trials", + "collapsed": false, + "id": 58, + "gridPos": { + "x": 0, + "y": 97, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "type": "stat", + "title": "Current Trial", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 59, + "fieldConfig": { + "defaults": { + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "background", + "graphMode": "area", + "textMode": "auto" + }, + "targets": [ + { + "expr": "foxhunt_hyperopt_trial_current{pod=~\"$pod\", namespace=\"foxhunt\"}", + "legendFormat": "", + "refId": "A" + } + ], + "gridPos": { + "x": 0, + "y": 98, + "w": 3, + "h": 4 + } + }, + { + "type": "stat", + "title": "Total Trials", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 60, + "fieldConfig": { + "defaults": { + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "background", + "graphMode": "area", + "textMode": "auto" + }, + "targets": [ + { + "expr": "foxhunt_hyperopt_trial_total{pod=~\"$pod\", namespace=\"foxhunt\"}", + "legendFormat": "", + "refId": "A" + } + ], + "gridPos": { + "x": 3, + "y": 98, + "w": 3, + "h": 4 + } + }, + { + "type": "stat", + "title": "Best Loss", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 61, + "fieldConfig": { + "defaults": { + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.5 + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "background", + "graphMode": "area", + "textMode": "auto" + }, + "targets": [ + { + "expr": "foxhunt_hyperopt_trial_best_loss{pod=~\"$pod\", namespace=\"foxhunt\"}", + "legendFormat": "", + "refId": "A" + } + ], + "gridPos": { + "x": 6, + "y": 98, + "w": 3, + "h": 4 + } + }, + { + "type": "stat", + "title": "Elapsed", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 62, + "fieldConfig": { + "defaults": { + "unit": "s", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "background", + "graphMode": "area", + "textMode": "auto" + }, + "targets": [ + { + "expr": "foxhunt_hyperopt_elapsed_seconds{pod=~\"$pod\", namespace=\"foxhunt\"}", + "legendFormat": "", + "refId": "A" + } + ], + "gridPos": { + "x": 9, + "y": 98, + "w": 3, + "h": 4 + } + }, + { + "type": "timeseries", + "title": "Best Loss Over Trials", + "description": "Convergence curve \u2014 should decrease over trials", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 63, + "fieldConfig": { + "defaults": { + "unit": "none", + "custom": { + "fillOpacity": 0, + "stacking": { + "mode": "none" + }, + "lineWidth": 1, + "spanNulls": true + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "foxhunt_hyperopt_trial_best_loss{pod=~\"$pod\", namespace=\"foxhunt\"}", + "legendFormat": "Best Loss", + "refId": "A" + }, + { + "expr": "foxhunt_hyperopt_best_objective{pod=~\"$pod\", namespace=\"foxhunt\"}", + "legendFormat": "Best Objective", + "refId": "B" + } + ], + "gridPos": { + "x": 12, + "y": 98, + "w": 12, + "h": 7 + } + }, + { + "type": "heatmap", + "title": "Trial Duration Distribution", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 64, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": { + "calculate": false, + "yAxis": { + "unit": "s" + }, + "color": { + "scheme": "Oranges", + "mode": "scheme" + }, + "cellGap": 1 + }, + "targets": [ + { + "expr": "rate(foxhunt_hyperopt_trial_duration_seconds_bucket{pod=~\"$pod\", namespace=\"foxhunt\"}[$__rate_interval])", + "legendFormat": "{{le}}", + "refId": "A", + "format": "heatmap" + } + ], + "gridPos": { + "x": 0, + "y": 102, + "w": 12, + "h": 7 + } + }, + { + "type": "row", + "title": "Container Resources", + "collapsed": false, + "id": 65, + "gridPos": { + "x": 0, + "y": 109, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "type": "timeseries", + "title": "Container CPU Usage", + "description": "CPU cores consumed by each container in the pod", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 66, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "fillOpacity": 0, + "stacking": { + "mode": "none" + }, + "lineWidth": 1, + "spanNulls": true + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "rate(container_cpu_usage_seconds_total{pod=~\"$pod\", namespace=\"foxhunt\", container!=\"\"}[5m])", + "legendFormat": "{{container}}", + "refId": "A" + } + ], + "gridPos": { + "x": 0, + "y": 110, + "w": 6, + "h": 7 + } + }, + { + "type": "timeseries", + "title": "Container Memory (RSS)", + "description": "Resident set size \u2014 actual physical memory used", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 67, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "fillOpacity": 0, + "stacking": { + "mode": "none" + }, + "lineWidth": 1, + "spanNulls": true + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "container_memory_rss{pod=~\"$pod\", namespace=\"foxhunt\", container!=\"\"}", + "legendFormat": "{{container}}", + "refId": "A" + } + ], + "gridPos": { + "x": 6, + "y": 110, + "w": 6, + "h": 7 + } + }, + { + "type": "timeseries", + "title": "Container Network I/O", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 68, + "fieldConfig": { + "defaults": { + "unit": "Bps", + "custom": { + "fillOpacity": 0, + "stacking": { + "mode": "none" + }, + "lineWidth": 1, + "spanNulls": true + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "rate(container_network_receive_bytes_total{pod=~\"$pod\", namespace=\"foxhunt\"}[5m])", + "legendFormat": "RX {{interface}}", + "refId": "A" + }, + { + "expr": "rate(container_network_transmit_bytes_total{pod=~\"$pod\", namespace=\"foxhunt\"}[5m])", + "legendFormat": "TX {{interface}}", + "refId": "B" + } + ], + "gridPos": { + "x": 12, + "y": 110, + "w": 6, + "h": 7 + } + }, + { + "type": "timeseries", + "title": "Pod Restarts", + "description": "Non-zero restarts indicate OOM kills or crashes", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 69, + "fieldConfig": { + "defaults": { + "unit": "none", + "custom": { + "fillOpacity": 0, + "stacking": { + "mode": "none" + }, + "lineWidth": 1, + "spanNulls": true + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "kube_pod_container_status_restarts_total{pod=~\"$pod\", namespace=\"foxhunt\"}", + "legendFormat": "{{container}}", + "refId": "A" + } + ], + "gridPos": { + "x": 18, + "y": 110, + "w": 6, + "h": 7 + } + }, + { + "type": "row", + "title": "Live Logs", + "collapsed": false, + "id": 70, + "gridPos": { + "x": 0, + "y": 117, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "type": "logs", + "title": "Full Log Stream", + "description": "All logs from the selected pod, filtered by level and search text", + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "id": 71, + "options": { + "showTime": true, + "showLabels": false, + "showCommonLabels": false, + "wrapLogMessage": true, + "prettifyLogMessage": false, + "enableLogDetails": true, + "sortOrder": "Descending", + "dedupStrategy": "none" + }, + "targets": [ + { + "expr": "{namespace=\"foxhunt\", pod=~\"$pod\", level=~\"$level\"} |~ \"$search\"", + "refId": "A" + } + ], + "gridPos": { + "x": 0, + "y": 118, + "w": 12, + "h": 9 + } + }, + { + "type": "logs", + "title": "Errors & Warnings", + "description": "Only ERROR and WARN level logs", + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "id": 72, + "options": { + "showTime": true, + "showLabels": false, + "showCommonLabels": false, + "wrapLogMessage": true, + "prettifyLogMessage": false, + "enableLogDetails": true, + "sortOrder": "Descending", + "dedupStrategy": "none" + }, + "targets": [ + { + "expr": "{namespace=\"foxhunt\", pod=~\"$pod\", level=~\"ERROR|WARN\"}", + "refId": "A" + } + ], + "gridPos": { + "x": 12, + "y": 118, + "w": 12, + "h": 9 + } + }, + { + "type": "logs", + "title": "Training Progress (Epoch/Loss/Sharpe)", + "description": "Log lines mentioning training metrics \u2014 epoch completions, loss values, Sharpe ratios", + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "id": 73, + "options": { + "showTime": true, + "showLabels": false, + "showCommonLabels": false, + "wrapLogMessage": true, + "prettifyLogMessage": false, + "enableLogDetails": true, + "sortOrder": "Descending", + "dedupStrategy": "none" + }, + "targets": [ + { + "expr": "{namespace=\"foxhunt\", pod=~\"$pod\"} |~ \"(?i)(epoch|loss|sharpe|max_?dd|sortino|reward|return|accuracy|f1|precision|recall)\"", + "refId": "A" + } + ], + "gridPos": { + "x": 0, + "y": 127, + "w": 12, + "h": 9 + } + }, + { + "type": "logs", + "title": "GPU / CUDA / OOM", + "description": "GPU-related log lines \u2014 CUDA errors, OOM events, NCCL issues", + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "id": 74, + "options": { + "showTime": true, + "showLabels": false, + "showCommonLabels": false, + "wrapLogMessage": true, + "prettifyLogMessage": false, + "enableLogDetails": true, + "sortOrder": "Descending", + "dedupStrategy": "none" + }, + "targets": [ + { + "expr": "{namespace=\"foxhunt\", pod=~\"$pod\"} |~ \"(?i)(cuda|gpu|vram|oom|out.of.memory|nccl|nvrtc|tensor.core|cublas)\"", + "refId": "A" + } + ], + "gridPos": { + "x": 12, + "y": 127, + "w": 12, + "h": 9 + } + } + ], + "links": [ + { + "title": "CI Pipeline Logs", + "url": "/d/foxhunt-ci-logs/foxhunt-ci-pipeline-logs", + "type": "link" + }, + { + "title": "Training Logs", + "url": "/d/foxhunt-training-logs/foxhunt-training-pipeline-logs", + "type": "link" + }, + { + "title": "Training & CI/CD", + "url": "/d/foxhunt-training/foxhunt-training-and-ci-cd", + "type": "link" + }, + { + "title": "Operations Cockpit", + "url": "/d/foxhunt-cockpit/foxhunt-operations-cockpit", + "type": "link" + } + ] +} \ No newline at end of file diff --git a/infra/k8s/monitoring/dashboards/import.sh b/infra/k8s/monitoring/dashboards/import.sh index ef393ed8b..bf75b5d2b 100755 --- a/infra/k8s/monitoring/dashboards/import.sh +++ b/infra/k8s/monitoring/dashboards/import.sh @@ -130,6 +130,7 @@ declare -A CONFIGMAP_GROUPS=( [grafana-dashboards-observability]="foxhunt-observability.json" [grafana-dashboards-ci-logs]="foxhunt-ci-logs.json" [grafana-dashboards-training-logs]="foxhunt-training-logs.json" + [grafana-dashboards-deep-dive]="foxhunt-training-deep-dive.json" ) # Folder mapping: ConfigMap name -> Grafana folder name (used by sidecar annotation) @@ -141,6 +142,7 @@ declare -A CONFIGMAP_FOLDERS=( [grafana-dashboards-observability]="Foxhunt" [grafana-dashboards-ci-logs]="Foxhunt" [grafana-dashboards-training-logs]="Foxhunt" + [grafana-dashboards-deep-dive]="Foxhunt" ) deploy_configmaps() { @@ -235,7 +237,8 @@ else "${SCRIPT_DIR}/foxhunt-infrastructure.json" \ "${SCRIPT_DIR}/foxhunt-observability.json" \ "${SCRIPT_DIR}/foxhunt-ci-logs.json" \ - "${SCRIPT_DIR}/foxhunt-training-logs.json"; do + "${SCRIPT_DIR}/foxhunt-training-logs.json" \ + "${SCRIPT_DIR}/foxhunt-training-deep-dive.json"; do if [[ -f "$dashboard" ]]; then if ! import_dashboard "$dashboard"; then failed=$((failed + 1))