dba50fa3be
Update all dashboard JSON files to use Helm chart's Prometheus datasource UID and add Grafana sidecar folder annotations for auto-provisioning. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
3349 lines
76 KiB
JSON
3349 lines
76 KiB
JSON
{
|
|
"annotations": {
|
|
"list": [
|
|
{
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "${DS_LOKI}"
|
|
},
|
|
"enable": true,
|
|
"expr": "{namespace=\"foxhunt\", pod=~\"$pod\", level=\"ERROR\"}",
|
|
"iconColor": "red",
|
|
"name": "Errors",
|
|
"tagKeys": "level",
|
|
"titleFormat": "Error"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"enable": true,
|
|
"expr": "changes(foxhunt_training_checkpoint_saves_total{pod=~\"$pod\", namespace=\"foxhunt\"}[1m]) > 0",
|
|
"iconColor": "blue",
|
|
"name": "Checkpoints",
|
|
"titleFormat": "Checkpoint Saved"
|
|
}
|
|
]
|
|
},
|
|
"description": "Unified training pipeline monitor \u2014 target a running pod to see training curves, GPU health, eval metrics, hyperopt progress, container resources, and live logs in one place.",
|
|
"editable": true,
|
|
"fiscalYearStartMonth": 0,
|
|
"links": [
|
|
{
|
|
"title": "CI Pipeline Logs",
|
|
"type": "link",
|
|
"url": "/d/foxhunt-ci-logs/foxhunt-ci-pipeline-logs"
|
|
},
|
|
{
|
|
"title": "Training Logs",
|
|
"type": "link",
|
|
"url": "/d/foxhunt-training-logs/foxhunt-training-pipeline-logs"
|
|
},
|
|
{
|
|
"title": "Training & CI/CD",
|
|
"type": "link",
|
|
"url": "/d/foxhunt-training/foxhunt-training-and-ci-cd"
|
|
},
|
|
{
|
|
"title": "Operations Cockpit",
|
|
"type": "link",
|
|
"url": "/d/foxhunt-cockpit/foxhunt-operations-cockpit"
|
|
}
|
|
],
|
|
"liveNow": true,
|
|
"panels": [
|
|
{
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 0
|
|
},
|
|
"id": 1,
|
|
"panels": [],
|
|
"title": "Job Overview",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "blue",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 4,
|
|
"w": 3,
|
|
"x": 0,
|
|
"y": 1
|
|
},
|
|
"id": 2,
|
|
"options": {
|
|
"colorMode": "background",
|
|
"graphMode": "area",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"textMode": "auto"
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "foxhunt_training_current_epoch{pod=~\"$pod\", namespace=\"foxhunt\"}",
|
|
"legendFormat": "",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Epoch",
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "blue",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 4,
|
|
"w": 3,
|
|
"x": 3,
|
|
"y": 1
|
|
},
|
|
"id": 3,
|
|
"options": {
|
|
"colorMode": "background",
|
|
"graphMode": "area",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"textMode": "auto"
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "foxhunt_training_step{pod=~\"$pod\", namespace=\"foxhunt\"}",
|
|
"legendFormat": "",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Step",
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "yellow",
|
|
"value": 0.5
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 1
|
|
}
|
|
]
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 4,
|
|
"w": 3,
|
|
"x": 6,
|
|
"y": 1
|
|
},
|
|
"id": 4,
|
|
"options": {
|
|
"colorMode": "background",
|
|
"graphMode": "area",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"textMode": "auto"
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "foxhunt_training_epoch_loss{pod=~\"$pod\", namespace=\"foxhunt\"}",
|
|
"legendFormat": "",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Loss",
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "yellow",
|
|
"value": 0.5
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 1
|
|
}
|
|
]
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 4,
|
|
"w": 3,
|
|
"x": 9,
|
|
"y": 1
|
|
},
|
|
"id": 5,
|
|
"options": {
|
|
"colorMode": "background",
|
|
"graphMode": "area",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"textMode": "auto"
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "foxhunt_training_validation_loss{pod=~\"$pod\", namespace=\"foxhunt\"}",
|
|
"legendFormat": "",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Val Loss",
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "red",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "yellow",
|
|
"value": 0.5
|
|
},
|
|
{
|
|
"color": "green",
|
|
"value": 1
|
|
}
|
|
]
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 4,
|
|
"w": 3,
|
|
"x": 12,
|
|
"y": 1
|
|
},
|
|
"id": 6,
|
|
"options": {
|
|
"colorMode": "background",
|
|
"graphMode": "area",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"textMode": "auto"
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "foxhunt_training_epoch_sharpe{pod=~\"$pod\", namespace=\"foxhunt\"}",
|
|
"legendFormat": "",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Sharpe",
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "yellow",
|
|
"value": 0.15
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 0.3
|
|
}
|
|
]
|
|
},
|
|
"unit": "percentunit"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 4,
|
|
"w": 3,
|
|
"x": 15,
|
|
"y": 1
|
|
},
|
|
"id": 7,
|
|
"options": {
|
|
"colorMode": "background",
|
|
"graphMode": "area",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"textMode": "auto"
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "foxhunt_training_epoch_max_drawdown{pod=~\"$pod\", namespace=\"foxhunt\"}",
|
|
"legendFormat": "",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "MaxDD",
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"max": 100,
|
|
"min": 0,
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "yellow",
|
|
"value": 70
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 90
|
|
}
|
|
]
|
|
},
|
|
"unit": "percent"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 4,
|
|
"w": 3,
|
|
"x": 18,
|
|
"y": 1
|
|
},
|
|
"id": 8,
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"showThresholdLabels": false,
|
|
"showThresholdMarkers": true
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "DCGM_FI_DEV_GPU_UTIL{pod=~\"$pod\", namespace=\"foxhunt\"}",
|
|
"legendFormat": "",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "GPU Util",
|
|
"type": "gauge"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"max": 100,
|
|
"min": 0,
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "yellow",
|
|
"value": 70
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 90
|
|
}
|
|
]
|
|
},
|
|
"unit": "percent"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 4,
|
|
"w": 3,
|
|
"x": 21,
|
|
"y": 1
|
|
},
|
|
"id": 9,
|
|
"options": {
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"showThresholdLabels": false,
|
|
"showThresholdMarkers": true
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "DCGM_FI_DEV_FB_USED{pod=~\"$pod\", namespace=\"foxhunt\"} / (DCGM_FI_DEV_FB_USED{pod=~\"$pod\", namespace=\"foxhunt\"} + DCGM_FI_DEV_FB_FREE{pod=~\"$pod\", namespace=\"foxhunt\"}) * 100",
|
|
"legendFormat": "",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "VRAM Used",
|
|
"type": "gauge"
|
|
},
|
|
{
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 5
|
|
},
|
|
"id": 10,
|
|
"panels": [],
|
|
"title": "CI Pipeline & Argo Workflows",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "Currently running Argo workflows",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "blue",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 4,
|
|
"w": 3,
|
|
"x": 0,
|
|
"y": 6
|
|
},
|
|
"id": 11,
|
|
"options": {
|
|
"colorMode": "background",
|
|
"graphMode": "area",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"textMode": "auto"
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "argo_workflows_gauge{status=\"Running\"}",
|
|
"legendFormat": "",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Active Workflows",
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "yellow",
|
|
"value": 3
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 10
|
|
}
|
|
]
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 4,
|
|
"w": 3,
|
|
"x": 3,
|
|
"y": 6
|
|
},
|
|
"id": 12,
|
|
"options": {
|
|
"colorMode": "background",
|
|
"graphMode": "area",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"textMode": "auto"
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "argo_workflows_pod_pending_count",
|
|
"legendFormat": "",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Pending Pods",
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 4,
|
|
"w": 3,
|
|
"x": 6,
|
|
"y": 6
|
|
},
|
|
"id": 13,
|
|
"options": {
|
|
"colorMode": "background",
|
|
"graphMode": "area",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"textMode": "auto"
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "argo_workflows_total_count{status=\"Succeeded\"}",
|
|
"legendFormat": "",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Total Completed",
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 1
|
|
}
|
|
]
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 4,
|
|
"w": 3,
|
|
"x": 9,
|
|
"y": 6
|
|
},
|
|
"id": 14,
|
|
"options": {
|
|
"colorMode": "background",
|
|
"graphMode": "area",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"textMode": "auto"
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "argo_workflows_error_count",
|
|
"legendFormat": "",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Errors",
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "Average workflow operation duration",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"fillOpacity": 0,
|
|
"lineWidth": 1,
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"mode": "none"
|
|
}
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 6,
|
|
"x": 12,
|
|
"y": 6
|
|
},
|
|
"id": 15,
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "rate(argo_workflows_operation_duration_seconds_sum[5m]) / rate(argo_workflows_operation_duration_seconds_count[5m])",
|
|
"legendFormat": "Avg Duration",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Workflow Duration",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "Argo controller K8s API calls \u2014 high rates may indicate thrashing",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"fillOpacity": 0,
|
|
"lineWidth": 1,
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"mode": "none"
|
|
}
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 6,
|
|
"x": 18,
|
|
"y": 6
|
|
},
|
|
"id": 16,
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "rate(argo_workflows_k8s_request_total[5m])",
|
|
"legendFormat": "{{verb}} {{status_code}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "K8s API Request Rate",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "CPU usage of selected CI/training pods",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"fillOpacity": 0,
|
|
"lineWidth": 1,
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"mode": "none"
|
|
}
|
|
},
|
|
"unit": "short"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 6,
|
|
"x": 0,
|
|
"y": 10
|
|
},
|
|
"id": 17,
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"$pod\"}[5m])",
|
|
"legendFormat": "{{container}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "CI Pod CPU",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "RSS memory of selected CI/training pods",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"fillOpacity": 0,
|
|
"lineWidth": 1,
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"mode": "none"
|
|
}
|
|
},
|
|
"unit": "bytes"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 6,
|
|
"x": 6,
|
|
"y": 10
|
|
},
|
|
"id": 18,
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "container_memory_rss{namespace=\"foxhunt\", pod=~\"$pod\"}",
|
|
"legendFormat": "{{container}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "CI Pod Memory",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 17
|
|
},
|
|
"id": 19,
|
|
"panels": [],
|
|
"title": "CI Logs",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "${DS_LOKI}"
|
|
},
|
|
"description": "All logs from selected CI/training pods",
|
|
"gridPos": {
|
|
"h": 9,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 18
|
|
},
|
|
"id": 20,
|
|
"options": {
|
|
"dedupStrategy": "none",
|
|
"enableLogDetails": true,
|
|
"prettifyLogMessage": false,
|
|
"showCommonLabels": false,
|
|
"showLabels": false,
|
|
"showTime": true,
|
|
"sortOrder": "Descending",
|
|
"wrapLogMessage": true
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "{namespace=\"foxhunt\", pod=~\"$pod\"} |~ \"$search\" | pattern `<_> <_> <_> <content>` | regexp `msg=\"(?P<clean_msg>[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "CI Full Log Stream",
|
|
"type": "logs"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "${DS_LOKI}"
|
|
},
|
|
"description": "Rust compile output, test results, clippy warnings",
|
|
"gridPos": {
|
|
"h": 9,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 18
|
|
},
|
|
"id": 21,
|
|
"options": {
|
|
"dedupStrategy": "none",
|
|
"enableLogDetails": true,
|
|
"prettifyLogMessage": false,
|
|
"showCommonLabels": false,
|
|
"showLabels": false,
|
|
"showTime": true,
|
|
"sortOrder": "Descending",
|
|
"wrapLogMessage": true
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "{namespace=\"foxhunt\", pod=~\"$pod\"} |~ \"(?i)(cargo (build|test|check|clippy)|Compiling|Finished|running \\\\d+ test|test result|FAILED|passed|warning:)\" | pattern `<_> <_> <_> <content>` | regexp `msg=\"(?P<clean_msg>[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Compile & Test Results",
|
|
"type": "logs"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "${DS_LOKI}"
|
|
},
|
|
"description": "S3 binary uploads, kubectl deploys, image builds",
|
|
"gridPos": {
|
|
"h": 9,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 27
|
|
},
|
|
"id": 22,
|
|
"options": {
|
|
"dedupStrategy": "none",
|
|
"enableLogDetails": true,
|
|
"prettifyLogMessage": false,
|
|
"showCommonLabels": false,
|
|
"showLabels": false,
|
|
"showTime": true,
|
|
"sortOrder": "Descending",
|
|
"wrapLogMessage": true
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "{namespace=\"foxhunt\", pod=~\"$pod\"} |~ \"(?i)(rclone|s3://|upload|download|kubectl|rollout|image|docker|kaniko|configmap)\" | pattern `<_> <_> <_> <content>` | regexp `msg=\"(?P<clean_msg>[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Build Artifacts & Deploy",
|
|
"type": "logs"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "${DS_LOKI}"
|
|
},
|
|
"description": "Data fetching, redis cache, dbn file operations",
|
|
"gridPos": {
|
|
"h": 9,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 27
|
|
},
|
|
"id": 23,
|
|
"options": {
|
|
"dedupStrategy": "none",
|
|
"enableLogDetails": true,
|
|
"prettifyLogMessage": false,
|
|
"showCommonLabels": false,
|
|
"showLabels": false,
|
|
"showTime": true,
|
|
"sortOrder": "Descending",
|
|
"wrapLogMessage": true
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "{namespace=\"foxhunt\", pod=~\"$pod\"} |~ \"(?i)(redis|rclone|dbn|data|cache|download|decompress)\" | pattern `<_> <_> <_> <content>` | regexp `msg=\"(?P<clean_msg>[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Redis Sidecar & Data",
|
|
"type": "logs"
|
|
},
|
|
{
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 36
|
|
},
|
|
"id": 24,
|
|
"panels": [],
|
|
"title": "Training Curves",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "Training and validation loss over time",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"fillOpacity": 0,
|
|
"lineWidth": 1,
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"mode": "none"
|
|
}
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 37
|
|
},
|
|
"id": 25,
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "foxhunt_training_epoch_loss{pod=~\"$pod\", namespace=\"foxhunt\"}",
|
|
"legendFormat": "Train Loss",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"expr": "foxhunt_training_validation_loss{pod=~\"$pod\", namespace=\"foxhunt\"}",
|
|
"legendFormat": "Val Loss",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"title": "Loss & Validation Loss",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "Learning rate schedule",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"fillOpacity": 0,
|
|
"lineWidth": 1,
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"mode": "none"
|
|
}
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 37
|
|
},
|
|
"id": 26,
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "foxhunt_training_learning_rate{pod=~\"$pod\", namespace=\"foxhunt\"}",
|
|
"legendFormat": "LR",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Learning Rate",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "Gradient L2 norm \u2014 spikes indicate instability",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"fillOpacity": 0,
|
|
"lineWidth": 1,
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"mode": "none"
|
|
}
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 37
|
|
},
|
|
"id": 27,
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "foxhunt_training_gradient_norm{pod=~\"$pod\", namespace=\"foxhunt\"}",
|
|
"legendFormat": "Grad Norm",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Gradient Norm",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 44
|
|
},
|
|
"id": 28,
|
|
"panels": [],
|
|
"title": "Trading Performance",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "Risk-adjusted return metrics per epoch",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"fillOpacity": 0,
|
|
"lineWidth": 1,
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"mode": "none"
|
|
}
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 45
|
|
},
|
|
"id": 29,
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "foxhunt_training_epoch_sharpe{pod=~\"$pod\", namespace=\"foxhunt\"}",
|
|
"legendFormat": "Sharpe",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"expr": "foxhunt_training_epoch_sortino{pod=~\"$pod\", namespace=\"foxhunt\"}",
|
|
"legendFormat": "Sortino",
|
|
"refId": "B"
|
|
},
|
|
{
|
|
"expr": "foxhunt_training_epoch_max_drawdown{pod=~\"$pod\", namespace=\"foxhunt\"}",
|
|
"legendFormat": "MaxDD",
|
|
"refId": "C"
|
|
}
|
|
],
|
|
"title": "Sharpe / Sortino / MaxDD",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"fillOpacity": 0,
|
|
"lineWidth": 1,
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"mode": "none"
|
|
}
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 45
|
|
},
|
|
"id": 30,
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "foxhunt_training_epoch_win_rate{pod=~\"$pod\", namespace=\"foxhunt\"}",
|
|
"legendFormat": "Win Rate",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"expr": "foxhunt_training_epoch_profit_factor{pod=~\"$pod\", namespace=\"foxhunt\"}",
|
|
"legendFormat": "Profit Factor",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"title": "Win Rate & Profit Factor",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"fillOpacity": 0,
|
|
"lineWidth": 1,
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"mode": "none"
|
|
}
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 45
|
|
},
|
|
"id": 31,
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "foxhunt_training_epoch_total_return{pod=~\"$pod\", namespace=\"foxhunt\"}",
|
|
"legendFormat": "Total Return",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"expr": "foxhunt_training_epoch_avg_return{pod=~\"$pod\", namespace=\"foxhunt\"}",
|
|
"legendFormat": "Avg Return",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"title": "Total Return & Avg Return",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "Action distribution per epoch \u2014 watch for collapse to single action",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"fillOpacity": 30,
|
|
"lineWidth": 1,
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"mode": "normal"
|
|
}
|
|
},
|
|
"max": 1,
|
|
"min": 0,
|
|
"unit": "percentunit"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 8,
|
|
"x": 0,
|
|
"y": 52
|
|
},
|
|
"id": 32,
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "foxhunt_training_epoch_action_buy_pct{pod=~\"$pod\", namespace=\"foxhunt\"}",
|
|
"legendFormat": "Buy %",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"expr": "foxhunt_training_epoch_action_hold_pct{pod=~\"$pod\", namespace=\"foxhunt\"}",
|
|
"legendFormat": "Hold %",
|
|
"refId": "B"
|
|
},
|
|
{
|
|
"expr": "foxhunt_training_epoch_action_sell_pct{pod=~\"$pod\", namespace=\"foxhunt\"}",
|
|
"legendFormat": "Sell %",
|
|
"refId": "C"
|
|
}
|
|
],
|
|
"title": "Action Distribution",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "Higher entropy = more exploration. Collapse \u2192 policy degeneration",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"fillOpacity": 0,
|
|
"lineWidth": 1,
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"mode": "none"
|
|
}
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 8,
|
|
"x": 8,
|
|
"y": 52
|
|
},
|
|
"id": 33,
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "foxhunt_training_action_entropy{pod=~\"$pod\", namespace=\"foxhunt\"}",
|
|
"legendFormat": "Entropy",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"expr": "foxhunt_training_action_diversity{pod=~\"$pod\", namespace=\"foxhunt\"}",
|
|
"legendFormat": "Diversity",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"title": "Action Entropy & Diversity",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"fillOpacity": 0,
|
|
"lineWidth": 1,
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"mode": "none"
|
|
}
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 52
|
|
},
|
|
"id": 34,
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "foxhunt_training_epoch_total_trades{pod=~\"$pod\", namespace=\"foxhunt\"}",
|
|
"legendFormat": "Trades/Epoch",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Total Trades",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 59
|
|
},
|
|
"id": 35,
|
|
"panels": [],
|
|
"title": "Evaluation Metrics",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"fillOpacity": 0,
|
|
"lineWidth": 1,
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"mode": "none"
|
|
}
|
|
},
|
|
"max": 1,
|
|
"min": 0,
|
|
"unit": "percentunit"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 6,
|
|
"x": 0,
|
|
"y": 60
|
|
},
|
|
"id": 36,
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "foxhunt_training_eval_accuracy{pod=~\"$pod\", namespace=\"foxhunt\"}",
|
|
"legendFormat": "Accuracy",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Eval Accuracy",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "Classification quality of directional predictions",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"fillOpacity": 0,
|
|
"lineWidth": 1,
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"mode": "none"
|
|
}
|
|
},
|
|
"max": 1,
|
|
"min": 0,
|
|
"unit": "percentunit"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 6,
|
|
"x": 6,
|
|
"y": 60
|
|
},
|
|
"id": 37,
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "foxhunt_training_eval_precision{pod=~\"$pod\", namespace=\"foxhunt\"}",
|
|
"legendFormat": "Precision",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"expr": "foxhunt_training_eval_recall{pod=~\"$pod\", namespace=\"foxhunt\"}",
|
|
"legendFormat": "Recall",
|
|
"refId": "B"
|
|
},
|
|
{
|
|
"expr": "foxhunt_training_eval_f1{pod=~\"$pod\", namespace=\"foxhunt\"}",
|
|
"legendFormat": "F1",
|
|
"refId": "C"
|
|
}
|
|
],
|
|
"title": "Precision / Recall / F1",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "DQN Q-value estimates \u2014 divergence from max indicates overestimation",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"fillOpacity": 0,
|
|
"lineWidth": 1,
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"mode": "none"
|
|
}
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 6,
|
|
"x": 12,
|
|
"y": 60
|
|
},
|
|
"id": 38,
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "foxhunt_training_q_value_mean{pod=~\"$pod\", namespace=\"foxhunt\"}",
|
|
"legendFormat": "Q Mean",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"expr": "foxhunt_training_q_value_max{pod=~\"$pod\", namespace=\"foxhunt\"}",
|
|
"legendFormat": "Q Max",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"title": "Q-Values (DQN)",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "Ratio > 1 = Q-values overestimate true returns",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"fillOpacity": 0,
|
|
"lineWidth": 1,
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"mode": "none"
|
|
}
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 6,
|
|
"x": 18,
|
|
"y": 60
|
|
},
|
|
"id": 39,
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "foxhunt_training_q_overestimation_ratio{pod=~\"$pod\", namespace=\"foxhunt\"}",
|
|
"legendFormat": "Overestimation Ratio",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Q-Overestimation & Replay Buffer",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 67
|
|
},
|
|
"id": 40,
|
|
"panels": [],
|
|
"title": "GPU & Hardware",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "Overall GPU and graphics engine utilization",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"fillOpacity": 10,
|
|
"lineWidth": 1,
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"mode": "none"
|
|
}
|
|
},
|
|
"max": 100,
|
|
"min": 0,
|
|
"unit": "percent"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 6,
|
|
"x": 0,
|
|
"y": 68
|
|
},
|
|
"id": 41,
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "DCGM_FI_DEV_GPU_UTIL{pod=~\"$pod\", namespace=\"foxhunt\"}",
|
|
"legendFormat": "GPU %",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"expr": "DCGM_FI_PROF_GR_ENGINE_ACTIVE{pod=~\"$pod\", namespace=\"foxhunt\"} * 100",
|
|
"legendFormat": "GR Engine %",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"title": "GPU Utilization",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "Tensor core and DRAM bandwidth utilization \u2014 key H100 saturation indicators",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"fillOpacity": 10,
|
|
"lineWidth": 1,
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"mode": "none"
|
|
}
|
|
},
|
|
"max": 100,
|
|
"min": 0,
|
|
"unit": "percent"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 6,
|
|
"x": 6,
|
|
"y": 68
|
|
},
|
|
"id": 42,
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{pod=~\"$pod\", namespace=\"foxhunt\"} * 100",
|
|
"legendFormat": "Tensor Cores %",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"expr": "DCGM_FI_PROF_DRAM_ACTIVE{pod=~\"$pod\", namespace=\"foxhunt\"} * 100",
|
|
"legendFormat": "DRAM Active %",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"title": "Tensor Core Activity",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "GPU framebuffer memory usage",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"fillOpacity": 20,
|
|
"lineWidth": 1,
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"mode": "normal"
|
|
}
|
|
},
|
|
"unit": "decmbytes"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 6,
|
|
"x": 12,
|
|
"y": 68
|
|
},
|
|
"id": 43,
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "DCGM_FI_DEV_FB_USED{pod=~\"$pod\", namespace=\"foxhunt\"}",
|
|
"legendFormat": "Used (MiB)",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"expr": "DCGM_FI_DEV_FB_FREE{pod=~\"$pod\", namespace=\"foxhunt\"}",
|
|
"legendFormat": "Free (MiB)",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"title": "VRAM (Framebuffer)",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "Thermal throttling starts at ~83\u00b0C on H100",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"fillOpacity": 0,
|
|
"lineWidth": 1,
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"mode": "none"
|
|
}
|
|
},
|
|
"unit": "celsius"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 6,
|
|
"x": 18,
|
|
"y": 68
|
|
},
|
|
"id": 44,
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "DCGM_FI_DEV_GPU_TEMP{pod=~\"$pod\", namespace=\"foxhunt\"}",
|
|
"legendFormat": "GPU Temp",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"expr": "DCGM_FI_DEV_MEMORY_TEMP{pod=~\"$pod\", namespace=\"foxhunt\"}",
|
|
"legendFormat": "Memory Temp",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"title": "GPU Temperature",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "GPU power draw \u2014 H100 TDP is 700W",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"fillOpacity": 0,
|
|
"lineWidth": 1,
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"mode": "none"
|
|
}
|
|
},
|
|
"unit": "watt"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 6,
|
|
"x": 0,
|
|
"y": 75
|
|
},
|
|
"id": 45,
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "DCGM_FI_DEV_POWER_USAGE{pod=~\"$pod\", namespace=\"foxhunt\"}",
|
|
"legendFormat": "Power (W)",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Power Usage",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "PCIe bandwidth \u2014 high RX during data loading, high TX during checkpoint saves",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"fillOpacity": 0,
|
|
"lineWidth": 1,
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"mode": "none"
|
|
}
|
|
},
|
|
"unit": "Bps"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 6,
|
|
"x": 6,
|
|
"y": 75
|
|
},
|
|
"id": 46,
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "rate(DCGM_FI_PROF_PCIE_TX_BYTES{pod=~\"$pod\", namespace=\"foxhunt\"}[1m])",
|
|
"legendFormat": "TX",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"expr": "rate(DCGM_FI_PROF_PCIE_RX_BYTES{pod=~\"$pod\", namespace=\"foxhunt\"}[1m])",
|
|
"legendFormat": "RX",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"title": "PCIe Throughput",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "Clock throttling indicates thermal or power issues",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"fillOpacity": 0,
|
|
"lineWidth": 1,
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"mode": "none"
|
|
}
|
|
},
|
|
"unit": "clockhertz"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 6,
|
|
"x": 12,
|
|
"y": 75
|
|
},
|
|
"id": 47,
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "DCGM_FI_DEV_SM_CLOCK{pod=~\"$pod\", namespace=\"foxhunt\"}",
|
|
"legendFormat": "SM Clock",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"expr": "DCGM_FI_DEV_MEM_CLOCK{pod=~\"$pod\", namespace=\"foxhunt\"}",
|
|
"legendFormat": "Mem Clock",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"title": "SM & Memory Clock",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"fillOpacity": 10,
|
|
"lineWidth": 1,
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"mode": "none"
|
|
}
|
|
},
|
|
"max": 100,
|
|
"min": 0,
|
|
"unit": "percent"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 6,
|
|
"x": 18,
|
|
"y": 75
|
|
},
|
|
"id": 48,
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "DCGM_FI_DEV_MEM_COPY_UTIL{pod=~\"$pod\", namespace=\"foxhunt\"}",
|
|
"legendFormat": "Mem Copy %",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Memory Copy Utilization",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 82
|
|
},
|
|
"id": 49,
|
|
"panels": [],
|
|
"title": "Throughput & Performance",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "Training throughput \u2014 higher is better",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"fillOpacity": 0,
|
|
"lineWidth": 1,
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"mode": "none"
|
|
}
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 6,
|
|
"x": 0,
|
|
"y": 83
|
|
},
|
|
"id": 50,
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "foxhunt_training_batches_per_second{pod=~\"$pod\", namespace=\"foxhunt\"}",
|
|
"legendFormat": "Batches/s",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Batches / Second",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "Wall-clock time per epoch",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"fillOpacity": 0,
|
|
"lineWidth": 1,
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"mode": "none"
|
|
}
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 6,
|
|
"x": 6,
|
|
"y": 83
|
|
},
|
|
"id": 51,
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "foxhunt_training_epoch_duration_seconds{pod=~\"$pod\", namespace=\"foxhunt\"}",
|
|
"legendFormat": "Epoch Time",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Epoch Duration",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "Per-step training iteration time",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"fillOpacity": 0,
|
|
"lineWidth": 1,
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"mode": "none"
|
|
}
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 6,
|
|
"x": 12,
|
|
"y": 83
|
|
},
|
|
"id": 52,
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "foxhunt_training_iteration_seconds{pod=~\"$pod\", namespace=\"foxhunt\"}",
|
|
"legendFormat": "Iter Time",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Iteration Time",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 6,
|
|
"x": 18,
|
|
"y": 83
|
|
},
|
|
"id": 53,
|
|
"options": {
|
|
"calculate": false,
|
|
"cellGap": 1,
|
|
"color": {
|
|
"mode": "scheme",
|
|
"scheme": "Oranges"
|
|
},
|
|
"yAxis": {
|
|
"unit": "s"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "rate(foxhunt_training_data_load_seconds_bucket{pod=~\"$pod\", namespace=\"foxhunt\"}[$__rate_interval])",
|
|
"format": "heatmap",
|
|
"legendFormat": "{{le}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Data Load Latency",
|
|
"type": "heatmap"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "Checkpoint save count over time",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"fillOpacity": 0,
|
|
"lineWidth": 1,
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"mode": "none"
|
|
}
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 6,
|
|
"x": 0,
|
|
"y": 90
|
|
},
|
|
"id": 54,
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "foxhunt_training_checkpoint_saves_total{pod=~\"$pod\", namespace=\"foxhunt\"}",
|
|
"legendFormat": "Total Saves",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Checkpoint Saves",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "Checkpoint file size",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"fillOpacity": 0,
|
|
"lineWidth": 1,
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"mode": "none"
|
|
}
|
|
},
|
|
"unit": "bytes"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 6,
|
|
"x": 6,
|
|
"y": 90
|
|
},
|
|
"id": 55,
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "foxhunt_training_checkpoint_size_bytes{pod=~\"$pod\", namespace=\"foxhunt\"}",
|
|
"legendFormat": "Size (bytes)",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Checkpoint Size & Duration",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "DQN/PER replay buffer population",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"fillOpacity": 0,
|
|
"lineWidth": 1,
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"mode": "none"
|
|
}
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 6,
|
|
"x": 12,
|
|
"y": 90
|
|
},
|
|
"id": 56,
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "foxhunt_training_replay_buffer_size{pod=~\"$pod\", namespace=\"foxhunt\"}",
|
|
"legendFormat": "Buffer Size",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Replay Buffer Size",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "Parallel training workers",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"fillOpacity": 0,
|
|
"lineWidth": 1,
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"mode": "none"
|
|
}
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 6,
|
|
"x": 18,
|
|
"y": 90
|
|
},
|
|
"id": 57,
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "foxhunt_training_active_workers{pod=~\"$pod\", namespace=\"foxhunt\"}",
|
|
"legendFormat": "Workers",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Active Workers",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 97
|
|
},
|
|
"id": 58,
|
|
"panels": [],
|
|
"title": "Hyperopt Trials",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "purple",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 4,
|
|
"w": 3,
|
|
"x": 0,
|
|
"y": 98
|
|
},
|
|
"id": 59,
|
|
"options": {
|
|
"colorMode": "background",
|
|
"graphMode": "area",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"textMode": "auto"
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "foxhunt_hyperopt_trial_current{pod=~\"$pod\", namespace=\"foxhunt\"}",
|
|
"legendFormat": "",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Current Trial",
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "purple",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 4,
|
|
"w": 3,
|
|
"x": 3,
|
|
"y": 98
|
|
},
|
|
"id": 60,
|
|
"options": {
|
|
"colorMode": "background",
|
|
"graphMode": "area",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"textMode": "auto"
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "foxhunt_hyperopt_trial_total{pod=~\"$pod\", namespace=\"foxhunt\"}",
|
|
"legendFormat": "",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Total Trials",
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "yellow",
|
|
"value": 0.5
|
|
}
|
|
]
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 4,
|
|
"w": 3,
|
|
"x": 6,
|
|
"y": 98
|
|
},
|
|
"id": 61,
|
|
"options": {
|
|
"colorMode": "background",
|
|
"graphMode": "area",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"textMode": "auto"
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "foxhunt_hyperopt_trial_best_loss{pod=~\"$pod\", namespace=\"foxhunt\"}",
|
|
"legendFormat": "",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Best Loss",
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "blue",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 4,
|
|
"w": 3,
|
|
"x": 9,
|
|
"y": 98
|
|
},
|
|
"id": 62,
|
|
"options": {
|
|
"colorMode": "background",
|
|
"graphMode": "area",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
]
|
|
},
|
|
"textMode": "auto"
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "foxhunt_hyperopt_elapsed_seconds{pod=~\"$pod\", namespace=\"foxhunt\"}",
|
|
"legendFormat": "",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Elapsed",
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "Convergence curve \u2014 should decrease over trials",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"fillOpacity": 0,
|
|
"lineWidth": 1,
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"mode": "none"
|
|
}
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 98
|
|
},
|
|
"id": 63,
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "foxhunt_hyperopt_trial_best_loss{pod=~\"$pod\", namespace=\"foxhunt\"}",
|
|
"legendFormat": "Best Loss",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"expr": "foxhunt_hyperopt_best_objective{pod=~\"$pod\", namespace=\"foxhunt\"}",
|
|
"legendFormat": "Best Objective",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"title": "Best Loss Over Trials",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 102
|
|
},
|
|
"id": 64,
|
|
"options": {
|
|
"calculate": false,
|
|
"cellGap": 1,
|
|
"color": {
|
|
"mode": "scheme",
|
|
"scheme": "Oranges"
|
|
},
|
|
"yAxis": {
|
|
"unit": "s"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "rate(foxhunt_hyperopt_trial_duration_seconds_bucket{pod=~\"$pod\", namespace=\"foxhunt\"}[$__rate_interval])",
|
|
"format": "heatmap",
|
|
"legendFormat": "{{le}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Trial Duration Distribution",
|
|
"type": "heatmap"
|
|
},
|
|
{
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 109
|
|
},
|
|
"id": 65,
|
|
"panels": [],
|
|
"title": "Container Resources",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "CPU cores consumed by each container in the pod",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"fillOpacity": 0,
|
|
"lineWidth": 1,
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"mode": "none"
|
|
}
|
|
},
|
|
"unit": "short"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 6,
|
|
"x": 0,
|
|
"y": 110
|
|
},
|
|
"id": 66,
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "rate(container_cpu_usage_seconds_total{pod=~\"$pod\", namespace=\"foxhunt\", container!=\"\"}[5m])",
|
|
"legendFormat": "{{container}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Container CPU Usage",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "Resident set size \u2014 actual physical memory used",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"fillOpacity": 0,
|
|
"lineWidth": 1,
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"mode": "none"
|
|
}
|
|
},
|
|
"unit": "bytes"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 6,
|
|
"x": 6,
|
|
"y": 110
|
|
},
|
|
"id": 67,
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "container_memory_rss{pod=~\"$pod\", namespace=\"foxhunt\", container!=\"\"}",
|
|
"legendFormat": "{{container}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Container Memory (RSS)",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"fillOpacity": 0,
|
|
"lineWidth": 1,
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"mode": "none"
|
|
}
|
|
},
|
|
"unit": "Bps"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 6,
|
|
"x": 12,
|
|
"y": 110
|
|
},
|
|
"id": 68,
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "rate(container_network_receive_bytes_total{pod=~\"$pod\", namespace=\"foxhunt\"}[5m])",
|
|
"legendFormat": "RX {{interface}}",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"expr": "rate(container_network_transmit_bytes_total{pod=~\"$pod\", namespace=\"foxhunt\"}[5m])",
|
|
"legendFormat": "TX {{interface}}",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"title": "Container Network I/O",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"description": "Non-zero restarts indicate OOM kills or crashes",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"fillOpacity": 0,
|
|
"lineWidth": 1,
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"mode": "none"
|
|
}
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 7,
|
|
"w": 6,
|
|
"x": 18,
|
|
"y": 110
|
|
},
|
|
"id": 69,
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "bottom"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "kube_pod_container_status_restarts_total{pod=~\"$pod\", namespace=\"foxhunt\"}",
|
|
"legendFormat": "{{container}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Pod Restarts",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 117
|
|
},
|
|
"id": 70,
|
|
"panels": [],
|
|
"title": "Live Logs",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "${DS_LOKI}"
|
|
},
|
|
"description": "All logs from the selected pod, filtered by level and search text",
|
|
"gridPos": {
|
|
"h": 9,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 118
|
|
},
|
|
"id": 71,
|
|
"options": {
|
|
"dedupStrategy": "none",
|
|
"enableLogDetails": true,
|
|
"prettifyLogMessage": false,
|
|
"showCommonLabels": false,
|
|
"showLabels": false,
|
|
"showTime": true,
|
|
"sortOrder": "Descending",
|
|
"wrapLogMessage": true
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "{namespace=\"foxhunt\", pod=~\"$pod\", level=~\"$level\"} |~ \"$search\" | pattern `<_> <_> <_> <content>` | regexp `msg=\"(?P<clean_msg>[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Full Log Stream",
|
|
"type": "logs"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "${DS_LOKI}"
|
|
},
|
|
"description": "Only ERROR and WARN level logs",
|
|
"gridPos": {
|
|
"h": 9,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 118
|
|
},
|
|
"id": 72,
|
|
"options": {
|
|
"dedupStrategy": "none",
|
|
"enableLogDetails": true,
|
|
"prettifyLogMessage": false,
|
|
"showCommonLabels": false,
|
|
"showLabels": false,
|
|
"showTime": true,
|
|
"sortOrder": "Descending",
|
|
"wrapLogMessage": true
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "{namespace=\"foxhunt\", pod=~\"$pod\", level=~\"ERROR|WARN\"} | pattern `<_> <_> <_> <content>` | regexp `msg=\"(?P<clean_msg>[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Errors & Warnings",
|
|
"type": "logs"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "${DS_LOKI}"
|
|
},
|
|
"description": "Log lines mentioning training metrics \u2014 epoch completions, loss values, Sharpe ratios",
|
|
"gridPos": {
|
|
"h": 9,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 127
|
|
},
|
|
"id": 73,
|
|
"options": {
|
|
"dedupStrategy": "none",
|
|
"enableLogDetails": true,
|
|
"prettifyLogMessage": false,
|
|
"showCommonLabels": false,
|
|
"showLabels": false,
|
|
"showTime": true,
|
|
"sortOrder": "Descending",
|
|
"wrapLogMessage": true
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "{namespace=\"foxhunt\", pod=~\"$pod\"} |~ \"(?i)(epoch|loss|sharpe|max_?dd|sortino|reward|return|accuracy|f1|precision|recall)\" | pattern `<_> <_> <_> <content>` | regexp `msg=\"(?P<clean_msg>[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Training Progress (Epoch/Loss/Sharpe)",
|
|
"type": "logs"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "${DS_LOKI}"
|
|
},
|
|
"description": "GPU-related log lines \u2014 CUDA errors, OOM events, NCCL issues",
|
|
"gridPos": {
|
|
"h": 9,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 127
|
|
},
|
|
"id": 74,
|
|
"options": {
|
|
"dedupStrategy": "none",
|
|
"enableLogDetails": true,
|
|
"prettifyLogMessage": false,
|
|
"showCommonLabels": false,
|
|
"showLabels": false,
|
|
"showTime": true,
|
|
"sortOrder": "Descending",
|
|
"wrapLogMessage": true
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "{namespace=\"foxhunt\", pod=~\"$pod\"} |~ \"(?i)(cuda|gpu|vram|oom|out.of.memory|nccl|nvrtc|tensor.core|cublas)\" | pattern `<_> <_> <_> <content>` | regexp `msg=\"(?P<clean_msg>[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "GPU / CUDA / OOM",
|
|
"type": "logs"
|
|
}
|
|
],
|
|
"refresh": "10s",
|
|
"schemaVersion": 39,
|
|
"tags": [
|
|
"foxhunt",
|
|
"training",
|
|
"gpu",
|
|
"loki",
|
|
"deep-dive"
|
|
],
|
|
"templating": {
|
|
"list": [
|
|
{
|
|
"current": {},
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "${DS_LOKI}"
|
|
},
|
|
"includeAll": true,
|
|
"label": "Pod",
|
|
"multi": true,
|
|
"name": "pod",
|
|
"query": "{namespace=\"foxhunt\", pod\\!~\"(grafana|prometheus|loki|promtail|${DS_TEMPO}|redis|postgres|mattermost|tailscale|gitlab|cilium|csi|konnectivity|nvidia-gpu|argo-workflows|questdb|web-gateway|trading-service).*\"}",
|
|
"refresh": 2,
|
|
"regex": "",
|
|
"sort": 2,
|
|
"type": "query"
|
|
},
|
|
{
|
|
"current": {},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"includeAll": true,
|
|
"label": "Prometheus Job",
|
|
"multi": false,
|
|
"name": "prom_job",
|
|
"query": "label_values(up{namespace=\"foxhunt\"}, job)",
|
|
"refresh": 2,
|
|
"regex": "/(train|hyperopt|evaluate|pushgateway|training-pods|dcgm)/",
|
|
"sort": 1,
|
|
"type": "query"
|
|
},
|
|
{
|
|
"current": {
|
|
"text": "All",
|
|
"value": "$__all"
|
|
},
|
|
"includeAll": true,
|
|
"label": "Log Level",
|
|
"multi": true,
|
|
"name": "level",
|
|
"query": "ERROR,WARN,INFO,DEBUG,TRACE",
|
|
"type": "custom"
|
|
},
|
|
{
|
|
"current": {
|
|
"text": "",
|
|
"value": ""
|
|
},
|
|
"label": "Search",
|
|
"name": "search",
|
|
"type": "textbox"
|
|
}
|
|
]
|
|
},
|
|
"time": {
|
|
"from": "now-3h",
|
|
"to": "now"
|
|
},
|
|
"timezone": "browser",
|
|
"title": "Training Deep-Dive",
|
|
"uid": "foxhunt-training-deep-dive",
|
|
"__inputs": [
|
|
{
|
|
"name": "DS_PROMETHEUS",
|
|
"type": "datasource",
|
|
"pluginId": "prometheus"
|
|
},
|
|
{
|
|
"name": "DS_LOKI",
|
|
"type": "datasource",
|
|
"pluginId": "loki"
|
|
},
|
|
{
|
|
"name": "DS_TEMPO",
|
|
"type": "datasource",
|
|
"pluginId": "${DS_TEMPO}"
|
|
}
|
|
]
|
|
}
|