Files
foxhunt/infra/k8s/monitoring/dashboards/foxhunt-training-logs.json
jgrusewski dba50fa3be fix(dashboards): update datasource UIDs and folder annotations for Helm Prometheus
Update all dashboard JSON files to use Helm chart's Prometheus datasource
UID and add Grafana sidecar folder annotations for auto-provisioning.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-14 11:34:14 +01:00

836 lines
21 KiB
JSON

{
"description": "Loki-powered training log viewer \u2014 hyperopt trials, epoch progress, GPU status, errors",
"editable": true,
"graphTooltip": 2,
"panels": [
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 0
},
"id": 1,
"title": "Training Overview",
"type": "row"
},
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"fieldConfig": {
"defaults": {
"color": {
"fixedColor": "red",
"mode": "fixed"
},
"mappings": [],
"noValue": "0",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 1
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 4,
"x": 0,
"y": 1
},
"id": 2,
"options": {
"colorMode": "background",
"graphMode": "none",
"justifyMode": "center",
"reduceOptions": {
"calcs": [
"sum"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"editorMode": "code",
"expr": "sum(count_over_time({namespace=\"foxhunt\", pod=~\"$job-.*\", level=\"ERROR\"} [$__range]))",
"queryType": "instant",
"refId": "A"
}
],
"title": "Errors",
"type": "stat"
},
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"fieldConfig": {
"defaults": {
"color": {
"fixedColor": "yellow",
"mode": "fixed"
},
"mappings": [],
"noValue": "0",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 1
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 4,
"x": 4,
"y": 1
},
"id": 3,
"options": {
"colorMode": "background",
"graphMode": "none",
"justifyMode": "center",
"reduceOptions": {
"calcs": [
"sum"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"editorMode": "code",
"expr": "sum(count_over_time({namespace=\"foxhunt\", pod=~\"$job-.*\", level=\"WARN\"} [$__range]))",
"queryType": "instant",
"refId": "A"
}
],
"title": "Warnings",
"type": "stat"
},
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"fieldConfig": {
"defaults": {
"color": {
"fixedColor": "blue",
"mode": "fixed"
},
"mappings": [],
"noValue": "0"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 4,
"x": 8,
"y": 1
},
"id": 4,
"options": {
"colorMode": "background",
"graphMode": "none",
"justifyMode": "center",
"reduceOptions": {
"calcs": [
"sum"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"editorMode": "code",
"expr": "sum(count_over_time({namespace=\"foxhunt\", pod=~\"$job-.*\"} [$__range]))",
"queryType": "instant",
"refId": "A"
}
],
"title": "Total Log Lines",
"type": "stat"
},
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisLabel": "lines/interval",
"drawStyle": "bars",
"fillOpacity": 80,
"lineWidth": 0,
"showPoints": "never",
"stacking": {
"mode": "normal"
}
}
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "ERROR"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "red",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "WARN"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "yellow",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "INFO"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "green",
"mode": "fixed"
}
}
]
}
]
},
"gridPos": {
"h": 6,
"w": 12,
"x": 12,
"y": 1
},
"id": 5,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"editorMode": "code",
"expr": "sum by (level) (count_over_time({namespace=\"foxhunt\", pod=~\"$job-.*\"} [$__interval]))",
"legendFormat": "{{level}}",
"queryType": "range",
"refId": "A"
}
],
"title": "Log Volume by Level",
"type": "timeseries"
},
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"drawStyle": "bars",
"fillOpacity": 60,
"lineWidth": 0,
"showPoints": "never",
"stacking": {
"mode": "normal"
}
}
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 12,
"x": 0,
"y": 5
},
"id": 6,
"options": {
"legend": {
"displayMode": "list",
"placement": "right"
},
"tooltip": {
"mode": "multi"
}
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"editorMode": "code",
"expr": "sum by (pod) (count_over_time({namespace=\"foxhunt\", pod=~\"$job-.*\"} [$__interval]))",
"legendFormat": "{{pod}}",
"queryType": "range",
"refId": "A"
}
],
"title": "Log Volume by Pod",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 9
},
"id": 10,
"title": "Full Training Log Stream",
"type": "row"
},
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"gridPos": {
"h": 14,
"w": 24,
"x": 0,
"y": 10
},
"id": 11,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": true,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": true
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"editorMode": "code",
"expr": "{namespace=\"foxhunt\", pod=~\"$job-.*\", level=~\"$level\"} | pattern `<_> <_> <_> <content>` | regexp `msg=\"(?P<clean_msg>[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg ${search:pipe}",
"queryType": "range",
"refId": "A"
}
],
"title": "Training Logs \u2014 $job",
"type": "logs"
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 24
},
"id": 20,
"panels": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"gridPos": {
"h": 12,
"w": 24,
"x": 0,
"y": 25
},
"id": 21,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"showLabels": true,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": true
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"editorMode": "code",
"expr": "{namespace=\"foxhunt\", pod=~\"$job-.*\"} |~ \"(?i)(epoch|loss|reward|sharpe|accuracy|val_loss|train_loss|best_trial|trial|fold|window|learning_rate|lr=|MaxDD|win.?rate|drawdown)\" | pattern `<_> <_> <_> <content>` | regexp `msg=\"(?P<clean_msg>[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg",
"queryType": "range",
"refId": "A"
}
],
"title": "Epoch / Loss / Reward / Sharpe",
"type": "logs"
}
],
"title": "Epoch Progress & Loss",
"type": "row"
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 25
},
"id": 30,
"panels": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"gridPos": {
"h": 12,
"w": 24,
"x": 0,
"y": 26
},
"id": 31,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"showLabels": true,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": true
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"editorMode": "code",
"expr": "{namespace=\"foxhunt\", pod=~\"(hyperopt|train)-.*\"} |~ \"(?i)(trial|best|cost|param|learning_rate|hidden_dim|batch_size|buffer_size|gamma|epsilon|clip|PSO|iteration|objective|optimal)\" | pattern `<_> <_> <_> <content>` | regexp `msg=\"(?P<clean_msg>[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg",
"queryType": "range",
"refId": "A"
}
],
"title": "Hyperopt Trial Results",
"type": "logs"
}
],
"title": "Hyperopt Trials",
"type": "row"
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 26
},
"id": 40,
"panels": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"gridPos": {
"h": 12,
"w": 24,
"x": 0,
"y": 27
},
"id": 41,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"showLabels": true,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": true
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"editorMode": "code",
"expr": "{namespace=\"foxhunt\", pod=~\"$job-.*\"} |~ \"(?i)(GPU|CUDA|VRAM|cuda|gpu|vram|OOM|out.of.memory|nvrtc|tensor.core|BF16|bf16|SM|occupancy|kernel|device|nvidia)\" | pattern `<_> <_> <_> <content>` | regexp `msg=\"(?P<clean_msg>[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg",
"queryType": "range",
"refId": "A"
}
],
"title": "GPU / CUDA / VRAM Logs",
"type": "logs"
}
],
"title": "GPU / CUDA / VRAM",
"type": "row"
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 27
},
"id": 50,
"panels": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"gridPos": {
"h": 10,
"w": 24,
"x": 0,
"y": 28
},
"id": 51,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"showLabels": true,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": true
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"editorMode": "code",
"expr": "{namespace=\"foxhunt\", pod=~\"$job-.*\"} |~ \"(?i)(walk.?forward|fold|backtest|window|train_start|train_end|val_start|val_end|test_start|test_end|embargo|purge|bars|features)\" | pattern `<_> <_> <_> <content>` | regexp `msg=\"(?P<clean_msg>[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg",
"queryType": "range",
"refId": "A"
}
],
"title": "Walk-Forward / Fold / Backtest",
"type": "logs"
}
],
"title": "Walk-Forward Windows",
"type": "row"
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 28
},
"id": 60,
"panels": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"gridPos": {
"h": 12,
"w": 24,
"x": 0,
"y": 29
},
"id": 61,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"showLabels": true,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": true
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"editorMode": "code",
"expr": "{namespace=\"foxhunt\", pod=~\"$job-.*\"} |~ \"(?i)(error|failed|failure|panic|fatal|SIGABRT|SIGSEGV|thread .* panicked|OOM|NaN|inf|diverge)\" | pattern `<_> <_> <_> <content>` | regexp `msg=\"(?P<clean_msg>[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg",
"queryType": "range",
"refId": "A"
}
],
"title": "All Training Errors & Failures",
"type": "logs"
}
],
"title": "Errors & Failures",
"type": "row"
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 29
},
"id": 70,
"panels": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"gridPos": {
"h": 10,
"w": 24,
"x": 0,
"y": 30
},
"id": 71,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"showLabels": true,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": true
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"editorMode": "code",
"expr": "{namespace=\"foxhunt\", pod=~\"$job-.*\"} |~ \"(?i)(loading|loaded|dbn|ofi|features|bars|MBP.?10|OHLCV|symbol|normalize|norm_stats|state_dim|rclone|S3|download)\" | pattern `<_> <_> <_> <content>` | regexp `msg=\"(?P<clean_msg>[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg",
"queryType": "range",
"refId": "A"
}
],
"title": "Data Loading / OFI / DBN / Features",
"type": "logs"
}
],
"title": "Data Loading & OFI",
"type": "row"
}
],
"refresh": "30s",
"schemaVersion": 39,
"tags": [
"foxhunt",
"training",
"loki",
"logs",
"gpu"
],
"templating": {
"list": [
{
"current": {
"selected": true,
"text": "All Training",
"value": "(train|hyperopt|compile-and-train)"
},
"label": "Job Type",
"name": "job",
"options": [
{
"selected": true,
"text": "All Training",
"value": "(train|hyperopt|compile-and-train)"
},
{
"selected": false,
"text": "DQN",
"value": "train-dqn"
},
{
"selected": false,
"text": "PPO",
"value": "train-ppo"
},
{
"selected": false,
"text": "Supervised",
"value": "train-supervised"
},
{
"selected": false,
"text": "Hyperopt",
"value": "hyperopt"
},
{
"selected": false,
"text": "Compile+Train",
"value": "compile-and-train"
}
],
"query": "All Training : (train|hyperopt|compile-and-train), DQN : train-dqn, PPO : train-ppo, Supervised : train-supervised, Hyperopt : hyperopt, Compile+Train : compile-and-train",
"type": "custom"
},
{
"allValue": ".*",
"current": {
"selected": true,
"text": "All",
"value": "$__all"
},
"includeAll": true,
"label": "Level",
"multi": true,
"name": "level",
"options": [
{
"selected": true,
"text": "All",
"value": "$__all"
},
{
"selected": false,
"text": "ERROR",
"value": "ERROR"
},
{
"selected": false,
"text": "WARN",
"value": "WARN"
},
{
"selected": false,
"text": "INFO",
"value": "INFO"
},
{
"selected": false,
"text": "DEBUG",
"value": "DEBUG"
},
{
"selected": false,
"text": "TRACE",
"value": "TRACE"
}
],
"query": "ERROR,WARN,INFO,DEBUG,TRACE",
"type": "custom"
},
{
"current": {
"selected": false,
"text": "",
"value": ""
},
"label": "Search",
"name": "search",
"type": "textbox"
}
]
},
"time": {
"from": "now-12h",
"to": "now"
},
"timepicker": {},
"timezone": "browser",
"title": "Training Pipeline Logs",
"uid": "foxhunt-training-logs",
"__inputs": [
{
"name": "DS_PROMETHEUS",
"type": "datasource",
"pluginId": "prometheus"
},
{
"name": "DS_LOKI",
"type": "datasource",
"pluginId": "loki"
},
{
"name": "DS_TEMPO",
"type": "datasource",
"pluginId": "${DS_TEMPO}"
}
]
}