Files
foxhunt/infra/k8s/monitoring/dashboards/foxhunt-ci-logs.json
jgrusewski dba50fa3be fix(dashboards): update datasource UIDs and folder annotations for Helm Prometheus
Update all dashboard JSON files to use Helm chart's Prometheus datasource
UID and add Grafana sidecar folder annotations for auto-provisioning.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-14 11:34:14 +01:00

810 lines
20 KiB
JSON

{
"description": "Loki-powered CI pipeline log viewer \u2014 test gate, clippy, builds, errors",
"editable": true,
"graphTooltip": 2,
"panels": [
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 0
},
"id": 1,
"title": "Pipeline Overview",
"type": "row"
},
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"fieldConfig": {
"defaults": {
"color": {
"fixedColor": "red",
"mode": "fixed"
},
"mappings": [],
"noValue": "0",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 1
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 4,
"x": 0,
"y": 1
},
"id": 2,
"options": {
"colorMode": "background",
"graphMode": "none",
"justifyMode": "center",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"sum"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"editorMode": "code",
"expr": "sum(count_over_time({namespace=\"foxhunt\", pod=~\"$pipeline-.*\", level=\"ERROR\"} [$__range]))",
"queryType": "instant",
"refId": "A"
}
],
"title": "Errors",
"type": "stat"
},
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"fieldConfig": {
"defaults": {
"color": {
"fixedColor": "yellow",
"mode": "fixed"
},
"mappings": [],
"noValue": "0",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 1
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 4,
"x": 4,
"y": 1
},
"id": 3,
"options": {
"colorMode": "background",
"graphMode": "none",
"justifyMode": "center",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"sum"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"editorMode": "code",
"expr": "sum(count_over_time({namespace=\"foxhunt\", pod=~\"$pipeline-.*\", level=\"WARN\"} [$__range]))",
"queryType": "instant",
"refId": "A"
}
],
"title": "Warnings",
"type": "stat"
},
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"fieldConfig": {
"defaults": {
"color": {
"fixedColor": "blue",
"mode": "fixed"
},
"mappings": [],
"noValue": "0"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 4,
"x": 8,
"y": 1
},
"id": 4,
"options": {
"colorMode": "background",
"graphMode": "none",
"justifyMode": "center",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"sum"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"editorMode": "code",
"expr": "sum(count_over_time({namespace=\"foxhunt\", pod=~\"$pipeline-.*\"} [$__range]))",
"queryType": "instant",
"refId": "A"
}
],
"title": "Total Log Lines",
"type": "stat"
},
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisLabel": "lines/interval",
"drawStyle": "bars",
"fillOpacity": 80,
"lineWidth": 0,
"pointSize": 5,
"showPoints": "never",
"stacking": {
"mode": "normal"
}
}
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "ERROR"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "red",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "WARN"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "yellow",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "INFO"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "green",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byName",
"options": "DEBUG"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "blue",
"mode": "fixed"
}
}
]
}
]
},
"gridPos": {
"h": 6,
"w": 12,
"x": 12,
"y": 1
},
"id": 5,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"editorMode": "code",
"expr": "sum by (level) (count_over_time({namespace=\"foxhunt\", pod=~\"$pipeline-.*\"} [$__interval]))",
"legendFormat": "{{level}}",
"queryType": "range",
"refId": "A"
}
],
"title": "Log Volume by Level",
"type": "timeseries"
},
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"drawStyle": "bars",
"fillOpacity": 60,
"lineWidth": 0,
"showPoints": "never",
"stacking": {
"mode": "normal"
}
}
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 12,
"x": 0,
"y": 5
},
"id": 6,
"options": {
"legend": {
"displayMode": "list",
"placement": "right"
},
"tooltip": {
"mode": "multi"
}
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"editorMode": "code",
"expr": "sum by (container) (count_over_time({namespace=\"foxhunt\", pod=~\"$pipeline-.*\"} [$__interval]))",
"legendFormat": "{{container}}",
"queryType": "range",
"refId": "A"
}
],
"title": "Log Volume by Container",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 9
},
"id": 10,
"title": "Full Pipeline Log Stream",
"type": "row"
},
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"gridPos": {
"h": 14,
"w": 24,
"x": 0,
"y": 10
},
"id": 11,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": true,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": true
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"editorMode": "code",
"expr": "{namespace=\"foxhunt\", pod=~\"$pipeline-.*\", level=~\"$level\"} | pattern `<_> <_> <_> <content>` | regexp `msg=\"(?P<clean_msg>[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg ${search:pipe}",
"queryType": "range",
"refId": "A"
}
],
"title": "Pipeline Logs \u2014 $pipeline",
"type": "logs"
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 24
},
"id": 20,
"panels": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"gridPos": {
"h": 12,
"w": 24,
"x": 0,
"y": 25
},
"id": 21,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"showLabels": false,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": true
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"editorMode": "code",
"expr": "{namespace=\"foxhunt\", pod=~\"ci-pipeline-.*\", container=\"main\"} |~ \"(clippy|cargo test|warning\\\\[|error\\\\[|FAILED|test result|running \\\\d+ test|failures:)\" | pattern `<_> <_> <_> <content>` | regexp `msg=\"(?P<clean_msg>[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg",
"queryType": "range",
"refId": "A"
}
],
"title": "Clippy + Test Output",
"type": "logs"
}
],
"title": "Test Gate (clippy + cargo test)",
"type": "row"
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 25
},
"id": 30,
"panels": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 26
},
"id": 31,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"showLabels": false,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": true
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"editorMode": "code",
"expr": "{namespace=\"foxhunt\", pod=~\"ci-pipeline-.*\", container=\"redis\"} | pattern `<_> <_> <_> <content>` | regexp `msg=\"(?P<clean_msg>[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg",
"queryType": "range",
"refId": "A"
}
],
"title": "Redis Sidecar Logs",
"type": "logs"
}
],
"title": "Redis Sidecar",
"type": "row"
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 26
},
"id": 40,
"panels": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"gridPos": {
"h": 12,
"w": 24,
"x": 0,
"y": 27
},
"id": 41,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"showLabels": true,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": true
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"editorMode": "code",
"expr": "{namespace=\"foxhunt\", pod=~\"$pipeline-.*\"} |~ \"(?i)(error|failed|failure|panic|fatal|SIGABRT|SIGSEGV|thread .* panicked)\" | pattern `<_> <_> <_> <content>` | regexp `msg=\"(?P<clean_msg>[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg",
"queryType": "range",
"refId": "A"
}
],
"title": "All Errors & Failures",
"type": "logs"
}
],
"title": "Errors & Failures",
"type": "row"
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 27
},
"id": 50,
"panels": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"gridPos": {
"h": 10,
"w": 12,
"x": 0,
"y": 28
},
"id": 51,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"showLabels": false,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": true
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"editorMode": "code",
"expr": "{namespace=\"foxhunt\", pod=~\"(ci-pipeline|compile-and-deploy|compile-and-train)-.*\", container=\"main\"} |~ \"(Compiling|Finished|warning|error|Downloading|Building|cargo build|rclone)\" | pattern `<_> <_> <_> <content>` | regexp `msg=\"(?P<clean_msg>[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg",
"queryType": "range",
"refId": "A"
}
],
"title": "Compile Logs",
"type": "logs"
},
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"gridPos": {
"h": 10,
"w": 12,
"x": 12,
"y": 28
},
"id": 52,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"showLabels": false,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": true
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"editorMode": "code",
"expr": "{namespace=\"foxhunt\", pod=~\"(ci-pipeline|compile-and-deploy)-.*\", container=\"main\"} |~ \"(kubectl|apply|rollout|deploy|configured|unchanged|created|image|kaniko)\" | pattern `<_> <_> <_> <content>` | regexp `msg=\"(?P<clean_msg>[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg",
"queryType": "range",
"refId": "A"
}
],
"title": "Deploy & Argo Template Logs",
"type": "logs"
}
],
"title": "Build & Deploy",
"type": "row"
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 28
},
"id": 60,
"panels": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"gridPos": {
"h": 12,
"w": 24,
"x": 0,
"y": 29
},
"id": 61,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"showLabels": true,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": true
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "${DS_LOKI}"
},
"editorMode": "code",
"expr": "{namespace=\"foxhunt\", pod=~\"gpu-test-.*\"} | pattern `<_> <_> <_> <content>` | regexp `msg=\"(?P<clean_msg>[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg",
"queryType": "range",
"refId": "A"
}
],
"title": "GPU Test Logs (DQN/PPO/TFT + Supervised)",
"type": "logs"
}
],
"title": "GPU Test Pipeline",
"type": "row"
}
],
"refresh": "30s",
"schemaVersion": 39,
"tags": [
"foxhunt",
"ci",
"loki",
"logs"
],
"templating": {
"list": [
{
"current": {
"selected": true,
"text": "ci-pipeline",
"value": "ci-pipeline"
},
"label": "Pipeline",
"name": "pipeline",
"options": [],
"query": "ci-pipeline,compile-and-deploy,compile-and-train,gpu-test,gpu-diag,gpu-walltime,build-ci-image,build-training-runtime,evaluate",
"type": "custom"
},
{
"allValue": ".*",
"current": {
"selected": true,
"text": "All",
"value": "$__all"
},
"includeAll": true,
"label": "Level",
"multi": true,
"name": "level",
"options": [
{
"selected": true,
"text": "All",
"value": "$__all"
},
{
"selected": false,
"text": "ERROR",
"value": "ERROR"
},
{
"selected": false,
"text": "WARN",
"value": "WARN"
},
{
"selected": false,
"text": "INFO",
"value": "INFO"
},
{
"selected": false,
"text": "DEBUG",
"value": "DEBUG"
},
{
"selected": false,
"text": "TRACE",
"value": "TRACE"
}
],
"query": "ERROR,WARN,INFO,DEBUG,TRACE",
"type": "custom"
},
{
"current": {
"selected": false,
"text": "",
"value": ""
},
"label": "Search",
"name": "search",
"type": "textbox"
}
]
},
"time": {
"from": "now-6h",
"to": "now"
},
"timepicker": {},
"timezone": "browser",
"title": "CI Pipeline Logs",
"uid": "foxhunt-ci-logs",
"__inputs": [
{
"name": "DS_PROMETHEUS",
"type": "datasource",
"pluginId": "prometheus"
},
{
"name": "DS_LOKI",
"type": "datasource",
"pluginId": "loki"
},
{
"name": "DS_TEMPO",
"type": "datasource",
"pluginId": "${DS_TEMPO}"
}
]
}