diff --git a/infra/k8s/monitoring/dashboards/foxhunt-ci-logs.json b/infra/k8s/monitoring/dashboards/foxhunt-ci-logs.json index 71c901efd..a5f24f455 100644 --- a/infra/k8s/monitoring/dashboards/foxhunt-ci-logs.json +++ b/infra/k8s/monitoring/dashboards/foxhunt-ci-logs.json @@ -412,7 +412,7 @@ "uid": "${DS_LOKI}" }, "editorMode": "code", - "expr": "{namespace=\"foxhunt\", pod=~\"$pipeline-.*\", level=~\"$level\"} ${search:pipe}", + "expr": "{namespace=\"foxhunt\", pod=~\"$pipeline-.*\", level=~\"$level\"} | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg ${search:pipe}", "queryType": "range", "refId": "A" } @@ -457,7 +457,7 @@ "uid": "${DS_LOKI}" }, "editorMode": "code", - "expr": "{namespace=\"foxhunt\", pod=~\"ci-pipeline-.*\", container=\"main\"} |~ \"(clippy|cargo test|warning\\\\[|error\\\\[|FAILED|test result|running \\\\d+ test|failures:)\"", + "expr": "{namespace=\"foxhunt\", pod=~\"ci-pipeline-.*\", container=\"main\"} |~ \"(clippy|cargo test|warning\\\\[|error\\\\[|FAILED|test result|running \\\\d+ test|failures:)\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", "queryType": "range", "refId": "A" } @@ -506,7 +506,7 @@ "uid": "${DS_LOKI}" }, "editorMode": "code", - "expr": "{namespace=\"foxhunt\", pod=~\"ci-pipeline-.*\", container=\"redis\"}", + "expr": "{namespace=\"foxhunt\", pod=~\"ci-pipeline-.*\", container=\"redis\"} | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", "queryType": "range", "refId": "A" } @@ -555,7 +555,7 @@ "uid": "${DS_LOKI}" }, "editorMode": "code", - "expr": "{namespace=\"foxhunt\", pod=~\"$pipeline-.*\"} |~ \"(?i)(error|failed|failure|panic|fatal|SIGABRT|SIGSEGV|thread .* panicked)\"", + "expr": "{namespace=\"foxhunt\", pod=~\"$pipeline-.*\"} |~ \"(?i)(error|failed|failure|panic|fatal|SIGABRT|SIGSEGV|thread .* panicked)\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", "queryType": "range", "refId": "A" } @@ -604,7 +604,7 @@ "uid": "${DS_LOKI}" }, "editorMode": "code", - "expr": "{namespace=\"foxhunt\", pod=~\"(ci-pipeline|compile-and-deploy|compile-and-train)-.*\", container=\"main\"} |~ \"(Compiling|Finished|warning|error|Downloading|Building|cargo build|rclone)\"", + "expr": "{namespace=\"foxhunt\", pod=~\"(ci-pipeline|compile-and-deploy|compile-and-train)-.*\", container=\"main\"} |~ \"(Compiling|Finished|warning|error|Downloading|Building|cargo build|rclone)\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", "queryType": "range", "refId": "A" } @@ -639,7 +639,7 @@ "uid": "${DS_LOKI}" }, "editorMode": "code", - "expr": "{namespace=\"foxhunt\", pod=~\"(ci-pipeline|compile-and-deploy)-.*\", container=\"main\"} |~ \"(kubectl|apply|rollout|deploy|configured|unchanged|created|image|kaniko)\"", + "expr": "{namespace=\"foxhunt\", pod=~\"(ci-pipeline|compile-and-deploy)-.*\", container=\"main\"} |~ \"(kubectl|apply|rollout|deploy|configured|unchanged|created|image|kaniko)\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", "queryType": "range", "refId": "A" } @@ -688,7 +688,7 @@ "uid": "${DS_LOKI}" }, "editorMode": "code", - "expr": "{namespace=\"foxhunt\", pod=~\"gpu-test-.*\"}", + "expr": "{namespace=\"foxhunt\", pod=~\"gpu-test-.*\"} | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", "queryType": "range", "refId": "A" } diff --git a/infra/k8s/monitoring/dashboards/foxhunt-observability.json b/infra/k8s/monitoring/dashboards/foxhunt-observability.json index c960124b6..df6a0a03b 100644 --- a/infra/k8s/monitoring/dashboards/foxhunt-observability.json +++ b/infra/k8s/monitoring/dashboards/foxhunt-observability.json @@ -1205,7 +1205,7 @@ }, "targets": [ { - "expr": "{namespace=\"foxhunt\"} |= \"error\"", + "expr": "{namespace=\"foxhunt\"} |= \"error\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", "refId": "A" } ], diff --git a/infra/k8s/monitoring/dashboards/foxhunt-training-deep-dive.json b/infra/k8s/monitoring/dashboards/foxhunt-training-deep-dive.json index 23425d785..df98ae908 100644 --- a/infra/k8s/monitoring/dashboards/foxhunt-training-deep-dive.json +++ b/infra/k8s/monitoring/dashboards/foxhunt-training-deep-dive.json @@ -977,7 +977,7 @@ }, "targets": [ { - "expr": "{namespace=\"foxhunt\", pod=~\"$pod\"} |~ \"$search\"", + "expr": "{namespace=\"foxhunt\", pod=~\"$pod\"} |~ \"$search\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", "refId": "A" } ], @@ -1009,7 +1009,7 @@ }, "targets": [ { - "expr": "{namespace=\"foxhunt\", pod=~\"$pod\"} |~ \"(?i)(cargo (build|test|check|clippy)|Compiling|Finished|running \\\\d+ test|test result|FAILED|passed|warning:)\"", + "expr": "{namespace=\"foxhunt\", pod=~\"$pod\"} |~ \"(?i)(cargo (build|test|check|clippy)|Compiling|Finished|running \\\\d+ test|test result|FAILED|passed|warning:)\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", "refId": "A" } ], @@ -1041,7 +1041,7 @@ }, "targets": [ { - "expr": "{namespace=\"foxhunt\", pod=~\"$pod\"} |~ \"(?i)(rclone|s3://|upload|download|kubectl|rollout|image|docker|kaniko|configmap)\"", + "expr": "{namespace=\"foxhunt\", pod=~\"$pod\"} |~ \"(?i)(rclone|s3://|upload|download|kubectl|rollout|image|docker|kaniko|configmap)\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", "refId": "A" } ], @@ -1073,7 +1073,7 @@ }, "targets": [ { - "expr": "{namespace=\"foxhunt\", pod=~\"$pod\"} |~ \"(?i)(redis|rclone|dbn|data|cache|download|decompress)\"", + "expr": "{namespace=\"foxhunt\", pod=~\"$pod\"} |~ \"(?i)(redis|rclone|dbn|data|cache|download|decompress)\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", "refId": "A" } ], @@ -3151,7 +3151,7 @@ }, "targets": [ { - "expr": "{namespace=\"foxhunt\", pod=~\"$pod\", level=~\"$level\"} |~ \"$search\"", + "expr": "{namespace=\"foxhunt\", pod=~\"$pod\", level=~\"$level\"} |~ \"$search\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", "refId": "A" } ], @@ -3183,7 +3183,7 @@ }, "targets": [ { - "expr": "{namespace=\"foxhunt\", pod=~\"$pod\", level=~\"ERROR|WARN\"}", + "expr": "{namespace=\"foxhunt\", pod=~\"$pod\", level=~\"ERROR|WARN\"} | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", "refId": "A" } ], @@ -3215,7 +3215,7 @@ }, "targets": [ { - "expr": "{namespace=\"foxhunt\", pod=~\"$pod\"} |~ \"(?i)(epoch|loss|sharpe|max_?dd|sortino|reward|return|accuracy|f1|precision|recall)\"", + "expr": "{namespace=\"foxhunt\", pod=~\"$pod\"} |~ \"(?i)(epoch|loss|sharpe|max_?dd|sortino|reward|return|accuracy|f1|precision|recall)\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", "refId": "A" } ], @@ -3247,7 +3247,7 @@ }, "targets": [ { - "expr": "{namespace=\"foxhunt\", pod=~\"$pod\"} |~ \"(?i)(cuda|gpu|vram|oom|out.of.memory|nccl|nvrtc|tensor.core|cublas)\"", + "expr": "{namespace=\"foxhunt\", pod=~\"$pod\"} |~ \"(?i)(cuda|gpu|vram|oom|out.of.memory|nccl|nvrtc|tensor.core|cublas)\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", "refId": "A" } ], diff --git a/infra/k8s/monitoring/dashboards/foxhunt-training-logs.json b/infra/k8s/monitoring/dashboards/foxhunt-training-logs.json index 43f5e3ca1..4e1832049 100644 --- a/infra/k8s/monitoring/dashboards/foxhunt-training-logs.json +++ b/infra/k8s/monitoring/dashboards/foxhunt-training-logs.json @@ -392,7 +392,7 @@ "uid": "${DS_LOKI}" }, "editorMode": "code", - "expr": "{namespace=\"foxhunt\", pod=~\"$job-.*\", level=~\"$level\"} ${search:pipe}", + "expr": "{namespace=\"foxhunt\", pod=~\"$job-.*\", level=~\"$level\"} | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg ${search:pipe}", "queryType": "range", "refId": "A" } @@ -437,7 +437,7 @@ "uid": "${DS_LOKI}" }, "editorMode": "code", - "expr": "{namespace=\"foxhunt\", pod=~\"$job-.*\"} |~ \"(?i)(epoch|loss|reward|sharpe|accuracy|val_loss|train_loss|best_trial|trial|fold|window|learning_rate|lr=|MaxDD|win.?rate|drawdown)\"", + "expr": "{namespace=\"foxhunt\", pod=~\"$job-.*\"} |~ \"(?i)(epoch|loss|reward|sharpe|accuracy|val_loss|train_loss|best_trial|trial|fold|window|learning_rate|lr=|MaxDD|win.?rate|drawdown)\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", "queryType": "range", "refId": "A" } @@ -486,7 +486,7 @@ "uid": "${DS_LOKI}" }, "editorMode": "code", - "expr": "{namespace=\"foxhunt\", pod=~\"(hyperopt|train)-.*\"} |~ \"(?i)(trial|best|cost|param|learning_rate|hidden_dim|batch_size|buffer_size|gamma|epsilon|clip|PSO|iteration|objective|optimal)\"", + "expr": "{namespace=\"foxhunt\", pod=~\"(hyperopt|train)-.*\"} |~ \"(?i)(trial|best|cost|param|learning_rate|hidden_dim|batch_size|buffer_size|gamma|epsilon|clip|PSO|iteration|objective|optimal)\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", "queryType": "range", "refId": "A" } @@ -535,7 +535,7 @@ "uid": "${DS_LOKI}" }, "editorMode": "code", - "expr": "{namespace=\"foxhunt\", pod=~\"$job-.*\"} |~ \"(?i)(GPU|CUDA|VRAM|cuda|gpu|vram|OOM|out.of.memory|nvrtc|tensor.core|BF16|bf16|SM|occupancy|kernel|device|nvidia)\"", + "expr": "{namespace=\"foxhunt\", pod=~\"$job-.*\"} |~ \"(?i)(GPU|CUDA|VRAM|cuda|gpu|vram|OOM|out.of.memory|nvrtc|tensor.core|BF16|bf16|SM|occupancy|kernel|device|nvidia)\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", "queryType": "range", "refId": "A" } @@ -584,7 +584,7 @@ "uid": "${DS_LOKI}" }, "editorMode": "code", - "expr": "{namespace=\"foxhunt\", pod=~\"$job-.*\"} |~ \"(?i)(walk.?forward|fold|backtest|window|train_start|train_end|val_start|val_end|test_start|test_end|embargo|purge|bars|features)\"", + "expr": "{namespace=\"foxhunt\", pod=~\"$job-.*\"} |~ \"(?i)(walk.?forward|fold|backtest|window|train_start|train_end|val_start|val_end|test_start|test_end|embargo|purge|bars|features)\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", "queryType": "range", "refId": "A" } @@ -633,7 +633,7 @@ "uid": "${DS_LOKI}" }, "editorMode": "code", - "expr": "{namespace=\"foxhunt\", pod=~\"$job-.*\"} |~ \"(?i)(error|failed|failure|panic|fatal|SIGABRT|SIGSEGV|thread .* panicked|OOM|NaN|inf|diverge)\"", + "expr": "{namespace=\"foxhunt\", pod=~\"$job-.*\"} |~ \"(?i)(error|failed|failure|panic|fatal|SIGABRT|SIGSEGV|thread .* panicked|OOM|NaN|inf|diverge)\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", "queryType": "range", "refId": "A" } @@ -682,7 +682,7 @@ "uid": "${DS_LOKI}" }, "editorMode": "code", - "expr": "{namespace=\"foxhunt\", pod=~\"$job-.*\"} |~ \"(?i)(loading|loaded|dbn|ofi|features|bars|MBP.?10|OHLCV|symbol|normalize|norm_stats|state_dim|rclone|S3|download)\"", + "expr": "{namespace=\"foxhunt\", pod=~\"$job-.*\"} |~ \"(?i)(loading|loaded|dbn|ofi|features|bars|MBP.?10|OHLCV|symbol|normalize|norm_stats|state_dim|rclone|S3|download)\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", "queryType": "range", "refId": "A" } diff --git a/infra/k8s/monitoring/dashboards/platform-argo.json b/infra/k8s/monitoring/dashboards/platform-argo.json index de3d7456a..4bf955d41 100644 --- a/infra/k8s/monitoring/dashboards/platform-argo.json +++ b/infra/k8s/monitoring/dashboards/platform-argo.json @@ -1531,7 +1531,7 @@ "type": "loki", "uid": "${DS_LOKI}" }, - "expr": "{namespace=\"foxhunt\", pod=~\"argo-workflows.*\"}", + "expr": "{namespace=\"foxhunt\", pod=~\"argo-workflows.*\"} | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", "refId": "A" } ], @@ -1566,7 +1566,7 @@ "type": "loki", "uid": "${DS_LOKI}" }, - "expr": "{namespace=\"foxhunt\", pod=~\"argo-workflows.*\"} |~ \"(?i)(error|warn|panic|fatal)\"", + "expr": "{namespace=\"foxhunt\", pod=~\"argo-workflows.*\"} | logfmt | level =~ \"error|warn|fatal\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", "refId": "A" } ], @@ -1629,4 +1629,4 @@ "timezone": "browser", "title": "Argo Workflows", "uid": "plat-argo" -} \ No newline at end of file +} diff --git a/infra/k8s/monitoring/dashboards/platform-grafana.json b/infra/k8s/monitoring/dashboards/platform-grafana.json index 6de38cfcb..d69f1c22f 100644 --- a/infra/k8s/monitoring/dashboards/platform-grafana.json +++ b/infra/k8s/monitoring/dashboards/platform-grafana.json @@ -702,7 +702,7 @@ "type": "loki", "uid": "${DS_LOKI}" }, - "expr": "{namespace=\"foxhunt\", pod=~\"grafana.*\"}", + "expr": "{namespace=\"foxhunt\", pod=~\"grafana.*\"} | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", "refId": "A" } ], @@ -737,7 +737,7 @@ "type": "loki", "uid": "${DS_LOKI}" }, - "expr": "{namespace=\"foxhunt\", pod=~\"grafana.*\"} |~ \"(?i)(error|warn|panic|fatal)\"", + "expr": "{namespace=\"foxhunt\", pod=~\"grafana.*\"} | logfmt | level =~ \"error|warn|fatal\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", "refId": "A" } ], @@ -799,4 +799,4 @@ "timezone": "browser", "title": "Grafana", "uid": "plat-grafana" -} \ No newline at end of file +} diff --git a/infra/k8s/monitoring/dashboards/platform-loki.json b/infra/k8s/monitoring/dashboards/platform-loki.json index 4a536d7bd..33f6dcc0b 100644 --- a/infra/k8s/monitoring/dashboards/platform-loki.json +++ b/infra/k8s/monitoring/dashboards/platform-loki.json @@ -1432,7 +1432,7 @@ "type": "loki", "uid": "${DS_LOKI}" }, - "expr": "{namespace=\"foxhunt\", pod=~\"loki.*\"}", + "expr": "{namespace=\"foxhunt\", pod=~\"loki.*\"} | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", "refId": "A" } ], @@ -1467,7 +1467,7 @@ "type": "loki", "uid": "${DS_LOKI}" }, - "expr": "{namespace=\"foxhunt\", pod=~\"loki.*\"} |~ \"(?i)(error|warn|panic|fatal)\"", + "expr": "{namespace=\"foxhunt\", pod=~\"loki.*\"} | logfmt | level =~ \"error|warn|fatal\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", "refId": "A" } ], @@ -1529,4 +1529,4 @@ "timezone": "browser", "title": "Loki & Promtail", "uid": "plat-loki" -} \ No newline at end of file +} diff --git a/infra/k8s/monitoring/dashboards/platform-postgres.json b/infra/k8s/monitoring/dashboards/platform-postgres.json index c1284e596..d8a639f7c 100644 --- a/infra/k8s/monitoring/dashboards/platform-postgres.json +++ b/infra/k8s/monitoring/dashboards/platform-postgres.json @@ -1046,7 +1046,7 @@ "type": "loki", "uid": "${DS_LOKI}" }, - "expr": "{namespace=\"foxhunt\", pod=~\"postgres.*\"}", + "expr": "{namespace=\"foxhunt\", pod=~\"postgres.*\"} | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", "refId": "A" } ], @@ -1081,7 +1081,7 @@ "type": "loki", "uid": "${DS_LOKI}" }, - "expr": "{namespace=\"foxhunt\", pod=~\"postgres.*\"} |~ \"(?i)(error|warn|panic|fatal)\"", + "expr": "{namespace=\"foxhunt\", pod=~\"postgres.*\"} |~ \"(ERROR|WARNING|FATAL|PANIC):\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", "refId": "A" } ], @@ -1143,4 +1143,4 @@ "timezone": "browser", "title": "PostgreSQL", "uid": "plat-postgres" -} \ No newline at end of file +} diff --git a/infra/k8s/monitoring/dashboards/platform-prometheus.json b/infra/k8s/monitoring/dashboards/platform-prometheus.json index 981977072..7a415ab1b 100644 --- a/infra/k8s/monitoring/dashboards/platform-prometheus.json +++ b/infra/k8s/monitoring/dashboards/platform-prometheus.json @@ -846,7 +846,7 @@ "type": "loki", "uid": "${DS_LOKI}" }, - "expr": "{namespace=\"foxhunt\", pod=~\"prometheus.*\"}", + "expr": "{namespace=\"foxhunt\", pod=~\"prometheus.*\"} | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", "refId": "A" } ], @@ -881,7 +881,7 @@ "type": "loki", "uid": "${DS_LOKI}" }, - "expr": "{namespace=\"foxhunt\", pod=~\"prometheus.*\"} |~ \"(?i)(error|warn|panic|fatal)\"", + "expr": "{namespace=\"foxhunt\", pod=~\"prometheus.*\"} | logfmt | level =~ \"error|warn|fatal\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", "refId": "A" } ], @@ -943,4 +943,4 @@ "timezone": "browser", "title": "Prometheus", "uid": "plat-prometheus" -} \ No newline at end of file +} diff --git a/infra/k8s/monitoring/dashboards/platform-questdb.json b/infra/k8s/monitoring/dashboards/platform-questdb.json index 9d9a0a617..87e299857 100644 --- a/infra/k8s/monitoring/dashboards/platform-questdb.json +++ b/infra/k8s/monitoring/dashboards/platform-questdb.json @@ -702,7 +702,7 @@ "type": "loki", "uid": "${DS_LOKI}" }, - "expr": "{namespace=\"foxhunt\", pod=~\"questdb.*\"}", + "expr": "{namespace=\"foxhunt\", pod=~\"questdb.*\"} | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", "refId": "A" } ], @@ -737,7 +737,7 @@ "type": "loki", "uid": "${DS_LOKI}" }, - "expr": "{namespace=\"foxhunt\", pod=~\"questdb.*\"} |~ \"(?i)(error|warn|panic|fatal)\"", + "expr": "{namespace=\"foxhunt\", pod=~\"questdb.*\"} | json | level =~ \"ERROR|WARN|FATAL\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", "refId": "A" } ], @@ -799,4 +799,4 @@ "timezone": "browser", "title": "QuestDB", "uid": "plat-questdb" -} \ No newline at end of file +} diff --git a/infra/k8s/monitoring/dashboards/platform-redis.json b/infra/k8s/monitoring/dashboards/platform-redis.json index a638e7c94..4eae4ed15 100644 --- a/infra/k8s/monitoring/dashboards/platform-redis.json +++ b/infra/k8s/monitoring/dashboards/platform-redis.json @@ -1215,7 +1215,7 @@ "type": "loki", "uid": "${DS_LOKI}" }, - "expr": "{namespace=\"foxhunt\", pod=~\"redis.*\"}", + "expr": "{namespace=\"foxhunt\", pod=~\"redis.*\"} | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", "refId": "A" } ], @@ -1250,7 +1250,7 @@ "type": "loki", "uid": "${DS_LOKI}" }, - "expr": "{namespace=\"foxhunt\", pod=~\"redis.*\"} |~ \"(?i)(error|warn|panic|fatal)\"", + "expr": "{namespace=\"foxhunt\", pod=~\"redis.*\"} |~ \"(# WARNING|# Error|ERR |FATAL)\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", "refId": "A" } ], @@ -1312,4 +1312,4 @@ "timezone": "browser", "title": "Redis", "uid": "plat-redis" -} \ No newline at end of file +} diff --git a/infra/k8s/monitoring/dashboards/platform-tempo.json b/infra/k8s/monitoring/dashboards/platform-tempo.json index aeaa0b2fb..c8d7c3db0 100644 --- a/infra/k8s/monitoring/dashboards/platform-tempo.json +++ b/infra/k8s/monitoring/dashboards/platform-tempo.json @@ -702,7 +702,7 @@ "type": "loki", "uid": "${DS_LOKI}" }, - "expr": "{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\"}", + "expr": "{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\"} | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", "refId": "A" } ], @@ -737,7 +737,7 @@ "type": "loki", "uid": "${DS_LOKI}" }, - "expr": "{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\"} |~ \"(?i)(error|warn|panic|fatal)\"", + "expr": "{namespace=\"foxhunt\", pod=~\"${DS_TEMPO}.*\"} | logfmt | level =~ \"error|warn|fatal\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", "refId": "A" } ], @@ -799,4 +799,4 @@ "timezone": "browser", "title": "Tempo", "uid": "plat-${DS_TEMPO}" -} \ No newline at end of file +} diff --git a/infra/k8s/monitoring/dashboards/svc-api-gateway.json b/infra/k8s/monitoring/dashboards/svc-api-gateway.json index b5fc824c5..27a4803b5 100644 --- a/infra/k8s/monitoring/dashboards/svc-api-gateway.json +++ b/infra/k8s/monitoring/dashboards/svc-api-gateway.json @@ -863,7 +863,7 @@ "type": "loki", "uid": "${DS_LOKI}" }, - "expr": "{namespace=\"foxhunt\", pod=~\"api.*\"}", + "expr": "{namespace=\"foxhunt\", pod=~\"api.*\"} | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", "refId": "A" } ], @@ -898,7 +898,7 @@ "type": "loki", "uid": "${DS_LOKI}" }, - "expr": "{namespace=\"foxhunt\", pod=~\"api.*\"} |~ \"(?i)(error|warn|panic|fatal)\"", + "expr": "{namespace=\"foxhunt\", pod=~\"api.*\"} | json | level =~ \"ERROR|WARN|FATAL\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", "refId": "A" } ], @@ -960,4 +960,4 @@ "timezone": "browser", "title": "API Gateway", "uid": "svc-api-gateway" -} \ No newline at end of file +} diff --git a/infra/k8s/monitoring/dashboards/svc-backtesting.json b/infra/k8s/monitoring/dashboards/svc-backtesting.json index 8f6060a2b..129bd2228 100644 --- a/infra/k8s/monitoring/dashboards/svc-backtesting.json +++ b/infra/k8s/monitoring/dashboards/svc-backtesting.json @@ -975,7 +975,7 @@ "type": "loki", "uid": "${DS_LOKI}" }, - "expr": "{namespace=\"foxhunt\", pod=~\"backtesting.*\"}", + "expr": "{namespace=\"foxhunt\", pod=~\"backtesting.*\"} | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", "refId": "A" } ], @@ -1010,7 +1010,7 @@ "type": "loki", "uid": "${DS_LOKI}" }, - "expr": "{namespace=\"foxhunt\", pod=~\"backtesting.*\"} |~ \"(?i)(error|warn|panic|fatal)\"", + "expr": "{namespace=\"foxhunt\", pod=~\"backtesting.*\"} | json | level =~ \"ERROR|WARN|FATAL\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", "refId": "A" } ], @@ -1072,4 +1072,4 @@ "timezone": "browser", "title": "Backtesting Service", "uid": "svc-backtesting" -} \ No newline at end of file +} diff --git a/infra/k8s/monitoring/dashboards/svc-data-acquisition.json b/infra/k8s/monitoring/dashboards/svc-data-acquisition.json index ae615e125..58b9a3d8c 100644 --- a/infra/k8s/monitoring/dashboards/svc-data-acquisition.json +++ b/infra/k8s/monitoring/dashboards/svc-data-acquisition.json @@ -931,7 +931,7 @@ "type": "loki", "uid": "${DS_LOKI}" }, - "expr": "{namespace=\"foxhunt\", pod=~\"data-acquisition.*\"}", + "expr": "{namespace=\"foxhunt\", pod=~\"data-acquisition.*\"} | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", "refId": "A" } ], @@ -966,7 +966,7 @@ "type": "loki", "uid": "${DS_LOKI}" }, - "expr": "{namespace=\"foxhunt\", pod=~\"data-acquisition.*\"} |~ \"(?i)(error|warn|panic|fatal)\"", + "expr": "{namespace=\"foxhunt\", pod=~\"data-acquisition.*\"} | json | level =~ \"ERROR|WARN|FATAL\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", "refId": "A" } ], @@ -1028,4 +1028,4 @@ "timezone": "browser", "title": "Data Acquisition", "uid": "svc-data-acq" -} \ No newline at end of file +} diff --git a/infra/k8s/monitoring/dashboards/svc-ml-training.json b/infra/k8s/monitoring/dashboards/svc-ml-training.json index 41a8d89ba..1d792ec13 100644 --- a/infra/k8s/monitoring/dashboards/svc-ml-training.json +++ b/infra/k8s/monitoring/dashboards/svc-ml-training.json @@ -925,7 +925,7 @@ "type": "loki", "uid": "${DS_LOKI}" }, - "expr": "{namespace=\"foxhunt\", pod=~\"ml-training.*\"}", + "expr": "{namespace=\"foxhunt\", pod=~\"ml-training.*\"} | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", "refId": "A" } ], @@ -960,7 +960,7 @@ "type": "loki", "uid": "${DS_LOKI}" }, - "expr": "{namespace=\"foxhunt\", pod=~\"ml-training.*\"} |~ \"(?i)(error|warn|panic|fatal)\"", + "expr": "{namespace=\"foxhunt\", pod=~\"ml-training.*\"} | json | level =~ \"ERROR|WARN|FATAL\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", "refId": "A" } ], @@ -1023,4 +1023,4 @@ "timezone": "browser", "title": "ML Training Service", "uid": "svc-ml-training" -} \ No newline at end of file +} diff --git a/infra/k8s/monitoring/dashboards/svc-trading-agent.json b/infra/k8s/monitoring/dashboards/svc-trading-agent.json index 30b3a387f..7e97104b7 100644 --- a/infra/k8s/monitoring/dashboards/svc-trading-agent.json +++ b/infra/k8s/monitoring/dashboards/svc-trading-agent.json @@ -975,7 +975,7 @@ "type": "loki", "uid": "${DS_LOKI}" }, - "expr": "{namespace=\"foxhunt\", pod=~\"trading-agent.*\"}", + "expr": "{namespace=\"foxhunt\", pod=~\"trading-agent.*\"} | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", "refId": "A" } ], @@ -1010,7 +1010,7 @@ "type": "loki", "uid": "${DS_LOKI}" }, - "expr": "{namespace=\"foxhunt\", pod=~\"trading-agent.*\"} |~ \"(?i)(error|warn|panic|fatal)\"", + "expr": "{namespace=\"foxhunt\", pod=~\"trading-agent.*\"} | json | level =~ \"ERROR|WARN|FATAL\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", "refId": "A" } ], @@ -1072,4 +1072,4 @@ "timezone": "browser", "title": "Trading Agent", "uid": "svc-trading-agent" -} \ No newline at end of file +} diff --git a/infra/k8s/monitoring/dashboards/svc-trading-service.json b/infra/k8s/monitoring/dashboards/svc-trading-service.json index 5fcfa3c3f..768f1c409 100644 --- a/infra/k8s/monitoring/dashboards/svc-trading-service.json +++ b/infra/k8s/monitoring/dashboards/svc-trading-service.json @@ -974,7 +974,7 @@ "type": "loki", "uid": "${DS_LOKI}" }, - "expr": "{namespace=\"foxhunt\", pod=~\"trading-service.*\"}", + "expr": "{namespace=\"foxhunt\", pod=~\"trading-service.*\"} | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", "refId": "A" } ], @@ -1009,7 +1009,7 @@ "type": "loki", "uid": "${DS_LOKI}" }, - "expr": "{namespace=\"foxhunt\", pod=~\"trading-service.*\"} |~ \"(?i)(error|warn|panic|fatal)\"", + "expr": "{namespace=\"foxhunt\", pod=~\"trading-service.*\"} | json | level =~ \"ERROR|WARN|FATAL\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", "refId": "A" } ], @@ -1071,4 +1071,4 @@ "timezone": "browser", "title": "Trading Service", "uid": "svc-trading" -} \ No newline at end of file +} diff --git a/infra/k8s/monitoring/dashboards/svc-web-gateway.json b/infra/k8s/monitoring/dashboards/svc-web-gateway.json index e0279af01..0d4a926d0 100644 --- a/infra/k8s/monitoring/dashboards/svc-web-gateway.json +++ b/infra/k8s/monitoring/dashboards/svc-web-gateway.json @@ -922,7 +922,7 @@ "type": "loki", "uid": "${DS_LOKI}" }, - "expr": "{namespace=\"foxhunt\", pod=~\"web-gateway.*\"}", + "expr": "{namespace=\"foxhunt\", pod=~\"web-gateway.*\"} | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", "refId": "A" } ], @@ -957,7 +957,7 @@ "type": "loki", "uid": "${DS_LOKI}" }, - "expr": "{namespace=\"foxhunt\", pod=~\"web-gateway.*\"} |~ \"(?i)(error|warn|panic|fatal)\"", + "expr": "{namespace=\"foxhunt\", pod=~\"web-gateway.*\"} | json | level =~ \"ERROR|WARN|FATAL\" | pattern `<_> <_> <_> ` | regexp `msg=\"(?P[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg", "refId": "A" } ], @@ -1019,4 +1019,4 @@ "timezone": "browser", "title": "Web Gateway", "uid": "svc-web-gateway" -} \ No newline at end of file +}