From 58c0f0ebc995f326efe57f53d854304ef7d21030 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sun, 1 Mar 2026 02:08:41 +0100 Subject: [PATCH] fix(dashboard): use real Tempo metric names in traces dashboard Replace placeholder metric names (traces_spanmetrics_calls_total, traces_spanmetrics_latency_*) with real Tempo metrics (tempo_distributor_spans_received_total, tempo_spanmetrics_latency_*). Switch Recent Traces panel from nativeSearch to TraceQL query. Co-Authored-By: Claude Opus 4.6 --- .../monitoring/dashboards/foxhunt-traces.json | 345 ++++++++++++++---- 1 file changed, 280 insertions(+), 65 deletions(-) diff --git a/infra/k8s/monitoring/dashboards/foxhunt-traces.json b/infra/k8s/monitoring/dashboards/foxhunt-traces.json index ad6373bf5..f69c52ee9 100644 --- a/infra/k8s/monitoring/dashboards/foxhunt-traces.json +++ b/infra/k8s/monitoring/dashboards/foxhunt-traces.json @@ -1,13 +1,29 @@ { "__inputs": [ - { "name": "DS_PROMETHEUS", "label": "Prometheus", "description": "", "type": "datasource", "pluginId": "prometheus" }, - { "name": "DS_TEMPO", "label": "Tempo", "description": "", "type": "datasource", "pluginId": "tempo" } + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus" + }, + { + "name": "DS_TEMPO", + "label": "Tempo", + "description": "", + "type": "datasource", + "pluginId": "tempo" + } ], "id": null, "uid": "foxhunt-traces", "title": "Foxhunt - Traces", "description": "Distributed tracing dashboard: span rates, error rates, latency percentiles, service map, trace search", - "tags": ["foxhunt", "traces", "tempo"], + "tags": [ + "foxhunt", + "traces", + "tempo" + ], "timezone": "browser", "schemaVersion": 39, "version": 1, @@ -25,12 +41,19 @@ { "name": "service", "type": "query", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "query": "label_values(traces_service_graph_request_total, client)", "refresh": 2, "multi": true, "includeAll": true, - "current": { "selected": true, "text": "All", "value": "$__all" } + "current": { + "selected": true, + "text": "All", + "value": "$__all" + } } ] }, @@ -39,7 +62,12 @@ "id": 1, "type": "row", "title": "Overview Stats", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, "collapsed": false, "panels": [] }, @@ -47,23 +75,42 @@ "id": 2, "type": "stat", "title": "Spans/sec", - "gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 1 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null } + { + "color": "green", + "value": null + } ] }, "mappings": [], - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -73,8 +120,11 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "sum(rate(traces_spanmetrics_calls_total[5m]))", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(tempo_distributor_spans_received_total[5m]))", "instant": true } ] @@ -83,26 +133,51 @@ "id": 3, "type": "stat", "title": "Error Rate", - "gridPos": { "h": 4, "w": 6, "x": 6, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 1 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "unit": "percent", "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 1 }, - { "color": "red", "value": 5 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 5 + } ] }, "mappings": [], - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -112,8 +187,11 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "sum(rate(traces_spanmetrics_calls_total{status_code=\"STATUS_CODE_ERROR\"}[5m])) / sum(rate(traces_spanmetrics_calls_total[5m])) * 100", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(tempo_distributor_spans_received_total{status=\"error\"}[5m])) / sum(rate(tempo_distributor_spans_received_total[5m])) * 100", "instant": true } ] @@ -122,24 +200,43 @@ "id": 4, "type": "stat", "title": "Avg Duration", - "gridPos": { "h": 4, "w": 6, "x": 12, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 1 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "unit": "s", "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null } + { + "color": "green", + "value": null + } ] }, "mappings": [], - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -149,8 +246,11 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "sum(rate(traces_spanmetrics_latency_sum[5m])) / sum(rate(traces_spanmetrics_calls_total[5m]))", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(tempo_spanmetrics_latency_sum[5m])) / sum(rate(tempo_spanmetrics_latency_count[5m]))", "instant": true } ] @@ -159,23 +259,42 @@ "id": 5, "type": "stat", "title": "Active Services", - "gridPos": { "h": 4, "w": 6, "x": 18, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 1 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null } + { + "color": "green", + "value": null + } ] }, "mappings": [], - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -185,8 +304,11 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "count(count by (service_name)(traces_spanmetrics_calls_total))", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "count(count by (service_name)(tempo_distributor_spans_received_total))", "instant": true } ] @@ -195,7 +317,12 @@ "id": 20, "type": "row", "title": "Service Map + Trace Search", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, "collapsed": false, "panels": [] }, @@ -203,12 +330,23 @@ "id": 6, "type": "nodeGraph", "title": "Service Map", - "gridPos": { "h": 10, "w": 24, "x": 0, "y": 6 }, - "datasource": { "type": "tempo", "uid": "${DS_TEMPO}" }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 6 + }, + "datasource": { + "type": "tempo", + "uid": "${DS_TEMPO}" + }, "targets": [ { "refId": "A", - "datasource": { "type": "tempo", "uid": "${DS_TEMPO}" }, + "datasource": { + "type": "tempo", + "uid": "${DS_TEMPO}" + }, "queryType": "serviceMap" } ] @@ -217,7 +355,12 @@ "id": 21, "type": "row", "title": "Latency and Errors by Service", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 16 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, "collapsed": false, "panels": [] }, @@ -225,15 +368,26 @@ "id": 7, "type": "timeseries", "title": "Latency by Service p95", - "gridPos": { "h": 7, "w": 12, "x": 0, "y": 17 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 17 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "unit": "s", "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null } + { + "color": "green", + "value": null + } ] }, "custom": { @@ -248,22 +402,39 @@ "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "scaleDistribution": { "type": "linear" } + "scaleDistribution": { + "type": "linear" + } + }, + "color": { + "mode": "palette-classic" }, - "color": { "mode": "palette-classic" }, "mappings": [] }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + } }, "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.95, sum by (le, service_name)(rate(traces_spanmetrics_latency_bucket[5m])))", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.95, sum by (le, service_name)(rate(tempo_spanmetrics_latency_bucket[5m])))", "legendFormat": "{{service_name}}", "instant": false, "range": true @@ -274,15 +445,26 @@ "id": 8, "type": "timeseries", "title": "Error Rate by Service", - "gridPos": { "h": 7, "w": 12, "x": 12, "y": 17 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 17 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "unit": "reqps", "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null } + { + "color": "green", + "value": null + } ] }, "custom": { @@ -297,22 +479,39 @@ "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "scaleDistribution": { "type": "linear" } + "scaleDistribution": { + "type": "linear" + } + }, + "color": { + "mode": "palette-classic" }, - "color": { "mode": "palette-classic" }, "mappings": [] }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + } }, "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "sum by (service_name)(rate(traces_spanmetrics_calls_total{status_code=\"STATUS_CODE_ERROR\"}[5m]))", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (service_name)(rate(tempo_distributor_spans_received_total{status=\"error\"}[5m]))", "legendFormat": "{{service_name}}", "instant": false, "range": true @@ -323,7 +522,12 @@ "id": 22, "type": "row", "title": "Trace Search", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 24 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 24 + }, "collapsed": false, "panels": [] }, @@ -331,14 +535,25 @@ "id": 10, "type": "table", "title": "Recent Traces", - "gridPos": { "h": 8, "w": 24, "x": 0, "y": 25 }, - "datasource": { "type": "tempo", "uid": "${DS_TEMPO}" }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 25 + }, + "datasource": { + "type": "tempo", + "uid": "${DS_TEMPO}" + }, "targets": [ { "refId": "A", - "datasource": { "type": "tempo", "uid": "${DS_TEMPO}" }, - "queryType": "nativeSearch", - "serviceName": "$service", + "datasource": { + "type": "tempo", + "uid": "${DS_TEMPO}" + }, + "queryType": "traceql", + "query": "{status = error} || {duration > 1s}", "limit": 20 } ]