gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(dashboard): use real Tempo metric names in traces dashboard

Browse Source 
Replace placeholder metric names (traces_spanmetrics_calls_total,
traces_spanmetrics_latency_*) with real Tempo metrics
(tempo_distributor_spans_received_total, tempo_spanmetrics_latency_*).
Switch Recent Traces panel from nativeSearch to TraceQL query.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-03-01 02:08:41 +01:00
parent 430b098f2c
commit 58c0f0ebc9
1 changed files with 280 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

345
infra/k8s/monitoring/dashboards/foxhunt-traces.json
View File

@@ -1,13 +1,29 @@
{
"__inputs": [
{ "name": "DS_PROMETHEUS", "label": "Prometheus", "description": "", "type": "datasource", "pluginId": "prometheus" },
{ "name": "DS_TEMPO", "label": "Tempo", "description": "", "type": "datasource", "pluginId": "tempo" }
{
"name": "DS_PROMETHEUS",
"label": "Prometheus",
"description": "",
"type": "datasource",
"pluginId": "prometheus"
},
{
"name": "DS_TEMPO",
"label": "Tempo",
"description": "",
"type": "datasource",
"pluginId": "tempo"
}
],
"id": null,
"uid": "foxhunt-traces",
"title": "Foxhunt - Traces",
"description": "Distributed tracing dashboard: span rates, error rates, latency percentiles, service map, trace search",
"tags": ["foxhunt", "traces", "tempo"],
"tags": [
"foxhunt",
"traces",
"tempo"
],
"timezone": "browser",
"schemaVersion": 39,
"version": 1,
@@ -25,12 +41,19 @@
{
"name": "service",
"type": "query",
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"query": "label_values(traces_service_graph_request_total, client)",
"refresh": 2,
"multi": true,
"includeAll": true,
"current": { "selected": true, "text": "All", "value": "$__all" }
"current": {
"selected": true,
"text": "All",
"value": "$__all"
}
}
]
},
@@ -39,7 +62,12 @@
"id": 1,
"type": "row",
"title": "Overview Stats",
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 0
},
"collapsed": false,
"panels": []
},
@@ -47,23 +75,42 @@
"id": 2,
"type": "stat",
"title": "Spans/sec",
"gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"gridPos": {
"h": 4,
"w": 6,
"x": 0,
"y": 1
},
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null }
{
"color": "green",
"value": null
}
]
},
"mappings": [],
"color": { "mode": "thresholds" }
"color": {
"mode": "thresholds"
}
},
"overrides": []
},
"options": {
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"orientation": "auto",
"textMode": "auto",
"colorMode": "background",
@@ -73,8 +120,11 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(rate(traces_spanmetrics_calls_total[5m]))",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(tempo_distributor_spans_received_total[5m]))",
"instant": true
}
]
@@ -83,26 +133,51 @@
"id": 3,
"type": "stat",
"title": "Error Rate",
"gridPos": { "h": 4, "w": 6, "x": 6, "y": 1 },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"gridPos": {
"h": 4,
"w": 6,
"x": 6,
"y": 1
},
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"unit": "percent",
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "yellow", "value": 1 },
{ "color": "red", "value": 5 }
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 1
},
{
"color": "red",
"value": 5
}
]
},
"mappings": [],
"color": { "mode": "thresholds" }
"color": {
"mode": "thresholds"
}
},
"overrides": []
},
"options": {
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"orientation": "auto",
"textMode": "auto",
"colorMode": "background",
@@ -112,8 +187,11 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(rate(traces_spanmetrics_calls_total{status_code=\"STATUS_CODE_ERROR\"}[5m])) / sum(rate(traces_spanmetrics_calls_total[5m])) * 100",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(tempo_distributor_spans_received_total{status=\"error\"}[5m])) / sum(rate(tempo_distributor_spans_received_total[5m])) * 100",
"instant": true
}
]
@@ -122,24 +200,43 @@
"id": 4,
"type": "stat",
"title": "Avg Duration",
"gridPos": { "h": 4, "w": 6, "x": 12, "y": 1 },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"gridPos": {
"h": 4,
"w": 6,
"x": 12,
"y": 1
},
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"unit": "s",
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null }
{
"color": "green",
"value": null
}
]
},
"mappings": [],
"color": { "mode": "thresholds" }
"color": {
"mode": "thresholds"
}
},
"overrides": []
},
"options": {
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"orientation": "auto",
"textMode": "auto",
"colorMode": "background",
@@ -149,8 +246,11 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(rate(traces_spanmetrics_latency_sum[5m])) / sum(rate(traces_spanmetrics_calls_total[5m]))",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(rate(tempo_spanmetrics_latency_sum[5m])) / sum(rate(tempo_spanmetrics_latency_count[5m]))",
"instant": true
}
]
@@ -159,23 +259,42 @@
"id": 5,
"type": "stat",
"title": "Active Services",
"gridPos": { "h": 4, "w": 6, "x": 18, "y": 1 },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"gridPos": {
"h": 4,
"w": 6,
"x": 18,
"y": 1
},
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null }
{
"color": "green",
"value": null
}
]
},
"mappings": [],
"color": { "mode": "thresholds" }
"color": {
"mode": "thresholds"
}
},
"overrides": []
},
"options": {
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"orientation": "auto",
"textMode": "auto",
"colorMode": "background",
@@ -185,8 +304,11 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "count(count by (service_name)(traces_spanmetrics_calls_total))",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "count(count by (service_name)(tempo_distributor_spans_received_total))",
"instant": true
}
]
@@ -195,7 +317,12 @@
"id": 20,
"type": "row",
"title": "Service Map + Trace Search",
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 },
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 5
},
"collapsed": false,
"panels": []
},
@@ -203,12 +330,23 @@
"id": 6,
"type": "nodeGraph",
"title": "Service Map",
"gridPos": { "h": 10, "w": 24, "x": 0, "y": 6 },
"datasource": { "type": "tempo", "uid": "${DS_TEMPO}" },
"gridPos": {
"h": 10,
"w": 24,
"x": 0,
"y": 6
},
"datasource": {
"type": "tempo",
"uid": "${DS_TEMPO}"
},
"targets": [
{
"refId": "A",
"datasource": { "type": "tempo", "uid": "${DS_TEMPO}" },
"datasource": {
"type": "tempo",
"uid": "${DS_TEMPO}"
},
"queryType": "serviceMap"
}
]
@@ -217,7 +355,12 @@
"id": 21,
"type": "row",
"title": "Latency and Errors by Service",
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 16 },
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 16
},
"collapsed": false,
"panels": []
},
@@ -225,15 +368,26 @@
"id": 7,
"type": "timeseries",
"title": "Latency by Service p95",
"gridPos": { "h": 7, "w": 12, "x": 0, "y": 17 },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
"y": 17
},
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"unit": "s",
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null }
{
"color": "green",
"value": null
}
]
},
"custom": {
@@ -248,22 +402,39 @@
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"scaleDistribution": { "type": "linear" }
"scaleDistribution": {
"type": "linear"
}
},
"color": {
"mode": "palette-classic"
},
"color": { "mode": "palette-classic" },
"mappings": []
},
"overrides": []
},
"options": {
"tooltip": { "mode": "multi", "sort": "desc" },
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] }
"tooltip": {
"mode": "multi",
"sort": "desc"
},
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": [
"mean",
"max"
]
}
},
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "histogram_quantile(0.95, sum by (le, service_name)(rate(traces_spanmetrics_latency_bucket[5m])))",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.95, sum by (le, service_name)(rate(tempo_spanmetrics_latency_bucket[5m])))",
"legendFormat": "{{service_name}}",
"instant": false,
"range": true
@@ -274,15 +445,26 @@
"id": 8,
"type": "timeseries",
"title": "Error Rate by Service",
"gridPos": { "h": 7, "w": 12, "x": 12, "y": 17 },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
"y": 17
},
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"unit": "reqps",
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null }
{
"color": "green",
"value": null
}
]
},
"custom": {
@@ -297,22 +479,39 @@
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"scaleDistribution": { "type": "linear" }
"scaleDistribution": {
"type": "linear"
}
},
"color": {
"mode": "palette-classic"
},
"color": { "mode": "palette-classic" },
"mappings": []
},
"overrides": []
},
"options": {
"tooltip": { "mode": "multi", "sort": "desc" },
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] }
"tooltip": {
"mode": "multi",
"sort": "desc"
},
"legend": {
"displayMode": "table",
"placement": "bottom",
"calcs": [
"mean",
"max"
]
}
},
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum by (service_name)(rate(traces_spanmetrics_calls_total{status_code=\"STATUS_CODE_ERROR\"}[5m]))",
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum by (service_name)(rate(tempo_distributor_spans_received_total{status=\"error\"}[5m]))",
"legendFormat": "{{service_name}}",
"instant": false,
"range": true
@@ -323,7 +522,12 @@
"id": 22,
"type": "row",
"title": "Trace Search",
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 24 },
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 24
},
"collapsed": false,
"panels": []
},
@@ -331,14 +535,25 @@
"id": 10,
"type": "table",
"title": "Recent Traces",
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 25 },
"datasource": { "type": "tempo", "uid": "${DS_TEMPO}" },
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 25
},
"datasource": {
"type": "tempo",
"uid": "${DS_TEMPO}"
},
"targets": [
{
"refId": "A",
"datasource": { "type": "tempo", "uid": "${DS_TEMPO}" },
"queryType": "nativeSearch",
"serviceName": "$service",
"datasource": {
"type": "tempo",
"uid": "${DS_TEMPO}"
},
"queryType": "traceql",
"query": "{status = error} || {duration > 1s}",
"limit": 20
}
]