gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(dashboards): correct metric prefixes, gRPC code filters, and service labels

Browse Source 
- Replace api_gateway_ with api_ prefix to match actual metric exports (cockpit, observability)
- Change grpc_code!="OK" to grpc_code!="0" — gRPC uses numeric status codes (cockpit, trading)
- Update gRPC service filter to actual exported_service labels: trading-agent-service,
  broker-gateway, backtesting-service, ml-training-service, data-acquisition-service (trading)

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-03-05 19:02:32 +01:00
parent 9dc4d2585f
commit 2b77df7dda
3 changed files with 50 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
infra/k8s/monitoring/dashboards/foxhunt-cockpit.json
View File

@@ -461,7 +461,7 @@
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"targets": [
{
"expr": "sum by (exported_service) (rate(grpc_server_handled_total{grpc_code!=\"OK\"}[$__rate_interval]))",
"expr": "sum by (exported_service) (rate(grpc_server_handled_total{grpc_code!=\"0\"}[$__rate_interval]))",
"legendFormat": "{{exported_service}}",
"refId": "A"
}
@@ -551,7 +551,7 @@
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"targets": [
{
"expr": "sum(rate(grpc_server_handled_total{grpc_code!=\"OK\"}[$__rate_interval])) / sum(rate(grpc_server_handled_total[$__rate_interval]))",
"expr": "sum(rate(grpc_server_handled_total{grpc_code!=\"0\"}[$__rate_interval])) / sum(rate(grpc_server_handled_total[$__rate_interval]))",
"legendFormat": "error ratio",
"refId": "A"
}
@@ -965,17 +965,17 @@
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"targets": [
{
"expr": "rate(api_gateway_auth_requests_total[$__rate_interval])",
"expr": "rate(api_auth_requests_total[$__rate_interval])",
"legendFormat": "total",
"refId": "A"
},
{
"expr": "rate(api_gateway_auth_requests_success[$__rate_interval])",
"expr": "rate(api_auth_requests_success[$__rate_interval])",
"legendFormat": "success",
"refId": "B"
},
{
"expr": "rate(api_gateway_auth_requests_failure[$__rate_interval])",
"expr": "rate(api_auth_requests_failure[$__rate_interval])",
"legendFormat": "failure",
"refId": "C"
}
@@ -1021,7 +1021,7 @@
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"targets": [
{
"expr": "api_gateway_rate_limiter_entries",
"expr": "api_rate_limiter_entries",
"legendFormat": "entries",
"refId": "A"
}
@@ -1059,7 +1059,7 @@
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"targets": [
{
"expr": "api_gateway_jwt_cache_hits / (api_gateway_jwt_cache_hits + api_gateway_jwt_cache_misses)",
"expr": "api_jwt_cache_hits / (api_jwt_cache_hits + api_jwt_cache_misses)",
"legendFormat": "hit ratio",
"refId": "A"
}
@@ -1099,17 +1099,17 @@
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"targets": [
{
"expr": "histogram_quantile(0.50, sum by (le) (rate(api_gateway_routing_duration_microseconds_bucket[$__rate_interval])))",
"expr": "histogram_quantile(0.50, sum by (le) (rate(api_routing_duration_microseconds_bucket[$__rate_interval])))",
"legendFormat": "p50",
"refId": "A"
},
{
"expr": "histogram_quantile(0.95, sum by (le) (rate(api_gateway_routing_duration_microseconds_bucket[$__rate_interval])))",
"expr": "histogram_quantile(0.95, sum by (le) (rate(api_routing_duration_microseconds_bucket[$__rate_interval])))",
"legendFormat": "p95",
"refId": "B"
},
{
"expr": "histogram_quantile(0.99, sum by (le) (rate(api_gateway_routing_duration_microseconds_bucket[$__rate_interval])))",
"expr": "histogram_quantile(0.99, sum by (le) (rate(api_routing_duration_microseconds_bucket[$__rate_interval])))",
"legendFormat": "p99",
"refId": "C"
}
@@ -1154,7 +1154,7 @@
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"targets": [
{
"expr": "rate(api_gateway_auth_requests_failure[$__rate_interval]) / rate(api_gateway_auth_requests_total[$__rate_interval])",
"expr": "rate(api_auth_requests_failure[$__rate_interval]) / rate(api_auth_requests_total[$__rate_interval])",
"legendFormat": "failure rate",
"refId": "A"
}
@@ -1194,7 +1194,7 @@
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"targets": [
{
"expr": "rate(api_gateway_jwt_cache_misses[$__rate_interval])",
"expr": "rate(api_jwt_cache_misses[$__rate_interval])",
"legendFormat": "misses/s",
"refId": "A"
}

66
infra/k8s/monitoring/dashboards/foxhunt-observability.json
View File

@@ -58,17 +58,17 @@
"type": "timeseries",
"targets": [
{
"expr": "rate(api_gateway_auth_requests_success[$interval])",
"expr": "rate(api_auth_requests_success[$interval])",
"legendFormat": "success",
"refId": "A"
},
{
"expr": "rate(api_gateway_auth_requests_failure[$interval])",
"expr": "rate(api_auth_requests_failure[$interval])",
"legendFormat": "failure",
"refId": "B"
},
{
"expr": "rate(api_gateway_auth_requests_total[$interval])",
"expr": "rate(api_auth_requests_total[$interval])",
"legendFormat": "total",
"refId": "C"
}
@@ -104,27 +104,27 @@
"type": "timeseries",
"targets": [
{
"expr": "rate(api_gateway_auth_errors_expired_jwt[$interval])",
"expr": "rate(api_auth_errors_expired_jwt[$interval])",
"legendFormat": "expired_jwt",
"refId": "A"
},
{
"expr": "rate(api_gateway_auth_errors_invalid_jwt[$interval])",
"expr": "rate(api_auth_errors_invalid_jwt[$interval])",
"legendFormat": "invalid_jwt",
"refId": "B"
},
{
"expr": "rate(api_gateway_auth_errors_missing_jwt[$interval])",
"expr": "rate(api_auth_errors_missing_jwt[$interval])",
"legendFormat": "missing_jwt",
"refId": "C"
},
{
"expr": "rate(api_gateway_auth_errors_permission_denied[$interval])",
"expr": "rate(api_auth_errors_permission_denied[$interval])",
"legendFormat": "permission_denied",
"refId": "D"
},
{
"expr": "rate(api_gateway_auth_errors_rate_limited[$interval])",
"expr": "rate(api_auth_errors_rate_limited[$interval])",
"legendFormat": "rate_limited",
"refId": "E"
}
@@ -160,17 +160,17 @@
"type": "timeseries",
"targets": [
{
"expr": "histogram_quantile(0.50, rate(api_gateway_jwt_validation_duration_microseconds_bucket[$interval]))",
"expr": "histogram_quantile(0.50, rate(api_jwt_validation_duration_microseconds_bucket[$interval]))",
"legendFormat": "p50",
"refId": "A"
},
{
"expr": "histogram_quantile(0.95, rate(api_gateway_jwt_validation_duration_microseconds_bucket[$interval]))",
"expr": "histogram_quantile(0.95, rate(api_jwt_validation_duration_microseconds_bucket[$interval]))",
"legendFormat": "p95",
"refId": "B"
},
{
"expr": "histogram_quantile(0.99, rate(api_gateway_jwt_validation_duration_microseconds_bucket[$interval]))",
"expr": "histogram_quantile(0.99, rate(api_jwt_validation_duration_microseconds_bucket[$interval]))",
"legendFormat": "p99",
"refId": "C"
}
@@ -202,7 +202,7 @@
"type": "stat",
"targets": [
{
"expr": "api_gateway_active_jwt_tokens",
"expr": "api_active_jwt_tokens",
"legendFormat": "",
"refId": "A"
}
@@ -243,7 +243,7 @@
"type": "stat",
"targets": [
{
"expr": "api_gateway_rate_limiter_entries",
"expr": "api_rate_limiter_entries",
"legendFormat": "",
"refId": "A"
}
@@ -281,7 +281,7 @@
"type": "timeseries",
"targets": [
{
"expr": "api_gateway_rbac_cache_hits / (api_gateway_rbac_cache_hits + api_gateway_rbac_cache_misses)",
"expr": "api_rbac_cache_hits / (api_rbac_cache_hits + api_rbac_cache_misses)",
"legendFormat": "hit ratio",
"refId": "A"
}
@@ -315,7 +315,7 @@
"type": "stat",
"targets": [
{
"expr": "api_gateway_revoked_tokens_cached",
"expr": "api_revoked_tokens_cached",
"legendFormat": "",
"refId": "A"
}
@@ -351,17 +351,17 @@
"type": "timeseries",
"targets": [
{
"expr": "histogram_quantile(0.50, rate(api_gateway_mfa_verification_duration_microseconds_bucket[$interval]))",
"expr": "histogram_quantile(0.50, rate(api_mfa_verification_duration_microseconds_bucket[$interval]))",
"legendFormat": "p50",
"refId": "A"
},
{
"expr": "histogram_quantile(0.95, rate(api_gateway_mfa_verification_duration_microseconds_bucket[$interval]))",
"expr": "histogram_quantile(0.95, rate(api_mfa_verification_duration_microseconds_bucket[$interval]))",
"legendFormat": "p95",
"refId": "B"
},
{
"expr": "histogram_quantile(0.99, rate(api_gateway_mfa_verification_duration_microseconds_bucket[$interval]))",
"expr": "histogram_quantile(0.99, rate(api_mfa_verification_duration_microseconds_bucket[$interval]))",
"legendFormat": "p99",
"refId": "C"
}
@@ -404,17 +404,17 @@
"type": "timeseries",
"targets": [
{
"expr": "histogram_quantile(0.50, rate(api_gateway_routing_duration_microseconds_bucket[$interval]))",
"expr": "histogram_quantile(0.50, rate(api_routing_duration_microseconds_bucket[$interval]))",
"legendFormat": "p50",
"refId": "A"
},
{
"expr": "histogram_quantile(0.95, rate(api_gateway_routing_duration_microseconds_bucket[$interval]))",
"expr": "histogram_quantile(0.95, rate(api_routing_duration_microseconds_bucket[$interval]))",
"legendFormat": "p95",
"refId": "B"
},
{
"expr": "histogram_quantile(0.99, rate(api_gateway_routing_duration_microseconds_bucket[$interval]))",
"expr": "histogram_quantile(0.99, rate(api_routing_duration_microseconds_bucket[$interval]))",
"legendFormat": "p99",
"refId": "C"
}
@@ -450,22 +450,22 @@
"type": "timeseries",
"targets": [
{
"expr": "rate(api_gateway_config_updates_auth[$interval])",
"expr": "rate(api_config_updates_auth[$interval])",
"legendFormat": "auth",
"refId": "A"
},
{
"expr": "rate(api_gateway_config_updates_backend[$interval])",
"expr": "rate(api_config_updates_backend[$interval])",
"legendFormat": "backend",
"refId": "B"
},
{
"expr": "rate(api_gateway_config_updates_rate_limit[$interval])",
"expr": "rate(api_config_updates_rate_limit[$interval])",
"legendFormat": "rate_limit",
"refId": "C"
},
{
"expr": "rate(api_gateway_config_updates_routing[$interval])",
"expr": "rate(api_config_updates_routing[$interval])",
"legendFormat": "routing",
"refId": "D"
}
@@ -503,12 +503,12 @@
"type": "timeseries",
"targets": [
{
"expr": "api_gateway_config_cache_hits / (api_gateway_config_cache_hits + api_gateway_config_cache_misses)",
"expr": "api_config_cache_hits / (api_config_cache_hits + api_config_cache_misses)",
"legendFormat": "config hit ratio",
"refId": "A"
},
{
"expr": "api_gateway_jwt_cache_hits / (api_gateway_jwt_cache_hits + api_gateway_jwt_cache_misses)",
"expr": "api_jwt_cache_hits / (api_jwt_cache_hits + api_jwt_cache_misses)",
"legendFormat": "jwt hit ratio",
"refId": "B"
}
@@ -564,17 +564,17 @@
"type": "timeseries",
"targets": [
{
"expr": "rate(api_gateway_notify_events_total[$interval])",
"expr": "rate(api_notify_events_total[$interval])",
"legendFormat": "total",
"refId": "A"
},
{
"expr": "rate(api_gateway_notify_events_success[$interval])",
"expr": "rate(api_notify_events_success[$interval])",
"legendFormat": "success",
"refId": "B"
},
{
"expr": "rate(api_gateway_notify_events_failure[$interval])",
"expr": "rate(api_notify_events_failure[$interval])",
"legendFormat": "failure",
"refId": "C"
}
@@ -606,7 +606,7 @@
"type": "stat",
"targets": [
{
"expr": "api_gateway_notify_listener_connected",
"expr": "api_notify_listener_connected",
"legendFormat": "",
"refId": "A"
}
@@ -655,12 +655,12 @@
"type": "timeseries",
"targets": [
{
"expr": "rate(api_gateway_notify_listener_errors[$interval])",
"expr": "rate(api_notify_listener_errors[$interval])",
"legendFormat": "errors",
"refId": "A"
},
{
"expr": "rate(api_gateway_notify_listener_reconnections[$interval])",
"expr": "rate(api_notify_listener_reconnections[$interval])",
"legendFormat": "reconnections",
"refId": "B"
}

10
infra/k8s/monitoring/dashboards/foxhunt-trading.json
View File

@@ -622,7 +622,7 @@
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"expr": "sum by (exported_service) (rate(grpc_server_handled_total{exported_service=~\"trading-service|trading-agent-service|broker-gateway\"}[$__rate_interval]))",
"expr": "sum by (exported_service) (rate(grpc_server_handled_total{exported_service=~\"trading-agent-service|broker-gateway|backtesting-service|ml-training-service|data-acquisition-service\"}[$__rate_interval]))",
"legendFormat": "{{exported_service}}"
}
]
@@ -661,7 +661,7 @@
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"expr": "sum by (exported_service, grpc_code) (rate(grpc_server_handled_total{exported_service=~\"trading-service|trading-agent-service|broker-gateway\", grpc_code!=\"OK\"}[$__rate_interval]))",
"expr": "sum by (exported_service, grpc_code) (rate(grpc_server_handled_total{exported_service=~\"trading-agent-service|broker-gateway|backtesting-service|ml-training-service|data-acquisition-service\", grpc_code!=\"0\"}[$__rate_interval]))",
"legendFormat": "{{exported_service}} {{grpc_code}}"
}
]
@@ -701,7 +701,7 @@
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"expr": "histogram_quantile(0.99, sum by (exported_service, le) (rate(grpc_server_handling_seconds_bucket{exported_service=~\"trading-service|trading-agent-service|broker-gateway\"}[$__rate_interval])))",
"expr": "histogram_quantile(0.99, sum by (exported_service, le) (rate(grpc_server_handling_seconds_bucket{exported_service=~\"trading-agent-service|broker-gateway|backtesting-service|ml-training-service|data-acquisition-service\"}[$__rate_interval])))",
"legendFormat": "{{exported_service}} p99"
}
]
@@ -740,7 +740,7 @@
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"expr": "sum(grpc_server_started_total{exported_service=~\"trading-service|trading-agent-service|broker-gateway\"})",
"expr": "sum(grpc_server_started_total{exported_service=~\"trading-agent-service|broker-gateway|backtesting-service|ml-training-service|data-acquisition-service\"})",
"instant": true,
"legendFormat": "total started"
}
@@ -783,7 +783,7 @@
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"expr": "sum(rate(grpc_server_handled_total{exported_service=~\"trading-service|trading-agent-service|broker-gateway\", grpc_code!=\"OK\"}[$__rate_interval])) / sum(rate(grpc_server_handled_total{exported_service=~\"trading-service|trading-agent-service|broker-gateway\"}[$__rate_interval]))",
"expr": "sum(rate(grpc_server_handled_total{exported_service=~\"trading-agent-service|broker-gateway|backtesting-service|ml-training-service|data-acquisition-service\", grpc_code!=\"0\"}[$__rate_interval])) / sum(rate(grpc_server_handled_total{exported_service=~\"trading-agent-service|broker-gateway|backtesting-service|ml-training-service|data-acquisition-service\"}[$__rate_interval]))",
"instant": true,
"legendFormat": "error ratio"
}