33b4020e06
Add foxhunt-cockpit.json — single-page overview with 7 rows (health bar,
service status, resources, GPU/training, restarts/network, errors, live
logs) pulling critical signals from all domain dashboards.
Migrate all 10 dashboards from hardcoded datasource UIDs to ${DS_PROMETHEUS},
${DS_LOKI}, ${DS_TEMPO} variables with __inputs sections for portable import.
Update import.sh to auto-detect datasource UIDs from Grafana API at deploy
time, using /api/dashboards/import endpoint.
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
587 lines
19 KiB
JSON
587 lines
19 KiB
JSON
{
|
|
"__inputs": [
|
|
{ "name": "DS_PROMETHEUS", "label": "Prometheus", "description": "", "type": "datasource", "pluginId": "prometheus" }
|
|
],
|
|
"id": null,
|
|
"uid": "foxhunt-gitlab-services",
|
|
"title": "GitLab + Foxhunt Services",
|
|
"description": "GitLab internals (Puma, Sidekiq, Gitaly) and Foxhunt microservice health",
|
|
"tags": ["gitlab", "foxhunt", "services", "kubernetes"],
|
|
"timezone": "browser",
|
|
"schemaVersion": 39,
|
|
"version": 1,
|
|
"refresh": "30s",
|
|
"time": {
|
|
"from": "now-1h",
|
|
"to": "now"
|
|
},
|
|
"fiscalYearStartMonth": 0,
|
|
"liveNow": false,
|
|
"weekStart": "",
|
|
"templating": {
|
|
"list": [
|
|
{
|
|
"name": "namespace",
|
|
"type": "query",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "${DS_PROMETHEUS}"
|
|
},
|
|
"query": "label_values(kube_pod_info, namespace)",
|
|
"refresh": 2,
|
|
"includeAll": true,
|
|
"multi": true,
|
|
"allValue": ".*",
|
|
"current": {
|
|
"selected": true,
|
|
"text": "All",
|
|
"value": "$__all"
|
|
},
|
|
"sort": 1
|
|
}
|
|
]
|
|
},
|
|
"panels": [
|
|
{
|
|
"id": 1,
|
|
"type": "row",
|
|
"title": "GitLab - Puma",
|
|
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
|
|
"collapsed": false,
|
|
"panels": []
|
|
},
|
|
{
|
|
"id": 2,
|
|
"type": "timeseries",
|
|
"title": "Puma HTTP Request Rate",
|
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 1 },
|
|
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineInterpolation": "smooth",
|
|
"fillOpacity": 15,
|
|
"gradientMode": "scheme",
|
|
"spanNulls": false,
|
|
"showPoints": "never",
|
|
"stacking": { "mode": "none", "group": "A" },
|
|
"axisPlacement": "auto",
|
|
"lineWidth": 1
|
|
},
|
|
"unit": "reqps",
|
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
|
"color": { "mode": "palette-classic" },
|
|
"mappings": []
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"tooltip": { "mode": "multi", "sort": "desc" },
|
|
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] }
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
|
"expr": "sum(rate(http_requests_total{job=~\".*puma.*\"}[5m])) by (method)",
|
|
"legendFormat": "{{ method }}"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 3,
|
|
"type": "timeseries",
|
|
"title": "Puma Request Duration (p95)",
|
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 1 },
|
|
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineInterpolation": "smooth",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"spanNulls": false,
|
|
"showPoints": "never",
|
|
"stacking": { "mode": "none", "group": "A" },
|
|
"axisPlacement": "auto",
|
|
"lineWidth": 1
|
|
},
|
|
"unit": "s",
|
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
|
"color": { "mode": "palette-classic" },
|
|
"mappings": []
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"tooltip": { "mode": "multi", "sort": "desc" },
|
|
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] }
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
|
"expr": "histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket{job=~\".*puma.*\"}[5m])) by (le))",
|
|
"legendFormat": "p95"
|
|
},
|
|
{
|
|
"refId": "B",
|
|
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
|
"expr": "histogram_quantile(0.50, sum(rate(http_request_duration_seconds_bucket{job=~\".*puma.*\"}[5m])) by (le))",
|
|
"legendFormat": "p50"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 10,
|
|
"type": "row",
|
|
"title": "GitLab - Sidekiq",
|
|
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 9 },
|
|
"collapsed": false,
|
|
"panels": []
|
|
},
|
|
{
|
|
"id": 11,
|
|
"type": "timeseries",
|
|
"title": "Sidekiq Job Processing Rate",
|
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 10 },
|
|
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineInterpolation": "smooth",
|
|
"fillOpacity": 15,
|
|
"gradientMode": "scheme",
|
|
"spanNulls": false,
|
|
"showPoints": "never",
|
|
"stacking": { "mode": "none", "group": "A" },
|
|
"axisPlacement": "auto",
|
|
"lineWidth": 1
|
|
},
|
|
"unit": "ops",
|
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
|
"color": { "mode": "palette-classic" },
|
|
"mappings": []
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"tooltip": { "mode": "multi", "sort": "desc" },
|
|
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] }
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
|
"expr": "sum(rate(sidekiq_jobs_processed_total[5m]))",
|
|
"legendFormat": "Processed"
|
|
},
|
|
{
|
|
"refId": "B",
|
|
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
|
"expr": "sum(rate(sidekiq_jobs_failed_total[5m]))",
|
|
"legendFormat": "Failed"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 12,
|
|
"type": "stat",
|
|
"title": "Sidekiq Queue Size",
|
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 10 },
|
|
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{ "color": "green", "value": null },
|
|
{ "color": "yellow", "value": 100 },
|
|
{ "color": "red", "value": 500 }
|
|
]
|
|
},
|
|
"mappings": [],
|
|
"color": { "mode": "thresholds" }
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
|
"orientation": "auto",
|
|
"textMode": "auto",
|
|
"colorMode": "value",
|
|
"graphMode": "area",
|
|
"justifyMode": "auto"
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
|
"expr": "sum(sidekiq_queue_size)",
|
|
"legendFormat": "Queue Size",
|
|
"instant": true
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 20,
|
|
"type": "row",
|
|
"title": "GitLab - Gitaly",
|
|
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 18 },
|
|
"collapsed": false,
|
|
"panels": []
|
|
},
|
|
{
|
|
"id": 21,
|
|
"type": "timeseries",
|
|
"title": "Gitaly RPC Latency (p95)",
|
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 19 },
|
|
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineInterpolation": "smooth",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"spanNulls": false,
|
|
"showPoints": "never",
|
|
"stacking": { "mode": "none", "group": "A" },
|
|
"axisPlacement": "auto",
|
|
"lineWidth": 1
|
|
},
|
|
"unit": "s",
|
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
|
"color": { "mode": "palette-classic" },
|
|
"mappings": []
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"tooltip": { "mode": "multi", "sort": "desc" },
|
|
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] }
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
|
"expr": "histogram_quantile(0.95, sum(rate(grpc_server_handling_seconds_bucket{job=~\".*gitaly.*\"}[5m])) by (le, grpc_method))",
|
|
"legendFormat": "{{ grpc_method }}"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 22,
|
|
"type": "timeseries",
|
|
"title": "Gitaly RPC Rate",
|
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 19 },
|
|
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineInterpolation": "smooth",
|
|
"fillOpacity": 15,
|
|
"gradientMode": "none",
|
|
"spanNulls": false,
|
|
"showPoints": "never",
|
|
"stacking": { "mode": "normal", "group": "A" },
|
|
"axisPlacement": "auto",
|
|
"lineWidth": 1
|
|
},
|
|
"unit": "reqps",
|
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
|
"color": { "mode": "palette-classic" },
|
|
"mappings": []
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"tooltip": { "mode": "multi", "sort": "desc" },
|
|
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] }
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
|
"expr": "sum(rate(grpc_server_handled_total{job=~\".*gitaly.*\"}[5m])) by (grpc_code)",
|
|
"legendFormat": "{{ grpc_code }}"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 30,
|
|
"type": "row",
|
|
"title": "Foxhunt Services - Resource Usage",
|
|
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 27 },
|
|
"collapsed": false,
|
|
"panels": []
|
|
},
|
|
{
|
|
"id": 31,
|
|
"type": "timeseries",
|
|
"title": "Container CPU Usage by Service",
|
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 28 },
|
|
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineInterpolation": "smooth",
|
|
"fillOpacity": 15,
|
|
"gradientMode": "none",
|
|
"spanNulls": false,
|
|
"showPoints": "never",
|
|
"stacking": { "mode": "none", "group": "A" },
|
|
"axisPlacement": "auto",
|
|
"lineWidth": 1
|
|
},
|
|
"unit": "short",
|
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
|
"color": { "mode": "palette-classic" },
|
|
"mappings": []
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"tooltip": { "mode": "multi", "sort": "desc" },
|
|
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] }
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
|
"expr": "sum by (container) (rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\", container=~\"web-gateway|api-gateway|trading-service|ml-training-service|backtesting-service|data-acquisition-service|broker-gateway-service|trading-agent-service\", image!=\"\"}[5m]))",
|
|
"legendFormat": "{{ container }}"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 32,
|
|
"type": "timeseries",
|
|
"title": "Container Memory Usage by Service",
|
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 28 },
|
|
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineInterpolation": "smooth",
|
|
"fillOpacity": 15,
|
|
"gradientMode": "none",
|
|
"spanNulls": false,
|
|
"showPoints": "never",
|
|
"stacking": { "mode": "none", "group": "A" },
|
|
"axisPlacement": "auto",
|
|
"lineWidth": 1
|
|
},
|
|
"unit": "bytes",
|
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
|
"color": { "mode": "palette-classic" },
|
|
"mappings": []
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"tooltip": { "mode": "multi", "sort": "desc" },
|
|
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] }
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
|
"expr": "sum by (container) (container_memory_working_set_bytes{namespace=~\"$namespace\", container=~\"web-gateway|api-gateway|trading-service|ml-training-service|backtesting-service|data-acquisition-service|broker-gateway-service|trading-agent-service\", image!=\"\"})",
|
|
"legendFormat": "{{ container }}"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 40,
|
|
"type": "row",
|
|
"title": "Foxhunt Services - Pod Health",
|
|
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 36 },
|
|
"collapsed": false,
|
|
"panels": []
|
|
},
|
|
{
|
|
"id": 41,
|
|
"type": "timeseries",
|
|
"title": "Pod Restarts (by Container)",
|
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 37 },
|
|
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"drawStyle": "bars",
|
|
"lineInterpolation": "linear",
|
|
"fillOpacity": 80,
|
|
"gradientMode": "none",
|
|
"spanNulls": false,
|
|
"showPoints": "never",
|
|
"stacking": { "mode": "normal", "group": "A" },
|
|
"axisPlacement": "auto",
|
|
"lineWidth": 1
|
|
},
|
|
"unit": "short",
|
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
|
"color": { "mode": "palette-classic" },
|
|
"mappings": []
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"tooltip": { "mode": "multi", "sort": "desc" },
|
|
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["sum", "max"] }
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
|
"expr": "increase(kube_pod_container_status_restarts_total{namespace=~\"$namespace\"}[1h])",
|
|
"legendFormat": "{{ namespace }}/{{ pod }} ({{ container }})"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 42,
|
|
"type": "table",
|
|
"title": "Pod Status by Namespace",
|
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 37 },
|
|
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
|
"mappings": [],
|
|
"color": { "mode": "thresholds" }
|
|
},
|
|
"overrides": [
|
|
{
|
|
"matcher": { "id": "byName", "options": "Failed" },
|
|
"properties": [
|
|
{
|
|
"id": "thresholds",
|
|
"value": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{ "color": "green", "value": null },
|
|
{ "color": "red", "value": 1 }
|
|
]
|
|
}
|
|
},
|
|
{ "id": "custom.displayMode", "value": "color-background" }
|
|
]
|
|
},
|
|
{
|
|
"matcher": { "id": "byName", "options": "Pending" },
|
|
"properties": [
|
|
{
|
|
"id": "thresholds",
|
|
"value": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{ "color": "green", "value": null },
|
|
{ "color": "yellow", "value": 1 }
|
|
]
|
|
}
|
|
},
|
|
{ "id": "custom.displayMode", "value": "color-background" }
|
|
]
|
|
}
|
|
]
|
|
},
|
|
"options": {
|
|
"showHeader": true,
|
|
"sortBy": [{ "displayName": "namespace", "desc": false }],
|
|
"footer": { "show": true, "reducer": ["sum"], "fields": "" }
|
|
},
|
|
"transformations": [
|
|
{
|
|
"id": "seriesToColumns",
|
|
"options": { "byField": "namespace" }
|
|
}
|
|
],
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
|
"expr": "sum by (namespace) (kube_pod_status_phase{namespace=~\"$namespace\", phase=\"Running\"})",
|
|
"legendFormat": "Running",
|
|
"instant": true,
|
|
"format": "table"
|
|
},
|
|
{
|
|
"refId": "B",
|
|
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
|
"expr": "sum by (namespace) (kube_pod_status_phase{namespace=~\"$namespace\", phase=\"Pending\"})",
|
|
"legendFormat": "Pending",
|
|
"instant": true,
|
|
"format": "table"
|
|
},
|
|
{
|
|
"refId": "C",
|
|
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
|
"expr": "sum by (namespace) (kube_pod_status_phase{namespace=~\"$namespace\", phase=\"Failed\"})",
|
|
"legendFormat": "Failed",
|
|
"instant": true,
|
|
"format": "table"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": 50,
|
|
"type": "row",
|
|
"title": "Kubernetes - Deployments",
|
|
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 45 },
|
|
"collapsed": false,
|
|
"panels": []
|
|
},
|
|
{
|
|
"id": 51,
|
|
"type": "timeseries",
|
|
"title": "Deployment Replicas: Desired vs Available",
|
|
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 46 },
|
|
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"lineInterpolation": "stepAfter",
|
|
"fillOpacity": 0,
|
|
"gradientMode": "none",
|
|
"spanNulls": false,
|
|
"showPoints": "never",
|
|
"stacking": { "mode": "none", "group": "A" },
|
|
"axisPlacement": "auto",
|
|
"lineWidth": 2
|
|
},
|
|
"unit": "short",
|
|
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
|
"color": { "mode": "palette-classic" },
|
|
"mappings": []
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"tooltip": { "mode": "multi", "sort": "desc" },
|
|
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["last"] }
|
|
},
|
|
"targets": [
|
|
{
|
|
"refId": "A",
|
|
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
|
"expr": "kube_deployment_spec_replicas{namespace=~\"$namespace\"}",
|
|
"legendFormat": "{{ deployment }} desired"
|
|
},
|
|
{
|
|
"refId": "B",
|
|
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
|
|
"expr": "kube_deployment_status_replicas_available{namespace=~\"$namespace\"}",
|
|
"legendFormat": "{{ deployment }} available"
|
|
}
|
|
]
|
|
}
|
|
]
|
|
} |