feat(monitoring): production dashboards, remove Cockpit, add Kapsule logs
- Remove Cockpit remote_write from GitLab Prometheus (all metrics local) - Uninstall k8s-monitoring Helm (Alloy no longer pushes to Cockpit) - Deploy standalone node-exporter DaemonSet + kube-state-metrics - Add 10 extra Prometheus scrape configs: node-exporter, kube-state-metrics, kubelet, cadvisor, DCGM GPU, foxhunt services, Loki, Tempo, Promtail, prometheus.io annotated pods - Provision 9 Grafana dashboards via sidecar (ConfigMap-based): overview, services, GPU/training, infrastructure, logs, CI/CD pipelines, cluster overview, GPU overview, GitLab services - Expand Promtail to ingest kube-system + GitLab runner + node-level logs - Add dashboard.fxhnt.ai DNS record + nginx routing to Grafana - Update CI deploy job to apply new monitoring manifests Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
@@ -1417,8 +1417,10 @@ deploy:
|
||||
# S3 binary share: services use generic runtime image + initContainer to fetch binaries from S3.
|
||||
# Deploy = apply K8s manifests (picks up YAML changes) + rollout restart (fetch new binaries).
|
||||
- echo "Deploying ${CI_COMMIT_SHA}..."
|
||||
# Deploy monitoring stack (Loki, Tempo, Promtail, Grafana datasources)
|
||||
- kubectl apply -f infra/k8s/monitoring/loki.yaml -f infra/k8s/monitoring/tempo.yaml -f infra/k8s/monitoring/promtail.yaml -f infra/k8s/monitoring/grafana-datasources.yaml
|
||||
# Deploy monitoring stack (Loki, Tempo, Promtail, node-exporter, kube-state-metrics)
|
||||
- kubectl apply -f infra/k8s/monitoring/loki.yaml -f infra/k8s/monitoring/tempo.yaml -f infra/k8s/monitoring/promtail.yaml -f infra/k8s/monitoring/node-exporter.yaml -f infra/k8s/monitoring/kube-state-metrics.yaml
|
||||
# Apply proxy (picks up nginx config changes for dashboard.fxhnt.ai routing)
|
||||
- kubectl apply -f infra/k8s/gitlab/tailscale-proxy.yaml
|
||||
# Apply service deployments and binary cache PVCs (idempotent — picks up any YAML changes)
|
||||
# Skip training/job-template.yaml (placeholder names, not directly applicable)
|
||||
# Skip storage/training-data-pv.yaml (PVC spec immutable after creation)
|
||||
|
||||
@@ -196,10 +196,10 @@ data:
|
||||
}
|
||||
}
|
||||
|
||||
# Grafana — grafana.fxhnt.ai
|
||||
# Grafana — grafana.fxhnt.ai + dashboard.fxhnt.ai
|
||||
server {
|
||||
listen 443 ssl;
|
||||
server_name grafana.fxhnt.ai;
|
||||
server_name grafana.fxhnt.ai dashboard.fxhnt.ai;
|
||||
|
||||
ssl_certificate /etc/nginx/certs/tls.crt;
|
||||
ssl_certificate_key /etc/nginx/certs/tls.key;
|
||||
|
||||
973
infra/k8s/monitoring/dashboards/foxhunt-ci-pipelines.json
Normal file
973
infra/k8s/monitoring/dashboards/foxhunt-ci-pipelines.json
Normal file
@@ -0,0 +1,973 @@
|
||||
{
|
||||
"id": null,
|
||||
"uid": "foxhunt-ci-pipelines",
|
||||
"title": "Foxhunt CI/CD Pipelines",
|
||||
"description": "GitLab CI pipeline metrics, failure analysis, runner performance, and K8s training jobs",
|
||||
"tags": ["foxhunt", "ci-cd", "pipelines"],
|
||||
"timezone": "browser",
|
||||
"schemaVersion": 39,
|
||||
"version": 1,
|
||||
"editable": true,
|
||||
"refresh": "30s",
|
||||
"time": {
|
||||
"from": "now-24h",
|
||||
"to": "now"
|
||||
},
|
||||
"fiscalYearStartMonth": 0,
|
||||
"liveNow": false,
|
||||
"weekStart": "",
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"name": "interval",
|
||||
"type": "interval",
|
||||
"query": "1m,5m,15m,30m,1h",
|
||||
"current": {
|
||||
"selected": true,
|
||||
"text": "5m",
|
||||
"value": "5m"
|
||||
},
|
||||
"auto": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "row",
|
||||
"title": "Pipeline Overview",
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
|
||||
"collapsed": false,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "stat",
|
||||
"title": "Pipeline Creation Rate",
|
||||
"description": "Pipelines created per second (averaged over $interval)",
|
||||
"gridPos": { "h": 6, "w": 6, "x": 0, "y": 1 },
|
||||
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "ops",
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "blue", "value": null },
|
||||
{ "color": "green", "value": 0.01 }
|
||||
]
|
||||
},
|
||||
"mappings": [],
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
|
||||
"expr": "sum(rate(gitlab_ci_pipeline_processing_events_total[$interval]))",
|
||||
"legendFormat": "pipelines/s",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "stat",
|
||||
"title": "Active CI Jobs",
|
||||
"description": "Current number of active CI jobs",
|
||||
"gridPos": { "h": 6, "w": 6, "x": 6, "y": 1 },
|
||||
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "yellow", "value": 5 },
|
||||
{ "color": "red", "value": 10 }
|
||||
]
|
||||
},
|
||||
"mappings": [],
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
|
||||
"expr": "sum(gitlab_ci_active_jobs_sum)",
|
||||
"legendFormat": "active jobs",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"type": "stat",
|
||||
"title": "Pipeline Failure Rate",
|
||||
"description": "Pipeline failures per second (averaged over $interval)",
|
||||
"gridPos": { "h": 6, "w": 6, "x": 12, "y": 1 },
|
||||
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "ops",
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "yellow", "value": 0.001 },
|
||||
{ "color": "red", "value": 0.01 }
|
||||
]
|
||||
},
|
||||
"mappings": [],
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
|
||||
"expr": "sum(rate(gitlab_ci_pipeline_failure_reasons[$interval]))",
|
||||
"legendFormat": "failures/s",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"type": "stat",
|
||||
"title": "Queue Operations Rate",
|
||||
"description": "CI queue operations per second (averaged over $interval)",
|
||||
"gridPos": { "h": 6, "w": 6, "x": 18, "y": 1 },
|
||||
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "ops",
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "blue", "value": null },
|
||||
{ "color": "green", "value": 0.01 }
|
||||
]
|
||||
},
|
||||
"mappings": [],
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
|
||||
"expr": "sum(rate(gitlab_ci_queue_operations_total[$interval]))",
|
||||
"legendFormat": "queue ops/s",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
{
|
||||
"id": 10,
|
||||
"type": "row",
|
||||
"title": "Pipeline Performance",
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 7 },
|
||||
"collapsed": false,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"id": 11,
|
||||
"type": "timeseries",
|
||||
"title": "Pipeline Creation Duration",
|
||||
"description": "Time to create a pipeline (P50, P90, P99)",
|
||||
"gridPos": { "h": 8, "w": 8, "x": 0, "y": 8 },
|
||||
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"lineWidth": 2
|
||||
},
|
||||
"unit": "s",
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
||||
"color": { "mode": "palette-classic" },
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": { "id": "byName", "options": "P99" },
|
||||
"properties": [
|
||||
{ "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } }
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": { "id": "byName", "options": "P90" },
|
||||
"properties": [
|
||||
{ "id": "color", "value": { "fixedColor": "orange", "mode": "fixed" } }
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": { "id": "byName", "options": "P50" },
|
||||
"properties": [
|
||||
{ "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } }
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
|
||||
"expr": "histogram_quantile(0.50, sum(rate(gitlab_ci_pipeline_creation_duration_seconds_bucket[$interval])) by (le))",
|
||||
"legendFormat": "P50"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
|
||||
"expr": "histogram_quantile(0.90, sum(rate(gitlab_ci_pipeline_creation_duration_seconds_bucket[$interval])) by (le))",
|
||||
"legendFormat": "P90"
|
||||
},
|
||||
{
|
||||
"refId": "C",
|
||||
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
|
||||
"expr": "histogram_quantile(0.99, sum(rate(gitlab_ci_pipeline_creation_duration_seconds_bucket[$interval])) by (le))",
|
||||
"legendFormat": "P99"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 12,
|
||||
"type": "timeseries",
|
||||
"title": "Pipeline Size (Builds per Pipeline)",
|
||||
"description": "Number of builds/jobs per pipeline (P50, P90, P99)",
|
||||
"gridPos": { "h": 8, "w": 8, "x": 8, "y": 8 },
|
||||
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"lineWidth": 2
|
||||
},
|
||||
"unit": "short",
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
||||
"color": { "mode": "palette-classic" },
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": { "id": "byName", "options": "P99" },
|
||||
"properties": [
|
||||
{ "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } }
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": { "id": "byName", "options": "P90" },
|
||||
"properties": [
|
||||
{ "id": "color", "value": { "fixedColor": "orange", "mode": "fixed" } }
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": { "id": "byName", "options": "P50" },
|
||||
"properties": [
|
||||
{ "id": "color", "value": { "fixedColor": "semi-dark-blue", "mode": "fixed" } }
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
|
||||
"expr": "histogram_quantile(0.50, sum(rate(gitlab_ci_pipeline_size_builds_bucket[$interval])) by (le))",
|
||||
"legendFormat": "P50"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
|
||||
"expr": "histogram_quantile(0.90, sum(rate(gitlab_ci_pipeline_size_builds_bucket[$interval])) by (le))",
|
||||
"legendFormat": "P90"
|
||||
},
|
||||
{
|
||||
"refId": "C",
|
||||
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
|
||||
"expr": "histogram_quantile(0.99, sum(rate(gitlab_ci_pipeline_size_builds_bucket[$interval])) by (le))",
|
||||
"legendFormat": "P99"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 13,
|
||||
"type": "heatmap",
|
||||
"title": "Active Jobs Distribution",
|
||||
"description": "Heatmap of active CI job counts over time",
|
||||
"gridPos": { "h": 8, "w": 8, "x": 16, "y": 8 },
|
||||
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"scaleDistribution": { "type": "linear" }
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"calculate": false,
|
||||
"yAxis": {
|
||||
"axisPlacement": "left",
|
||||
"unit": "short"
|
||||
},
|
||||
"rowsFrame": {
|
||||
"layout": "auto"
|
||||
},
|
||||
"color": {
|
||||
"mode": "scheme",
|
||||
"scheme": "Oranges",
|
||||
"fill": "dark",
|
||||
"scale": "exponential",
|
||||
"exponent": 0.5,
|
||||
"steps": 64
|
||||
},
|
||||
"cellGap": 1,
|
||||
"filterValues": { "le": 1e-9 },
|
||||
"tooltip": { "show": true, "yHistogram": true },
|
||||
"legend": { "show": true }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
|
||||
"expr": "sum(increase(gitlab_ci_active_jobs_bucket[$interval])) by (le)",
|
||||
"legendFormat": "{{le}}",
|
||||
"format": "heatmap"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
{
|
||||
"id": 20,
|
||||
"type": "row",
|
||||
"title": "Failure Analysis",
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 16 },
|
||||
"collapsed": false,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"id": 21,
|
||||
"type": "table",
|
||||
"title": "Pipeline Failure Reasons",
|
||||
"description": "Breakdown of pipeline failure reasons over the selected time range",
|
||||
"gridPos": { "h": 8, "w": 8, "x": 0, "y": 17 },
|
||||
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "yellow", "value": 5 },
|
||||
{ "color": "red", "value": 20 }
|
||||
]
|
||||
},
|
||||
"mappings": [],
|
||||
"color": { "mode": "thresholds" },
|
||||
"custom": {
|
||||
"align": "auto",
|
||||
"cellOptions": {
|
||||
"type": "color-background",
|
||||
"mode": "basic"
|
||||
},
|
||||
"filterable": true
|
||||
}
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": { "id": "byName", "options": "Reason" },
|
||||
"properties": [
|
||||
{ "id": "custom.width", "value": 200 }
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": { "id": "byName", "options": "Time" },
|
||||
"properties": [
|
||||
{ "id": "custom.hidden", "value": true }
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"showHeader": true,
|
||||
"sortBy": [{ "displayName": "Value", "desc": true }],
|
||||
"footer": {
|
||||
"show": true,
|
||||
"reducer": ["sum"],
|
||||
"countRows": false
|
||||
}
|
||||
},
|
||||
"transformations": [
|
||||
{
|
||||
"id": "sortBy",
|
||||
"options": {
|
||||
"fields": {},
|
||||
"sort": [{ "field": "Value", "desc": true }]
|
||||
}
|
||||
}
|
||||
],
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
|
||||
"expr": "sum by (reason) (increase(gitlab_ci_pipeline_failure_reasons[$__range]))",
|
||||
"legendFormat": "{{reason}}",
|
||||
"instant": true,
|
||||
"format": "table"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 22,
|
||||
"type": "table",
|
||||
"title": "Job Failure Reasons",
|
||||
"description": "Breakdown of individual job failure reasons over the selected time range",
|
||||
"gridPos": { "h": 8, "w": 8, "x": 8, "y": 17 },
|
||||
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "yellow", "value": 5 },
|
||||
{ "color": "red", "value": 20 }
|
||||
]
|
||||
},
|
||||
"mappings": [],
|
||||
"color": { "mode": "thresholds" },
|
||||
"custom": {
|
||||
"align": "auto",
|
||||
"cellOptions": {
|
||||
"type": "color-background",
|
||||
"mode": "basic"
|
||||
},
|
||||
"filterable": true
|
||||
}
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": { "id": "byName", "options": "Failure Reason" },
|
||||
"properties": [
|
||||
{ "id": "custom.width", "value": 200 }
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": { "id": "byName", "options": "Time" },
|
||||
"properties": [
|
||||
{ "id": "custom.hidden", "value": true }
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"showHeader": true,
|
||||
"sortBy": [{ "displayName": "Value", "desc": true }],
|
||||
"footer": {
|
||||
"show": true,
|
||||
"reducer": ["sum"],
|
||||
"countRows": false
|
||||
}
|
||||
},
|
||||
"transformations": [
|
||||
{
|
||||
"id": "sortBy",
|
||||
"options": {
|
||||
"fields": {},
|
||||
"sort": [{ "field": "Value", "desc": true }]
|
||||
}
|
||||
}
|
||||
],
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
|
||||
"expr": "sum by (failure_reason) (increase(gitlab_ci_job_failure_reasons[$__range]))",
|
||||
"legendFormat": "{{failure_reason}}",
|
||||
"instant": true,
|
||||
"format": "table"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 23,
|
||||
"type": "timeseries",
|
||||
"title": "Build Trace Errors",
|
||||
"description": "Rate of build trace errors over time",
|
||||
"gridPos": { "h": 8, "w": 8, "x": 16, "y": 17 },
|
||||
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"fillOpacity": 20,
|
||||
"gradientMode": "scheme",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"lineWidth": 2
|
||||
},
|
||||
"unit": "ops",
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "red", "value": 0.1 }
|
||||
]
|
||||
},
|
||||
"color": { "fixedColor": "red", "mode": "fixed" },
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "sum"] }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
|
||||
"expr": "sum(rate(gitlab_ci_build_trace_errors_total[$interval]))",
|
||||
"legendFormat": "trace errors/s"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
{
|
||||
"id": 30,
|
||||
"type": "row",
|
||||
"title": "Runner Performance",
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 25 },
|
||||
"collapsed": false,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"id": 31,
|
||||
"type": "timeseries",
|
||||
"title": "Runner Authentication Success",
|
||||
"description": "Rate of successful runner authentication attempts",
|
||||
"gridPos": { "h": 8, "w": 8, "x": 0, "y": 26 },
|
||||
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"fillOpacity": 15,
|
||||
"gradientMode": "scheme",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"lineWidth": 2
|
||||
},
|
||||
"unit": "ops",
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
||||
"color": { "fixedColor": "green", "mode": "fixed" },
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "sum"] }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
|
||||
"expr": "sum(rate(gitlab_ci_runner_authentication_success_total[$interval]))",
|
||||
"legendFormat": "auth success/s"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 32,
|
||||
"type": "timeseries",
|
||||
"title": "Trace Operations & Volume",
|
||||
"description": "CI trace operations rate and data volume",
|
||||
"gridPos": { "h": 8, "w": 8, "x": 8, "y": 26 },
|
||||
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"lineWidth": 2
|
||||
},
|
||||
"unit": "ops",
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
||||
"color": { "mode": "palette-classic" },
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": { "id": "byName", "options": "trace bytes/s" },
|
||||
"properties": [
|
||||
{ "id": "custom.axisPlacement", "value": "right" },
|
||||
{ "id": "unit", "value": "Bps" }
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
|
||||
"expr": "sum(rate(gitlab_ci_trace_operations_total[$interval]))",
|
||||
"legendFormat": "trace ops/s"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
|
||||
"expr": "sum(rate(gitlab_ci_trace_bytes_total[$interval]))",
|
||||
"legendFormat": "trace bytes/s"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 33,
|
||||
"type": "timeseries",
|
||||
"title": "Artifacts Cleanup",
|
||||
"description": "Rate and volume of destroyed job artifacts",
|
||||
"gridPos": { "h": 8, "w": 8, "x": 16, "y": 26 },
|
||||
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "bars",
|
||||
"lineInterpolation": "smooth",
|
||||
"fillOpacity": 60,
|
||||
"gradientMode": "scheme",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"lineWidth": 0,
|
||||
"barAlignment": 0
|
||||
},
|
||||
"unit": "ops",
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
||||
"color": { "mode": "palette-classic" },
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": { "id": "byName", "options": "bytes destroyed/s" },
|
||||
"properties": [
|
||||
{ "id": "custom.axisPlacement", "value": "right" },
|
||||
{ "id": "unit", "value": "Bps" },
|
||||
{ "id": "custom.drawStyle", "value": "line" },
|
||||
{ "id": "custom.fillOpacity", "value": 0 },
|
||||
{ "id": "custom.lineWidth", "value": 2 }
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "sum"] }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
|
||||
"expr": "sum(rate(destroyed_job_artifacts_count_total[$interval]))",
|
||||
"legendFormat": "artifacts destroyed/s"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
|
||||
"expr": "sum(rate(destroyed_job_artifacts_bytes_total[$interval]))",
|
||||
"legendFormat": "bytes destroyed/s"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
{
|
||||
"id": 40,
|
||||
"type": "row",
|
||||
"title": "K8s Training Jobs",
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 34 },
|
||||
"collapsed": false,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"id": 41,
|
||||
"type": "timeseries",
|
||||
"title": "Training Jobs: Created vs Completed vs Failed",
|
||||
"description": "K8s training job lifecycle in the foxhunt namespace",
|
||||
"gridPos": { "h": 8, "w": 8, "x": 0, "y": 35 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "stepAfter",
|
||||
"fillOpacity": 15,
|
||||
"gradientMode": "none",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"lineWidth": 2
|
||||
},
|
||||
"unit": "short",
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
||||
"color": { "mode": "palette-classic" },
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": { "id": "byName", "options": "Created" },
|
||||
"properties": [
|
||||
{ "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } }
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": { "id": "byName", "options": "Succeeded" },
|
||||
"properties": [
|
||||
{ "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } }
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": { "id": "byName", "options": "Failed" },
|
||||
"properties": [
|
||||
{ "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } }
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "max"] }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "count(kube_job_info{namespace=\"foxhunt\", job_name=~\".*train.*\"})",
|
||||
"legendFormat": "Created"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "count(kube_job_status_succeeded{namespace=\"foxhunt\", job_name=~\".*train.*\"} == 1)",
|
||||
"legendFormat": "Succeeded"
|
||||
},
|
||||
{
|
||||
"refId": "C",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "count(kube_job_status_failed{namespace=\"foxhunt\", job_name=~\".*train.*\"} > 0)",
|
||||
"legendFormat": "Failed"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 42,
|
||||
"type": "timeseries",
|
||||
"title": "Training Job Duration",
|
||||
"description": "Age of active training jobs (time since creation)",
|
||||
"gridPos": { "h": 8, "w": 8, "x": 8, "y": 35 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"lineWidth": 2
|
||||
},
|
||||
"unit": "s",
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "yellow", "value": 3600 },
|
||||
{ "color": "red", "value": 14400 }
|
||||
]
|
||||
},
|
||||
"color": { "mode": "palette-classic" },
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "lastNotNull"] }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "time() - kube_job_created{namespace=\"foxhunt\", job_name=~\".*train.*\"} and on(job_name, namespace) kube_job_status_succeeded{namespace=\"foxhunt\"} == 0 and on(job_name, namespace) (kube_job_status_failed{namespace=\"foxhunt\"} == 0 or absent(kube_job_status_failed{namespace=\"foxhunt\"}))",
|
||||
"legendFormat": "{{job_name}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 43,
|
||||
"type": "timeseries",
|
||||
"title": "Training Pod CPU Usage",
|
||||
"description": "CPU usage of training pods in the foxhunt namespace",
|
||||
"gridPos": { "h": 8, "w": 4, "x": 16, "y": 35 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"fillOpacity": 20,
|
||||
"gradientMode": "scheme",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"stacking": { "mode": "normal", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"lineWidth": 1
|
||||
},
|
||||
"unit": "short",
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
||||
"color": { "mode": "palette-classic" },
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\".*train.*\", container!=\"POD\", container!=\"\"}[$interval]))",
|
||||
"legendFormat": "{{pod}}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 44,
|
||||
"type": "timeseries",
|
||||
"title": "Training Pod Memory",
|
||||
"description": "Memory working set of training pods in the foxhunt namespace",
|
||||
"gridPos": { "h": 8, "w": 4, "x": 20, "y": 35 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"fillOpacity": 20,
|
||||
"gradientMode": "scheme",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"stacking": { "mode": "normal", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"lineWidth": 1
|
||||
},
|
||||
"unit": "bytes",
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
||||
"color": { "mode": "palette-classic" },
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "sum by (pod) (container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\".*train.*\", container!=\"POD\", container!=\"\"})",
|
||||
"legendFormat": "{{pod}}"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
761
infra/k8s/monitoring/dashboards/foxhunt-gpu-training.json
Normal file
761
infra/k8s/monitoring/dashboards/foxhunt-gpu-training.json
Normal file
@@ -0,0 +1,761 @@
|
||||
{
|
||||
"id": null,
|
||||
"uid": "foxhunt-gpu-training",
|
||||
"title": "Foxhunt - GPU & Training Jobs",
|
||||
"description": "GPU node pools, DCGM metrics, K8s training jobs, CI pipelines",
|
||||
"tags": ["foxhunt", "production"],
|
||||
"timezone": "browser",
|
||||
"schemaVersion": 39,
|
||||
"version": 1,
|
||||
"editable": true,
|
||||
"refresh": "30s",
|
||||
"time": {
|
||||
"from": "now-1h",
|
||||
"to": "now"
|
||||
},
|
||||
"fiscalYearStartMonth": 0,
|
||||
"liveNow": false,
|
||||
"weekStart": "",
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"name": "gpu",
|
||||
"type": "query",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"query": "label_values(DCGM_FI_DEV_GPU_UTIL, gpu)",
|
||||
"refresh": 2,
|
||||
"includeAll": true,
|
||||
"multi": true,
|
||||
"allValue": ".*",
|
||||
"current": { "selected": true, "text": "All", "value": "$__all" },
|
||||
"sort": 1
|
||||
},
|
||||
{
|
||||
"name": "interval",
|
||||
"type": "interval",
|
||||
"query": "1m,5m,15m,30m,1h",
|
||||
"current": { "selected": true, "text": "5m", "value": "5m" },
|
||||
"auto": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "row",
|
||||
"title": "GPU Node Pool Status",
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
|
||||
"collapsed": false,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "stat",
|
||||
"title": "ci-compile Nodes",
|
||||
"description": "L4-1-24G GPU nodes (0-1 autoscale)",
|
||||
"gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "blue", "value": null },
|
||||
{ "color": "green", "value": 1 }
|
||||
]
|
||||
},
|
||||
"mappings": [],
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "center"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "count(kube_node_labels{label_k8s_scaleway_com_pool_name=\"ci-compile\"}) or vector(0)",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "stat",
|
||||
"title": "ci-training Nodes",
|
||||
"description": "L40S-1-48G GPU nodes (0-1 autoscale)",
|
||||
"gridPos": { "h": 4, "w": 4, "x": 4, "y": 1 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "blue", "value": null },
|
||||
{ "color": "green", "value": 1 }
|
||||
]
|
||||
},
|
||||
"mappings": [],
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "center"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "count(kube_node_labels{label_k8s_scaleway_com_pool_name=\"ci-training\"}) or vector(0)",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"type": "stat",
|
||||
"title": "ci-compile-cpu Nodes",
|
||||
"description": "POP2-32C-128G CPU nodes (0-1 autoscale)",
|
||||
"gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "blue", "value": null },
|
||||
{ "color": "green", "value": 1 }
|
||||
]
|
||||
},
|
||||
"mappings": [],
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "center"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "count(kube_node_labels{label_k8s_scaleway_com_pool_name=\"ci-compile-cpu\"}) or vector(0)",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"type": "stat",
|
||||
"title": "Active Training Jobs",
|
||||
"gridPos": { "h": 4, "w": 4, "x": 12, "y": 1 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "blue", "value": null },
|
||||
{ "color": "green", "value": 1 }
|
||||
]
|
||||
},
|
||||
"mappings": [],
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "center"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "sum(kube_job_status_active{namespace=\"foxhunt\"}) or vector(0)",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"type": "stat",
|
||||
"title": "Jobs Succeeded (24h)",
|
||||
"gridPos": { "h": 4, "w": 4, "x": 16, "y": 1 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null }
|
||||
]
|
||||
},
|
||||
"mappings": [],
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "center"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "sum(increase(kube_job_status_succeeded{namespace=\"foxhunt\"}[24h])) or vector(0)",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"type": "stat",
|
||||
"title": "Jobs Failed (24h)",
|
||||
"gridPos": { "h": 4, "w": 4, "x": 20, "y": 1 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "red", "value": 1 }
|
||||
]
|
||||
},
|
||||
"mappings": [],
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "center"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "sum(increase(kube_job_status_failed{namespace=\"foxhunt\"}[24h])) or vector(0)",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"type": "row",
|
||||
"title": "GPU Utilization (DCGM)",
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 },
|
||||
"collapsed": false,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"id": 11,
|
||||
"type": "timeseries",
|
||||
"title": "GPU Utilization %",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"fillOpacity": 20,
|
||||
"gradientMode": "scheme",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"lineWidth": 2
|
||||
},
|
||||
"unit": "percent",
|
||||
"min": 0,
|
||||
"max": 100,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "yellow", "value": 80 },
|
||||
{ "color": "red", "value": 95 }
|
||||
]
|
||||
},
|
||||
"color": { "mode": "palette-classic" },
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "DCGM_FI_DEV_GPU_UTIL{gpu=~\"$gpu\"}",
|
||||
"legendFormat": "GPU {{ gpu }} ({{ instance }})"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 12,
|
||||
"type": "timeseries",
|
||||
"title": "GPU Memory Used (MB)",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"fillOpacity": 20,
|
||||
"gradientMode": "scheme",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"lineWidth": 2
|
||||
},
|
||||
"unit": "decmbytes",
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
||||
"color": { "mode": "palette-classic" },
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "DCGM_FI_DEV_FB_USED{gpu=~\"$gpu\"}",
|
||||
"legendFormat": "GPU {{ gpu }} used"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "DCGM_FI_DEV_FB_TOTAL{gpu=~\"$gpu\"}",
|
||||
"legendFormat": "GPU {{ gpu }} total"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 13,
|
||||
"type": "timeseries",
|
||||
"title": "GPU Temperature",
|
||||
"gridPos": { "h": 6, "w": 12, "x": 0, "y": 14 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "scheme",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"lineWidth": 2
|
||||
},
|
||||
"unit": "celsius",
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "blue", "value": null },
|
||||
{ "color": "green", "value": 40 },
|
||||
{ "color": "yellow", "value": 75 },
|
||||
{ "color": "red", "value": 85 }
|
||||
]
|
||||
},
|
||||
"color": { "mode": "palette-classic" },
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "DCGM_FI_DEV_GPU_TEMP{gpu=~\"$gpu\"}",
|
||||
"legendFormat": "GPU {{ gpu }}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 14,
|
||||
"type": "timeseries",
|
||||
"title": "GPU Power (Watts)",
|
||||
"gridPos": { "h": 6, "w": 12, "x": 12, "y": 14 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "scheme",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"lineWidth": 2
|
||||
},
|
||||
"unit": "watt",
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
||||
"color": { "mode": "palette-classic" },
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "DCGM_FI_DEV_POWER_USAGE{gpu=~\"$gpu\"}",
|
||||
"legendFormat": "GPU {{ gpu }}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 20,
|
||||
"type": "row",
|
||||
"title": "Training Jobs",
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 20 },
|
||||
"collapsed": false,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"id": 21,
|
||||
"type": "timeseries",
|
||||
"title": "Active Jobs Over Time",
|
||||
"gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "stepAfter",
|
||||
"fillOpacity": 30,
|
||||
"gradientMode": "none",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"lineWidth": 2
|
||||
},
|
||||
"unit": "short",
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
||||
"color": { "mode": "palette-classic" },
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["last"] }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "kube_job_status_active{namespace=\"foxhunt\"}",
|
||||
"legendFormat": "{{ job_name }}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 22,
|
||||
"type": "table",
|
||||
"title": "Job Status Summary",
|
||||
"gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
||||
"mappings": [],
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": { "id": "byName", "options": "Failed" },
|
||||
"properties": [
|
||||
{
|
||||
"id": "thresholds",
|
||||
"value": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "red", "value": 1 }
|
||||
]
|
||||
}
|
||||
},
|
||||
{ "id": "custom.displayMode", "value": "color-background" }
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"showHeader": true,
|
||||
"sortBy": [{ "displayName": "job_name", "desc": false }],
|
||||
"footer": { "show": false }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "kube_job_status_active{namespace=\"foxhunt\"}",
|
||||
"legendFormat": "Active",
|
||||
"instant": true,
|
||||
"format": "table"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "kube_job_status_succeeded{namespace=\"foxhunt\"}",
|
||||
"legendFormat": "Succeeded",
|
||||
"instant": true,
|
||||
"format": "table"
|
||||
},
|
||||
{
|
||||
"refId": "C",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "kube_job_status_failed{namespace=\"foxhunt\"}",
|
||||
"legendFormat": "Failed",
|
||||
"instant": true,
|
||||
"format": "table"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 25,
|
||||
"type": "timeseries",
|
||||
"title": "GPU Node CPU Usage (Compute Pools)",
|
||||
"description": "CPU usage on GPU/compute node pools when no DCGM data is available",
|
||||
"gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"fillOpacity": 15,
|
||||
"gradientMode": "scheme",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"lineWidth": 1
|
||||
},
|
||||
"unit": "percentunit",
|
||||
"min": 0,
|
||||
"max": 1,
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
||||
"color": { "mode": "palette-classic" },
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[$interval]))",
|
||||
"legendFormat": "{{ instance }}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 26,
|
||||
"type": "timeseries",
|
||||
"title": "GPU Node Memory Usage (Compute Pools)",
|
||||
"gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"fillOpacity": 15,
|
||||
"gradientMode": "scheme",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"lineWidth": 1
|
||||
},
|
||||
"unit": "bytes",
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
||||
"color": { "mode": "palette-classic" },
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes",
|
||||
"legendFormat": "{{ instance }}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 30,
|
||||
"type": "row",
|
||||
"title": "Training & CI Logs",
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 35 },
|
||||
"collapsed": false,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"id": 31,
|
||||
"type": "logs",
|
||||
"title": "Training Job Logs",
|
||||
"gridPos": { "h": 10, "w": 12, "x": 0, "y": 36 },
|
||||
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
|
||||
"options": {
|
||||
"showTime": true,
|
||||
"showLabels": true,
|
||||
"showCommonLabels": false,
|
||||
"wrapLogMessage": true,
|
||||
"prettifyLogMessage": false,
|
||||
"enableLogDetails": true,
|
||||
"sortOrder": "Descending",
|
||||
"dedupStrategy": "none"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
|
||||
"expr": "{namespace=\"foxhunt\", container=~\".*train.*\"}",
|
||||
"maxLines": 200
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 32,
|
||||
"type": "logs",
|
||||
"title": "CI / GitLab Runner Logs",
|
||||
"gridPos": { "h": 10, "w": 12, "x": 12, "y": 36 },
|
||||
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
|
||||
"options": {
|
||||
"showTime": true,
|
||||
"showLabels": true,
|
||||
"showCommonLabels": false,
|
||||
"wrapLogMessage": true,
|
||||
"prettifyLogMessage": false,
|
||||
"enableLogDetails": true,
|
||||
"sortOrder": "Descending",
|
||||
"dedupStrategy": "none"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
|
||||
"expr": "{namespace=\"foxhunt\", app=~\".*runner.*|.*gitlab.*\"}",
|
||||
"maxLines": 200
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 40,
|
||||
"type": "row",
|
||||
"title": "Node Autoscale Events",
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 46 },
|
||||
"collapsed": false,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"id": 41,
|
||||
"type": "timeseries",
|
||||
"title": "Node Count Over Time",
|
||||
"description": "Tracks autoscaling of ci-compile and ci-training pools",
|
||||
"gridPos": { "h": 7, "w": 24, "x": 0, "y": 47 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "stepAfter",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"spanNulls": false,
|
||||
"showPoints": "auto",
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"lineWidth": 2
|
||||
},
|
||||
"unit": "short",
|
||||
"decimals": 0,
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
||||
"color": { "mode": "palette-classic" },
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["min", "max", "last"] }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "count by (label_k8s_scaleway_com_pool_name) (kube_node_labels)",
|
||||
"legendFormat": "{{ label_k8s_scaleway_com_pool_name }}"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
751
infra/k8s/monitoring/dashboards/foxhunt-infrastructure.json
Normal file
751
infra/k8s/monitoring/dashboards/foxhunt-infrastructure.json
Normal file
@@ -0,0 +1,751 @@
|
||||
{
|
||||
"id": null,
|
||||
"uid": "foxhunt-infrastructure",
|
||||
"title": "Foxhunt - K8s Infrastructure",
|
||||
"description": "Node health, per-node resources, PVC usage, scheduling, OOMKill events",
|
||||
"tags": ["foxhunt", "production"],
|
||||
"timezone": "browser",
|
||||
"schemaVersion": 39,
|
||||
"version": 1,
|
||||
"editable": true,
|
||||
"refresh": "30s",
|
||||
"time": {
|
||||
"from": "now-1h",
|
||||
"to": "now"
|
||||
},
|
||||
"fiscalYearStartMonth": 0,
|
||||
"liveNow": false,
|
||||
"weekStart": "",
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"name": "node",
|
||||
"type": "query",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"query": "label_values(kube_node_info, node)",
|
||||
"refresh": 2,
|
||||
"includeAll": true,
|
||||
"multi": true,
|
||||
"allValue": ".*",
|
||||
"current": { "selected": true, "text": "All", "value": "$__all" },
|
||||
"sort": 1
|
||||
},
|
||||
{
|
||||
"name": "interval",
|
||||
"type": "interval",
|
||||
"query": "1m,5m,15m,30m,1h",
|
||||
"current": { "selected": true, "text": "5m", "value": "5m" },
|
||||
"auto": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "row",
|
||||
"title": "Node Status",
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
|
||||
"collapsed": false,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "table",
|
||||
"title": "Node Status by Pool",
|
||||
"gridPos": { "h": 7, "w": 24, "x": 0, "y": 1 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
||||
"mappings": [
|
||||
{ "type": "value", "options": { "1": { "text": "Ready", "color": "green" }, "0": { "text": "NotReady", "color": "red" } } }
|
||||
],
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": { "id": "byName", "options": "Value" },
|
||||
"properties": [
|
||||
{ "id": "custom.displayMode", "value": "color-background-solid" },
|
||||
{
|
||||
"id": "thresholds",
|
||||
"value": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "red", "value": null },
|
||||
{ "color": "green", "value": 1 }
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"showHeader": true,
|
||||
"sortBy": [{ "displayName": "node", "desc": false }],
|
||||
"footer": { "show": false }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "kube_node_status_condition{condition=\"Ready\", status=\"true\", node=~\"$node\"}",
|
||||
"legendFormat": "{{ node }}",
|
||||
"instant": true,
|
||||
"format": "table"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"type": "stat",
|
||||
"title": "services Pool",
|
||||
"description": "DEV1-M, 2 fixed",
|
||||
"gridPos": { "h": 3, "w": 4, "x": 0, "y": 8 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "red", "value": null },
|
||||
{ "color": "yellow", "value": 1 },
|
||||
{ "color": "green", "value": 2 }
|
||||
]
|
||||
},
|
||||
"mappings": [],
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "value_and_name",
|
||||
"colorMode": "background",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "center"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "count(kube_node_labels{label_k8s_scaleway_com_pool_name=\"services\"} * on(node) kube_node_status_condition{condition=\"Ready\", status=\"true\"}) or vector(0)",
|
||||
"legendFormat": "services",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"type": "stat",
|
||||
"title": "gitlab Pool",
|
||||
"description": "GP1-XS, 1 fixed",
|
||||
"gridPos": { "h": 3, "w": 4, "x": 4, "y": 8 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "red", "value": null },
|
||||
{ "color": "green", "value": 1 }
|
||||
]
|
||||
},
|
||||
"mappings": [],
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "value_and_name",
|
||||
"colorMode": "background",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "center"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "count(kube_node_labels{label_k8s_scaleway_com_pool_name=\"gitlab\"} * on(node) kube_node_status_condition{condition=\"Ready\", status=\"true\"}) or vector(0)",
|
||||
"legendFormat": "gitlab",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"type": "stat",
|
||||
"title": "ci-compile-cpu Pool",
|
||||
"description": "POP2-32C-128G, 0-1 autoscale",
|
||||
"gridPos": { "h": 3, "w": 4, "x": 8, "y": 8 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "blue", "value": null },
|
||||
{ "color": "green", "value": 1 }
|
||||
]
|
||||
},
|
||||
"mappings": [],
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "value_and_name",
|
||||
"colorMode": "background",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "center"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "count(kube_node_labels{label_k8s_scaleway_com_pool_name=\"ci-compile-cpu\"}) or vector(0)",
|
||||
"legendFormat": "ci-compile-cpu",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"type": "stat",
|
||||
"title": "ci-compile Pool (GPU)",
|
||||
"description": "L4-1-24G, 0-1 autoscale",
|
||||
"gridPos": { "h": 3, "w": 4, "x": 12, "y": 8 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "blue", "value": null },
|
||||
{ "color": "green", "value": 1 }
|
||||
]
|
||||
},
|
||||
"mappings": [],
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "value_and_name",
|
||||
"colorMode": "background",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "center"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "count(kube_node_labels{label_k8s_scaleway_com_pool_name=\"ci-compile\"}) or vector(0)",
|
||||
"legendFormat": "ci-compile",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
"type": "stat",
|
||||
"title": "ci-training Pool (GPU)",
|
||||
"description": "L40S-1-48G, 0-1 autoscale",
|
||||
"gridPos": { "h": 3, "w": 4, "x": 16, "y": 8 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "blue", "value": null },
|
||||
{ "color": "green", "value": 1 }
|
||||
]
|
||||
},
|
||||
"mappings": [],
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "value_and_name",
|
||||
"colorMode": "background",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "center"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "count(kube_node_labels{label_k8s_scaleway_com_pool_name=\"ci-training\"}) or vector(0)",
|
||||
"legendFormat": "ci-training",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"type": "stat",
|
||||
"title": "Total Pods",
|
||||
"gridPos": { "h": 3, "w": 4, "x": 20, "y": 8 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "blue", "value": null }
|
||||
]
|
||||
},
|
||||
"mappings": [],
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "center"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "sum(kubelet_running_pods) or count(kube_pod_info)",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 15,
|
||||
"type": "row",
|
||||
"title": "Per-Node Resources",
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 11 },
|
||||
"collapsed": false,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"id": 16,
|
||||
"type": "timeseries",
|
||||
"title": "Node CPU Usage",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"fillOpacity": 15,
|
||||
"gradientMode": "scheme",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"lineWidth": 1
|
||||
},
|
||||
"unit": "percentunit",
|
||||
"min": 0,
|
||||
"max": 1,
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
||||
"color": { "mode": "palette-classic" },
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[$interval]))",
|
||||
"legendFormat": "{{ instance }}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 17,
|
||||
"type": "timeseries",
|
||||
"title": "Node Memory Usage",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"fillOpacity": 15,
|
||||
"gradientMode": "scheme",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"lineWidth": 1
|
||||
},
|
||||
"unit": "bytes",
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
||||
"color": { "mode": "palette-classic" },
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes",
|
||||
"legendFormat": "{{ instance }} used"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "node_memory_MemTotal_bytes",
|
||||
"legendFormat": "{{ instance }} total"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 18,
|
||||
"type": "timeseries",
|
||||
"title": "Node Filesystem Usage",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 20 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"fillOpacity": 15,
|
||||
"gradientMode": "scheme",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"lineWidth": 1
|
||||
},
|
||||
"unit": "percentunit",
|
||||
"min": 0,
|
||||
"max": 1,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "yellow", "value": 0.7 },
|
||||
{ "color": "red", "value": 0.9 }
|
||||
]
|
||||
},
|
||||
"color": { "mode": "palette-classic" },
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "1 - (node_filesystem_avail_bytes{mountpoint=\"/\", fstype!=\"tmpfs\"} / node_filesystem_size_bytes{mountpoint=\"/\", fstype!=\"tmpfs\"})",
|
||||
"legendFormat": "{{ instance }}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 19,
|
||||
"type": "bargauge",
|
||||
"title": "Pods per Node",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 20 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "yellow", "value": 80 },
|
||||
{ "color": "red", "value": 100 }
|
||||
]
|
||||
},
|
||||
"mappings": [],
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "horizontal",
|
||||
"displayMode": "gradient",
|
||||
"showUnfilled": true,
|
||||
"minVizWidth": 8,
|
||||
"minVizHeight": 16,
|
||||
"valueMode": "color"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "count by (node) (kube_pod_info{node=~\"$node\"})",
|
||||
"legendFormat": "{{ node }}",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 25,
|
||||
"type": "row",
|
||||
"title": "Network & PVC",
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 28 },
|
||||
"collapsed": false,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"id": 26,
|
||||
"type": "timeseries",
|
||||
"title": "Network I/O per Node",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 29 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"lineWidth": 1
|
||||
},
|
||||
"unit": "Bps",
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
||||
"color": { "mode": "palette-classic" },
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "sum by (instance) (rate(node_network_receive_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$interval]))",
|
||||
"legendFormat": "{{ instance }} RX"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "sum by (instance) (rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$interval]))",
|
||||
"legendFormat": "{{ instance }} TX"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 27,
|
||||
"type": "bargauge",
|
||||
"title": "PVC Usage",
|
||||
"description": "Persistent Volume Claim usage percentage. Requires kubelet_volume_stats metrics.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 29 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "yellow", "value": 70 },
|
||||
{ "color": "red", "value": 90 }
|
||||
]
|
||||
},
|
||||
"unit": "percent",
|
||||
"min": 0,
|
||||
"max": 100,
|
||||
"mappings": [],
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "horizontal",
|
||||
"displayMode": "gradient",
|
||||
"showUnfilled": true,
|
||||
"minVizWidth": 8,
|
||||
"minVizHeight": 16,
|
||||
"valueMode": "color"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "(1 - (kubelet_volume_stats_available_bytes{namespace=\"foxhunt\"} / kubelet_volume_stats_capacity_bytes{namespace=\"foxhunt\"})) * 100",
|
||||
"legendFormat": "{{ persistentvolumeclaim }}",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 30,
|
||||
"type": "row",
|
||||
"title": "Pod Scheduling & OOMKills",
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 37 },
|
||||
"collapsed": false,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"id": 31,
|
||||
"type": "timeseries",
|
||||
"title": "Scheduling Latency (p99)",
|
||||
"description": "Time to schedule pods. From scheduler_scheduling_algorithm_duration_seconds or scheduler_e2e_scheduling_duration_seconds.",
|
||||
"gridPos": { "h": 7, "w": 12, "x": 0, "y": 38 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"lineWidth": 2
|
||||
},
|
||||
"unit": "s",
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
||||
"color": { "mode": "palette-classic" },
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "histogram_quantile(0.99, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket[$interval])) by (le))",
|
||||
"legendFormat": "p99 e2e scheduling"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "histogram_quantile(0.50, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket[$interval])) by (le))",
|
||||
"legendFormat": "p50 e2e scheduling"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 32,
|
||||
"type": "timeseries",
|
||||
"title": "OOMKill Events",
|
||||
"description": "Container OOMKill events. Each spike indicates a container was killed due to memory pressure.",
|
||||
"gridPos": { "h": 7, "w": 12, "x": 12, "y": 38 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "bars",
|
||||
"lineInterpolation": "linear",
|
||||
"fillOpacity": 80,
|
||||
"gradientMode": "none",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"stacking": { "mode": "normal", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"lineWidth": 1
|
||||
},
|
||||
"unit": "short",
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
||||
"color": { "mode": "palette-classic" },
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["sum"] }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "increase(kube_pod_container_status_last_terminated_reason{reason=\"OOMKilled\", namespace=\"foxhunt\"}[30m]) > 0",
|
||||
"legendFormat": "{{ pod }} ({{ container }})"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 35,
|
||||
"type": "timeseries",
|
||||
"title": "Disk I/O per Node",
|
||||
"gridPos": { "h": 7, "w": 24, "x": 0, "y": 45 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"lineWidth": 1
|
||||
},
|
||||
"unit": "Bps",
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
||||
"color": { "mode": "palette-classic" },
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "sum by (instance) (rate(node_disk_read_bytes_total[$interval]))",
|
||||
"legendFormat": "{{ instance }} read"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "sum by (instance) (rate(node_disk_written_bytes_total[$interval]))",
|
||||
"legendFormat": "{{ instance }} write"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
419
infra/k8s/monitoring/dashboards/foxhunt-logs.json
Normal file
419
infra/k8s/monitoring/dashboards/foxhunt-logs.json
Normal file
@@ -0,0 +1,419 @@
|
||||
{
|
||||
"id": null,
|
||||
"uid": "foxhunt-logs",
|
||||
"title": "Foxhunt - Log Explorer",
|
||||
"description": "Log volume, error rates, top errors, live log stream, trace search",
|
||||
"tags": ["foxhunt", "production"],
|
||||
"timezone": "browser",
|
||||
"schemaVersion": 39,
|
||||
"version": 1,
|
||||
"editable": true,
|
||||
"refresh": "30s",
|
||||
"time": {
|
||||
"from": "now-1h",
|
||||
"to": "now"
|
||||
},
|
||||
"fiscalYearStartMonth": 0,
|
||||
"liveNow": false,
|
||||
"weekStart": "",
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"name": "service",
|
||||
"type": "custom",
|
||||
"query": "api-gateway,trading-service,ml-training-service,backtesting-service,trading-agent-service,broker-gateway,data-acquisition-service,web-gateway",
|
||||
"current": { "selected": false, "text": "All", "value": "$__all" },
|
||||
"multi": true,
|
||||
"includeAll": true,
|
||||
"allValue": ".*",
|
||||
"sort": 0
|
||||
},
|
||||
{
|
||||
"name": "search",
|
||||
"type": "textbox",
|
||||
"query": "",
|
||||
"current": { "selected": false, "text": "", "value": "" },
|
||||
"label": "Log Search"
|
||||
},
|
||||
{
|
||||
"name": "interval",
|
||||
"type": "interval",
|
||||
"query": "1m,5m,15m,30m,1h",
|
||||
"current": { "selected": true, "text": "5m", "value": "5m" },
|
||||
"auto": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "row",
|
||||
"title": "Log Volume Overview",
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
|
||||
"collapsed": false,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "timeseries",
|
||||
"title": "Log Volume by Service",
|
||||
"gridPos": { "h": 8, "w": 16, "x": 0, "y": 1 },
|
||||
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "bars",
|
||||
"lineInterpolation": "linear",
|
||||
"fillOpacity": 60,
|
||||
"gradientMode": "none",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"stacking": { "mode": "normal", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"lineWidth": 0
|
||||
},
|
||||
"unit": "short",
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
||||
"color": { "mode": "palette-classic" },
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["sum", "mean"] }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
|
||||
"expr": "sum by (app) (count_over_time({namespace=\"foxhunt\", app=~\"$service\"} [$interval]))",
|
||||
"legendFormat": "{{ app }}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "stat",
|
||||
"title": "Total Logs (1h)",
|
||||
"gridPos": { "h": 4, "w": 4, "x": 16, "y": 1 },
|
||||
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "blue", "value": null }
|
||||
]
|
||||
},
|
||||
"mappings": [],
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["sum"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "center"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
|
||||
"expr": "sum(count_over_time({namespace=\"foxhunt\", app=~\"$service\"} [1h]))",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"type": "stat",
|
||||
"title": "Errors (1h)",
|
||||
"gridPos": { "h": 4, "w": 4, "x": 20, "y": 1 },
|
||||
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "yellow", "value": 10 },
|
||||
{ "color": "red", "value": 50 }
|
||||
]
|
||||
},
|
||||
"mappings": [],
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["sum"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "center"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
|
||||
"expr": "sum(count_over_time({namespace=\"foxhunt\", app=~\"$service\"} |= \"ERROR\" [1h]))",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"type": "stat",
|
||||
"title": "Warnings (1h)",
|
||||
"gridPos": { "h": 4, "w": 4, "x": 16, "y": 5 },
|
||||
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "yellow", "value": 50 }
|
||||
]
|
||||
},
|
||||
"mappings": [],
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["sum"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "center"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
|
||||
"expr": "sum(count_over_time({namespace=\"foxhunt\", app=~\"$service\"} |= \"WARN\" [1h]))",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"type": "stat",
|
||||
"title": "Error Rate /min",
|
||||
"gridPos": { "h": 4, "w": 4, "x": 20, "y": 5 },
|
||||
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "yellow", "value": 1 },
|
||||
{ "color": "red", "value": 5 }
|
||||
]
|
||||
},
|
||||
"unit": "short",
|
||||
"decimals": 1,
|
||||
"mappings": [],
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "center"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
|
||||
"expr": "sum(count_over_time({namespace=\"foxhunt\", app=~\"$service\"} |= \"ERROR\" [1m]))",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"type": "row",
|
||||
"title": "Error Analysis",
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 9 },
|
||||
"collapsed": false,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"id": 11,
|
||||
"type": "timeseries",
|
||||
"title": "Error Rate by Service",
|
||||
"gridPos": { "h": 7, "w": 24, "x": 0, "y": 10 },
|
||||
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"fillOpacity": 20,
|
||||
"gradientMode": "scheme",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"lineWidth": 2
|
||||
},
|
||||
"unit": "short",
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
||||
"color": { "mode": "palette-classic" },
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "right", "calcs": ["sum", "max", "last"] }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
|
||||
"expr": "sum by (app) (count_over_time({namespace=\"foxhunt\", app=~\"$service\"} |= \"ERROR\" [$interval]))",
|
||||
"legendFormat": "{{ app }}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 12,
|
||||
"type": "table",
|
||||
"title": "Top Error Messages",
|
||||
"description": "Most frequent error messages across all services in the time range",
|
||||
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 17 },
|
||||
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
||||
"mappings": [],
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"showHeader": true,
|
||||
"sortBy": [{ "displayName": "Value", "desc": true }],
|
||||
"footer": { "show": false }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
|
||||
"expr": "topk(20, sum by (app, level) (count_over_time({namespace=\"foxhunt\", app=~\"$service\"} |= \"ERROR\" | logfmt [$interval])))",
|
||||
"legendFormat": "{{ app }}",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 20,
|
||||
"type": "row",
|
||||
"title": "Live Logs",
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 25 },
|
||||
"collapsed": false,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"id": 21,
|
||||
"type": "logs",
|
||||
"title": "Live Log Stream",
|
||||
"description": "Streaming logs from selected services. Use the $search variable to filter.",
|
||||
"gridPos": { "h": 14, "w": 24, "x": 0, "y": 26 },
|
||||
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
|
||||
"options": {
|
||||
"showTime": true,
|
||||
"showLabels": true,
|
||||
"showCommonLabels": false,
|
||||
"wrapLogMessage": true,
|
||||
"prettifyLogMessage": false,
|
||||
"enableLogDetails": true,
|
||||
"sortOrder": "Descending",
|
||||
"dedupStrategy": "none"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
|
||||
"expr": "{namespace=\"foxhunt\", app=~\"$service\"} |= \"$search\"",
|
||||
"maxLines": 500
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 30,
|
||||
"type": "row",
|
||||
"title": "Traces",
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 40 },
|
||||
"collapsed": false,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"id": 31,
|
||||
"type": "table",
|
||||
"title": "Recent Traces",
|
||||
"description": "Search for distributed traces via Tempo. Click a trace ID to view the full trace.",
|
||||
"gridPos": { "h": 10, "w": 24, "x": 0, "y": 41 },
|
||||
"datasource": { "type": "tempo", "uid": "tempo" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
||||
"mappings": [],
|
||||
"color": { "mode": "thresholds" },
|
||||
"links": [
|
||||
{
|
||||
"title": "View Trace",
|
||||
"url": "",
|
||||
"internal": {
|
||||
"datasourceUid": "tempo",
|
||||
"datasourceName": "Tempo",
|
||||
"query": {
|
||||
"queryType": "traceql",
|
||||
"query": "${__data.fields.traceID}"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": { "id": "byName", "options": "Duration" },
|
||||
"properties": [{ "id": "unit", "value": "ns" }]
|
||||
}
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"showHeader": true,
|
||||
"sortBy": [{ "displayName": "Start time", "desc": true }],
|
||||
"footer": { "show": false }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "tempo", "uid": "tempo" },
|
||||
"queryType": "nativeSearch",
|
||||
"serviceName": "",
|
||||
"spanName": "",
|
||||
"search": "",
|
||||
"minDuration": "",
|
||||
"maxDuration": "",
|
||||
"limit": 20
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
949
infra/k8s/monitoring/dashboards/foxhunt-overview.json
Normal file
949
infra/k8s/monitoring/dashboards/foxhunt-overview.json
Normal file
@@ -0,0 +1,949 @@
|
||||
{
|
||||
"id": null,
|
||||
"uid": "foxhunt-overview",
|
||||
"title": "Foxhunt - System Overview",
|
||||
"description": "Home dashboard: cluster health, service status, resource utilization, deployments, training jobs, error rates",
|
||||
"tags": ["foxhunt", "production"],
|
||||
"timezone": "browser",
|
||||
"schemaVersion": 39,
|
||||
"version": 1,
|
||||
"editable": true,
|
||||
"refresh": "30s",
|
||||
"time": {
|
||||
"from": "now-1h",
|
||||
"to": "now"
|
||||
},
|
||||
"fiscalYearStartMonth": 0,
|
||||
"liveNow": false,
|
||||
"weekStart": "",
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"name": "interval",
|
||||
"type": "interval",
|
||||
"query": "1m,5m,15m,30m,1h",
|
||||
"current": {
|
||||
"selected": true,
|
||||
"text": "5m",
|
||||
"value": "5m"
|
||||
},
|
||||
"auto": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "row",
|
||||
"title": "Cluster Health",
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
|
||||
"collapsed": false,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "stat",
|
||||
"title": "Nodes Ready",
|
||||
"gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "red", "value": null },
|
||||
{ "color": "yellow", "value": 2 },
|
||||
{ "color": "green", "value": 3 }
|
||||
]
|
||||
},
|
||||
"mappings": [],
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "auto"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "sum(kube_node_status_condition{condition=\"Ready\", status=\"true\"})",
|
||||
"legendFormat": "Ready",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "stat",
|
||||
"title": "Total Nodes",
|
||||
"gridPos": { "h": 4, "w": 4, "x": 4, "y": 1 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "blue", "value": null }
|
||||
]
|
||||
},
|
||||
"mappings": [],
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "auto"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "count(kube_node_info)",
|
||||
"legendFormat": "Nodes",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"type": "stat",
|
||||
"title": "Pods Running",
|
||||
"gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null }
|
||||
]
|
||||
},
|
||||
"mappings": [],
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "auto"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", phase=\"Running\"})",
|
||||
"legendFormat": "Running",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"type": "stat",
|
||||
"title": "Pods Failed",
|
||||
"gridPos": { "h": 4, "w": 4, "x": 12, "y": 1 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "red", "value": 1 }
|
||||
]
|
||||
},
|
||||
"mappings": [],
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "auto"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", phase=\"Failed\"}) or vector(0)",
|
||||
"legendFormat": "Failed",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"type": "stat",
|
||||
"title": "Active Training Jobs",
|
||||
"gridPos": { "h": 4, "w": 4, "x": 16, "y": 1 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "blue", "value": null },
|
||||
{ "color": "green", "value": 1 }
|
||||
]
|
||||
},
|
||||
"mappings": [],
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "auto"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "sum(kube_job_status_active{namespace=\"foxhunt\"}) or vector(0)",
|
||||
"legendFormat": "Active Jobs",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"type": "stat",
|
||||
"title": "Log Error Rate (/min)",
|
||||
"gridPos": { "h": 4, "w": 4, "x": 20, "y": 1 },
|
||||
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "yellow", "value": 5 },
|
||||
{ "color": "red", "value": 20 }
|
||||
]
|
||||
},
|
||||
"unit": "short",
|
||||
"decimals": 1,
|
||||
"mappings": [],
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
|
||||
"expr": "sum(count_over_time({namespace=\"foxhunt\"} |= \"ERROR\" [1m]))",
|
||||
"legendFormat": "Errors/min",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"type": "row",
|
||||
"title": "Service Status",
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 },
|
||||
"collapsed": false,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"id": 11,
|
||||
"type": "stat",
|
||||
"title": "api-gateway",
|
||||
"gridPos": { "h": 3, "w": 3, "x": 0, "y": 6 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "red", "value": null },
|
||||
{ "color": "green", "value": 1 }
|
||||
]
|
||||
},
|
||||
"mappings": [
|
||||
{ "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }
|
||||
],
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "center"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"api-gateway.*\", phase=\"Running\"}) > 0 or vector(0)",
|
||||
"legendFormat": "",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 12,
|
||||
"type": "stat",
|
||||
"title": "trading-service",
|
||||
"gridPos": { "h": 3, "w": 3, "x": 3, "y": 6 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "red", "value": null },
|
||||
{ "color": "green", "value": 1 }
|
||||
]
|
||||
},
|
||||
"mappings": [
|
||||
{ "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }
|
||||
],
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "center"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"trading-service.*\", phase=\"Running\"}) > 0 or vector(0)",
|
||||
"legendFormat": "",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 13,
|
||||
"type": "stat",
|
||||
"title": "ml-training-service",
|
||||
"gridPos": { "h": 3, "w": 3, "x": 6, "y": 6 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "red", "value": null },
|
||||
{ "color": "green", "value": 1 }
|
||||
]
|
||||
},
|
||||
"mappings": [
|
||||
{ "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }
|
||||
],
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "center"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"ml-training-service.*\", phase=\"Running\"}) > 0 or vector(0)",
|
||||
"legendFormat": "",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 14,
|
||||
"type": "stat",
|
||||
"title": "backtesting-service",
|
||||
"gridPos": { "h": 3, "w": 3, "x": 9, "y": 6 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "red", "value": null },
|
||||
{ "color": "green", "value": 1 }
|
||||
]
|
||||
},
|
||||
"mappings": [
|
||||
{ "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }
|
||||
],
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "center"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"backtesting-service.*\", phase=\"Running\"}) > 0 or vector(0)",
|
||||
"legendFormat": "",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 15,
|
||||
"type": "stat",
|
||||
"title": "trading-agent",
|
||||
"gridPos": { "h": 3, "w": 3, "x": 12, "y": 6 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "red", "value": null },
|
||||
{ "color": "green", "value": 1 }
|
||||
]
|
||||
},
|
||||
"mappings": [
|
||||
{ "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }
|
||||
],
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "center"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"trading-agent.*\", phase=\"Running\"}) > 0 or vector(0)",
|
||||
"legendFormat": "",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 16,
|
||||
"type": "stat",
|
||||
"title": "broker-gateway",
|
||||
"gridPos": { "h": 3, "w": 3, "x": 15, "y": 6 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "red", "value": null },
|
||||
{ "color": "green", "value": 1 }
|
||||
]
|
||||
},
|
||||
"mappings": [
|
||||
{ "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }
|
||||
],
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "center"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"broker-gateway.*\", phase=\"Running\"}) > 0 or vector(0)",
|
||||
"legendFormat": "",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 17,
|
||||
"type": "stat",
|
||||
"title": "data-acquisition",
|
||||
"gridPos": { "h": 3, "w": 3, "x": 18, "y": 6 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "red", "value": null },
|
||||
{ "color": "green", "value": 1 }
|
||||
]
|
||||
},
|
||||
"mappings": [
|
||||
{ "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }
|
||||
],
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "center"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"data-acquisition.*\", phase=\"Running\"}) > 0 or vector(0)",
|
||||
"legendFormat": "",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 18,
|
||||
"type": "stat",
|
||||
"title": "web-gateway",
|
||||
"gridPos": { "h": 3, "w": 3, "x": 21, "y": 6 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "red", "value": null },
|
||||
{ "color": "green", "value": 1 }
|
||||
]
|
||||
},
|
||||
"mappings": [
|
||||
{ "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }
|
||||
],
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "center"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"web-gateway.*\", phase=\"Running\"}) > 0 or vector(0)",
|
||||
"legendFormat": "",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 20,
|
||||
"type": "row",
|
||||
"title": "Resource Utilization",
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 9 },
|
||||
"collapsed": false,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"id": 21,
|
||||
"type": "gauge",
|
||||
"title": "Cluster CPU %",
|
||||
"gridPos": { "h": 6, "w": 6, "x": 0, "y": 10 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "yellow", "value": 70 },
|
||||
{ "color": "red", "value": 90 }
|
||||
]
|
||||
},
|
||||
"min": 0,
|
||||
"max": 100,
|
||||
"unit": "percent",
|
||||
"mappings": [],
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"showThresholdLabels": false,
|
||||
"showThresholdMarkers": true,
|
||||
"orientation": "auto"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "100 - (avg(rate(node_cpu_seconds_total{mode=\"idle\"}[$interval])) * 100)",
|
||||
"legendFormat": "CPU",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 22,
|
||||
"type": "gauge",
|
||||
"title": "Cluster Memory %",
|
||||
"gridPos": { "h": 6, "w": 6, "x": 6, "y": 10 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "yellow", "value": 70 },
|
||||
{ "color": "red", "value": 90 }
|
||||
]
|
||||
},
|
||||
"min": 0,
|
||||
"max": 100,
|
||||
"unit": "percent",
|
||||
"mappings": [],
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"showThresholdLabels": false,
|
||||
"showThresholdMarkers": true,
|
||||
"orientation": "auto"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "(1 - (sum(node_memory_MemAvailable_bytes) / sum(node_memory_MemTotal_bytes))) * 100",
|
||||
"legendFormat": "Memory",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 23,
|
||||
"type": "timeseries",
|
||||
"title": "CPU by Node Pool",
|
||||
"gridPos": { "h": 6, "w": 12, "x": 12, "y": 10 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"fillOpacity": 15,
|
||||
"gradientMode": "scheme",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"lineWidth": 1
|
||||
},
|
||||
"unit": "percentunit",
|
||||
"min": 0,
|
||||
"max": 1,
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
||||
"color": { "mode": "palette-classic" },
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[$interval]))",
|
||||
"legendFormat": "{{ instance }}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 24,
|
||||
"type": "timeseries",
|
||||
"title": "Memory by Node Pool",
|
||||
"gridPos": { "h": 6, "w": 12, "x": 0, "y": 16 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"fillOpacity": 15,
|
||||
"gradientMode": "scheme",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"lineWidth": 1
|
||||
},
|
||||
"unit": "bytes",
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
||||
"color": { "mode": "palette-classic" },
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes",
|
||||
"legendFormat": "{{ instance }}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 25,
|
||||
"type": "timeseries",
|
||||
"title": "Foxhunt Container CPU Usage",
|
||||
"gridPos": { "h": 6, "w": 12, "x": 12, "y": 16 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"fillOpacity": 15,
|
||||
"gradientMode": "none",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"stacking": { "mode": "normal", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"lineWidth": 1
|
||||
},
|
||||
"unit": "short",
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
||||
"color": { "mode": "palette-classic" },
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "sum by (container) (rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", container=~\"api-gateway|trading-service|ml-training-service|backtesting-service|trading-agent-service|broker-gateway|data-acquisition-service|web-gateway\", image!=\"\"}[$interval]))",
|
||||
"legendFormat": "{{ container }}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 30,
|
||||
"type": "row",
|
||||
"title": "Deployments & Restarts",
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 22 },
|
||||
"collapsed": false,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"id": 31,
|
||||
"type": "timeseries",
|
||||
"title": "Pod Restarts (last 1h)",
|
||||
"description": "Pod restart events indicating crashes or OOMKills",
|
||||
"gridPos": { "h": 7, "w": 12, "x": 0, "y": 23 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "bars",
|
||||
"lineInterpolation": "linear",
|
||||
"fillOpacity": 80,
|
||||
"gradientMode": "none",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"stacking": { "mode": "normal", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"lineWidth": 1
|
||||
},
|
||||
"unit": "short",
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
||||
"color": { "mode": "palette-classic" },
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "increase(kube_pod_container_status_restarts_total{namespace=\"foxhunt\"}[1h]) > 0",
|
||||
"legendFormat": "{{ pod }}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 32,
|
||||
"type": "timeseries",
|
||||
"title": "Deployment Replicas: Desired vs Available",
|
||||
"gridPos": { "h": 7, "w": 12, "x": 12, "y": 23 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "stepAfter",
|
||||
"fillOpacity": 0,
|
||||
"gradientMode": "none",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"lineWidth": 2
|
||||
},
|
||||
"unit": "short",
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
||||
"color": { "mode": "palette-classic" },
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["last"] }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "kube_deployment_spec_replicas{namespace=\"foxhunt\"}",
|
||||
"legendFormat": "{{ deployment }} desired"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "kube_deployment_status_replicas_available{namespace=\"foxhunt\"}",
|
||||
"legendFormat": "{{ deployment }} available"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 40,
|
||||
"type": "row",
|
||||
"title": "Error Rate Trend",
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 30 },
|
||||
"collapsed": false,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"id": 41,
|
||||
"type": "timeseries",
|
||||
"title": "Log Errors Over Time",
|
||||
"gridPos": { "h": 7, "w": 24, "x": 0, "y": 31 },
|
||||
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "bars",
|
||||
"lineInterpolation": "linear",
|
||||
"fillOpacity": 50,
|
||||
"gradientMode": "scheme",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"stacking": { "mode": "normal", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"lineWidth": 1
|
||||
},
|
||||
"unit": "short",
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
||||
"color": { "mode": "palette-classic" },
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
|
||||
"expr": "sum by (app) (count_over_time({namespace=\"foxhunt\"} |= \"ERROR\" [$interval]))",
|
||||
"legendFormat": "{{ app }}"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
675
infra/k8s/monitoring/dashboards/foxhunt-services.json
Normal file
675
infra/k8s/monitoring/dashboards/foxhunt-services.json
Normal file
@@ -0,0 +1,675 @@
|
||||
{
|
||||
"id": null,
|
||||
"uid": "foxhunt-services",
|
||||
"title": "Foxhunt - Microservice Detail",
|
||||
"description": "Per-service deep dive: CPU, memory, restarts, gRPC metrics, logs",
|
||||
"tags": ["foxhunt", "production"],
|
||||
"timezone": "browser",
|
||||
"schemaVersion": 39,
|
||||
"version": 1,
|
||||
"editable": true,
|
||||
"refresh": "30s",
|
||||
"time": {
|
||||
"from": "now-1h",
|
||||
"to": "now"
|
||||
},
|
||||
"fiscalYearStartMonth": 0,
|
||||
"liveNow": false,
|
||||
"weekStart": "",
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"name": "service",
|
||||
"type": "custom",
|
||||
"query": "api-gateway,trading-service,ml-training-service,backtesting-service,trading-agent-service,broker-gateway,data-acquisition-service,web-gateway",
|
||||
"current": {
|
||||
"selected": true,
|
||||
"text": "api-gateway",
|
||||
"value": "api-gateway"
|
||||
},
|
||||
"multi": false,
|
||||
"includeAll": false,
|
||||
"sort": 0
|
||||
},
|
||||
{
|
||||
"name": "interval",
|
||||
"type": "interval",
|
||||
"query": "1m,5m,15m,30m,1h",
|
||||
"current": {
|
||||
"selected": true,
|
||||
"text": "5m",
|
||||
"value": "5m"
|
||||
},
|
||||
"auto": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "row",
|
||||
"title": "Pod Status - $service",
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
|
||||
"collapsed": false,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "stat",
|
||||
"title": "Status",
|
||||
"gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "red", "value": null },
|
||||
{ "color": "green", "value": 1 }
|
||||
]
|
||||
},
|
||||
"mappings": [
|
||||
{ "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }
|
||||
],
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "center"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"$service.*\", phase=\"Running\"}) > 0 or vector(0)",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "stat",
|
||||
"title": "Replicas",
|
||||
"gridPos": { "h": 4, "w": 4, "x": 4, "y": 1 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "blue", "value": null }
|
||||
]
|
||||
},
|
||||
"mappings": [],
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "value",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "auto"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "count(kube_pod_info{namespace=\"foxhunt\", pod=~\"$service.*\"})",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"type": "stat",
|
||||
"title": "Restarts (24h)",
|
||||
"gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "yellow", "value": 1 },
|
||||
{ "color": "red", "value": 5 }
|
||||
]
|
||||
},
|
||||
"mappings": [],
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "background",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "sum(increase(kube_pod_container_status_restarts_total{namespace=\"foxhunt\", pod=~\"$service.*\"}[24h])) or vector(0)",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"type": "stat",
|
||||
"title": "Container Uptime",
|
||||
"gridPos": { "h": 4, "w": 4, "x": 12, "y": 1 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "red", "value": null },
|
||||
{ "color": "yellow", "value": 3600 },
|
||||
{ "color": "green", "value": 86400 }
|
||||
]
|
||||
},
|
||||
"unit": "s",
|
||||
"mappings": [],
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "value",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "auto"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "min(time() - kube_pod_start_time{namespace=\"foxhunt\", pod=~\"$service.*\"})",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"type": "stat",
|
||||
"title": "CPU Cores",
|
||||
"gridPos": { "h": 4, "w": 4, "x": 16, "y": 1 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "blue", "value": null }
|
||||
]
|
||||
},
|
||||
"unit": "short",
|
||||
"decimals": 3,
|
||||
"mappings": [],
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", container=~\"$service\", image!=\"\"}[$interval]))",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"type": "stat",
|
||||
"title": "Memory",
|
||||
"gridPos": { "h": 4, "w": 4, "x": 20, "y": 1 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "blue", "value": null }
|
||||
]
|
||||
},
|
||||
"unit": "bytes",
|
||||
"mappings": [],
|
||||
"color": { "mode": "thresholds" }
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
|
||||
"orientation": "auto",
|
||||
"textMode": "auto",
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", container=~\"$service\", image!=\"\"})",
|
||||
"instant": true
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"type": "row",
|
||||
"title": "Resource Usage Over Time",
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 },
|
||||
"collapsed": false,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"id": 11,
|
||||
"type": "timeseries",
|
||||
"title": "CPU Usage",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"fillOpacity": 20,
|
||||
"gradientMode": "scheme",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"lineWidth": 2
|
||||
},
|
||||
"unit": "short",
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
||||
"color": { "mode": "palette-classic" },
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", container=~\"$service\", image!=\"\"}[$interval]))",
|
||||
"legendFormat": "{{ pod }}"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "sum(kube_pod_container_resource_requests{namespace=\"foxhunt\", container=~\"$service\", resource=\"cpu\"})",
|
||||
"legendFormat": "Request"
|
||||
},
|
||||
{
|
||||
"refId": "C",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "sum(kube_pod_container_resource_limits{namespace=\"foxhunt\", container=~\"$service\", resource=\"cpu\"})",
|
||||
"legendFormat": "Limit"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 12,
|
||||
"type": "timeseries",
|
||||
"title": "Memory Usage",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"fillOpacity": 20,
|
||||
"gradientMode": "scheme",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"lineWidth": 2
|
||||
},
|
||||
"unit": "bytes",
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
||||
"color": { "mode": "palette-classic" },
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "sum by (pod) (container_memory_working_set_bytes{namespace=\"foxhunt\", container=~\"$service\", image!=\"\"})",
|
||||
"legendFormat": "{{ pod }}"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "sum(kube_pod_container_resource_requests{namespace=\"foxhunt\", container=~\"$service\", resource=\"memory\"})",
|
||||
"legendFormat": "Request"
|
||||
},
|
||||
{
|
||||
"refId": "C",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "sum(kube_pod_container_resource_limits{namespace=\"foxhunt\", container=~\"$service\", resource=\"memory\"})",
|
||||
"legendFormat": "Limit"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 15,
|
||||
"type": "row",
|
||||
"title": "Restart History",
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 },
|
||||
"collapsed": false,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"id": 16,
|
||||
"type": "timeseries",
|
||||
"title": "Pod Restart Count",
|
||||
"gridPos": { "h": 6, "w": 12, "x": 0, "y": 15 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "bars",
|
||||
"lineInterpolation": "linear",
|
||||
"fillOpacity": 80,
|
||||
"gradientMode": "none",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"stacking": { "mode": "normal", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"lineWidth": 1
|
||||
},
|
||||
"unit": "short",
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
||||
"color": { "mode": "palette-classic" },
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "increase(kube_pod_container_status_restarts_total{namespace=\"foxhunt\", pod=~\"$service.*\"}[30m])",
|
||||
"legendFormat": "{{ pod }}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 17,
|
||||
"type": "timeseries",
|
||||
"title": "Network I/O",
|
||||
"gridPos": { "h": 6, "w": 12, "x": 12, "y": 15 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"lineWidth": 1
|
||||
},
|
||||
"unit": "Bps",
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
||||
"color": { "mode": "palette-classic" },
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "sum by (pod) (rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"$service.*\"}[$interval]))",
|
||||
"legendFormat": "{{ pod }} RX"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "sum by (pod) (rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"$service.*\"}[$interval]))",
|
||||
"legendFormat": "{{ pod }} TX"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 20,
|
||||
"type": "row",
|
||||
"title": "gRPC Metrics",
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 21 },
|
||||
"collapsed": false,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"id": 21,
|
||||
"type": "timeseries",
|
||||
"title": "gRPC Request Rate",
|
||||
"description": "gRPC requests per second by method and status code. Requires grpc_server_handled_total metric from service instrumentation.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 22 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"fillOpacity": 15,
|
||||
"gradientMode": "scheme",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"lineWidth": 1
|
||||
},
|
||||
"unit": "reqps",
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
||||
"color": { "mode": "palette-classic" },
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "sum by (grpc_method, grpc_code) (rate(grpc_server_handled_total{grpc_service=~\".*$service.*\"}[$interval]))",
|
||||
"legendFormat": "{{ grpc_method }} [{{ grpc_code }}]"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 22,
|
||||
"type": "timeseries",
|
||||
"title": "gRPC Latency (p50 / p95 / p99)",
|
||||
"description": "gRPC server-side latency percentiles. Requires grpc_server_handling_seconds_bucket histogram.",
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 22 },
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"lineInterpolation": "smooth",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"stacking": { "mode": "none", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"lineWidth": 2
|
||||
},
|
||||
"unit": "s",
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
||||
"color": { "mode": "palette-classic" },
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "histogram_quantile(0.50, sum(rate(grpc_server_handling_seconds_bucket{grpc_service=~\".*$service.*\"}[$interval])) by (le))",
|
||||
"legendFormat": "p50"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "histogram_quantile(0.95, sum(rate(grpc_server_handling_seconds_bucket{grpc_service=~\".*$service.*\"}[$interval])) by (le))",
|
||||
"legendFormat": "p95"
|
||||
},
|
||||
{
|
||||
"refId": "C",
|
||||
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
|
||||
"expr": "histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{grpc_service=~\".*$service.*\"}[$interval])) by (le))",
|
||||
"legendFormat": "p99"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 30,
|
||||
"type": "row",
|
||||
"title": "Logs",
|
||||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 30 },
|
||||
"collapsed": false,
|
||||
"panels": []
|
||||
},
|
||||
{
|
||||
"id": 31,
|
||||
"type": "timeseries",
|
||||
"title": "Log Volume by Level",
|
||||
"gridPos": { "h": 6, "w": 24, "x": 0, "y": 31 },
|
||||
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "bars",
|
||||
"lineInterpolation": "linear",
|
||||
"fillOpacity": 60,
|
||||
"gradientMode": "none",
|
||||
"spanNulls": false,
|
||||
"showPoints": "never",
|
||||
"stacking": { "mode": "normal", "group": "A" },
|
||||
"axisPlacement": "auto",
|
||||
"lineWidth": 1
|
||||
},
|
||||
"unit": "short",
|
||||
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
|
||||
"color": { "mode": "palette-classic" },
|
||||
"mappings": []
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": { "id": "byName", "options": "ERROR" },
|
||||
"properties": [{ "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } }]
|
||||
},
|
||||
{
|
||||
"matcher": { "id": "byName", "options": "WARN" },
|
||||
"properties": [{ "id": "color", "value": { "fixedColor": "yellow", "mode": "fixed" } }]
|
||||
},
|
||||
{
|
||||
"matcher": { "id": "byName", "options": "INFO" },
|
||||
"properties": [{ "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } }]
|
||||
},
|
||||
{
|
||||
"matcher": { "id": "byName", "options": "DEBUG" },
|
||||
"properties": [{ "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } }]
|
||||
}
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"tooltip": { "mode": "multi", "sort": "desc" },
|
||||
"legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
|
||||
"expr": "sum by (level) (count_over_time({app=\"$service\", namespace=\"foxhunt\"} | logfmt | level != \"\" [$interval]))",
|
||||
"legendFormat": "{{ level }}"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 32,
|
||||
"type": "logs",
|
||||
"title": "Recent Error Logs",
|
||||
"gridPos": { "h": 10, "w": 24, "x": 0, "y": 37 },
|
||||
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
|
||||
"options": {
|
||||
"showTime": true,
|
||||
"showLabels": true,
|
||||
"showCommonLabels": false,
|
||||
"wrapLogMessage": true,
|
||||
"prettifyLogMessage": false,
|
||||
"enableLogDetails": true,
|
||||
"sortOrder": "Descending",
|
||||
"dedupStrategy": "none"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
|
||||
"expr": "{app=\"$service\", namespace=\"foxhunt\"} |= \"ERROR\"",
|
||||
"maxLines": 200
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -4,19 +4,16 @@ set -euo pipefail
|
||||
# Import Grafana dashboards to Scaleway Cockpit via the Grafana API.
|
||||
#
|
||||
# Required environment variables:
|
||||
# GRAFANA_URL - Grafana base URL (e.g. https://xxxx.grafana.scw.cloud)
|
||||
# GRAFANA_PASS - Password for foxhunt-admin user
|
||||
# GRAFANA_URL - Grafana base URL (default: https://grafana.fxhnt.ai)
|
||||
# GRAFANA_PASS - Password for admin user
|
||||
#
|
||||
# Usage:
|
||||
# GRAFANA_URL="https://..." GRAFANA_PASS="..." ./import.sh
|
||||
# GRAFANA_PASS="Welcome01" ./import.sh
|
||||
# GRAFANA_URL="https://grafana.fxhnt.ai" GRAFANA_PASS="Welcome01" ./import.sh
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
GRAFANA_USER="${GRAFANA_USER:-foxhunt-admin}"
|
||||
|
||||
if [[ -z "${GRAFANA_URL:-}" ]]; then
|
||||
echo "ERROR: GRAFANA_URL is not set" >&2
|
||||
exit 1
|
||||
fi
|
||||
GRAFANA_USER="${GRAFANA_USER:-admin}"
|
||||
GRAFANA_URL="${GRAFANA_URL:-https://grafana.fxhnt.ai}"
|
||||
|
||||
if [[ -z "${GRAFANA_PASS:-}" ]]; then
|
||||
echo "ERROR: GRAFANA_PASS is not set" >&2
|
||||
@@ -57,7 +54,12 @@ failed=0
|
||||
|
||||
for dashboard in "${SCRIPT_DIR}/cluster-overview.json" \
|
||||
"${SCRIPT_DIR}/gpu-overview.json" \
|
||||
"${SCRIPT_DIR}/gitlab-services.json"; do
|
||||
"${SCRIPT_DIR}/gitlab-services.json" \
|
||||
"${SCRIPT_DIR}/foxhunt-overview.json" \
|
||||
"${SCRIPT_DIR}/foxhunt-services.json" \
|
||||
"${SCRIPT_DIR}/foxhunt-gpu-training.json" \
|
||||
"${SCRIPT_DIR}/foxhunt-infrastructure.json" \
|
||||
"${SCRIPT_DIR}/foxhunt-logs.json"; do
|
||||
if [[ -f "$dashboard" ]]; then
|
||||
if ! import_dashboard "$dashboard"; then
|
||||
failed=$((failed + 1))
|
||||
|
||||
164
infra/k8s/monitoring/kube-state-metrics.yaml
Normal file
164
infra/k8s/monitoring/kube-state-metrics.yaml
Normal file
@@ -0,0 +1,164 @@
|
||||
# kube-state-metrics — Kubernetes object state metrics
|
||||
# Replaces k8s-monitoring chart's kube-state-metrics after Cockpit removal
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: kube-state-metrics
|
||||
namespace: foxhunt
|
||||
labels:
|
||||
app.kubernetes.io/name: kube-state-metrics
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: kube-state-metrics
|
||||
labels:
|
||||
app.kubernetes.io/name: kube-state-metrics
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources:
|
||||
- configmaps
|
||||
- secrets
|
||||
- nodes
|
||||
- pods
|
||||
- services
|
||||
- serviceaccounts
|
||||
- resourcequotas
|
||||
- replicationcontrollers
|
||||
- limitranges
|
||||
- persistentvolumeclaims
|
||||
- persistentvolumes
|
||||
- namespaces
|
||||
- endpoints
|
||||
verbs: ["list", "watch"]
|
||||
- apiGroups: ["apps"]
|
||||
resources:
|
||||
- statefulsets
|
||||
- daemonsets
|
||||
- deployments
|
||||
- replicasets
|
||||
verbs: ["list", "watch"]
|
||||
- apiGroups: ["batch"]
|
||||
resources:
|
||||
- cronjobs
|
||||
- jobs
|
||||
verbs: ["list", "watch"]
|
||||
- apiGroups: ["autoscaling"]
|
||||
resources:
|
||||
- horizontalpodautoscalers
|
||||
verbs: ["list", "watch"]
|
||||
- apiGroups: ["networking.k8s.io"]
|
||||
resources:
|
||||
- networkpolicies
|
||||
- ingresses
|
||||
verbs: ["list", "watch"]
|
||||
- apiGroups: ["policy"]
|
||||
resources:
|
||||
- poddisruptionbudgets
|
||||
verbs: ["list", "watch"]
|
||||
- apiGroups: ["storage.k8s.io"]
|
||||
resources:
|
||||
- storageclasses
|
||||
- volumeattachments
|
||||
verbs: ["list", "watch"]
|
||||
- apiGroups: ["coordination.k8s.io"]
|
||||
resources:
|
||||
- leases
|
||||
verbs: ["list", "watch"]
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: kube-state-metrics
|
||||
labels:
|
||||
app.kubernetes.io/name: kube-state-metrics
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: kube-state-metrics
|
||||
namespace: foxhunt
|
||||
roleRef:
|
||||
kind: ClusterRole
|
||||
name: kube-state-metrics
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: kube-state-metrics
|
||||
namespace: foxhunt
|
||||
labels:
|
||||
app.kubernetes.io/name: kube-state-metrics
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: kube-state-metrics
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: kube-state-metrics
|
||||
annotations:
|
||||
prometheus.io/scrape: "true"
|
||||
prometheus.io/port: "8080"
|
||||
spec:
|
||||
serviceAccountName: kube-state-metrics
|
||||
nodeSelector:
|
||||
k8s.scaleway.com/pool-name: gitlab
|
||||
tolerations:
|
||||
- key: gitlab
|
||||
operator: Equal
|
||||
value: "true"
|
||||
effect: NoSchedule
|
||||
containers:
|
||||
- name: kube-state-metrics
|
||||
image: registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.14.0
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
name: http
|
||||
- containerPort: 8081
|
||||
name: telemetry
|
||||
resources:
|
||||
requests:
|
||||
cpu: 25m
|
||||
memory: 64Mi
|
||||
limits:
|
||||
cpu: 100m
|
||||
memory: 128Mi
|
||||
securityContext:
|
||||
readOnlyRootFilesystem: true
|
||||
allowPrivilegeEscalation: false
|
||||
runAsNonRoot: true
|
||||
runAsUser: 65534
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /healthz
|
||||
port: 8080
|
||||
initialDelaySeconds: 5
|
||||
timeoutSeconds: 5
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /
|
||||
port: 8080
|
||||
initialDelaySeconds: 5
|
||||
timeoutSeconds: 5
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: kube-state-metrics
|
||||
namespace: foxhunt
|
||||
labels:
|
||||
app.kubernetes.io/name: kube-state-metrics
|
||||
annotations:
|
||||
prometheus.io/scrape: "true"
|
||||
prometheus.io/port: "8080"
|
||||
spec:
|
||||
selector:
|
||||
app.kubernetes.io/name: kube-state-metrics
|
||||
ports:
|
||||
- port: 8080
|
||||
targetPort: 8080
|
||||
name: http
|
||||
- port: 8081
|
||||
targetPort: 8081
|
||||
name: telemetry
|
||||
104
infra/k8s/monitoring/node-exporter.yaml
Normal file
104
infra/k8s/monitoring/node-exporter.yaml
Normal file
@@ -0,0 +1,104 @@
|
||||
# Prometheus Node Exporter — host-level metrics (CPU, memory, disk, network)
|
||||
# Replaces k8s-monitoring chart's node-exporter after Cockpit removal
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: node-exporter
|
||||
namespace: foxhunt
|
||||
labels:
|
||||
app.kubernetes.io/name: node-exporter
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: DaemonSet
|
||||
metadata:
|
||||
name: node-exporter
|
||||
namespace: foxhunt
|
||||
labels:
|
||||
app.kubernetes.io/name: node-exporter
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: node-exporter
|
||||
updateStrategy:
|
||||
type: RollingUpdate
|
||||
rollingUpdate:
|
||||
maxUnavailable: 1
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: node-exporter
|
||||
annotations:
|
||||
prometheus.io/scrape: "true"
|
||||
prometheus.io/port: "9100"
|
||||
spec:
|
||||
serviceAccountName: node-exporter
|
||||
hostPID: true
|
||||
hostNetwork: true
|
||||
tolerations:
|
||||
- effect: NoSchedule
|
||||
operator: Exists
|
||||
containers:
|
||||
- name: node-exporter
|
||||
image: quay.io/prometheus/node-exporter:v1.9.0
|
||||
args:
|
||||
- --path.procfs=/host/proc
|
||||
- --path.sysfs=/host/sys
|
||||
- --path.rootfs=/host/root
|
||||
- --collector.filesystem.mount-points-exclude=^/(dev|proc|sys|var/lib/docker/.+|var/lib/kubelet/.+)($|/)
|
||||
- --collector.filesystem.fs-types-exclude=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$
|
||||
- --web.listen-address=:9100
|
||||
ports:
|
||||
- containerPort: 9100
|
||||
name: metrics
|
||||
hostPort: 9100
|
||||
resources:
|
||||
requests:
|
||||
cpu: 25m
|
||||
memory: 32Mi
|
||||
limits:
|
||||
cpu: 100m
|
||||
memory: 64Mi
|
||||
securityContext:
|
||||
readOnlyRootFilesystem: true
|
||||
allowPrivilegeEscalation: false
|
||||
volumeMounts:
|
||||
- name: proc
|
||||
mountPath: /host/proc
|
||||
readOnly: true
|
||||
- name: sys
|
||||
mountPath: /host/sys
|
||||
readOnly: true
|
||||
- name: root
|
||||
mountPath: /host/root
|
||||
readOnly: true
|
||||
mountPropagation: HostToContainer
|
||||
volumes:
|
||||
- name: proc
|
||||
hostPath:
|
||||
path: /proc
|
||||
- name: sys
|
||||
hostPath:
|
||||
path: /sys
|
||||
- name: root
|
||||
hostPath:
|
||||
path: /
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: node-exporter
|
||||
namespace: foxhunt
|
||||
labels:
|
||||
app.kubernetes.io/name: node-exporter
|
||||
annotations:
|
||||
prometheus.io/scrape: "true"
|
||||
prometheus.io/port: "9100"
|
||||
spec:
|
||||
type: ClusterIP
|
||||
clusterIP: None
|
||||
selector:
|
||||
app.kubernetes.io/name: node-exporter
|
||||
ports:
|
||||
- port: 9100
|
||||
targetPort: 9100
|
||||
name: metrics
|
||||
166
infra/k8s/monitoring/prometheus-values.yaml
Normal file
166
infra/k8s/monitoring/prometheus-values.yaml
Normal file
@@ -0,0 +1,166 @@
|
||||
# Prometheus override values for GitLab Helm chart
|
||||
# Removes Cockpit remote_write, adds comprehensive cluster scrape configs
|
||||
#
|
||||
# Apply with:
|
||||
# helm upgrade gitlab gitlab/gitlab -n foxhunt --reuse-values \
|
||||
# -f infra/k8s/monitoring/prometheus-values.yaml
|
||||
prometheus:
|
||||
server:
|
||||
# Remove Cockpit remote_write — all metrics stay local
|
||||
remoteWrite: []
|
||||
|
||||
# Extra scrape configs for full cluster observability
|
||||
# NOTE: this is at prometheus.extraScrapeConfigs (chart root), NOT prometheus.server.extraScrapeConfigs
|
||||
extraScrapeConfigs: |
|
||||
# --- Host metrics from node-exporter ---
|
||||
- job_name: node-exporter
|
||||
kubernetes_sd_configs:
|
||||
- role: endpoints
|
||||
namespaces:
|
||||
names: [foxhunt]
|
||||
relabel_configs:
|
||||
- source_labels: [__meta_kubernetes_service_label_app_kubernetes_io_name]
|
||||
action: keep
|
||||
regex: node-exporter
|
||||
- source_labels: [__meta_kubernetes_endpoints_name]
|
||||
action: keep
|
||||
regex: node-exporter
|
||||
- source_labels: [__meta_kubernetes_pod_node_name]
|
||||
target_label: node
|
||||
|
||||
# --- Kubernetes object state ---
|
||||
- job_name: kube-state-metrics
|
||||
kubernetes_sd_configs:
|
||||
- role: endpoints
|
||||
namespaces:
|
||||
names: [foxhunt]
|
||||
relabel_configs:
|
||||
- source_labels: [__meta_kubernetes_service_label_app_kubernetes_io_name]
|
||||
action: keep
|
||||
regex: kube-state-metrics
|
||||
- source_labels: [__meta_kubernetes_endpoint_port_name]
|
||||
action: keep
|
||||
regex: http
|
||||
|
||||
# --- Kubelet metrics (container resource usage) ---
|
||||
- job_name: kubelet
|
||||
scheme: https
|
||||
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
|
||||
tls_config:
|
||||
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
|
||||
insecure_skip_verify: true
|
||||
kubernetes_sd_configs:
|
||||
- role: node
|
||||
relabel_configs:
|
||||
- target_label: __address__
|
||||
replacement: kubernetes.default.svc:443
|
||||
- source_labels: [__meta_kubernetes_node_name]
|
||||
regex: (.+)
|
||||
target_label: __metrics_path__
|
||||
replacement: /api/v1/nodes/$1/proxy/metrics
|
||||
|
||||
# --- cAdvisor (container CPU, memory, network, filesystem) ---
|
||||
- job_name: cadvisor
|
||||
scheme: https
|
||||
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
|
||||
tls_config:
|
||||
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
|
||||
insecure_skip_verify: true
|
||||
kubernetes_sd_configs:
|
||||
- role: node
|
||||
relabel_configs:
|
||||
- target_label: __address__
|
||||
replacement: kubernetes.default.svc:443
|
||||
- source_labels: [__meta_kubernetes_node_name]
|
||||
regex: (.+)
|
||||
target_label: __metrics_path__
|
||||
replacement: /api/v1/nodes/$1/proxy/metrics/cadvisor
|
||||
metric_relabel_configs:
|
||||
- source_labels: [namespace]
|
||||
action: keep
|
||||
regex: (foxhunt|monitoring|kube-system)
|
||||
|
||||
# --- GPU metrics from NVIDIA DCGM Exporter ---
|
||||
- job_name: dcgm-exporter
|
||||
kubernetes_sd_configs:
|
||||
- role: endpoints
|
||||
namespaces:
|
||||
names: [kube-system, monitoring]
|
||||
relabel_configs:
|
||||
- source_labels: [__meta_kubernetes_service_name]
|
||||
action: keep
|
||||
regex: .*dcgm.*
|
||||
- source_labels: [__meta_kubernetes_pod_node_name]
|
||||
target_label: node
|
||||
|
||||
# --- Foxhunt services (app metrics ports) ---
|
||||
- job_name: foxhunt-services
|
||||
kubernetes_sd_configs:
|
||||
- role: endpoints
|
||||
namespaces:
|
||||
names: [foxhunt]
|
||||
relabel_configs:
|
||||
- source_labels: [__meta_kubernetes_service_label_app_kubernetes_io_part_of]
|
||||
action: keep
|
||||
regex: foxhunt
|
||||
- source_labels: [__meta_kubernetes_endpoint_port_name]
|
||||
action: keep
|
||||
regex: metrics
|
||||
- source_labels: [__meta_kubernetes_service_name]
|
||||
target_label: service
|
||||
- source_labels: [__meta_kubernetes_namespace]
|
||||
target_label: namespace
|
||||
|
||||
# --- General prometheus.io annotation-based scraping ---
|
||||
- job_name: prometheus-annotated-pods
|
||||
kubernetes_sd_configs:
|
||||
- role: pod
|
||||
relabel_configs:
|
||||
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
|
||||
action: keep
|
||||
regex: "true"
|
||||
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scheme]
|
||||
action: replace
|
||||
regex: (https?)
|
||||
target_label: __scheme__
|
||||
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
|
||||
action: replace
|
||||
regex: (.+)
|
||||
target_label: __metrics_path__
|
||||
- source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
|
||||
action: replace
|
||||
regex: ([^:]+)(?::\d+)?;(\d+)
|
||||
replacement: $1:$2
|
||||
target_label: __address__
|
||||
- action: labelmap
|
||||
regex: __meta_kubernetes_pod_label_(.+)
|
||||
- source_labels: [__meta_kubernetes_namespace]
|
||||
target_label: namespace
|
||||
- source_labels: [__meta_kubernetes_pod_name]
|
||||
target_label: pod
|
||||
- source_labels: [__meta_kubernetes_pod_node_name]
|
||||
target_label: node
|
||||
|
||||
# --- Loki metrics ---
|
||||
- job_name: loki
|
||||
static_configs:
|
||||
- targets: ['loki.foxhunt.svc.cluster.local:3100']
|
||||
|
||||
# --- Tempo metrics ---
|
||||
- job_name: tempo
|
||||
static_configs:
|
||||
- targets: ['tempo.foxhunt.svc.cluster.local:3200']
|
||||
|
||||
# --- Promtail metrics ---
|
||||
- job_name: promtail
|
||||
kubernetes_sd_configs:
|
||||
- role: pod
|
||||
namespaces:
|
||||
names: [foxhunt]
|
||||
relabel_configs:
|
||||
- source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name]
|
||||
action: keep
|
||||
regex: promtail
|
||||
- source_labels: [__meta_kubernetes_pod_ip]
|
||||
target_label: __address__
|
||||
replacement: $1:9080
|
||||
@@ -49,19 +49,22 @@ data:
|
||||
clients:
|
||||
- url: http://loki.foxhunt.svc.cluster.local:3100/loki/api/v1/push
|
||||
scrape_configs:
|
||||
- job_name: kubernetes-pods
|
||||
# Foxhunt application pods (structured Rust logs)
|
||||
- job_name: foxhunt-pods
|
||||
kubernetes_sd_configs:
|
||||
- role: pod
|
||||
relabel_configs:
|
||||
# Only scrape pods in foxhunt namespace
|
||||
- source_labels: [__meta_kubernetes_namespace]
|
||||
action: keep
|
||||
regex: foxhunt
|
||||
# Drop gitlab/prometheus/grafana pods (too noisy)
|
||||
# Drop noisy infra pods (gitlab internals, prometheus, grafana)
|
||||
- source_labels: [__meta_kubernetes_pod_label_app]
|
||||
action: drop
|
||||
regex: (gitlab|prometheus|grafana)
|
||||
# Map to containerd log path: /var/log/pods/<ns>_<pod>_<uid>/<container>/*.log
|
||||
regex: (gitlab|prometheus|grafana|promtail)
|
||||
- source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name]
|
||||
action: drop
|
||||
regex: (promtail|node-exporter|kube-state-metrics)
|
||||
# Map to containerd log path
|
||||
- source_labels: [__meta_kubernetes_pod_uid, __meta_kubernetes_pod_container_name]
|
||||
separator: /
|
||||
target_label: __path__
|
||||
@@ -79,8 +82,9 @@ data:
|
||||
target_label: job_type
|
||||
- source_labels: [__meta_kubernetes_pod_label_foxhunt_model]
|
||||
target_label: model
|
||||
- source_labels: [__meta_kubernetes_pod_node_name]
|
||||
target_label: node
|
||||
pipeline_stages:
|
||||
# Try to parse JSON logs (from structured logging)
|
||||
- json:
|
||||
expressions:
|
||||
level: level
|
||||
@@ -90,7 +94,6 @@ data:
|
||||
- labels:
|
||||
level:
|
||||
target:
|
||||
# Fallback: if not JSON, keep as plain text
|
||||
- match:
|
||||
selector: '{level=""}'
|
||||
stages:
|
||||
@@ -98,6 +101,96 @@ data:
|
||||
expression: '(?P<level>TRACE|DEBUG|INFO|WARN|ERROR)'
|
||||
- labels:
|
||||
level:
|
||||
|
||||
# Kapsule system pods (kube-system, kube-node-lease, kube-public)
|
||||
- job_name: kapsule-system
|
||||
kubernetes_sd_configs:
|
||||
- role: pod
|
||||
relabel_configs:
|
||||
- source_labels: [__meta_kubernetes_namespace]
|
||||
action: keep
|
||||
regex: (kube-system|kube-node-lease|kube-public)
|
||||
# Map to containerd log path
|
||||
- source_labels: [__meta_kubernetes_pod_uid, __meta_kubernetes_pod_container_name]
|
||||
separator: /
|
||||
target_label: __path__
|
||||
replacement: /var/log/pods/*$1/*.log
|
||||
- source_labels: [__meta_kubernetes_namespace]
|
||||
target_label: namespace
|
||||
- source_labels: [__meta_kubernetes_pod_name]
|
||||
target_label: pod
|
||||
- source_labels: [__meta_kubernetes_pod_container_name]
|
||||
target_label: container
|
||||
- source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name]
|
||||
target_label: app
|
||||
- source_labels: [__meta_kubernetes_pod_label_app]
|
||||
target_label: kube_app
|
||||
- source_labels: [__meta_kubernetes_pod_node_name]
|
||||
target_label: node
|
||||
pipeline_stages:
|
||||
- match:
|
||||
selector: '{namespace="kube-system"}'
|
||||
stages:
|
||||
- regex:
|
||||
expression: '(?P<level>TRACE|DEBUG|INFO|WARN|ERROR|warning|error|fatal)'
|
||||
- labels:
|
||||
level:
|
||||
|
||||
# GitLab CI runner logs (foxhunt namespace, runner pods)
|
||||
- job_name: gitlab-runners
|
||||
kubernetes_sd_configs:
|
||||
- role: pod
|
||||
relabel_configs:
|
||||
- source_labels: [__meta_kubernetes_namespace]
|
||||
action: keep
|
||||
regex: foxhunt
|
||||
- source_labels: [__meta_kubernetes_pod_label_app]
|
||||
action: keep
|
||||
regex: gitlab-runner
|
||||
- source_labels: [__meta_kubernetes_pod_uid, __meta_kubernetes_pod_container_name]
|
||||
separator: /
|
||||
target_label: __path__
|
||||
replacement: /var/log/pods/*$1/*.log
|
||||
- source_labels: [__meta_kubernetes_namespace]
|
||||
target_label: namespace
|
||||
- source_labels: [__meta_kubernetes_pod_name]
|
||||
target_label: pod
|
||||
- source_labels: [__meta_kubernetes_pod_container_name]
|
||||
target_label: container
|
||||
- replacement: gitlab-runner
|
||||
target_label: app
|
||||
- source_labels: [__meta_kubernetes_pod_node_name]
|
||||
target_label: node
|
||||
pipeline_stages:
|
||||
- regex:
|
||||
expression: '(?P<level>TRACE|DEBUG|INFO|WARN|ERROR|WARNING)'
|
||||
- labels:
|
||||
level:
|
||||
|
||||
# Kubernetes node-level logs (kubelet, containerd)
|
||||
- job_name: kapsule-node-logs
|
||||
static_configs:
|
||||
- targets: [localhost]
|
||||
labels:
|
||||
job: kapsule-nodes
|
||||
__path__: /var/log/syslog
|
||||
- targets: [localhost]
|
||||
labels:
|
||||
job: kapsule-nodes
|
||||
__path__: /var/log/kern.log
|
||||
- targets: [localhost]
|
||||
labels:
|
||||
job: kapsule-nodes
|
||||
__path__: /var/log/kubelet.log
|
||||
relabel_configs:
|
||||
- source_labels: [__path__]
|
||||
regex: .*/(.+)\.log
|
||||
target_label: log_type
|
||||
pipeline_stages:
|
||||
- regex:
|
||||
expression: '(?P<level>TRACE|DEBUG|INFO|WARN|ERROR|warning|error|fatal)'
|
||||
- labels:
|
||||
level:
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: DaemonSet
|
||||
@@ -147,11 +240,11 @@ spec:
|
||||
fieldPath: spec.nodeName
|
||||
resources:
|
||||
requests:
|
||||
cpu: 25m
|
||||
memory: 32Mi
|
||||
cpu: 50m
|
||||
memory: 64Mi
|
||||
limits:
|
||||
cpu: 100m
|
||||
memory: 128Mi
|
||||
cpu: 200m
|
||||
memory: 256Mi
|
||||
securityContext:
|
||||
readOnlyRootFilesystem: true
|
||||
allowPrivilegeEscalation: false
|
||||
|
||||
@@ -23,3 +23,11 @@ resource "scaleway_domain_record" "prometheus" {
|
||||
data = var.git_ip
|
||||
ttl = 300
|
||||
}
|
||||
|
||||
resource "scaleway_domain_record" "dashboard" {
|
||||
dns_zone = var.dns_zone
|
||||
name = "dashboard"
|
||||
type = "A"
|
||||
data = var.git_ip
|
||||
ttl = 300
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user