From 3bdefe86d5d058986a82241afd6bd09deb688a8d Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sat, 28 Feb 2026 22:20:32 +0100 Subject: [PATCH] feat(monitoring): production dashboards, remove Cockpit, add Kapsule logs - Remove Cockpit remote_write from GitLab Prometheus (all metrics local) - Uninstall k8s-monitoring Helm (Alloy no longer pushes to Cockpit) - Deploy standalone node-exporter DaemonSet + kube-state-metrics - Add 10 extra Prometheus scrape configs: node-exporter, kube-state-metrics, kubelet, cadvisor, DCGM GPU, foxhunt services, Loki, Tempo, Promtail, prometheus.io annotated pods - Provision 9 Grafana dashboards via sidecar (ConfigMap-based): overview, services, GPU/training, infrastructure, logs, CI/CD pipelines, cluster overview, GPU overview, GitLab services - Expand Promtail to ingest kube-system + GitLab runner + node-level logs - Add dashboard.fxhnt.ai DNS record + nginx routing to Grafana - Update CI deploy job to apply new monitoring manifests Co-Authored-By: Claude Opus 4.6 --- .gitlab-ci.yml | 6 +- infra/k8s/gitlab/tailscale-proxy.yaml | 4 +- .../dashboards/foxhunt-ci-pipelines.json | 973 ++++++++++++++++++ .../dashboards/foxhunt-gpu-training.json | 761 ++++++++++++++ .../dashboards/foxhunt-infrastructure.json | 751 ++++++++++++++ .../monitoring/dashboards/foxhunt-logs.json | 419 ++++++++ .../dashboards/foxhunt-overview.json | 949 +++++++++++++++++ .../dashboards/foxhunt-services.json | 675 ++++++++++++ infra/k8s/monitoring/dashboards/import.sh | 22 +- infra/k8s/monitoring/kube-state-metrics.yaml | 164 +++ infra/k8s/monitoring/node-exporter.yaml | 104 ++ infra/k8s/monitoring/prometheus-values.yaml | 166 +++ infra/k8s/monitoring/promtail.yaml | 115 ++- infra/modules/dns/main.tf | 8 + 14 files changed, 5092 insertions(+), 25 deletions(-) create mode 100644 infra/k8s/monitoring/dashboards/foxhunt-ci-pipelines.json create mode 100644 infra/k8s/monitoring/dashboards/foxhunt-gpu-training.json create mode 100644 infra/k8s/monitoring/dashboards/foxhunt-infrastructure.json create mode 100644 infra/k8s/monitoring/dashboards/foxhunt-logs.json create mode 100644 infra/k8s/monitoring/dashboards/foxhunt-overview.json create mode 100644 infra/k8s/monitoring/dashboards/foxhunt-services.json create mode 100644 infra/k8s/monitoring/kube-state-metrics.yaml create mode 100644 infra/k8s/monitoring/node-exporter.yaml create mode 100644 infra/k8s/monitoring/prometheus-values.yaml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 51ff401d1..34bbfd315 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1417,8 +1417,10 @@ deploy: # S3 binary share: services use generic runtime image + initContainer to fetch binaries from S3. # Deploy = apply K8s manifests (picks up YAML changes) + rollout restart (fetch new binaries). - echo "Deploying ${CI_COMMIT_SHA}..." - # Deploy monitoring stack (Loki, Tempo, Promtail, Grafana datasources) - - kubectl apply -f infra/k8s/monitoring/loki.yaml -f infra/k8s/monitoring/tempo.yaml -f infra/k8s/monitoring/promtail.yaml -f infra/k8s/monitoring/grafana-datasources.yaml + # Deploy monitoring stack (Loki, Tempo, Promtail, node-exporter, kube-state-metrics) + - kubectl apply -f infra/k8s/monitoring/loki.yaml -f infra/k8s/monitoring/tempo.yaml -f infra/k8s/monitoring/promtail.yaml -f infra/k8s/monitoring/node-exporter.yaml -f infra/k8s/monitoring/kube-state-metrics.yaml + # Apply proxy (picks up nginx config changes for dashboard.fxhnt.ai routing) + - kubectl apply -f infra/k8s/gitlab/tailscale-proxy.yaml # Apply service deployments and binary cache PVCs (idempotent — picks up any YAML changes) # Skip training/job-template.yaml (placeholder names, not directly applicable) # Skip storage/training-data-pv.yaml (PVC spec immutable after creation) diff --git a/infra/k8s/gitlab/tailscale-proxy.yaml b/infra/k8s/gitlab/tailscale-proxy.yaml index a06b535fe..aa1dc838d 100644 --- a/infra/k8s/gitlab/tailscale-proxy.yaml +++ b/infra/k8s/gitlab/tailscale-proxy.yaml @@ -196,10 +196,10 @@ data: } } - # Grafana — grafana.fxhnt.ai + # Grafana — grafana.fxhnt.ai + dashboard.fxhnt.ai server { listen 443 ssl; - server_name grafana.fxhnt.ai; + server_name grafana.fxhnt.ai dashboard.fxhnt.ai; ssl_certificate /etc/nginx/certs/tls.crt; ssl_certificate_key /etc/nginx/certs/tls.key; diff --git a/infra/k8s/monitoring/dashboards/foxhunt-ci-pipelines.json b/infra/k8s/monitoring/dashboards/foxhunt-ci-pipelines.json new file mode 100644 index 000000000..246005b83 --- /dev/null +++ b/infra/k8s/monitoring/dashboards/foxhunt-ci-pipelines.json @@ -0,0 +1,973 @@ +{ + "id": null, + "uid": "foxhunt-ci-pipelines", + "title": "Foxhunt CI/CD Pipelines", + "description": "GitLab CI pipeline metrics, failure analysis, runner performance, and K8s training jobs", + "tags": ["foxhunt", "ci-cd", "pipelines"], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "editable": true, + "refresh": "30s", + "time": { + "from": "now-24h", + "to": "now" + }, + "fiscalYearStartMonth": 0, + "liveNow": false, + "weekStart": "", + "templating": { + "list": [ + { + "name": "interval", + "type": "interval", + "query": "1m,5m,15m,30m,1h", + "current": { + "selected": true, + "text": "5m", + "value": "5m" + }, + "auto": false + } + ] + }, + "panels": [ + { + "id": 1, + "type": "row", + "title": "Pipeline Overview", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "collapsed": false, + "panels": [] + }, + { + "id": 2, + "type": "stat", + "title": "Pipeline Creation Rate", + "description": "Pipelines created per second (averaged over $interval)", + "gridPos": { "h": 6, "w": 6, "x": 0, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "blue", "value": null }, + { "color": "green", "value": 0.01 } + ] + }, + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "sum(rate(gitlab_ci_pipeline_processing_events_total[$interval]))", + "legendFormat": "pipelines/s", + "instant": true + } + ] + }, + { + "id": 3, + "type": "stat", + "title": "Active CI Jobs", + "description": "Current number of active CI jobs", + "gridPos": { "h": 6, "w": 6, "x": 6, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 5 }, + { "color": "red", "value": 10 } + ] + }, + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "sum(gitlab_ci_active_jobs_sum)", + "legendFormat": "active jobs", + "instant": true + } + ] + }, + { + "id": 4, + "type": "stat", + "title": "Pipeline Failure Rate", + "description": "Pipeline failures per second (averaged over $interval)", + "gridPos": { "h": 6, "w": 6, "x": 12, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 0.001 }, + { "color": "red", "value": 0.01 } + ] + }, + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "sum(rate(gitlab_ci_pipeline_failure_reasons[$interval]))", + "legendFormat": "failures/s", + "instant": true + } + ] + }, + { + "id": 5, + "type": "stat", + "title": "Queue Operations Rate", + "description": "CI queue operations per second (averaged over $interval)", + "gridPos": { "h": 6, "w": 6, "x": 18, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "blue", "value": null }, + { "color": "green", "value": 0.01 } + ] + }, + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "sum(rate(gitlab_ci_queue_operations_total[$interval]))", + "legendFormat": "queue ops/s", + "instant": true + } + ] + }, + + { + "id": 10, + "type": "row", + "title": "Pipeline Performance", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 7 }, + "collapsed": false, + "panels": [] + }, + { + "id": 11, + "type": "timeseries", + "title": "Pipeline Creation Duration", + "description": "Time to create a pipeline (P50, P90, P99)", + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 8 }, + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 10, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "none", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 2 + }, + "unit": "s", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "P99" }, + "properties": [ + { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } + ] + }, + { + "matcher": { "id": "byName", "options": "P90" }, + "properties": [ + { "id": "color", "value": { "fixedColor": "orange", "mode": "fixed" } } + ] + }, + { + "matcher": { "id": "byName", "options": "P50" }, + "properties": [ + { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } + ] + } + ] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "lastNotNull"] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "histogram_quantile(0.50, sum(rate(gitlab_ci_pipeline_creation_duration_seconds_bucket[$interval])) by (le))", + "legendFormat": "P50" + }, + { + "refId": "B", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "histogram_quantile(0.90, sum(rate(gitlab_ci_pipeline_creation_duration_seconds_bucket[$interval])) by (le))", + "legendFormat": "P90" + }, + { + "refId": "C", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "histogram_quantile(0.99, sum(rate(gitlab_ci_pipeline_creation_duration_seconds_bucket[$interval])) by (le))", + "legendFormat": "P99" + } + ] + }, + { + "id": 12, + "type": "timeseries", + "title": "Pipeline Size (Builds per Pipeline)", + "description": "Number of builds/jobs per pipeline (P50, P90, P99)", + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 8 }, + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 10, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "none", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 2 + }, + "unit": "short", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "P99" }, + "properties": [ + { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } + ] + }, + { + "matcher": { "id": "byName", "options": "P90" }, + "properties": [ + { "id": "color", "value": { "fixedColor": "orange", "mode": "fixed" } } + ] + }, + { + "matcher": { "id": "byName", "options": "P50" }, + "properties": [ + { "id": "color", "value": { "fixedColor": "semi-dark-blue", "mode": "fixed" } } + ] + } + ] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "lastNotNull"] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "histogram_quantile(0.50, sum(rate(gitlab_ci_pipeline_size_builds_bucket[$interval])) by (le))", + "legendFormat": "P50" + }, + { + "refId": "B", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "histogram_quantile(0.90, sum(rate(gitlab_ci_pipeline_size_builds_bucket[$interval])) by (le))", + "legendFormat": "P90" + }, + { + "refId": "C", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "histogram_quantile(0.99, sum(rate(gitlab_ci_pipeline_size_builds_bucket[$interval])) by (le))", + "legendFormat": "P99" + } + ] + }, + { + "id": 13, + "type": "heatmap", + "title": "Active Jobs Distribution", + "description": "Heatmap of active CI job counts over time", + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 8 }, + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "fieldConfig": { + "defaults": { + "custom": { + "scaleDistribution": { "type": "linear" } + } + }, + "overrides": [] + }, + "options": { + "calculate": false, + "yAxis": { + "axisPlacement": "left", + "unit": "short" + }, + "rowsFrame": { + "layout": "auto" + }, + "color": { + "mode": "scheme", + "scheme": "Oranges", + "fill": "dark", + "scale": "exponential", + "exponent": 0.5, + "steps": 64 + }, + "cellGap": 1, + "filterValues": { "le": 1e-9 }, + "tooltip": { "show": true, "yHistogram": true }, + "legend": { "show": true } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "sum(increase(gitlab_ci_active_jobs_bucket[$interval])) by (le)", + "legendFormat": "{{le}}", + "format": "heatmap" + } + ] + }, + + { + "id": 20, + "type": "row", + "title": "Failure Analysis", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 16 }, + "collapsed": false, + "panels": [] + }, + { + "id": 21, + "type": "table", + "title": "Pipeline Failure Reasons", + "description": "Breakdown of pipeline failure reasons over the selected time range", + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 17 }, + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 5 }, + { "color": "red", "value": 20 } + ] + }, + "mappings": [], + "color": { "mode": "thresholds" }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "color-background", + "mode": "basic" + }, + "filterable": true + } + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "Reason" }, + "properties": [ + { "id": "custom.width", "value": 200 } + ] + }, + { + "matcher": { "id": "byName", "options": "Time" }, + "properties": [ + { "id": "custom.hidden", "value": true } + ] + } + ] + }, + "options": { + "showHeader": true, + "sortBy": [{ "displayName": "Value", "desc": true }], + "footer": { + "show": true, + "reducer": ["sum"], + "countRows": false + } + }, + "transformations": [ + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [{ "field": "Value", "desc": true }] + } + } + ], + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "sum by (reason) (increase(gitlab_ci_pipeline_failure_reasons[$__range]))", + "legendFormat": "{{reason}}", + "instant": true, + "format": "table" + } + ] + }, + { + "id": 22, + "type": "table", + "title": "Job Failure Reasons", + "description": "Breakdown of individual job failure reasons over the selected time range", + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 17 }, + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 5 }, + { "color": "red", "value": 20 } + ] + }, + "mappings": [], + "color": { "mode": "thresholds" }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "color-background", + "mode": "basic" + }, + "filterable": true + } + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "Failure Reason" }, + "properties": [ + { "id": "custom.width", "value": 200 } + ] + }, + { + "matcher": { "id": "byName", "options": "Time" }, + "properties": [ + { "id": "custom.hidden", "value": true } + ] + } + ] + }, + "options": { + "showHeader": true, + "sortBy": [{ "displayName": "Value", "desc": true }], + "footer": { + "show": true, + "reducer": ["sum"], + "countRows": false + } + }, + "transformations": [ + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [{ "field": "Value", "desc": true }] + } + } + ], + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "sum by (failure_reason) (increase(gitlab_ci_job_failure_reasons[$__range]))", + "legendFormat": "{{failure_reason}}", + "instant": true, + "format": "table" + } + ] + }, + { + "id": 23, + "type": "timeseries", + "title": "Build Trace Errors", + "description": "Rate of build trace errors over time", + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 17 }, + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 20, + "gradientMode": "scheme", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "none", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 2 + }, + "unit": "ops", + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "red", "value": 0.1 } + ] + }, + "color": { "fixedColor": "red", "mode": "fixed" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "sum"] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "sum(rate(gitlab_ci_build_trace_errors_total[$interval]))", + "legendFormat": "trace errors/s" + } + ] + }, + + { + "id": 30, + "type": "row", + "title": "Runner Performance", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 25 }, + "collapsed": false, + "panels": [] + }, + { + "id": 31, + "type": "timeseries", + "title": "Runner Authentication Success", + "description": "Rate of successful runner authentication attempts", + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 26 }, + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 15, + "gradientMode": "scheme", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "none", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 2 + }, + "unit": "ops", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "fixedColor": "green", "mode": "fixed" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "sum"] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "sum(rate(gitlab_ci_runner_authentication_success_total[$interval]))", + "legendFormat": "auth success/s" + } + ] + }, + { + "id": 32, + "type": "timeseries", + "title": "Trace Operations & Volume", + "description": "CI trace operations rate and data volume", + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 26 }, + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 10, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "none", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 2 + }, + "unit": "ops", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "trace bytes/s" }, + "properties": [ + { "id": "custom.axisPlacement", "value": "right" }, + { "id": "unit", "value": "Bps" } + ] + } + ] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "sum(rate(gitlab_ci_trace_operations_total[$interval]))", + "legendFormat": "trace ops/s" + }, + { + "refId": "B", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "sum(rate(gitlab_ci_trace_bytes_total[$interval]))", + "legendFormat": "trace bytes/s" + } + ] + }, + { + "id": 33, + "type": "timeseries", + "title": "Artifacts Cleanup", + "description": "Rate and volume of destroyed job artifacts", + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 26 }, + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "bars", + "lineInterpolation": "smooth", + "fillOpacity": 60, + "gradientMode": "scheme", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "none", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 0, + "barAlignment": 0 + }, + "unit": "ops", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "bytes destroyed/s" }, + "properties": [ + { "id": "custom.axisPlacement", "value": "right" }, + { "id": "unit", "value": "Bps" }, + { "id": "custom.drawStyle", "value": "line" }, + { "id": "custom.fillOpacity", "value": 0 }, + { "id": "custom.lineWidth", "value": 2 } + ] + } + ] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "sum"] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "sum(rate(destroyed_job_artifacts_count_total[$interval]))", + "legendFormat": "artifacts destroyed/s" + }, + { + "refId": "B", + "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "expr": "sum(rate(destroyed_job_artifacts_bytes_total[$interval]))", + "legendFormat": "bytes destroyed/s" + } + ] + }, + + { + "id": 40, + "type": "row", + "title": "K8s Training Jobs", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 34 }, + "collapsed": false, + "panels": [] + }, + { + "id": 41, + "type": "timeseries", + "title": "Training Jobs: Created vs Completed vs Failed", + "description": "K8s training job lifecycle in the foxhunt namespace", + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 35 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "stepAfter", + "fillOpacity": 15, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "none", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 2 + }, + "unit": "short", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "Created" }, + "properties": [ + { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } + ] + }, + { + "matcher": { "id": "byName", "options": "Succeeded" }, + "properties": [ + { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } + ] + }, + { + "matcher": { "id": "byName", "options": "Failed" }, + "properties": [ + { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } + ] + } + ] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "max"] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "count(kube_job_info{namespace=\"foxhunt\", job_name=~\".*train.*\"})", + "legendFormat": "Created" + }, + { + "refId": "B", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "count(kube_job_status_succeeded{namespace=\"foxhunt\", job_name=~\".*train.*\"} == 1)", + "legendFormat": "Succeeded" + }, + { + "refId": "C", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "count(kube_job_status_failed{namespace=\"foxhunt\", job_name=~\".*train.*\"} > 0)", + "legendFormat": "Failed" + } + ] + }, + { + "id": 42, + "type": "timeseries", + "title": "Training Job Duration", + "description": "Age of active training jobs (time since creation)", + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 35 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 10, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "none", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 2 + }, + "unit": "s", + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 3600 }, + { "color": "red", "value": 14400 } + ] + }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "lastNotNull"] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "time() - kube_job_created{namespace=\"foxhunt\", job_name=~\".*train.*\"} and on(job_name, namespace) kube_job_status_succeeded{namespace=\"foxhunt\"} == 0 and on(job_name, namespace) (kube_job_status_failed{namespace=\"foxhunt\"} == 0 or absent(kube_job_status_failed{namespace=\"foxhunt\"}))", + "legendFormat": "{{job_name}}" + } + ] + }, + { + "id": 43, + "type": "timeseries", + "title": "Training Pod CPU Usage", + "description": "CPU usage of training pods in the foxhunt namespace", + "gridPos": { "h": 8, "w": 4, "x": 16, "y": 35 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 20, + "gradientMode": "scheme", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "normal", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 1 + }, + "unit": "short", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\".*train.*\", container!=\"POD\", container!=\"\"}[$interval]))", + "legendFormat": "{{pod}}" + } + ] + }, + { + "id": 44, + "type": "timeseries", + "title": "Training Pod Memory", + "description": "Memory working set of training pods in the foxhunt namespace", + "gridPos": { "h": 8, "w": 4, "x": 20, "y": 35 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 20, + "gradientMode": "scheme", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "normal", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 1 + }, + "unit": "bytes", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "sum by (pod) (container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\".*train.*\", container!=\"POD\", container!=\"\"})", + "legendFormat": "{{pod}}" + } + ] + } + ] +} diff --git a/infra/k8s/monitoring/dashboards/foxhunt-gpu-training.json b/infra/k8s/monitoring/dashboards/foxhunt-gpu-training.json new file mode 100644 index 000000000..7c8486d02 --- /dev/null +++ b/infra/k8s/monitoring/dashboards/foxhunt-gpu-training.json @@ -0,0 +1,761 @@ +{ + "id": null, + "uid": "foxhunt-gpu-training", + "title": "Foxhunt - GPU & Training Jobs", + "description": "GPU node pools, DCGM metrics, K8s training jobs, CI pipelines", + "tags": ["foxhunt", "production"], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "editable": true, + "refresh": "30s", + "time": { + "from": "now-1h", + "to": "now" + }, + "fiscalYearStartMonth": 0, + "liveNow": false, + "weekStart": "", + "templating": { + "list": [ + { + "name": "gpu", + "type": "query", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "query": "label_values(DCGM_FI_DEV_GPU_UTIL, gpu)", + "refresh": 2, + "includeAll": true, + "multi": true, + "allValue": ".*", + "current": { "selected": true, "text": "All", "value": "$__all" }, + "sort": 1 + }, + { + "name": "interval", + "type": "interval", + "query": "1m,5m,15m,30m,1h", + "current": { "selected": true, "text": "5m", "value": "5m" }, + "auto": false + } + ] + }, + "panels": [ + { + "id": 1, + "type": "row", + "title": "GPU Node Pool Status", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "collapsed": false, + "panels": [] + }, + { + "id": 2, + "type": "stat", + "title": "ci-compile Nodes", + "description": "L4-1-24G GPU nodes (0-1 autoscale)", + "gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "blue", "value": null }, + { "color": "green", "value": 1 } + ] + }, + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "count(kube_node_labels{label_k8s_scaleway_com_pool_name=\"ci-compile\"}) or vector(0)", + "instant": true + } + ] + }, + { + "id": 3, + "type": "stat", + "title": "ci-training Nodes", + "description": "L40S-1-48G GPU nodes (0-1 autoscale)", + "gridPos": { "h": 4, "w": 4, "x": 4, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "blue", "value": null }, + { "color": "green", "value": 1 } + ] + }, + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "count(kube_node_labels{label_k8s_scaleway_com_pool_name=\"ci-training\"}) or vector(0)", + "instant": true + } + ] + }, + { + "id": 4, + "type": "stat", + "title": "ci-compile-cpu Nodes", + "description": "POP2-32C-128G CPU nodes (0-1 autoscale)", + "gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "blue", "value": null }, + { "color": "green", "value": 1 } + ] + }, + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "count(kube_node_labels{label_k8s_scaleway_com_pool_name=\"ci-compile-cpu\"}) or vector(0)", + "instant": true + } + ] + }, + { + "id": 5, + "type": "stat", + "title": "Active Training Jobs", + "gridPos": { "h": 4, "w": 4, "x": 12, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "blue", "value": null }, + { "color": "green", "value": 1 } + ] + }, + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "sum(kube_job_status_active{namespace=\"foxhunt\"}) or vector(0)", + "instant": true + } + ] + }, + { + "id": 6, + "type": "stat", + "title": "Jobs Succeeded (24h)", + "gridPos": { "h": 4, "w": 4, "x": 16, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "sum(increase(kube_job_status_succeeded{namespace=\"foxhunt\"}[24h])) or vector(0)", + "instant": true + } + ] + }, + { + "id": 7, + "type": "stat", + "title": "Jobs Failed (24h)", + "gridPos": { "h": 4, "w": 4, "x": 20, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "red", "value": 1 } + ] + }, + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "sum(increase(kube_job_status_failed{namespace=\"foxhunt\"}[24h])) or vector(0)", + "instant": true + } + ] + }, + { + "id": 10, + "type": "row", + "title": "GPU Utilization (DCGM)", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, + "collapsed": false, + "panels": [] + }, + { + "id": 11, + "type": "timeseries", + "title": "GPU Utilization %", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 20, + "gradientMode": "scheme", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "none", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 2 + }, + "unit": "percent", + "min": 0, + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 80 }, + { "color": "red", "value": 95 } + ] + }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "DCGM_FI_DEV_GPU_UTIL{gpu=~\"$gpu\"}", + "legendFormat": "GPU {{ gpu }} ({{ instance }})" + } + ] + }, + { + "id": 12, + "type": "timeseries", + "title": "GPU Memory Used (MB)", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 20, + "gradientMode": "scheme", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "none", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 2 + }, + "unit": "decmbytes", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "DCGM_FI_DEV_FB_USED{gpu=~\"$gpu\"}", + "legendFormat": "GPU {{ gpu }} used" + }, + { + "refId": "B", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "DCGM_FI_DEV_FB_TOTAL{gpu=~\"$gpu\"}", + "legendFormat": "GPU {{ gpu }} total" + } + ] + }, + { + "id": 13, + "type": "timeseries", + "title": "GPU Temperature", + "gridPos": { "h": 6, "w": 12, "x": 0, "y": 14 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 10, + "gradientMode": "scheme", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "none", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 2 + }, + "unit": "celsius", + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "blue", "value": null }, + { "color": "green", "value": 40 }, + { "color": "yellow", "value": 75 }, + { "color": "red", "value": 85 } + ] + }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "DCGM_FI_DEV_GPU_TEMP{gpu=~\"$gpu\"}", + "legendFormat": "GPU {{ gpu }}" + } + ] + }, + { + "id": 14, + "type": "timeseries", + "title": "GPU Power (Watts)", + "gridPos": { "h": 6, "w": 12, "x": 12, "y": 14 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 10, + "gradientMode": "scheme", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "none", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 2 + }, + "unit": "watt", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "DCGM_FI_DEV_POWER_USAGE{gpu=~\"$gpu\"}", + "legendFormat": "GPU {{ gpu }}" + } + ] + }, + { + "id": 20, + "type": "row", + "title": "Training Jobs", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 20 }, + "collapsed": false, + "panels": [] + }, + { + "id": 21, + "type": "timeseries", + "title": "Active Jobs Over Time", + "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "stepAfter", + "fillOpacity": 30, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "none", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 2 + }, + "unit": "short", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["last"] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "kube_job_status_active{namespace=\"foxhunt\"}", + "legendFormat": "{{ job_name }}" + } + ] + }, + { + "id": 22, + "type": "table", + "title": "Job Status Summary", + "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "Failed" }, + "properties": [ + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "red", "value": 1 } + ] + } + }, + { "id": "custom.displayMode", "value": "color-background" } + ] + } + ] + }, + "options": { + "showHeader": true, + "sortBy": [{ "displayName": "job_name", "desc": false }], + "footer": { "show": false } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "kube_job_status_active{namespace=\"foxhunt\"}", + "legendFormat": "Active", + "instant": true, + "format": "table" + }, + { + "refId": "B", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "kube_job_status_succeeded{namespace=\"foxhunt\"}", + "legendFormat": "Succeeded", + "instant": true, + "format": "table" + }, + { + "refId": "C", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "kube_job_status_failed{namespace=\"foxhunt\"}", + "legendFormat": "Failed", + "instant": true, + "format": "table" + } + ] + }, + { + "id": 25, + "type": "timeseries", + "title": "GPU Node CPU Usage (Compute Pools)", + "description": "CPU usage on GPU/compute node pools when no DCGM data is available", + "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 15, + "gradientMode": "scheme", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "none", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 1 + }, + "unit": "percentunit", + "min": 0, + "max": 1, + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[$interval]))", + "legendFormat": "{{ instance }}" + } + ] + }, + { + "id": 26, + "type": "timeseries", + "title": "GPU Node Memory Usage (Compute Pools)", + "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 15, + "gradientMode": "scheme", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "none", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 1 + }, + "unit": "bytes", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes", + "legendFormat": "{{ instance }}" + } + ] + }, + { + "id": 30, + "type": "row", + "title": "Training & CI Logs", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 35 }, + "collapsed": false, + "panels": [] + }, + { + "id": 31, + "type": "logs", + "title": "Training Job Logs", + "gridPos": { "h": 10, "w": 12, "x": 0, "y": 36 }, + "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "options": { + "showTime": true, + "showLabels": true, + "showCommonLabels": false, + "wrapLogMessage": true, + "prettifyLogMessage": false, + "enableLogDetails": true, + "sortOrder": "Descending", + "dedupStrategy": "none" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "expr": "{namespace=\"foxhunt\", container=~\".*train.*\"}", + "maxLines": 200 + } + ] + }, + { + "id": 32, + "type": "logs", + "title": "CI / GitLab Runner Logs", + "gridPos": { "h": 10, "w": 12, "x": 12, "y": 36 }, + "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "options": { + "showTime": true, + "showLabels": true, + "showCommonLabels": false, + "wrapLogMessage": true, + "prettifyLogMessage": false, + "enableLogDetails": true, + "sortOrder": "Descending", + "dedupStrategy": "none" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "expr": "{namespace=\"foxhunt\", app=~\".*runner.*|.*gitlab.*\"}", + "maxLines": 200 + } + ] + }, + { + "id": 40, + "type": "row", + "title": "Node Autoscale Events", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 46 }, + "collapsed": false, + "panels": [] + }, + { + "id": 41, + "type": "timeseries", + "title": "Node Count Over Time", + "description": "Tracks autoscaling of ci-compile and ci-training pools", + "gridPos": { "h": 7, "w": 24, "x": 0, "y": 47 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "stepAfter", + "fillOpacity": 10, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "auto", + "stacking": { "mode": "none", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 2 + }, + "unit": "short", + "decimals": 0, + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["min", "max", "last"] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "count by (label_k8s_scaleway_com_pool_name) (kube_node_labels)", + "legendFormat": "{{ label_k8s_scaleway_com_pool_name }}" + } + ] + } + ] +} \ No newline at end of file diff --git a/infra/k8s/monitoring/dashboards/foxhunt-infrastructure.json b/infra/k8s/monitoring/dashboards/foxhunt-infrastructure.json new file mode 100644 index 000000000..2591ab769 --- /dev/null +++ b/infra/k8s/monitoring/dashboards/foxhunt-infrastructure.json @@ -0,0 +1,751 @@ +{ + "id": null, + "uid": "foxhunt-infrastructure", + "title": "Foxhunt - K8s Infrastructure", + "description": "Node health, per-node resources, PVC usage, scheduling, OOMKill events", + "tags": ["foxhunt", "production"], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "editable": true, + "refresh": "30s", + "time": { + "from": "now-1h", + "to": "now" + }, + "fiscalYearStartMonth": 0, + "liveNow": false, + "weekStart": "", + "templating": { + "list": [ + { + "name": "node", + "type": "query", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "query": "label_values(kube_node_info, node)", + "refresh": 2, + "includeAll": true, + "multi": true, + "allValue": ".*", + "current": { "selected": true, "text": "All", "value": "$__all" }, + "sort": 1 + }, + { + "name": "interval", + "type": "interval", + "query": "1m,5m,15m,30m,1h", + "current": { "selected": true, "text": "5m", "value": "5m" }, + "auto": false + } + ] + }, + "panels": [ + { + "id": 1, + "type": "row", + "title": "Node Status", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "collapsed": false, + "panels": [] + }, + { + "id": 2, + "type": "table", + "title": "Node Status by Pool", + "gridPos": { "h": 7, "w": 24, "x": 0, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "mappings": [ + { "type": "value", "options": { "1": { "text": "Ready", "color": "green" }, "0": { "text": "NotReady", "color": "red" } } } + ], + "color": { "mode": "thresholds" } + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "Value" }, + "properties": [ + { "id": "custom.displayMode", "value": "color-background-solid" }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "green", "value": 1 } + ] + } + } + ] + } + ] + }, + "options": { + "showHeader": true, + "sortBy": [{ "displayName": "node", "desc": false }], + "footer": { "show": false } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "kube_node_status_condition{condition=\"Ready\", status=\"true\", node=~\"$node\"}", + "legendFormat": "{{ node }}", + "instant": true, + "format": "table" + } + ] + }, + { + "id": 5, + "type": "stat", + "title": "services Pool", + "description": "DEV1-M, 2 fixed", + "gridPos": { "h": 3, "w": 4, "x": 0, "y": 8 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "yellow", "value": 1 }, + { "color": "green", "value": 2 } + ] + }, + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "value_and_name", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "count(kube_node_labels{label_k8s_scaleway_com_pool_name=\"services\"} * on(node) kube_node_status_condition{condition=\"Ready\", status=\"true\"}) or vector(0)", + "legendFormat": "services", + "instant": true + } + ] + }, + { + "id": 6, + "type": "stat", + "title": "gitlab Pool", + "description": "GP1-XS, 1 fixed", + "gridPos": { "h": 3, "w": 4, "x": 4, "y": 8 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "green", "value": 1 } + ] + }, + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "value_and_name", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "count(kube_node_labels{label_k8s_scaleway_com_pool_name=\"gitlab\"} * on(node) kube_node_status_condition{condition=\"Ready\", status=\"true\"}) or vector(0)", + "legendFormat": "gitlab", + "instant": true + } + ] + }, + { + "id": 7, + "type": "stat", + "title": "ci-compile-cpu Pool", + "description": "POP2-32C-128G, 0-1 autoscale", + "gridPos": { "h": 3, "w": 4, "x": 8, "y": 8 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "blue", "value": null }, + { "color": "green", "value": 1 } + ] + }, + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "value_and_name", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "count(kube_node_labels{label_k8s_scaleway_com_pool_name=\"ci-compile-cpu\"}) or vector(0)", + "legendFormat": "ci-compile-cpu", + "instant": true + } + ] + }, + { + "id": 8, + "type": "stat", + "title": "ci-compile Pool (GPU)", + "description": "L4-1-24G, 0-1 autoscale", + "gridPos": { "h": 3, "w": 4, "x": 12, "y": 8 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "blue", "value": null }, + { "color": "green", "value": 1 } + ] + }, + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "value_and_name", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "count(kube_node_labels{label_k8s_scaleway_com_pool_name=\"ci-compile\"}) or vector(0)", + "legendFormat": "ci-compile", + "instant": true + } + ] + }, + { + "id": 9, + "type": "stat", + "title": "ci-training Pool (GPU)", + "description": "L40S-1-48G, 0-1 autoscale", + "gridPos": { "h": 3, "w": 4, "x": 16, "y": 8 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "blue", "value": null }, + { "color": "green", "value": 1 } + ] + }, + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "value_and_name", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "count(kube_node_labels{label_k8s_scaleway_com_pool_name=\"ci-training\"}) or vector(0)", + "legendFormat": "ci-training", + "instant": true + } + ] + }, + { + "id": 10, + "type": "stat", + "title": "Total Pods", + "gridPos": { "h": 3, "w": 4, "x": 20, "y": 8 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "blue", "value": null } + ] + }, + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "sum(kubelet_running_pods) or count(kube_pod_info)", + "instant": true + } + ] + }, + { + "id": 15, + "type": "row", + "title": "Per-Node Resources", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 11 }, + "collapsed": false, + "panels": [] + }, + { + "id": 16, + "type": "timeseries", + "title": "Node CPU Usage", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 15, + "gradientMode": "scheme", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "none", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 1 + }, + "unit": "percentunit", + "min": 0, + "max": 1, + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[$interval]))", + "legendFormat": "{{ instance }}" + } + ] + }, + { + "id": 17, + "type": "timeseries", + "title": "Node Memory Usage", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 15, + "gradientMode": "scheme", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "none", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 1 + }, + "unit": "bytes", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes", + "legendFormat": "{{ instance }} used" + }, + { + "refId": "B", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "node_memory_MemTotal_bytes", + "legendFormat": "{{ instance }} total" + } + ] + }, + { + "id": 18, + "type": "timeseries", + "title": "Node Filesystem Usage", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 20 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 15, + "gradientMode": "scheme", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "none", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 1 + }, + "unit": "percentunit", + "min": 0, + "max": 1, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 0.7 }, + { "color": "red", "value": 0.9 } + ] + }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "1 - (node_filesystem_avail_bytes{mountpoint=\"/\", fstype!=\"tmpfs\"} / node_filesystem_size_bytes{mountpoint=\"/\", fstype!=\"tmpfs\"})", + "legendFormat": "{{ instance }}" + } + ] + }, + { + "id": 19, + "type": "bargauge", + "title": "Pods per Node", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 20 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 80 }, + { "color": "red", "value": 100 } + ] + }, + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "horizontal", + "displayMode": "gradient", + "showUnfilled": true, + "minVizWidth": 8, + "minVizHeight": 16, + "valueMode": "color" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "count by (node) (kube_pod_info{node=~\"$node\"})", + "legendFormat": "{{ node }}", + "instant": true + } + ] + }, + { + "id": 25, + "type": "row", + "title": "Network & PVC", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 28 }, + "collapsed": false, + "panels": [] + }, + { + "id": 26, + "type": "timeseries", + "title": "Network I/O per Node", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 29 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 10, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "none", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 1 + }, + "unit": "Bps", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "sum by (instance) (rate(node_network_receive_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$interval]))", + "legendFormat": "{{ instance }} RX" + }, + { + "refId": "B", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "sum by (instance) (rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$interval]))", + "legendFormat": "{{ instance }} TX" + } + ] + }, + { + "id": 27, + "type": "bargauge", + "title": "PVC Usage", + "description": "Persistent Volume Claim usage percentage. Requires kubelet_volume_stats metrics.", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 29 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 70 }, + { "color": "red", "value": 90 } + ] + }, + "unit": "percent", + "min": 0, + "max": 100, + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "horizontal", + "displayMode": "gradient", + "showUnfilled": true, + "minVizWidth": 8, + "minVizHeight": 16, + "valueMode": "color" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "(1 - (kubelet_volume_stats_available_bytes{namespace=\"foxhunt\"} / kubelet_volume_stats_capacity_bytes{namespace=\"foxhunt\"})) * 100", + "legendFormat": "{{ persistentvolumeclaim }}", + "instant": true + } + ] + }, + { + "id": 30, + "type": "row", + "title": "Pod Scheduling & OOMKills", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 37 }, + "collapsed": false, + "panels": [] + }, + { + "id": 31, + "type": "timeseries", + "title": "Scheduling Latency (p99)", + "description": "Time to schedule pods. From scheduler_scheduling_algorithm_duration_seconds or scheduler_e2e_scheduling_duration_seconds.", + "gridPos": { "h": 7, "w": 12, "x": 0, "y": 38 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 10, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "none", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 2 + }, + "unit": "s", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "histogram_quantile(0.99, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket[$interval])) by (le))", + "legendFormat": "p99 e2e scheduling" + }, + { + "refId": "B", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "histogram_quantile(0.50, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket[$interval])) by (le))", + "legendFormat": "p50 e2e scheduling" + } + ] + }, + { + "id": 32, + "type": "timeseries", + "title": "OOMKill Events", + "description": "Container OOMKill events. Each spike indicates a container was killed due to memory pressure.", + "gridPos": { "h": 7, "w": 12, "x": 12, "y": 38 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "bars", + "lineInterpolation": "linear", + "fillOpacity": 80, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "normal", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 1 + }, + "unit": "short", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["sum"] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "increase(kube_pod_container_status_last_terminated_reason{reason=\"OOMKilled\", namespace=\"foxhunt\"}[30m]) > 0", + "legendFormat": "{{ pod }} ({{ container }})" + } + ] + }, + { + "id": 35, + "type": "timeseries", + "title": "Disk I/O per Node", + "gridPos": { "h": 7, "w": 24, "x": 0, "y": 45 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 10, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "none", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 1 + }, + "unit": "Bps", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "sum by (instance) (rate(node_disk_read_bytes_total[$interval]))", + "legendFormat": "{{ instance }} read" + }, + { + "refId": "B", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "sum by (instance) (rate(node_disk_written_bytes_total[$interval]))", + "legendFormat": "{{ instance }} write" + } + ] + } + ] +} \ No newline at end of file diff --git a/infra/k8s/monitoring/dashboards/foxhunt-logs.json b/infra/k8s/monitoring/dashboards/foxhunt-logs.json new file mode 100644 index 000000000..b17884567 --- /dev/null +++ b/infra/k8s/monitoring/dashboards/foxhunt-logs.json @@ -0,0 +1,419 @@ +{ + "id": null, + "uid": "foxhunt-logs", + "title": "Foxhunt - Log Explorer", + "description": "Log volume, error rates, top errors, live log stream, trace search", + "tags": ["foxhunt", "production"], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "editable": true, + "refresh": "30s", + "time": { + "from": "now-1h", + "to": "now" + }, + "fiscalYearStartMonth": 0, + "liveNow": false, + "weekStart": "", + "templating": { + "list": [ + { + "name": "service", + "type": "custom", + "query": "api-gateway,trading-service,ml-training-service,backtesting-service,trading-agent-service,broker-gateway,data-acquisition-service,web-gateway", + "current": { "selected": false, "text": "All", "value": "$__all" }, + "multi": true, + "includeAll": true, + "allValue": ".*", + "sort": 0 + }, + { + "name": "search", + "type": "textbox", + "query": "", + "current": { "selected": false, "text": "", "value": "" }, + "label": "Log Search" + }, + { + "name": "interval", + "type": "interval", + "query": "1m,5m,15m,30m,1h", + "current": { "selected": true, "text": "5m", "value": "5m" }, + "auto": false + } + ] + }, + "panels": [ + { + "id": 1, + "type": "row", + "title": "Log Volume Overview", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "collapsed": false, + "panels": [] + }, + { + "id": 2, + "type": "timeseries", + "title": "Log Volume by Service", + "gridPos": { "h": 8, "w": 16, "x": 0, "y": 1 }, + "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "bars", + "lineInterpolation": "linear", + "fillOpacity": 60, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "normal", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 0 + }, + "unit": "short", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "right", "calcs": ["sum", "mean"] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "expr": "sum by (app) (count_over_time({namespace=\"foxhunt\", app=~\"$service\"} [$interval]))", + "legendFormat": "{{ app }}" + } + ] + }, + { + "id": 3, + "type": "stat", + "title": "Total Logs (1h)", + "gridPos": { "h": 4, "w": 4, "x": 16, "y": 1 }, + "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "blue", "value": null } + ] + }, + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["sum"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "expr": "sum(count_over_time({namespace=\"foxhunt\", app=~\"$service\"} [1h]))", + "instant": true + } + ] + }, + { + "id": 4, + "type": "stat", + "title": "Errors (1h)", + "gridPos": { "h": 4, "w": 4, "x": 20, "y": 1 }, + "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 10 }, + { "color": "red", "value": 50 } + ] + }, + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["sum"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "area", + "justifyMode": "center" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "expr": "sum(count_over_time({namespace=\"foxhunt\", app=~\"$service\"} |= \"ERROR\" [1h]))", + "instant": true + } + ] + }, + { + "id": 5, + "type": "stat", + "title": "Warnings (1h)", + "gridPos": { "h": 4, "w": 4, "x": 16, "y": 5 }, + "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 50 } + ] + }, + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["sum"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "expr": "sum(count_over_time({namespace=\"foxhunt\", app=~\"$service\"} |= \"WARN\" [1h]))", + "instant": true + } + ] + }, + { + "id": 6, + "type": "stat", + "title": "Error Rate /min", + "gridPos": { "h": 4, "w": 4, "x": 20, "y": 5 }, + "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 1 }, + { "color": "red", "value": 5 } + ] + }, + "unit": "short", + "decimals": 1, + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "area", + "justifyMode": "center" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "expr": "sum(count_over_time({namespace=\"foxhunt\", app=~\"$service\"} |= \"ERROR\" [1m]))", + "instant": true + } + ] + }, + { + "id": 10, + "type": "row", + "title": "Error Analysis", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 9 }, + "collapsed": false, + "panels": [] + }, + { + "id": 11, + "type": "timeseries", + "title": "Error Rate by Service", + "gridPos": { "h": 7, "w": 24, "x": 0, "y": 10 }, + "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 20, + "gradientMode": "scheme", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "none", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 2 + }, + "unit": "short", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "right", "calcs": ["sum", "max", "last"] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "expr": "sum by (app) (count_over_time({namespace=\"foxhunt\", app=~\"$service\"} |= \"ERROR\" [$interval]))", + "legendFormat": "{{ app }}" + } + ] + }, + { + "id": 12, + "type": "table", + "title": "Top Error Messages", + "description": "Most frequent error messages across all services in the time range", + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 17 }, + "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "fieldConfig": { + "defaults": { + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "showHeader": true, + "sortBy": [{ "displayName": "Value", "desc": true }], + "footer": { "show": false } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "expr": "topk(20, sum by (app, level) (count_over_time({namespace=\"foxhunt\", app=~\"$service\"} |= \"ERROR\" | logfmt [$interval])))", + "legendFormat": "{{ app }}", + "instant": true + } + ] + }, + { + "id": 20, + "type": "row", + "title": "Live Logs", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 25 }, + "collapsed": false, + "panels": [] + }, + { + "id": 21, + "type": "logs", + "title": "Live Log Stream", + "description": "Streaming logs from selected services. Use the $search variable to filter.", + "gridPos": { "h": 14, "w": 24, "x": 0, "y": 26 }, + "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "options": { + "showTime": true, + "showLabels": true, + "showCommonLabels": false, + "wrapLogMessage": true, + "prettifyLogMessage": false, + "enableLogDetails": true, + "sortOrder": "Descending", + "dedupStrategy": "none" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "expr": "{namespace=\"foxhunt\", app=~\"$service\"} |= \"$search\"", + "maxLines": 500 + } + ] + }, + { + "id": 30, + "type": "row", + "title": "Traces", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 40 }, + "collapsed": false, + "panels": [] + }, + { + "id": 31, + "type": "table", + "title": "Recent Traces", + "description": "Search for distributed traces via Tempo. Click a trace ID to view the full trace.", + "gridPos": { "h": 10, "w": 24, "x": 0, "y": 41 }, + "datasource": { "type": "tempo", "uid": "tempo" }, + "fieldConfig": { + "defaults": { + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "mappings": [], + "color": { "mode": "thresholds" }, + "links": [ + { + "title": "View Trace", + "url": "", + "internal": { + "datasourceUid": "tempo", + "datasourceName": "Tempo", + "query": { + "queryType": "traceql", + "query": "${__data.fields.traceID}" + } + } + } + ] + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "Duration" }, + "properties": [{ "id": "unit", "value": "ns" }] + } + ] + }, + "options": { + "showHeader": true, + "sortBy": [{ "displayName": "Start time", "desc": true }], + "footer": { "show": false } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "tempo", "uid": "tempo" }, + "queryType": "nativeSearch", + "serviceName": "", + "spanName": "", + "search": "", + "minDuration": "", + "maxDuration": "", + "limit": 20 + } + ] + } + ] +} \ No newline at end of file diff --git a/infra/k8s/monitoring/dashboards/foxhunt-overview.json b/infra/k8s/monitoring/dashboards/foxhunt-overview.json new file mode 100644 index 000000000..b75010478 --- /dev/null +++ b/infra/k8s/monitoring/dashboards/foxhunt-overview.json @@ -0,0 +1,949 @@ +{ + "id": null, + "uid": "foxhunt-overview", + "title": "Foxhunt - System Overview", + "description": "Home dashboard: cluster health, service status, resource utilization, deployments, training jobs, error rates", + "tags": ["foxhunt", "production"], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "editable": true, + "refresh": "30s", + "time": { + "from": "now-1h", + "to": "now" + }, + "fiscalYearStartMonth": 0, + "liveNow": false, + "weekStart": "", + "templating": { + "list": [ + { + "name": "interval", + "type": "interval", + "query": "1m,5m,15m,30m,1h", + "current": { + "selected": true, + "text": "5m", + "value": "5m" + }, + "auto": false + } + ] + }, + "panels": [ + { + "id": 1, + "type": "row", + "title": "Cluster Health", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "collapsed": false, + "panels": [] + }, + { + "id": 2, + "type": "stat", + "title": "Nodes Ready", + "gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "yellow", "value": 2 }, + { "color": "green", "value": 3 } + ] + }, + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "sum(kube_node_status_condition{condition=\"Ready\", status=\"true\"})", + "legendFormat": "Ready", + "instant": true + } + ] + }, + { + "id": 3, + "type": "stat", + "title": "Total Nodes", + "gridPos": { "h": 4, "w": 4, "x": 4, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "blue", "value": null } + ] + }, + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "count(kube_node_info)", + "legendFormat": "Nodes", + "instant": true + } + ] + }, + { + "id": 4, + "type": "stat", + "title": "Pods Running", + "gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", phase=\"Running\"})", + "legendFormat": "Running", + "instant": true + } + ] + }, + { + "id": 5, + "type": "stat", + "title": "Pods Failed", + "gridPos": { "h": 4, "w": 4, "x": 12, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "red", "value": 1 } + ] + }, + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", phase=\"Failed\"}) or vector(0)", + "legendFormat": "Failed", + "instant": true + } + ] + }, + { + "id": 6, + "type": "stat", + "title": "Active Training Jobs", + "gridPos": { "h": 4, "w": 4, "x": 16, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "blue", "value": null }, + { "color": "green", "value": 1 } + ] + }, + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "sum(kube_job_status_active{namespace=\"foxhunt\"}) or vector(0)", + "legendFormat": "Active Jobs", + "instant": true + } + ] + }, + { + "id": 7, + "type": "stat", + "title": "Log Error Rate (/min)", + "gridPos": { "h": 4, "w": 4, "x": 20, "y": 1 }, + "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 5 }, + { "color": "red", "value": 20 } + ] + }, + "unit": "short", + "decimals": 1, + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "expr": "sum(count_over_time({namespace=\"foxhunt\"} |= \"ERROR\" [1m]))", + "legendFormat": "Errors/min", + "instant": true + } + ] + }, + { + "id": 10, + "type": "row", + "title": "Service Status", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, + "collapsed": false, + "panels": [] + }, + { + "id": 11, + "type": "stat", + "title": "api-gateway", + "gridPos": { "h": 3, "w": 3, "x": 0, "y": 6 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "green", "value": 1 } + ] + }, + "mappings": [ + { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } } + ], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"api-gateway.*\", phase=\"Running\"}) > 0 or vector(0)", + "legendFormat": "", + "instant": true + } + ] + }, + { + "id": 12, + "type": "stat", + "title": "trading-service", + "gridPos": { "h": 3, "w": 3, "x": 3, "y": 6 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "green", "value": 1 } + ] + }, + "mappings": [ + { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } } + ], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"trading-service.*\", phase=\"Running\"}) > 0 or vector(0)", + "legendFormat": "", + "instant": true + } + ] + }, + { + "id": 13, + "type": "stat", + "title": "ml-training-service", + "gridPos": { "h": 3, "w": 3, "x": 6, "y": 6 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "green", "value": 1 } + ] + }, + "mappings": [ + { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } } + ], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"ml-training-service.*\", phase=\"Running\"}) > 0 or vector(0)", + "legendFormat": "", + "instant": true + } + ] + }, + { + "id": 14, + "type": "stat", + "title": "backtesting-service", + "gridPos": { "h": 3, "w": 3, "x": 9, "y": 6 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "green", "value": 1 } + ] + }, + "mappings": [ + { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } } + ], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"backtesting-service.*\", phase=\"Running\"}) > 0 or vector(0)", + "legendFormat": "", + "instant": true + } + ] + }, + { + "id": 15, + "type": "stat", + "title": "trading-agent", + "gridPos": { "h": 3, "w": 3, "x": 12, "y": 6 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "green", "value": 1 } + ] + }, + "mappings": [ + { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } } + ], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"trading-agent.*\", phase=\"Running\"}) > 0 or vector(0)", + "legendFormat": "", + "instant": true + } + ] + }, + { + "id": 16, + "type": "stat", + "title": "broker-gateway", + "gridPos": { "h": 3, "w": 3, "x": 15, "y": 6 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "green", "value": 1 } + ] + }, + "mappings": [ + { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } } + ], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"broker-gateway.*\", phase=\"Running\"}) > 0 or vector(0)", + "legendFormat": "", + "instant": true + } + ] + }, + { + "id": 17, + "type": "stat", + "title": "data-acquisition", + "gridPos": { "h": 3, "w": 3, "x": 18, "y": 6 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "green", "value": 1 } + ] + }, + "mappings": [ + { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } } + ], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"data-acquisition.*\", phase=\"Running\"}) > 0 or vector(0)", + "legendFormat": "", + "instant": true + } + ] + }, + { + "id": 18, + "type": "stat", + "title": "web-gateway", + "gridPos": { "h": 3, "w": 3, "x": 21, "y": 6 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "green", "value": 1 } + ] + }, + "mappings": [ + { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } } + ], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"web-gateway.*\", phase=\"Running\"}) > 0 or vector(0)", + "legendFormat": "", + "instant": true + } + ] + }, + { + "id": 20, + "type": "row", + "title": "Resource Utilization", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 9 }, + "collapsed": false, + "panels": [] + }, + { + "id": 21, + "type": "gauge", + "title": "Cluster CPU %", + "gridPos": { "h": 6, "w": 6, "x": 0, "y": 10 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 70 }, + { "color": "red", "value": 90 } + ] + }, + "min": 0, + "max": 100, + "unit": "percent", + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "orientation": "auto" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "100 - (avg(rate(node_cpu_seconds_total{mode=\"idle\"}[$interval])) * 100)", + "legendFormat": "CPU", + "instant": true + } + ] + }, + { + "id": 22, + "type": "gauge", + "title": "Cluster Memory %", + "gridPos": { "h": 6, "w": 6, "x": 6, "y": 10 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 70 }, + { "color": "red", "value": 90 } + ] + }, + "min": 0, + "max": 100, + "unit": "percent", + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "orientation": "auto" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "(1 - (sum(node_memory_MemAvailable_bytes) / sum(node_memory_MemTotal_bytes))) * 100", + "legendFormat": "Memory", + "instant": true + } + ] + }, + { + "id": 23, + "type": "timeseries", + "title": "CPU by Node Pool", + "gridPos": { "h": 6, "w": 12, "x": 12, "y": 10 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 15, + "gradientMode": "scheme", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "none", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 1 + }, + "unit": "percentunit", + "min": 0, + "max": 1, + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[$interval]))", + "legendFormat": "{{ instance }}" + } + ] + }, + { + "id": 24, + "type": "timeseries", + "title": "Memory by Node Pool", + "gridPos": { "h": 6, "w": 12, "x": 0, "y": 16 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 15, + "gradientMode": "scheme", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "none", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 1 + }, + "unit": "bytes", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes", + "legendFormat": "{{ instance }}" + } + ] + }, + { + "id": 25, + "type": "timeseries", + "title": "Foxhunt Container CPU Usage", + "gridPos": { "h": 6, "w": 12, "x": 12, "y": 16 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 15, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "normal", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 1 + }, + "unit": "short", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "sum by (container) (rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", container=~\"api-gateway|trading-service|ml-training-service|backtesting-service|trading-agent-service|broker-gateway|data-acquisition-service|web-gateway\", image!=\"\"}[$interval]))", + "legendFormat": "{{ container }}" + } + ] + }, + { + "id": 30, + "type": "row", + "title": "Deployments & Restarts", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 22 }, + "collapsed": false, + "panels": [] + }, + { + "id": 31, + "type": "timeseries", + "title": "Pod Restarts (last 1h)", + "description": "Pod restart events indicating crashes or OOMKills", + "gridPos": { "h": 7, "w": 12, "x": 0, "y": 23 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "bars", + "lineInterpolation": "linear", + "fillOpacity": 80, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "normal", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 1 + }, + "unit": "short", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "increase(kube_pod_container_status_restarts_total{namespace=\"foxhunt\"}[1h]) > 0", + "legendFormat": "{{ pod }}" + } + ] + }, + { + "id": 32, + "type": "timeseries", + "title": "Deployment Replicas: Desired vs Available", + "gridPos": { "h": 7, "w": 12, "x": 12, "y": 23 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "stepAfter", + "fillOpacity": 0, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "none", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 2 + }, + "unit": "short", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["last"] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "kube_deployment_spec_replicas{namespace=\"foxhunt\"}", + "legendFormat": "{{ deployment }} desired" + }, + { + "refId": "B", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "kube_deployment_status_replicas_available{namespace=\"foxhunt\"}", + "legendFormat": "{{ deployment }} available" + } + ] + }, + { + "id": 40, + "type": "row", + "title": "Error Rate Trend", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 30 }, + "collapsed": false, + "panels": [] + }, + { + "id": 41, + "type": "timeseries", + "title": "Log Errors Over Time", + "gridPos": { "h": 7, "w": 24, "x": 0, "y": 31 }, + "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "bars", + "lineInterpolation": "linear", + "fillOpacity": 50, + "gradientMode": "scheme", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "normal", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 1 + }, + "unit": "short", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "expr": "sum by (app) (count_over_time({namespace=\"foxhunt\"} |= \"ERROR\" [$interval]))", + "legendFormat": "{{ app }}" + } + ] + } + ] +} \ No newline at end of file diff --git a/infra/k8s/monitoring/dashboards/foxhunt-services.json b/infra/k8s/monitoring/dashboards/foxhunt-services.json new file mode 100644 index 000000000..7cb11a9ac --- /dev/null +++ b/infra/k8s/monitoring/dashboards/foxhunt-services.json @@ -0,0 +1,675 @@ +{ + "id": null, + "uid": "foxhunt-services", + "title": "Foxhunt - Microservice Detail", + "description": "Per-service deep dive: CPU, memory, restarts, gRPC metrics, logs", + "tags": ["foxhunt", "production"], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "editable": true, + "refresh": "30s", + "time": { + "from": "now-1h", + "to": "now" + }, + "fiscalYearStartMonth": 0, + "liveNow": false, + "weekStart": "", + "templating": { + "list": [ + { + "name": "service", + "type": "custom", + "query": "api-gateway,trading-service,ml-training-service,backtesting-service,trading-agent-service,broker-gateway,data-acquisition-service,web-gateway", + "current": { + "selected": true, + "text": "api-gateway", + "value": "api-gateway" + }, + "multi": false, + "includeAll": false, + "sort": 0 + }, + { + "name": "interval", + "type": "interval", + "query": "1m,5m,15m,30m,1h", + "current": { + "selected": true, + "text": "5m", + "value": "5m" + }, + "auto": false + } + ] + }, + "panels": [ + { + "id": 1, + "type": "row", + "title": "Pod Status - $service", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "collapsed": false, + "panels": [] + }, + { + "id": 2, + "type": "stat", + "title": "Status", + "gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "green", "value": 1 } + ] + }, + "mappings": [ + { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } } + ], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"$service.*\", phase=\"Running\"}) > 0 or vector(0)", + "instant": true + } + ] + }, + { + "id": 3, + "type": "stat", + "title": "Replicas", + "gridPos": { "h": 4, "w": 4, "x": 4, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "blue", "value": null } + ] + }, + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "count(kube_pod_info{namespace=\"foxhunt\", pod=~\"$service.*\"})", + "instant": true + } + ] + }, + { + "id": 4, + "type": "stat", + "title": "Restarts (24h)", + "gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 1 }, + { "color": "red", "value": 5 } + ] + }, + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "sum(increase(kube_pod_container_status_restarts_total{namespace=\"foxhunt\", pod=~\"$service.*\"}[24h])) or vector(0)", + "instant": true + } + ] + }, + { + "id": 5, + "type": "stat", + "title": "Container Uptime", + "gridPos": { "h": 4, "w": 4, "x": 12, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "yellow", "value": 3600 }, + { "color": "green", "value": 86400 } + ] + }, + "unit": "s", + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "min(time() - kube_pod_start_time{namespace=\"foxhunt\", pod=~\"$service.*\"})", + "instant": true + } + ] + }, + { + "id": 6, + "type": "stat", + "title": "CPU Cores", + "gridPos": { "h": 4, "w": 4, "x": 16, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "blue", "value": null } + ] + }, + "unit": "short", + "decimals": 3, + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", container=~\"$service\", image!=\"\"}[$interval]))", + "instant": true + } + ] + }, + { + "id": 7, + "type": "stat", + "title": "Memory", + "gridPos": { "h": 4, "w": 4, "x": 20, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "blue", "value": null } + ] + }, + "unit": "bytes", + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", container=~\"$service\", image!=\"\"})", + "instant": true + } + ] + }, + { + "id": 10, + "type": "row", + "title": "Resource Usage Over Time", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, + "collapsed": false, + "panels": [] + }, + { + "id": 11, + "type": "timeseries", + "title": "CPU Usage", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 20, + "gradientMode": "scheme", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "none", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 2 + }, + "unit": "short", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", container=~\"$service\", image!=\"\"}[$interval]))", + "legendFormat": "{{ pod }}" + }, + { + "refId": "B", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "sum(kube_pod_container_resource_requests{namespace=\"foxhunt\", container=~\"$service\", resource=\"cpu\"})", + "legendFormat": "Request" + }, + { + "refId": "C", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "sum(kube_pod_container_resource_limits{namespace=\"foxhunt\", container=~\"$service\", resource=\"cpu\"})", + "legendFormat": "Limit" + } + ] + }, + { + "id": 12, + "type": "timeseries", + "title": "Memory Usage", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 20, + "gradientMode": "scheme", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "none", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 2 + }, + "unit": "bytes", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "sum by (pod) (container_memory_working_set_bytes{namespace=\"foxhunt\", container=~\"$service\", image!=\"\"})", + "legendFormat": "{{ pod }}" + }, + { + "refId": "B", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "sum(kube_pod_container_resource_requests{namespace=\"foxhunt\", container=~\"$service\", resource=\"memory\"})", + "legendFormat": "Request" + }, + { + "refId": "C", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "sum(kube_pod_container_resource_limits{namespace=\"foxhunt\", container=~\"$service\", resource=\"memory\"})", + "legendFormat": "Limit" + } + ] + }, + { + "id": 15, + "type": "row", + "title": "Restart History", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 }, + "collapsed": false, + "panels": [] + }, + { + "id": 16, + "type": "timeseries", + "title": "Pod Restart Count", + "gridPos": { "h": 6, "w": 12, "x": 0, "y": 15 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "bars", + "lineInterpolation": "linear", + "fillOpacity": 80, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "normal", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 1 + }, + "unit": "short", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "increase(kube_pod_container_status_restarts_total{namespace=\"foxhunt\", pod=~\"$service.*\"}[30m])", + "legendFormat": "{{ pod }}" + } + ] + }, + { + "id": 17, + "type": "timeseries", + "title": "Network I/O", + "gridPos": { "h": 6, "w": 12, "x": 12, "y": 15 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 10, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "none", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 1 + }, + "unit": "Bps", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "sum by (pod) (rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"$service.*\"}[$interval]))", + "legendFormat": "{{ pod }} RX" + }, + { + "refId": "B", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "sum by (pod) (rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"$service.*\"}[$interval]))", + "legendFormat": "{{ pod }} TX" + } + ] + }, + { + "id": 20, + "type": "row", + "title": "gRPC Metrics", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 21 }, + "collapsed": false, + "panels": [] + }, + { + "id": 21, + "type": "timeseries", + "title": "gRPC Request Rate", + "description": "gRPC requests per second by method and status code. Requires grpc_server_handled_total metric from service instrumentation.", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 22 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 15, + "gradientMode": "scheme", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "none", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 1 + }, + "unit": "reqps", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "sum by (grpc_method, grpc_code) (rate(grpc_server_handled_total{grpc_service=~\".*$service.*\"}[$interval]))", + "legendFormat": "{{ grpc_method }} [{{ grpc_code }}]" + } + ] + }, + { + "id": 22, + "type": "timeseries", + "title": "gRPC Latency (p50 / p95 / p99)", + "description": "gRPC server-side latency percentiles. Requires grpc_server_handling_seconds_bucket histogram.", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 22 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 10, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "none", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 2 + }, + "unit": "s", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "histogram_quantile(0.50, sum(rate(grpc_server_handling_seconds_bucket{grpc_service=~\".*$service.*\"}[$interval])) by (le))", + "legendFormat": "p50" + }, + { + "refId": "B", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "histogram_quantile(0.95, sum(rate(grpc_server_handling_seconds_bucket{grpc_service=~\".*$service.*\"}[$interval])) by (le))", + "legendFormat": "p95" + }, + { + "refId": "C", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{grpc_service=~\".*$service.*\"}[$interval])) by (le))", + "legendFormat": "p99" + } + ] + }, + { + "id": 30, + "type": "row", + "title": "Logs", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 30 }, + "collapsed": false, + "panels": [] + }, + { + "id": 31, + "type": "timeseries", + "title": "Log Volume by Level", + "gridPos": { "h": 6, "w": 24, "x": 0, "y": 31 }, + "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "bars", + "lineInterpolation": "linear", + "fillOpacity": 60, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "normal", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 1 + }, + "unit": "short", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "ERROR" }, + "properties": [{ "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } }] + }, + { + "matcher": { "id": "byName", "options": "WARN" }, + "properties": [{ "id": "color", "value": { "fixedColor": "yellow", "mode": "fixed" } }] + }, + { + "matcher": { "id": "byName", "options": "INFO" }, + "properties": [{ "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } }] + }, + { + "matcher": { "id": "byName", "options": "DEBUG" }, + "properties": [{ "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } }] + } + ] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "expr": "sum by (level) (count_over_time({app=\"$service\", namespace=\"foxhunt\"} | logfmt | level != \"\" [$interval]))", + "legendFormat": "{{ level }}" + } + ] + }, + { + "id": 32, + "type": "logs", + "title": "Recent Error Logs", + "gridPos": { "h": 10, "w": 24, "x": 0, "y": 37 }, + "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "options": { + "showTime": true, + "showLabels": true, + "showCommonLabels": false, + "wrapLogMessage": true, + "prettifyLogMessage": false, + "enableLogDetails": true, + "sortOrder": "Descending", + "dedupStrategy": "none" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "expr": "{app=\"$service\", namespace=\"foxhunt\"} |= \"ERROR\"", + "maxLines": 200 + } + ] + } + ] +} \ No newline at end of file diff --git a/infra/k8s/monitoring/dashboards/import.sh b/infra/k8s/monitoring/dashboards/import.sh index b7d041f3c..d1d33bcde 100755 --- a/infra/k8s/monitoring/dashboards/import.sh +++ b/infra/k8s/monitoring/dashboards/import.sh @@ -4,19 +4,16 @@ set -euo pipefail # Import Grafana dashboards to Scaleway Cockpit via the Grafana API. # # Required environment variables: -# GRAFANA_URL - Grafana base URL (e.g. https://xxxx.grafana.scw.cloud) -# GRAFANA_PASS - Password for foxhunt-admin user +# GRAFANA_URL - Grafana base URL (default: https://grafana.fxhnt.ai) +# GRAFANA_PASS - Password for admin user # # Usage: -# GRAFANA_URL="https://..." GRAFANA_PASS="..." ./import.sh +# GRAFANA_PASS="Welcome01" ./import.sh +# GRAFANA_URL="https://grafana.fxhnt.ai" GRAFANA_PASS="Welcome01" ./import.sh SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -GRAFANA_USER="${GRAFANA_USER:-foxhunt-admin}" - -if [[ -z "${GRAFANA_URL:-}" ]]; then - echo "ERROR: GRAFANA_URL is not set" >&2 - exit 1 -fi +GRAFANA_USER="${GRAFANA_USER:-admin}" +GRAFANA_URL="${GRAFANA_URL:-https://grafana.fxhnt.ai}" if [[ -z "${GRAFANA_PASS:-}" ]]; then echo "ERROR: GRAFANA_PASS is not set" >&2 @@ -57,7 +54,12 @@ failed=0 for dashboard in "${SCRIPT_DIR}/cluster-overview.json" \ "${SCRIPT_DIR}/gpu-overview.json" \ - "${SCRIPT_DIR}/gitlab-services.json"; do + "${SCRIPT_DIR}/gitlab-services.json" \ + "${SCRIPT_DIR}/foxhunt-overview.json" \ + "${SCRIPT_DIR}/foxhunt-services.json" \ + "${SCRIPT_DIR}/foxhunt-gpu-training.json" \ + "${SCRIPT_DIR}/foxhunt-infrastructure.json" \ + "${SCRIPT_DIR}/foxhunt-logs.json"; do if [[ -f "$dashboard" ]]; then if ! import_dashboard "$dashboard"; then failed=$((failed + 1)) diff --git a/infra/k8s/monitoring/kube-state-metrics.yaml b/infra/k8s/monitoring/kube-state-metrics.yaml new file mode 100644 index 000000000..9b24d6aa2 --- /dev/null +++ b/infra/k8s/monitoring/kube-state-metrics.yaml @@ -0,0 +1,164 @@ +# kube-state-metrics — Kubernetes object state metrics +# Replaces k8s-monitoring chart's kube-state-metrics after Cockpit removal +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-state-metrics + namespace: foxhunt + labels: + app.kubernetes.io/name: kube-state-metrics +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kube-state-metrics + labels: + app.kubernetes.io/name: kube-state-metrics +rules: + - apiGroups: [""] + resources: + - configmaps + - secrets + - nodes + - pods + - services + - serviceaccounts + - resourcequotas + - replicationcontrollers + - limitranges + - persistentvolumeclaims + - persistentvolumes + - namespaces + - endpoints + verbs: ["list", "watch"] + - apiGroups: ["apps"] + resources: + - statefulsets + - daemonsets + - deployments + - replicasets + verbs: ["list", "watch"] + - apiGroups: ["batch"] + resources: + - cronjobs + - jobs + verbs: ["list", "watch"] + - apiGroups: ["autoscaling"] + resources: + - horizontalpodautoscalers + verbs: ["list", "watch"] + - apiGroups: ["networking.k8s.io"] + resources: + - networkpolicies + - ingresses + verbs: ["list", "watch"] + - apiGroups: ["policy"] + resources: + - poddisruptionbudgets + verbs: ["list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: + - storageclasses + - volumeattachments + verbs: ["list", "watch"] + - apiGroups: ["coordination.k8s.io"] + resources: + - leases + verbs: ["list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kube-state-metrics + labels: + app.kubernetes.io/name: kube-state-metrics +subjects: + - kind: ServiceAccount + name: kube-state-metrics + namespace: foxhunt +roleRef: + kind: ClusterRole + name: kube-state-metrics + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kube-state-metrics + namespace: foxhunt + labels: + app.kubernetes.io/name: kube-state-metrics +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: kube-state-metrics + template: + metadata: + labels: + app.kubernetes.io/name: kube-state-metrics + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "8080" + spec: + serviceAccountName: kube-state-metrics + nodeSelector: + k8s.scaleway.com/pool-name: gitlab + tolerations: + - key: gitlab + operator: Equal + value: "true" + effect: NoSchedule + containers: + - name: kube-state-metrics + image: registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.14.0 + ports: + - containerPort: 8080 + name: http + - containerPort: 8081 + name: telemetry + resources: + requests: + cpu: 25m + memory: 64Mi + limits: + cpu: 100m + memory: 128Mi + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 65534 + livenessProbe: + httpGet: + path: /healthz + port: 8080 + initialDelaySeconds: 5 + timeoutSeconds: 5 + readinessProbe: + httpGet: + path: / + port: 8080 + initialDelaySeconds: 5 + timeoutSeconds: 5 +--- +apiVersion: v1 +kind: Service +metadata: + name: kube-state-metrics + namespace: foxhunt + labels: + app.kubernetes.io/name: kube-state-metrics + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "8080" +spec: + selector: + app.kubernetes.io/name: kube-state-metrics + ports: + - port: 8080 + targetPort: 8080 + name: http + - port: 8081 + targetPort: 8081 + name: telemetry diff --git a/infra/k8s/monitoring/node-exporter.yaml b/infra/k8s/monitoring/node-exporter.yaml new file mode 100644 index 000000000..50457c26c --- /dev/null +++ b/infra/k8s/monitoring/node-exporter.yaml @@ -0,0 +1,104 @@ +# Prometheus Node Exporter — host-level metrics (CPU, memory, disk, network) +# Replaces k8s-monitoring chart's node-exporter after Cockpit removal +apiVersion: v1 +kind: ServiceAccount +metadata: + name: node-exporter + namespace: foxhunt + labels: + app.kubernetes.io/name: node-exporter +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: node-exporter + namespace: foxhunt + labels: + app.kubernetes.io/name: node-exporter +spec: + selector: + matchLabels: + app.kubernetes.io/name: node-exporter + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + template: + metadata: + labels: + app.kubernetes.io/name: node-exporter + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9100" + spec: + serviceAccountName: node-exporter + hostPID: true + hostNetwork: true + tolerations: + - effect: NoSchedule + operator: Exists + containers: + - name: node-exporter + image: quay.io/prometheus/node-exporter:v1.9.0 + args: + - --path.procfs=/host/proc + - --path.sysfs=/host/sys + - --path.rootfs=/host/root + - --collector.filesystem.mount-points-exclude=^/(dev|proc|sys|var/lib/docker/.+|var/lib/kubelet/.+)($|/) + - --collector.filesystem.fs-types-exclude=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ + - --web.listen-address=:9100 + ports: + - containerPort: 9100 + name: metrics + hostPort: 9100 + resources: + requests: + cpu: 25m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + volumeMounts: + - name: proc + mountPath: /host/proc + readOnly: true + - name: sys + mountPath: /host/sys + readOnly: true + - name: root + mountPath: /host/root + readOnly: true + mountPropagation: HostToContainer + volumes: + - name: proc + hostPath: + path: /proc + - name: sys + hostPath: + path: /sys + - name: root + hostPath: + path: / +--- +apiVersion: v1 +kind: Service +metadata: + name: node-exporter + namespace: foxhunt + labels: + app.kubernetes.io/name: node-exporter + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9100" +spec: + type: ClusterIP + clusterIP: None + selector: + app.kubernetes.io/name: node-exporter + ports: + - port: 9100 + targetPort: 9100 + name: metrics diff --git a/infra/k8s/monitoring/prometheus-values.yaml b/infra/k8s/monitoring/prometheus-values.yaml new file mode 100644 index 000000000..6b44f3074 --- /dev/null +++ b/infra/k8s/monitoring/prometheus-values.yaml @@ -0,0 +1,166 @@ +# Prometheus override values for GitLab Helm chart +# Removes Cockpit remote_write, adds comprehensive cluster scrape configs +# +# Apply with: +# helm upgrade gitlab gitlab/gitlab -n foxhunt --reuse-values \ +# -f infra/k8s/monitoring/prometheus-values.yaml +prometheus: + server: + # Remove Cockpit remote_write — all metrics stay local + remoteWrite: [] + + # Extra scrape configs for full cluster observability + # NOTE: this is at prometheus.extraScrapeConfigs (chart root), NOT prometheus.server.extraScrapeConfigs + extraScrapeConfigs: | + # --- Host metrics from node-exporter --- + - job_name: node-exporter + kubernetes_sd_configs: + - role: endpoints + namespaces: + names: [foxhunt] + relabel_configs: + - source_labels: [__meta_kubernetes_service_label_app_kubernetes_io_name] + action: keep + regex: node-exporter + - source_labels: [__meta_kubernetes_endpoints_name] + action: keep + regex: node-exporter + - source_labels: [__meta_kubernetes_pod_node_name] + target_label: node + + # --- Kubernetes object state --- + - job_name: kube-state-metrics + kubernetes_sd_configs: + - role: endpoints + namespaces: + names: [foxhunt] + relabel_configs: + - source_labels: [__meta_kubernetes_service_label_app_kubernetes_io_name] + action: keep + regex: kube-state-metrics + - source_labels: [__meta_kubernetes_endpoint_port_name] + action: keep + regex: http + + # --- Kubelet metrics (container resource usage) --- + - job_name: kubelet + scheme: https + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecure_skip_verify: true + kubernetes_sd_configs: + - role: node + relabel_configs: + - target_label: __address__ + replacement: kubernetes.default.svc:443 + - source_labels: [__meta_kubernetes_node_name] + regex: (.+) + target_label: __metrics_path__ + replacement: /api/v1/nodes/$1/proxy/metrics + + # --- cAdvisor (container CPU, memory, network, filesystem) --- + - job_name: cadvisor + scheme: https + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + insecure_skip_verify: true + kubernetes_sd_configs: + - role: node + relabel_configs: + - target_label: __address__ + replacement: kubernetes.default.svc:443 + - source_labels: [__meta_kubernetes_node_name] + regex: (.+) + target_label: __metrics_path__ + replacement: /api/v1/nodes/$1/proxy/metrics/cadvisor + metric_relabel_configs: + - source_labels: [namespace] + action: keep + regex: (foxhunt|monitoring|kube-system) + + # --- GPU metrics from NVIDIA DCGM Exporter --- + - job_name: dcgm-exporter + kubernetes_sd_configs: + - role: endpoints + namespaces: + names: [kube-system, monitoring] + relabel_configs: + - source_labels: [__meta_kubernetes_service_name] + action: keep + regex: .*dcgm.* + - source_labels: [__meta_kubernetes_pod_node_name] + target_label: node + + # --- Foxhunt services (app metrics ports) --- + - job_name: foxhunt-services + kubernetes_sd_configs: + - role: endpoints + namespaces: + names: [foxhunt] + relabel_configs: + - source_labels: [__meta_kubernetes_service_label_app_kubernetes_io_part_of] + action: keep + regex: foxhunt + - source_labels: [__meta_kubernetes_endpoint_port_name] + action: keep + regex: metrics + - source_labels: [__meta_kubernetes_service_name] + target_label: service + - source_labels: [__meta_kubernetes_namespace] + target_label: namespace + + # --- General prometheus.io annotation-based scraping --- + - job_name: prometheus-annotated-pods + kubernetes_sd_configs: + - role: pod + relabel_configs: + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] + action: keep + regex: "true" + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scheme] + action: replace + regex: (https?) + target_label: __scheme__ + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] + action: replace + regex: (.+) + target_label: __metrics_path__ + - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] + action: replace + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + target_label: namespace + - source_labels: [__meta_kubernetes_pod_name] + target_label: pod + - source_labels: [__meta_kubernetes_pod_node_name] + target_label: node + + # --- Loki metrics --- + - job_name: loki + static_configs: + - targets: ['loki.foxhunt.svc.cluster.local:3100'] + + # --- Tempo metrics --- + - job_name: tempo + static_configs: + - targets: ['tempo.foxhunt.svc.cluster.local:3200'] + + # --- Promtail metrics --- + - job_name: promtail + kubernetes_sd_configs: + - role: pod + namespaces: + names: [foxhunt] + relabel_configs: + - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name] + action: keep + regex: promtail + - source_labels: [__meta_kubernetes_pod_ip] + target_label: __address__ + replacement: $1:9080 diff --git a/infra/k8s/monitoring/promtail.yaml b/infra/k8s/monitoring/promtail.yaml index 8c9e7377c..16c4b354f 100644 --- a/infra/k8s/monitoring/promtail.yaml +++ b/infra/k8s/monitoring/promtail.yaml @@ -49,19 +49,22 @@ data: clients: - url: http://loki.foxhunt.svc.cluster.local:3100/loki/api/v1/push scrape_configs: - - job_name: kubernetes-pods + # Foxhunt application pods (structured Rust logs) + - job_name: foxhunt-pods kubernetes_sd_configs: - role: pod relabel_configs: - # Only scrape pods in foxhunt namespace - source_labels: [__meta_kubernetes_namespace] action: keep regex: foxhunt - # Drop gitlab/prometheus/grafana pods (too noisy) + # Drop noisy infra pods (gitlab internals, prometheus, grafana) - source_labels: [__meta_kubernetes_pod_label_app] action: drop - regex: (gitlab|prometheus|grafana) - # Map to containerd log path: /var/log/pods/__//*.log + regex: (gitlab|prometheus|grafana|promtail) + - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name] + action: drop + regex: (promtail|node-exporter|kube-state-metrics) + # Map to containerd log path - source_labels: [__meta_kubernetes_pod_uid, __meta_kubernetes_pod_container_name] separator: / target_label: __path__ @@ -79,8 +82,9 @@ data: target_label: job_type - source_labels: [__meta_kubernetes_pod_label_foxhunt_model] target_label: model + - source_labels: [__meta_kubernetes_pod_node_name] + target_label: node pipeline_stages: - # Try to parse JSON logs (from structured logging) - json: expressions: level: level @@ -90,7 +94,6 @@ data: - labels: level: target: - # Fallback: if not JSON, keep as plain text - match: selector: '{level=""}' stages: @@ -98,6 +101,96 @@ data: expression: '(?PTRACE|DEBUG|INFO|WARN|ERROR)' - labels: level: + + # Kapsule system pods (kube-system, kube-node-lease, kube-public) + - job_name: kapsule-system + kubernetes_sd_configs: + - role: pod + relabel_configs: + - source_labels: [__meta_kubernetes_namespace] + action: keep + regex: (kube-system|kube-node-lease|kube-public) + # Map to containerd log path + - source_labels: [__meta_kubernetes_pod_uid, __meta_kubernetes_pod_container_name] + separator: / + target_label: __path__ + replacement: /var/log/pods/*$1/*.log + - source_labels: [__meta_kubernetes_namespace] + target_label: namespace + - source_labels: [__meta_kubernetes_pod_name] + target_label: pod + - source_labels: [__meta_kubernetes_pod_container_name] + target_label: container + - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name] + target_label: app + - source_labels: [__meta_kubernetes_pod_label_app] + target_label: kube_app + - source_labels: [__meta_kubernetes_pod_node_name] + target_label: node + pipeline_stages: + - match: + selector: '{namespace="kube-system"}' + stages: + - regex: + expression: '(?PTRACE|DEBUG|INFO|WARN|ERROR|warning|error|fatal)' + - labels: + level: + + # GitLab CI runner logs (foxhunt namespace, runner pods) + - job_name: gitlab-runners + kubernetes_sd_configs: + - role: pod + relabel_configs: + - source_labels: [__meta_kubernetes_namespace] + action: keep + regex: foxhunt + - source_labels: [__meta_kubernetes_pod_label_app] + action: keep + regex: gitlab-runner + - source_labels: [__meta_kubernetes_pod_uid, __meta_kubernetes_pod_container_name] + separator: / + target_label: __path__ + replacement: /var/log/pods/*$1/*.log + - source_labels: [__meta_kubernetes_namespace] + target_label: namespace + - source_labels: [__meta_kubernetes_pod_name] + target_label: pod + - source_labels: [__meta_kubernetes_pod_container_name] + target_label: container + - replacement: gitlab-runner + target_label: app + - source_labels: [__meta_kubernetes_pod_node_name] + target_label: node + pipeline_stages: + - regex: + expression: '(?PTRACE|DEBUG|INFO|WARN|ERROR|WARNING)' + - labels: + level: + + # Kubernetes node-level logs (kubelet, containerd) + - job_name: kapsule-node-logs + static_configs: + - targets: [localhost] + labels: + job: kapsule-nodes + __path__: /var/log/syslog + - targets: [localhost] + labels: + job: kapsule-nodes + __path__: /var/log/kern.log + - targets: [localhost] + labels: + job: kapsule-nodes + __path__: /var/log/kubelet.log + relabel_configs: + - source_labels: [__path__] + regex: .*/(.+)\.log + target_label: log_type + pipeline_stages: + - regex: + expression: '(?PTRACE|DEBUG|INFO|WARN|ERROR|warning|error|fatal)' + - labels: + level: --- apiVersion: apps/v1 kind: DaemonSet @@ -147,11 +240,11 @@ spec: fieldPath: spec.nodeName resources: requests: - cpu: 25m - memory: 32Mi + cpu: 50m + memory: 64Mi limits: - cpu: 100m - memory: 128Mi + cpu: 200m + memory: 256Mi securityContext: readOnlyRootFilesystem: true allowPrivilegeEscalation: false diff --git a/infra/modules/dns/main.tf b/infra/modules/dns/main.tf index 132232222..d6ccfe516 100644 --- a/infra/modules/dns/main.tf +++ b/infra/modules/dns/main.tf @@ -23,3 +23,11 @@ resource "scaleway_domain_record" "prometheus" { data = var.git_ip ttl = 300 } + +resource "scaleway_domain_record" "dashboard" { + dns_zone = var.dns_zone + name = "dashboard" + type = "A" + data = var.git_ip + ttl = 300 +}