diff --git a/infra/k8s/monitoring/dashboards/cluster-overview.json b/infra/k8s/monitoring/dashboards/cluster-overview.json index 24b74b30c..8d35c198f 100644 --- a/infra/k8s/monitoring/dashboards/cluster-overview.json +++ b/infra/k8s/monitoring/dashboards/cluster-overview.json @@ -1,4 +1,7 @@ { + "__inputs": [ + { "name": "DS_PROMETHEUS", "label": "Prometheus", "description": "", "type": "datasource", "pluginId": "prometheus" } + ], "id": null, "uid": "foxhunt-cluster-overview", "title": "K8s Cluster Overview", @@ -22,7 +25,7 @@ "type": "query", "datasource": { "type": "prometheus", - "uid": "cfegaydmscvswe" + "uid": "${DS_PROMETHEUS}" }, "query": "label_values(kube_pod_info, namespace)", "refresh": 2, @@ -41,7 +44,7 @@ "type": "query", "datasource": { "type": "prometheus", - "uid": "cfegaydmscvswe" + "uid": "${DS_PROMETHEUS}" }, "query": "label_values(node_uname_info, nodename)", "refresh": 2, @@ -71,7 +74,7 @@ "type": "stat", "title": "Total Nodes", "gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "thresholds": { @@ -97,7 +100,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "count(kube_node_info)", "legendFormat": "Nodes", "instant": true @@ -109,7 +112,7 @@ "type": "stat", "title": "Pods Running", "gridPos": { "h": 4, "w": 4, "x": 4, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "thresholds": { @@ -134,7 +137,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(kube_pod_status_phase{namespace=~\"$namespace\", phase=\"Running\"})", "legendFormat": "Running", "instant": true @@ -146,7 +149,7 @@ "type": "stat", "title": "Pods Pending", "gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "thresholds": { @@ -173,7 +176,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(kube_pod_status_phase{namespace=~\"$namespace\", phase=\"Pending\"}) or vector(0)", "legendFormat": "Pending", "instant": true @@ -185,7 +188,7 @@ "type": "stat", "title": "Pods Failed", "gridPos": { "h": 4, "w": 4, "x": 12, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "thresholds": { @@ -211,7 +214,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(kube_pod_status_phase{namespace=~\"$namespace\", phase=\"Failed\"}) or vector(0)", "legendFormat": "Failed", "instant": true @@ -223,7 +226,7 @@ "type": "gauge", "title": "Cluster CPU Usage %", "gridPos": { "h": 4, "w": 4, "x": 16, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "thresholds": { @@ -251,7 +254,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "100 - (avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m])) * 100)", "legendFormat": "CPU %", "instant": true @@ -263,7 +266,7 @@ "type": "gauge", "title": "Cluster Memory Usage %", "gridPos": { "h": 4, "w": 4, "x": 20, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "thresholds": { @@ -291,7 +294,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "(1 - (sum(node_memory_MemAvailable_bytes) / sum(node_memory_MemTotal_bytes))) * 100", "legendFormat": "Memory %", "instant": true @@ -311,7 +314,7 @@ "type": "timeseries", "title": "Per-Node CPU Usage", "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 }, - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { @@ -341,7 +344,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))", "legendFormat": "{{ instance }}" } @@ -352,7 +355,7 @@ "type": "timeseries", "title": "CPU Requests vs Limits vs Usage", "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 }, - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { @@ -380,19 +383,19 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(kube_pod_container_resource_requests{namespace=~\"$namespace\", resource=\"cpu\"})", "legendFormat": "Requests" }, { "refId": "B", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(kube_pod_container_resource_limits{namespace=~\"$namespace\", resource=\"cpu\"})", "legendFormat": "Limits" }, { "refId": "C", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\", image!=\"\"}[5m]))", "legendFormat": "Usage" } @@ -411,7 +414,7 @@ "type": "timeseries", "title": "Per-Node Memory Usage", "gridPos": { "h": 8, "w": 12, "x": 0, "y": 15 }, - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { @@ -439,7 +442,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes", "legendFormat": "{{ instance }}" } @@ -450,7 +453,7 @@ "type": "timeseries", "title": "Memory Requests vs Limits vs Usage", "gridPos": { "h": 8, "w": 12, "x": 12, "y": 15 }, - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { @@ -478,19 +481,19 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(kube_pod_container_resource_requests{namespace=~\"$namespace\", resource=\"memory\"})", "legendFormat": "Requests" }, { "refId": "B", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(kube_pod_container_resource_limits{namespace=~\"$namespace\", resource=\"memory\"})", "legendFormat": "Limits" }, { "refId": "C", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(container_memory_working_set_bytes{namespace=~\"$namespace\", image!=\"\"})", "legendFormat": "Usage" } @@ -509,7 +512,7 @@ "type": "timeseries", "title": "Node Filesystem Usage", "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 }, - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { @@ -539,7 +542,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "1 - (node_filesystem_avail_bytes{mountpoint=\"/\", fstype!=\"tmpfs\"} / node_filesystem_size_bytes{mountpoint=\"/\", fstype!=\"tmpfs\"})", "legendFormat": "{{ instance }}" } @@ -550,7 +553,7 @@ "type": "timeseries", "title": "Disk I/O Throughput", "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 }, - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { @@ -578,13 +581,13 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum by (instance) (rate(node_disk_read_bytes_total[5m]))", "legendFormat": "{{ instance }} read" }, { "refId": "B", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum by (instance) (rate(node_disk_written_bytes_total[5m]))", "legendFormat": "{{ instance }} write" } @@ -603,7 +606,7 @@ "type": "timeseries", "title": "Node Network Receive / Transmit", "gridPos": { "h": 8, "w": 12, "x": 0, "y": 33 }, - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { @@ -631,13 +634,13 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum by (instance) (rate(node_network_receive_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[5m]))", "legendFormat": "{{ instance }} rx" }, { "refId": "B", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum by (instance) (rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[5m]))", "legendFormat": "{{ instance }} tx" } @@ -648,7 +651,7 @@ "type": "timeseries", "title": "Pod Network I/O", "gridPos": { "h": 8, "w": 12, "x": 12, "y": 33 }, - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { @@ -676,13 +679,13 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum by (namespace) (rate(container_network_receive_bytes_total{namespace=~\"$namespace\", image!=\"\"}[5m]))", "legendFormat": "{{ namespace }} rx" }, { "refId": "B", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum by (namespace) (rate(container_network_transmit_bytes_total{namespace=~\"$namespace\", image!=\"\"}[5m]))", "legendFormat": "{{ namespace }} tx" } diff --git a/infra/k8s/monitoring/dashboards/foxhunt-ci-pipelines.json b/infra/k8s/monitoring/dashboards/foxhunt-ci-pipelines.json index 246005b83..f3654fdff 100644 --- a/infra/k8s/monitoring/dashboards/foxhunt-ci-pipelines.json +++ b/infra/k8s/monitoring/dashboards/foxhunt-ci-pipelines.json @@ -1,4 +1,7 @@ { + "__inputs": [ + { "name": "DS_PROMETHEUS", "label": "Prometheus", "description": "", "type": "datasource", "pluginId": "prometheus" } + ], "id": null, "uid": "foxhunt-ci-pipelines", "title": "Foxhunt CI/CD Pipelines", @@ -46,7 +49,7 @@ "title": "Pipeline Creation Rate", "description": "Pipelines created per second (averaged over $interval)", "gridPos": { "h": 6, "w": 6, "x": 0, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "unit": "ops", @@ -73,7 +76,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(rate(gitlab_ci_pipeline_processing_events_total[$interval]))", "legendFormat": "pipelines/s", "instant": true @@ -86,7 +89,7 @@ "title": "Active CI Jobs", "description": "Current number of active CI jobs", "gridPos": { "h": 6, "w": 6, "x": 6, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "unit": "short", @@ -114,7 +117,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(gitlab_ci_active_jobs_sum)", "legendFormat": "active jobs", "instant": true @@ -127,7 +130,7 @@ "title": "Pipeline Failure Rate", "description": "Pipeline failures per second (averaged over $interval)", "gridPos": { "h": 6, "w": 6, "x": 12, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "unit": "ops", @@ -155,7 +158,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(rate(gitlab_ci_pipeline_failure_reasons[$interval]))", "legendFormat": "failures/s", "instant": true @@ -168,7 +171,7 @@ "title": "Queue Operations Rate", "description": "CI queue operations per second (averaged over $interval)", "gridPos": { "h": 6, "w": 6, "x": 18, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "unit": "ops", @@ -195,7 +198,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(rate(gitlab_ci_queue_operations_total[$interval]))", "legendFormat": "queue ops/s", "instant": true @@ -217,7 +220,7 @@ "title": "Pipeline Creation Duration", "description": "Time to create a pipeline (P50, P90, P99)", "gridPos": { "h": 8, "w": 8, "x": 0, "y": 8 }, - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { @@ -264,19 +267,19 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.50, sum(rate(gitlab_ci_pipeline_creation_duration_seconds_bucket[$interval])) by (le))", "legendFormat": "P50" }, { "refId": "B", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.90, sum(rate(gitlab_ci_pipeline_creation_duration_seconds_bucket[$interval])) by (le))", "legendFormat": "P90" }, { "refId": "C", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.99, sum(rate(gitlab_ci_pipeline_creation_duration_seconds_bucket[$interval])) by (le))", "legendFormat": "P99" } @@ -288,7 +291,7 @@ "title": "Pipeline Size (Builds per Pipeline)", "description": "Number of builds/jobs per pipeline (P50, P90, P99)", "gridPos": { "h": 8, "w": 8, "x": 8, "y": 8 }, - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { @@ -335,19 +338,19 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.50, sum(rate(gitlab_ci_pipeline_size_builds_bucket[$interval])) by (le))", "legendFormat": "P50" }, { "refId": "B", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.90, sum(rate(gitlab_ci_pipeline_size_builds_bucket[$interval])) by (le))", "legendFormat": "P90" }, { "refId": "C", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.99, sum(rate(gitlab_ci_pipeline_size_builds_bucket[$interval])) by (le))", "legendFormat": "P99" } @@ -359,7 +362,7 @@ "title": "Active Jobs Distribution", "description": "Heatmap of active CI job counts over time", "gridPos": { "h": 8, "w": 8, "x": 16, "y": 8 }, - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { @@ -393,7 +396,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(increase(gitlab_ci_active_jobs_bucket[$interval])) by (le)", "legendFormat": "{{le}}", "format": "heatmap" @@ -415,7 +418,7 @@ "title": "Pipeline Failure Reasons", "description": "Breakdown of pipeline failure reasons over the selected time range", "gridPos": { "h": 8, "w": 8, "x": 0, "y": 17 }, - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "unit": "short", @@ -474,7 +477,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum by (reason) (increase(gitlab_ci_pipeline_failure_reasons[$__range]))", "legendFormat": "{{reason}}", "instant": true, @@ -488,7 +491,7 @@ "title": "Job Failure Reasons", "description": "Breakdown of individual job failure reasons over the selected time range", "gridPos": { "h": 8, "w": 8, "x": 8, "y": 17 }, - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "unit": "short", @@ -547,7 +550,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum by (failure_reason) (increase(gitlab_ci_job_failure_reasons[$__range]))", "legendFormat": "{{failure_reason}}", "instant": true, @@ -561,7 +564,7 @@ "title": "Build Trace Errors", "description": "Rate of build trace errors over time", "gridPos": { "h": 8, "w": 8, "x": 16, "y": 17 }, - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { @@ -595,7 +598,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(rate(gitlab_ci_build_trace_errors_total[$interval]))", "legendFormat": "trace errors/s" } @@ -616,7 +619,7 @@ "title": "Runner Authentication Success", "description": "Rate of successful runner authentication attempts", "gridPos": { "h": 8, "w": 8, "x": 0, "y": 26 }, - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { @@ -644,7 +647,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(rate(gitlab_ci_runner_authentication_success_total[$interval]))", "legendFormat": "auth success/s" } @@ -656,7 +659,7 @@ "title": "Trace Operations & Volume", "description": "CI trace operations rate and data volume", "gridPos": { "h": 8, "w": 8, "x": 8, "y": 26 }, - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { @@ -692,13 +695,13 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(rate(gitlab_ci_trace_operations_total[$interval]))", "legendFormat": "trace ops/s" }, { "refId": "B", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(rate(gitlab_ci_trace_bytes_total[$interval]))", "legendFormat": "trace bytes/s" } @@ -710,7 +713,7 @@ "title": "Artifacts Cleanup", "description": "Rate and volume of destroyed job artifacts", "gridPos": { "h": 8, "w": 8, "x": 16, "y": 26 }, - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { @@ -750,13 +753,13 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(rate(destroyed_job_artifacts_count_total[$interval]))", "legendFormat": "artifacts destroyed/s" }, { "refId": "B", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(rate(destroyed_job_artifacts_bytes_total[$interval]))", "legendFormat": "bytes destroyed/s" } @@ -777,7 +780,7 @@ "title": "Training Jobs: Created vs Completed vs Failed", "description": "K8s training job lifecycle in the foxhunt namespace", "gridPos": { "h": 8, "w": 8, "x": 0, "y": 35 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { @@ -824,19 +827,19 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "count(kube_job_info{namespace=\"foxhunt\", job_name=~\".*train.*\"})", "legendFormat": "Created" }, { "refId": "B", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "count(kube_job_status_succeeded{namespace=\"foxhunt\", job_name=~\".*train.*\"} == 1)", "legendFormat": "Succeeded" }, { "refId": "C", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "count(kube_job_status_failed{namespace=\"foxhunt\", job_name=~\".*train.*\"} > 0)", "legendFormat": "Failed" } @@ -848,7 +851,7 @@ "title": "Training Job Duration", "description": "Age of active training jobs (time since creation)", "gridPos": { "h": 8, "w": 8, "x": 8, "y": 35 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { @@ -883,7 +886,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "time() - kube_job_created{namespace=\"foxhunt\", job_name=~\".*train.*\"} and on(job_name, namespace) kube_job_status_succeeded{namespace=\"foxhunt\"} == 0 and on(job_name, namespace) (kube_job_status_failed{namespace=\"foxhunt\"} == 0 or absent(kube_job_status_failed{namespace=\"foxhunt\"}))", "legendFormat": "{{job_name}}" } @@ -895,7 +898,7 @@ "title": "Training Pod CPU Usage", "description": "CPU usage of training pods in the foxhunt namespace", "gridPos": { "h": 8, "w": 4, "x": 16, "y": 35 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { @@ -923,7 +926,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\".*train.*\", container!=\"POD\", container!=\"\"}[$interval]))", "legendFormat": "{{pod}}" } @@ -935,7 +938,7 @@ "title": "Training Pod Memory", "description": "Memory working set of training pods in the foxhunt namespace", "gridPos": { "h": 8, "w": 4, "x": 20, "y": 35 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { @@ -963,7 +966,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum by (pod) (container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\".*train.*\", container!=\"POD\", container!=\"\"})", "legendFormat": "{{pod}}" } diff --git a/infra/k8s/monitoring/dashboards/foxhunt-cockpit.json b/infra/k8s/monitoring/dashboards/foxhunt-cockpit.json new file mode 100644 index 000000000..6a3281fb0 --- /dev/null +++ b/infra/k8s/monitoring/dashboards/foxhunt-cockpit.json @@ -0,0 +1,931 @@ +{ + "__inputs": [ + { "name": "DS_PROMETHEUS", "label": "Prometheus", "description": "", "type": "datasource", "pluginId": "prometheus" }, + { "name": "DS_LOKI", "label": "Loki", "description": "", "type": "datasource", "pluginId": "loki" } + ], + "id": null, + "uid": "foxhunt-cockpit", + "title": "Foxhunt - Cockpit", + "description": "Single-page overview: cluster health, all services, GPU/training, resources, errors, live logs", + "tags": ["foxhunt", "production"], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "editable": true, + "refresh": "30s", + "time": { "from": "now-1h", "to": "now" }, + "fiscalYearStartMonth": 0, + "liveNow": false, + "weekStart": "", + "templating": { + "list": [ + { + "name": "interval", + "type": "interval", + "query": "1m,5m,15m,30m,1h", + "current": { "selected": true, "text": "5m", "value": "5m" }, + "auto": false + } + ] + }, + "panels": [ + { + "id": 1, + "type": "row", + "title": "Health Bar", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "collapsed": false, + "panels": [] + }, + { + "id": 2, + "type": "stat", + "title": "Nodes Ready", + "gridPos": { "h": 3, "w": 4, "x": 0, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] }, + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "sum(kube_node_status_condition{condition=\"Ready\",status=\"true\"})", + "instant": true + } + ] + }, + { + "id": 3, + "type": "stat", + "title": "Pods Running", + "gridPos": { "h": 3, "w": 4, "x": 4, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "thresholds": { "mode": "absolute", "steps": [{ "color": "yellow", "value": null }, { "color": "green", "value": 1 }] }, + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\",phase=\"Running\"})", + "instant": true + } + ] + }, + { + "id": 4, + "type": "stat", + "title": "Pods Not Ready", + "gridPos": { "h": 3, "w": 4, "x": 8, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 1 }, { "color": "red", "value": 3 }] }, + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\",phase=~\"Pending|Failed|Unknown\"}) or vector(0)", + "instant": true + } + ] + }, + { + "id": 5, + "type": "stat", + "title": "Training Jobs", + "gridPos": { "h": 3, "w": 4, "x": 12, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "thresholds": { "mode": "absolute", "steps": [{ "color": "blue", "value": null }] }, + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "sum(kube_job_status_active{namespace=\"foxhunt\"}) or vector(0)", + "instant": true + } + ] + }, + { + "id": 6, + "type": "stat", + "title": "GPU Available", + "gridPos": { "h": 3, "w": 4, "x": 16, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] }, + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "sum(kube_node_status_allocatable{resource=\"nvidia_com_gpu\"}) or vector(0)", + "instant": true + } + ] + }, + { + "id": 7, + "type": "stat", + "title": "Errors /min", + "gridPos": { "h": 3, "w": 4, "x": 20, "y": 1 }, + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "fieldConfig": { + "defaults": { + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 5 }, { "color": "red", "value": 20 }] }, + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "expr": "sum(count_over_time({namespace=\"foxhunt\"} |= \"ERROR\" [1m]))", + "instant": true + } + ] + }, + + { + "id": 8, + "type": "row", + "title": "Service Status", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 4 }, + "collapsed": false, + "panels": [] + }, + { + "id": 9, + "type": "stat", + "title": "api-gateway", + "gridPos": { "h": 3, "w": 3, "x": 0, "y": 5 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] }, + "mappings": [{ "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center" + }, + "links": [{ "title": "Service Detail", "url": "/d/foxhunt-services?var-service=api-gateway", "targetBlank": false }], + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"api-gateway.*\", phase=\"Running\"}) > 0 or vector(0)", + "instant": true + } + ] + }, + { + "id": 10, + "type": "stat", + "title": "trading-service", + "gridPos": { "h": 3, "w": 3, "x": 3, "y": 5 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] }, + "mappings": [{ "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center" + }, + "links": [{ "title": "Service Detail", "url": "/d/foxhunt-services?var-service=trading-service", "targetBlank": false }], + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"trading-service.*\", phase=\"Running\"}) > 0 or vector(0)", + "instant": true + } + ] + }, + { + "id": 11, + "type": "stat", + "title": "ml-training", + "gridPos": { "h": 3, "w": 3, "x": 6, "y": 5 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] }, + "mappings": [{ "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center" + }, + "links": [{ "title": "Service Detail", "url": "/d/foxhunt-services?var-service=ml-training-service", "targetBlank": false }], + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"ml-training-service.*\", phase=\"Running\"}) > 0 or vector(0)", + "instant": true + } + ] + }, + { + "id": 12, + "type": "stat", + "title": "backtesting", + "gridPos": { "h": 3, "w": 3, "x": 9, "y": 5 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] }, + "mappings": [{ "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center" + }, + "links": [{ "title": "Service Detail", "url": "/d/foxhunt-services?var-service=backtesting-service", "targetBlank": false }], + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"backtesting-service.*\", phase=\"Running\"}) > 0 or vector(0)", + "instant": true + } + ] + }, + { + "id": 13, + "type": "stat", + "title": "trading-agent", + "gridPos": { "h": 3, "w": 3, "x": 12, "y": 5 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] }, + "mappings": [{ "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center" + }, + "links": [{ "title": "Service Detail", "url": "/d/foxhunt-services?var-service=trading-agent-service", "targetBlank": false }], + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"trading-agent.*\", phase=\"Running\"}) > 0 or vector(0)", + "instant": true + } + ] + }, + { + "id": 14, + "type": "stat", + "title": "broker-gw", + "gridPos": { "h": 3, "w": 3, "x": 15, "y": 5 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] }, + "mappings": [{ "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center" + }, + "links": [{ "title": "Service Detail", "url": "/d/foxhunt-services?var-service=broker-gateway", "targetBlank": false }], + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"broker-gateway.*\", phase=\"Running\"}) > 0 or vector(0)", + "instant": true + } + ] + }, + { + "id": 15, + "type": "stat", + "title": "data-acq", + "gridPos": { "h": 3, "w": 3, "x": 18, "y": 5 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] }, + "mappings": [{ "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center" + }, + "links": [{ "title": "Service Detail", "url": "/d/foxhunt-services?var-service=data-acquisition-service", "targetBlank": false }], + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"data-acquisition.*\", phase=\"Running\"}) > 0 or vector(0)", + "instant": true + } + ] + }, + { + "id": 16, + "type": "stat", + "title": "web-gateway", + "gridPos": { "h": 3, "w": 3, "x": 21, "y": 5 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] }, + "mappings": [{ "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center" + }, + "links": [{ "title": "Service Detail", "url": "/d/foxhunt-services?var-service=web-gateway", "targetBlank": false }], + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"web-gateway.*\", phase=\"Running\"}) > 0 or vector(0)", + "instant": true + } + ] + }, + + { + "id": 17, + "type": "row", + "title": "Resources", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 8 }, + "collapsed": false, + "panels": [] + }, + { + "id": 18, + "type": "gauge", + "title": "Cluster CPU %", + "gridPos": { "h": 5, "w": 4, "x": 0, "y": 9 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 70 }, { "color": "red", "value": 90 }] }, + "unit": "percent", + "min": 0, + "max": 100, + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "100 * (1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[$interval])))", + "instant": true + } + ] + }, + { + "id": 19, + "type": "gauge", + "title": "Cluster Memory %", + "gridPos": { "h": 5, "w": 4, "x": 4, "y": 9 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 75 }, { "color": "red", "value": 90 }] }, + "unit": "percent", + "min": 0, + "max": 100, + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "100 * (1 - sum(node_memory_MemAvailable_bytes) / sum(node_memory_MemTotal_bytes))", + "instant": true + } + ] + }, + { + "id": 20, + "type": "gauge", + "title": "Cluster Disk %", + "gridPos": { "h": 5, "w": 4, "x": 8, "y": 9 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 75 }, { "color": "red", "value": 90 }] }, + "unit": "percent", + "min": 0, + "max": 100, + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "100 * (1 - sum(node_filesystem_avail_bytes{mountpoint=\"/\",fstype!=\"tmpfs\"}) / sum(node_filesystem_size_bytes{mountpoint=\"/\",fstype!=\"tmpfs\"}))", + "instant": true + } + ] + }, + { + "id": 21, + "type": "timeseries", + "title": "CPU by Service", + "gridPos": { "h": 5, "w": 12, "x": 12, "y": 9 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 30, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "normal", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 1 + }, + "unit": "short", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "desc" } }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\",container!=\"\",container!=\"POD\"}[$interval]))", + "legendFormat": "{{ pod }}" + } + ] + }, + + { + "id": 22, + "type": "row", + "title": "GPU & Training", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 }, + "collapsed": false, + "panels": [] + }, + { + "id": 23, + "type": "gauge", + "title": "GPU Utilization %", + "gridPos": { "h": 5, "w": 6, "x": 0, "y": 15 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "thresholds": { "mode": "absolute", "steps": [{ "color": "blue", "value": null }, { "color": "green", "value": 30 }, { "color": "yellow", "value": 80 }, { "color": "red", "value": 95 }] }, + "unit": "percent", + "min": 0, + "max": 100, + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true }, + "links": [{ "title": "GPU & Training Detail", "url": "/d/foxhunt-gpu-training", "targetBlank": false }], + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "avg(DCGM_FI_DEV_GPU_UTIL) or vector(0)", + "instant": true + } + ] + }, + { + "id": 24, + "type": "gauge", + "title": "GPU Memory %", + "gridPos": { "h": 5, "w": 6, "x": 6, "y": 15 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 70 }, { "color": "red", "value": 90 }] }, + "unit": "percent", + "min": 0, + "max": 100, + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true }, + "links": [{ "title": "GPU & Training Detail", "url": "/d/foxhunt-gpu-training", "targetBlank": false }], + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "100 * avg(DCGM_FI_DEV_FB_USED / (DCGM_FI_DEV_FB_USED + DCGM_FI_DEV_FB_FREE)) or vector(0)", + "instant": true + } + ] + }, + { + "id": 25, + "type": "timeseries", + "title": "Training Jobs Timeline", + "gridPos": { "h": 5, "w": 12, "x": 12, "y": 15 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "stepAfter", + "fillOpacity": 20, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "none", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 2 + }, + "unit": "short", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [ + { "matcher": { "id": "byName", "options": "active" }, "properties": [{ "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } }] }, + { "matcher": { "id": "byName", "options": "succeeded" }, "properties": [{ "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } }] }, + { "matcher": { "id": "byName", "options": "failed" }, "properties": [{ "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } }] } + ] + }, + "options": { "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "desc" } }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "sum(kube_job_status_active{namespace=\"foxhunt\"}) or vector(0)", + "legendFormat": "active" + }, + { + "refId": "B", + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "sum(kube_job_status_succeeded{namespace=\"foxhunt\"}) or vector(0)", + "legendFormat": "succeeded" + }, + { + "refId": "C", + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "sum(kube_job_status_failed{namespace=\"foxhunt\"}) or vector(0)", + "legendFormat": "failed" + } + ] + }, + + { + "id": 26, + "type": "row", + "title": "Restarts & Network", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 20 }, + "collapsed": false, + "panels": [] + }, + { + "id": 27, + "type": "timeseries", + "title": "Pod Restarts (1h)", + "gridPos": { "h": 5, "w": 12, "x": 0, "y": 21 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "bars", + "lineInterpolation": "linear", + "fillOpacity": 80, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "normal", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 0 + }, + "unit": "short", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "red", "value": 1 }] }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "desc" } }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "increase(kube_pod_container_status_restarts_total{namespace=\"foxhunt\"}[1h]) > 0", + "legendFormat": "{{ pod }}" + } + ] + }, + { + "id": 28, + "type": "timeseries", + "title": "Network I/O by Pod", + "gridPos": { "h": 5, "w": 12, "x": 12, "y": 21 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 10, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "none", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 1 + }, + "unit": "Bps", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "desc" } }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "sum by (pod) (rate(container_network_receive_bytes_total{namespace=\"foxhunt\"}[$interval]))", + "legendFormat": "{{ pod }} rx" + }, + { + "refId": "B", + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "sum by (pod) (rate(container_network_transmit_bytes_total{namespace=\"foxhunt\"}[$interval]))", + "legendFormat": "{{ pod }} tx" + } + ] + }, + + { + "id": 29, + "type": "row", + "title": "Errors", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 26 }, + "collapsed": false, + "panels": [] + }, + { + "id": 30, + "type": "timeseries", + "title": "Error Rate by Service", + "gridPos": { "h": 5, "w": 12, "x": 0, "y": 27 }, + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "bars", + "lineInterpolation": "linear", + "fillOpacity": 60, + "gradientMode": "none", + "spanNulls": false, + "showPoints": "never", + "stacking": { "mode": "normal", "group": "A" }, + "axisPlacement": "auto", + "lineWidth": 0 + }, + "unit": "short", + "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }] }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "desc" } }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "expr": "sum by (app) (count_over_time({namespace=\"foxhunt\"} |= \"ERROR\" [$interval]))", + "legendFormat": "{{ app }}" + } + ] + }, + { + "id": 31, + "type": "table", + "title": "Top Error Messages", + "gridPos": { "h": 5, "w": 12, "x": 12, "y": 27 }, + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "fieldConfig": { + "defaults": { + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "showHeader": true, + "sortBy": [{ "displayName": "Value", "desc": true }], + "footer": { "show": false } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "expr": "topk(10, sum by (app) (count_over_time({namespace=\"foxhunt\"} |= \"ERROR\" | logfmt [$interval])))", + "legendFormat": "{{ app }}", + "instant": true + } + ] + }, + + { + "id": 32, + "type": "row", + "title": "Live Error Log", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 32 }, + "collapsed": false, + "panels": [] + }, + { + "id": 33, + "type": "logs", + "title": "Recent Errors", + "description": "Live stream of ERROR-level log lines from all foxhunt services", + "gridPos": { "h": 7, "w": 24, "x": 0, "y": 33 }, + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "options": { + "showTime": true, + "showLabels": true, + "showCommonLabels": false, + "wrapLogMessage": true, + "prettifyLogMessage": false, + "enableLogDetails": true, + "sortOrder": "Descending", + "dedupStrategy": "none" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "expr": "{namespace=\"foxhunt\"} |= \"ERROR\"", + "maxLines": 200 + } + ] + } + ] +} diff --git a/infra/k8s/monitoring/dashboards/foxhunt-gpu-training.json b/infra/k8s/monitoring/dashboards/foxhunt-gpu-training.json index 7c8486d02..a95ec2529 100644 --- a/infra/k8s/monitoring/dashboards/foxhunt-gpu-training.json +++ b/infra/k8s/monitoring/dashboards/foxhunt-gpu-training.json @@ -1,4 +1,8 @@ { + "__inputs": [ + { "name": "DS_PROMETHEUS", "label": "Prometheus", "description": "", "type": "datasource", "pluginId": "prometheus" }, + { "name": "DS_LOKI", "label": "Loki", "description": "", "type": "datasource", "pluginId": "loki" } + ], "id": null, "uid": "foxhunt-gpu-training", "title": "Foxhunt - GPU & Training Jobs", @@ -21,7 +25,7 @@ { "name": "gpu", "type": "query", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "query": "label_values(DCGM_FI_DEV_GPU_UTIL, gpu)", "refresh": 2, "includeAll": true, @@ -54,7 +58,7 @@ "title": "ci-compile Nodes", "description": "L4-1-24G GPU nodes (0-1 autoscale)", "gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "thresholds": { @@ -80,7 +84,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "count(kube_node_labels{label_k8s_scaleway_com_pool_name=\"ci-compile\"}) or vector(0)", "instant": true } @@ -92,7 +96,7 @@ "title": "ci-training Nodes", "description": "L40S-1-48G GPU nodes (0-1 autoscale)", "gridPos": { "h": 4, "w": 4, "x": 4, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "thresholds": { @@ -118,7 +122,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "count(kube_node_labels{label_k8s_scaleway_com_pool_name=\"ci-training\"}) or vector(0)", "instant": true } @@ -130,7 +134,7 @@ "title": "ci-compile-cpu Nodes", "description": "POP2-32C-128G CPU nodes (0-1 autoscale)", "gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "thresholds": { @@ -156,7 +160,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "count(kube_node_labels{label_k8s_scaleway_com_pool_name=\"ci-compile-cpu\"}) or vector(0)", "instant": true } @@ -167,7 +171,7 @@ "type": "stat", "title": "Active Training Jobs", "gridPos": { "h": 4, "w": 4, "x": 12, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "thresholds": { @@ -193,7 +197,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(kube_job_status_active{namespace=\"foxhunt\"}) or vector(0)", "instant": true } @@ -204,7 +208,7 @@ "type": "stat", "title": "Jobs Succeeded (24h)", "gridPos": { "h": 4, "w": 4, "x": 16, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "thresholds": { @@ -229,7 +233,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(increase(kube_job_status_succeeded{namespace=\"foxhunt\"}[24h])) or vector(0)", "instant": true } @@ -240,7 +244,7 @@ "type": "stat", "title": "Jobs Failed (24h)", "gridPos": { "h": 4, "w": 4, "x": 20, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "thresholds": { @@ -266,7 +270,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(increase(kube_job_status_failed{namespace=\"foxhunt\"}[24h])) or vector(0)", "instant": true } @@ -285,7 +289,7 @@ "type": "timeseries", "title": "GPU Utilization %", "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { @@ -322,7 +326,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "DCGM_FI_DEV_GPU_UTIL{gpu=~\"$gpu\"}", "legendFormat": "GPU {{ gpu }} ({{ instance }})" } @@ -333,7 +337,7 @@ "type": "timeseries", "title": "GPU Memory Used (MB)", "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { @@ -361,13 +365,13 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "DCGM_FI_DEV_FB_USED{gpu=~\"$gpu\"}", "legendFormat": "GPU {{ gpu }} used" }, { "refId": "B", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "DCGM_FI_DEV_FB_TOTAL{gpu=~\"$gpu\"}", "legendFormat": "GPU {{ gpu }} total" } @@ -378,7 +382,7 @@ "type": "timeseries", "title": "GPU Temperature", "gridPos": { "h": 6, "w": 12, "x": 0, "y": 14 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { @@ -414,7 +418,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "DCGM_FI_DEV_GPU_TEMP{gpu=~\"$gpu\"}", "legendFormat": "GPU {{ gpu }}" } @@ -425,7 +429,7 @@ "type": "timeseries", "title": "GPU Power (Watts)", "gridPos": { "h": 6, "w": 12, "x": 12, "y": 14 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { @@ -453,7 +457,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "DCGM_FI_DEV_POWER_USAGE{gpu=~\"$gpu\"}", "legendFormat": "GPU {{ gpu }}" } @@ -472,7 +476,7 @@ "type": "timeseries", "title": "Active Jobs Over Time", "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { @@ -500,7 +504,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "kube_job_status_active{namespace=\"foxhunt\"}", "legendFormat": "{{ job_name }}" } @@ -511,7 +515,7 @@ "type": "table", "title": "Job Status Summary", "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, @@ -545,7 +549,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "kube_job_status_active{namespace=\"foxhunt\"}", "legendFormat": "Active", "instant": true, @@ -553,7 +557,7 @@ }, { "refId": "B", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "kube_job_status_succeeded{namespace=\"foxhunt\"}", "legendFormat": "Succeeded", "instant": true, @@ -561,7 +565,7 @@ }, { "refId": "C", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "kube_job_status_failed{namespace=\"foxhunt\"}", "legendFormat": "Failed", "instant": true, @@ -575,7 +579,7 @@ "title": "GPU Node CPU Usage (Compute Pools)", "description": "CPU usage on GPU/compute node pools when no DCGM data is available", "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { @@ -605,7 +609,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[$interval]))", "legendFormat": "{{ instance }}" } @@ -616,7 +620,7 @@ "type": "timeseries", "title": "GPU Node Memory Usage (Compute Pools)", "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { @@ -644,7 +648,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes", "legendFormat": "{{ instance }}" } @@ -663,7 +667,7 @@ "type": "logs", "title": "Training Job Logs", "gridPos": { "h": 10, "w": 12, "x": 0, "y": 36 }, - "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "options": { "showTime": true, "showLabels": true, @@ -677,7 +681,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "expr": "{namespace=\"foxhunt\", container=~\".*train.*\"}", "maxLines": 200 } @@ -688,7 +692,7 @@ "type": "logs", "title": "CI / GitLab Runner Logs", "gridPos": { "h": 10, "w": 12, "x": 12, "y": 36 }, - "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "options": { "showTime": true, "showLabels": true, @@ -702,7 +706,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "expr": "{namespace=\"foxhunt\", app=~\".*runner.*|.*gitlab.*\"}", "maxLines": 200 } @@ -722,7 +726,7 @@ "title": "Node Count Over Time", "description": "Tracks autoscaling of ci-compile and ci-training pools", "gridPos": { "h": 7, "w": 24, "x": 0, "y": 47 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { @@ -751,7 +755,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "count by (label_k8s_scaleway_com_pool_name) (kube_node_labels)", "legendFormat": "{{ label_k8s_scaleway_com_pool_name }}" } diff --git a/infra/k8s/monitoring/dashboards/foxhunt-infrastructure.json b/infra/k8s/monitoring/dashboards/foxhunt-infrastructure.json index 2591ab769..443aaa8cf 100644 --- a/infra/k8s/monitoring/dashboards/foxhunt-infrastructure.json +++ b/infra/k8s/monitoring/dashboards/foxhunt-infrastructure.json @@ -1,4 +1,7 @@ { + "__inputs": [ + { "name": "DS_PROMETHEUS", "label": "Prometheus", "description": "", "type": "datasource", "pluginId": "prometheus" } + ], "id": null, "uid": "foxhunt-infrastructure", "title": "Foxhunt - K8s Infrastructure", @@ -21,7 +24,7 @@ { "name": "node", "type": "query", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "query": "label_values(kube_node_info, node)", "refresh": 2, "includeAll": true, @@ -53,7 +56,7 @@ "type": "table", "title": "Node Status by Pool", "gridPos": { "h": 7, "w": 24, "x": 0, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, @@ -89,7 +92,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "kube_node_status_condition{condition=\"Ready\", status=\"true\", node=~\"$node\"}", "legendFormat": "{{ node }}", "instant": true, @@ -103,7 +106,7 @@ "title": "services Pool", "description": "DEV1-M, 2 fixed", "gridPos": { "h": 3, "w": 4, "x": 0, "y": 8 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "thresholds": { @@ -130,7 +133,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "count(kube_node_labels{label_k8s_scaleway_com_pool_name=\"services\"} * on(node) kube_node_status_condition{condition=\"Ready\", status=\"true\"}) or vector(0)", "legendFormat": "services", "instant": true @@ -143,7 +146,7 @@ "title": "gitlab Pool", "description": "GP1-XS, 1 fixed", "gridPos": { "h": 3, "w": 4, "x": 4, "y": 8 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "thresholds": { @@ -169,7 +172,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "count(kube_node_labels{label_k8s_scaleway_com_pool_name=\"gitlab\"} * on(node) kube_node_status_condition{condition=\"Ready\", status=\"true\"}) or vector(0)", "legendFormat": "gitlab", "instant": true @@ -182,7 +185,7 @@ "title": "ci-compile-cpu Pool", "description": "POP2-32C-128G, 0-1 autoscale", "gridPos": { "h": 3, "w": 4, "x": 8, "y": 8 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "thresholds": { @@ -208,7 +211,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "count(kube_node_labels{label_k8s_scaleway_com_pool_name=\"ci-compile-cpu\"}) or vector(0)", "legendFormat": "ci-compile-cpu", "instant": true @@ -221,7 +224,7 @@ "title": "ci-compile Pool (GPU)", "description": "L4-1-24G, 0-1 autoscale", "gridPos": { "h": 3, "w": 4, "x": 12, "y": 8 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "thresholds": { @@ -247,7 +250,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "count(kube_node_labels{label_k8s_scaleway_com_pool_name=\"ci-compile\"}) or vector(0)", "legendFormat": "ci-compile", "instant": true @@ -260,7 +263,7 @@ "title": "ci-training Pool (GPU)", "description": "L40S-1-48G, 0-1 autoscale", "gridPos": { "h": 3, "w": 4, "x": 16, "y": 8 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "thresholds": { @@ -286,7 +289,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "count(kube_node_labels{label_k8s_scaleway_com_pool_name=\"ci-training\"}) or vector(0)", "legendFormat": "ci-training", "instant": true @@ -298,7 +301,7 @@ "type": "stat", "title": "Total Pods", "gridPos": { "h": 3, "w": 4, "x": 20, "y": 8 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "thresholds": { @@ -323,7 +326,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(kubelet_running_pods) or count(kube_pod_info)", "instant": true } @@ -342,7 +345,7 @@ "type": "timeseries", "title": "Node CPU Usage", "gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { @@ -372,7 +375,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[$interval]))", "legendFormat": "{{ instance }}" } @@ -383,7 +386,7 @@ "type": "timeseries", "title": "Node Memory Usage", "gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { @@ -411,13 +414,13 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes", "legendFormat": "{{ instance }} used" }, { "refId": "B", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "node_memory_MemTotal_bytes", "legendFormat": "{{ instance }} total" } @@ -428,7 +431,7 @@ "type": "timeseries", "title": "Node Filesystem Usage", "gridPos": { "h": 8, "w": 12, "x": 0, "y": 20 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { @@ -465,7 +468,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "1 - (node_filesystem_avail_bytes{mountpoint=\"/\", fstype!=\"tmpfs\"} / node_filesystem_size_bytes{mountpoint=\"/\", fstype!=\"tmpfs\"})", "legendFormat": "{{ instance }}" } @@ -476,7 +479,7 @@ "type": "bargauge", "title": "Pods per Node", "gridPos": { "h": 8, "w": 12, "x": 12, "y": 20 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "thresholds": { @@ -504,7 +507,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "count by (node) (kube_pod_info{node=~\"$node\"})", "legendFormat": "{{ node }}", "instant": true @@ -524,7 +527,7 @@ "type": "timeseries", "title": "Network I/O per Node", "gridPos": { "h": 8, "w": 12, "x": 0, "y": 29 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { @@ -552,13 +555,13 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum by (instance) (rate(node_network_receive_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$interval]))", "legendFormat": "{{ instance }} RX" }, { "refId": "B", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum by (instance) (rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$interval]))", "legendFormat": "{{ instance }} TX" } @@ -570,7 +573,7 @@ "title": "PVC Usage", "description": "Persistent Volume Claim usage percentage. Requires kubelet_volume_stats metrics.", "gridPos": { "h": 8, "w": 12, "x": 12, "y": 29 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "thresholds": { @@ -601,7 +604,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "(1 - (kubelet_volume_stats_available_bytes{namespace=\"foxhunt\"} / kubelet_volume_stats_capacity_bytes{namespace=\"foxhunt\"})) * 100", "legendFormat": "{{ persistentvolumeclaim }}", "instant": true @@ -622,7 +625,7 @@ "title": "Scheduling Latency (p99)", "description": "Time to schedule pods. From scheduler_scheduling_algorithm_duration_seconds or scheduler_e2e_scheduling_duration_seconds.", "gridPos": { "h": 7, "w": 12, "x": 0, "y": 38 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { @@ -650,13 +653,13 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.99, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket[$interval])) by (le))", "legendFormat": "p99 e2e scheduling" }, { "refId": "B", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.50, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket[$interval])) by (le))", "legendFormat": "p50 e2e scheduling" } @@ -668,7 +671,7 @@ "title": "OOMKill Events", "description": "Container OOMKill events. Each spike indicates a container was killed due to memory pressure.", "gridPos": { "h": 7, "w": 12, "x": 12, "y": 38 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { @@ -696,7 +699,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "increase(kube_pod_container_status_last_terminated_reason{reason=\"OOMKilled\", namespace=\"foxhunt\"}[30m]) > 0", "legendFormat": "{{ pod }} ({{ container }})" } @@ -707,7 +710,7 @@ "type": "timeseries", "title": "Disk I/O per Node", "gridPos": { "h": 7, "w": 24, "x": 0, "y": 45 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { @@ -735,13 +738,13 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum by (instance) (rate(node_disk_read_bytes_total[$interval]))", "legendFormat": "{{ instance }} read" }, { "refId": "B", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum by (instance) (rate(node_disk_written_bytes_total[$interval]))", "legendFormat": "{{ instance }} write" } diff --git a/infra/k8s/monitoring/dashboards/foxhunt-logs.json b/infra/k8s/monitoring/dashboards/foxhunt-logs.json index b17884567..aa98d99f9 100644 --- a/infra/k8s/monitoring/dashboards/foxhunt-logs.json +++ b/infra/k8s/monitoring/dashboards/foxhunt-logs.json @@ -1,4 +1,8 @@ { + "__inputs": [ + { "name": "DS_LOKI", "label": "Loki", "description": "", "type": "datasource", "pluginId": "loki" }, + { "name": "DS_TEMPO", "label": "Tempo", "description": "", "type": "datasource", "pluginId": "tempo" } + ], "id": null, "uid": "foxhunt-logs", "title": "Foxhunt - Log Explorer", @@ -58,7 +62,7 @@ "type": "timeseries", "title": "Log Volume by Service", "gridPos": { "h": 8, "w": 16, "x": 0, "y": 1 }, - "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "fieldConfig": { "defaults": { "custom": { @@ -86,7 +90,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "expr": "sum by (app) (count_over_time({namespace=\"foxhunt\", app=~\"$service\"} [$interval]))", "legendFormat": "{{ app }}" } @@ -97,7 +101,7 @@ "type": "stat", "title": "Total Logs (1h)", "gridPos": { "h": 4, "w": 4, "x": 16, "y": 1 }, - "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "fieldConfig": { "defaults": { "thresholds": { @@ -122,7 +126,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "expr": "sum(count_over_time({namespace=\"foxhunt\", app=~\"$service\"} [1h]))", "instant": true } @@ -133,7 +137,7 @@ "type": "stat", "title": "Errors (1h)", "gridPos": { "h": 4, "w": 4, "x": 20, "y": 1 }, - "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "fieldConfig": { "defaults": { "thresholds": { @@ -160,7 +164,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "expr": "sum(count_over_time({namespace=\"foxhunt\", app=~\"$service\"} |= \"ERROR\" [1h]))", "instant": true } @@ -171,7 +175,7 @@ "type": "stat", "title": "Warnings (1h)", "gridPos": { "h": 4, "w": 4, "x": 16, "y": 5 }, - "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "fieldConfig": { "defaults": { "thresholds": { @@ -197,7 +201,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "expr": "sum(count_over_time({namespace=\"foxhunt\", app=~\"$service\"} |= \"WARN\" [1h]))", "instant": true } @@ -208,7 +212,7 @@ "type": "stat", "title": "Error Rate /min", "gridPos": { "h": 4, "w": 4, "x": 20, "y": 5 }, - "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "fieldConfig": { "defaults": { "thresholds": { @@ -237,7 +241,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "expr": "sum(count_over_time({namespace=\"foxhunt\", app=~\"$service\"} |= \"ERROR\" [1m]))", "instant": true } @@ -256,7 +260,7 @@ "type": "timeseries", "title": "Error Rate by Service", "gridPos": { "h": 7, "w": 24, "x": 0, "y": 10 }, - "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "fieldConfig": { "defaults": { "custom": { @@ -284,7 +288,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "expr": "sum by (app) (count_over_time({namespace=\"foxhunt\", app=~\"$service\"} |= \"ERROR\" [$interval]))", "legendFormat": "{{ app }}" } @@ -296,7 +300,7 @@ "title": "Top Error Messages", "description": "Most frequent error messages across all services in the time range", "gridPos": { "h": 8, "w": 24, "x": 0, "y": 17 }, - "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, @@ -313,7 +317,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "expr": "topk(20, sum by (app, level) (count_over_time({namespace=\"foxhunt\", app=~\"$service\"} |= \"ERROR\" | logfmt [$interval])))", "legendFormat": "{{ app }}", "instant": true @@ -334,7 +338,7 @@ "title": "Live Log Stream", "description": "Streaming logs from selected services. Use the $search variable to filter.", "gridPos": { "h": 14, "w": 24, "x": 0, "y": 26 }, - "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "options": { "showTime": true, "showLabels": true, @@ -348,7 +352,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "expr": "{namespace=\"foxhunt\", app=~\"$service\"} |= \"$search\"", "maxLines": 500 } @@ -368,7 +372,7 @@ "title": "Recent Traces", "description": "Search for distributed traces via Tempo. Click a trace ID to view the full trace.", "gridPos": { "h": 10, "w": 24, "x": 0, "y": 41 }, - "datasource": { "type": "tempo", "uid": "tempo" }, + "datasource": { "type": "tempo", "uid": "${DS_TEMPO}" }, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, @@ -379,7 +383,7 @@ "title": "View Trace", "url": "", "internal": { - "datasourceUid": "tempo", + "datasourceUid": "${DS_TEMPO}", "datasourceName": "Tempo", "query": { "queryType": "traceql", @@ -404,7 +408,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "tempo", "uid": "tempo" }, + "datasource": { "type": "tempo", "uid": "${DS_TEMPO}" }, "queryType": "nativeSearch", "serviceName": "", "spanName": "", diff --git a/infra/k8s/monitoring/dashboards/foxhunt-overview.json b/infra/k8s/monitoring/dashboards/foxhunt-overview.json index b75010478..b42d3829e 100644 --- a/infra/k8s/monitoring/dashboards/foxhunt-overview.json +++ b/infra/k8s/monitoring/dashboards/foxhunt-overview.json @@ -1,4 +1,8 @@ { + "__inputs": [ + { "name": "DS_PROMETHEUS", "label": "Prometheus", "description": "", "type": "datasource", "pluginId": "prometheus" }, + { "name": "DS_LOKI", "label": "Loki", "description": "", "type": "datasource", "pluginId": "loki" } + ], "id": null, "uid": "foxhunt-overview", "title": "Foxhunt - System Overview", @@ -45,7 +49,7 @@ "type": "stat", "title": "Nodes Ready", "gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "thresholds": { @@ -72,7 +76,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(kube_node_status_condition{condition=\"Ready\", status=\"true\"})", "legendFormat": "Ready", "instant": true @@ -84,7 +88,7 @@ "type": "stat", "title": "Total Nodes", "gridPos": { "h": 4, "w": 4, "x": 4, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "thresholds": { @@ -109,7 +113,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "count(kube_node_info)", "legendFormat": "Nodes", "instant": true @@ -121,7 +125,7 @@ "type": "stat", "title": "Pods Running", "gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "thresholds": { @@ -146,7 +150,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", phase=\"Running\"})", "legendFormat": "Running", "instant": true @@ -158,7 +162,7 @@ "type": "stat", "title": "Pods Failed", "gridPos": { "h": 4, "w": 4, "x": 12, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "thresholds": { @@ -184,7 +188,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", phase=\"Failed\"}) or vector(0)", "legendFormat": "Failed", "instant": true @@ -196,7 +200,7 @@ "type": "stat", "title": "Active Training Jobs", "gridPos": { "h": 4, "w": 4, "x": 16, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "thresholds": { @@ -222,7 +226,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(kube_job_status_active{namespace=\"foxhunt\"}) or vector(0)", "legendFormat": "Active Jobs", "instant": true @@ -234,7 +238,7 @@ "type": "stat", "title": "Log Error Rate (/min)", "gridPos": { "h": 4, "w": 4, "x": 20, "y": 1 }, - "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "fieldConfig": { "defaults": { "thresholds": { @@ -263,7 +267,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "expr": "sum(count_over_time({namespace=\"foxhunt\"} |= \"ERROR\" [1m]))", "legendFormat": "Errors/min", "instant": true @@ -283,7 +287,7 @@ "type": "stat", "title": "api-gateway", "gridPos": { "h": 3, "w": 3, "x": 0, "y": 6 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "thresholds": { @@ -311,7 +315,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"api-gateway.*\", phase=\"Running\"}) > 0 or vector(0)", "legendFormat": "", "instant": true @@ -323,7 +327,7 @@ "type": "stat", "title": "trading-service", "gridPos": { "h": 3, "w": 3, "x": 3, "y": 6 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "thresholds": { @@ -351,7 +355,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"trading-service.*\", phase=\"Running\"}) > 0 or vector(0)", "legendFormat": "", "instant": true @@ -363,7 +367,7 @@ "type": "stat", "title": "ml-training-service", "gridPos": { "h": 3, "w": 3, "x": 6, "y": 6 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "thresholds": { @@ -391,7 +395,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"ml-training-service.*\", phase=\"Running\"}) > 0 or vector(0)", "legendFormat": "", "instant": true @@ -403,7 +407,7 @@ "type": "stat", "title": "backtesting-service", "gridPos": { "h": 3, "w": 3, "x": 9, "y": 6 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "thresholds": { @@ -431,7 +435,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"backtesting-service.*\", phase=\"Running\"}) > 0 or vector(0)", "legendFormat": "", "instant": true @@ -443,7 +447,7 @@ "type": "stat", "title": "trading-agent", "gridPos": { "h": 3, "w": 3, "x": 12, "y": 6 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "thresholds": { @@ -471,7 +475,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"trading-agent.*\", phase=\"Running\"}) > 0 or vector(0)", "legendFormat": "", "instant": true @@ -483,7 +487,7 @@ "type": "stat", "title": "broker-gateway", "gridPos": { "h": 3, "w": 3, "x": 15, "y": 6 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "thresholds": { @@ -511,7 +515,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"broker-gateway.*\", phase=\"Running\"}) > 0 or vector(0)", "legendFormat": "", "instant": true @@ -523,7 +527,7 @@ "type": "stat", "title": "data-acquisition", "gridPos": { "h": 3, "w": 3, "x": 18, "y": 6 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "thresholds": { @@ -551,7 +555,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"data-acquisition.*\", phase=\"Running\"}) > 0 or vector(0)", "legendFormat": "", "instant": true @@ -563,7 +567,7 @@ "type": "stat", "title": "web-gateway", "gridPos": { "h": 3, "w": 3, "x": 21, "y": 6 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "thresholds": { @@ -591,7 +595,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"web-gateway.*\", phase=\"Running\"}) > 0 or vector(0)", "legendFormat": "", "instant": true @@ -611,7 +615,7 @@ "type": "gauge", "title": "Cluster CPU %", "gridPos": { "h": 6, "w": 6, "x": 0, "y": 10 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "thresholds": { @@ -639,7 +643,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "100 - (avg(rate(node_cpu_seconds_total{mode=\"idle\"}[$interval])) * 100)", "legendFormat": "CPU", "instant": true @@ -651,7 +655,7 @@ "type": "gauge", "title": "Cluster Memory %", "gridPos": { "h": 6, "w": 6, "x": 6, "y": 10 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "thresholds": { @@ -679,7 +683,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "(1 - (sum(node_memory_MemAvailable_bytes) / sum(node_memory_MemTotal_bytes))) * 100", "legendFormat": "Memory", "instant": true @@ -691,7 +695,7 @@ "type": "timeseries", "title": "CPU by Node Pool", "gridPos": { "h": 6, "w": 12, "x": 12, "y": 10 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { @@ -721,7 +725,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[$interval]))", "legendFormat": "{{ instance }}" } @@ -732,7 +736,7 @@ "type": "timeseries", "title": "Memory by Node Pool", "gridPos": { "h": 6, "w": 12, "x": 0, "y": 16 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { @@ -760,7 +764,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes", "legendFormat": "{{ instance }}" } @@ -771,7 +775,7 @@ "type": "timeseries", "title": "Foxhunt Container CPU Usage", "gridPos": { "h": 6, "w": 12, "x": 12, "y": 16 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { @@ -799,7 +803,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum by (container) (rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", container=~\"api-gateway|trading-service|ml-training-service|backtesting-service|trading-agent-service|broker-gateway|data-acquisition-service|web-gateway\", image!=\"\"}[$interval]))", "legendFormat": "{{ container }}" } @@ -819,7 +823,7 @@ "title": "Pod Restarts (last 1h)", "description": "Pod restart events indicating crashes or OOMKills", "gridPos": { "h": 7, "w": 12, "x": 0, "y": 23 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { @@ -847,7 +851,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "increase(kube_pod_container_status_restarts_total{namespace=\"foxhunt\"}[1h]) > 0", "legendFormat": "{{ pod }}" } @@ -858,7 +862,7 @@ "type": "timeseries", "title": "Deployment Replicas: Desired vs Available", "gridPos": { "h": 7, "w": 12, "x": 12, "y": 23 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { @@ -886,13 +890,13 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "kube_deployment_spec_replicas{namespace=\"foxhunt\"}", "legendFormat": "{{ deployment }} desired" }, { "refId": "B", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "kube_deployment_status_replicas_available{namespace=\"foxhunt\"}", "legendFormat": "{{ deployment }} available" } @@ -911,7 +915,7 @@ "type": "timeseries", "title": "Log Errors Over Time", "gridPos": { "h": 7, "w": 24, "x": 0, "y": 31 }, - "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "fieldConfig": { "defaults": { "custom": { @@ -939,7 +943,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "expr": "sum by (app) (count_over_time({namespace=\"foxhunt\"} |= \"ERROR\" [$interval]))", "legendFormat": "{{ app }}" } diff --git a/infra/k8s/monitoring/dashboards/foxhunt-services.json b/infra/k8s/monitoring/dashboards/foxhunt-services.json index 7cb11a9ac..5ef6389ea 100644 --- a/infra/k8s/monitoring/dashboards/foxhunt-services.json +++ b/infra/k8s/monitoring/dashboards/foxhunt-services.json @@ -1,4 +1,8 @@ { + "__inputs": [ + { "name": "DS_PROMETHEUS", "label": "Prometheus", "description": "", "type": "datasource", "pluginId": "prometheus" }, + { "name": "DS_LOKI", "label": "Loki", "description": "", "type": "datasource", "pluginId": "loki" } + ], "id": null, "uid": "foxhunt-services", "title": "Foxhunt - Microservice Detail", @@ -58,7 +62,7 @@ "type": "stat", "title": "Status", "gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "thresholds": { @@ -86,7 +90,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"$service.*\", phase=\"Running\"}) > 0 or vector(0)", "instant": true } @@ -97,7 +101,7 @@ "type": "stat", "title": "Replicas", "gridPos": { "h": 4, "w": 4, "x": 4, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "thresholds": { @@ -122,7 +126,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "count(kube_pod_info{namespace=\"foxhunt\", pod=~\"$service.*\"})", "instant": true } @@ -133,7 +137,7 @@ "type": "stat", "title": "Restarts (24h)", "gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "thresholds": { @@ -160,7 +164,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(increase(kube_pod_container_status_restarts_total{namespace=\"foxhunt\", pod=~\"$service.*\"}[24h])) or vector(0)", "instant": true } @@ -171,7 +175,7 @@ "type": "stat", "title": "Container Uptime", "gridPos": { "h": 4, "w": 4, "x": 12, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "thresholds": { @@ -199,7 +203,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "min(time() - kube_pod_start_time{namespace=\"foxhunt\", pod=~\"$service.*\"})", "instant": true } @@ -210,7 +214,7 @@ "type": "stat", "title": "CPU Cores", "gridPos": { "h": 4, "w": 4, "x": 16, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "thresholds": { @@ -237,7 +241,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", container=~\"$service\", image!=\"\"}[$interval]))", "instant": true } @@ -248,7 +252,7 @@ "type": "stat", "title": "Memory", "gridPos": { "h": 4, "w": 4, "x": 20, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "thresholds": { @@ -274,7 +278,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", container=~\"$service\", image!=\"\"})", "instant": true } @@ -293,7 +297,7 @@ "type": "timeseries", "title": "CPU Usage", "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { @@ -321,19 +325,19 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", container=~\"$service\", image!=\"\"}[$interval]))", "legendFormat": "{{ pod }}" }, { "refId": "B", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(kube_pod_container_resource_requests{namespace=\"foxhunt\", container=~\"$service\", resource=\"cpu\"})", "legendFormat": "Request" }, { "refId": "C", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(kube_pod_container_resource_limits{namespace=\"foxhunt\", container=~\"$service\", resource=\"cpu\"})", "legendFormat": "Limit" } @@ -344,7 +348,7 @@ "type": "timeseries", "title": "Memory Usage", "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { @@ -372,19 +376,19 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum by (pod) (container_memory_working_set_bytes{namespace=\"foxhunt\", container=~\"$service\", image!=\"\"})", "legendFormat": "{{ pod }}" }, { "refId": "B", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(kube_pod_container_resource_requests{namespace=\"foxhunt\", container=~\"$service\", resource=\"memory\"})", "legendFormat": "Request" }, { "refId": "C", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(kube_pod_container_resource_limits{namespace=\"foxhunt\", container=~\"$service\", resource=\"memory\"})", "legendFormat": "Limit" } @@ -403,7 +407,7 @@ "type": "timeseries", "title": "Pod Restart Count", "gridPos": { "h": 6, "w": 12, "x": 0, "y": 15 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { @@ -431,7 +435,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "increase(kube_pod_container_status_restarts_total{namespace=\"foxhunt\", pod=~\"$service.*\"}[30m])", "legendFormat": "{{ pod }}" } @@ -442,7 +446,7 @@ "type": "timeseries", "title": "Network I/O", "gridPos": { "h": 6, "w": 12, "x": 12, "y": 15 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { @@ -470,13 +474,13 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum by (pod) (rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"$service.*\"}[$interval]))", "legendFormat": "{{ pod }} RX" }, { "refId": "B", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum by (pod) (rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"$service.*\"}[$interval]))", "legendFormat": "{{ pod }} TX" } @@ -496,7 +500,7 @@ "title": "gRPC Request Rate", "description": "gRPC requests per second by method and status code. Requires grpc_server_handled_total metric from service instrumentation.", "gridPos": { "h": 8, "w": 12, "x": 0, "y": 22 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { @@ -524,7 +528,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum by (grpc_method, grpc_code) (rate(grpc_server_handled_total{grpc_service=~\".*$service.*\"}[$interval]))", "legendFormat": "{{ grpc_method }} [{{ grpc_code }}]" } @@ -536,7 +540,7 @@ "title": "gRPC Latency (p50 / p95 / p99)", "description": "gRPC server-side latency percentiles. Requires grpc_server_handling_seconds_bucket histogram.", "gridPos": { "h": 8, "w": 12, "x": 12, "y": 22 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { @@ -564,19 +568,19 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.50, sum(rate(grpc_server_handling_seconds_bucket{grpc_service=~\".*$service.*\"}[$interval])) by (le))", "legendFormat": "p50" }, { "refId": "B", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.95, sum(rate(grpc_server_handling_seconds_bucket{grpc_service=~\".*$service.*\"}[$interval])) by (le))", "legendFormat": "p95" }, { "refId": "C", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{grpc_service=~\".*$service.*\"}[$interval])) by (le))", "legendFormat": "p99" } @@ -595,7 +599,7 @@ "type": "timeseries", "title": "Log Volume by Level", "gridPos": { "h": 6, "w": 24, "x": 0, "y": 31 }, - "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "fieldConfig": { "defaults": { "custom": { @@ -640,7 +644,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "expr": "sum by (level) (count_over_time({app=\"$service\", namespace=\"foxhunt\"} | logfmt | level != \"\" [$interval]))", "legendFormat": "{{ level }}" } @@ -651,7 +655,7 @@ "type": "logs", "title": "Recent Error Logs", "gridPos": { "h": 10, "w": 24, "x": 0, "y": 37 }, - "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "options": { "showTime": true, "showLabels": true, @@ -665,7 +669,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "expr": "{app=\"$service\", namespace=\"foxhunt\"} |= \"ERROR\"", "maxLines": 200 } diff --git a/infra/k8s/monitoring/dashboards/gitlab-services.json b/infra/k8s/monitoring/dashboards/gitlab-services.json index 95aa33a72..7f2762bc5 100644 --- a/infra/k8s/monitoring/dashboards/gitlab-services.json +++ b/infra/k8s/monitoring/dashboards/gitlab-services.json @@ -1,4 +1,7 @@ { + "__inputs": [ + { "name": "DS_PROMETHEUS", "label": "Prometheus", "description": "", "type": "datasource", "pluginId": "prometheus" } + ], "id": null, "uid": "foxhunt-gitlab-services", "title": "GitLab + Foxhunt Services", @@ -22,7 +25,7 @@ "type": "query", "datasource": { "type": "prometheus", - "uid": "cfegaydmscvswe" + "uid": "${DS_PROMETHEUS}" }, "query": "label_values(kube_pod_info, namespace)", "refresh": 2, @@ -52,7 +55,7 @@ "type": "timeseries", "title": "Puma HTTP Request Rate", "gridPos": { "h": 8, "w": 12, "x": 0, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { @@ -80,7 +83,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(rate(http_requests_total{job=~\".*puma.*\"}[5m])) by (method)", "legendFormat": "{{ method }}" } @@ -91,7 +94,7 @@ "type": "timeseries", "title": "Puma Request Duration (p95)", "gridPos": { "h": 8, "w": 12, "x": 12, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { @@ -119,13 +122,13 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket{job=~\".*puma.*\"}[5m])) by (le))", "legendFormat": "p95" }, { "refId": "B", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.50, sum(rate(http_request_duration_seconds_bucket{job=~\".*puma.*\"}[5m])) by (le))", "legendFormat": "p50" } @@ -144,7 +147,7 @@ "type": "timeseries", "title": "Sidekiq Job Processing Rate", "gridPos": { "h": 8, "w": 12, "x": 0, "y": 10 }, - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { @@ -172,13 +175,13 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(rate(sidekiq_jobs_processed_total[5m]))", "legendFormat": "Processed" }, { "refId": "B", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(rate(sidekiq_jobs_failed_total[5m]))", "legendFormat": "Failed" } @@ -189,7 +192,7 @@ "type": "stat", "title": "Sidekiq Queue Size", "gridPos": { "h": 8, "w": 12, "x": 12, "y": 10 }, - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "thresholds": { @@ -216,7 +219,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(sidekiq_queue_size)", "legendFormat": "Queue Size", "instant": true @@ -236,7 +239,7 @@ "type": "timeseries", "title": "Gitaly RPC Latency (p95)", "gridPos": { "h": 8, "w": 12, "x": 0, "y": 19 }, - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { @@ -264,7 +267,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.95, sum(rate(grpc_server_handling_seconds_bucket{job=~\".*gitaly.*\"}[5m])) by (le, grpc_method))", "legendFormat": "{{ grpc_method }}" } @@ -275,7 +278,7 @@ "type": "timeseries", "title": "Gitaly RPC Rate", "gridPos": { "h": 8, "w": 12, "x": 12, "y": 19 }, - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { @@ -303,7 +306,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(rate(grpc_server_handled_total{job=~\".*gitaly.*\"}[5m])) by (grpc_code)", "legendFormat": "{{ grpc_code }}" } @@ -322,7 +325,7 @@ "type": "timeseries", "title": "Container CPU Usage by Service", "gridPos": { "h": 8, "w": 12, "x": 0, "y": 28 }, - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { @@ -350,7 +353,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum by (container) (rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\", container=~\"web-gateway|api-gateway|trading-service|ml-training-service|backtesting-service|data-acquisition-service|broker-gateway-service|trading-agent-service\", image!=\"\"}[5m]))", "legendFormat": "{{ container }}" } @@ -361,7 +364,7 @@ "type": "timeseries", "title": "Container Memory Usage by Service", "gridPos": { "h": 8, "w": 12, "x": 12, "y": 28 }, - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { @@ -389,7 +392,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum by (container) (container_memory_working_set_bytes{namespace=~\"$namespace\", container=~\"web-gateway|api-gateway|trading-service|ml-training-service|backtesting-service|data-acquisition-service|broker-gateway-service|trading-agent-service\", image!=\"\"})", "legendFormat": "{{ container }}" } @@ -408,7 +411,7 @@ "type": "timeseries", "title": "Pod Restarts (by Container)", "gridPos": { "h": 8, "w": 12, "x": 0, "y": 37 }, - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { @@ -436,7 +439,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "increase(kube_pod_container_status_restarts_total{namespace=~\"$namespace\"}[1h])", "legendFormat": "{{ namespace }}/{{ pod }} ({{ container }})" } @@ -447,7 +450,7 @@ "type": "table", "title": "Pod Status by Namespace", "gridPos": { "h": 8, "w": 12, "x": 12, "y": 37 }, - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, @@ -503,7 +506,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum by (namespace) (kube_pod_status_phase{namespace=~\"$namespace\", phase=\"Running\"})", "legendFormat": "Running", "instant": true, @@ -511,7 +514,7 @@ }, { "refId": "B", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum by (namespace) (kube_pod_status_phase{namespace=~\"$namespace\", phase=\"Pending\"})", "legendFormat": "Pending", "instant": true, @@ -519,7 +522,7 @@ }, { "refId": "C", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum by (namespace) (kube_pod_status_phase{namespace=~\"$namespace\", phase=\"Failed\"})", "legendFormat": "Failed", "instant": true, @@ -540,7 +543,7 @@ "type": "timeseries", "title": "Deployment Replicas: Desired vs Available", "gridPos": { "h": 8, "w": 24, "x": 0, "y": 46 }, - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { @@ -568,13 +571,13 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "kube_deployment_spec_replicas{namespace=~\"$namespace\"}", "legendFormat": "{{ deployment }} desired" }, { "refId": "B", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "kube_deployment_status_replicas_available{namespace=~\"$namespace\"}", "legendFormat": "{{ deployment }} available" } diff --git a/infra/k8s/monitoring/dashboards/gpu-overview.json b/infra/k8s/monitoring/dashboards/gpu-overview.json index 8d51efd5c..d3d40eaef 100644 --- a/infra/k8s/monitoring/dashboards/gpu-overview.json +++ b/infra/k8s/monitoring/dashboards/gpu-overview.json @@ -1,4 +1,7 @@ { + "__inputs": [ + { "name": "DS_PROMETHEUS", "label": "Prometheus", "description": "", "type": "datasource", "pluginId": "prometheus" } + ], "id": null, "uid": "foxhunt-gpu-overview", "title": "GPU Overview", @@ -22,7 +25,7 @@ "type": "query", "datasource": { "type": "prometheus", - "uid": "cfegaydmscvswe" + "uid": "${DS_PROMETHEUS}" }, "query": "label_values(DCGM_FI_DEV_GPU_UTIL, gpu)", "refresh": 2, @@ -41,7 +44,7 @@ "type": "query", "datasource": { "type": "prometheus", - "uid": "cfegaydmscvswe" + "uid": "${DS_PROMETHEUS}" }, "query": "label_values(DCGM_FI_DEV_GPU_UTIL, instance)", "refresh": 2, @@ -60,7 +63,7 @@ "type": "query", "datasource": { "type": "prometheus", - "uid": "cfegaydmscvswe" + "uid": "${DS_PROMETHEUS}" }, "query": "label_values(DCGM_FI_DEV_GPU_UTIL, exported_namespace)", "refresh": 2, @@ -90,7 +93,7 @@ "type": "timeseries", "title": "GPU Utilization %", "gridPos": { "h": 8, "w": 12, "x": 0, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { @@ -127,7 +130,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "DCGM_FI_DEV_GPU_UTIL{gpu=~\"$gpu\", instance=~\"$instance\"}", "legendFormat": "GPU {{ gpu }} ({{ instance }})" } @@ -138,7 +141,7 @@ "type": "gauge", "title": "Current GPU Utilization", "gridPos": { "h": 8, "w": 12, "x": 12, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "thresholds": { @@ -167,7 +170,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "DCGM_FI_DEV_GPU_UTIL{gpu=~\"$gpu\", instance=~\"$instance\"}", "legendFormat": "GPU {{ gpu }}", "instant": true @@ -187,7 +190,7 @@ "type": "timeseries", "title": "GPU Memory Used", "gridPos": { "h": 8, "w": 12, "x": 0, "y": 10 }, - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { @@ -215,13 +218,13 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "DCGM_FI_DEV_FB_USED{gpu=~\"$gpu\", instance=~\"$instance\"}", "legendFormat": "GPU {{ gpu }} used" }, { "refId": "B", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "DCGM_FI_DEV_FB_TOTAL{gpu=~\"$gpu\", instance=~\"$instance\"}", "legendFormat": "GPU {{ gpu }} total" } @@ -232,7 +235,7 @@ "type": "gauge", "title": "GPU Memory Usage %", "gridPos": { "h": 8, "w": 12, "x": 12, "y": 10 }, - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "thresholds": { @@ -260,7 +263,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "(DCGM_FI_DEV_FB_USED{gpu=~\"$gpu\", instance=~\"$instance\"} / DCGM_FI_DEV_FB_TOTAL{gpu=~\"$gpu\", instance=~\"$instance\"}) * 100", "legendFormat": "GPU {{ gpu }}", "instant": true @@ -280,7 +283,7 @@ "type": "timeseries", "title": "GPU Temperature", "gridPos": { "h": 8, "w": 12, "x": 0, "y": 19 }, - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { @@ -316,7 +319,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "DCGM_FI_DEV_GPU_TEMP{gpu=~\"$gpu\", instance=~\"$instance\"}", "legendFormat": "GPU {{ gpu }} ({{ instance }})" } @@ -327,7 +330,7 @@ "type": "timeseries", "title": "GPU Power Usage", "gridPos": { "h": 8, "w": 12, "x": 12, "y": 19 }, - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { @@ -355,7 +358,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "DCGM_FI_DEV_POWER_USAGE{gpu=~\"$gpu\", instance=~\"$instance\"}", "legendFormat": "GPU {{ gpu }} ({{ instance }})" } @@ -374,7 +377,7 @@ "type": "timeseries", "title": "GPU Memory Clock", "gridPos": { "h": 8, "w": 12, "x": 0, "y": 28 }, - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { @@ -403,7 +406,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "DCGM_FI_DEV_MEM_CLK{gpu=~\"$gpu\", instance=~\"$instance\"} * 1000000", "legendFormat": "GPU {{ gpu }} mem clock" } @@ -415,7 +418,7 @@ "title": "Per-Pod GPU Utilization", "description": "GPU utilization broken down by pod. Requires DCGM exporter pod-level labels (exported_pod, exported_namespace).", "gridPos": { "h": 8, "w": 12, "x": 12, "y": 28 }, - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { @@ -445,7 +448,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "DCGM_FI_DEV_GPU_UTIL{exported_namespace=~\"$namespace\", gpu=~\"$gpu\", instance=~\"$instance\"}", "legendFormat": "{{ exported_namespace }}/{{ exported_pod }} GPU {{ gpu }}" } @@ -457,7 +460,7 @@ "title": "Per-Pod GPU Memory Used", "description": "GPU framebuffer memory used per pod. Requires DCGM exporter pod-level labels.", "gridPos": { "h": 8, "w": 24, "x": 0, "y": 36 }, - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "custom": { @@ -485,7 +488,7 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "DCGM_FI_DEV_FB_USED{exported_namespace=~\"$namespace\", gpu=~\"$gpu\", instance=~\"$instance\"}", "legendFormat": "{{ exported_namespace }}/{{ exported_pod }} GPU {{ gpu }}" } diff --git a/infra/k8s/monitoring/dashboards/import.sh b/infra/k8s/monitoring/dashboards/import.sh index d1d33bcde..3c5a70d05 100755 --- a/infra/k8s/monitoring/dashboards/import.sh +++ b/infra/k8s/monitoring/dashboards/import.sh @@ -1,27 +1,97 @@ #!/usr/bin/env bash set -euo pipefail -# Import Grafana dashboards to Scaleway Cockpit via the Grafana API. +# Import Grafana dashboards with dynamic datasource resolution. +# +# Dashboards use ${DS_PROMETHEUS}, ${DS_LOKI}, ${DS_TEMPO} variables. +# This script auto-detects datasource UIDs from the Grafana instance +# and maps them at import time via /api/dashboards/import. # # Required environment variables: -# GRAFANA_URL - Grafana base URL (default: https://grafana.fxhnt.ai) # GRAFANA_PASS - Password for admin user # +# Optional overrides: +# GRAFANA_URL - Grafana base URL (default: https://grafana.fxhnt.ai) +# GRAFANA_USER - Grafana user (default: admin) +# GRAFANA_PROM_UID - Override Prometheus datasource UID +# GRAFANA_LOKI_UID - Override Loki datasource UID +# GRAFANA_TEMPO_UID- Override Tempo datasource UID +# # Usage: # GRAFANA_PASS="Welcome01" ./import.sh -# GRAFANA_URL="https://grafana.fxhnt.ai" GRAFANA_PASS="Welcome01" ./import.sh SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" GRAFANA_USER="${GRAFANA_USER:-admin}" GRAFANA_URL="${GRAFANA_URL:-https://grafana.fxhnt.ai}" +GRAFANA_URL="${GRAFANA_URL%/}" if [[ -z "${GRAFANA_PASS:-}" ]]; then echo "ERROR: GRAFANA_PASS is not set" >&2 exit 1 fi -# Strip trailing slash from URL -GRAFANA_URL="${GRAFANA_URL%/}" +# --- Datasource auto-detection --- + +detect_datasource_uid() { + local ds_type="$1" + curl -sf -u "${GRAFANA_USER}:${GRAFANA_PASS}" "${GRAFANA_URL}/api/datasources" 2>/dev/null | \ + python3 -c " +import sys, json +for ds in json.load(sys.stdin): + if ds.get('type') == '${ds_type}': + print(ds['uid']) + break +" 2>/dev/null || true +} + +echo "Detecting datasource UIDs..." +PROM_UID="${GRAFANA_PROM_UID:-$(detect_datasource_uid prometheus)}" +LOKI_UID="${GRAFANA_LOKI_UID:-$(detect_datasource_uid loki)}" +TEMPO_UID="${GRAFANA_TEMPO_UID:-$(detect_datasource_uid tempo)}" + +echo " Prometheus: ${PROM_UID:-NOT FOUND}" +echo " Loki: ${LOKI_UID:-NOT FOUND}" +echo " Tempo: ${TEMPO_UID:-NOT FOUND}" + +if [[ -z "$PROM_UID" ]]; then + echo "ERROR: Could not detect Prometheus datasource. Set GRAFANA_PROM_UID." >&2 + exit 1 +fi + +# --- Build inputs array for a dashboard --- + +build_inputs() { + local file="$1" + local inputs="[" + local first=true + + if grep -q 'DS_PROMETHEUS' "$file"; then + $first || inputs+="," + inputs+="{\"name\":\"DS_PROMETHEUS\",\"type\":\"datasource\",\"pluginId\":\"prometheus\",\"value\":\"${PROM_UID}\"}" + first=false + fi + if grep -q 'DS_LOKI' "$file"; then + if [[ -z "$LOKI_UID" ]]; then + echo " WARNING: Dashboard requires Loki but no datasource found" >&2 + fi + $first || inputs+="," + inputs+="{\"name\":\"DS_LOKI\",\"type\":\"datasource\",\"pluginId\":\"loki\",\"value\":\"${LOKI_UID}\"}" + first=false + fi + if grep -q 'DS_TEMPO' "$file"; then + if [[ -z "$TEMPO_UID" ]]; then + echo " WARNING: Dashboard requires Tempo but no datasource found" >&2 + fi + $first || inputs+="," + inputs+="{\"name\":\"DS_TEMPO\",\"type\":\"datasource\",\"pluginId\":\"tempo\",\"value\":\"${TEMPO_UID}\"}" + first=false + fi + + inputs+="]" + echo "$inputs" +} + +# --- Import a single dashboard --- import_dashboard() { local file="$1" @@ -30,12 +100,15 @@ import_dashboard() { echo "Importing ${name}..." + local inputs + inputs=$(build_inputs "$file") + local response response=$(curl -s -w "\n%{http_code}" \ -u "${GRAFANA_USER}:${GRAFANA_PASS}" \ -H "Content-Type: application/json" \ - -d "{\"dashboard\": $(cat "$file"), \"overwrite\": true}" \ - "${GRAFANA_URL}/api/dashboards/db") + -d "{\"dashboard\": $(cat "$file"), \"inputs\": ${inputs}, \"overwrite\": true}" \ + "${GRAFANA_URL}/api/dashboards/import") local http_code http_code=$(echo "$response" | tail -1) @@ -43,13 +116,15 @@ import_dashboard() { body=$(echo "$response" | sed '$d') if [[ "$http_code" -ge 200 && "$http_code" -lt 300 ]]; then - echo " OK (${http_code}): ${body}" + echo " OK (${http_code})" else echo " FAILED (${http_code}): ${body}" >&2 return 1 fi } +# --- Import all dashboards --- + failed=0 for dashboard in "${SCRIPT_DIR}/cluster-overview.json" \ @@ -59,7 +134,9 @@ for dashboard in "${SCRIPT_DIR}/cluster-overview.json" \ "${SCRIPT_DIR}/foxhunt-services.json" \ "${SCRIPT_DIR}/foxhunt-gpu-training.json" \ "${SCRIPT_DIR}/foxhunt-infrastructure.json" \ - "${SCRIPT_DIR}/foxhunt-logs.json"; do + "${SCRIPT_DIR}/foxhunt-logs.json" \ + "${SCRIPT_DIR}/foxhunt-ci-pipelines.json" \ + "${SCRIPT_DIR}/foxhunt-cockpit.json"; do if [[ -f "$dashboard" ]]; then if ! import_dashboard "$dashboard"; then failed=$((failed + 1))