gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(monitoring): add cockpit dashboard, migrate all dashboards to dynamic datasources

Browse Source 
Add foxhunt-cockpit.json — single-page overview with 7 rows (health bar,
service status, resources, GPU/training, restarts/network, errors, live
logs) pulling critical signals from all domain dashboards.

Migrate all 10 dashboards from hardcoded datasource UIDs to ${DS_PROMETHEUS},
${DS_LOKI}, ${DS_TEMPO} variables with __inputs sections for portable import.
Update import.sh to auto-detect datasource UIDs from Grafana API at deploy
time, using /api/dashboards/import endpoint.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-02-28 23:04:42 +01:00
parent 6b94045bcd
commit 33b4020e06
11 changed files with 1351 additions and 312 deletions
Download Patch File Download Diff File Expand all files Collapse all files

77
infra/k8s/monitoring/dashboards/cluster-overview.json
View File

@@ -1,4 +1,7 @@
{
"__inputs": [
{ "name": "DS_PROMETHEUS", "label": "Prometheus", "description": "", "type": "datasource", "pluginId": "prometheus" }
],
"id": null,
"uid": "foxhunt-cluster-overview",
"title": "K8s Cluster Overview",
@@ -22,7 +25,7 @@
"type": "query",
"datasource": {
"type": "prometheus",
"uid": "cfegaydmscvswe"
"uid": "${DS_PROMETHEUS}"
},
"query": "label_values(kube_pod_info, namespace)",
"refresh": 2,
@@ -41,7 +44,7 @@
"type": "query",
"datasource": {
"type": "prometheus",
"uid": "cfegaydmscvswe"
"uid": "${DS_PROMETHEUS}"
},
"query": "label_values(node_uname_info, nodename)",
"refresh": 2,
@@ -71,7 +74,7 @@
"type": "stat",
"title": "Total Nodes",
"gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": {
@@ -97,7 +100,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "count(kube_node_info)",
"legendFormat": "Nodes",
"instant": true
@@ -109,7 +112,7 @@
"type": "stat",
"title": "Pods Running",
"gridPos": { "h": 4, "w": 4, "x": 4, "y": 1 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": {
@@ -134,7 +137,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(kube_pod_status_phase{namespace=~\"$namespace\", phase=\"Running\"})",
"legendFormat": "Running",
"instant": true
@@ -146,7 +149,7 @@
"type": "stat",
"title": "Pods Pending",
"gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": {
@@ -173,7 +176,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(kube_pod_status_phase{namespace=~\"$namespace\", phase=\"Pending\"}) or vector(0)",
"legendFormat": "Pending",
"instant": true
@@ -185,7 +188,7 @@
"type": "stat",
"title": "Pods Failed",
"gridPos": { "h": 4, "w": 4, "x": 12, "y": 1 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": {
@@ -211,7 +214,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(kube_pod_status_phase{namespace=~\"$namespace\", phase=\"Failed\"}) or vector(0)",
"legendFormat": "Failed",
"instant": true
@@ -223,7 +226,7 @@
"type": "gauge",
"title": "Cluster CPU Usage %",
"gridPos": { "h": 4, "w": 4, "x": 16, "y": 1 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": {
@@ -251,7 +254,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "100 - (avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m])) * 100)",
"legendFormat": "CPU %",
"instant": true
@@ -263,7 +266,7 @@
"type": "gauge",
"title": "Cluster Memory Usage %",
"gridPos": { "h": 4, "w": 4, "x": 20, "y": 1 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": {
@@ -291,7 +294,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "(1 - (sum(node_memory_MemAvailable_bytes) / sum(node_memory_MemTotal_bytes))) * 100",
"legendFormat": "Memory %",
"instant": true
@@ -311,7 +314,7 @@
"type": "timeseries",
"title": "Per-Node CPU Usage",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -341,7 +344,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))",
"legendFormat": "{{ instance }}"
}
@@ -352,7 +355,7 @@
"type": "timeseries",
"title": "CPU Requests vs Limits vs Usage",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -380,19 +383,19 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(kube_pod_container_resource_requests{namespace=~\"$namespace\", resource=\"cpu\"})",
"legendFormat": "Requests"
},
{
"refId": "B",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(kube_pod_container_resource_limits{namespace=~\"$namespace\", resource=\"cpu\"})",
"legendFormat": "Limits"
},
{
"refId": "C",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\", image!=\"\"}[5m]))",
"legendFormat": "Usage"
}
@@ -411,7 +414,7 @@
"type": "timeseries",
"title": "Per-Node Memory Usage",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 15 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -439,7 +442,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes",
"legendFormat": "{{ instance }}"
}
@@ -450,7 +453,7 @@
"type": "timeseries",
"title": "Memory Requests vs Limits vs Usage",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 15 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -478,19 +481,19 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(kube_pod_container_resource_requests{namespace=~\"$namespace\", resource=\"memory\"})",
"legendFormat": "Requests"
},
{
"refId": "B",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(kube_pod_container_resource_limits{namespace=~\"$namespace\", resource=\"memory\"})",
"legendFormat": "Limits"
},
{
"refId": "C",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(container_memory_working_set_bytes{namespace=~\"$namespace\", image!=\"\"})",
"legendFormat": "Usage"
}
@@ -509,7 +512,7 @@
"type": "timeseries",
"title": "Node Filesystem Usage",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -539,7 +542,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "1 - (node_filesystem_avail_bytes{mountpoint=\"/\", fstype!=\"tmpfs\"} / node_filesystem_size_bytes{mountpoint=\"/\", fstype!=\"tmpfs\"})",
"legendFormat": "{{ instance }}"
}
@@ -550,7 +553,7 @@
"type": "timeseries",
"title": "Disk I/O Throughput",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -578,13 +581,13 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum by (instance) (rate(node_disk_read_bytes_total[5m]))",
"legendFormat": "{{ instance }} read"
},
{
"refId": "B",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum by (instance) (rate(node_disk_written_bytes_total[5m]))",
"legendFormat": "{{ instance }} write"
}
@@ -603,7 +606,7 @@
"type": "timeseries",
"title": "Node Network Receive / Transmit",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 33 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -631,13 +634,13 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum by (instance) (rate(node_network_receive_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[5m]))",
"legendFormat": "{{ instance }} rx"
},
{
"refId": "B",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum by (instance) (rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[5m]))",
"legendFormat": "{{ instance }} tx"
}
@@ -648,7 +651,7 @@
"type": "timeseries",
"title": "Pod Network I/O",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 33 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -676,13 +679,13 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum by (namespace) (rate(container_network_receive_bytes_total{namespace=~\"$namespace\", image!=\"\"}[5m]))",
"legendFormat": "{{ namespace }} rx"
},
{
"refId": "B",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum by (namespace) (rate(container_network_transmit_bytes_total{namespace=~\"$namespace\", image!=\"\"}[5m]))",
"legendFormat": "{{ namespace }} tx"
}

87
infra/k8s/monitoring/dashboards/foxhunt-ci-pipelines.json
View File

@@ -1,4 +1,7 @@
{
"__inputs": [
{ "name": "DS_PROMETHEUS", "label": "Prometheus", "description": "", "type": "datasource", "pluginId": "prometheus" }
],
"id": null,
"uid": "foxhunt-ci-pipelines",
"title": "Foxhunt CI/CD Pipelines",
@@ -46,7 +49,7 @@
"title": "Pipeline Creation Rate",
"description": "Pipelines created per second (averaged over $interval)",
"gridPos": { "h": 6, "w": 6, "x": 0, "y": 1 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"unit": "ops",
@@ -73,7 +76,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(rate(gitlab_ci_pipeline_processing_events_total[$interval]))",
"legendFormat": "pipelines/s",
"instant": true
@@ -86,7 +89,7 @@
"title": "Active CI Jobs",
"description": "Current number of active CI jobs",
"gridPos": { "h": 6, "w": 6, "x": 6, "y": 1 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"unit": "short",
@@ -114,7 +117,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(gitlab_ci_active_jobs_sum)",
"legendFormat": "active jobs",
"instant": true
@@ -127,7 +130,7 @@
"title": "Pipeline Failure Rate",
"description": "Pipeline failures per second (averaged over $interval)",
"gridPos": { "h": 6, "w": 6, "x": 12, "y": 1 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"unit": "ops",
@@ -155,7 +158,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(rate(gitlab_ci_pipeline_failure_reasons[$interval]))",
"legendFormat": "failures/s",
"instant": true
@@ -168,7 +171,7 @@
"title": "Queue Operations Rate",
"description": "CI queue operations per second (averaged over $interval)",
"gridPos": { "h": 6, "w": 6, "x": 18, "y": 1 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"unit": "ops",
@@ -195,7 +198,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(rate(gitlab_ci_queue_operations_total[$interval]))",
"legendFormat": "queue ops/s",
"instant": true
@@ -217,7 +220,7 @@
"title": "Pipeline Creation Duration",
"description": "Time to create a pipeline (P50, P90, P99)",
"gridPos": { "h": 8, "w": 8, "x": 0, "y": 8 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -264,19 +267,19 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "histogram_quantile(0.50, sum(rate(gitlab_ci_pipeline_creation_duration_seconds_bucket[$interval])) by (le))",
"legendFormat": "P50"
},
{
"refId": "B",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "histogram_quantile(0.90, sum(rate(gitlab_ci_pipeline_creation_duration_seconds_bucket[$interval])) by (le))",
"legendFormat": "P90"
},
{
"refId": "C",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "histogram_quantile(0.99, sum(rate(gitlab_ci_pipeline_creation_duration_seconds_bucket[$interval])) by (le))",
"legendFormat": "P99"
}
@@ -288,7 +291,7 @@
"title": "Pipeline Size (Builds per Pipeline)",
"description": "Number of builds/jobs per pipeline (P50, P90, P99)",
"gridPos": { "h": 8, "w": 8, "x": 8, "y": 8 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -335,19 +338,19 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "histogram_quantile(0.50, sum(rate(gitlab_ci_pipeline_size_builds_bucket[$interval])) by (le))",
"legendFormat": "P50"
},
{
"refId": "B",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "histogram_quantile(0.90, sum(rate(gitlab_ci_pipeline_size_builds_bucket[$interval])) by (le))",
"legendFormat": "P90"
},
{
"refId": "C",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "histogram_quantile(0.99, sum(rate(gitlab_ci_pipeline_size_builds_bucket[$interval])) by (le))",
"legendFormat": "P99"
}
@@ -359,7 +362,7 @@
"title": "Active Jobs Distribution",
"description": "Heatmap of active CI job counts over time",
"gridPos": { "h": 8, "w": 8, "x": 16, "y": 8 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -393,7 +396,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(increase(gitlab_ci_active_jobs_bucket[$interval])) by (le)",
"legendFormat": "{{le}}",
"format": "heatmap"
@@ -415,7 +418,7 @@
"title": "Pipeline Failure Reasons",
"description": "Breakdown of pipeline failure reasons over the selected time range",
"gridPos": { "h": 8, "w": 8, "x": 0, "y": 17 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"unit": "short",
@@ -474,7 +477,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum by (reason) (increase(gitlab_ci_pipeline_failure_reasons[$__range]))",
"legendFormat": "{{reason}}",
"instant": true,
@@ -488,7 +491,7 @@
"title": "Job Failure Reasons",
"description": "Breakdown of individual job failure reasons over the selected time range",
"gridPos": { "h": 8, "w": 8, "x": 8, "y": 17 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"unit": "short",
@@ -547,7 +550,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum by (failure_reason) (increase(gitlab_ci_job_failure_reasons[$__range]))",
"legendFormat": "{{failure_reason}}",
"instant": true,
@@ -561,7 +564,7 @@
"title": "Build Trace Errors",
"description": "Rate of build trace errors over time",
"gridPos": { "h": 8, "w": 8, "x": 16, "y": 17 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -595,7 +598,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(rate(gitlab_ci_build_trace_errors_total[$interval]))",
"legendFormat": "trace errors/s"
}
@@ -616,7 +619,7 @@
"title": "Runner Authentication Success",
"description": "Rate of successful runner authentication attempts",
"gridPos": { "h": 8, "w": 8, "x": 0, "y": 26 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -644,7 +647,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(rate(gitlab_ci_runner_authentication_success_total[$interval]))",
"legendFormat": "auth success/s"
}
@@ -656,7 +659,7 @@
"title": "Trace Operations & Volume",
"description": "CI trace operations rate and data volume",
"gridPos": { "h": 8, "w": 8, "x": 8, "y": 26 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -692,13 +695,13 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(rate(gitlab_ci_trace_operations_total[$interval]))",
"legendFormat": "trace ops/s"
},
{
"refId": "B",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(rate(gitlab_ci_trace_bytes_total[$interval]))",
"legendFormat": "trace bytes/s"
}
@@ -710,7 +713,7 @@
"title": "Artifacts Cleanup",
"description": "Rate and volume of destroyed job artifacts",
"gridPos": { "h": 8, "w": 8, "x": 16, "y": 26 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -750,13 +753,13 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(rate(destroyed_job_artifacts_count_total[$interval]))",
"legendFormat": "artifacts destroyed/s"
},
{
"refId": "B",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(rate(destroyed_job_artifacts_bytes_total[$interval]))",
"legendFormat": "bytes destroyed/s"
}
@@ -777,7 +780,7 @@
"title": "Training Jobs: Created vs Completed vs Failed",
"description": "K8s training job lifecycle in the foxhunt namespace",
"gridPos": { "h": 8, "w": 8, "x": 0, "y": 35 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -824,19 +827,19 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "count(kube_job_info{namespace=\"foxhunt\", job_name=~\".*train.*\"})",
"legendFormat": "Created"
},
{
"refId": "B",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "count(kube_job_status_succeeded{namespace=\"foxhunt\", job_name=~\".*train.*\"} == 1)",
"legendFormat": "Succeeded"
},
{
"refId": "C",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "count(kube_job_status_failed{namespace=\"foxhunt\", job_name=~\".*train.*\"} > 0)",
"legendFormat": "Failed"
}
@@ -848,7 +851,7 @@
"title": "Training Job Duration",
"description": "Age of active training jobs (time since creation)",
"gridPos": { "h": 8, "w": 8, "x": 8, "y": 35 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -883,7 +886,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "time() - kube_job_created{namespace=\"foxhunt\", job_name=~\".*train.*\"} and on(job_name, namespace) kube_job_status_succeeded{namespace=\"foxhunt\"} == 0 and on(job_name, namespace) (kube_job_status_failed{namespace=\"foxhunt\"} == 0 or absent(kube_job_status_failed{namespace=\"foxhunt\"}))",
"legendFormat": "{{job_name}}"
}
@@ -895,7 +898,7 @@
"title": "Training Pod CPU Usage",
"description": "CPU usage of training pods in the foxhunt namespace",
"gridPos": { "h": 8, "w": 4, "x": 16, "y": 35 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -923,7 +926,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\".*train.*\", container!=\"POD\", container!=\"\"}[$interval]))",
"legendFormat": "{{pod}}"
}
@@ -935,7 +938,7 @@
"title": "Training Pod Memory",
"description": "Memory working set of training pods in the foxhunt namespace",
"gridPos": { "h": 8, "w": 4, "x": 20, "y": 35 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -963,7 +966,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum by (pod) (container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\".*train.*\", container!=\"POD\", container!=\"\"})",
"legendFormat": "{{pod}}"
}

931
infra/k8s/monitoring/dashboards/foxhunt-cockpit.json Normal file
View File

@@ -0,0 +1,931 @@
{
"__inputs": [
{ "name": "DS_PROMETHEUS", "label": "Prometheus", "description": "", "type": "datasource", "pluginId": "prometheus" },
{ "name": "DS_LOKI", "label": "Loki", "description": "", "type": "datasource", "pluginId": "loki" }
],
"id": null,
"uid": "foxhunt-cockpit",
"title": "Foxhunt - Cockpit",
"description": "Single-page overview: cluster health, all services, GPU/training, resources, errors, live logs",
"tags": ["foxhunt", "production"],
"timezone": "browser",
"schemaVersion": 39,
"version": 1,
"editable": true,
"refresh": "30s",
"time": { "from": "now-1h", "to": "now" },
"fiscalYearStartMonth": 0,
"liveNow": false,
"weekStart": "",
"templating": {
"list": [
{
"name": "interval",
"type": "interval",
"query": "1m,5m,15m,30m,1h",
"current": { "selected": true, "text": "5m", "value": "5m" },
"auto": false
}
]
},
"panels": [
{
"id": 1,
"type": "row",
"title": "Health Bar",
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
"collapsed": false,
"panels": []
},
{
"id": 2,
"type": "stat",
"title": "Nodes Ready",
"gridPos": { "h": 3, "w": 4, "x": 0, "y": 1 },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] },
"mappings": [],
"color": { "mode": "thresholds" }
},
"overrides": []
},
"options": {
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
"orientation": "auto",
"textMode": "auto",
"colorMode": "background",
"graphMode": "none",
"justifyMode": "center"
},
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(kube_node_status_condition{condition=\"Ready\",status=\"true\"})",
"instant": true
}
]
},
{
"id": 3,
"type": "stat",
"title": "Pods Running",
"gridPos": { "h": 3, "w": 4, "x": 4, "y": 1 },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": { "mode": "absolute", "steps": [{ "color": "yellow", "value": null }, { "color": "green", "value": 1 }] },
"mappings": [],
"color": { "mode": "thresholds" }
},
"overrides": []
},
"options": {
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
"orientation": "auto",
"textMode": "auto",
"colorMode": "background",
"graphMode": "none",
"justifyMode": "center"
},
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\",phase=\"Running\"})",
"instant": true
}
]
},
{
"id": 4,
"type": "stat",
"title": "Pods Not Ready",
"gridPos": { "h": 3, "w": 4, "x": 8, "y": 1 },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 1 }, { "color": "red", "value": 3 }] },
"mappings": [],
"color": { "mode": "thresholds" }
},
"overrides": []
},
"options": {
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
"orientation": "auto",
"textMode": "auto",
"colorMode": "background",
"graphMode": "none",
"justifyMode": "center"
},
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\",phase=~\"Pending|Failed|Unknown\"}) or vector(0)",
"instant": true
}
]
},
{
"id": 5,
"type": "stat",
"title": "Training Jobs",
"gridPos": { "h": 3, "w": 4, "x": 12, "y": 1 },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": { "mode": "absolute", "steps": [{ "color": "blue", "value": null }] },
"mappings": [],
"color": { "mode": "thresholds" }
},
"overrides": []
},
"options": {
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
"orientation": "auto",
"textMode": "auto",
"colorMode": "background",
"graphMode": "none",
"justifyMode": "center"
},
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(kube_job_status_active{namespace=\"foxhunt\"}) or vector(0)",
"instant": true
}
]
},
{
"id": 6,
"type": "stat",
"title": "GPU Available",
"gridPos": { "h": 3, "w": 4, "x": 16, "y": 1 },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] },
"mappings": [],
"color": { "mode": "thresholds" }
},
"overrides": []
},
"options": {
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
"orientation": "auto",
"textMode": "auto",
"colorMode": "background",
"graphMode": "none",
"justifyMode": "center"
},
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(kube_node_status_allocatable{resource=\"nvidia_com_gpu\"}) or vector(0)",
"instant": true
}
]
},
{
"id": 7,
"type": "stat",
"title": "Errors /min",
"gridPos": { "h": 3, "w": 4, "x": 20, "y": 1 },
"datasource": { "type": "loki", "uid": "${DS_LOKI}" },
"fieldConfig": {
"defaults": {
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 5 }, { "color": "red", "value": 20 }] },
"mappings": [],
"color": { "mode": "thresholds" }
},
"overrides": []
},
"options": {
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
"orientation": "auto",
"textMode": "auto",
"colorMode": "background",
"graphMode": "none",
"justifyMode": "center"
},
"targets": [
{
"refId": "A",
"datasource": { "type": "loki", "uid": "${DS_LOKI}" },
"expr": "sum(count_over_time({namespace=\"foxhunt\"} |= \"ERROR\" [1m]))",
"instant": true
}
]
},
{
"id": 8,
"type": "row",
"title": "Service Status",
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 4 },
"collapsed": false,
"panels": []
},
{
"id": 9,
"type": "stat",
"title": "api-gateway",
"gridPos": { "h": 3, "w": 3, "x": 0, "y": 5 },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] },
"mappings": [{ "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }],
"color": { "mode": "thresholds" }
},
"overrides": []
},
"options": {
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
"orientation": "auto",
"textMode": "auto",
"colorMode": "background",
"graphMode": "none",
"justifyMode": "center"
},
"links": [{ "title": "Service Detail", "url": "/d/foxhunt-services?var-service=api-gateway", "targetBlank": false }],
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"api-gateway.*\", phase=\"Running\"}) > 0 or vector(0)",
"instant": true
}
]
},
{
"id": 10,
"type": "stat",
"title": "trading-service",
"gridPos": { "h": 3, "w": 3, "x": 3, "y": 5 },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] },
"mappings": [{ "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }],
"color": { "mode": "thresholds" }
},
"overrides": []
},
"options": {
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
"orientation": "auto",
"textMode": "auto",
"colorMode": "background",
"graphMode": "none",
"justifyMode": "center"
},
"links": [{ "title": "Service Detail", "url": "/d/foxhunt-services?var-service=trading-service", "targetBlank": false }],
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"trading-service.*\", phase=\"Running\"}) > 0 or vector(0)",
"instant": true
}
]
},
{
"id": 11,
"type": "stat",
"title": "ml-training",
"gridPos": { "h": 3, "w": 3, "x": 6, "y": 5 },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] },
"mappings": [{ "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }],
"color": { "mode": "thresholds" }
},
"overrides": []
},
"options": {
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
"orientation": "auto",
"textMode": "auto",
"colorMode": "background",
"graphMode": "none",
"justifyMode": "center"
},
"links": [{ "title": "Service Detail", "url": "/d/foxhunt-services?var-service=ml-training-service", "targetBlank": false }],
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"ml-training-service.*\", phase=\"Running\"}) > 0 or vector(0)",
"instant": true
}
]
},
{
"id": 12,
"type": "stat",
"title": "backtesting",
"gridPos": { "h": 3, "w": 3, "x": 9, "y": 5 },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] },
"mappings": [{ "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }],
"color": { "mode": "thresholds" }
},
"overrides": []
},
"options": {
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
"orientation": "auto",
"textMode": "auto",
"colorMode": "background",
"graphMode": "none",
"justifyMode": "center"
},
"links": [{ "title": "Service Detail", "url": "/d/foxhunt-services?var-service=backtesting-service", "targetBlank": false }],
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"backtesting-service.*\", phase=\"Running\"}) > 0 or vector(0)",
"instant": true
}
]
},
{
"id": 13,
"type": "stat",
"title": "trading-agent",
"gridPos": { "h": 3, "w": 3, "x": 12, "y": 5 },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] },
"mappings": [{ "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }],
"color": { "mode": "thresholds" }
},
"overrides": []
},
"options": {
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
"orientation": "auto",
"textMode": "auto",
"colorMode": "background",
"graphMode": "none",
"justifyMode": "center"
},
"links": [{ "title": "Service Detail", "url": "/d/foxhunt-services?var-service=trading-agent-service", "targetBlank": false }],
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"trading-agent.*\", phase=\"Running\"}) > 0 or vector(0)",
"instant": true
}
]
},
{
"id": 14,
"type": "stat",
"title": "broker-gw",
"gridPos": { "h": 3, "w": 3, "x": 15, "y": 5 },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] },
"mappings": [{ "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }],
"color": { "mode": "thresholds" }
},
"overrides": []
},
"options": {
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
"orientation": "auto",
"textMode": "auto",
"colorMode": "background",
"graphMode": "none",
"justifyMode": "center"
},
"links": [{ "title": "Service Detail", "url": "/d/foxhunt-services?var-service=broker-gateway", "targetBlank": false }],
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"broker-gateway.*\", phase=\"Running\"}) > 0 or vector(0)",
"instant": true
}
]
},
{
"id": 15,
"type": "stat",
"title": "data-acq",
"gridPos": { "h": 3, "w": 3, "x": 18, "y": 5 },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] },
"mappings": [{ "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }],
"color": { "mode": "thresholds" }
},
"overrides": []
},
"options": {
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
"orientation": "auto",
"textMode": "auto",
"colorMode": "background",
"graphMode": "none",
"justifyMode": "center"
},
"links": [{ "title": "Service Detail", "url": "/d/foxhunt-services?var-service=data-acquisition-service", "targetBlank": false }],
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"data-acquisition.*\", phase=\"Running\"}) > 0 or vector(0)",
"instant": true
}
]
},
{
"id": 16,
"type": "stat",
"title": "web-gateway",
"gridPos": { "h": 3, "w": 3, "x": 21, "y": 5 },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] },
"mappings": [{ "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }],
"color": { "mode": "thresholds" }
},
"overrides": []
},
"options": {
"reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
"orientation": "auto",
"textMode": "auto",
"colorMode": "background",
"graphMode": "none",
"justifyMode": "center"
},
"links": [{ "title": "Service Detail", "url": "/d/foxhunt-services?var-service=web-gateway", "targetBlank": false }],
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"web-gateway.*\", phase=\"Running\"}) > 0 or vector(0)",
"instant": true
}
]
},
{
"id": 17,
"type": "row",
"title": "Resources",
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 8 },
"collapsed": false,
"panels": []
},
{
"id": 18,
"type": "gauge",
"title": "Cluster CPU %",
"gridPos": { "h": 5, "w": 4, "x": 0, "y": 9 },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 70 }, { "color": "red", "value": 90 }] },
"unit": "percent",
"min": 0,
"max": 100,
"mappings": [],
"color": { "mode": "thresholds" }
},
"overrides": []
},
"options": { "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true },
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "100 * (1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[$interval])))",
"instant": true
}
]
},
{
"id": 19,
"type": "gauge",
"title": "Cluster Memory %",
"gridPos": { "h": 5, "w": 4, "x": 4, "y": 9 },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 75 }, { "color": "red", "value": 90 }] },
"unit": "percent",
"min": 0,
"max": 100,
"mappings": [],
"color": { "mode": "thresholds" }
},
"overrides": []
},
"options": { "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true },
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "100 * (1 - sum(node_memory_MemAvailable_bytes) / sum(node_memory_MemTotal_bytes))",
"instant": true
}
]
},
{
"id": 20,
"type": "gauge",
"title": "Cluster Disk %",
"gridPos": { "h": 5, "w": 4, "x": 8, "y": 9 },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 75 }, { "color": "red", "value": 90 }] },
"unit": "percent",
"min": 0,
"max": 100,
"mappings": [],
"color": { "mode": "thresholds" }
},
"overrides": []
},
"options": { "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true },
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "100 * (1 - sum(node_filesystem_avail_bytes{mountpoint=\"/\",fstype!=\"tmpfs\"}) / sum(node_filesystem_size_bytes{mountpoint=\"/\",fstype!=\"tmpfs\"}))",
"instant": true
}
]
},
{
"id": 21,
"type": "timeseries",
"title": "CPU by Service",
"gridPos": { "h": 5, "w": 12, "x": 12, "y": 9 },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 30,
"gradientMode": "none",
"spanNulls": false,
"showPoints": "never",
"stacking": { "mode": "normal", "group": "A" },
"axisPlacement": "auto",
"lineWidth": 1
},
"unit": "short",
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
"color": { "mode": "palette-classic" },
"mappings": []
},
"overrides": []
},
"options": { "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "desc" } },
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\",container!=\"\",container!=\"POD\"}[$interval]))",
"legendFormat": "{{ pod }}"
}
]
},
{
"id": 22,
"type": "row",
"title": "GPU & Training",
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 },
"collapsed": false,
"panels": []
},
{
"id": 23,
"type": "gauge",
"title": "GPU Utilization %",
"gridPos": { "h": 5, "w": 6, "x": 0, "y": 15 },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": { "mode": "absolute", "steps": [{ "color": "blue", "value": null }, { "color": "green", "value": 30 }, { "color": "yellow", "value": 80 }, { "color": "red", "value": 95 }] },
"unit": "percent",
"min": 0,
"max": 100,
"mappings": [],
"color": { "mode": "thresholds" }
},
"overrides": []
},
"options": { "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true },
"links": [{ "title": "GPU & Training Detail", "url": "/d/foxhunt-gpu-training", "targetBlank": false }],
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "avg(DCGM_FI_DEV_GPU_UTIL) or vector(0)",
"instant": true
}
]
},
{
"id": 24,
"type": "gauge",
"title": "GPU Memory %",
"gridPos": { "h": 5, "w": 6, "x": 6, "y": 15 },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 70 }, { "color": "red", "value": 90 }] },
"unit": "percent",
"min": 0,
"max": 100,
"mappings": [],
"color": { "mode": "thresholds" }
},
"overrides": []
},
"options": { "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true },
"links": [{ "title": "GPU & Training Detail", "url": "/d/foxhunt-gpu-training", "targetBlank": false }],
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "100 * avg(DCGM_FI_DEV_FB_USED / (DCGM_FI_DEV_FB_USED + DCGM_FI_DEV_FB_FREE)) or vector(0)",
"instant": true
}
]
},
{
"id": 25,
"type": "timeseries",
"title": "Training Jobs Timeline",
"gridPos": { "h": 5, "w": 12, "x": 12, "y": 15 },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"lineInterpolation": "stepAfter",
"fillOpacity": 20,
"gradientMode": "none",
"spanNulls": false,
"showPoints": "never",
"stacking": { "mode": "none", "group": "A" },
"axisPlacement": "auto",
"lineWidth": 2
},
"unit": "short",
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
"color": { "mode": "palette-classic" },
"mappings": []
},
"overrides": [
{ "matcher": { "id": "byName", "options": "active" }, "properties": [{ "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } }] },
{ "matcher": { "id": "byName", "options": "succeeded" }, "properties": [{ "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } }] },
{ "matcher": { "id": "byName", "options": "failed" }, "properties": [{ "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } }] }
]
},
"options": { "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "desc" } },
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(kube_job_status_active{namespace=\"foxhunt\"}) or vector(0)",
"legendFormat": "active"
},
{
"refId": "B",
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(kube_job_status_succeeded{namespace=\"foxhunt\"}) or vector(0)",
"legendFormat": "succeeded"
},
{
"refId": "C",
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(kube_job_status_failed{namespace=\"foxhunt\"}) or vector(0)",
"legendFormat": "failed"
}
]
},
{
"id": 26,
"type": "row",
"title": "Restarts & Network",
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 20 },
"collapsed": false,
"panels": []
},
{
"id": 27,
"type": "timeseries",
"title": "Pod Restarts (1h)",
"gridPos": { "h": 5, "w": 12, "x": 0, "y": 21 },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "bars",
"lineInterpolation": "linear",
"fillOpacity": 80,
"gradientMode": "none",
"spanNulls": false,
"showPoints": "never",
"stacking": { "mode": "normal", "group": "A" },
"axisPlacement": "auto",
"lineWidth": 0
},
"unit": "short",
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "red", "value": 1 }] },
"color": { "mode": "palette-classic" },
"mappings": []
},
"overrides": []
},
"options": { "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "desc" } },
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "increase(kube_pod_container_status_restarts_total{namespace=\"foxhunt\"}[1h]) > 0",
"legendFormat": "{{ pod }}"
}
]
},
{
"id": 28,
"type": "timeseries",
"title": "Network I/O by Pod",
"gridPos": { "h": 5, "w": 12, "x": 12, "y": 21 },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"fillOpacity": 10,
"gradientMode": "none",
"spanNulls": false,
"showPoints": "never",
"stacking": { "mode": "none", "group": "A" },
"axisPlacement": "auto",
"lineWidth": 1
},
"unit": "Bps",
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
"color": { "mode": "palette-classic" },
"mappings": []
},
"overrides": []
},
"options": { "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "desc" } },
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum by (pod) (rate(container_network_receive_bytes_total{namespace=\"foxhunt\"}[$interval]))",
"legendFormat": "{{ pod }} rx"
},
{
"refId": "B",
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum by (pod) (rate(container_network_transmit_bytes_total{namespace=\"foxhunt\"}[$interval]))",
"legendFormat": "{{ pod }} tx"
}
]
},
{
"id": 29,
"type": "row",
"title": "Errors",
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 26 },
"collapsed": false,
"panels": []
},
{
"id": 30,
"type": "timeseries",
"title": "Error Rate by Service",
"gridPos": { "h": 5, "w": 12, "x": 0, "y": 27 },
"datasource": { "type": "loki", "uid": "${DS_LOKI}" },
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "bars",
"lineInterpolation": "linear",
"fillOpacity": 60,
"gradientMode": "none",
"spanNulls": false,
"showPoints": "never",
"stacking": { "mode": "normal", "group": "A" },
"axisPlacement": "auto",
"lineWidth": 0
},
"unit": "short",
"thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }] },
"color": { "mode": "palette-classic" },
"mappings": []
},
"overrides": []
},
"options": { "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "desc" } },
"targets": [
{
"refId": "A",
"datasource": { "type": "loki", "uid": "${DS_LOKI}" },
"expr": "sum by (app) (count_over_time({namespace=\"foxhunt\"} |= \"ERROR\" [$interval]))",
"legendFormat": "{{ app }}"
}
]
},
{
"id": 31,
"type": "table",
"title": "Top Error Messages",
"gridPos": { "h": 5, "w": 12, "x": 12, "y": 27 },
"datasource": { "type": "loki", "uid": "${DS_LOKI}" },
"fieldConfig": {
"defaults": {
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
"mappings": [],
"color": { "mode": "thresholds" }
},
"overrides": []
},
"options": {
"showHeader": true,
"sortBy": [{ "displayName": "Value", "desc": true }],
"footer": { "show": false }
},
"targets": [
{
"refId": "A",
"datasource": { "type": "loki", "uid": "${DS_LOKI}" },
"expr": "topk(10, sum by (app) (count_over_time({namespace=\"foxhunt\"} |= \"ERROR\" | logfmt [$interval])))",
"legendFormat": "{{ app }}",
"instant": true
}
]
},
{
"id": 32,
"type": "row",
"title": "Live Error Log",
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 32 },
"collapsed": false,
"panels": []
},
{
"id": 33,
"type": "logs",
"title": "Recent Errors",
"description": "Live stream of ERROR-level log lines from all foxhunt services",
"gridPos": { "h": 7, "w": 24, "x": 0, "y": 33 },
"datasource": { "type": "loki", "uid": "${DS_LOKI}" },
"options": {
"showTime": true,
"showLabels": true,
"showCommonLabels": false,
"wrapLogMessage": true,
"prettifyLogMessage": false,
"enableLogDetails": true,
"sortOrder": "Descending",
"dedupStrategy": "none"
},
"targets": [
{
"refId": "A",
"datasource": { "type": "loki", "uid": "${DS_LOKI}" },
"expr": "{namespace=\"foxhunt\"} |= \"ERROR\"",
"maxLines": 200
}
]
}
]
}

80
infra/k8s/monitoring/dashboards/foxhunt-gpu-training.json
View File

@@ -1,4 +1,8 @@
{
"__inputs": [
{ "name": "DS_PROMETHEUS", "label": "Prometheus", "description": "", "type": "datasource", "pluginId": "prometheus" },
{ "name": "DS_LOKI", "label": "Loki", "description": "", "type": "datasource", "pluginId": "loki" }
],
"id": null,
"uid": "foxhunt-gpu-training",
"title": "Foxhunt - GPU & Training Jobs",
@@ -21,7 +25,7 @@
{
"name": "gpu",
"type": "query",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"query": "label_values(DCGM_FI_DEV_GPU_UTIL, gpu)",
"refresh": 2,
"includeAll": true,
@@ -54,7 +58,7 @@
"title": "ci-compile Nodes",
"description": "L4-1-24G GPU nodes (0-1 autoscale)",
"gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": {
@@ -80,7 +84,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "count(kube_node_labels{label_k8s_scaleway_com_pool_name=\"ci-compile\"}) or vector(0)",
"instant": true
}
@@ -92,7 +96,7 @@
"title": "ci-training Nodes",
"description": "L40S-1-48G GPU nodes (0-1 autoscale)",
"gridPos": { "h": 4, "w": 4, "x": 4, "y": 1 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": {
@@ -118,7 +122,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "count(kube_node_labels{label_k8s_scaleway_com_pool_name=\"ci-training\"}) or vector(0)",
"instant": true
}
@@ -130,7 +134,7 @@
"title": "ci-compile-cpu Nodes",
"description": "POP2-32C-128G CPU nodes (0-1 autoscale)",
"gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": {
@@ -156,7 +160,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "count(kube_node_labels{label_k8s_scaleway_com_pool_name=\"ci-compile-cpu\"}) or vector(0)",
"instant": true
}
@@ -167,7 +171,7 @@
"type": "stat",
"title": "Active Training Jobs",
"gridPos": { "h": 4, "w": 4, "x": 12, "y": 1 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": {
@@ -193,7 +197,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(kube_job_status_active{namespace=\"foxhunt\"}) or vector(0)",
"instant": true
}
@@ -204,7 +208,7 @@
"type": "stat",
"title": "Jobs Succeeded (24h)",
"gridPos": { "h": 4, "w": 4, "x": 16, "y": 1 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": {
@@ -229,7 +233,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(increase(kube_job_status_succeeded{namespace=\"foxhunt\"}[24h])) or vector(0)",
"instant": true
}
@@ -240,7 +244,7 @@
"type": "stat",
"title": "Jobs Failed (24h)",
"gridPos": { "h": 4, "w": 4, "x": 20, "y": 1 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": {
@@ -266,7 +270,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(increase(kube_job_status_failed{namespace=\"foxhunt\"}[24h])) or vector(0)",
"instant": true
}
@@ -285,7 +289,7 @@
"type": "timeseries",
"title": "GPU Utilization %",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -322,7 +326,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "DCGM_FI_DEV_GPU_UTIL{gpu=~\"$gpu\"}",
"legendFormat": "GPU {{ gpu }} ({{ instance }})"
}
@@ -333,7 +337,7 @@
"type": "timeseries",
"title": "GPU Memory Used (MB)",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -361,13 +365,13 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "DCGM_FI_DEV_FB_USED{gpu=~\"$gpu\"}",
"legendFormat": "GPU {{ gpu }} used"
},
{
"refId": "B",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "DCGM_FI_DEV_FB_TOTAL{gpu=~\"$gpu\"}",
"legendFormat": "GPU {{ gpu }} total"
}
@@ -378,7 +382,7 @@
"type": "timeseries",
"title": "GPU Temperature",
"gridPos": { "h": 6, "w": 12, "x": 0, "y": 14 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -414,7 +418,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "DCGM_FI_DEV_GPU_TEMP{gpu=~\"$gpu\"}",
"legendFormat": "GPU {{ gpu }}"
}
@@ -425,7 +429,7 @@
"type": "timeseries",
"title": "GPU Power (Watts)",
"gridPos": { "h": 6, "w": 12, "x": 12, "y": 14 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -453,7 +457,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "DCGM_FI_DEV_POWER_USAGE{gpu=~\"$gpu\"}",
"legendFormat": "GPU {{ gpu }}"
}
@@ -472,7 +476,7 @@
"type": "timeseries",
"title": "Active Jobs Over Time",
"gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -500,7 +504,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "kube_job_status_active{namespace=\"foxhunt\"}",
"legendFormat": "{{ job_name }}"
}
@@ -511,7 +515,7 @@
"type": "table",
"title": "Job Status Summary",
"gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
@@ -545,7 +549,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "kube_job_status_active{namespace=\"foxhunt\"}",
"legendFormat": "Active",
"instant": true,
@@ -553,7 +557,7 @@
},
{
"refId": "B",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "kube_job_status_succeeded{namespace=\"foxhunt\"}",
"legendFormat": "Succeeded",
"instant": true,
@@ -561,7 +565,7 @@
},
{
"refId": "C",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "kube_job_status_failed{namespace=\"foxhunt\"}",
"legendFormat": "Failed",
"instant": true,
@@ -575,7 +579,7 @@
"title": "GPU Node CPU Usage (Compute Pools)",
"description": "CPU usage on GPU/compute node pools when no DCGM data is available",
"gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -605,7 +609,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[$interval]))",
"legendFormat": "{{ instance }}"
}
@@ -616,7 +620,7 @@
"type": "timeseries",
"title": "GPU Node Memory Usage (Compute Pools)",
"gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -644,7 +648,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes",
"legendFormat": "{{ instance }}"
}
@@ -663,7 +667,7 @@
"type": "logs",
"title": "Training Job Logs",
"gridPos": { "h": 10, "w": 12, "x": 0, "y": 36 },
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
"datasource": { "type": "loki", "uid": "${DS_LOKI}" },
"options": {
"showTime": true,
"showLabels": true,
@@ -677,7 +681,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
"datasource": { "type": "loki", "uid": "${DS_LOKI}" },
"expr": "{namespace=\"foxhunt\", container=~\".*train.*\"}",
"maxLines": 200
}
@@ -688,7 +692,7 @@
"type": "logs",
"title": "CI / GitLab Runner Logs",
"gridPos": { "h": 10, "w": 12, "x": 12, "y": 36 },
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
"datasource": { "type": "loki", "uid": "${DS_LOKI}" },
"options": {
"showTime": true,
"showLabels": true,
@@ -702,7 +706,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
"datasource": { "type": "loki", "uid": "${DS_LOKI}" },
"expr": "{namespace=\"foxhunt\", app=~\".*runner.*|.*gitlab.*\"}",
"maxLines": 200
}
@@ -722,7 +726,7 @@
"title": "Node Count Over Time",
"description": "Tracks autoscaling of ci-compile and ci-training pools",
"gridPos": { "h": 7, "w": 24, "x": 0, "y": 47 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -751,7 +755,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "count by (label_k8s_scaleway_com_pool_name) (kube_node_labels)",
"legendFormat": "{{ label_k8s_scaleway_com_pool_name }}"
}

77
infra/k8s/monitoring/dashboards/foxhunt-infrastructure.json
View File

@@ -1,4 +1,7 @@
{
"__inputs": [
{ "name": "DS_PROMETHEUS", "label": "Prometheus", "description": "", "type": "datasource", "pluginId": "prometheus" }
],
"id": null,
"uid": "foxhunt-infrastructure",
"title": "Foxhunt - K8s Infrastructure",
@@ -21,7 +24,7 @@
{
"name": "node",
"type": "query",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"query": "label_values(kube_node_info, node)",
"refresh": 2,
"includeAll": true,
@@ -53,7 +56,7 @@
"type": "table",
"title": "Node Status by Pool",
"gridPos": { "h": 7, "w": 24, "x": 0, "y": 1 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
@@ -89,7 +92,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "kube_node_status_condition{condition=\"Ready\", status=\"true\", node=~\"$node\"}",
"legendFormat": "{{ node }}",
"instant": true,
@@ -103,7 +106,7 @@
"title": "services Pool",
"description": "DEV1-M, 2 fixed",
"gridPos": { "h": 3, "w": 4, "x": 0, "y": 8 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": {
@@ -130,7 +133,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "count(kube_node_labels{label_k8s_scaleway_com_pool_name=\"services\"} * on(node) kube_node_status_condition{condition=\"Ready\", status=\"true\"}) or vector(0)",
"legendFormat": "services",
"instant": true
@@ -143,7 +146,7 @@
"title": "gitlab Pool",
"description": "GP1-XS, 1 fixed",
"gridPos": { "h": 3, "w": 4, "x": 4, "y": 8 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": {
@@ -169,7 +172,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "count(kube_node_labels{label_k8s_scaleway_com_pool_name=\"gitlab\"} * on(node) kube_node_status_condition{condition=\"Ready\", status=\"true\"}) or vector(0)",
"legendFormat": "gitlab",
"instant": true
@@ -182,7 +185,7 @@
"title": "ci-compile-cpu Pool",
"description": "POP2-32C-128G, 0-1 autoscale",
"gridPos": { "h": 3, "w": 4, "x": 8, "y": 8 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": {
@@ -208,7 +211,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "count(kube_node_labels{label_k8s_scaleway_com_pool_name=\"ci-compile-cpu\"}) or vector(0)",
"legendFormat": "ci-compile-cpu",
"instant": true
@@ -221,7 +224,7 @@
"title": "ci-compile Pool (GPU)",
"description": "L4-1-24G, 0-1 autoscale",
"gridPos": { "h": 3, "w": 4, "x": 12, "y": 8 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": {
@@ -247,7 +250,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "count(kube_node_labels{label_k8s_scaleway_com_pool_name=\"ci-compile\"}) or vector(0)",
"legendFormat": "ci-compile",
"instant": true
@@ -260,7 +263,7 @@
"title": "ci-training Pool (GPU)",
"description": "L40S-1-48G, 0-1 autoscale",
"gridPos": { "h": 3, "w": 4, "x": 16, "y": 8 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": {
@@ -286,7 +289,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "count(kube_node_labels{label_k8s_scaleway_com_pool_name=\"ci-training\"}) or vector(0)",
"legendFormat": "ci-training",
"instant": true
@@ -298,7 +301,7 @@
"type": "stat",
"title": "Total Pods",
"gridPos": { "h": 3, "w": 4, "x": 20, "y": 8 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": {
@@ -323,7 +326,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(kubelet_running_pods) or count(kube_pod_info)",
"instant": true
}
@@ -342,7 +345,7 @@
"type": "timeseries",
"title": "Node CPU Usage",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -372,7 +375,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[$interval]))",
"legendFormat": "{{ instance }}"
}
@@ -383,7 +386,7 @@
"type": "timeseries",
"title": "Node Memory Usage",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -411,13 +414,13 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes",
"legendFormat": "{{ instance }} used"
},
{
"refId": "B",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "node_memory_MemTotal_bytes",
"legendFormat": "{{ instance }} total"
}
@@ -428,7 +431,7 @@
"type": "timeseries",
"title": "Node Filesystem Usage",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 20 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -465,7 +468,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "1 - (node_filesystem_avail_bytes{mountpoint=\"/\", fstype!=\"tmpfs\"} / node_filesystem_size_bytes{mountpoint=\"/\", fstype!=\"tmpfs\"})",
"legendFormat": "{{ instance }}"
}
@@ -476,7 +479,7 @@
"type": "bargauge",
"title": "Pods per Node",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 20 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": {
@@ -504,7 +507,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "count by (node) (kube_pod_info{node=~\"$node\"})",
"legendFormat": "{{ node }}",
"instant": true
@@ -524,7 +527,7 @@
"type": "timeseries",
"title": "Network I/O per Node",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 29 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -552,13 +555,13 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum by (instance) (rate(node_network_receive_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$interval]))",
"legendFormat": "{{ instance }} RX"
},
{
"refId": "B",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum by (instance) (rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$interval]))",
"legendFormat": "{{ instance }} TX"
}
@@ -570,7 +573,7 @@
"title": "PVC Usage",
"description": "Persistent Volume Claim usage percentage. Requires kubelet_volume_stats metrics.",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 29 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": {
@@ -601,7 +604,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "(1 - (kubelet_volume_stats_available_bytes{namespace=\"foxhunt\"} / kubelet_volume_stats_capacity_bytes{namespace=\"foxhunt\"})) * 100",
"legendFormat": "{{ persistentvolumeclaim }}",
"instant": true
@@ -622,7 +625,7 @@
"title": "Scheduling Latency (p99)",
"description": "Time to schedule pods. From scheduler_scheduling_algorithm_duration_seconds or scheduler_e2e_scheduling_duration_seconds.",
"gridPos": { "h": 7, "w": 12, "x": 0, "y": 38 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -650,13 +653,13 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "histogram_quantile(0.99, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket[$interval])) by (le))",
"legendFormat": "p99 e2e scheduling"
},
{
"refId": "B",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "histogram_quantile(0.50, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket[$interval])) by (le))",
"legendFormat": "p50 e2e scheduling"
}
@@ -668,7 +671,7 @@
"title": "OOMKill Events",
"description": "Container OOMKill events. Each spike indicates a container was killed due to memory pressure.",
"gridPos": { "h": 7, "w": 12, "x": 12, "y": 38 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -696,7 +699,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "increase(kube_pod_container_status_last_terminated_reason{reason=\"OOMKilled\", namespace=\"foxhunt\"}[30m]) > 0",
"legendFormat": "{{ pod }} ({{ container }})"
}
@@ -707,7 +710,7 @@
"type": "timeseries",
"title": "Disk I/O per Node",
"gridPos": { "h": 7, "w": 24, "x": 0, "y": 45 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -735,13 +738,13 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum by (instance) (rate(node_disk_read_bytes_total[$interval]))",
"legendFormat": "{{ instance }} read"
},
{
"refId": "B",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum by (instance) (rate(node_disk_written_bytes_total[$interval]))",
"legendFormat": "{{ instance }} write"
}

42
infra/k8s/monitoring/dashboards/foxhunt-logs.json
View File

@@ -1,4 +1,8 @@
{
"__inputs": [
{ "name": "DS_LOKI", "label": "Loki", "description": "", "type": "datasource", "pluginId": "loki" },
{ "name": "DS_TEMPO", "label": "Tempo", "description": "", "type": "datasource", "pluginId": "tempo" }
],
"id": null,
"uid": "foxhunt-logs",
"title": "Foxhunt - Log Explorer",
@@ -58,7 +62,7 @@
"type": "timeseries",
"title": "Log Volume by Service",
"gridPos": { "h": 8, "w": 16, "x": 0, "y": 1 },
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
"datasource": { "type": "loki", "uid": "${DS_LOKI}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -86,7 +90,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
"datasource": { "type": "loki", "uid": "${DS_LOKI}" },
"expr": "sum by (app) (count_over_time({namespace=\"foxhunt\", app=~\"$service\"} [$interval]))",
"legendFormat": "{{ app }}"
}
@@ -97,7 +101,7 @@
"type": "stat",
"title": "Total Logs (1h)",
"gridPos": { "h": 4, "w": 4, "x": 16, "y": 1 },
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
"datasource": { "type": "loki", "uid": "${DS_LOKI}" },
"fieldConfig": {
"defaults": {
"thresholds": {
@@ -122,7 +126,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
"datasource": { "type": "loki", "uid": "${DS_LOKI}" },
"expr": "sum(count_over_time({namespace=\"foxhunt\", app=~\"$service\"} [1h]))",
"instant": true
}
@@ -133,7 +137,7 @@
"type": "stat",
"title": "Errors (1h)",
"gridPos": { "h": 4, "w": 4, "x": 20, "y": 1 },
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
"datasource": { "type": "loki", "uid": "${DS_LOKI}" },
"fieldConfig": {
"defaults": {
"thresholds": {
@@ -160,7 +164,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
"datasource": { "type": "loki", "uid": "${DS_LOKI}" },
"expr": "sum(count_over_time({namespace=\"foxhunt\", app=~\"$service\"} |= \"ERROR\" [1h]))",
"instant": true
}
@@ -171,7 +175,7 @@
"type": "stat",
"title": "Warnings (1h)",
"gridPos": { "h": 4, "w": 4, "x": 16, "y": 5 },
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
"datasource": { "type": "loki", "uid": "${DS_LOKI}" },
"fieldConfig": {
"defaults": {
"thresholds": {
@@ -197,7 +201,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
"datasource": { "type": "loki", "uid": "${DS_LOKI}" },
"expr": "sum(count_over_time({namespace=\"foxhunt\", app=~\"$service\"} |= \"WARN\" [1h]))",
"instant": true
}
@@ -208,7 +212,7 @@
"type": "stat",
"title": "Error Rate /min",
"gridPos": { "h": 4, "w": 4, "x": 20, "y": 5 },
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
"datasource": { "type": "loki", "uid": "${DS_LOKI}" },
"fieldConfig": {
"defaults": {
"thresholds": {
@@ -237,7 +241,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
"datasource": { "type": "loki", "uid": "${DS_LOKI}" },
"expr": "sum(count_over_time({namespace=\"foxhunt\", app=~\"$service\"} |= \"ERROR\" [1m]))",
"instant": true
}
@@ -256,7 +260,7 @@
"type": "timeseries",
"title": "Error Rate by Service",
"gridPos": { "h": 7, "w": 24, "x": 0, "y": 10 },
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
"datasource": { "type": "loki", "uid": "${DS_LOKI}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -284,7 +288,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
"datasource": { "type": "loki", "uid": "${DS_LOKI}" },
"expr": "sum by (app) (count_over_time({namespace=\"foxhunt\", app=~\"$service\"} |= \"ERROR\" [$interval]))",
"legendFormat": "{{ app }}"
}
@@ -296,7 +300,7 @@
"title": "Top Error Messages",
"description": "Most frequent error messages across all services in the time range",
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 17 },
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
"datasource": { "type": "loki", "uid": "${DS_LOKI}" },
"fieldConfig": {
"defaults": {
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
@@ -313,7 +317,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
"datasource": { "type": "loki", "uid": "${DS_LOKI}" },
"expr": "topk(20, sum by (app, level) (count_over_time({namespace=\"foxhunt\", app=~\"$service\"} |= \"ERROR\" | logfmt [$interval])))",
"legendFormat": "{{ app }}",
"instant": true
@@ -334,7 +338,7 @@
"title": "Live Log Stream",
"description": "Streaming logs from selected services. Use the $search variable to filter.",
"gridPos": { "h": 14, "w": 24, "x": 0, "y": 26 },
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
"datasource": { "type": "loki", "uid": "${DS_LOKI}" },
"options": {
"showTime": true,
"showLabels": true,
@@ -348,7 +352,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
"datasource": { "type": "loki", "uid": "${DS_LOKI}" },
"expr": "{namespace=\"foxhunt\", app=~\"$service\"} |= \"$search\"",
"maxLines": 500
}
@@ -368,7 +372,7 @@
"title": "Recent Traces",
"description": "Search for distributed traces via Tempo. Click a trace ID to view the full trace.",
"gridPos": { "h": 10, "w": 24, "x": 0, "y": 41 },
"datasource": { "type": "tempo", "uid": "tempo" },
"datasource": { "type": "tempo", "uid": "${DS_TEMPO}" },
"fieldConfig": {
"defaults": {
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
@@ -379,7 +383,7 @@
"title": "View Trace",
"url": "",
"internal": {
"datasourceUid": "tempo",
"datasourceUid": "${DS_TEMPO}",
"datasourceName": "Tempo",
"query": {
"queryType": "traceql",
@@ -404,7 +408,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "tempo", "uid": "tempo" },
"datasource": { "type": "tempo", "uid": "${DS_TEMPO}" },
"queryType": "nativeSearch",
"serviceName": "",
"spanName": "",

94
infra/k8s/monitoring/dashboards/foxhunt-overview.json
View File

@@ -1,4 +1,8 @@
{
"__inputs": [
{ "name": "DS_PROMETHEUS", "label": "Prometheus", "description": "", "type": "datasource", "pluginId": "prometheus" },
{ "name": "DS_LOKI", "label": "Loki", "description": "", "type": "datasource", "pluginId": "loki" }
],
"id": null,
"uid": "foxhunt-overview",
"title": "Foxhunt - System Overview",
@@ -45,7 +49,7 @@
"type": "stat",
"title": "Nodes Ready",
"gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": {
@@ -72,7 +76,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(kube_node_status_condition{condition=\"Ready\", status=\"true\"})",
"legendFormat": "Ready",
"instant": true
@@ -84,7 +88,7 @@
"type": "stat",
"title": "Total Nodes",
"gridPos": { "h": 4, "w": 4, "x": 4, "y": 1 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": {
@@ -109,7 +113,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "count(kube_node_info)",
"legendFormat": "Nodes",
"instant": true
@@ -121,7 +125,7 @@
"type": "stat",
"title": "Pods Running",
"gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": {
@@ -146,7 +150,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", phase=\"Running\"})",
"legendFormat": "Running",
"instant": true
@@ -158,7 +162,7 @@
"type": "stat",
"title": "Pods Failed",
"gridPos": { "h": 4, "w": 4, "x": 12, "y": 1 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": {
@@ -184,7 +188,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", phase=\"Failed\"}) or vector(0)",
"legendFormat": "Failed",
"instant": true
@@ -196,7 +200,7 @@
"type": "stat",
"title": "Active Training Jobs",
"gridPos": { "h": 4, "w": 4, "x": 16, "y": 1 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": {
@@ -222,7 +226,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(kube_job_status_active{namespace=\"foxhunt\"}) or vector(0)",
"legendFormat": "Active Jobs",
"instant": true
@@ -234,7 +238,7 @@
"type": "stat",
"title": "Log Error Rate (/min)",
"gridPos": { "h": 4, "w": 4, "x": 20, "y": 1 },
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
"datasource": { "type": "loki", "uid": "${DS_LOKI}" },
"fieldConfig": {
"defaults": {
"thresholds": {
@@ -263,7 +267,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
"datasource": { "type": "loki", "uid": "${DS_LOKI}" },
"expr": "sum(count_over_time({namespace=\"foxhunt\"} |= \"ERROR\" [1m]))",
"legendFormat": "Errors/min",
"instant": true
@@ -283,7 +287,7 @@
"type": "stat",
"title": "api-gateway",
"gridPos": { "h": 3, "w": 3, "x": 0, "y": 6 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": {
@@ -311,7 +315,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"api-gateway.*\", phase=\"Running\"}) > 0 or vector(0)",
"legendFormat": "",
"instant": true
@@ -323,7 +327,7 @@
"type": "stat",
"title": "trading-service",
"gridPos": { "h": 3, "w": 3, "x": 3, "y": 6 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": {
@@ -351,7 +355,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"trading-service.*\", phase=\"Running\"}) > 0 or vector(0)",
"legendFormat": "",
"instant": true
@@ -363,7 +367,7 @@
"type": "stat",
"title": "ml-training-service",
"gridPos": { "h": 3, "w": 3, "x": 6, "y": 6 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": {
@@ -391,7 +395,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"ml-training-service.*\", phase=\"Running\"}) > 0 or vector(0)",
"legendFormat": "",
"instant": true
@@ -403,7 +407,7 @@
"type": "stat",
"title": "backtesting-service",
"gridPos": { "h": 3, "w": 3, "x": 9, "y": 6 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": {
@@ -431,7 +435,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"backtesting-service.*\", phase=\"Running\"}) > 0 or vector(0)",
"legendFormat": "",
"instant": true
@@ -443,7 +447,7 @@
"type": "stat",
"title": "trading-agent",
"gridPos": { "h": 3, "w": 3, "x": 12, "y": 6 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": {
@@ -471,7 +475,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"trading-agent.*\", phase=\"Running\"}) > 0 or vector(0)",
"legendFormat": "",
"instant": true
@@ -483,7 +487,7 @@
"type": "stat",
"title": "broker-gateway",
"gridPos": { "h": 3, "w": 3, "x": 15, "y": 6 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": {
@@ -511,7 +515,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"broker-gateway.*\", phase=\"Running\"}) > 0 or vector(0)",
"legendFormat": "",
"instant": true
@@ -523,7 +527,7 @@
"type": "stat",
"title": "data-acquisition",
"gridPos": { "h": 3, "w": 3, "x": 18, "y": 6 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": {
@@ -551,7 +555,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"data-acquisition.*\", phase=\"Running\"}) > 0 or vector(0)",
"legendFormat": "",
"instant": true
@@ -563,7 +567,7 @@
"type": "stat",
"title": "web-gateway",
"gridPos": { "h": 3, "w": 3, "x": 21, "y": 6 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": {
@@ -591,7 +595,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"web-gateway.*\", phase=\"Running\"}) > 0 or vector(0)",
"legendFormat": "",
"instant": true
@@ -611,7 +615,7 @@
"type": "gauge",
"title": "Cluster CPU %",
"gridPos": { "h": 6, "w": 6, "x": 0, "y": 10 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": {
@@ -639,7 +643,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "100 - (avg(rate(node_cpu_seconds_total{mode=\"idle\"}[$interval])) * 100)",
"legendFormat": "CPU",
"instant": true
@@ -651,7 +655,7 @@
"type": "gauge",
"title": "Cluster Memory %",
"gridPos": { "h": 6, "w": 6, "x": 6, "y": 10 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": {
@@ -679,7 +683,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "(1 - (sum(node_memory_MemAvailable_bytes) / sum(node_memory_MemTotal_bytes))) * 100",
"legendFormat": "Memory",
"instant": true
@@ -691,7 +695,7 @@
"type": "timeseries",
"title": "CPU by Node Pool",
"gridPos": { "h": 6, "w": 12, "x": 12, "y": 10 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -721,7 +725,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[$interval]))",
"legendFormat": "{{ instance }}"
}
@@ -732,7 +736,7 @@
"type": "timeseries",
"title": "Memory by Node Pool",
"gridPos": { "h": 6, "w": 12, "x": 0, "y": 16 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -760,7 +764,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes",
"legendFormat": "{{ instance }}"
}
@@ -771,7 +775,7 @@
"type": "timeseries",
"title": "Foxhunt Container CPU Usage",
"gridPos": { "h": 6, "w": 12, "x": 12, "y": 16 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -799,7 +803,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum by (container) (rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", container=~\"api-gateway|trading-service|ml-training-service|backtesting-service|trading-agent-service|broker-gateway|data-acquisition-service|web-gateway\", image!=\"\"}[$interval]))",
"legendFormat": "{{ container }}"
}
@@ -819,7 +823,7 @@
"title": "Pod Restarts (last 1h)",
"description": "Pod restart events indicating crashes or OOMKills",
"gridPos": { "h": 7, "w": 12, "x": 0, "y": 23 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -847,7 +851,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "increase(kube_pod_container_status_restarts_total{namespace=\"foxhunt\"}[1h]) > 0",
"legendFormat": "{{ pod }}"
}
@@ -858,7 +862,7 @@
"type": "timeseries",
"title": "Deployment Replicas: Desired vs Available",
"gridPos": { "h": 7, "w": 12, "x": 12, "y": 23 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -886,13 +890,13 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "kube_deployment_spec_replicas{namespace=\"foxhunt\"}",
"legendFormat": "{{ deployment }} desired"
},
{
"refId": "B",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "kube_deployment_status_replicas_available{namespace=\"foxhunt\"}",
"legendFormat": "{{ deployment }} available"
}
@@ -911,7 +915,7 @@
"type": "timeseries",
"title": "Log Errors Over Time",
"gridPos": { "h": 7, "w": 24, "x": 0, "y": 31 },
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
"datasource": { "type": "loki", "uid": "${DS_LOKI}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -939,7 +943,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
"datasource": { "type": "loki", "uid": "${DS_LOKI}" },
"expr": "sum by (app) (count_over_time({namespace=\"foxhunt\"} |= \"ERROR\" [$interval]))",
"legendFormat": "{{ app }}"
}

74
infra/k8s/monitoring/dashboards/foxhunt-services.json
View File

@@ -1,4 +1,8 @@
{
"__inputs": [
{ "name": "DS_PROMETHEUS", "label": "Prometheus", "description": "", "type": "datasource", "pluginId": "prometheus" },
{ "name": "DS_LOKI", "label": "Loki", "description": "", "type": "datasource", "pluginId": "loki" }
],
"id": null,
"uid": "foxhunt-services",
"title": "Foxhunt - Microservice Detail",
@@ -58,7 +62,7 @@
"type": "stat",
"title": "Status",
"gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": {
@@ -86,7 +90,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"$service.*\", phase=\"Running\"}) > 0 or vector(0)",
"instant": true
}
@@ -97,7 +101,7 @@
"type": "stat",
"title": "Replicas",
"gridPos": { "h": 4, "w": 4, "x": 4, "y": 1 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": {
@@ -122,7 +126,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "count(kube_pod_info{namespace=\"foxhunt\", pod=~\"$service.*\"})",
"instant": true
}
@@ -133,7 +137,7 @@
"type": "stat",
"title": "Restarts (24h)",
"gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": {
@@ -160,7 +164,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(increase(kube_pod_container_status_restarts_total{namespace=\"foxhunt\", pod=~\"$service.*\"}[24h])) or vector(0)",
"instant": true
}
@@ -171,7 +175,7 @@
"type": "stat",
"title": "Container Uptime",
"gridPos": { "h": 4, "w": 4, "x": 12, "y": 1 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": {
@@ -199,7 +203,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "min(time() - kube_pod_start_time{namespace=\"foxhunt\", pod=~\"$service.*\"})",
"instant": true
}
@@ -210,7 +214,7 @@
"type": "stat",
"title": "CPU Cores",
"gridPos": { "h": 4, "w": 4, "x": 16, "y": 1 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": {
@@ -237,7 +241,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", container=~\"$service\", image!=\"\"}[$interval]))",
"instant": true
}
@@ -248,7 +252,7 @@
"type": "stat",
"title": "Memory",
"gridPos": { "h": 4, "w": 4, "x": 20, "y": 1 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": {
@@ -274,7 +278,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", container=~\"$service\", image!=\"\"})",
"instant": true
}
@@ -293,7 +297,7 @@
"type": "timeseries",
"title": "CPU Usage",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -321,19 +325,19 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", container=~\"$service\", image!=\"\"}[$interval]))",
"legendFormat": "{{ pod }}"
},
{
"refId": "B",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(kube_pod_container_resource_requests{namespace=\"foxhunt\", container=~\"$service\", resource=\"cpu\"})",
"legendFormat": "Request"
},
{
"refId": "C",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(kube_pod_container_resource_limits{namespace=\"foxhunt\", container=~\"$service\", resource=\"cpu\"})",
"legendFormat": "Limit"
}
@@ -344,7 +348,7 @@
"type": "timeseries",
"title": "Memory Usage",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -372,19 +376,19 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum by (pod) (container_memory_working_set_bytes{namespace=\"foxhunt\", container=~\"$service\", image!=\"\"})",
"legendFormat": "{{ pod }}"
},
{
"refId": "B",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(kube_pod_container_resource_requests{namespace=\"foxhunt\", container=~\"$service\", resource=\"memory\"})",
"legendFormat": "Request"
},
{
"refId": "C",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(kube_pod_container_resource_limits{namespace=\"foxhunt\", container=~\"$service\", resource=\"memory\"})",
"legendFormat": "Limit"
}
@@ -403,7 +407,7 @@
"type": "timeseries",
"title": "Pod Restart Count",
"gridPos": { "h": 6, "w": 12, "x": 0, "y": 15 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -431,7 +435,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "increase(kube_pod_container_status_restarts_total{namespace=\"foxhunt\", pod=~\"$service.*\"}[30m])",
"legendFormat": "{{ pod }}"
}
@@ -442,7 +446,7 @@
"type": "timeseries",
"title": "Network I/O",
"gridPos": { "h": 6, "w": 12, "x": 12, "y": 15 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -470,13 +474,13 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum by (pod) (rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"$service.*\"}[$interval]))",
"legendFormat": "{{ pod }} RX"
},
{
"refId": "B",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum by (pod) (rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"$service.*\"}[$interval]))",
"legendFormat": "{{ pod }} TX"
}
@@ -496,7 +500,7 @@
"title": "gRPC Request Rate",
"description": "gRPC requests per second by method and status code. Requires grpc_server_handled_total metric from service instrumentation.",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 22 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -524,7 +528,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum by (grpc_method, grpc_code) (rate(grpc_server_handled_total{grpc_service=~\".*$service.*\"}[$interval]))",
"legendFormat": "{{ grpc_method }} [{{ grpc_code }}]"
}
@@ -536,7 +540,7 @@
"title": "gRPC Latency (p50 / p95 / p99)",
"description": "gRPC server-side latency percentiles. Requires grpc_server_handling_seconds_bucket histogram.",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 22 },
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -564,19 +568,19 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "histogram_quantile(0.50, sum(rate(grpc_server_handling_seconds_bucket{grpc_service=~\".*$service.*\"}[$interval])) by (le))",
"legendFormat": "p50"
},
{
"refId": "B",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "histogram_quantile(0.95, sum(rate(grpc_server_handling_seconds_bucket{grpc_service=~\".*$service.*\"}[$interval])) by (le))",
"legendFormat": "p95"
},
{
"refId": "C",
"datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{grpc_service=~\".*$service.*\"}[$interval])) by (le))",
"legendFormat": "p99"
}
@@ -595,7 +599,7 @@
"type": "timeseries",
"title": "Log Volume by Level",
"gridPos": { "h": 6, "w": 24, "x": 0, "y": 31 },
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
"datasource": { "type": "loki", "uid": "${DS_LOKI}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -640,7 +644,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
"datasource": { "type": "loki", "uid": "${DS_LOKI}" },
"expr": "sum by (level) (count_over_time({app=\"$service\", namespace=\"foxhunt\"} | logfmt | level != \"\" [$interval]))",
"legendFormat": "{{ level }}"
}
@@ -651,7 +655,7 @@
"type": "logs",
"title": "Recent Error Logs",
"gridPos": { "h": 10, "w": 24, "x": 0, "y": 37 },
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
"datasource": { "type": "loki", "uid": "${DS_LOKI}" },
"options": {
"showTime": true,
"showLabels": true,
@@ -665,7 +669,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
"datasource": { "type": "loki", "uid": "${DS_LOKI}" },
"expr": "{app=\"$service\", namespace=\"foxhunt\"} |= \"ERROR\"",
"maxLines": 200
}

59
infra/k8s/monitoring/dashboards/gitlab-services.json
View File

@@ -1,4 +1,7 @@
{
"__inputs": [
{ "name": "DS_PROMETHEUS", "label": "Prometheus", "description": "", "type": "datasource", "pluginId": "prometheus" }
],
"id": null,
"uid": "foxhunt-gitlab-services",
"title": "GitLab + Foxhunt Services",
@@ -22,7 +25,7 @@
"type": "query",
"datasource": {
"type": "prometheus",
"uid": "cfegaydmscvswe"
"uid": "${DS_PROMETHEUS}"
},
"query": "label_values(kube_pod_info, namespace)",
"refresh": 2,
@@ -52,7 +55,7 @@
"type": "timeseries",
"title": "Puma HTTP Request Rate",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 1 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -80,7 +83,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(rate(http_requests_total{job=~\".*puma.*\"}[5m])) by (method)",
"legendFormat": "{{ method }}"
}
@@ -91,7 +94,7 @@
"type": "timeseries",
"title": "Puma Request Duration (p95)",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 1 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -119,13 +122,13 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket{job=~\".*puma.*\"}[5m])) by (le))",
"legendFormat": "p95"
},
{
"refId": "B",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "histogram_quantile(0.50, sum(rate(http_request_duration_seconds_bucket{job=~\".*puma.*\"}[5m])) by (le))",
"legendFormat": "p50"
}
@@ -144,7 +147,7 @@
"type": "timeseries",
"title": "Sidekiq Job Processing Rate",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 10 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -172,13 +175,13 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(rate(sidekiq_jobs_processed_total[5m]))",
"legendFormat": "Processed"
},
{
"refId": "B",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(rate(sidekiq_jobs_failed_total[5m]))",
"legendFormat": "Failed"
}
@@ -189,7 +192,7 @@
"type": "stat",
"title": "Sidekiq Queue Size",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 10 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": {
@@ -216,7 +219,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(sidekiq_queue_size)",
"legendFormat": "Queue Size",
"instant": true
@@ -236,7 +239,7 @@
"type": "timeseries",
"title": "Gitaly RPC Latency (p95)",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 19 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -264,7 +267,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "histogram_quantile(0.95, sum(rate(grpc_server_handling_seconds_bucket{job=~\".*gitaly.*\"}[5m])) by (le, grpc_method))",
"legendFormat": "{{ grpc_method }}"
}
@@ -275,7 +278,7 @@
"type": "timeseries",
"title": "Gitaly RPC Rate",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 19 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -303,7 +306,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum(rate(grpc_server_handled_total{job=~\".*gitaly.*\"}[5m])) by (grpc_code)",
"legendFormat": "{{ grpc_code }}"
}
@@ -322,7 +325,7 @@
"type": "timeseries",
"title": "Container CPU Usage by Service",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 28 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -350,7 +353,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum by (container) (rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\", container=~\"web-gateway|api-gateway|trading-service|ml-training-service|backtesting-service|data-acquisition-service|broker-gateway-service|trading-agent-service\", image!=\"\"}[5m]))",
"legendFormat": "{{ container }}"
}
@@ -361,7 +364,7 @@
"type": "timeseries",
"title": "Container Memory Usage by Service",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 28 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -389,7 +392,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum by (container) (container_memory_working_set_bytes{namespace=~\"$namespace\", container=~\"web-gateway|api-gateway|trading-service|ml-training-service|backtesting-service|data-acquisition-service|broker-gateway-service|trading-agent-service\", image!=\"\"})",
"legendFormat": "{{ container }}"
}
@@ -408,7 +411,7 @@
"type": "timeseries",
"title": "Pod Restarts (by Container)",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 37 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -436,7 +439,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "increase(kube_pod_container_status_restarts_total{namespace=~\"$namespace\"}[1h])",
"legendFormat": "{{ namespace }}/{{ pod }} ({{ container }})"
}
@@ -447,7 +450,7 @@
"type": "table",
"title": "Pod Status by Namespace",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 37 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
@@ -503,7 +506,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum by (namespace) (kube_pod_status_phase{namespace=~\"$namespace\", phase=\"Running\"})",
"legendFormat": "Running",
"instant": true,
@@ -511,7 +514,7 @@
},
{
"refId": "B",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum by (namespace) (kube_pod_status_phase{namespace=~\"$namespace\", phase=\"Pending\"})",
"legendFormat": "Pending",
"instant": true,
@@ -519,7 +522,7 @@
},
{
"refId": "C",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "sum by (namespace) (kube_pod_status_phase{namespace=~\"$namespace\", phase=\"Failed\"})",
"legendFormat": "Failed",
"instant": true,
@@ -540,7 +543,7 @@
"type": "timeseries",
"title": "Deployment Replicas: Desired vs Available",
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 46 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -568,13 +571,13 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "kube_deployment_spec_replicas{namespace=~\"$namespace\"}",
"legendFormat": "{{ deployment }} desired"
},
{
"refId": "B",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "kube_deployment_status_replicas_available{namespace=~\"$namespace\"}",
"legendFormat": "{{ deployment }} available"
}

47
infra/k8s/monitoring/dashboards/gpu-overview.json
View File

@@ -1,4 +1,7 @@
{
"__inputs": [
{ "name": "DS_PROMETHEUS", "label": "Prometheus", "description": "", "type": "datasource", "pluginId": "prometheus" }
],
"id": null,
"uid": "foxhunt-gpu-overview",
"title": "GPU Overview",
@@ -22,7 +25,7 @@
"type": "query",
"datasource": {
"type": "prometheus",
"uid": "cfegaydmscvswe"
"uid": "${DS_PROMETHEUS}"
},
"query": "label_values(DCGM_FI_DEV_GPU_UTIL, gpu)",
"refresh": 2,
@@ -41,7 +44,7 @@
"type": "query",
"datasource": {
"type": "prometheus",
"uid": "cfegaydmscvswe"
"uid": "${DS_PROMETHEUS}"
},
"query": "label_values(DCGM_FI_DEV_GPU_UTIL, instance)",
"refresh": 2,
@@ -60,7 +63,7 @@
"type": "query",
"datasource": {
"type": "prometheus",
"uid": "cfegaydmscvswe"
"uid": "${DS_PROMETHEUS}"
},
"query": "label_values(DCGM_FI_DEV_GPU_UTIL, exported_namespace)",
"refresh": 2,
@@ -90,7 +93,7 @@
"type": "timeseries",
"title": "GPU Utilization %",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 1 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -127,7 +130,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "DCGM_FI_DEV_GPU_UTIL{gpu=~\"$gpu\", instance=~\"$instance\"}",
"legendFormat": "GPU {{ gpu }} ({{ instance }})"
}
@@ -138,7 +141,7 @@
"type": "gauge",
"title": "Current GPU Utilization",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 1 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": {
@@ -167,7 +170,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "DCGM_FI_DEV_GPU_UTIL{gpu=~\"$gpu\", instance=~\"$instance\"}",
"legendFormat": "GPU {{ gpu }}",
"instant": true
@@ -187,7 +190,7 @@
"type": "timeseries",
"title": "GPU Memory Used",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 10 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -215,13 +218,13 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "DCGM_FI_DEV_FB_USED{gpu=~\"$gpu\", instance=~\"$instance\"}",
"legendFormat": "GPU {{ gpu }} used"
},
{
"refId": "B",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "DCGM_FI_DEV_FB_TOTAL{gpu=~\"$gpu\", instance=~\"$instance\"}",
"legendFormat": "GPU {{ gpu }} total"
}
@@ -232,7 +235,7 @@
"type": "gauge",
"title": "GPU Memory Usage %",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 10 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"thresholds": {
@@ -260,7 +263,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "(DCGM_FI_DEV_FB_USED{gpu=~\"$gpu\", instance=~\"$instance\"} / DCGM_FI_DEV_FB_TOTAL{gpu=~\"$gpu\", instance=~\"$instance\"}) * 100",
"legendFormat": "GPU {{ gpu }}",
"instant": true
@@ -280,7 +283,7 @@
"type": "timeseries",
"title": "GPU Temperature",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 19 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -316,7 +319,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "DCGM_FI_DEV_GPU_TEMP{gpu=~\"$gpu\", instance=~\"$instance\"}",
"legendFormat": "GPU {{ gpu }} ({{ instance }})"
}
@@ -327,7 +330,7 @@
"type": "timeseries",
"title": "GPU Power Usage",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 19 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -355,7 +358,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "DCGM_FI_DEV_POWER_USAGE{gpu=~\"$gpu\", instance=~\"$instance\"}",
"legendFormat": "GPU {{ gpu }} ({{ instance }})"
}
@@ -374,7 +377,7 @@
"type": "timeseries",
"title": "GPU Memory Clock",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 28 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -403,7 +406,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "DCGM_FI_DEV_MEM_CLK{gpu=~\"$gpu\", instance=~\"$instance\"} * 1000000",
"legendFormat": "GPU {{ gpu }} mem clock"
}
@@ -415,7 +418,7 @@
"title": "Per-Pod GPU Utilization",
"description": "GPU utilization broken down by pod. Requires DCGM exporter pod-level labels (exported_pod, exported_namespace).",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 28 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -445,7 +448,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "DCGM_FI_DEV_GPU_UTIL{exported_namespace=~\"$namespace\", gpu=~\"$gpu\", instance=~\"$instance\"}",
"legendFormat": "{{ exported_namespace }}/{{ exported_pod }} GPU {{ gpu }}"
}
@@ -457,7 +460,7 @@
"title": "Per-Pod GPU Memory Used",
"description": "GPU framebuffer memory used per pod. Requires DCGM exporter pod-level labels.",
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 36 },
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"fieldConfig": {
"defaults": {
"custom": {
@@ -485,7 +488,7 @@
"targets": [
{
"refId": "A",
"datasource": { "type": "prometheus", "uid": "cfegaydmscvswe" },
"datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
"expr": "DCGM_FI_DEV_FB_USED{exported_namespace=~\"$namespace\", gpu=~\"$gpu\", instance=~\"$instance\"}",
"legendFormat": "{{ exported_namespace }}/{{ exported_pod }} GPU {{ gpu }}"
}

95
infra/k8s/monitoring/dashboards/import.sh
View File

@@ -1,27 +1,97 @@
#!/usr/bin/env bash
set -euo pipefail
# Import Grafana dashboards to Scaleway Cockpit via the Grafana API.
# Import Grafana dashboards with dynamic datasource resolution.
#
# Dashboards use ${DS_PROMETHEUS}, ${DS_LOKI}, ${DS_TEMPO} variables.
# This script auto-detects datasource UIDs from the Grafana instance
# and maps them at import time via /api/dashboards/import.
#
# Required environment variables:
# GRAFANA_URL - Grafana base URL (default: https://grafana.fxhnt.ai)
# GRAFANA_PASS - Password for admin user
#
# Optional overrides:
# GRAFANA_URL - Grafana base URL (default: https://grafana.fxhnt.ai)
# GRAFANA_USER - Grafana user (default: admin)
# GRAFANA_PROM_UID - Override Prometheus datasource UID
# GRAFANA_LOKI_UID - Override Loki datasource UID
# GRAFANA_TEMPO_UID- Override Tempo datasource UID
#
# Usage:
# GRAFANA_PASS="Welcome01" ./import.sh
# GRAFANA_URL="https://grafana.fxhnt.ai" GRAFANA_PASS="Welcome01" ./import.sh
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
GRAFANA_USER="${GRAFANA_USER:-admin}"
GRAFANA_URL="${GRAFANA_URL:-https://grafana.fxhnt.ai}"
GRAFANA_URL="${GRAFANA_URL%/}"
if [[ -z "${GRAFANA_PASS:-}" ]]; then
echo "ERROR: GRAFANA_PASS is not set" >&2
exit 1
fi
# Strip trailing slash from URL
GRAFANA_URL="${GRAFANA_URL%/}"
# --- Datasource auto-detection ---
detect_datasource_uid() {
local ds_type="$1"
curl -sf -u "${GRAFANA_USER}:${GRAFANA_PASS}" "${GRAFANA_URL}/api/datasources" 2>/dev/null | \
python3 -c "
import sys, json
for ds in json.load(sys.stdin):
if ds.get('type') == '${ds_type}':
print(ds['uid'])
break
" 2>/dev/null || true
}
echo "Detecting datasource UIDs..."
PROM_UID="${GRAFANA_PROM_UID:-$(detect_datasource_uid prometheus)}"
LOKI_UID="${GRAFANA_LOKI_UID:-$(detect_datasource_uid loki)}"
TEMPO_UID="${GRAFANA_TEMPO_UID:-$(detect_datasource_uid tempo)}"
echo " Prometheus: ${PROM_UID:-NOT FOUND}"
echo " Loki: ${LOKI_UID:-NOT FOUND}"
echo " Tempo: ${TEMPO_UID:-NOT FOUND}"
if [[ -z "$PROM_UID" ]]; then
echo "ERROR: Could not detect Prometheus datasource. Set GRAFANA_PROM_UID." >&2
exit 1
fi
# --- Build inputs array for a dashboard ---
build_inputs() {
local file="$1"
local inputs="["
local first=true
if grep -q 'DS_PROMETHEUS' "$file"; then
$first || inputs+=","
inputs+="{\"name\":\"DS_PROMETHEUS\",\"type\":\"datasource\",\"pluginId\":\"prometheus\",\"value\":\"${PROM_UID}\"}"
first=false
fi
if grep -q 'DS_LOKI' "$file"; then
if [[ -z "$LOKI_UID" ]]; then
echo " WARNING: Dashboard requires Loki but no datasource found" >&2
fi
$first || inputs+=","
inputs+="{\"name\":\"DS_LOKI\",\"type\":\"datasource\",\"pluginId\":\"loki\",\"value\":\"${LOKI_UID}\"}"
first=false
fi
if grep -q 'DS_TEMPO' "$file"; then
if [[ -z "$TEMPO_UID" ]]; then
echo " WARNING: Dashboard requires Tempo but no datasource found" >&2
fi
$first || inputs+=","
inputs+="{\"name\":\"DS_TEMPO\",\"type\":\"datasource\",\"pluginId\":\"tempo\",\"value\":\"${TEMPO_UID}\"}"
first=false
fi
inputs+="]"
echo "$inputs"
}
# --- Import a single dashboard ---
import_dashboard() {
local file="$1"
@@ -30,12 +100,15 @@ import_dashboard() {
echo "Importing ${name}..."
local inputs
inputs=$(build_inputs "$file")
local response
response=$(curl -s -w "\n%{http_code}" \
-u "${GRAFANA_USER}:${GRAFANA_PASS}" \
-H "Content-Type: application/json" \
-d "{\"dashboard\": $(cat "$file"), \"overwrite\": true}" \
"${GRAFANA_URL}/api/dashboards/db")
-d "{\"dashboard\": $(cat "$file"), \"inputs\": ${inputs}, \"overwrite\": true}" \
"${GRAFANA_URL}/api/dashboards/import")
local http_code
http_code=$(echo "$response" | tail -1)
@@ -43,13 +116,15 @@ import_dashboard() {
body=$(echo "$response" | sed '$d')
if [[ "$http_code" -ge 200 && "$http_code" -lt 300 ]]; then
echo " OK (${http_code}): ${body}"
echo " OK (${http_code})"
else
echo " FAILED (${http_code}): ${body}" >&2
return 1
fi
}
# --- Import all dashboards ---
failed=0
for dashboard in "${SCRIPT_DIR}/cluster-overview.json" \
@@ -59,7 +134,9 @@ for dashboard in "${SCRIPT_DIR}/cluster-overview.json" \
"${SCRIPT_DIR}/foxhunt-services.json" \
"${SCRIPT_DIR}/foxhunt-gpu-training.json" \
"${SCRIPT_DIR}/foxhunt-infrastructure.json" \
"${SCRIPT_DIR}/foxhunt-logs.json"; do
"${SCRIPT_DIR}/foxhunt-logs.json" \
"${SCRIPT_DIR}/foxhunt-ci-pipelines.json" \
"${SCRIPT_DIR}/foxhunt-cockpit.json"; do
if [[ -f "$dashboard" ]]; then
if ! import_dashboard "$dashboard"; then
failed=$((failed + 1))