diff --git a/infra/k8s/gitlab/grafana-values.yaml b/infra/k8s/gitlab/grafana-values.yaml index f206242c0..73a63e3dc 100644 --- a/infra/k8s/gitlab/grafana-values.yaml +++ b/infra/k8s/gitlab/grafana-values.yaml @@ -42,7 +42,7 @@ datasources: datasources: - name: Prometheus type: prometheus - url: http://gitlab-prometheus-server.foxhunt.svc.cluster.local:80 + url: http://prometheus.foxhunt.svc.cluster.local:80 access: proxy isDefault: true - name: Loki diff --git a/infra/k8s/monitoring/dashboards/cluster-overview.json b/infra/k8s/monitoring/dashboards/cluster-overview.json deleted file mode 100644 index 8d35c198f..000000000 --- a/infra/k8s/monitoring/dashboards/cluster-overview.json +++ /dev/null @@ -1,695 +0,0 @@ -{ - "__inputs": [ - { "name": "DS_PROMETHEUS", "label": "Prometheus", "description": "", "type": "datasource", "pluginId": "prometheus" } - ], - "id": null, - "uid": "foxhunt-cluster-overview", - "title": "K8s Cluster Overview", - "description": "Kubernetes cluster health: nodes, pods, CPU, memory, disk, and network", - "tags": ["kubernetes", "foxhunt", "cluster"], - "timezone": "browser", - "schemaVersion": 39, - "version": 1, - "refresh": "30s", - "time": { - "from": "now-1h", - "to": "now" - }, - "fiscalYearStartMonth": 0, - "liveNow": false, - "weekStart": "", - "templating": { - "list": [ - { - "name": "namespace", - "type": "query", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "query": "label_values(kube_pod_info, namespace)", - "refresh": 2, - "includeAll": true, - "multi": true, - "allValue": ".*", - "current": { - "selected": true, - "text": "All", - "value": "$__all" - }, - "sort": 1 - }, - { - "name": "node", - "type": "query", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "query": "label_values(node_uname_info, nodename)", - "refresh": 2, - "includeAll": true, - "multi": true, - "allValue": ".*", - "current": { - "selected": true, - "text": "All", - "value": "$__all" - }, - "sort": 1 - } - ] - }, - "panels": [ - { - "id": 1, - "type": "row", - "title": "Cluster Summary", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, - "collapsed": false, - "panels": [] - }, - { - "id": 2, - "type": "stat", - "title": "Total Nodes", - "gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "red", "value": null }, - { "color": "green", "value": 1 } - ] - }, - "mappings": [], - "color": { "mode": "thresholds" } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto" - }, - "targets": [ - { - "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "count(kube_node_info)", - "legendFormat": "Nodes", - "instant": true - } - ] - }, - { - "id": 3, - "type": "stat", - "title": "Pods Running", - "gridPos": { "h": 4, "w": 4, "x": 4, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "green", "value": null } - ] - }, - "mappings": [], - "color": { "mode": "thresholds" } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto" - }, - "targets": [ - { - "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "sum(kube_pod_status_phase{namespace=~\"$namespace\", phase=\"Running\"})", - "legendFormat": "Running", - "instant": true - } - ] - }, - { - "id": 4, - "type": "stat", - "title": "Pods Pending", - "gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 1 }, - { "color": "red", "value": 5 } - ] - }, - "mappings": [], - "color": { "mode": "thresholds" } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto" - }, - "targets": [ - { - "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "sum(kube_pod_status_phase{namespace=~\"$namespace\", phase=\"Pending\"}) or vector(0)", - "legendFormat": "Pending", - "instant": true - } - ] - }, - { - "id": 5, - "type": "stat", - "title": "Pods Failed", - "gridPos": { "h": 4, "w": 4, "x": 12, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "green", "value": null }, - { "color": "red", "value": 1 } - ] - }, - "mappings": [], - "color": { "mode": "thresholds" } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto" - }, - "targets": [ - { - "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "sum(kube_pod_status_phase{namespace=~\"$namespace\", phase=\"Failed\"}) or vector(0)", - "legendFormat": "Failed", - "instant": true - } - ] - }, - { - "id": 6, - "type": "gauge", - "title": "Cluster CPU Usage %", - "gridPos": { "h": 4, "w": 4, "x": 16, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 70 }, - { "color": "red", "value": 90 } - ] - }, - "min": 0, - "max": 100, - "unit": "percent", - "mappings": [], - "color": { "mode": "thresholds" } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "orientation": "auto" - }, - "targets": [ - { - "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "100 - (avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m])) * 100)", - "legendFormat": "CPU %", - "instant": true - } - ] - }, - { - "id": 7, - "type": "gauge", - "title": "Cluster Memory Usage %", - "gridPos": { "h": 4, "w": 4, "x": 20, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 70 }, - { "color": "red", "value": 90 } - ] - }, - "min": 0, - "max": 100, - "unit": "percent", - "mappings": [], - "color": { "mode": "thresholds" } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "orientation": "auto" - }, - "targets": [ - { - "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "(1 - (sum(node_memory_MemAvailable_bytes) / sum(node_memory_MemTotal_bytes))) * 100", - "legendFormat": "Memory %", - "instant": true - } - ] - }, - { - "id": 10, - "type": "row", - "title": "CPU", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, - "collapsed": false, - "panels": [] - }, - { - "id": 11, - "type": "timeseries", - "title": "Per-Node CPU Usage", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 10, - "gradientMode": "scheme", - "spanNulls": false, - "showPoints": "never", - "stacking": { "mode": "none", "group": "A" }, - "axisPlacement": "auto", - "lineWidth": 1 - }, - "unit": "percentunit", - "min": 0, - "max": 1, - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, - "color": { "mode": "palette-classic" }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } - }, - "targets": [ - { - "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))", - "legendFormat": "{{ instance }}" - } - ] - }, - { - "id": 12, - "type": "timeseries", - "title": "CPU Requests vs Limits vs Usage", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 10, - "gradientMode": "none", - "spanNulls": false, - "showPoints": "never", - "stacking": { "mode": "none", "group": "A" }, - "axisPlacement": "auto", - "lineWidth": 1 - }, - "unit": "short", - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, - "color": { "mode": "palette-classic" }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } - }, - "targets": [ - { - "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "sum(kube_pod_container_resource_requests{namespace=~\"$namespace\", resource=\"cpu\"})", - "legendFormat": "Requests" - }, - { - "refId": "B", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "sum(kube_pod_container_resource_limits{namespace=~\"$namespace\", resource=\"cpu\"})", - "legendFormat": "Limits" - }, - { - "refId": "C", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\", image!=\"\"}[5m]))", - "legendFormat": "Usage" - } - ] - }, - { - "id": 20, - "type": "row", - "title": "Memory", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 }, - "collapsed": false, - "panels": [] - }, - { - "id": 21, - "type": "timeseries", - "title": "Per-Node Memory Usage", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 15 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 10, - "gradientMode": "scheme", - "spanNulls": false, - "showPoints": "never", - "stacking": { "mode": "none", "group": "A" }, - "axisPlacement": "auto", - "lineWidth": 1 - }, - "unit": "bytes", - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, - "color": { "mode": "palette-classic" }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } - }, - "targets": [ - { - "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes", - "legendFormat": "{{ instance }}" - } - ] - }, - { - "id": 22, - "type": "timeseries", - "title": "Memory Requests vs Limits vs Usage", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 15 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 10, - "gradientMode": "none", - "spanNulls": false, - "showPoints": "never", - "stacking": { "mode": "none", "group": "A" }, - "axisPlacement": "auto", - "lineWidth": 1 - }, - "unit": "bytes", - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, - "color": { "mode": "palette-classic" }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } - }, - "targets": [ - { - "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "sum(kube_pod_container_resource_requests{namespace=~\"$namespace\", resource=\"memory\"})", - "legendFormat": "Requests" - }, - { - "refId": "B", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "sum(kube_pod_container_resource_limits{namespace=~\"$namespace\", resource=\"memory\"})", - "legendFormat": "Limits" - }, - { - "refId": "C", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "sum(container_memory_working_set_bytes{namespace=~\"$namespace\", image!=\"\"})", - "legendFormat": "Usage" - } - ] - }, - { - "id": 30, - "type": "row", - "title": "Disk", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 }, - "collapsed": false, - "panels": [] - }, - { - "id": 31, - "type": "timeseries", - "title": "Node Filesystem Usage", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 10, - "gradientMode": "scheme", - "spanNulls": false, - "showPoints": "never", - "stacking": { "mode": "none", "group": "A" }, - "axisPlacement": "auto", - "lineWidth": 1 - }, - "unit": "percentunit", - "min": 0, - "max": 1, - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, - "color": { "mode": "palette-classic" }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } - }, - "targets": [ - { - "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "1 - (node_filesystem_avail_bytes{mountpoint=\"/\", fstype!=\"tmpfs\"} / node_filesystem_size_bytes{mountpoint=\"/\", fstype!=\"tmpfs\"})", - "legendFormat": "{{ instance }}" - } - ] - }, - { - "id": 32, - "type": "timeseries", - "title": "Disk I/O Throughput", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 10, - "gradientMode": "none", - "spanNulls": false, - "showPoints": "never", - "stacking": { "mode": "none", "group": "A" }, - "axisPlacement": "auto", - "lineWidth": 1 - }, - "unit": "Bps", - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, - "color": { "mode": "palette-classic" }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } - }, - "targets": [ - { - "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "sum by (instance) (rate(node_disk_read_bytes_total[5m]))", - "legendFormat": "{{ instance }} read" - }, - { - "refId": "B", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "sum by (instance) (rate(node_disk_written_bytes_total[5m]))", - "legendFormat": "{{ instance }} write" - } - ] - }, - { - "id": 40, - "type": "row", - "title": "Network", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 32 }, - "collapsed": false, - "panels": [] - }, - { - "id": 41, - "type": "timeseries", - "title": "Node Network Receive / Transmit", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 33 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 10, - "gradientMode": "none", - "spanNulls": false, - "showPoints": "never", - "stacking": { "mode": "none", "group": "A" }, - "axisPlacement": "auto", - "lineWidth": 1 - }, - "unit": "Bps", - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, - "color": { "mode": "palette-classic" }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } - }, - "targets": [ - { - "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "sum by (instance) (rate(node_network_receive_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[5m]))", - "legendFormat": "{{ instance }} rx" - }, - { - "refId": "B", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "sum by (instance) (rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[5m]))", - "legendFormat": "{{ instance }} tx" - } - ] - }, - { - "id": 42, - "type": "timeseries", - "title": "Pod Network I/O", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 33 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 10, - "gradientMode": "none", - "spanNulls": false, - "showPoints": "never", - "stacking": { "mode": "normal", "group": "A" }, - "axisPlacement": "auto", - "lineWidth": 1 - }, - "unit": "Bps", - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, - "color": { "mode": "palette-classic" }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } - }, - "targets": [ - { - "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "sum by (namespace) (rate(container_network_receive_bytes_total{namespace=~\"$namespace\", image!=\"\"}[5m]))", - "legendFormat": "{{ namespace }} rx" - }, - { - "refId": "B", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "sum by (namespace) (rate(container_network_transmit_bytes_total{namespace=~\"$namespace\", image!=\"\"}[5m]))", - "legendFormat": "{{ namespace }} tx" - } - ] - } - ] -} \ No newline at end of file diff --git a/infra/k8s/monitoring/dashboards/foxhunt-cockpit.json b/infra/k8s/monitoring/dashboards/foxhunt-cockpit.json index 271eaf556..709f0beb8 100644 --- a/infra/k8s/monitoring/dashboards/foxhunt-cockpit.json +++ b/infra/k8s/monitoring/dashboards/foxhunt-cockpit.json @@ -6,35 +6,21 @@ "description": "", "type": "datasource", "pluginId": "prometheus" - }, - { - "name": "DS_LOKI", - "label": "Loki", - "description": "", - "type": "datasource", - "pluginId": "loki" - }, - { - "name": "DS_TEMPO", - "label": "Tempo", - "description": "", - "type": "datasource", - "pluginId": "tempo" } ], "id": null, "uid": "foxhunt-cockpit", - "title": "Infrastructure Cockpit", - "description": "Single-page overview: cluster health, all services, GPU/training, resources, errors, live logs", + "title": "Foxhunt Operations Cockpit", + "description": "Single-pane-of-glass for the foxhunt HFT trading platform. Service health, CI/CD pipelines, resource utilization, gRPC backends, and security at a glance.", "tags": [ "foxhunt", - "production" + "cockpit" ], "timezone": "browser", "schemaVersion": 39, "version": 1, "editable": true, - "refresh": "30s", + "refresh": "10s", "time": { "from": "now-1h", "to": "now" @@ -53,7 +39,10 @@ "text": "5m", "value": "5m" }, - "auto": false + "auto": true, + "auto_count": 30, + "auto_min": "1m", + "label": "Interval" } ] }, @@ -61,3187 +50,1091 @@ { "id": 1, "type": "row", - "title": "Health Bar", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, + "title": "Health Status Bar", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "collapsed": false, "panels": [] }, { "id": 2, "type": "stat", - "title": "Nodes Ready", - "gridPos": { - "h": 3, - "w": 4, - "x": 0, - "y": 1 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "title": "Services Running", + "gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "targets": [ + { + "expr": "count(up{job=~\"foxhunt.*|pushgateway|argo-workflows\"} == 1)", + "legendFormat": "Up", + "refId": "A" + }, + { + "expr": "count(up{job=~\"foxhunt.*|pushgateway|argo-workflows\"})", + "legendFormat": "Total", + "refId": "B" + } + ], "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } + { "color": "red", "value": null }, + { "color": "orange", "value": 3 }, + { "color": "green", "value": 6 } ] }, - "mappings": [], - "color": { - "mode": "thresholds" - } + "unit": "none", + "mappings": [] }, "overrides": [] }, "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", + "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "background", "graphMode": "none", - "justifyMode": "center" + "textMode": "auto", + "justifyMode": "auto" }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kube_node_status_condition{condition=\"Ready\",status=\"true\"})", - "instant": true - } - ] + "transformations": [] }, { "id": 3, "type": "stat", - "title": "Pods Running", - "gridPos": { - "h": 3, - "w": 4, - "x": 4, - "y": 1 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "title": "Total Request Rate", + "gridPos": { "h": 4, "w": 4, "x": 4, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "targets": [ + { + "expr": "sum(rate(api_gateway_http_requests_total[5m]))", + "legendFormat": "req/s", + "refId": "A" + } + ], "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ - { - "color": "yellow", - "value": null - }, - { - "color": "green", - "value": 1 - } + { "color": "blue", "value": null } ] }, - "mappings": [], - "color": { - "mode": "thresholds" - } + "unit": "reqps", + "decimals": 1, + "mappings": [] }, "overrides": [] }, "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", + "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "background", "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\",phase=\"Running\"})", - "instant": true - } - ] + "textMode": "auto", + "justifyMode": "auto" + } }, { "id": 4, "type": "stat", - "title": "Pods Not Ready", - "gridPos": { - "h": 3, - "w": 4, - "x": 8, - "y": 1 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "title": "Error Rate", + "gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "targets": [ + { + "expr": "sum(rate(api_gateway_http_requests_total{status=~\"5..\"}[5m])) / sum(rate(api_gateway_http_requests_total[5m])) * 100", + "legendFormat": "Error %", + "refId": "A" + } + ], "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 1 - }, - { - "color": "red", - "value": 3 - } + { "color": "green", "value": null }, + { "color": "yellow", "value": 1 }, + { "color": "red", "value": 5 } ] }, + "unit": "percent", + "decimals": 2, "mappings": [], - "color": { - "mode": "thresholds" - } + "min": 0, + "max": 100 }, "overrides": [] }, "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", + "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "background", "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\",phase=~\"Pending|Failed|Unknown\"}) or vector(0)", - "instant": true - } - ] + "textMode": "auto", + "justifyMode": "auto" + } }, { "id": 5, "type": "stat", - "title": "Training Jobs", - "gridPos": { - "h": 3, - "w": 4, - "x": 12, - "y": 1 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "title": "Active Workflows", + "gridPos": { "h": 4, "w": 4, "x": 12, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "targets": [ + { + "expr": "argo_workflows_gauge{status=\"Running\"}", + "legendFormat": "Running", + "refId": "A" + } + ], "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ - { - "color": "blue", - "value": null - } + { "color": "blue", "value": null }, + { "color": "green", "value": 1 } ] }, - "mappings": [], - "color": { - "mode": "thresholds" - } + "unit": "none", + "mappings": [] }, "overrides": [] }, "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", + "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "background", "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kube_job_status_active{namespace=\"foxhunt\"}) or vector(0)", - "instant": true - } - ] + "textMode": "auto", + "justifyMode": "auto" + } }, { "id": 6, "type": "stat", - "title": "GPU Available", - "gridPos": { - "h": 3, - "w": 4, - "x": 16, - "y": 1 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "title": "GPU Utilization", + "gridPos": { "h": 4, "w": 4, "x": 16, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "targets": [ + { + "expr": "avg(DCGM_FI_DEV_GPU_UTIL)", + "legendFormat": "GPU %", + "refId": "A" + } + ], "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } + { "color": "green", "value": null }, + { "color": "yellow", "value": 70 }, + { "color": "red", "value": 90 } ] }, - "mappings": [], - "color": { - "mode": "thresholds" - } + "unit": "percent", + "decimals": 0, + "min": 0, + "max": 100, + "mappings": [] }, "overrides": [] }, "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", + "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "background", "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kube_node_status_allocatable{resource=\"nvidia_com_gpu\"}) or vector(0)", - "instant": true - } - ] + "textMode": "auto", + "justifyMode": "auto" + } }, { "id": 7, "type": "stat", - "title": "Errors /min", - "gridPos": { - "h": 3, - "w": 4, - "x": 20, - "y": 1 - }, - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, + "title": "Training Active", + "gridPos": { "h": 4, "w": 4, "x": 20, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "targets": [ + { + "expr": "ml_training_active_workers or count(foxhunt_training_current_epoch > 0)", + "legendFormat": "Workers", + "refId": "A" + } + ], "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 5 - }, - { - "color": "red", - "value": 20 - } + { "color": "semi-dark-blue", "value": null }, + { "color": "green", "value": 1 } ] }, - "mappings": [], - "color": { - "mode": "thresholds" - } + "unit": "none", + "mappings": [ + { + "type": "special", + "options": { + "match": "null", + "result": { "text": "0", "color": "text" } + } + } + ] }, "overrides": [] }, "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", + "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "background", "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "expr": "sum(count_over_time({namespace=\"foxhunt\", container!~\"loki|grafana|grafana-sc-dashboard|prometheus-server\"} |= \"ERROR\" [1m]))", - "instant": true - } - ] + "textMode": "auto", + "justifyMode": "auto" + } }, { "id": 8, "type": "row", - "title": "Service Status", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 4 - }, + "title": "Service Health Matrix", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, "collapsed": false, "panels": [] }, { "id": 9, - "type": "stat", - "title": "api", - "gridPos": { - "h": 3, - "w": 3, - "x": 0, - "y": 5 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "type": "table", + "title": "Service Status Table", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "targets": [ + { + "expr": "up{job=~\"foxhunt.*\"}", + "legendFormat": "{{job}}", + "refId": "A", + "format": "table", + "instant": true + }, + { + "expr": "time() - process_start_time_seconds{job=~\"foxhunt.*\"}", + "legendFormat": "{{job}}", + "refId": "B", + "format": "table", + "instant": true + }, + { + "expr": "sum by (job) (rate(api_gateway_http_requests_total{job=~\"foxhunt.*\"}[5m]))", + "legendFormat": "{{job}}", + "refId": "C", + "format": "table", + "instant": true + }, + { + "expr": "sum by (job) (rate(api_gateway_http_requests_total{job=~\"foxhunt.*\", status=~\"5..\"}[5m])) / sum by (job) (rate(api_gateway_http_requests_total{job=~\"foxhunt.*\"}[5m])) * 100", + "legendFormat": "{{job}}", + "refId": "D", + "format": "table", + "instant": true + }, + { + "expr": "histogram_quantile(0.99, sum by (job, le) (rate(api_gateway_http_request_duration_seconds_bucket{job=~\"foxhunt.*\"}[5m])))", + "legendFormat": "{{job}}", + "refId": "E", + "format": "table", + "instant": true + } + ], "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } + { "color": "green", "value": null } ] }, - "mappings": [ - { - "type": "value", - "options": { - "0": { - "text": "DOWN", - "color": "red" - }, - "1": { - "text": "UP", - "color": "green" - } - } - } - ], - "color": { - "mode": "thresholds" - } + "mappings": [] }, - "overrides": [] + "overrides": [ + { + "matcher": { "id": "byName", "options": "Value #A" }, + "properties": [ + { "id": "displayName", "value": "Status" }, + { + "id": "mappings", + "value": [ + { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } } + ] + }, + { "id": "custom.cellOptions", "value": { "type": "color-background" } } + ] + }, + { + "matcher": { "id": "byName", "options": "Value #B" }, + "properties": [ + { "id": "displayName", "value": "Uptime" }, + { "id": "unit", "value": "s" } + ] + }, + { + "matcher": { "id": "byName", "options": "Value #C" }, + "properties": [ + { "id": "displayName", "value": "Request Rate" }, + { "id": "unit", "value": "reqps" }, + { "id": "decimals", "value": 2 } + ] + }, + { + "matcher": { "id": "byName", "options": "Value #D" }, + "properties": [ + { "id": "displayName", "value": "Error Rate %" }, + { "id": "unit", "value": "percent" }, + { "id": "decimals", "value": 2 }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 1 }, + { "color": "red", "value": 5 } + ] + } + }, + { "id": "custom.cellOptions", "value": { "type": "color-background" } } + ] + }, + { + "matcher": { "id": "byName", "options": "Value #E" }, + "properties": [ + { "id": "displayName", "value": "P99 Latency" }, + { "id": "unit", "value": "s" }, + { "id": "decimals", "value": 3 } + ] + }, + { + "matcher": { "id": "byName", "options": "job" }, + "properties": [ + { "id": "displayName", "value": "Service" } + ] + } + ] }, "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" + "showHeader": true, + "footer": { "show": false }, + "sortBy": [ + { "displayName": "Service", "desc": false } + ] }, - "links": [ + "transformations": [ { - "title": "Service Detail", - "url": "/d/foxhunt-services?var-service=api", - "targetBlank": false - } - ], - "targets": [ + "id": "merge", + "options": {} + }, { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"api.*\", phase=\"Running\"}) > 0 or vector(0)", - "instant": true + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "__name__": true, + "instance": true + } + } } ] }, { "id": 10, - "type": "stat", - "title": "trading-service", - "gridPos": { - "h": 3, - "w": 3, - "x": 3, - "y": 5 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - }, - "mappings": [ - { - "type": "value", - "options": { - "0": { - "text": "DOWN", - "color": "red" - }, - "1": { - "text": "UP", - "color": "green" - } - } - } - ], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "links": [ - { - "title": "Service Detail", - "url": "/d/foxhunt-services?var-service=trading-service", - "targetBlank": false - } - ], - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"trading-service.*\", phase=\"Running\"}) > 0 or vector(0)", - "instant": true - } - ] - }, - { - "id": 11, - "type": "stat", - "title": "ml-training", - "gridPos": { - "h": 3, - "w": 3, - "x": 6, - "y": 5 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - }, - "mappings": [ - { - "type": "value", - "options": { - "0": { - "text": "DOWN", - "color": "red" - }, - "1": { - "text": "UP", - "color": "green" - } - } - } - ], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "links": [ - { - "title": "Service Detail", - "url": "/d/foxhunt-services?var-service=ml-training-service", - "targetBlank": false - } - ], - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"ml-training-service.*\", phase=\"Running\"}) > 0 or vector(0)", - "instant": true - } - ] - }, - { - "id": 12, - "type": "stat", - "title": "backtesting", - "gridPos": { - "h": 3, - "w": 3, - "x": 9, - "y": 5 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - }, - "mappings": [ - { - "type": "value", - "options": { - "0": { - "text": "DOWN", - "color": "red" - }, - "1": { - "text": "UP", - "color": "green" - } - } - } - ], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "links": [ - { - "title": "Service Detail", - "url": "/d/foxhunt-services?var-service=backtesting-service", - "targetBlank": false - } - ], - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"backtesting-service.*\", phase=\"Running\"}) > 0 or vector(0)", - "instant": true - } - ] - }, - { - "id": 13, - "type": "stat", - "title": "trading-agent", - "gridPos": { - "h": 3, - "w": 3, - "x": 12, - "y": 5 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - }, - "mappings": [ - { - "type": "value", - "options": { - "0": { - "text": "DOWN", - "color": "red" - }, - "1": { - "text": "UP", - "color": "green" - } - } - } - ], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "links": [ - { - "title": "Service Detail", - "url": "/d/foxhunt-services?var-service=trading-agent-service", - "targetBlank": false - } - ], - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"trading-agent.*\", phase=\"Running\"}) > 0 or vector(0)", - "instant": true - } - ] - }, - { - "id": 14, - "type": "stat", - "title": "broker-gw", - "gridPos": { - "h": 3, - "w": 3, - "x": 15, - "y": 5 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - }, - "mappings": [ - { - "type": "value", - "options": { - "0": { - "text": "DOWN", - "color": "red" - }, - "1": { - "text": "UP", - "color": "green" - } - } - } - ], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "links": [ - { - "title": "Service Detail", - "url": "/d/foxhunt-services?var-service=broker-gateway", - "targetBlank": false - } - ], - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"broker-gateway.*\", phase=\"Running\"}) > 0 or vector(0)", - "instant": true - } - ] - }, - { - "id": 15, - "type": "stat", - "title": "data-acq", - "gridPos": { - "h": 3, - "w": 3, - "x": 18, - "y": 5 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - }, - "mappings": [ - { - "type": "value", - "options": { - "0": { - "text": "DOWN", - "color": "red" - }, - "1": { - "text": "UP", - "color": "green" - } - } - } - ], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "links": [ - { - "title": "Service Detail", - "url": "/d/foxhunt-services?var-service=data-acquisition-service", - "targetBlank": false - } - ], - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"data-acquisition.*\", phase=\"Running\"}) > 0 or vector(0)", - "instant": true - } - ] - }, - { - "id": 100, - "type": "row", - "title": "Platform Services", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 8 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 101, - "type": "stat", - "title": "postgres", - "gridPos": { - "h": 3, - "w": 4, - "x": 0, - "y": 9 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - }, - "mappings": [ - { - "type": "value", - "options": { - "0": { - "text": "DOWN", - "color": "red" - }, - "1": { - "text": "UP", - "color": "green" - } - } - } - ], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"postgres-.*\", phase=\"Running\"}) > 0 or vector(0)", - "instant": true - } - ] - }, - { - "id": 102, - "type": "stat", - "title": "redis", - "gridPos": { - "h": 3, - "w": 4, - "x": 4, - "y": 9 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - }, - "mappings": [ - { - "type": "value", - "options": { - "0": { - "text": "DOWN", - "color": "red" - }, - "1": { - "text": "UP", - "color": "green" - } - } - } - ], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"redis-.*\", phase=\"Running\"}) > 0 or vector(0)", - "instant": true - } - ] - }, - { - "id": 103, - "type": "stat", - "title": "minio", - "gridPos": { - "h": 3, - "w": 4, - "x": 8, - "y": 9 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - }, - "mappings": [ - { - "type": "value", - "options": { - "0": { - "text": "DOWN", - "color": "red" - }, - "1": { - "text": "UP", - "color": "green" - } - } - } - ], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"minio-[^i].*\", phase=\"Running\"}) > 0 or vector(0)", - "instant": true - } - ] - }, - { - "id": 104, - "type": "stat", - "title": "questdb", - "gridPos": { - "h": 3, - "w": 4, - "x": 12, - "y": 9 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - }, - "mappings": [ - { - "type": "value", - "options": { - "0": { - "text": "DOWN", - "color": "red" - }, - "1": { - "text": "UP", - "color": "green" - } - } - } - ], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"questdb-.*\", phase=\"Running\"}) > 0 or vector(0)", - "instant": true - } - ] - }, - { - "id": 105, - "type": "stat", - "title": "gitlab", - "gridPos": { - "h": 3, - "w": 4, - "x": 16, - "y": 9 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - }, - "mappings": [ - { - "type": "value", - "options": { - "0": { - "text": "DOWN", - "color": "red" - }, - "1": { - "text": "UP", - "color": "green" - } - } - } - ], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"gitlab-webservice.*\", phase=\"Running\"}) > 0 or vector(0)", - "instant": true - } - ], - "links": [ - { - "title": "Detail", - "url": "/d/foxhunt-gitlab-services", - "targetBlank": false - } - ] - }, - { - "id": 106, - "type": "stat", - "title": "tailscale", - "gridPos": { - "h": 3, - "w": 4, - "x": 20, - "y": 9 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - }, - "mappings": [ - { - "type": "value", - "options": { - "0": { - "text": "DOWN", - "color": "red" - }, - "1": { - "text": "UP", - "color": "green" - } - } - } - ], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"tailscale-.*\", phase=\"Running\"}) > 0 or vector(0)", - "instant": true - } - ] - }, - { - "id": 107, - "type": "stat", - "title": "grafana", - "gridPos": { - "h": 3, - "w": 4, - "x": 0, - "y": 12 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - }, - "mappings": [ - { - "type": "value", - "options": { - "0": { - "text": "DOWN", - "color": "red" - }, - "1": { - "text": "UP", - "color": "green" - } - } - } - ], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"grafana-.*\", phase=\"Running\"}) > 0 or vector(0)", - "instant": true - } - ] - }, - { - "id": 108, - "type": "stat", - "title": "loki", - "gridPos": { - "h": 3, - "w": 4, - "x": 4, - "y": 12 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - }, - "mappings": [ - { - "type": "value", - "options": { - "0": { - "text": "DOWN", - "color": "red" - }, - "1": { - "text": "UP", - "color": "green" - } - } - } - ], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"loki-.*\", phase=\"Running\"}) > 0 or vector(0)", - "instant": true - } - ] - }, - { - "id": 109, - "type": "stat", - "title": "tempo", - "gridPos": { - "h": 3, - "w": 4, - "x": 8, - "y": 12 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - }, - "mappings": [ - { - "type": "value", - "options": { - "0": { - "text": "DOWN", - "color": "red" - }, - "1": { - "text": "UP", - "color": "green" - } - } - } - ], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"tempo-.*\", phase=\"Running\"}) > 0 or vector(0)", - "instant": true - } - ] - }, - { - "id": 110, - "type": "stat", - "title": "ksm", - "gridPos": { - "h": 3, - "w": 4, - "x": 12, - "y": 12 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - }, - "mappings": [ - { - "type": "value", - "options": { - "0": { - "text": "DOWN", - "color": "red" - }, - "1": { - "text": "UP", - "color": "green" - } - } - } - ], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"kube-state-metrics-.*\", phase=\"Running\"}) > 0 or vector(0)", - "instant": true - } - ] - }, - { - "id": 111, - "type": "stat", - "title": "runners", - "gridPos": { - "h": 3, - "w": 4, - "x": 16, - "y": 12 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "yellow", - "value": 1 - }, - { - "color": "green", - "value": 2 - } - ] - }, - "mappings": [], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "value_and_name", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "count(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"gitlab-runner.*|runner-.*\", phase=\"Running\"}) or vector(0)", - "instant": true - } - ] - }, - { - "id": 112, - "type": "stat", - "title": "collectors", - "gridPos": { - "h": 3, - "w": 4, - "x": 20, - "y": 12 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "yellow", - "value": 1 - }, - { - "color": "green", - "value": 2 - } - ] - }, - "mappings": [], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "value_and_name", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "count(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"promtail-.*|node-exporter-.*\", phase=\"Running\"}) or vector(0)", - "instant": true - } - ] - }, - { - "id": 17, - "type": "row", - "title": "Resources", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 15 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 18, - "type": "gauge", - "title": "Cluster CPU %", - "gridPos": { - "h": 4, - "w": 4, - "x": 0, - "y": 16 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 70 - }, - { - "color": "red", - "value": 90 - } - ] - }, - "unit": "percent", - "min": 0, - "max": 100, - "mappings": [], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "100 * (1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[$interval])))", - "instant": true - } - ] - }, - { - "id": 19, - "type": "gauge", - "title": "Cluster Memory %", - "gridPos": { - "h": 4, - "w": 4, - "x": 4, - "y": 16 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 75 - }, - { - "color": "red", - "value": 90 - } - ] - }, - "unit": "percent", - "min": 0, - "max": 100, - "mappings": [], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "100 * (1 - sum(node_memory_MemAvailable_bytes) / sum(node_memory_MemTotal_bytes))", - "instant": true - } - ] - }, - { - "id": 20, - "type": "gauge", - "title": "Cluster Disk %", - "gridPos": { - "h": 4, - "w": 4, - "x": 8, - "y": 16 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 75 - }, - { - "color": "red", - "value": 90 - } - ] - }, - "unit": "percent", - "min": 0, - "max": 100, - "mappings": [], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "100 * (1 - sum(node_filesystem_avail_bytes{mountpoint=\"/\",fstype!=\"tmpfs\"}) / sum(node_filesystem_size_bytes{mountpoint=\"/\",fstype!=\"tmpfs\"}))", - "instant": true - } - ] - }, - { - "id": 21, "type": "timeseries", - "title": "CPU by Service", - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 20 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "title": "Service Request Rate", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "targets": [ + { + "expr": "sum by (service) (rate(api_gateway_http_requests_total[$interval]))", + "legendFormat": "{{service}}", + "refId": "A" + } + ], "fieldConfig": { "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "reqps", "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 30, - "gradientMode": "none", - "spanNulls": false, - "showPoints": "never", - "stacking": { - "mode": "normal", - "group": "A" - }, - "axisPlacement": "auto", - "lineWidth": 1 - }, - "unit": "short", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "color": { - "mode": "palette-classic" - }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "calcs": [] - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\",container!=\"\",container!=\"POD\"}[$interval]))", - "legendFormat": "{{ pod }}" - } - ] - }, - { - "id": 38, - "type": "timeseries", - "title": "Memory by Service", - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 20 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", + "stacking": { "mode": "normal", "group": "A" }, "lineWidth": 1, - "fillOpacity": 20, + "pointSize": 5, + "showPoints": "never", "gradientMode": "opacity", - "spanNulls": false, - "showPoints": "never", - "stacking": { - "mode": "normal", - "group": "A" - }, - "axisPlacement": "auto", - "scaleDistribution": { - "type": "linear" - } - }, - "unit": "bytes", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "color": { - "mode": "palette-classic" - } - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "list", - "placement": "bottom", - "calcs": [ - "mean", - "max" - ] - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum by (pod) (container_memory_working_set_bytes{namespace=\"foxhunt\",container!=\"POD\",container!=\"\",pod=~\"(api|trading-service|ml-training|backtesting|trading-agent|broker-gateway|data-acquisition).*\"})", - "legendFormat": "{{pod}}", - "range": true - } - ] - }, - { - "id": 22, - "type": "row", - "title": "GPU & Training", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 26 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 23, - "type": "timeseries", - "title": "GPU Utilization", - "description": "SM compute utilization and memory copy utilization over time", - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 27 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 20, - "gradientMode": "opacity", - "spanNulls": false, - "showPoints": "never", - "stacking": { - "mode": "none", - "group": "A" - }, - "axisPlacement": "auto", - "axisLabel": "", - "scaleDistribution": { - "type": "linear" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "unit": "percent", - "min": 0, - "max": 100, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 80 - }, - { - "color": "red", - "value": 95 - } - ] - }, - "color": { - "mode": "palette-classic" - }, - "noValue": "GPU offline" - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "list", - "placement": "bottom", - "calcs": [ - "mean", - "max" - ] - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "dcgm_gpu_utilization{gpu=\"0\"}", - "legendFormat": "SM Utilization", - "range": true - }, - { - "refId": "B", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "dcgm_mem_copy_utilization{gpu=\"0\"}", - "legendFormat": "Memory Copy", - "range": true - } - ] - }, - { - "id": 24, - "type": "timeseries", - "title": "GPU VRAM", - "description": "Framebuffer memory used vs total (NVIDIA L40S 48GB)", - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 27 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 20, - "gradientMode": "opacity", - "spanNulls": false, - "showPoints": "never", - "stacking": { - "mode": "none", - "group": "A" - }, - "axisPlacement": "auto", - "axisLabel": "", - "scaleDistribution": { - "type": "linear" - } - }, - "unit": "decmbytes", - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "color": { - "mode": "palette-classic" - }, - "noValue": "GPU offline" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Total" - }, - "properties": [ - { - "id": "custom.lineStyle", - "value": { - "fill": "dash", - "dash": [ - 10, - 10 - ] - } - }, - { - "id": "custom.fillOpacity", - "value": 0 - }, - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "white" - } - } - ] - } - ] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "list", - "placement": "bottom", - "calcs": [ - "mean", - "max" - ] - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "dcgm_fb_used{gpu=\"0\"}", - "legendFormat": "Used", - "range": true - }, - { - "refId": "B", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "dcgm_fb_used{gpu=\"0\"} + dcgm_fb_free{gpu=\"0\"}", - "legendFormat": "Total", - "range": true - } - ] - }, - { - "id": 25, - "type": "timeseries", - "title": "Training Jobs Timeline", - "gridPos": { - "h": 5, - "w": 24, - "x": 0, - "y": 33 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "stepAfter", - "fillOpacity": 20, - "gradientMode": "none", - "spanNulls": false, - "showPoints": "never", - "stacking": { - "mode": "none", - "group": "A" - }, - "axisPlacement": "auto", - "lineWidth": 2 - }, - "unit": "short", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "color": { - "mode": "palette-classic" - }, - "mappings": [] - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "active" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "succeeded" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "failed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - } - ] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "calcs": [] - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kube_job_status_active{namespace=\"foxhunt\"}) or vector(0)", - "legendFormat": "active" - }, - { - "refId": "B", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kube_job_status_succeeded{namespace=\"foxhunt\"}) or vector(0)", - "legendFormat": "succeeded" - }, - { - "refId": "C", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kube_job_status_failed{namespace=\"foxhunt\"}) or vector(0)", - "legendFormat": "failed" - } - ] - }, - { - "id": 26, - "type": "row", - "title": "Restarts & Network", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 32 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 27, - "type": "timeseries", - "title": "Pod Restarts (1h)", - "gridPos": { - "h": 5, - "w": 12, - "x": 0, - "y": 34 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "bars", - "lineInterpolation": "linear", - "fillOpacity": 80, - "gradientMode": "none", - "spanNulls": false, - "showPoints": "never", - "stacking": { - "mode": "normal", - "group": "A" - }, - "axisPlacement": "auto", - "lineWidth": 0 - }, - "unit": "short", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - }, - "color": { - "mode": "palette-classic" + "axisPlacement": "auto" }, "mappings": [] }, "overrides": [] }, "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "calcs": [] - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "increase(kube_pod_container_status_restarts_total{namespace=\"foxhunt\"}[1h]) > 0", - "legendFormat": "{{ pod }}" - } - ] + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "lastNotNull"] } + } }, { - "id": 28, - "type": "timeseries", - "title": "Network I/O by Pod", - "gridPos": { - "h": 5, - "w": 12, - "x": 12, - "y": 34 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "id": 11, + "type": "row", + "title": "Argo CI/CD Pipeline", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 }, + "collapsed": false, + "panels": [] + }, + { + "id": 12, + "type": "stat", + "title": "Workflow Status", + "gridPos": { "h": 8, "w": 6, "x": 0, "y": 15 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "targets": [ + { + "expr": "argo_workflows_gauge{status=\"Running\"}", + "legendFormat": "Running", + "refId": "A" + }, + { + "expr": "argo_workflows_gauge{status=\"Succeeded\"}", + "legendFormat": "Succeeded", + "refId": "B" + }, + { + "expr": "argo_workflows_gauge{status=\"Failed\"}", + "legendFormat": "Failed", + "refId": "C" + }, + { + "expr": "argo_workflows_gauge{status=\"Pending\"}", + "legendFormat": "Pending", + "refId": "D" + } + ], "fieldConfig": { "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "text", "value": null } + ] + }, + "unit": "none", + "mappings": [] + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "Running" }, + "properties": [ + { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } + ] + }, + { + "matcher": { "id": "byName", "options": "Succeeded" }, + "properties": [ + { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } + ] + }, + { + "matcher": { "id": "byName", "options": "Failed" }, + "properties": [ + { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } + ] + }, + { + "matcher": { "id": "byName", "options": "Pending" }, + "properties": [ + { "id": "color", "value": { "fixedColor": "yellow", "mode": "fixed" } } + ] + } + ] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"] }, + "colorMode": "background", + "graphMode": "none", + "textMode": "auto", + "justifyMode": "auto", + "orientation": "vertical" + } + }, + { + "id": 13, + "type": "timeseries", + "title": "Workflow Duration", + "gridPos": { "h": 8, "w": 9, "x": 6, "y": 15 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "targets": [ + { + "expr": "histogram_quantile(0.50, sum by (le) (rate(argo_workflows_operation_duration_seconds_bucket[5m])))", + "legendFormat": "p50", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, sum by (le) (rate(argo_workflows_operation_duration_seconds_bucket[5m])))", + "legendFormat": "p95", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "s", "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 10, - "gradientMode": "none", - "spanNulls": false, + "lineWidth": 2, + "pointSize": 5, "showPoints": "never", - "stacking": { - "mode": "none", - "group": "A" - }, - "axisPlacement": "auto", - "lineWidth": 1 - }, - "unit": "Bps", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "color": { - "mode": "palette-classic" + "gradientMode": "scheme", + "axisPlacement": "auto" }, "mappings": [] }, - "overrides": [] + "overrides": [ + { + "matcher": { "id": "byName", "options": "p95" }, + "properties": [ + { "id": "color", "value": { "fixedColor": "orange", "mode": "fixed" } } + ] + }, + { + "matcher": { "id": "byName", "options": "p50" }, + "properties": [ + { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } + ] + } + ] }, "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "calcs": [] - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "lastNotNull"] } + } + }, + { + "id": 14, + "type": "timeseries", + "title": "Workflow Errors", + "gridPos": { "h": 8, "w": 9, "x": 15, "y": 15 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "targets": [ { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum by (pod) (rate(container_network_receive_bytes_total{namespace=\"foxhunt\"}[$interval]))", - "legendFormat": "{{ pod }} rx" - }, - { - "refId": "B", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum by (pod) (rate(container_network_transmit_bytes_total{namespace=\"foxhunt\"}[$interval]))", - "legendFormat": "{{ pod }} tx" + "expr": "rate(argo_workflows_error_count[5m])", + "legendFormat": "{{cause}}", + "refId": "A" } - ] - }, - { - "id": 29, - "type": "row", - "title": "Errors", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 39 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 30, - "type": "timeseries", - "title": "Error Rate by Service", - "gridPos": { - "h": 5, - "w": 12, - "x": 0, - "y": 40 - }, - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, + ], "fieldConfig": { "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "ops", "custom": { "drawStyle": "bars", "lineInterpolation": "linear", - "fillOpacity": 60, - "gradientMode": "none", - "spanNulls": false, + "fillOpacity": 80, + "stacking": { "mode": "normal", "group": "A" }, + "lineWidth": 1, + "pointSize": 5, "showPoints": "never", - "stacking": { - "mode": "normal", - "group": "A" - }, - "axisPlacement": "auto", - "lineWidth": 0 - }, - "unit": "short", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - } - ] - }, - "color": { - "mode": "palette-classic" + "gradientMode": "none", + "axisPlacement": "auto" }, + "color": { "mode": "palette-classic" }, "mappings": [] }, "overrides": [] }, "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "calcs": [] - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "expr": "sum by (app) (count_over_time({namespace=\"foxhunt\", container!~\"loki|grafana|grafana-sc-dashboard|prometheus-server\"} |= \"ERROR\" [$interval]))", - "legendFormat": "{{ app }}" - } - ] + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["sum", "lastNotNull"] } + } }, { - "id": 31, - "type": "table", - "title": "Top Error Messages", - "gridPos": { - "h": 5, - "w": 12, - "x": 12, - "y": 40 - }, - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "mappings": [], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "showHeader": true, - "sortBy": [ - { - "displayName": "Value", - "desc": true - } - ], - "footer": { - "show": false - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "expr": "topk(10, sum by (app) (count_over_time({namespace=\"foxhunt\", container!~\"loki|grafana|grafana-sc-dashboard|prometheus-server\"} |= \"ERROR\" | logfmt [$interval])))", - "legendFormat": "{{ app }}", - "instant": true - } - ] - }, - { - "id": 32, + "id": 15, "type": "row", - "title": "Live Error Log", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 45 - }, + "title": "Resource Overview", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 }, "collapsed": false, "panels": [] }, { - "id": 33, - "type": "logs", - "title": "Recent Errors", - "description": "Live stream of ERROR-level log lines from all foxhunt services", - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 46 - }, - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "options": { - "showTime": true, - "showLabels": true, - "showCommonLabels": false, - "wrapLogMessage": true, - "prettifyLogMessage": false, - "enableLogDetails": true, - "sortOrder": "Descending", - "dedupStrategy": "none" - }, + "id": 16, + "type": "gauge", + "title": "CPU Usage", + "gridPos": { "h": 8, "w": 6, "x": 0, "y": 24 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "targets": [ { - "refId": "A", - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "expr": "{namespace=\"foxhunt\", container!~\"loki|grafana|grafana-sc-dashboard|prometheus-server\"} |= \"ERROR\"", - "maxLines": 200 + "expr": "avg(1 - rate(node_cpu_seconds_total{mode=\"idle\"}[5m])) * 100", + "legendFormat": "CPU %", + "refId": "A" } - ] - }, - { - "id": 34, - "type": "row", - "title": "Trace Health", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 53 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 35, - "type": "stat", - "title": "Spans/sec", - "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 54 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + ], "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ - { - "color": "blue", - "value": null - } - ] - }, - "mappings": [], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "area", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(tempo_distributor_spans_received_total[5m]))", - "instant": true - } - ] - }, - { - "id": 36, - "type": "stat", - "title": "Trace Error Rate", - "gridPos": { - "h": 4, - "w": 6, - "x": 6, - "y": 54 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 1 - }, - { - "color": "red", - "value": 5 - } + { "color": "green", "value": null }, + { "color": "yellow", "value": 60 }, + { "color": "red", "value": 80 } ] }, "unit": "percent", - "mappings": [], - "color": { - "mode": "thresholds" - } + "decimals": 1, + "min": 0, + "max": 100, + "mappings": [] }, "overrides": [] }, "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(tempo_distributor_spans_received_total{status=\"error\"}[5m])) / sum(rate(tempo_distributor_spans_received_total[5m]))", - "instant": true - } - ] + "reduceOptions": { "calcs": ["lastNotNull"] }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "orientation": "auto" + } }, { - "id": 37, - "type": "timeseries", - "title": "Spans/sec by Service", - "gridPos": { - "h": 4, - "w": 12, - "x": 12, - "y": 54 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "id": 17, + "type": "gauge", + "title": "Memory Usage", + "gridPos": { "h": 8, "w": 6, "x": 6, "y": 24 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "targets": [ + { + "expr": "(1 - (node_memory_AvailableBytes / node_memory_MemoryTotal_bytes)) * 100", + "legendFormat": "Memory %", + "refId": "A" + } + ], "fieldConfig": { "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 60 }, + { "color": "red", "value": 80 } + ] + }, + "unit": "percent", + "decimals": 1, + "min": 0, + "max": 100, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"] }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "orientation": "auto" + } + }, + { + "id": 18, + "type": "gauge", + "title": "Disk Usage", + "gridPos": { "h": 8, "w": 6, "x": 12, "y": 24 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "targets": [ + { + "expr": "(1 - (node_filesystem_avail_bytes{mountpoint=\"/\"} / node_filesystem_size_bytes{mountpoint=\"/\"})) * 100", + "legendFormat": "Disk %", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 60 }, + { "color": "red", "value": 80 } + ] + }, + "unit": "percent", + "decimals": 1, + "min": 0, + "max": 100, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"] }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "orientation": "auto" + } + }, + { + "id": 19, + "type": "timeseries", + "title": "Network IO", + "gridPos": { "h": 8, "w": 6, "x": 18, "y": 24 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "targets": [ + { + "expr": "rate(node_network_receive_bytes_total{device!=\"lo\"}[5m])", + "legendFormat": "Receive {{device}}", + "refId": "A" + }, + { + "expr": "rate(node_network_transmit_bytes_total{device!=\"lo\"}[5m])", + "legendFormat": "Transmit {{device}}", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "Bps", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 15, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "gradientMode": "opacity", + "axisPlacement": "auto" + }, + "mappings": [] + }, + "overrides": [ + { + "matcher": { "id": "byRegexp", "options": "/Transmit.*/" }, + "properties": [ + { "id": "custom.transform", "value": "negative-Y" } + ] + } + ] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } + } + }, + { + "id": 20, + "type": "row", + "title": "gRPC & Backend", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 32 }, + "collapsed": false, + "panels": [] + }, + { + "id": 21, + "type": "timeseries", + "title": "gRPC Request Rate", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 33 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "targets": [ + { + "expr": "sum by (grpc_service, grpc_method) (rate(grpc_server_handled_total[5m]))", + "legendFormat": "{{grpc_service}}/{{grpc_method}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "ops", "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 20, - "gradientMode": "none", - "spanNulls": false, + "stacking": { "mode": "normal", "group": "A" }, + "lineWidth": 1, + "pointSize": 5, "showPoints": "never", - "stacking": { - "mode": "normal", - "group": "A" - }, - "axisPlacement": "auto", - "lineWidth": 1 - }, - "unit": "ops", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "color": { - "mode": "palette-classic" + "gradientMode": "opacity", + "axisPlacement": "auto" }, + "color": { "mode": "palette-classic" }, "mappings": [] }, "overrides": [] }, "options": { - "legend": { - "displayMode": "list", - "placement": "bottom", - "calcs": [] - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "lastNotNull"] } + } + }, + { + "id": 22, + "type": "timeseries", + "title": "gRPC Latency", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 33 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "targets": [ { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum by (service_name)(rate(tempo_distributor_spans_received_total[5m]))", - "legendFormat": "{{ service_name }}" + "expr": "histogram_quantile(0.50, sum by (le, grpc_service) (rate(grpc_server_handling_seconds_bucket[5m])))", + "legendFormat": "p50 {{grpc_service}}", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, sum by (le, grpc_service) (rate(grpc_server_handling_seconds_bucket[5m])))", + "legendFormat": "p95 {{grpc_service}}", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.99, sum by (le, grpc_service) (rate(grpc_server_handling_seconds_bucket[5m])))", + "legendFormat": "p99 {{grpc_service}}", + "refId": "C" } - ] + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "s", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 0, + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "gradientMode": "none", + "axisPlacement": "auto" + }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [ + { + "matcher": { "id": "byRegexp", "options": "/p99.*/" }, + "properties": [ + { "id": "custom.lineStyle", "value": { "fill": "solid" } }, + { "id": "custom.lineWidth", "value": 2 } + ] + }, + { + "matcher": { "id": "byRegexp", "options": "/p50.*/" }, + "properties": [ + { "id": "custom.lineStyle", "value": { "fill": "dash", "dash": [10, 10] } }, + { "id": "custom.lineWidth", "value": 1 } + ] + } + ] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "lastNotNull"] } + } + }, + { + "id": 23, + "type": "row", + "title": "Rate Limiting & Auth", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 41 }, + "collapsed": false, + "panels": [] + }, + { + "id": 24, + "type": "timeseries", + "title": "Rate Limit Rejections", + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 42 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "targets": [ + { + "expr": "rate(api_gateway_rate_limit_rejected_total[5m])", + "legendFormat": "{{tier}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "ops", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 20, + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "gradientMode": "opacity", + "axisPlacement": "auto" + }, + "color": { "mode": "palette-classic" }, + "mappings": [] + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["sum", "mean", "lastNotNull"] } + } + }, + { + "id": 25, + "type": "timeseries", + "title": "Auth Success/Fail", + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 42 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "targets": [ + { + "expr": "rate(api_gateway_auth_attempts_total{result=\"success\"}[5m])", + "legendFormat": "Success", + "refId": "A" + }, + { + "expr": "rate(api_gateway_auth_attempts_total{result=\"failure\"}[5m])", + "legendFormat": "Failure", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "ops", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 20, + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "gradientMode": "opacity", + "axisPlacement": "auto" + }, + "mappings": [] + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "Success" }, + "properties": [ + { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } + ] + }, + { + "matcher": { "id": "byName", "options": "Failure" }, + "properties": [ + { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } + ] + } + ] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["sum", "mean", "lastNotNull"] } + } + }, + { + "id": 26, + "type": "stat", + "title": "JWT Cache Hit Rate", + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 42 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "targets": [ + { + "expr": "api_gateway_jwt_cache_hit_total / (api_gateway_jwt_cache_hit_total + api_gateway_jwt_cache_miss_total) * 100", + "legendFormat": "Hit Rate", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "yellow", "value": 50 }, + { "color": "green", "value": 80 } + ] + }, + "unit": "percent", + "decimals": 1, + "min": 0, + "max": 100, + "mappings": [ + { + "type": "special", + "options": { + "match": "null", + "result": { "text": "N/A", "color": "text" } + } + }, + { + "type": "special", + "options": { + "match": "nan", + "result": { "text": "N/A", "color": "text" } + } + } + ] + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"] }, + "colorMode": "background", + "graphMode": "none", + "textMode": "auto", + "justifyMode": "auto" + } } - ] + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { "type": "grafana", "uid": "-- Grafana --" }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "links": [], + "graphTooltip": 1 } diff --git a/infra/k8s/monitoring/dashboards/foxhunt-gpu-training.json b/infra/k8s/monitoring/dashboards/foxhunt-gpu-training.json deleted file mode 100644 index 7873f8768..000000000 --- a/infra/k8s/monitoring/dashboards/foxhunt-gpu-training.json +++ /dev/null @@ -1,1301 +0,0 @@ -{ - "__inputs": [ - { - "name": "DS_PROMETHEUS", - "label": "Prometheus", - "description": "", - "type": "datasource", - "pluginId": "prometheus" - }, - { - "name": "DS_LOKI", - "label": "Loki", - "description": "", - "type": "datasource", - "pluginId": "loki" - } - ], - "id": null, - "uid": "foxhunt-gpu-training", - "title": "GPU & Training Jobs", - "description": "GPU node pools, DCGM metrics, K8s training jobs, CI pipelines", - "tags": [ - "foxhunt", - "production" - ], - "timezone": "browser", - "schemaVersion": 39, - "version": 1, - "editable": true, - "refresh": "30s", - "time": { - "from": "now-1h", - "to": "now" - }, - "fiscalYearStartMonth": 0, - "liveNow": false, - "weekStart": "", - "templating": { - "list": [ - { - "name": "gpu", - "type": "query", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "query": "label_values(dcgm_gpu_utilization, gpu)", - "refresh": 2, - "includeAll": true, - "multi": true, - "allValue": ".*", - "current": { - "selected": true, - "text": "All", - "value": "$__all" - }, - "sort": 1 - }, - { - "name": "interval", - "type": "interval", - "query": "1m,5m,15m,30m,1h", - "current": { - "selected": true, - "text": "5m", - "value": "5m" - }, - "auto": false - } - ] - }, - "panels": [ - { - "id": 1, - "type": "row", - "title": "GPU Node Pool Status", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 2, - "type": "stat", - "title": "ci-compile Nodes", - "description": "L4-1-24G GPU nodes (0-1 autoscale)", - "gridPos": { - "h": 4, - "w": 4, - "x": 0, - "y": 1 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - }, - "mappings": [], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "count(kube_node_labels{label_k8s_scaleway_com_pool_name=\"ci-compile\"}) or vector(0)", - "instant": true - } - ] - }, - { - "id": 3, - "type": "stat", - "title": "ci-training Nodes", - "description": "L40S-1-48G GPU nodes (0-1 autoscale)", - "gridPos": { - "h": 4, - "w": 4, - "x": 4, - "y": 1 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - }, - "mappings": [], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "count(kube_node_labels{label_k8s_scaleway_com_pool_name=\"ci-training\"}) or vector(0)", - "instant": true - } - ] - }, - { - "id": 4, - "type": "stat", - "title": "ci-compile-cpu Nodes", - "description": "POP2-32C-128G CPU nodes (0-1 autoscale)", - "gridPos": { - "h": 4, - "w": 4, - "x": 8, - "y": 1 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - }, - "mappings": [], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "count(kube_node_labels{label_k8s_scaleway_com_pool_name=\"ci-compile-cpu\"}) or vector(0)", - "instant": true - } - ] - }, - { - "id": 5, - "type": "stat", - "title": "Active Training Jobs", - "gridPos": { - "h": 4, - "w": 4, - "x": 12, - "y": 1 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - }, - "mappings": [], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kube_job_status_active{namespace=\"foxhunt\"}) or vector(0)", - "instant": true - } - ] - }, - { - "id": 6, - "type": "stat", - "title": "Jobs Succeeded (24h)", - "gridPos": { - "h": 4, - "w": 4, - "x": 16, - "y": 1 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "mappings": [], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(increase(kube_job_status_succeeded{namespace=\"foxhunt\"}[24h])) or vector(0)", - "instant": true - } - ] - }, - { - "id": 7, - "type": "stat", - "title": "Jobs Failed (24h)", - "gridPos": { - "h": 4, - "w": 4, - "x": 20, - "y": 1 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - }, - "mappings": [], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(increase(kube_job_status_failed{namespace=\"foxhunt\"}[24h])) or vector(0)", - "instant": true - } - ] - }, - { - "id": 10, - "type": "row", - "title": "GPU Utilization (DCGM)", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 5 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 11, - "type": "timeseries", - "title": "GPU Utilization %", - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 6 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 20, - "gradientMode": "scheme", - "spanNulls": false, - "showPoints": "never", - "stacking": { - "mode": "none", - "group": "A" - }, - "axisPlacement": "auto", - "lineWidth": 2 - }, - "unit": "percent", - "min": 0, - "max": 100, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 80 - }, - { - "color": "red", - "value": 95 - } - ] - }, - "color": { - "mode": "palette-classic" - }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "max", - "last" - ] - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "dcgm_gpu_utilization{gpu=~\"$gpu\"}", - "legendFormat": "GPU {{ gpu }} ({{ instance }})" - } - ] - }, - { - "id": 12, - "type": "timeseries", - "title": "GPU Memory Used (MB)", - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 6 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 20, - "gradientMode": "scheme", - "spanNulls": false, - "showPoints": "never", - "stacking": { - "mode": "none", - "group": "A" - }, - "axisPlacement": "auto", - "lineWidth": 2 - }, - "unit": "decmbytes", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "color": { - "mode": "palette-classic" - }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "max", - "last" - ] - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "dcgm_fb_used{gpu=~\"$gpu\"}", - "legendFormat": "GPU {{ gpu }} used" - }, - { - "refId": "B", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "(dcgm_fb_used{gpu=~\"$gpu\"} + dcgm_fb_free{gpu=~\"$gpu\"})", - "legendFormat": "GPU {{ gpu }} total" - } - ] - }, - { - "id": 13, - "type": "timeseries", - "title": "GPU Temperature", - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 14 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 10, - "gradientMode": "scheme", - "spanNulls": false, - "showPoints": "never", - "stacking": { - "mode": "none", - "group": "A" - }, - "axisPlacement": "auto", - "lineWidth": 2 - }, - "unit": "celsius", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - }, - { - "color": "green", - "value": 40 - }, - { - "color": "yellow", - "value": 75 - }, - { - "color": "red", - "value": 85 - } - ] - }, - "color": { - "mode": "palette-classic" - }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "list", - "placement": "bottom", - "calcs": [] - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "dcgm_gpu_temp{gpu=~\"$gpu\"}", - "legendFormat": "GPU {{ gpu }}" - } - ] - }, - { - "id": 14, - "type": "timeseries", - "title": "GPU Power (Watts)", - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 14 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 10, - "gradientMode": "scheme", - "spanNulls": false, - "showPoints": "never", - "stacking": { - "mode": "none", - "group": "A" - }, - "axisPlacement": "auto", - "lineWidth": 2 - }, - "unit": "watt", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "color": { - "mode": "palette-classic" - }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "list", - "placement": "bottom", - "calcs": [] - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "dcgm_power_usage{gpu=~\"$gpu\"}", - "legendFormat": "GPU {{ gpu }}" - } - ] - }, - { - "id": 20, - "type": "row", - "title": "Training Jobs", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 20 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 21, - "type": "timeseries", - "title": "Active Jobs Over Time", - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 21 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "stepAfter", - "fillOpacity": 30, - "gradientMode": "none", - "spanNulls": false, - "showPoints": "never", - "stacking": { - "mode": "none", - "group": "A" - }, - "axisPlacement": "auto", - "lineWidth": 2 - }, - "unit": "short", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "color": { - "mode": "palette-classic" - }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "last" - ] - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "kube_job_status_active{namespace=\"foxhunt\"}", - "legendFormat": "{{ job_name }}" - } - ] - }, - { - "id": 22, - "type": "table", - "title": "Job Status Summary", - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 21 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "mappings": [], - "color": { - "mode": "thresholds" - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Failed" - }, - "properties": [ - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - } - }, - { - "id": "custom.displayMode", - "value": "color-background" - } - ] - } - ] - }, - "options": { - "showHeader": true, - "sortBy": [ - { - "displayName": "job_name", - "desc": false - } - ], - "footer": { - "show": false - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "kube_job_status_active{namespace=\"foxhunt\"}", - "legendFormat": "Active", - "instant": true, - "format": "table" - }, - { - "refId": "B", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "kube_job_status_succeeded{namespace=\"foxhunt\"}", - "legendFormat": "Succeeded", - "instant": true, - "format": "table" - }, - { - "refId": "C", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "kube_job_status_failed{namespace=\"foxhunt\"}", - "legendFormat": "Failed", - "instant": true, - "format": "table" - } - ] - }, - { - "id": 25, - "type": "timeseries", - "title": "GPU Node CPU Usage (Compute Pools)", - "description": "CPU usage on GPU/compute node pools when no DCGM data is available", - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 28 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 15, - "gradientMode": "scheme", - "spanNulls": false, - "showPoints": "never", - "stacking": { - "mode": "none", - "group": "A" - }, - "axisPlacement": "auto", - "lineWidth": 1 - }, - "unit": "percentunit", - "min": 0, - "max": 1, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "color": { - "mode": "palette-classic" - }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "list", - "placement": "bottom", - "calcs": [] - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[$interval]))", - "legendFormat": "{{ instance }}" - } - ] - }, - { - "id": 26, - "type": "timeseries", - "title": "GPU Node Memory Usage (Compute Pools)", - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 28 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 15, - "gradientMode": "scheme", - "spanNulls": false, - "showPoints": "never", - "stacking": { - "mode": "none", - "group": "A" - }, - "axisPlacement": "auto", - "lineWidth": 1 - }, - "unit": "bytes", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "color": { - "mode": "palette-classic" - }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "list", - "placement": "bottom", - "calcs": [] - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes", - "legendFormat": "{{ instance }}" - } - ] - }, - { - "id": 30, - "type": "row", - "title": "Training & CI Logs", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 35 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 31, - "type": "logs", - "title": "Training Job Logs", - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 36 - }, - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "options": { - "showTime": true, - "showLabels": true, - "showCommonLabels": false, - "wrapLogMessage": true, - "prettifyLogMessage": false, - "enableLogDetails": true, - "sortOrder": "Descending", - "dedupStrategy": "none" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "expr": "{namespace=\"foxhunt\", container=~\".*train.*\"}", - "maxLines": 200 - } - ] - }, - { - "id": 32, - "type": "logs", - "title": "CI / GitLab Runner Logs", - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 36 - }, - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "options": { - "showTime": true, - "showLabels": true, - "showCommonLabels": false, - "wrapLogMessage": true, - "prettifyLogMessage": false, - "enableLogDetails": true, - "sortOrder": "Descending", - "dedupStrategy": "none" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "expr": "{namespace=\"foxhunt\", app=~\".*runner.*|.*gitlab.*\"}", - "maxLines": 200 - } - ] - }, - { - "id": 40, - "type": "row", - "title": "Node Autoscale Events", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 46 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 41, - "type": "timeseries", - "title": "Node Count Over Time", - "description": "Tracks autoscaling of ci-compile and ci-training pools", - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 47 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "stepAfter", - "fillOpacity": 10, - "gradientMode": "none", - "spanNulls": false, - "showPoints": "auto", - "stacking": { - "mode": "none", - "group": "A" - }, - "axisPlacement": "auto", - "lineWidth": 2 - }, - "unit": "short", - "decimals": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "color": { - "mode": "palette-classic" - }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "min", - "max", - "last" - ] - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "count by (label_k8s_scaleway_com_pool_name) (kube_node_labels)", - "legendFormat": "{{ label_k8s_scaleway_com_pool_name }}" - } - ] - } - ] -} \ No newline at end of file diff --git a/infra/k8s/monitoring/dashboards/foxhunt-infrastructure.json b/infra/k8s/monitoring/dashboards/foxhunt-infrastructure.json index 042f8a499..027a97f16 100644 --- a/infra/k8s/monitoring/dashboards/foxhunt-infrastructure.json +++ b/infra/k8s/monitoring/dashboards/foxhunt-infrastructure.json @@ -10,11 +10,12 @@ ], "id": null, "uid": "foxhunt-infrastructure", - "title": "K8s Infrastructure", - "description": "Node health, per-node resources, PVC usage, scheduling, OOMKill events", + "title": "Infrastructure & Cluster", + "description": "K8s cluster health, node pools, pods, storage, and networking for the foxhunt platform", "tags": [ "foxhunt", - "production" + "infrastructure", + "k8s" ], "timezone": "browser", "schemaVersion": 39, @@ -22,7 +23,7 @@ "editable": true, "refresh": "30s", "time": { - "from": "now-1h", + "from": "now-3h", "to": "now" }, "fiscalYearStartMonth": 0, @@ -30,6 +31,21 @@ "weekStart": "", "templating": { "list": [ + { + "name": "interval", + "type": "interval", + "query": "1m,5m,15m,30m,1h", + "auto": true, + "auto_count": 30, + "auto_min": "1m", + "current": { + "selected": true, + "text": "auto", + "value": "$__auto_interval_interval" + }, + "hide": 0, + "refresh": 2 + }, { "name": "node", "type": "query", @@ -37,7 +53,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "query": "label_values(kube_node_info, node)", + "query": "label_values(node_uname_info, nodename)", "refresh": 2, "includeAll": true, "multi": true, @@ -47,1564 +63,365 @@ "text": "All", "value": "$__all" }, - "sort": 1 - }, - { - "name": "interval", - "type": "interval", - "query": "1m,5m,15m,30m,1h", - "current": { - "selected": true, - "text": "5m", - "value": "5m" - }, - "auto": false + "sort": 1, + "hide": 0 } ] }, "panels": [ { - "id": 1, + "id": 100, "type": "row", - "title": "Node Status", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, + "title": "Cluster Health", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "collapsed": false, "panels": [] }, { - "id": 2, - "type": "table", - "title": "Node Status by Pool", - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 1 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "id": 1, + "type": "stat", + "title": "Nodes Ready", + "gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "targets": [ + { + "expr": "sum(kube_node_status_condition{condition=\"Ready\",status=\"true\"})", + "legendFormat": "Ready Nodes", + "refId": "A", + "instant": true + } + ], "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ - { - "color": "green", - "value": null - } + { "color": "red", "value": null }, + { "color": "green", "value": 1 } ] }, - "mappings": [ - { - "type": "value", - "options": { - "1": { - "text": "Ready", - "color": "green" - }, - "0": { - "text": "NotReady", - "color": "red" - } - } - } - ], - "color": { - "mode": "thresholds" - } + "unit": "none", + "noValue": "0" }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Value" - }, - "properties": [ - { - "id": "custom.displayMode", - "value": "color-background-solid" - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - } - } - ] - } - ] + "overrides": [] }, "options": { - "showHeader": true, - "sortBy": [ - { - "displayName": "node", - "desc": false - } - ], - "footer": { - "show": false - } - }, + "reduceOptions": { "calcs": ["lastNotNull"] }, + "colorMode": "background", + "graphMode": "none", + "textMode": "auto", + "justifyMode": "auto" + } + }, + { + "id": 2, + "type": "stat", + "title": "Total Pods", + "gridPos": { "h": 4, "w": 4, "x": 4, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "targets": [ { + "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\",phase=\"Running\"})", + "legendFormat": "Running", "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "kube_node_status_condition{condition=\"Ready\", status=\"true\", node=~\"$node\"}", - "legendFormat": "{{ node }}", - "instant": true, - "format": "table" + "instant": true } - ] + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "green", "value": 1 } + ] + }, + "unit": "none", + "noValue": "0" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"] }, + "colorMode": "background", + "graphMode": "none", + "textMode": "auto", + "justifyMode": "auto" + } + }, + { + "id": 3, + "type": "stat", + "title": "Pending Pods", + "gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "targets": [ + { + "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\",phase=\"Pending\"})", + "legendFormat": "Pending", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "red", "value": 1 } + ] + }, + "unit": "none", + "noValue": "0" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"] }, + "colorMode": "background", + "graphMode": "none", + "textMode": "auto", + "justifyMode": "auto" + } + }, + { + "id": 4, + "type": "stat", + "title": "Restarts (1h)", + "gridPos": { "h": 4, "w": 4, "x": 12, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "targets": [ + { + "expr": "sum(increase(kube_pod_container_status_restarts_total{namespace=\"foxhunt\"}[1h]))", + "legendFormat": "Restarts", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "orange", "value": 3 }, + { "color": "red", "value": 5 } + ] + }, + "unit": "none", + "noValue": "0", + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"] }, + "colorMode": "background", + "graphMode": "none", + "textMode": "auto", + "justifyMode": "auto" + } }, { "id": 5, "type": "stat", - "title": "services Pool", - "description": "DEV1-M, 2 fixed", - "gridPos": { - "h": 3, - "w": 4, - "x": 0, - "y": 8 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "title": "CPU Request %", + "gridPos": { "h": 4, "w": 4, "x": 16, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "targets": [ + { + "expr": "sum(kube_pod_container_resource_requests{namespace=\"foxhunt\",resource=\"cpu\"}) / sum(kube_node_status_allocatable{resource=\"cpu\"}) * 100", + "legendFormat": "CPU Req %", + "refId": "A", + "instant": true + } + ], "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "yellow", - "value": 1 - }, - { - "color": "green", - "value": 2 - } + { "color": "green", "value": null }, + { "color": "orange", "value": 70 }, + { "color": "red", "value": 90 } ] }, - "mappings": [], - "color": { - "mode": "thresholds" - } + "unit": "percent", + "noValue": "0", + "decimals": 1 }, "overrides": [] }, "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "value_and_name", + "reduceOptions": { "calcs": ["lastNotNull"] }, "colorMode": "background", "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "count(kube_node_labels{label_k8s_scaleway_com_pool_name=\"services\"} * on(node) kube_node_status_condition{condition=\"Ready\", status=\"true\"}) or vector(0)", - "legendFormat": "services", - "instant": true - } - ] + "textMode": "auto", + "justifyMode": "auto" + } }, { "id": 6, "type": "stat", - "title": "gitlab Pool", - "description": "GP1-XS, 1 fixed", - "gridPos": { - "h": 3, - "w": 4, - "x": 4, - "y": 8 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - }, - "mappings": [], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "value_and_name", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, + "title": "Memory Request %", + "gridPos": { "h": 4, "w": 4, "x": 20, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "targets": [ { + "expr": "sum(kube_pod_container_resource_requests{namespace=\"foxhunt\",resource=\"memory\"}) / sum(kube_node_status_allocatable{resource=\"memory\"}) * 100", + "legendFormat": "Mem Req %", "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "count(kube_node_labels{label_k8s_scaleway_com_pool_name=\"gitlab\"} * on(node) kube_node_status_condition{condition=\"Ready\", status=\"true\"}) or vector(0)", - "legendFormat": "gitlab", "instant": true } - ] - }, - { - "id": 7, - "type": "stat", - "title": "ci-compile-cpu Pool", - "description": "POP2-32C-128G, 0-1 autoscale", - "gridPos": { - "h": 3, - "w": 4, - "x": 8, - "y": 8 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + ], "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ - { - "color": "blue", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - }, - "mappings": [], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "value_and_name", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "count(kube_node_labels{label_k8s_scaleway_com_pool_name=\"ci-compile-cpu\"}) or vector(0)", - "legendFormat": "ci-compile-cpu", - "instant": true - } - ] - }, - { - "id": 8, - "type": "stat", - "title": "ci-compile Pool (GPU)", - "description": "L4-1-24G, 0-1 autoscale", - "gridPos": { - "h": 3, - "w": 4, - "x": 12, - "y": 8 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - }, - "mappings": [], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "value_and_name", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "count(kube_node_labels{label_k8s_scaleway_com_pool_name=\"ci-compile\"}) or vector(0)", - "legendFormat": "ci-compile", - "instant": true - } - ] - }, - { - "id": 9, - "type": "stat", - "title": "ci-training Pool (GPU)", - "description": "L40S-1-48G, 0-1 autoscale", - "gridPos": { - "h": 3, - "w": 4, - "x": 16, - "y": 8 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - }, - "mappings": [], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "value_and_name", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "count(kube_node_labels{label_k8s_scaleway_com_pool_name=\"ci-training\"}) or vector(0)", - "legendFormat": "ci-training", - "instant": true - } - ] - }, - { - "id": 10, - "type": "stat", - "title": "Total Pods", - "gridPos": { - "h": 3, - "w": 4, - "x": 20, - "y": 8 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - } - ] - }, - "mappings": [], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kubelet_running_pods) or count(kube_pod_info)", - "instant": true - } - ] - }, - { - "id": 15, - "type": "row", - "title": "Per-Node Resources", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 11 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 16, - "type": "timeseries", - "title": "Node CPU Usage", - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 12 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 15, - "gradientMode": "scheme", - "spanNulls": false, - "showPoints": "never", - "stacking": { - "mode": "none", - "group": "A" - }, - "axisPlacement": "auto", - "lineWidth": 1 - }, - "unit": "percentunit", - "min": 0, - "max": 1, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "color": { - "mode": "palette-classic" - }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "max" - ] - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[$interval]))", - "legendFormat": "{{ instance }}" - } - ] - }, - { - "id": 17, - "type": "timeseries", - "title": "Node Memory Usage", - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 12 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 15, - "gradientMode": "scheme", - "spanNulls": false, - "showPoints": "never", - "stacking": { - "mode": "none", - "group": "A" - }, - "axisPlacement": "auto", - "lineWidth": 1 - }, - "unit": "bytes", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "color": { - "mode": "palette-classic" - }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "max" - ] - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes", - "legendFormat": "{{ instance }} used" - }, - { - "refId": "B", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_MemTotal_bytes", - "legendFormat": "{{ instance }} total" - } - ] - }, - { - "id": 18, - "type": "timeseries", - "title": "Node Filesystem Usage", - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 20 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 15, - "gradientMode": "scheme", - "spanNulls": false, - "showPoints": "never", - "stacking": { - "mode": "none", - "group": "A" - }, - "axisPlacement": "auto", - "lineWidth": 1 - }, - "unit": "percentunit", - "min": 0, - "max": 1, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.7 - }, - { - "color": "red", - "value": 0.9 - } - ] - }, - "color": { - "mode": "palette-classic" - }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "max" - ] - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "1 - (node_filesystem_avail_bytes{mountpoint=\"/\", fstype!=\"tmpfs\"} / node_filesystem_size_bytes{mountpoint=\"/\", fstype!=\"tmpfs\"})", - "legendFormat": "{{ instance }}" - } - ] - }, - { - "id": 19, - "type": "bargauge", - "title": "Pods per Node", - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 20 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 80 - }, - { - "color": "red", - "value": 100 - } - ] - }, - "mappings": [], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "horizontal", - "displayMode": "gradient", - "showUnfilled": true, - "minVizWidth": 8, - "minVizHeight": 16, - "valueMode": "color" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "count by (node) (kube_pod_info{node=~\"$node\"})", - "legendFormat": "{{ node }}", - "instant": true - } - ] - }, - { - "id": 25, - "type": "row", - "title": "Network & PVC", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 28 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 26, - "type": "timeseries", - "title": "Network I/O per Node", - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 29 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 10, - "gradientMode": "none", - "spanNulls": false, - "showPoints": "never", - "stacking": { - "mode": "none", - "group": "A" - }, - "axisPlacement": "auto", - "lineWidth": 1 - }, - "unit": "Bps", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "color": { - "mode": "palette-classic" - }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "max" - ] - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum by (instance) (rate(node_network_receive_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$interval]))", - "legendFormat": "{{ instance }} RX" - }, - { - "refId": "B", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum by (instance) (rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|docker.*|flannel.*|cali.*|cbr.*\"}[$interval]))", - "legendFormat": "{{ instance }} TX" - } - ] - }, - { - "id": 27, - "type": "bargauge", - "title": "PVC Usage", - "description": "Persistent Volume Claim usage percentage. Requires kubelet_volume_stats metrics.", - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 29 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 70 - }, - { - "color": "red", - "value": 90 - } + { "color": "green", "value": null }, + { "color": "orange", "value": 70 }, + { "color": "red", "value": 90 } ] }, "unit": "percent", + "noValue": "0", + "decimals": 1 + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"] }, + "colorMode": "background", + "graphMode": "none", + "textMode": "auto", + "justifyMode": "auto" + } + }, + { + "id": 101, + "type": "row", + "title": "Node CPU & Memory", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, + "collapsed": false, + "panels": [] + }, + { + "id": 7, + "type": "timeseries", + "title": "CPU Usage by Node", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "targets": [ + { + "expr": "1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[$interval]))", + "legendFormat": "{{ instance }}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "orange", "value": 0.7 }, + { "color": "red", "value": 0.9 } + ] + }, + "unit": "percentunit", "min": 0, - "max": 100, - "mappings": [], - "color": { - "mode": "thresholds" + "max": 1, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 15, + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": true, + "axisCenteredZero": false, + "axisColorMode": "text", + "scaleDistribution": { "type": "linear" }, + "thresholdsStyle": { "mode": "off" } } }, "overrides": [] }, "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "horizontal", - "displayMode": "gradient", - "showUnfilled": true, - "minVizWidth": 8, - "minVizHeight": 16, - "valueMode": "color" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "(1 - (kubelet_volume_stats_available_bytes{namespace=\"foxhunt\"} / kubelet_volume_stats_capacity_bytes{namespace=\"foxhunt\"})) * 100", - "legendFormat": "{{ persistentvolumeclaim }}", - "instant": true - } - ] + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } + } }, { - "id": 30, - "type": "row", - "title": "Pod Scheduling & OOMKills", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 37 + "id": 8, + "type": "timeseries", + "title": "Memory Usage by Node", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "targets": [ + { + "expr": "1 - (node_memory_AvailableBytes / node_memory_MemoryTotal_bytes)", + "legendFormat": "{{ instance }}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "orange", "value": 0.7 }, + { "color": "red", "value": 0.9 } + ] + }, + "unit": "percentunit", + "min": 0, + "max": 1, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 15, + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": true, + "axisCenteredZero": false, + "axisColorMode": "text", + "scaleDistribution": { "type": "linear" }, + "thresholdsStyle": { "mode": "off" } + } + }, + "overrides": [] }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } + } + }, + { + "id": 102, + "type": "row", + "title": "Pod Resources", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 }, "collapsed": false, "panels": [] }, { - "id": 31, + "id": 9, "type": "timeseries", - "title": "Scheduling Latency (p99)", - "description": "Time to schedule pods. From scheduler_scheduling_algorithm_duration_seconds or scheduler_e2e_scheduling_duration_seconds.", - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 38 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 10, - "gradientMode": "none", - "spanNulls": false, - "showPoints": "never", - "stacking": { - "mode": "none", - "group": "A" - }, - "axisPlacement": "auto", - "lineWidth": 2 - }, - "unit": "s", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "color": { - "mode": "palette-classic" - }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "list", - "placement": "bottom", - "calcs": [] - } - }, + "title": "Pod CPU Usage", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 15 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "targets": [ { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "histogram_quantile(0.99, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket[$interval])) by (le))", - "legendFormat": "p99 e2e scheduling" - }, - { - "refId": "B", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "histogram_quantile(0.50, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket[$interval])) by (le))", - "legendFormat": "p50 e2e scheduling" + "expr": "topk(10, sum by (pod) (rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\",container!=\"POD\",container!=\"\"}[$interval])))", + "legendFormat": "{{ pod }}", + "refId": "A" } - ] - }, - { - "id": 32, - "type": "timeseries", - "title": "OOMKill Events", - "description": "Container OOMKill events. Each spike indicates a container was killed due to memory pressure.", - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 38 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "bars", - "lineInterpolation": "linear", - "fillOpacity": 80, - "gradientMode": "none", - "spanNulls": false, - "showPoints": "never", - "stacking": { - "mode": "normal", - "group": "A" - }, - "axisPlacement": "auto", - "lineWidth": 1 - }, - "unit": "short", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "color": { - "mode": "palette-classic" - }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "sum" - ] - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "increase(kube_pod_container_status_last_terminated_reason{reason=\"OOMKilled\", namespace=\"foxhunt\"}[30m]) > 0", - "legendFormat": "{{ pod }} ({{ container }})" - } - ] - }, - { - "id": 35, - "type": "timeseries", - "title": "Disk I/O per Node", - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 45 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 10, - "gradientMode": "none", - "spanNulls": false, - "showPoints": "never", - "stacking": { - "mode": "none", - "group": "A" - }, - "axisPlacement": "auto", - "lineWidth": 1 - }, - "unit": "Bps", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "color": { - "mode": "palette-classic" - }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "max" - ] - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum by (instance) (rate(node_disk_read_bytes_total[$interval]))", - "legendFormat": "{{ instance }} read" - }, - { - "refId": "B", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum by (instance) (rate(node_disk_written_bytes_total[$interval]))", - "legendFormat": "{{ instance }} write" - } - ] - }, - { - "id": 40, - "type": "row", - "title": "Platform Services", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 52 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 41, - "type": "stat", - "title": "GitLab", - "description": "Ready pods: webservice, sidekiq, gitaly, shell, registry, toolbox", - "gridPos": { - "h": 3, - "w": 4, - "x": 0, - "y": 53 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "red", "value": null }, - { "color": "yellow", "value": 4 }, - { "color": "green", "value": 6 } - ] - }, - "mappings": [], - "color": { "mode": "thresholds" } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, - "orientation": "auto", - "textMode": "value_and_name", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "count(kube_pod_status_ready{namespace=\"foxhunt\", condition=\"true\"} * on(pod) group_left() kube_pod_labels{namespace=\"foxhunt\", label_release=\"gitlab\"}) or vector(0)", - "legendFormat": "GitLab", - "instant": true - } - ] - }, - { - "id": 42, - "type": "stat", - "title": "CI Runners", - "description": "GitLab CI runner manager + active job pods", - "gridPos": { - "h": 3, - "w": 4, - "x": 4, - "y": 53 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "blue", "value": null }, - { "color": "green", "value": 1 } - ] - }, - "mappings": [], - "color": { "mode": "thresholds" } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, - "orientation": "auto", - "textMode": "value_and_name", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "count(kube_pod_status_ready{namespace=\"foxhunt\", condition=\"true\"} * on(pod) group_left() kube_pod_info{namespace=\"foxhunt\", pod=~\"gitlab-runner.*|runner-.*\"}) or vector(0)", - "legendFormat": "Runners", - "instant": true - } - ] - }, - { - "id": 43, - "type": "stat", - "title": "Monitoring", - "description": "Grafana, Prometheus, Loki, Tempo, kube-state-metrics", - "gridPos": { - "h": 3, - "w": 4, - "x": 8, - "y": 53 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "red", "value": null }, - { "color": "yellow", "value": 3 }, - { "color": "green", "value": 5 } - ] - }, - "mappings": [], - "color": { "mode": "thresholds" } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, - "orientation": "auto", - "textMode": "value_and_name", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "count(kube_pod_status_ready{namespace=\"foxhunt\", condition=\"true\"} * on(pod) group_left() kube_pod_info{namespace=\"foxhunt\", pod=~\"grafana.*|gitlab-prometheus.*|loki.*|tempo.*|kube-state-metrics.*\"}) or vector(0)", - "legendFormat": "Monitoring", - "instant": true - } - ] - }, - { - "id": 44, - "type": "stat", - "title": "Collectors", - "description": "Promtail + node-exporter (DaemonSets, scales with nodes)", - "gridPos": { - "h": 3, - "w": 4, - "x": 12, - "y": 53 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "red", "value": null }, - { "color": "green", "value": 4 } - ] - }, - "mappings": [], - "color": { "mode": "thresholds" } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, - "orientation": "auto", - "textMode": "value_and_name", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "count(kube_pod_status_ready{namespace=\"foxhunt\", condition=\"true\"} * on(pod) group_left() kube_pod_info{namespace=\"foxhunt\", pod=~\"promtail.*|node-exporter.*\"}) or vector(0)", - "legendFormat": "Collectors", - "instant": true - } - ] - }, - { - "id": 45, - "type": "stat", - "title": "Tailscale", - "description": "Tailscale proxy (VPN ingress)", - "gridPos": { - "h": 3, - "w": 4, - "x": 16, - "y": 53 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "red", "value": null }, - { "color": "green", "value": 1 } - ] - }, - "mappings": [], - "color": { "mode": "thresholds" } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, - "orientation": "auto", - "textMode": "value_and_name", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "count(kube_pod_status_ready{namespace=\"foxhunt\", condition=\"true\"} * on(pod) group_left() kube_pod_info{namespace=\"foxhunt\", pod=~\"tailscale.*\"}) or vector(0)", - "legendFormat": "Tailscale", - "instant": true - } - ] - }, - { - "id": 46, - "type": "stat", - "title": "Platform Total", - "description": "All non-foxhunt pods (Running & Ready)", - "gridPos": { - "h": 3, - "w": 4, - "x": 20, - "y": 53 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "blue", "value": null } - ] - }, - "mappings": [], - "color": { "mode": "thresholds" } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "count(kube_pod_status_ready{namespace=\"foxhunt\", condition=\"true\"} * on(pod) group_left() kube_pod_info{namespace=\"foxhunt\", pod=~\"gitlab-.*|grafana.*|loki.*|tempo.*|promtail.*|node-exporter.*|kube-state-metrics.*|tailscale.*|runner-.*\"}) or vector(0)", - "instant": true - } - ] - }, - { - "id": 47, - "type": "table", - "title": "Platform Pod Status", - "description": "All non-foxhunt service pods: GitLab, monitoring, CI runners, networking", - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 56 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + ], "fieldConfig": { "defaults": { "thresholds": { @@ -1613,32 +430,165 @@ { "color": "green", "value": null } ] }, - "mappings": [ - { - "type": "value", - "options": { - "1": { "text": "Ready", "color": "green" }, - "0": { "text": "Not Ready", "color": "red" } - } - } - ], - "color": { "mode": "thresholds" } + "unit": "short", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 10, + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": true, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "CPU cores", + "scaleDistribution": { "type": "linear" }, + "thresholdsStyle": { "mode": "off" } + } + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } + } + }, + { + "id": 10, + "type": "timeseries", + "title": "Pod Memory Usage", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 15 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "targets": [ + { + "expr": "topk(10, sum by (pod) (container_memory_working_set_bytes{namespace=\"foxhunt\",container!=\"POD\",container!=\"\"}))", + "legendFormat": "{{ pod }}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "bytes", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 10, + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": true, + "axisCenteredZero": false, + "axisColorMode": "text", + "scaleDistribution": { "type": "linear" }, + "thresholdsStyle": { "mode": "off" } + } + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } + } + }, + { + "id": 103, + "type": "row", + "title": "Storage", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 }, + "collapsed": false, + "panels": [] + }, + { + "id": 11, + "type": "table", + "title": "PVC Usage", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "targets": [ + { + "expr": "kubelet_volume_stats_used_bytes{namespace=\"foxhunt\"}", + "legendFormat": "", + "refId": "A", + "instant": true, + "format": "table" + }, + { + "expr": "kubelet_volume_stats_capacity_bytes{namespace=\"foxhunt\"}", + "legendFormat": "", + "refId": "B", + "instant": true, + "format": "table" + }, + { + "expr": "kubelet_volume_stats_used_bytes{namespace=\"foxhunt\"} / kubelet_volume_stats_capacity_bytes{namespace=\"foxhunt\"} * 100", + "legendFormat": "", + "refId": "C", + "instant": true, + "format": "table" + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "orange", "value": 70 }, + { "color": "red", "value": 90 } + ] + }, + "custom": { + "align": "auto", + "cellOptions": { "type": "auto" }, + "filterable": true + } }, "overrides": [ { - "matcher": { "id": "byName", "options": "Value" }, + "matcher": { "id": "byName", "options": "Value #A" }, "properties": [ - { "id": "custom.displayMode", "value": "color-background-solid" }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { "color": "red", "value": null }, - { "color": "green", "value": 1 } - ] - } - } + { "id": "displayName", "value": "Used" }, + { "id": "unit", "value": "bytes" } + ] + }, + { + "matcher": { "id": "byName", "options": "Value #B" }, + "properties": [ + { "id": "displayName", "value": "Total" }, + { "id": "unit", "value": "bytes" } + ] + }, + { + "matcher": { "id": "byName", "options": "Value #C" }, + "properties": [ + { "id": "displayName", "value": "Used %" }, + { "id": "unit", "value": "percent" }, + { "id": "custom.cellOptions", "value": { "mode": "gradient", "type": "gauge" } }, + { "id": "thresholds", "value": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "orange", "value": 70 }, { "color": "red", "value": 90 }] } } + ] + }, + { + "matcher": { "id": "byName", "options": "persistentvolumeclaim" }, + "properties": [ + { "id": "displayName", "value": "PVC" } + ] + }, + { + "matcher": { "id": "byName", "options": "Time" }, + "properties": [ + { "id": "custom.hidden", "value": true } + ] + }, + { + "matcher": { "id": "byName", "options": "__name__" }, + "properties": [ + { "id": "custom.hidden", "value": true } ] } ] @@ -1646,185 +596,566 @@ "options": { "showHeader": true, "sortBy": [ - { "displayName": "pod", "desc": false } + { "displayName": "Used %", "desc": true } ], "footer": { "show": false } }, + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "__name__": true, + "endpoint": true, + "instance": true, + "job": true, + "metrics_path": true, + "node": true, + "service": true + }, + "indexByName": {}, + "renameByName": { + "persistentvolumeclaim": "PVC", + "namespace": "Namespace" + } + } + } + ] + }, + { + "id": 12, + "type": "timeseries", + "title": "Disk I/O", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "targets": [ { - "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "kube_pod_status_ready{namespace=\"foxhunt\", condition=\"true\"} * on(pod) group_left() kube_pod_info{namespace=\"foxhunt\", pod=~\"gitlab-.*|grafana.*|loki.*|tempo.*|promtail.*|node-exporter.*|kube-state-metrics.*|tailscale.*|runner-.*\"}", + "expr": "rate(node_disk_read_bytes_total{device=~\"sd.*|nvme.*\"}[$interval])", + "legendFormat": "{{ instance }} {{ device }} read", + "refId": "A" + }, + { + "expr": "rate(node_disk_written_bytes_total{device=~\"sd.*|nvme.*\"}[$interval])", + "legendFormat": "{{ instance }} {{ device }} write", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "Bps", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 10, + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": true, + "axisCenteredZero": false, + "axisColorMode": "text", + "scaleDistribution": { "type": "linear" }, + "thresholdsStyle": { "mode": "off" } + } + }, + "overrides": [ + { + "matcher": { "id": "byRegexp", "options": ".*write.*" }, + "properties": [ + { "id": "custom.transform", "value": "negative-Y" } + ] + } + ] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } + } + }, + { + "id": 104, + "type": "row", + "title": "Network", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 32 }, + "collapsed": false, + "panels": [] + }, + { + "id": 13, + "type": "timeseries", + "title": "Network Receive", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 33 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "targets": [ + { + "expr": "sum by (instance) (rate(node_network_receive_bytes_total{device!=\"lo\"}[$interval]))", + "legendFormat": "{{ instance }}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "Bps", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 15, + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": true, + "axisCenteredZero": false, + "axisColorMode": "text", + "scaleDistribution": { "type": "linear" }, + "thresholdsStyle": { "mode": "off" } + } + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } + } + }, + { + "id": 14, + "type": "timeseries", + "title": "Network Transmit", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 33 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "targets": [ + { + "expr": "sum by (instance) (rate(node_network_transmit_bytes_total{device!=\"lo\"}[$interval]))", + "legendFormat": "{{ instance }}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "Bps", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 15, + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": true, + "axisCenteredZero": false, + "axisColorMode": "text", + "scaleDistribution": { "type": "linear" }, + "thresholdsStyle": { "mode": "off" } + } + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } + } + }, + { + "id": 105, + "type": "row", + "title": "Pod Status Detail", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 41 }, + "collapsed": false, + "panels": [] + }, + { + "id": 15, + "type": "timeseries", + "title": "Pod Restarts", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 42 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "targets": [ + { + "expr": "increase(kube_pod_container_status_restarts_total{namespace=\"foxhunt\"}[$interval])", "legendFormat": "{{ pod }}", - "instant": true, - "format": "table" + "refId": "A" } - ] - }, - { - "id": 48, - "type": "timeseries", - "title": "Platform CPU Usage", - "description": "CPU usage by platform service pods (GitLab, monitoring, runners)", - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 66 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + ], "fieldConfig": { "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 15, - "gradientMode": "scheme", - "spanNulls": false, - "showPoints": "never", - "stacking": { "mode": "none", "group": "A" }, - "axisPlacement": "auto", - "lineWidth": 1 - }, - "unit": "short", "thresholds": { "mode": "absolute", - "steps": [{ "color": "green", "value": null }] + "steps": [ + { "color": "green", "value": null }, + { "color": "orange", "value": 1 }, + { "color": "red", "value": 5 } + ] }, - "color": { "mode": "palette-classic" }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": ["mean", "max"] - } - }, - "targets": [ - { - "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"gitlab-webservice.*|gitlab-sidekiq.*|gitlab-gitaly.*|gitlab-registry.*|grafana.*|loki.*|tempo.*\"}[$interval]))", - "legendFormat": "{{ pod }}" - } - ] - }, - { - "id": 49, - "type": "timeseries", - "title": "Platform Memory Usage", - "description": "Memory working set by platform service pods", - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 66 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 15, - "gradientMode": "scheme", - "spanNulls": false, - "showPoints": "never", - "stacking": { "mode": "none", "group": "A" }, - "axisPlacement": "auto", - "lineWidth": 1 - }, - "unit": "bytes", - "thresholds": { - "mode": "absolute", - "steps": [{ "color": "green", "value": null }] - }, - "color": { "mode": "palette-classic" }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": ["mean", "max"] - } - }, - "targets": [ - { - "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "sum by (pod) (container_memory_working_set_bytes{namespace=\"foxhunt\", pod=~\"gitlab-webservice.*|gitlab-sidekiq.*|gitlab-gitaly.*|gitlab-registry.*|grafana.*|loki.*|tempo.*\"})", - "legendFormat": "{{ pod }}" - } - ] - }, - { - "id": 50, - "type": "timeseries", - "title": "Container Restarts (Platform)", - "description": "Restart rate for platform pods — spikes indicate instability", - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 74 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { + "unit": "none", + "decimals": 0, "custom": { "drawStyle": "bars", "lineInterpolation": "linear", "fillOpacity": 80, - "gradientMode": "none", - "spanNulls": false, + "lineWidth": 1, + "pointSize": 5, "showPoints": "never", - "stacking": { "mode": "normal", "group": "A" }, - "axisPlacement": "auto", - "lineWidth": 1 - }, - "unit": "short", - "thresholds": { - "mode": "absolute", - "steps": [{ "color": "green", "value": null }] - }, - "color": { "mode": "palette-classic" }, - "mappings": [] + "spanNulls": true, + "axisCenteredZero": false, + "axisColorMode": "text", + "scaleDistribution": { "type": "linear" }, + "thresholdsStyle": { "mode": "off" }, + "stacking": { "mode": "normal", "group": "A" } + } }, "overrides": [] }, "options": { "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": ["sum"] - } - }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["sum"] } + } + }, + { + "id": 16, + "type": "table", + "title": "Container Status", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 42 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "targets": [ { + "expr": "kube_pod_container_status_running{namespace=\"foxhunt\"}", + "legendFormat": "", "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "increase(kube_pod_container_status_restarts_total{namespace=\"foxhunt\", pod=~\"gitlab-.*|grafana.*|loki.*|tempo.*|promtail.*|kube-state-metrics.*|tailscale.*|runner-.*\"}[30m]) > 0", - "legendFormat": "{{ pod }} ({{ container }})" + "instant": true, + "format": "table" + }, + { + "expr": "kube_pod_container_status_waiting{namespace=\"foxhunt\"}", + "legendFormat": "", + "refId": "B", + "instant": true, + "format": "table" + }, + { + "expr": "kube_pod_container_status_terminated{namespace=\"foxhunt\"}", + "legendFormat": "", + "refId": "C", + "instant": true, + "format": "table" + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "custom": { + "align": "auto", + "cellOptions": { "type": "auto" }, + "filterable": true + } + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "Value #A" }, + "properties": [ + { "id": "displayName", "value": "Running" } + ] + }, + { + "matcher": { "id": "byName", "options": "Value #B" }, + "properties": [ + { "id": "displayName", "value": "Waiting" }, + { "id": "custom.cellOptions", "value": { "mode": "basic", "type": "color-background" } }, + { "id": "thresholds", "value": { "mode": "absolute", "steps": [{ "color": "transparent", "value": null }, { "color": "orange", "value": 1 }] } } + ] + }, + { + "matcher": { "id": "byName", "options": "Value #C" }, + "properties": [ + { "id": "displayName", "value": "Terminated" }, + { "id": "custom.cellOptions", "value": { "mode": "basic", "type": "color-background" } }, + { "id": "thresholds", "value": { "mode": "absolute", "steps": [{ "color": "transparent", "value": null }, { "color": "red", "value": 1 }] } } + ] + }, + { + "matcher": { "id": "byName", "options": "Time" }, + "properties": [ + { "id": "custom.hidden", "value": true } + ] + }, + { + "matcher": { "id": "byName", "options": "__name__" }, + "properties": [ + { "id": "custom.hidden", "value": true } + ] + } + ] + }, + "options": { + "showHeader": true, + "sortBy": [ + { "displayName": "Waiting", "desc": true } + ], + "footer": { "show": false } + }, + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "__name__": true, + "endpoint": true, + "instance": true, + "job": true, + "metrics_path": true, + "node": true, + "service": true, + "uid": true + }, + "indexByName": {}, + "renameByName": { + "pod": "Pod", + "container": "Container", + "namespace": "Namespace" + } + } } ] + }, + { + "id": 106, + "type": "row", + "title": "kube-state-metrics", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 50 }, + "collapsed": false, + "panels": [] + }, + { + "id": 17, + "type": "table", + "title": "Deployment Replicas", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 51 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "targets": [ + { + "expr": "kube_deployment_spec_replicas{namespace=\"foxhunt\"}", + "legendFormat": "", + "refId": "A", + "instant": true, + "format": "table" + }, + { + "expr": "kube_deployment_status_replicas_available{namespace=\"foxhunt\"}", + "legendFormat": "", + "refId": "B", + "instant": true, + "format": "table" + }, + { + "expr": "kube_deployment_status_replicas_unavailable{namespace=\"foxhunt\"}", + "legendFormat": "", + "refId": "C", + "instant": true, + "format": "table" + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "custom": { + "align": "auto", + "cellOptions": { "type": "auto" }, + "filterable": true + } + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "Value #A" }, + "properties": [ + { "id": "displayName", "value": "Desired" } + ] + }, + { + "matcher": { "id": "byName", "options": "Value #B" }, + "properties": [ + { "id": "displayName", "value": "Available" } + ] + }, + { + "matcher": { "id": "byName", "options": "Value #C" }, + "properties": [ + { "id": "displayName", "value": "Unavailable" }, + { "id": "custom.cellOptions", "value": { "mode": "basic", "type": "color-background" } }, + { "id": "thresholds", "value": { "mode": "absolute", "steps": [{ "color": "transparent", "value": null }, { "color": "red", "value": 1 }] } } + ] + }, + { + "matcher": { "id": "byName", "options": "Time" }, + "properties": [ + { "id": "custom.hidden", "value": true } + ] + }, + { + "matcher": { "id": "byName", "options": "__name__" }, + "properties": [ + { "id": "custom.hidden", "value": true } + ] + } + ] + }, + "options": { + "showHeader": true, + "sortBy": [ + { "displayName": "Unavailable", "desc": true } + ], + "footer": { "show": false } + }, + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "__name__": true, + "endpoint": true, + "instance": true, + "job": true, + "metrics_path": true, + "node": true, + "service": true, + "uid": true + }, + "indexByName": {}, + "renameByName": { + "deployment": "Deployment", + "namespace": "Namespace" + } + } + } + ] + }, + { + "id": 18, + "type": "timeseries", + "title": "HPA Status", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 51 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "targets": [ + { + "expr": "kube_horizontalpodautoscaler_status_current_replicas{namespace=\"foxhunt\"}", + "legendFormat": "{{ horizontalpodautoscaler }} current", + "refId": "A" + }, + { + "expr": "kube_horizontalpodautoscaler_spec_min_replicas{namespace=\"foxhunt\"}", + "legendFormat": "{{ horizontalpodautoscaler }} min", + "refId": "B" + }, + { + "expr": "kube_horizontalpodautoscaler_spec_max_replicas{namespace=\"foxhunt\"}", + "legendFormat": "{{ horizontalpodautoscaler }} max", + "refId": "C" + }, + { + "expr": "kube_horizontalpodautoscaler_status_desired_replicas{namespace=\"foxhunt\"}", + "legendFormat": "{{ horizontalpodautoscaler }} desired", + "refId": "D" + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "none", + "decimals": 0, + "custom": { + "drawStyle": "line", + "lineInterpolation": "stepAfter", + "fillOpacity": 0, + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": true, + "axisCenteredZero": false, + "axisColorMode": "text", + "scaleDistribution": { "type": "linear" }, + "thresholdsStyle": { "mode": "off" } + } + }, + "overrides": [ + { + "matcher": { "id": "byRegexp", "options": ".*min.*" }, + "properties": [ + { "id": "custom.lineStyle", "value": { "fill": "dash", "dash": [10, 10] } }, + { "id": "color", "value": { "mode": "fixed", "fixedColor": "orange" } } + ] + }, + { + "matcher": { "id": "byRegexp", "options": ".*max.*" }, + "properties": [ + { "id": "custom.lineStyle", "value": { "fill": "dash", "dash": [10, 10] } }, + { "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } } + ] + } + ] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull"] } + } } - ] -} + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { "type": "grafana", "uid": "-- Grafana --" }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "links": [], + "style": "dark" +} \ No newline at end of file diff --git a/infra/k8s/monitoring/dashboards/foxhunt-logs.json b/infra/k8s/monitoring/dashboards/foxhunt-logs.json deleted file mode 100644 index e6ac2d576..000000000 --- a/infra/k8s/monitoring/dashboards/foxhunt-logs.json +++ /dev/null @@ -1,717 +0,0 @@ -{ - "__inputs": [ - { - "name": "DS_LOKI", - "label": "Loki", - "description": "", - "type": "datasource", - "pluginId": "loki" - }, - { - "name": "DS_TEMPO", - "label": "Tempo", - "description": "", - "type": "datasource", - "pluginId": "tempo" - } - ], - "id": null, - "uid": "foxhunt-logs", - "title": "Log Explorer", - "description": "Log volume, error rates, top errors, live log stream, trace search", - "tags": [ - "foxhunt", - "production" - ], - "timezone": "browser", - "schemaVersion": 39, - "version": 1, - "editable": true, - "refresh": "30s", - "time": { - "from": "now-1h", - "to": "now" - }, - "fiscalYearStartMonth": 0, - "liveNow": false, - "weekStart": "", - "templating": { - "list": [ - { - "name": "service", - "type": "custom", - "query": "api,trading-service,ml-training-service,backtesting-service,trading-agent-service,broker-gateway,data-acquisition-service", - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "multi": true, - "includeAll": true, - "allValue": ".*", - "sort": 0 - }, - { - "name": "search", - "type": "textbox", - "query": "", - "current": { - "selected": false, - "text": "", - "value": "" - }, - "label": "Log Search" - }, - { - "name": "interval", - "type": "interval", - "query": "1m,5m,15m,30m,1h", - "current": { - "selected": true, - "text": "5m", - "value": "5m" - }, - "auto": false - } - ] - }, - "panels": [ - { - "id": 1, - "type": "row", - "title": "Log Volume Overview", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 2, - "type": "timeseries", - "title": "Log Volume by Service", - "gridPos": { - "h": 8, - "w": 16, - "x": 0, - "y": 1 - }, - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "bars", - "lineInterpolation": "linear", - "fillOpacity": 60, - "gradientMode": "none", - "spanNulls": false, - "showPoints": "never", - "stacking": { - "mode": "normal", - "group": "A" - }, - "axisPlacement": "auto", - "lineWidth": 0 - }, - "unit": "short", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "color": { - "mode": "palette-classic" - }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "right", - "calcs": [ - "sum", - "mean" - ] - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "expr": "sum by (app) (count_over_time({namespace=\"foxhunt\", app=~\"$service\"} [$interval]))", - "legendFormat": "{{ app }}" - } - ] - }, - { - "id": 3, - "type": "stat", - "title": "Total Logs (1h)", - "gridPos": { - "h": 4, - "w": 4, - "x": 16, - "y": 1 - }, - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - } - ] - }, - "mappings": [], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "sum" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "expr": "sum(count_over_time({namespace=\"foxhunt\", app=~\"$service\"} [1h]))", - "instant": true - } - ] - }, - { - "id": 4, - "type": "stat", - "title": "Errors (1h)", - "gridPos": { - "h": 4, - "w": 4, - "x": 20, - "y": 1 - }, - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 10 - }, - { - "color": "red", - "value": 50 - } - ] - }, - "mappings": [], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "sum" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "area", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "expr": "sum(count_over_time({namespace=\"foxhunt\", app=~\"$service\", container!~\"loki|grafana|grafana-sc-dashboard|prometheus-server\"} |= \"ERROR\" [1h]))", - "instant": true - } - ] - }, - { - "id": 5, - "type": "stat", - "title": "Warnings (1h)", - "gridPos": { - "h": 4, - "w": 4, - "x": 16, - "y": 5 - }, - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 50 - } - ] - }, - "mappings": [], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "sum" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "expr": "sum(count_over_time({namespace=\"foxhunt\", app=~\"$service\"} |= \"WARN\" [1h]))", - "instant": true - } - ] - }, - { - "id": 6, - "type": "stat", - "title": "Error Rate /min", - "gridPos": { - "h": 4, - "w": 4, - "x": 20, - "y": 5 - }, - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 1 - }, - { - "color": "red", - "value": 5 - } - ] - }, - "unit": "short", - "decimals": 1, - "mappings": [], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "area", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "expr": "sum(count_over_time({namespace=\"foxhunt\", app=~\"$service\", container!~\"loki|grafana|grafana-sc-dashboard|prometheus-server\"} |= \"ERROR\" [1m]))", - "instant": true - } - ] - }, - { - "id": 10, - "type": "row", - "title": "Error Analysis", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 9 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 11, - "type": "timeseries", - "title": "Error Rate by Service", - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 10 - }, - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 20, - "gradientMode": "scheme", - "spanNulls": false, - "showPoints": "never", - "stacking": { - "mode": "none", - "group": "A" - }, - "axisPlacement": "auto", - "lineWidth": 2 - }, - "unit": "short", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "color": { - "mode": "palette-classic" - }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "right", - "calcs": [ - "sum", - "max", - "last" - ] - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "expr": "sum by (app) (count_over_time({namespace=\"foxhunt\", app=~\"$service\", container!~\"loki|grafana|grafana-sc-dashboard|prometheus-server\"} |= \"ERROR\" [$interval]))", - "legendFormat": "{{ app }}" - } - ] - }, - { - "id": 12, - "type": "table", - "title": "Top Error Messages", - "description": "Most frequent error messages across all services in the time range", - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 17 - }, - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "mappings": [], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "showHeader": true, - "sortBy": [ - { - "displayName": "Value", - "desc": true - } - ], - "footer": { - "show": false - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "expr": "topk(20, sum by (app, level) (count_over_time({namespace=\"foxhunt\", app=~\"$service\", container!~\"loki|grafana|grafana-sc-dashboard|prometheus-server\"} |= \"ERROR\" | logfmt [$interval])))", - "legendFormat": "{{ app }}", - "instant": true - } - ] - }, - { - "id": 20, - "type": "row", - "title": "Live Logs", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 25 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 21, - "type": "logs", - "title": "Live Log Stream", - "description": "Streaming logs from selected services. Use the $search variable to filter.", - "gridPos": { - "h": 14, - "w": 24, - "x": 0, - "y": 26 - }, - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "options": { - "showTime": true, - "showLabels": true, - "showCommonLabels": false, - "wrapLogMessage": true, - "prettifyLogMessage": false, - "enableLogDetails": true, - "sortOrder": "Descending", - "dedupStrategy": "none" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "expr": "{namespace=\"foxhunt\", app=~\"$service\"} |= \"$search\"", - "maxLines": 500 - } - ] - }, - { - "id": 30, - "type": "row", - "title": "Traces", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 40 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 31, - "type": "table", - "title": "Recent Traces", - "description": "Search for distributed traces via Tempo. Click a trace ID to view the full trace.", - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 41 - }, - "datasource": { - "type": "tempo", - "uid": "${DS_TEMPO}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "mappings": [], - "color": { - "mode": "thresholds" - }, - "links": [ - { - "title": "View Trace", - "url": "", - "internal": { - "datasourceUid": "${DS_TEMPO}", - "datasourceName": "Tempo", - "query": { - "queryType": "traceql", - "query": "${__data.fields.traceID}" - } - } - } - ] - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Duration" - }, - "properties": [ - { - "id": "unit", - "value": "ns" - } - ] - } - ] - }, - "options": { - "showHeader": true, - "sortBy": [ - { - "displayName": "Start time", - "desc": true - } - ], - "footer": { - "show": false - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "tempo", - "uid": "${DS_TEMPO}" - }, - "queryType": "nativeSearch", - "serviceName": "", - "spanName": "", - "search": "", - "minDuration": "", - "maxDuration": "", - "limit": 20 - } - ] - } - ] -} diff --git a/infra/k8s/monitoring/dashboards/foxhunt-observability.json b/infra/k8s/monitoring/dashboards/foxhunt-observability.json new file mode 100644 index 000000000..0ba731e7b --- /dev/null +++ b/infra/k8s/monitoring/dashboards/foxhunt-observability.json @@ -0,0 +1,767 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "type": "datasource", + "pluginId": "prometheus" + }, + { + "name": "DS_LOKI", + "label": "Loki", + "type": "datasource", + "pluginId": "loki" + }, + { + "name": "DS_TEMPO", + "label": "Tempo", + "type": "datasource", + "pluginId": "tempo" + } + ], + "annotations": { + "list": [] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "id": 100, + "title": "API Gateway Overview", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 }, + "id": 1, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"] }, + "colorMode": "background", + "graphMode": "none" + }, + "title": "Total Request Rate", + "type": "stat", + "targets": [ + { + "expr": "sum(rate(api_gateway_http_requests_total[$interval]))", + "legendFormat": "", + "refId": "A" + } + ] + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 1 }, + { "color": "red", "value": 5 } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 4, "x": 4, "y": 1 }, + "id": 2, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"] }, + "colorMode": "background", + "graphMode": "none" + }, + "title": "Error Rate %", + "type": "stat", + "targets": [ + { + "expr": "sum(rate(api_gateway_http_requests_total{status=~\"5..\"}[$interval])) / sum(rate(api_gateway_http_requests_total[$interval])) * 100", + "legendFormat": "", + "refId": "A" + } + ] + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 0.5 }, + { "color": "red", "value": 1 } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 }, + "id": 3, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"] }, + "colorMode": "background", + "graphMode": "none" + }, + "title": "P99 Latency", + "type": "stat", + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(api_gateway_request_duration_seconds_bucket[$interval])) by (le))", + "legendFormat": "", + "refId": "A" + } + ] + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "blue", "value": null } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 4, "x": 12, "y": 1 }, + "id": 4, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"] }, + "colorMode": "background", + "graphMode": "none" + }, + "title": "Active WebSockets", + "type": "stat", + "targets": [ + { + "expr": "api_gateway_websocket_connections_active or vector(0)", + "legendFormat": "", + "refId": "A" + } + ] + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "orange", "value": 1 }, + { "color": "red", "value": 10 } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 4, "x": 16, "y": 1 }, + "id": 5, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"] }, + "colorMode": "background", + "graphMode": "none" + }, + "title": "Rate Limit Rejections/s", + "type": "stat", + "targets": [ + { + "expr": "sum(rate(api_gateway_rate_limit_rejected_total[$interval]))", + "legendFormat": "", + "refId": "A" + } + ] + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "orange", "value": 0.5 }, + { "color": "red", "value": 2 } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 4, "x": 20, "y": 1 }, + "id": 6, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"] }, + "colorMode": "background", + "graphMode": "none" + }, + "title": "Auth Failures/s", + "type": "stat", + "targets": [ + { + "expr": "sum(rate(api_gateway_auth_attempts_total{result=\"failure\"}[$interval]))", + "legendFormat": "", + "refId": "A" + } + ] + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, + "id": 101, + "title": "Request Patterns", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisLabel": "", + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "auto", + "stacking": { "mode": "none" } + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 }, + "id": 7, + "options": { + "tooltip": { "mode": "multi" }, + "legend": { "displayMode": "table", "placement": "bottom" } + }, + "title": "Request Rate by Handler", + "type": "timeseries", + "targets": [ + { + "expr": "topk(10, sum by (handler) (rate(api_gateway_http_requests_total[$interval])))", + "legendFormat": "{{handler}}", + "refId": "A" + } + ] + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisLabel": "", + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "auto", + "stacking": { "mode": "none" } + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 }, + "id": 8, + "options": { + "tooltip": { "mode": "multi" }, + "legend": { "displayMode": "table", "placement": "bottom" } + }, + "title": "Latency by Handler", + "type": "timeseries", + "targets": [ + { + "expr": "histogram_quantile(0.50, sum by (le, handler) (rate(api_gateway_request_duration_seconds_bucket[$interval])))", + "legendFormat": "p50 {{handler}}", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, sum by (le, handler) (rate(api_gateway_request_duration_seconds_bucket[$interval])))", + "legendFormat": "p95 {{handler}}", + "refId": "B" + } + ] + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 }, + "id": 102, + "title": "Auth & Security", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisLabel": "", + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "auto", + "stacking": { "mode": "none" } + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 15 }, + "id": 9, + "options": { + "tooltip": { "mode": "multi" }, + "legend": { "displayMode": "table", "placement": "bottom" } + }, + "title": "Auth Timing", + "type": "timeseries", + "targets": [ + { + "expr": "histogram_quantile(0.50, rate(api_gateway_auth_duration_seconds_bucket[$interval]))", + "legendFormat": "p50", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, rate(api_gateway_auth_duration_seconds_bucket[$interval]))", + "legendFormat": "p95", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.99, rate(api_gateway_auth_duration_seconds_bucket[$interval]))", + "legendFormat": "p99", + "refId": "C" + } + ] + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisLabel": "", + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "auto", + "stacking": { "mode": "none" } + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 15 }, + "id": 10, + "options": { + "tooltip": { "mode": "multi" }, + "legend": { "displayMode": "table", "placement": "bottom" } + }, + "title": "RBAC Checks", + "type": "timeseries", + "targets": [ + { + "expr": "sum by (result) (rate(api_gateway_rbac_checks_total[$interval]))", + "legendFormat": "{{result}}", + "refId": "A" + } + ] + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisLabel": "", + "drawStyle": "bars", + "fillOpacity": 50, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "stacking": { "mode": "normal" } + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 15 }, + "id": 11, + "options": { + "tooltip": { "mode": "multi" }, + "legend": { "displayMode": "table", "placement": "bottom" } + }, + "title": "Rate Limiting by Tier", + "type": "timeseries", + "targets": [ + { + "expr": "sum by (tier) (rate(api_gateway_rate_limit_allowed_total[$interval]))", + "legendFormat": "allowed {{tier}}", + "refId": "A" + }, + { + "expr": "sum by (tier) (rate(api_gateway_rate_limit_rejected_total[$interval]))", + "legendFormat": "rejected {{tier}}", + "refId": "B" + } + ] + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 }, + "id": 103, + "title": "Log Volume", + "type": "row" + }, + { + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisLabel": "", + "drawStyle": "line", + "fillOpacity": 30, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "stacking": { "mode": "normal" } + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "error" }, + "properties": [{ "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } }] + }, + { + "matcher": { "id": "byName", "options": "warn" }, + "properties": [{ "id": "color", "value": { "fixedColor": "yellow", "mode": "fixed" } }] + }, + { + "matcher": { "id": "byName", "options": "info" }, + "properties": [{ "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } }] + }, + { + "matcher": { "id": "byName", "options": "debug" }, + "properties": [{ "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } }] + } + ] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 }, + "id": 12, + "options": { + "tooltip": { "mode": "multi" }, + "legend": { "displayMode": "table", "placement": "bottom" } + }, + "title": "Log Rate by Level", + "type": "timeseries", + "targets": [ + { + "expr": "sum by (level) (count_over_time({namespace=\"foxhunt\"} | json | level=~\"$level\" [$interval]))", + "legendFormat": "{{level}}", + "refId": "A" + } + ] + }, + { + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 }, + "id": 13, + "options": { + "showTime": true, + "showLabels": true, + "showCommonLabels": false, + "wrapLogMessage": true, + "enableLogDetails": true + }, + "title": "Error Logs", + "type": "logs", + "targets": [ + { + "expr": "{namespace=\"foxhunt\"} | json | level=\"error\"", + "refId": "A" + } + ] + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 32 }, + "id": 104, + "title": "Log Detail", + "type": "row" + }, + { + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisLabel": "", + "drawStyle": "bars", + "fillOpacity": 50, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "stacking": { "mode": "normal" } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 33 }, + "id": 14, + "options": { + "tooltip": { "mode": "multi" }, + "legend": { "displayMode": "table", "placement": "bottom" } + }, + "title": "Logs by Service", + "type": "timeseries", + "targets": [ + { + "expr": "sum by (app) (count_over_time({namespace=\"foxhunt\", app=~\".+\"} [$interval]))", + "legendFormat": "{{app}}", + "refId": "A" + } + ] + }, + { + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 33 }, + "id": 15, + "options": { + "showTime": true, + "showLabels": true, + "showCommonLabels": false, + "wrapLogMessage": true, + "enableLogDetails": true + }, + "title": "Log Stream", + "type": "logs", + "targets": [ + { + "expr": "{namespace=\"foxhunt\", app=~\"$service\"} | json | level=~\"$level\"", + "refId": "A" + } + ] + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 41 }, + "id": 105, + "title": "Traces", + "type": "row" + }, + { + "datasource": { "type": "tempo", "uid": "${DS_TEMPO}" }, + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 42 }, + "id": 16, + "options": {}, + "title": "Trace Search", + "type": "traces", + "targets": [ + { + "queryType": "traceqlSearch", + "refId": "A" + } + ] + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 50 }, + "id": 106, + "title": "Prometheus Self-Monitoring", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisLabel": "", + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "auto", + "stacking": { "mode": "none" } + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 51 }, + "id": 17, + "options": { + "tooltip": { "mode": "multi" }, + "legend": { "displayMode": "table", "placement": "bottom" } + }, + "title": "Prometheus Scrape Duration", + "type": "timeseries", + "targets": [ + { + "expr": "scrape_duration_seconds", + "legendFormat": "{{job}}", + "refId": "A" + } + ] + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisLabel": "", + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "stacking": { "mode": "none" } + }, + "mappings": [ + { "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } }, "type": "value" } + ], + "unit": "short", + "min": 0, + "max": 1 + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 51 }, + "id": 18, + "options": { + "tooltip": { "mode": "multi" }, + "legend": { "displayMode": "table", "placement": "bottom" } + }, + "title": "Targets Up/Down", + "type": "timeseries", + "targets": [ + { + "expr": "up", + "legendFormat": "{{job}}", + "refId": "A" + } + ] + } + ], + "refresh": "15s", + "schemaVersion": 39, + "tags": ["foxhunt", "observability", "logs", "traces"], + "templating": { + "list": [ + { + "auto": true, + "auto_count": 30, + "auto_min": "10s", + "current": { "selected": false, "text": "auto", "value": "$__auto_interval_interval" }, + "hide": 0, + "label": "Interval", + "name": "interval", + "options": [ + { "selected": false, "text": "1m", "value": "1m" }, + { "selected": false, "text": "5m", "value": "5m" }, + { "selected": false, "text": "15m", "value": "15m" }, + { "selected": false, "text": "30m", "value": "30m" }, + { "selected": false, "text": "1h", "value": "1h" } + ], + "query": "1m,5m,15m,30m,1h", + "type": "interval" + }, + { + "allValue": ".*", + "current": { "selected": true, "text": ["All"], "value": ["$__all"] }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "definition": "label_values(up{job=~\"foxhunt.*\"}, job)", + "hide": 0, + "includeAll": true, + "label": "Service", + "multi": true, + "name": "service", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(up{job=~\"foxhunt.*\"}, job)" + }, + "refresh": 2, + "type": "query" + }, + { + "allValue": ".*", + "current": { "selected": true, "text": ["All"], "value": ["$__all"] }, + "hide": 0, + "includeAll": true, + "label": "Level", + "multi": true, + "name": "level", + "options": [ + { "selected": false, "text": "error", "value": "error" }, + { "selected": false, "text": "warn", "value": "warn" }, + { "selected": false, "text": "info", "value": "info" }, + { "selected": false, "text": "debug", "value": "debug" } + ], + "query": "error,warn,info,debug", + "type": "custom" + } + ] + }, + "time": { "from": "now-1h", "to": "now" }, + "timepicker": {}, + "timezone": "browser", + "title": "Observability & API Gateway", + "uid": "foxhunt-observability", + "version": 1 +} \ No newline at end of file diff --git a/infra/k8s/monitoring/dashboards/foxhunt-overview.json b/infra/k8s/monitoring/dashboards/foxhunt-overview.json deleted file mode 100644 index 9fbd98a8b..000000000 --- a/infra/k8s/monitoring/dashboards/foxhunt-overview.json +++ /dev/null @@ -1,1609 +0,0 @@ -{ - "__inputs": [ - { - "name": "DS_PROMETHEUS", - "label": "Prometheus", - "description": "", - "type": "datasource", - "pluginId": "prometheus" - }, - { - "name": "DS_LOKI", - "label": "Loki", - "description": "", - "type": "datasource", - "pluginId": "loki" - } - ], - "id": null, - "uid": "foxhunt-overview", - "title": "System Overview", - "description": "Home dashboard: cluster health, service status, resource utilization, deployments, training jobs, error rates", - "tags": [ - "foxhunt", - "production" - ], - "timezone": "browser", - "schemaVersion": 39, - "version": 1, - "editable": true, - "refresh": "30s", - "time": { - "from": "now-1h", - "to": "now" - }, - "fiscalYearStartMonth": 0, - "liveNow": false, - "weekStart": "", - "templating": { - "list": [ - { - "name": "interval", - "type": "interval", - "query": "1m,5m,15m,30m,1h", - "current": { - "selected": true, - "text": "5m", - "value": "5m" - }, - "auto": false - } - ] - }, - "panels": [ - { - "id": 1, - "type": "row", - "title": "Cluster Health", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 2, - "type": "stat", - "title": "Nodes Ready", - "gridPos": { - "h": 4, - "w": 4, - "x": 0, - "y": 1 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "yellow", - "value": 2 - }, - { - "color": "green", - "value": 3 - } - ] - }, - "mappings": [], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kube_node_status_condition{condition=\"Ready\", status=\"true\"})", - "legendFormat": "Ready", - "instant": true - } - ] - }, - { - "id": 3, - "type": "stat", - "title": "Total Nodes", - "gridPos": { - "h": 4, - "w": 4, - "x": 4, - "y": 1 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - } - ] - }, - "mappings": [], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "count(kube_node_info)", - "legendFormat": "Nodes", - "instant": true - } - ] - }, - { - "id": 4, - "type": "stat", - "title": "Pods Running", - "gridPos": { - "h": 4, - "w": 4, - "x": 8, - "y": 1 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "mappings": [], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", phase=\"Running\"})", - "legendFormat": "Running", - "instant": true - } - ] - }, - { - "id": 5, - "type": "stat", - "title": "Pods Failed", - "gridPos": { - "h": 4, - "w": 4, - "x": 12, - "y": 1 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - }, - "mappings": [], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", phase=\"Failed\"}) or vector(0)", - "legendFormat": "Failed", - "instant": true - } - ] - }, - { - "id": 6, - "type": "stat", - "title": "Active Training Jobs", - "gridPos": { - "h": 4, - "w": 4, - "x": 16, - "y": 1 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - }, - "mappings": [], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kube_job_status_active{namespace=\"foxhunt\"}) or vector(0)", - "legendFormat": "Active Jobs", - "instant": true - } - ] - }, - { - "id": 7, - "type": "stat", - "title": "Log Error Rate (/min)", - "gridPos": { - "h": 4, - "w": 4, - "x": 20, - "y": 1 - }, - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 5 - }, - { - "color": "red", - "value": 20 - } - ] - }, - "unit": "short", - "decimals": 1, - "mappings": [], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "area", - "justifyMode": "auto" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "expr": "sum(count_over_time({namespace=\"foxhunt\", container!~\"loki|grafana|grafana-sc-dashboard|prometheus-server\"} |= \"ERROR\" [1m]))", - "legendFormat": "Errors/min", - "instant": true - } - ] - }, - { - "id": 10, - "type": "row", - "title": "Service Status", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 5 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 11, - "type": "stat", - "title": "api", - "gridPos": { - "h": 3, - "w": 3, - "x": 0, - "y": 6 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - }, - "mappings": [ - { - "type": "value", - "options": { - "0": { - "text": "DOWN", - "color": "red" - }, - "1": { - "text": "UP", - "color": "green" - } - } - } - ], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"api.*\", phase=\"Running\"}) > 0 or vector(0)", - "legendFormat": "", - "instant": true - } - ] - }, - { - "id": 12, - "type": "stat", - "title": "trading-service", - "gridPos": { - "h": 3, - "w": 3, - "x": 3, - "y": 6 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - }, - "mappings": [ - { - "type": "value", - "options": { - "0": { - "text": "DOWN", - "color": "red" - }, - "1": { - "text": "UP", - "color": "green" - } - } - } - ], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"trading-service.*\", phase=\"Running\"}) > 0 or vector(0)", - "legendFormat": "", - "instant": true - } - ] - }, - { - "id": 13, - "type": "stat", - "title": "ml-training-service", - "gridPos": { - "h": 3, - "w": 3, - "x": 6, - "y": 6 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - }, - "mappings": [ - { - "type": "value", - "options": { - "0": { - "text": "DOWN", - "color": "red" - }, - "1": { - "text": "UP", - "color": "green" - } - } - } - ], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"ml-training-service.*\", phase=\"Running\"}) > 0 or vector(0)", - "legendFormat": "", - "instant": true - } - ] - }, - { - "id": 14, - "type": "stat", - "title": "backtesting-service", - "gridPos": { - "h": 3, - "w": 3, - "x": 9, - "y": 6 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - }, - "mappings": [ - { - "type": "value", - "options": { - "0": { - "text": "DOWN", - "color": "red" - }, - "1": { - "text": "UP", - "color": "green" - } - } - } - ], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"backtesting-service.*\", phase=\"Running\"}) > 0 or vector(0)", - "legendFormat": "", - "instant": true - } - ] - }, - { - "id": 15, - "type": "stat", - "title": "trading-agent", - "gridPos": { - "h": 3, - "w": 3, - "x": 12, - "y": 6 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - }, - "mappings": [ - { - "type": "value", - "options": { - "0": { - "text": "DOWN", - "color": "red" - }, - "1": { - "text": "UP", - "color": "green" - } - } - } - ], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"trading-agent.*\", phase=\"Running\"}) > 0 or vector(0)", - "legendFormat": "", - "instant": true - } - ] - }, - { - "id": 16, - "type": "stat", - "title": "broker-gateway", - "gridPos": { - "h": 3, - "w": 3, - "x": 15, - "y": 6 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - }, - "mappings": [ - { - "type": "value", - "options": { - "0": { - "text": "DOWN", - "color": "red" - }, - "1": { - "text": "UP", - "color": "green" - } - } - } - ], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"broker-gateway.*\", phase=\"Running\"}) > 0 or vector(0)", - "legendFormat": "", - "instant": true - } - ] - }, - { - "id": 17, - "type": "stat", - "title": "data-acquisition", - "gridPos": { - "h": 3, - "w": 3, - "x": 18, - "y": 6 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - }, - "mappings": [ - { - "type": "value", - "options": { - "0": { - "text": "DOWN", - "color": "red" - }, - "1": { - "text": "UP", - "color": "green" - } - } - } - ], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"data-acquisition.*\", phase=\"Running\"}) > 0 or vector(0)", - "legendFormat": "", - "instant": true - } - ] - }, - { - "id": 20, - "type": "row", - "title": "Resource Utilization", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 9 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 21, - "type": "gauge", - "title": "Cluster CPU %", - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 10 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 70 - }, - { - "color": "red", - "value": 90 - } - ] - }, - "min": 0, - "max": 100, - "unit": "percent", - "mappings": [], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "orientation": "auto" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "100 - (avg(rate(node_cpu_seconds_total{mode=\"idle\"}[$interval])) * 100)", - "legendFormat": "CPU", - "instant": true - } - ] - }, - { - "id": 22, - "type": "gauge", - "title": "Cluster Memory %", - "gridPos": { - "h": 6, - "w": 6, - "x": 6, - "y": 10 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 70 - }, - { - "color": "red", - "value": 90 - } - ] - }, - "min": 0, - "max": 100, - "unit": "percent", - "mappings": [], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "orientation": "auto" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "(1 - (sum(node_memory_MemAvailable_bytes) / sum(node_memory_MemTotal_bytes))) * 100", - "legendFormat": "Memory", - "instant": true - } - ] - }, - { - "id": 23, - "type": "timeseries", - "title": "CPU by Node Pool", - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 10 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 15, - "gradientMode": "scheme", - "spanNulls": false, - "showPoints": "never", - "stacking": { - "mode": "none", - "group": "A" - }, - "axisPlacement": "auto", - "lineWidth": 1 - }, - "unit": "percentunit", - "min": 0, - "max": 1, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "color": { - "mode": "palette-classic" - }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "list", - "placement": "bottom", - "calcs": [] - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[$interval]))", - "legendFormat": "{{ instance }}" - } - ] - }, - { - "id": 24, - "type": "timeseries", - "title": "Memory by Node Pool", - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 16 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 15, - "gradientMode": "scheme", - "spanNulls": false, - "showPoints": "never", - "stacking": { - "mode": "none", - "group": "A" - }, - "axisPlacement": "auto", - "lineWidth": 1 - }, - "unit": "bytes", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "color": { - "mode": "palette-classic" - }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "list", - "placement": "bottom", - "calcs": [] - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes", - "legendFormat": "{{ instance }}" - } - ] - }, - { - "id": 25, - "type": "timeseries", - "title": "Foxhunt Container CPU Usage", - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 16 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 15, - "gradientMode": "none", - "spanNulls": false, - "showPoints": "never", - "stacking": { - "mode": "normal", - "group": "A" - }, - "axisPlacement": "auto", - "lineWidth": 1 - }, - "unit": "short", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "color": { - "mode": "palette-classic" - }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "list", - "placement": "bottom", - "calcs": [] - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum by (container) (rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", container=~\"api|trading-service|ml-training-service|backtesting-service|trading-agent-service|broker-gateway|data-acquisition-service\", image!=\"\"}[$interval]))", - "legendFormat": "{{ container }}" - } - ] - }, - { - "id": 30, - "type": "row", - "title": "Deployments & Restarts", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 22 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 31, - "type": "timeseries", - "title": "Pod Restarts (last 1h)", - "description": "Pod restart events indicating crashes or OOMKills", - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 23 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "bars", - "lineInterpolation": "linear", - "fillOpacity": 80, - "gradientMode": "none", - "spanNulls": false, - "showPoints": "never", - "stacking": { - "mode": "normal", - "group": "A" - }, - "axisPlacement": "auto", - "lineWidth": 1 - }, - "unit": "short", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "color": { - "mode": "palette-classic" - }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "list", - "placement": "bottom", - "calcs": [] - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "increase(kube_pod_container_status_restarts_total{namespace=\"foxhunt\"}[1h]) > 0", - "legendFormat": "{{ pod }}" - } - ] - }, - { - "id": 32, - "type": "timeseries", - "title": "Deployment Replicas: Desired vs Available", - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 23 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "stepAfter", - "fillOpacity": 0, - "gradientMode": "none", - "spanNulls": false, - "showPoints": "never", - "stacking": { - "mode": "none", - "group": "A" - }, - "axisPlacement": "auto", - "lineWidth": 2 - }, - "unit": "short", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "color": { - "mode": "palette-classic" - }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "last" - ] - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "kube_deployment_spec_replicas{namespace=\"foxhunt\"}", - "legendFormat": "{{ deployment }} desired" - }, - { - "refId": "B", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "kube_deployment_status_replicas_available{namespace=\"foxhunt\"}", - "legendFormat": "{{ deployment }} available" - } - ] - }, - { - "id": 40, - "type": "row", - "title": "Error Rate Trend", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 30 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 41, - "type": "timeseries", - "title": "Log Errors Over Time", - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 31 - }, - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "bars", - "lineInterpolation": "linear", - "fillOpacity": 50, - "gradientMode": "scheme", - "spanNulls": false, - "showPoints": "never", - "stacking": { - "mode": "normal", - "group": "A" - }, - "axisPlacement": "auto", - "lineWidth": 1 - }, - "unit": "short", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "color": { - "mode": "palette-classic" - }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "list", - "placement": "bottom", - "calcs": [] - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "expr": "sum by (app) (count_over_time({namespace=\"foxhunt\", container!~\"loki|grafana|grafana-sc-dashboard|prometheus-server\"} |= \"ERROR\" [$interval]))", - "legendFormat": "{{ app }}" - } - ] - } - ] -} diff --git a/infra/k8s/monitoring/dashboards/foxhunt-services.json b/infra/k8s/monitoring/dashboards/foxhunt-services.json deleted file mode 100644 index 01367a762..000000000 --- a/infra/k8s/monitoring/dashboards/foxhunt-services.json +++ /dev/null @@ -1,1185 +0,0 @@ -{ - "__inputs": [ - { - "name": "DS_PROMETHEUS", - "label": "Prometheus", - "description": "", - "type": "datasource", - "pluginId": "prometheus" - }, - { - "name": "DS_LOKI", - "label": "Loki", - "description": "", - "type": "datasource", - "pluginId": "loki" - } - ], - "id": null, - "uid": "foxhunt-services", - "title": "Microservice Detail", - "description": "Per-service deep dive: CPU, memory, restarts, gRPC metrics, logs", - "tags": [ - "foxhunt", - "production" - ], - "timezone": "browser", - "schemaVersion": 39, - "version": 1, - "editable": true, - "refresh": "30s", - "time": { - "from": "now-1h", - "to": "now" - }, - "fiscalYearStartMonth": 0, - "liveNow": false, - "weekStart": "", - "templating": { - "list": [ - { - "name": "service", - "type": "custom", - "query": "api,trading-service,ml-training-service,backtesting-service,trading-agent-service,broker-gateway,data-acquisition-service", - "current": { - "selected": true, - "text": "api", - "value": "api" - }, - "multi": false, - "includeAll": false, - "sort": 0 - }, - { - "name": "interval", - "type": "interval", - "query": "1m,5m,15m,30m,1h", - "current": { - "selected": true, - "text": "5m", - "value": "5m" - }, - "auto": false - } - ] - }, - "panels": [ - { - "id": 1, - "type": "row", - "title": "Pod Status - $service", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 2, - "type": "stat", - "title": "Status", - "gridPos": { - "h": 4, - "w": 4, - "x": 0, - "y": 1 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - }, - "mappings": [ - { - "type": "value", - "options": { - "0": { - "text": "DOWN", - "color": "red" - }, - "1": { - "text": "UP", - "color": "green" - } - } - } - ], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"$service.*\", phase=\"Running\"}) > 0 or vector(0)", - "instant": true - } - ] - }, - { - "id": 3, - "type": "stat", - "title": "Replicas", - "gridPos": { - "h": 4, - "w": 4, - "x": 4, - "y": 1 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - } - ] - }, - "mappings": [], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "count(kube_pod_info{namespace=\"foxhunt\", pod=~\"$service.*\"})", - "instant": true - } - ] - }, - { - "id": 4, - "type": "stat", - "title": "Restarts (24h)", - "gridPos": { - "h": 4, - "w": 4, - "x": 8, - "y": 1 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 1 - }, - { - "color": "red", - "value": 5 - } - ] - }, - "mappings": [], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "area", - "justifyMode": "auto" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(increase(kube_pod_container_status_restarts_total{namespace=\"foxhunt\", pod=~\"$service.*\"}[24h])) or vector(0)", - "instant": true - } - ] - }, - { - "id": 5, - "type": "stat", - "title": "Container Uptime", - "gridPos": { - "h": 4, - "w": 4, - "x": 12, - "y": 1 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "yellow", - "value": 3600 - }, - { - "color": "green", - "value": 86400 - } - ] - }, - "unit": "s", - "mappings": [], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "min(time() - kube_pod_start_time{namespace=\"foxhunt\", pod=~\"$service.*\"})", - "instant": true - } - ] - }, - { - "id": 6, - "type": "stat", - "title": "CPU Cores", - "gridPos": { - "h": 4, - "w": 4, - "x": 16, - "y": 1 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - } - ] - }, - "unit": "short", - "decimals": 3, - "mappings": [], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", container=~\"$service\", image!=\"\"}[$interval]))", - "instant": true - } - ] - }, - { - "id": 7, - "type": "stat", - "title": "Memory", - "gridPos": { - "h": 4, - "w": 4, - "x": 20, - "y": 1 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - } - ] - }, - "unit": "bytes", - "mappings": [], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(container_memory_working_set_bytes{namespace=\"foxhunt\", container=~\"$service\", image!=\"\"})", - "instant": true - } - ] - }, - { - "id": 10, - "type": "row", - "title": "Resource Usage Over Time", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 5 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 11, - "type": "timeseries", - "title": "CPU Usage", - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 6 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 20, - "gradientMode": "scheme", - "spanNulls": false, - "showPoints": "never", - "stacking": { - "mode": "none", - "group": "A" - }, - "axisPlacement": "auto", - "lineWidth": 2 - }, - "unit": "short", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "color": { - "mode": "palette-classic" - }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "max", - "last" - ] - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", container=~\"$service\", image!=\"\"}[$interval]))", - "legendFormat": "{{ pod }}" - }, - { - "refId": "B", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kube_pod_container_resource_requests{namespace=\"foxhunt\", container=~\"$service\", resource=\"cpu\"})", - "legendFormat": "Request" - }, - { - "refId": "C", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kube_pod_container_resource_limits{namespace=\"foxhunt\", container=~\"$service\", resource=\"cpu\"})", - "legendFormat": "Limit" - } - ] - }, - { - "id": 12, - "type": "timeseries", - "title": "Memory Usage", - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 6 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 20, - "gradientMode": "scheme", - "spanNulls": false, - "showPoints": "never", - "stacking": { - "mode": "none", - "group": "A" - }, - "axisPlacement": "auto", - "lineWidth": 2 - }, - "unit": "bytes", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "color": { - "mode": "palette-classic" - }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "max", - "last" - ] - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum by (pod) (container_memory_working_set_bytes{namespace=\"foxhunt\", container=~\"$service\", image!=\"\"})", - "legendFormat": "{{ pod }}" - }, - { - "refId": "B", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kube_pod_container_resource_requests{namespace=\"foxhunt\", container=~\"$service\", resource=\"memory\"})", - "legendFormat": "Request" - }, - { - "refId": "C", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(kube_pod_container_resource_limits{namespace=\"foxhunt\", container=~\"$service\", resource=\"memory\"})", - "legendFormat": "Limit" - } - ] - }, - { - "id": 15, - "type": "row", - "title": "Restart History", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 14 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 16, - "type": "timeseries", - "title": "Pod Restart Count", - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 15 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "bars", - "lineInterpolation": "linear", - "fillOpacity": 80, - "gradientMode": "none", - "spanNulls": false, - "showPoints": "never", - "stacking": { - "mode": "normal", - "group": "A" - }, - "axisPlacement": "auto", - "lineWidth": 1 - }, - "unit": "short", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "color": { - "mode": "palette-classic" - }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "list", - "placement": "bottom", - "calcs": [] - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "increase(kube_pod_container_status_restarts_total{namespace=\"foxhunt\", pod=~\"$service.*\"}[30m])", - "legendFormat": "{{ pod }}" - } - ] - }, - { - "id": 17, - "type": "timeseries", - "title": "Network I/O", - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 15 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 10, - "gradientMode": "none", - "spanNulls": false, - "showPoints": "never", - "stacking": { - "mode": "none", - "group": "A" - }, - "axisPlacement": "auto", - "lineWidth": 1 - }, - "unit": "Bps", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "color": { - "mode": "palette-classic" - }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "list", - "placement": "bottom", - "calcs": [] - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum by (pod) (rate(container_network_receive_bytes_total{namespace=\"foxhunt\", pod=~\"$service.*\"}[$interval]))", - "legendFormat": "{{ pod }} RX" - }, - { - "refId": "B", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum by (pod) (rate(container_network_transmit_bytes_total{namespace=\"foxhunt\", pod=~\"$service.*\"}[$interval]))", - "legendFormat": "{{ pod }} TX" - } - ] - }, - { - "id": 20, - "type": "row", - "title": "gRPC Metrics", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 21 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 21, - "type": "timeseries", - "title": "gRPC Request Rate", - "description": "gRPC requests per second by method and status code. Requires grpc_server_handled_total metric from service instrumentation.", - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 22 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 15, - "gradientMode": "scheme", - "spanNulls": false, - "showPoints": "never", - "stacking": { - "mode": "none", - "group": "A" - }, - "axisPlacement": "auto", - "lineWidth": 1 - }, - "unit": "reqps", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "color": { - "mode": "palette-classic" - }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "max" - ] - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum by (grpc_method, grpc_code) (rate(grpc_server_handled_total{grpc_service=~\".*$service.*\"}[$interval]))", - "legendFormat": "{{ grpc_method }} [{{ grpc_code }}]" - } - ] - }, - { - "id": 22, - "type": "timeseries", - "title": "gRPC Latency (p50 / p95 / p99)", - "description": "gRPC server-side latency percentiles. Requires grpc_server_handling_seconds_bucket histogram.", - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 22 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 10, - "gradientMode": "none", - "spanNulls": false, - "showPoints": "never", - "stacking": { - "mode": "none", - "group": "A" - }, - "axisPlacement": "auto", - "lineWidth": 2 - }, - "unit": "s", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "color": { - "mode": "palette-classic" - }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "max" - ] - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "histogram_quantile(0.50, sum(rate(grpc_server_handling_seconds_bucket{grpc_service=~\".*$service.*\"}[$interval])) by (le))", - "legendFormat": "p50" - }, - { - "refId": "B", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "histogram_quantile(0.95, sum(rate(grpc_server_handling_seconds_bucket{grpc_service=~\".*$service.*\"}[$interval])) by (le))", - "legendFormat": "p95" - }, - { - "refId": "C", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{grpc_service=~\".*$service.*\"}[$interval])) by (le))", - "legendFormat": "p99" - } - ] - }, - { - "id": 30, - "type": "row", - "title": "Logs", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 30 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 31, - "type": "timeseries", - "title": "Log Volume by Level", - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 31 - }, - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "bars", - "lineInterpolation": "linear", - "fillOpacity": 60, - "gradientMode": "none", - "spanNulls": false, - "showPoints": "never", - "stacking": { - "mode": "normal", - "group": "A" - }, - "axisPlacement": "auto", - "lineWidth": 1 - }, - "unit": "short", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "color": { - "mode": "palette-classic" - }, - "mappings": [] - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "ERROR" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "WARN" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "INFO" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "DEBUG" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - } - ] - } - ] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "list", - "placement": "bottom", - "calcs": [] - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "expr": "sum by (level) (count_over_time({app=\"$service\", namespace=\"foxhunt\"} | logfmt | level != \"\" [$interval]))", - "legendFormat": "{{ level }}" - } - ] - }, - { - "id": 32, - "type": "logs", - "title": "Recent Error Logs", - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 37 - }, - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "options": { - "showTime": true, - "showLabels": true, - "showCommonLabels": false, - "wrapLogMessage": true, - "prettifyLogMessage": false, - "enableLogDetails": true, - "sortOrder": "Descending", - "dedupStrategy": "none" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "loki", - "uid": "${DS_LOKI}" - }, - "expr": "{app=\"$service\", namespace=\"foxhunt\", container!~\"loki|grafana|grafana-sc-dashboard|prometheus-server\"} |= \"ERROR\"", - "maxLines": 200 - } - ] - } - ] -} diff --git a/infra/k8s/monitoring/dashboards/foxhunt-traces.json b/infra/k8s/monitoring/dashboards/foxhunt-traces.json deleted file mode 100644 index 093f3881b..000000000 --- a/infra/k8s/monitoring/dashboards/foxhunt-traces.json +++ /dev/null @@ -1,562 +0,0 @@ -{ - "__inputs": [ - { - "name": "DS_PROMETHEUS", - "label": "Prometheus", - "description": "", - "type": "datasource", - "pluginId": "prometheus" - }, - { - "name": "DS_TEMPO", - "label": "Tempo", - "description": "", - "type": "datasource", - "pluginId": "tempo" - } - ], - "id": null, - "uid": "foxhunt-traces", - "title": "Traces", - "description": "Distributed tracing dashboard: span rates, error rates, latency percentiles, service map, trace search", - "tags": [ - "foxhunt", - "traces", - "tempo" - ], - "timezone": "browser", - "schemaVersion": 39, - "version": 1, - "editable": true, - "refresh": "30s", - "time": { - "from": "now-1h", - "to": "now" - }, - "fiscalYearStartMonth": 0, - "liveNow": false, - "weekStart": "", - "templating": { - "list": [ - { - "name": "service", - "type": "query", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "query": "label_values(traces_service_graph_request_total, client)", - "refresh": 2, - "multi": true, - "includeAll": true, - "current": { - "selected": true, - "text": "All", - "value": "$__all" - } - } - ] - }, - "panels": [ - { - "id": 1, - "type": "row", - "title": "Overview Stats", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 2, - "type": "stat", - "title": "Spans/sec", - "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 1 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "mappings": [], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(tempo_distributor_spans_received_total[5m]))", - "instant": true - } - ] - }, - { - "id": 3, - "type": "stat", - "title": "Error Rate", - "gridPos": { - "h": 4, - "w": 6, - "x": 6, - "y": 1 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "unit": "percent", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 1 - }, - { - "color": "red", - "value": 5 - } - ] - }, - "mappings": [], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(tempo_distributor_spans_received_total{status=\"error\"}[5m])) / sum(rate(tempo_distributor_spans_received_total[5m])) * 100", - "instant": true - } - ] - }, - { - "id": 4, - "type": "stat", - "title": "Avg Duration", - "gridPos": { - "h": 4, - "w": 6, - "x": 12, - "y": 1 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "unit": "s", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "mappings": [], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum(rate(tempo_spanmetrics_latency_sum[5m])) / sum(rate(tempo_spanmetrics_latency_count[5m]))", - "instant": true - } - ] - }, - { - "id": 5, - "type": "stat", - "title": "Active Services", - "gridPos": { - "h": 4, - "w": 6, - "x": 18, - "y": 1 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "mappings": [], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "count(count by (service_name)(tempo_distributor_spans_received_total))", - "instant": true - } - ] - }, - { - "id": 20, - "type": "row", - "title": "Service Map + Trace Search", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 5 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 6, - "type": "nodeGraph", - "title": "Service Map", - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 6 - }, - "datasource": { - "type": "tempo", - "uid": "${DS_TEMPO}" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "tempo", - "uid": "${DS_TEMPO}" - }, - "queryType": "serviceMap" - } - ] - }, - { - "id": 21, - "type": "row", - "title": "Latency and Errors by Service", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 16 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 7, - "type": "timeseries", - "title": "Latency by Service p95", - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 17 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "unit": "s", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 10, - "gradientMode": "none", - "showPoints": "auto", - "pointSize": 5, - "lineWidth": 1, - "spanNulls": false, - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "scaleDistribution": { - "type": "linear" - } - }, - "color": { - "mode": "palette-classic" - }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "max" - ] - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "histogram_quantile(0.95, sum by (le, service_name)(rate(tempo_spanmetrics_latency_bucket[5m])))", - "legendFormat": "{{service_name}}", - "instant": false, - "range": true - } - ] - }, - { - "id": 8, - "type": "timeseries", - "title": "Error Rate by Service", - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 17 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "unit": "reqps", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 10, - "gradientMode": "none", - "showPoints": "auto", - "pointSize": 5, - "lineWidth": 1, - "spanNulls": false, - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "scaleDistribution": { - "type": "linear" - } - }, - "color": { - "mode": "palette-classic" - }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "max" - ] - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "sum by (service_name)(rate(tempo_distributor_spans_received_total{status=\"error\"}[5m]))", - "legendFormat": "{{service_name}}", - "instant": false, - "range": true - } - ] - }, - { - "id": 22, - "type": "row", - "title": "Trace Search", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 24 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 10, - "type": "table", - "title": "Recent Traces", - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 25 - }, - "datasource": { - "type": "tempo", - "uid": "${DS_TEMPO}" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "tempo", - "uid": "${DS_TEMPO}" - }, - "queryType": "traceql", - "query": "{status = error} || {duration > 1s}", - "limit": 20 - } - ] - } - ] -} diff --git a/infra/k8s/monitoring/dashboards/foxhunt-trading-cockpit.json b/infra/k8s/monitoring/dashboards/foxhunt-trading.json similarity index 63% rename from infra/k8s/monitoring/dashboards/foxhunt-trading-cockpit.json rename to infra/k8s/monitoring/dashboards/foxhunt-trading.json index d3206b12f..54b1b6a9b 100644 --- a/infra/k8s/monitoring/dashboards/foxhunt-trading-cockpit.json +++ b/infra/k8s/monitoring/dashboards/foxhunt-trading.json @@ -9,12 +9,13 @@ } ], "id": null, - "uid": "foxhunt-trading-cockpit", - "title": "Trading Cockpit", - "description": "Real-time trading signals, execution, risk, ML model health, and latency monitoring", + "uid": "foxhunt-trading", + "title": "Trading & ML Intelligence", + "description": "ML signal quality, execution latency, risk checks, and gRPC service health for the foxhunt HFT platform", "tags": [ "foxhunt", - "trading" + "trading", + "ml" ], "timezone": "browser", "schemaVersion": 39, @@ -44,7 +45,7 @@ { "name": "model", "type": "query", - "query": "label_values(foxhunt_prediction_accuracy, model_id)", + "query": "label_values(foxhunt_predictions_confidence_bucket, model)", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" @@ -60,23 +61,7 @@ { "name": "symbol", "type": "query", - "query": "label_values(foxhunt_ensemble_agreement_rate, symbol)", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "refresh": 2, - "includeAll": true, - "multi": true, - "current": { - "text": "All", - "value": "$__all" - } - }, - { - "name": "service", - "type": "query", - "query": "label_values(grpc_server_handling_seconds_bucket, service)", + "query": "label_values(foxhunt_predictions_confidence_bucket, symbol)", "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" @@ -95,7 +80,7 @@ { "id": 1, "type": "row", - "title": "Signal Quality", + "title": "ML Signal Quality", "gridPos": { "h": 1, "w": 24, @@ -139,23 +124,10 @@ ] }, "unit": "percentunit", - "mappings": [ - { - "type": "special", - "options": { - "match": "null", - "result": { - "text": "Inactive", - "color": "dark-blue", - "index": 99 - } - } - } - ], "color": { "mode": "thresholds" }, - "noValue": "Inactive" + "noValue": "No data" }, "overrides": [] }, @@ -181,7 +153,8 @@ "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_ensemble_agreement_rate{symbol=~\"$symbol\"}", - "instant": true + "instant": true, + "legendFormat": "{{symbol}}" } ] }, @@ -219,23 +192,10 @@ ] }, "unit": "percentunit", - "mappings": [ - { - "type": "special", - "options": { - "match": "null", - "result": { - "text": "Inactive", - "color": "dark-blue", - "index": 99 - } - } - } - ], "color": { "mode": "thresholds" }, - "noValue": "Inactive" + "noValue": "No data" }, "overrides": [] }, @@ -260,15 +220,16 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.5, rate(foxhunt_predictions_confidence_bucket[5m]))", - "instant": true + "expr": "histogram_quantile(0.5, rate(foxhunt_predictions_confidence_bucket{model=~\"$model\"}[5m]))", + "instant": true, + "legendFormat": "p50" } ] }, { "id": 4, "type": "stat", - "title": "Model Accuracy", + "title": "Model Predictions/s", "gridPos": { "h": 4, "w": 6, @@ -290,32 +251,19 @@ }, { "color": "yellow", - "value": 0.5 + "value": 1 }, { "color": "green", - "value": 0.6 + "value": 10 } ] }, - "unit": "percentunit", - "mappings": [ - { - "type": "special", - "options": { - "match": "null", - "result": { - "text": "Inactive", - "color": "dark-blue", - "index": 99 - } - } - } - ], + "unit": "ops", "color": { "mode": "thresholds" }, - "noValue": "Inactive" + "noValue": "0" }, "overrides": [] }, @@ -340,15 +288,16 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "foxhunt_prediction_accuracy{model_id=~\"$model\"}", - "instant": true + "expr": "sum(rate(foxhunt_predictions_total{model=~\"$model\"}[5m]))", + "instant": true, + "legendFormat": "predictions/s" } ] }, { "id": 5, "type": "stat", - "title": "Ensemble Sharpe", + "title": "Active Models", "gridPos": { "h": 4, "w": 6, @@ -374,27 +323,15 @@ }, { "color": "green", - "value": 2 + "value": 5 } ] }, - "mappings": [ - { - "type": "special", - "options": { - "match": "null", - "result": { - "text": "Inactive", - "color": "dark-blue", - "index": 99 - } - } - } - ], + "unit": "none", "color": { "mode": "thresholds" }, - "noValue": "Inactive" + "noValue": "0" }, "overrides": [] }, @@ -419,15 +356,16 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "foxhunt_model_sharpe_ratio{model_id=\"ensemble\"}", - "instant": true + "expr": "count(foxhunt_model_loaded{model=~\"$model\"} == 1)", + "instant": true, + "legendFormat": "models" } ] }, { "id": 6, "type": "row", - "title": "Execution", + "title": "Prediction Detail", "gridPos": { "h": 1, "w": 24, @@ -439,11 +377,11 @@ }, { "id": 7, - "type": "stat", - "title": "Active Positions", + "type": "timeseries", + "title": "Confidence Distribution", "gridPos": { - "h": 4, - "w": 4, + "h": 8, + "w": 12, "x": 0, "y": 6 }, @@ -451,6 +389,166 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 10, + "gradientMode": "none", + "showPoints": "auto", + "pointSize": 5, + "spanNulls": false, + "axisCenteredZero": false, + "axisLabel": "", + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "mean" + ] + } + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.1, rate(foxhunt_predictions_confidence_bucket{model=~\"$model\"}[$interval]))", + "legendFormat": "p10" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.5, rate(foxhunt_predictions_confidence_bucket{model=~\"$model\"}[$interval]))", + "legendFormat": "p50" + }, + { + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.9, rate(foxhunt_predictions_confidence_bucket{model=~\"$model\"}[$interval]))", + "legendFormat": "p90" + } + ] + }, + { + "id": 8, + "type": "timeseries", + "title": "Prediction Rate by Model", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 6 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "bars", + "lineWidth": 1, + "fillOpacity": 80, + "gradientMode": "none", + "showPoints": "never", + "stacking": { + "mode": "normal", + "group": "A" + }, + "axisCenteredZero": false, + "axisLabel": "", + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "mean" + ] + } + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (model) (rate(foxhunt_predictions_total{model=~\"$model\"}[$interval]))", + "legendFormat": "{{model}}" + } + ] + }, + { + "id": 9, + "type": "row", + "title": "Order Execution", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "collapsed": false, + "panels": [] + }, + { + "id": 10, + "type": "stat", + "title": "Orders/s", + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 15 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "thresholds": { @@ -459,234 +557,26 @@ { "color": "blue", "value": null - } - ] - }, - "mappings": [], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_trading_agent_assets_selected", - "instant": true - } - ] - }, - { - "id": 8, - "type": "stat", - "title": "Realized PnL", - "gridPos": { - "h": 4, - "w": 4, - "x": 4, - "y": 6 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null }, { "color": "green", - "value": 0 - } - ] - }, - "unit": "currencyUSD", - "mappings": [ - { - "type": "special", - "options": { - "match": "null", - "result": { - "text": "Inactive", - "color": "dark-blue", - "index": 99 - } - } - } - ], - "color": { - "mode": "thresholds" - }, - "noValue": "Inactive" - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_model_cumulative_pnl{model_id=~\"$model\"}", - "instant": true - } - ] - }, - { - "id": 9, - "type": "stat", - "title": "Unrealized PnL", - "gridPos": { - "h": 4, - "w": 4, - "x": 8, - "y": 6 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "currencyUSD", - "mappings": [ - { - "type": "special", - "options": { - "match": "null", - "result": { - "text": "Inactive", - "color": "dark-blue", - "index": 99 - } - } - } - ], - "color": { - "mode": "thresholds" - }, - "noValue": "Inactive" - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_trading_agent_portfolio_value_usd", - "instant": true - } - ] - }, - { - "id": 10, - "type": "gauge", - "title": "Max Drawdown", - "gridPos": { - "h": 4, - "w": 4, - "x": 12, - "y": 6 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "value": 1 }, { "color": "yellow", - "value": 5 + "value": 100 }, { "color": "red", - "value": 10 + "value": 500 } ] }, - "unit": "percent", - "min": 0, - "max": 25, - "mappings": [], + "unit": "ops", "color": { "mode": "thresholds" - } + }, + "noValue": "0" }, "overrides": [] }, @@ -699,8 +589,10 @@ "values": false }, "orientation": "auto", - "showThresholdLabels": false, - "showThresholdMarkers": true + "textMode": "auto", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center" }, "targets": [ { @@ -709,20 +601,21 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "foxhunt_model_max_drawdown{model_id=~\"$model\"}", - "instant": true + "expr": "sum(rate(trading_orders_total[5m]))", + "instant": true, + "legendFormat": "orders/s" } ] }, { "id": 11, - "type": "timeseries", - "title": "PnL Over Time", + "type": "stat", + "title": "Order Latency P99", "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 6 + "h": 4, + "w": 6, + "x": 6, + "y": 15 }, "datasource": { "type": "prometheus", @@ -730,52 +623,44 @@ }, "fieldConfig": { "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 10, - "gradientMode": "none", - "spanNulls": false, - "showPoints": "never", - "pointSize": 5, - "stacking": { - "mode": "none", - "group": "A" - }, - "axisPlacement": "auto", - "axisLabel": "", - "scaleDistribution": { - "type": "linear" - } - }, - "unit": "currencyUSD", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null + }, + { + "color": "yellow", + "value": 0.01 + }, + { + "color": "red", + "value": 0.1 } ] }, - "mappings": [], + "unit": "ms", "color": { - "mode": "palette-classic" - } + "mode": "thresholds" + }, + "noValue": "N/A" }, "overrides": [] }, "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false }, - "legend": { - "displayMode": "list", - "placement": "bottom", - "calcs": [] - } + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center" }, "targets": [ { @@ -784,46 +669,21 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "foxhunt_model_cumulative_pnl{model_id=~\"$model\"}", - "legendFormat": "Realized PnL", - "instant": false, - "range": true - }, - { - "refId": "B", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_trading_agent_portfolio_value_usd", - "legendFormat": "Unrealized PnL", - "instant": false, - "range": true + "expr": "histogram_quantile(0.99, rate(trading_order_processing_seconds_bucket[5m]))", + "instant": true, + "legendFormat": "p99" } ] }, { "id": 12, - "type": "row", - "title": "Risk", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 12 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 13, "type": "stat", - "title": "Portfolio VaR", + "title": "Risk Check Latency", "gridPos": { "h": 4, "w": 6, - "x": 0, - "y": 13 + "x": 12, + "y": 15 }, "datasource": { "type": "prometheus", @@ -840,19 +700,19 @@ }, { "color": "yellow", - "value": 50000 + "value": 0.005 }, { "color": "red", - "value": 100000 + "value": 0.05 } ] }, - "unit": "currencyUSD", - "mappings": [], + "unit": "ms", "color": { "mode": "thresholds" - } + }, + "noValue": "N/A" }, "overrides": [] }, @@ -877,117 +737,102 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "foxhunt_model_max_drawdown{model_id=\"ensemble\"}", - "instant": true + "expr": "histogram_quantile(0.99, rate(trading_risk_check_seconds_bucket[5m]))", + "instant": true, + "legendFormat": "p99" + } + ] + }, + { + "id": 13, + "type": "stat", + "title": "Total Latency P99", + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 15 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.05 + }, + { + "color": "red", + "value": 0.5 + } + ] + }, + "unit": "ms", + "color": { + "mode": "thresholds" + }, + "noValue": "N/A" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.99, rate(trading_total_latency_seconds_bucket[5m]))", + "instant": true, + "legendFormat": "p99" } ] }, { "id": 14, - "type": "stat", - "title": "Circuit Breaker", + "type": "row", + "title": "Execution Quality", "gridPos": { - "h": 4, - "w": 6, - "x": 6, - "y": 13 + "h": 1, + "w": 24, + "x": 0, + "y": 19 }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - }, - { - "color": "yellow", - "value": 2 - } - ] - }, - "mappings": [ - { - "type": "value", - "options": { - "0": { - "text": "Closed", - "color": "green", - "index": 0 - } - } - }, - { - "type": "value", - "options": { - "1": { - "text": "Open", - "color": "red", - "index": 1 - } - } - }, - { - "type": "value", - "options": { - "2": { - "text": "HalfOpen", - "color": "yellow", - "index": 2 - } - } - } - ], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_circuit_breaker_state{model_id=~\"$model\"}", - "instant": true - } - ] + "collapsed": false, + "panels": [] }, { "id": 15, - "type": "stat", - "title": "Risk Alerts", + "type": "timeseries", + "title": "Order Processing Latency", "gridPos": { - "h": 4, - "w": 6, - "x": 12, - "y": 13 + "h": 8, + "w": 12, + "x": 0, + "y": 20 }, "datasource": { "type": "prometheus", @@ -995,197 +840,41 @@ }, "fieldConfig": { "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 1 - }, - { - "color": "red", - "value": 5 - } - ] - }, - "mappings": [], + "unit": "s", "color": { - "mode": "thresholds" + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 10, + "gradientMode": "none", + "showPoints": "auto", + "pointSize": 5, + "spanNulls": false, + "axisCenteredZero": false, + "axisLabel": "", + "scaleDistribution": { + "type": "linear" + } } }, "overrides": [] }, "options": { - "reduceOptions": { + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "rate(foxhunt_alerts_total{severity=\"critical\"}[5m])", - "instant": true - } - ] - }, - { - "id": 16, - "type": "gauge", - "title": "Position HHI", - "gridPos": { - "h": 4, - "w": 6, - "x": 18, - "y": 13 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.25 - }, - { - "color": "red", - "value": 0.5 - } - ] - }, - "unit": "percentunit", - "min": 0, - "max": 1, - "mappings": [], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_ensemble_agreement_rate{symbol=~\"$symbol\"}", - "instant": true - } - ] - }, - { - "id": 17, - "type": "row", - "title": "ML Model Health", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 17 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 18, - "type": "timeseries", - "title": "Inference Latency p95", - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 18 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 10, - "gradientMode": "none", - "spanNulls": false, - "showPoints": "never", - "pointSize": 5, - "stacking": { - "mode": "none", - "group": "A" - }, - "axisPlacement": "auto", - "axisLabel": "", - "scaleDistribution": { - "type": "linear" - } - }, - "unit": "s", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "mappings": [], - "color": { - "mode": "palette-classic" - } - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "list", - "placement": "bottom", - "calcs": [] + "lastNotNull", + "mean", + "max" + ] } }, "targets": [ @@ -1195,257 +884,8 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.95, rate(foxhunt_model_inference_latency_bucket[5m]))", - "legendFormat": "{{model_id}}", - "instant": false, - "range": true - } - ] - }, - { - "id": 19, - "type": "timeseries", - "title": "Model Drift", - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 18 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 10, - "gradientMode": "none", - "spanNulls": false, - "showPoints": "never", - "pointSize": 5, - "stacking": { - "mode": "none", - "group": "A" - }, - "axisPlacement": "auto", - "axisLabel": "", - "scaleDistribution": { - "type": "linear" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 0.5 - } - ] - }, - "mappings": [], - "color": { - "mode": "palette-classic" - } - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "list", - "placement": "bottom", - "calcs": [] - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_model_drift_score{model_id=~\"$model\"}", - "legendFormat": "{{model_id}}", - "instant": false, - "range": true - } - ] - }, - { - "id": 20, - "type": "row", - "title": "Latency", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 24 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 21, - "type": "timeseries", - "title": "gRPC Latency p95", - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 25 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 10, - "gradientMode": "none", - "spanNulls": false, - "showPoints": "never", - "pointSize": 5, - "stacking": { - "mode": "none", - "group": "A" - }, - "axisPlacement": "auto", - "axisLabel": "", - "scaleDistribution": { - "type": "linear" - } - }, - "unit": "s", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "mappings": [], - "color": { - "mode": "palette-classic" - } - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "list", - "placement": "bottom", - "calcs": [] - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "histogram_quantile(0.95, rate(grpc_server_handling_seconds_bucket{service=~\"$service\"}[5m]))", - "legendFormat": "{{service}}", - "instant": false, - "range": true - } - ] - }, - { - "id": 22, - "type": "timeseries", - "title": "ML Inference Distribution", - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 25 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 10, - "gradientMode": "none", - "spanNulls": false, - "showPoints": "never", - "pointSize": 5, - "stacking": { - "mode": "none", - "group": "A" - }, - "axisPlacement": "auto", - "axisLabel": "", - "scaleDistribution": { - "type": "linear" - } - }, - "unit": "s", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "mappings": [], - "color": { - "mode": "palette-classic" - } - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "list", - "placement": "bottom", - "calcs": [] - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "histogram_quantile(0.50, rate(foxhunt_model_inference_latency_bucket[5m]))", - "legendFormat": "p50", - "instant": false, - "range": true + "expr": "histogram_quantile(0.5, rate(trading_order_processing_seconds_bucket[$interval]))", + "legendFormat": "p50" }, { "refId": "B", @@ -1453,10 +893,8 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.95, rate(foxhunt_model_inference_latency_bucket[5m]))", - "legendFormat": "p95", - "instant": false, - "range": true + "expr": "histogram_quantile(0.95, rate(trading_order_processing_seconds_bucket[$interval]))", + "legendFormat": "p95" }, { "refId": "C", @@ -1464,12 +902,432 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.99, rate(foxhunt_model_inference_latency_bucket[5m]))", - "legendFormat": "p99", - "instant": false, - "range": true + "expr": "histogram_quantile(0.99, rate(trading_order_processing_seconds_bucket[$interval]))", + "legendFormat": "p99" + } + ] + }, + { + "id": 16, + "type": "timeseries", + "title": "Risk Check Latency", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 20 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 10, + "gradientMode": "none", + "showPoints": "auto", + "pointSize": 5, + "spanNulls": false, + "axisCenteredZero": false, + "axisLabel": "", + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "mean", + "max" + ] + } + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.5, rate(trading_risk_check_seconds_bucket[$interval]))", + "legendFormat": "p50" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.95, rate(trading_risk_check_seconds_bucket[$interval]))", + "legendFormat": "p95" + }, + { + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.99, rate(trading_risk_check_seconds_bucket[$interval]))", + "legendFormat": "p99" + } + ] + }, + { + "id": 17, + "type": "row", + "title": "API Gateway Trading Path", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 28 + }, + "collapsed": false, + "panels": [] + }, + { + "id": 18, + "type": "timeseries", + "title": "Trading Endpoint Latency", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 29 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 10, + "gradientMode": "none", + "showPoints": "auto", + "pointSize": 5, + "spanNulls": false, + "axisCenteredZero": false, + "axisLabel": "", + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "mean", + "max" + ] + } + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.99, rate(api_gateway_request_duration_seconds_bucket{handler=~\".*trade.*|.*order.*\"}[$interval]))", + "legendFormat": "{{handler}} p99" + } + ] + }, + { + "id": 19, + "type": "timeseries", + "title": "Trading Rate Limit", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 29 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15, + "gradientMode": "none", + "showPoints": "auto", + "pointSize": 5, + "spanNulls": false, + "axisCenteredZero": false, + "axisLabel": "", + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "rejected" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "allowed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "mean" + ] + } + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(api_gateway_rate_limit_rejected_total{tier=\"trading\"}[$interval])", + "legendFormat": "rejected" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(api_gateway_rate_limit_allowed_total{tier=\"trading\"}[$interval])", + "legendFormat": "allowed" + } + ] + }, + { + "id": 20, + "type": "row", + "title": "gRPC Services", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 37 + }, + "collapsed": false, + "panels": [] + }, + { + "id": 21, + "type": "timeseries", + "title": "gRPC Trading Calls", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 38 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 10, + "gradientMode": "none", + "showPoints": "auto", + "pointSize": 5, + "spanNulls": false, + "axisCenteredZero": false, + "axisLabel": "", + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "mean" + ] + } + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (grpc_method) (rate(grpc_server_handled_total{grpc_service=~\".*trading.*|.*ml.*\"}[$interval]))", + "legendFormat": "{{grpc_method}}" + } + ] + }, + { + "id": 22, + "type": "timeseries", + "title": "gRPC Error Rate", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 38 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15, + "gradientMode": "none", + "showPoints": "auto", + "pointSize": 5, + "spanNulls": false, + "axisCenteredZero": false, + "axisLabel": "", + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "options": { + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "sum" + ] + } + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (grpc_service, grpc_code) (rate(grpc_server_handled_total{grpc_code!=\"OK\"}[$interval]))", + "legendFormat": "{{grpc_service}} {{grpc_code}}" } ] } - ] + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + } } diff --git a/infra/k8s/monitoring/dashboards/foxhunt-training-cockpit.json b/infra/k8s/monitoring/dashboards/foxhunt-training-cockpit.json deleted file mode 100644 index 46e6422ff..000000000 --- a/infra/k8s/monitoring/dashboards/foxhunt-training-cockpit.json +++ /dev/null @@ -1,2041 +0,0 @@ -{ - "__inputs": [ - { - "name": "DS_PROMETHEUS", - "label": "Prometheus", - "description": "", - "type": "datasource", - "pluginId": "prometheus" - } - ], - "id": null, - "uid": "foxhunt-training-cockpit", - "title": "Training Cockpit", - "description": "ML training job monitoring: loss curves, GPU resources, model quality, checkpoints, data pipeline, errors, job history", - "tags": [ - "foxhunt", - "training", - "ml" - ], - "timezone": "browser", - "schemaVersion": 39, - "version": 1, - "editable": true, - "refresh": "10s", - "time": { - "from": "now-6h", - "to": "now" - }, - "fiscalYearStartMonth": 0, - "liveNow": false, - "weekStart": "", - "templating": { - "list": [ - { - "name": "model_type", - "type": "query", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "query": "label_values(foxhunt_training_epoch_loss, model)", - "refresh": 2, - "includeAll": true, - "multi": true, - "allValue": ".*", - "current": { - "selected": true, - "text": "All", - "value": "$__all" - }, - "sort": 1 - }, - { - "name": "interval", - "type": "interval", - "query": "1m,5m,15m,30m,1h", - "current": { - "selected": true, - "text": "5m", - "value": "5m" - }, - "auto": false - } - ] - }, - "panels": [ - { - "id": 1, - "type": "row", - "title": "Active Jobs", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 2, - "type": "stat", - "title": "Running Jobs", - "gridPos": { - "h": 4, - "w": 4, - "x": 0, - "y": 1 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - }, - "mappings": [], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "kube_job_status_active{namespace=\"foxhunt\", job_name=~\"training-.*\"}", - "instant": true - } - ] - }, - { - "id": 3, - "type": "stat", - "title": "Active Workers", - "gridPos": { - "h": 4, - "w": 4, - "x": 4, - "y": 1 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - }, - "mappings": [], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_training_active_workers", - "instant": true - } - ] - }, - { - "id": 4, - "type": "stat", - "title": "Jobs Started", - "gridPos": { - "h": 4, - "w": 4, - "x": 8, - "y": 1 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "mappings": [], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_training_jobs_started_total", - "instant": true - } - ] - }, - { - "id": 5, - "type": "stat", - "title": "Jobs Completed", - "gridPos": { - "h": 4, - "w": 4, - "x": 12, - "y": 1 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "mappings": [], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_training_jobs_completed_total", - "instant": true - } - ] - }, - { - "id": 6, - "type": "stat", - "title": "Failed Jobs", - "gridPos": { - "h": 4, - "w": 4, - "x": 16, - "y": 1 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - }, - "mappings": [], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_training_errors_total", - "instant": true - } - ] - }, - { - "id": 7, - "type": "stat", - "title": "Pending Jobs", - "gridPos": { - "h": 4, - "w": 4, - "x": 20, - "y": 1 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "yellow", - "value": null - } - ] - }, - "mappings": [], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "kube_job_status_active{namespace=\"foxhunt\", job_name=~\"training-.*\"} - kube_job_status_succeeded{namespace=\"foxhunt\", job_name=~\"training-.*\"}", - "instant": true - } - ] - }, - { - "id": 8, - "type": "row", - "title": "Training Progress", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 5 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 9, - "type": "timeseries", - "title": "Loss Curves", - "description": "Training and validation loss over time by model type", - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 6 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 10, - "gradientMode": "scheme", - "spanNulls": false, - "showPoints": "never", - "axisCenteredZero": false, - "axisLabel": "Loss" - }, - "thresholds": { - "mode": "off", - "steps": [] - }, - "mappings": [], - "color": { - "mode": "palette-classic" - }, - "unit": "none" - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "lastNotNull", - "min" - ] - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_training_epoch_loss{model=~\"$model_type\"}", - "legendFormat": "{{model}} - train", - "instant": false, - "range": true - }, - { - "refId": "B", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_training_validation_loss{model=~\"$model_type\"}", - "legendFormat": "{{model}} - val", - "instant": false, - "range": true - } - ] - }, - { - "id": 10, - "type": "gauge", - "title": "Epoch Progress", - "gridPos": { - "h": 7, - "w": 4, - "x": 12, - "y": 6 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "yellow", - "value": 25 - }, - { - "color": "green", - "value": 75 - } - ] - }, - "mappings": [], - "color": { - "mode": "thresholds" - }, - "min": 0, - "max": 100, - "unit": "percent" - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_training_current_epoch{model=~\"$model_type\"}", - "instant": true - } - ] - }, - { - "id": 11, - "type": "stat", - "title": "Convergence Rate", - "gridPos": { - "h": 3, - "w": 4, - "x": 16, - "y": 6 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "mappings": [], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "area", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "deriv(foxhunt_training_epoch_loss{model=~\"$model_type\"}[10m])", - "instant": true - } - ] - }, - { - "id": 12, - "type": "stat", - "title": "Training Speed", - "description": "Epochs per second", - "gridPos": { - "h": 4, - "w": 4, - "x": 16, - "y": 9 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "mappings": [], - "color": { - "mode": "thresholds" - }, - "unit": "ops" - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "area", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_training_batches_per_second{model=~\"$model_type\"}", - "instant": true - } - ] - }, - { - "id": 13, - "type": "row", - "title": "GPU Resources", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 13 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 14, - "type": "timeseries", - "title": "GPU Utilization", - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 14 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 20, - "gradientMode": "scheme", - "spanNulls": false, - "showPoints": "never", - "axisCenteredZero": false, - "stacking": { - "mode": "none" - } - }, - "thresholds": { - "mode": "off", - "steps": [] - }, - "mappings": [], - "color": { - "mode": "palette-classic" - }, - "min": 0, - "max": 100, - "unit": "percent" - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "list", - "placement": "bottom" - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "dcgm_gpu_utilization{gpu=\"0\"}", - "legendFormat": "GPU Core", - "instant": false, - "range": true - }, - { - "refId": "B", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "dcgm_mem_copy_utilization{gpu=\"0\"}", - "legendFormat": "Memory Copy", - "instant": false, - "range": true - } - ] - }, - { - "id": 15, - "type": "timeseries", - "title": "GPU VRAM", - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 14 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 20, - "gradientMode": "scheme", - "spanNulls": false, - "showPoints": "never", - "axisCenteredZero": false, - "stacking": { - "mode": "none" - } - }, - "thresholds": { - "mode": "off", - "steps": [] - }, - "mappings": [], - "color": { - "mode": "palette-classic" - }, - "unit": "decmbytes" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Total" - }, - "properties": [ - { - "id": "custom.lineStyle", - "value": { - "fill": "dash", - "dash": [ - 10, - 10 - ] - } - }, - { - "id": "custom.fillOpacity", - "value": 0 - }, - { - "id": "color", - "value": { - "fixedColor": "white", - "mode": "fixed" - } - } - ] - } - ] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "list", - "placement": "bottom" - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "dcgm_fb_used{gpu=\"0\"}", - "legendFormat": "Used", - "instant": false, - "range": true - }, - { - "refId": "B", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "dcgm_fb_used{gpu=\"0\"} + dcgm_fb_free{gpu=\"0\"}", - "legendFormat": "Total", - "instant": false, - "range": true - } - ] - }, - { - "id": 16, - "type": "timeseries", - "title": "GPU Temperature", - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 20 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 15, - "gradientMode": "scheme", - "spanNulls": false, - "showPoints": "never", - "axisCenteredZero": false, - "axisLabel": "Celsius", - "thresholdsStyle": { - "mode": "line" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 85 - } - ] - }, - "mappings": [], - "color": { - "mode": "palette-classic" - }, - "unit": "celsius" - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "list", - "placement": "bottom" - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "dcgm_gpu_temp", - "legendFormat": "GPU Temp", - "instant": false, - "range": true - } - ] - }, - { - "id": 17, - "type": "timeseries", - "title": "GPU Power", - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 20 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 15, - "gradientMode": "scheme", - "spanNulls": false, - "showPoints": "never", - "axisCenteredZero": false, - "axisLabel": "Watts" - }, - "thresholds": { - "mode": "off", - "steps": [] - }, - "mappings": [], - "color": { - "mode": "palette-classic" - }, - "unit": "watt" - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "list", - "placement": "bottom" - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "dcgm_power_usage", - "legendFormat": "GPU Power", - "instant": false, - "range": true - } - ] - }, - { - "id": 18, - "type": "row", - "title": "Model Quality", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 26 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 19, - "type": "timeseries", - "title": "Win Rate / Directional Accuracy", - "description": "Win rate (RL models) or directional accuracy (supervised models)", - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 27 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 10, - "gradientMode": "scheme", - "spanNulls": false, - "showPoints": "never", - "axisCenteredZero": false, - "axisLabel": "Rate" - }, - "thresholds": { - "mode": "off", - "steps": [] - }, - "mappings": [], - "color": { - "mode": "palette-classic" - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "lastNotNull", - "max" - ] - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_training_eval_accuracy{model=~\"$model_type\"}", - "legendFormat": "{{model}}", - "instant": false, - "range": true - } - ] - }, - { - "id": 20, - "type": "timeseries", - "title": "Sharpe / Profit Factor / Return", - "description": "Sharpe ratio, profit factor, and total return per fold", - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 27 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 10, - "gradientMode": "scheme", - "spanNulls": false, - "showPoints": "never", - "axisCenteredZero": false, - "axisLabel": "Score" - }, - "thresholds": { - "mode": "off", - "steps": [] - }, - "mappings": [], - "color": { - "mode": "palette-classic" - }, - "unit": "none" - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "lastNotNull", - "max" - ] - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_training_eval_f1{model=~\"$model_type\"}", - "legendFormat": "{{model}} - Return", - "instant": false, - "range": true - }, - { - "refId": "B", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_training_eval_precision{model=~\"$model_type\"}", - "legendFormat": "{{model}} - Sharpe", - "instant": false, - "range": true - }, - { - "refId": "C", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_training_eval_recall{model=~\"$model_type\"}", - "legendFormat": "{{model}} - Profit Factor", - "instant": false, - "range": true - } - ] - }, - { - "id": 21, - "type": "row", - "title": "Checkpoints", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 33 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 22, - "type": "stat", - "title": "Total Saves", - "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 34 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "mappings": [], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_training_checkpoint_saves_total{model=~\"$model_type\"}", - "instant": true - } - ] - }, - { - "id": 23, - "type": "stat", - "title": "Save Failures", - "gridPos": { - "h": 4, - "w": 6, - "x": 6, - "y": 34 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - }, - "mappings": [], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_training_checkpoint_failures_total{model=~\"$model_type\"}", - "instant": true - } - ] - }, - { - "id": 24, - "type": "stat", - "title": "Save Duration p95", - "gridPos": { - "h": 4, - "w": 6, - "x": 12, - "y": 34 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 10 - }, - { - "color": "red", - "value": 30 - } - ] - }, - "mappings": [], - "color": { - "mode": "thresholds" - }, - "unit": "s" - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "histogram_quantile(0.95, rate(foxhunt_training_checkpoint_duration_seconds_bucket[5m]))", - "instant": true - } - ] - }, - { - "id": 25, - "type": "stat", - "title": "Checkpoint Size", - "gridPos": { - "h": 4, - "w": 6, - "x": 18, - "y": 34 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - } - ] - }, - "mappings": [], - "color": { - "mode": "thresholds" - }, - "unit": "bytes" - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_training_checkpoint_size_bytes{model=~\"$model_type\"}", - "instant": true - } - ] - }, - { - "id": 26, - "type": "row", - "title": "Data Pipeline", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 38 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 27, - "type": "timeseries", - "title": "Batches Processed", - "description": "Rate of training data batches processed per second", - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 39 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 15, - "gradientMode": "scheme", - "spanNulls": false, - "showPoints": "never", - "axisCenteredZero": false, - "axisLabel": "Batches/s" - }, - "thresholds": { - "mode": "off", - "steps": [] - }, - "mappings": [], - "color": { - "mode": "palette-classic" - }, - "unit": "ops" - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "list", - "placement": "bottom" - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_training_batches_processed{model=~\"$model_type\"}", - "legendFormat": "Batches/s", - "instant": false, - "range": true - } - ] - }, - { - "id": 28, - "type": "timeseries", - "title": "Data Loading Latency", - "description": "Time spent loading training data batches", - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 39 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 15, - "gradientMode": "scheme", - "spanNulls": false, - "showPoints": "never", - "axisCenteredZero": false, - "axisLabel": "Seconds" - }, - "thresholds": { - "mode": "off", - "steps": [] - }, - "mappings": [], - "color": { - "mode": "palette-classic" - }, - "unit": "s" - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "list", - "placement": "bottom" - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "histogram_quantile(0.95, rate(foxhunt_training_data_load_seconds_bucket[5m]))", - "legendFormat": "Load Latency", - "instant": false, - "range": true - } - ] - }, - { - "id": 29, - "type": "stat", - "title": "Feature Errors", - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 39 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - }, - "mappings": [], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_training_feature_errors_total{model=~\"$model_type\"}", - "instant": true - } - ] - }, - { - "id": 30, - "type": "row", - "title": "Errors", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 45 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 31, - "type": "stat", - "title": "NaN Detections", - "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 46 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - }, - "mappings": [], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_training_nan_detected_total{model=~\"$model_type\"}", - "instant": true - } - ] - }, - { - "id": 32, - "type": "stat", - "title": "Gradient Explosions", - "gridPos": { - "h": 4, - "w": 6, - "x": 6, - "y": 46 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - }, - "mappings": [], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_training_gradient_explosion_total{model=~\"$model_type\"}", - "instant": true - } - ] - }, - { - "id": 33, - "type": "timeseries", - "title": "Training Failures", - "description": "Hourly increase of training failures by error type", - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 46 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "bars", - "lineWidth": 1, - "fillOpacity": 80, - "gradientMode": "none", - "spanNulls": false, - "showPoints": "never", - "axisCenteredZero": false, - "axisLabel": "Failures", - "stacking": { - "mode": "normal", - "group": "A" - } - }, - "thresholds": { - "mode": "off", - "steps": [] - }, - "mappings": [], - "color": { - "mode": "palette-classic" - } - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "sum" - ] - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_training_errors_total", - "legendFormat": "{{error_type}}", - "instant": false, - "range": true - } - ] - }, - { - "id": 34, - "type": "row", - "title": "Job History", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 52 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 35, - "type": "timeseries", - "title": "Job Duration Distribution", - "description": "Training job durations over time by model type", - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 53 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "points", - "lineWidth": 0, - "fillOpacity": 0, - "spanNulls": false, - "showPoints": "always", - "pointSize": 6, - "axisCenteredZero": false, - "axisLabel": "Duration" - }, - "thresholds": { - "mode": "off", - "steps": [] - }, - "mappings": [], - "color": { - "mode": "palette-classic" - }, - "unit": "s" - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "max" - ] - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "(kube_job_complete_time{namespace=\"foxhunt\", job_name=~\"training-.*\"} - kube_job_start_time{namespace=\"foxhunt\", job_name=~\"training-.*\"}) / 3600", - "legendFormat": "{{model}}", - "instant": false, - "range": true - } - ] - }, - { - "id": 36, - "type": "timeseries", - "title": "Iteration Duration", - "description": "Per-iteration training time by model type", - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 53 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 10, - "gradientMode": "scheme", - "spanNulls": false, - "showPoints": "never", - "axisCenteredZero": false, - "axisLabel": "Seconds" - }, - "thresholds": { - "mode": "off", - "steps": [] - }, - "mappings": [], - "color": { - "mode": "palette-classic" - }, - "unit": "s" - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "max", - "lastNotNull" - ] - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "foxhunt_training_iteration_seconds{model=~\"$model_type\"}", - "legendFormat": "{{model}}", - "instant": false, - "range": true - } - ] - } - ] -} diff --git a/infra/k8s/monitoring/dashboards/foxhunt-training.json b/infra/k8s/monitoring/dashboards/foxhunt-training.json new file mode 100644 index 000000000..771e2edf1 --- /dev/null +++ b/infra/k8s/monitoring/dashboards/foxhunt-training.json @@ -0,0 +1,1284 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "Prometheus datasource for foxhunt metrics", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "10.0.0" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + }, + { + "type": "panel", + "id": "gauge", + "name": "Gauge", + "version": "" + } + ], + "uid": "foxhunt-training", + "title": "Training & CI/CD", + "description": "ML model training, hyperopt, GPU utilization, and Argo CI pipelines for the foxhunt HFT platform", + "tags": ["foxhunt", "training", "gpu", "ci"], + "editable": true, + "graphTooltip": 1, + "refresh": "15s", + "schemaVersion": 39, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "templating": { + "list": [ + { + "name": "interval", + "label": "Interval", + "type": "interval", + "auto": true, + "auto_count": 30, + "auto_min": "1m", + "current": { + "selected": true, + "text": "auto", + "value": "$__auto_interval_interval" + }, + "options": [ + { "selected": false, "text": "1m", "value": "1m" }, + { "selected": false, "text": "5m", "value": "5m" }, + { "selected": false, "text": "15m", "value": "15m" }, + { "selected": false, "text": "30m", "value": "30m" }, + { "selected": false, "text": "1h", "value": "1h" } + ], + "query": "1m,5m,15m,30m,1h" + } + ] + }, + "panels": [ + { + "type": "row", + "title": "Training Status", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "collapsed": false, + "panels": [] + }, + { + "id": 1, + "type": "stat", + "title": "Active Workers", + "gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "targets": [ + { + "expr": "ml_training_active_workers", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "blue", "value": null }, + { "color": "green", "value": 1 } + ] + }, + "mappings": [], + "unit": "none" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"] }, + "colorMode": "background", + "graphMode": "none" + } + }, + { + "id": 2, + "type": "stat", + "title": "Jobs Started", + "gridPos": { "h": 4, "w": 4, "x": 4, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "targets": [ + { + "expr": "ml_training_jobs_started_total", + "legendFormat": "started", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "blue", "value": null } + ] + }, + "mappings": [], + "unit": "none" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"] }, + "colorMode": "background", + "graphMode": "none" + } + }, + { + "id": 3, + "type": "stat", + "title": "Jobs Completed", + "gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "targets": [ + { + "expr": "ml_training_jobs_completed_total", + "legendFormat": "completed", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "mappings": [], + "unit": "none" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"] }, + "colorMode": "background", + "graphMode": "none" + } + }, + { + "id": 4, + "type": "stat", + "title": "Training Errors", + "gridPos": { "h": 4, "w": 4, "x": 12, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "targets": [ + { + "expr": "ml_training_errors_total", + "legendFormat": "errors", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "red", "value": 1 } + ] + }, + "mappings": [], + "unit": "none" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"] }, + "colorMode": "background", + "graphMode": "none" + } + }, + { + "id": 5, + "type": "stat", + "title": "Current Epoch", + "gridPos": { "h": 4, "w": 4, "x": 16, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "targets": [ + { + "expr": "foxhunt_training_current_epoch", + "legendFormat": "epoch", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "purple", "value": null } + ] + }, + "mappings": [], + "unit": "none" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"] }, + "colorMode": "background", + "graphMode": "none" + } + }, + { + "id": 6, + "type": "stat", + "title": "Hyperopt Mode", + "gridPos": { "h": 4, "w": 4, "x": 20, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "targets": [ + { + "expr": "foxhunt_hyperopt_mode", + "legendFormat": "{{mode}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "orange", "value": null } + ] + }, + "mappings": [ + { "type": "value", "options": { "0": { "text": "Idle", "color": "text" }, "1": { "text": "PSO", "color": "orange" }, "2": { "text": "Grid", "color": "blue" } } } + ], + "unit": "none" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"] }, + "colorMode": "background", + "graphMode": "none", + "textMode": "value_and_name" + } + }, + { + "type": "row", + "title": "Training Curves", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, + "collapsed": false, + "panels": [] + }, + { + "id": 7, + "type": "timeseries", + "title": "Epoch Loss", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "targets": [ + { + "expr": "foxhunt_training_epoch_loss", + "legendFormat": "{{model}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 10, + "gradientMode": "scheme", + "spanNulls": false, + "lineWidth": 2, + "pointSize": 5, + "showPoints": "auto", + "axisLabel": "Loss", + "scaleDistribution": { "type": "linear" } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "none", + "color": { "mode": "palette-classic" } + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["min", "max", "last"] } + } + }, + { + "id": 8, + "type": "timeseries", + "title": "Epoch Sharpe Ratio", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "targets": [ + { + "expr": "foxhunt_training_epoch_sharpe", + "legendFormat": "{{model}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 10, + "gradientMode": "scheme", + "spanNulls": false, + "lineWidth": 2, + "pointSize": 5, + "showPoints": "auto", + "axisLabel": "Sharpe", + "scaleDistribution": { "type": "linear" }, + "thresholdsStyle": { "mode": "line" } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "green", "value": 0 } + ] + }, + "unit": "none", + "color": { "mode": "palette-classic" } + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["min", "max", "last"] } + } + }, + { + "type": "row", + "title": "Training Quality", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 13 }, + "collapsed": false, + "panels": [] + }, + { + "id": 9, + "type": "timeseries", + "title": "Win Rate", + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 14 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "targets": [ + { + "expr": "foxhunt_training_epoch_win_rate", + "legendFormat": "{{model}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 10, + "gradientMode": "scheme", + "spanNulls": false, + "lineWidth": 2, + "pointSize": 5, + "showPoints": "auto", + "axisLabel": "Win Rate", + "scaleDistribution": { "type": "linear" } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "yellow", "value": 40 }, + { "color": "green", "value": 55 } + ] + }, + "unit": "percent", + "min": 0, + "max": 100, + "color": { "mode": "palette-classic" } + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["last"] } + } + }, + { + "id": 10, + "type": "timeseries", + "title": "Learning Rate", + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 14 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "targets": [ + { + "expr": "foxhunt_training_learning_rate", + "legendFormat": "{{model}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 5, + "spanNulls": false, + "lineWidth": 2, + "pointSize": 5, + "showPoints": "auto", + "axisLabel": "LR", + "scaleDistribution": { "type": "log", "log": 10 } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "blue", "value": null } + ] + }, + "unit": "none", + "color": { "mode": "palette-classic" } + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["last"] } + } + }, + { + "id": 11, + "type": "timeseries", + "title": "Gradient Norm", + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 14 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "targets": [ + { + "expr": "foxhunt_training_gradient_norm", + "legendFormat": "{{model}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 10, + "spanNulls": false, + "lineWidth": 2, + "pointSize": 5, + "showPoints": "auto", + "axisLabel": "Grad Norm", + "scaleDistribution": { "type": "linear" }, + "thresholdsStyle": { "mode": "line+area" } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 10 }, + { "color": "red", "value": 100 } + ] + }, + "unit": "none", + "color": { "mode": "palette-classic" } + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["max", "last"] } + } + }, + { + "type": "row", + "title": "Hyperopt Progress", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 21 }, + "collapsed": false, + "panels": [] + }, + { + "id": 12, + "type": "gauge", + "title": "Trial Progress", + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 22 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "targets": [ + { + "expr": "foxhunt_hyperopt_trial_current / foxhunt_hyperopt_trial_total * 100", + "legendFormat": "progress", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "orange", "value": 25 }, + { "color": "yellow", "value": 50 }, + { "color": "light-green", "value": 75 }, + { "color": "green", "value": 95 } + ] + }, + "unit": "percent", + "min": 0, + "max": 100 + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"] }, + "showThresholdLabels": false, + "showThresholdMarkers": true + } + }, + { + "id": 13, + "type": "timeseries", + "title": "Best Objective", + "gridPos": { "h": 8, "w": 16, "x": 8, "y": 22 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "targets": [ + { + "expr": "foxhunt_training_best_objective", + "legendFormat": "{{model}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "stepAfter", + "fillOpacity": 15, + "gradientMode": "scheme", + "spanNulls": true, + "lineWidth": 2, + "pointSize": 5, + "showPoints": "auto", + "axisLabel": "Objective", + "scaleDistribution": { "type": "linear" } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "none", + "color": { "mode": "palette-classic" } + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["min", "max", "last"] } + } + }, + { + "type": "row", + "title": "GPU Metrics", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 29 }, + "collapsed": false, + "panels": [] + }, + { + "id": 14, + "type": "timeseries", + "title": "GPU Utilization", + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 30 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "targets": [ + { + "expr": "DCGM_FI_DEV_GPU_UTIL", + "legendFormat": "{{Hostname}} gpu{{gpu}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 20, + "gradientMode": "scheme", + "spanNulls": false, + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "axisLabel": "Utilization", + "scaleDistribution": { "type": "linear" } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "yellow", "value": 50 }, + { "color": "green", "value": 80 } + ] + }, + "unit": "percent", + "min": 0, + "max": 100, + "color": { "mode": "palette-classic" } + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] } + } + }, + { + "id": 15, + "type": "timeseries", + "title": "GPU Memory Used", + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 30 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "targets": [ + { + "expr": "DCGM_FI_DEV_FB_USED / (DCGM_FI_DEV_FB_USED + DCGM_FI_DEV_FB_FREE) * 100", + "legendFormat": "{{Hostname}} gpu{{gpu}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 20, + "gradientMode": "scheme", + "spanNulls": false, + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "axisLabel": "Memory %", + "scaleDistribution": { "type": "linear" } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 75 }, + { "color": "red", "value": 90 } + ] + }, + "unit": "percent", + "min": 0, + "max": 100, + "color": { "mode": "palette-classic" } + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] } + } + }, + { + "id": 16, + "type": "timeseries", + "title": "GPU Temperature", + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 30 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "targets": [ + { + "expr": "DCGM_FI_DEV_GPU_TEMP", + "legendFormat": "{{Hostname}} gpu{{gpu}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 10, + "spanNulls": false, + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "axisLabel": "Temp", + "scaleDistribution": { "type": "linear" }, + "thresholdsStyle": { "mode": "line+area" } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "orange", "value": 80 }, + { "color": "red", "value": 90 } + ] + }, + "unit": "celsius", + "color": { "mode": "palette-classic" } + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] } + } + }, + { + "type": "row", + "title": "GPU Detail", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 37 }, + "collapsed": false, + "panels": [] + }, + { + "id": 17, + "type": "timeseries", + "title": "GPU Power Draw", + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 38 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "targets": [ + { + "expr": "DCGM_FI_DEV_POWER_USAGE", + "legendFormat": "{{Hostname}} gpu{{gpu}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 15, + "gradientMode": "scheme", + "spanNulls": false, + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "axisLabel": "Power", + "scaleDistribution": { "type": "linear" } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "watt", + "color": { "mode": "palette-classic" } + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] } + } + }, + { + "id": 18, + "type": "timeseries", + "title": "SM Clock", + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 38 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "targets": [ + { + "expr": "DCGM_FI_DEV_SM_CLOCK", + "legendFormat": "{{Hostname}} gpu{{gpu}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 10, + "spanNulls": false, + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "axisLabel": "Clock", + "scaleDistribution": { "type": "linear" } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "blue", "value": null } + ] + }, + "unit": "clockHz", + "color": { "mode": "palette-classic" } + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "last"] } + } + }, + { + "id": 19, + "type": "timeseries", + "title": "PCIe TX/RX", + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 38 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "targets": [ + { + "expr": "DCGM_FI_DEV_PCIE_TX_THROUGHPUT", + "legendFormat": "TX {{Hostname}} gpu{{gpu}}", + "refId": "A" + }, + { + "expr": "DCGM_FI_DEV_PCIE_RX_THROUGHPUT", + "legendFormat": "RX {{Hostname}} gpu{{gpu}}", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 10, + "spanNulls": false, + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "axisLabel": "Throughput", + "scaleDistribution": { "type": "linear" } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "Bps", + "color": { "mode": "palette-classic" } + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] } + } + }, + { + "type": "row", + "title": "Argo CI Pipelines", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 45 }, + "collapsed": false, + "panels": [] + }, + { + "id": 20, + "type": "stat", + "title": "Active Workflows", + "gridPos": { "h": 4, "w": 6, "x": 0, "y": 46 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "targets": [ + { + "expr": "argo_workflows_gauge{status=\"Running\"}", + "legendFormat": "running", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "blue", "value": null }, + { "color": "green", "value": 1 } + ] + }, + "mappings": [], + "unit": "none" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"] }, + "colorMode": "background", + "graphMode": "none" + } + }, + { + "id": 21, + "type": "stat", + "title": "Succeeded", + "gridPos": { "h": 4, "w": 6, "x": 6, "y": 46 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "targets": [ + { + "expr": "argo_workflows_gauge{status=\"Succeeded\"}", + "legendFormat": "succeeded", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "mappings": [], + "unit": "none" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"] }, + "colorMode": "background", + "graphMode": "none" + } + }, + { + "id": 22, + "type": "stat", + "title": "Failed", + "gridPos": { "h": 4, "w": 6, "x": 12, "y": 46 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "targets": [ + { + "expr": "argo_workflows_gauge{status=\"Failed\"}", + "legendFormat": "failed", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "red", "value": 1 } + ] + }, + "mappings": [], + "unit": "none" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"] }, + "colorMode": "background", + "graphMode": "none" + } + }, + { + "id": 23, + "type": "stat", + "title": "Pending", + "gridPos": { "h": 4, "w": 6, "x": 18, "y": 46 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "targets": [ + { + "expr": "argo_workflows_gauge{status=\"Pending\"}", + "legendFormat": "pending", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 1 } + ] + }, + "mappings": [], + "unit": "none" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"] }, + "colorMode": "background", + "graphMode": "none" + } + }, + { + "type": "row", + "title": "Argo Performance", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 50 }, + "collapsed": false, + "panels": [] + }, + { + "id": 24, + "type": "timeseries", + "title": "Operation Duration", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 51 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "targets": [ + { + "expr": "histogram_quantile(0.50, rate(argo_workflows_operation_duration_seconds_bucket[$interval]))", + "legendFormat": "p50", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, rate(argo_workflows_operation_duration_seconds_bucket[$interval]))", + "legendFormat": "p95", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 10, + "gradientMode": "scheme", + "spanNulls": false, + "lineWidth": 2, + "pointSize": 5, + "showPoints": "auto", + "axisLabel": "Duration", + "scaleDistribution": { "type": "linear" } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 30 }, + { "color": "red", "value": 60 } + ] + }, + "unit": "s", + "color": { "mode": "palette-classic" } + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "p95" }, + "properties": [ + { "id": "custom.lineStyle", "value": { "fill": "dash", "dash": [10, 10] } } + ] + } + ] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] } + } + }, + { + "id": 25, + "type": "timeseries", + "title": "K8s API Requests", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 51 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "targets": [ + { + "expr": "topk(10, sum by (verb, kind) (rate(argo_workflows_k8s_request_total[$interval])))", + "legendFormat": "{{verb}} {{kind}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 10, + "gradientMode": "scheme", + "spanNulls": false, + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "axisLabel": "req/s", + "scaleDistribution": { "type": "linear" } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "reqps", + "color": { "mode": "palette-classic" } + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] } + } + }, + { + "type": "row", + "title": "Argo Internals", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 58 }, + "collapsed": false, + "panels": [] + }, + { + "id": 26, + "type": "timeseries", + "title": "Queue Depth", + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 59 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "targets": [ + { + "expr": "argo_workflows_queue_depth_gauge", + "legendFormat": "{{queue_name}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "stepAfter", + "fillOpacity": 20, + "gradientMode": "scheme", + "spanNulls": false, + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "axisLabel": "Depth", + "scaleDistribution": { "type": "linear" } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 50 }, + { "color": "red", "value": 100 } + ] + }, + "unit": "none", + "min": 0, + "color": { "mode": "palette-classic" } + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] } + } + }, + { + "id": 27, + "type": "timeseries", + "title": "Worker Busy", + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 59 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "targets": [ + { + "expr": "argo_workflows_workers_busy_count", + "legendFormat": "{{worker_type}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "stepAfter", + "fillOpacity": 20, + "gradientMode": "scheme", + "spanNulls": false, + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "axisLabel": "Busy", + "scaleDistribution": { "type": "linear" } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "none", + "min": 0, + "color": { "mode": "palette-classic" } + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] } + } + }, + { + "id": 28, + "type": "timeseries", + "title": "Error Count", + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 59 }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "targets": [ + { + "expr": "rate(argo_workflows_error_count[$interval])", + "legendFormat": "{{cause}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 15, + "gradientMode": "scheme", + "spanNulls": false, + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "axisLabel": "Errors/s", + "scaleDistribution": { "type": "linear" } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "red", "value": 0.1 } + ] + }, + "unit": "none", + "min": 0, + "color": { "mode": "palette-classic" } + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["sum", "max", "last"] } + } + } + ] +} \ No newline at end of file diff --git a/infra/k8s/monitoring/dashboards/gpu-overview.json b/infra/k8s/monitoring/dashboards/gpu-overview.json deleted file mode 100644 index c63aa267a..000000000 --- a/infra/k8s/monitoring/dashboards/gpu-overview.json +++ /dev/null @@ -1,841 +0,0 @@ -{ - "__inputs": [ - { - "name": "DS_PROMETHEUS", - "label": "Prometheus", - "description": "", - "type": "datasource", - "pluginId": "prometheus" - } - ], - "id": null, - "uid": "foxhunt-gpu-overview", - "title": "GPU Overview", - "description": "NVIDIA GPU metrics from DCGM exporter: utilization, memory, temperature, power", - "tags": [ - "gpu", - "nvidia", - "dcgm", - "foxhunt" - ], - "timezone": "browser", - "schemaVersion": 39, - "version": 1, - "refresh": "15s", - "time": { - "from": "now-1h", - "to": "now" - }, - "fiscalYearStartMonth": 0, - "liveNow": false, - "weekStart": "", - "templating": { - "list": [ - { - "name": "gpu", - "type": "query", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "query": "label_values(dcgm_gpu_utilization, gpu)", - "refresh": 2, - "includeAll": true, - "multi": true, - "allValue": ".*", - "current": { - "selected": true, - "text": "All", - "value": "$__all" - }, - "sort": 1 - }, - { - "name": "instance", - "type": "query", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "query": "label_values(dcgm_gpu_utilization, instance)", - "refresh": 2, - "includeAll": true, - "multi": true, - "allValue": ".*", - "current": { - "selected": true, - "text": "All", - "value": "$__all" - }, - "sort": 1 - }, - { - "name": "namespace", - "type": "query", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "query": "label_values(dcgm_gpu_utilization, exported_namespace)", - "refresh": 2, - "includeAll": true, - "multi": true, - "allValue": ".*", - "current": { - "selected": true, - "text": "All", - "value": "$__all" - }, - "sort": 1 - } - ] - }, - "panels": [ - { - "id": 1, - "type": "row", - "title": "GPU Utilization", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 2, - "type": "timeseries", - "title": "GPU Utilization %", - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 1 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 20, - "gradientMode": "scheme", - "spanNulls": false, - "showPoints": "never", - "stacking": { - "mode": "none", - "group": "A" - }, - "axisPlacement": "auto", - "lineWidth": 2 - }, - "unit": "percent", - "min": 0, - "max": 100, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 80 - }, - { - "color": "red", - "value": 95 - } - ] - }, - "color": { - "mode": "palette-classic" - }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "max", - "last" - ] - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "dcgm_gpu_utilization{gpu=~\"$gpu\", instance=~\"$instance\"}", - "legendFormat": "GPU {{ gpu }} ({{ instance }})" - } - ] - }, - { - "id": 3, - "type": "gauge", - "title": "Current GPU Utilization", - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 1 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - }, - { - "color": "green", - "value": 20 - }, - { - "color": "yellow", - "value": 80 - }, - { - "color": "red", - "value": 95 - } - ] - }, - "min": 0, - "max": 100, - "unit": "percent", - "mappings": [], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "orientation": "auto" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "dcgm_gpu_utilization{gpu=~\"$gpu\", instance=~\"$instance\"}", - "legendFormat": "GPU {{ gpu }}", - "instant": true - } - ] - }, - { - "id": 10, - "type": "row", - "title": "GPU Memory", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 9 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 11, - "type": "timeseries", - "title": "GPU Memory Used", - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 10 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 20, - "gradientMode": "scheme", - "spanNulls": false, - "showPoints": "never", - "stacking": { - "mode": "none", - "group": "A" - }, - "axisPlacement": "auto", - "lineWidth": 2 - }, - "unit": "decmbytes", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "color": { - "mode": "palette-classic" - }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "max", - "last" - ] - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "dcgm_fb_used{gpu=~\"$gpu\", instance=~\"$instance\"}", - "legendFormat": "GPU {{ gpu }} used" - }, - { - "refId": "B", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "(dcgm_fb_used{gpu=~\"$gpu\", instance=~\"$instance\"} + dcgm_fb_free{gpu=~\"$gpu\", instance=~\"$instance\"})", - "legendFormat": "GPU {{ gpu }} total" - } - ] - }, - { - "id": 12, - "type": "gauge", - "title": "GPU Memory Usage %", - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 10 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 70 - }, - { - "color": "red", - "value": 90 - } - ] - }, - "min": 0, - "max": 100, - "unit": "percent", - "mappings": [], - "color": { - "mode": "thresholds" - } - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "orientation": "auto" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "(dcgm_fb_used{gpu=~\"$gpu\", instance=~\"$instance\"} / (dcgm_fb_used{gpu=~\"$gpu\", instance=~\"$instance\"} + dcgm_fb_free{gpu=~\"$gpu\", instance=~\"$instance\"})) * 100", - "legendFormat": "GPU {{ gpu }}", - "instant": true - } - ] - }, - { - "id": 20, - "type": "row", - "title": "GPU Temperature & Power", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 18 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 21, - "type": "timeseries", - "title": "GPU Temperature", - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 19 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 10, - "gradientMode": "scheme", - "spanNulls": false, - "showPoints": "never", - "stacking": { - "mode": "none", - "group": "A" - }, - "axisPlacement": "auto", - "lineWidth": 2 - }, - "unit": "celsius", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - }, - { - "color": "green", - "value": 40 - }, - { - "color": "yellow", - "value": 75 - }, - { - "color": "red", - "value": 85 - } - ] - }, - "color": { - "mode": "palette-classic" - }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "max", - "last" - ] - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "dcgm_gpu_temp{gpu=~\"$gpu\", instance=~\"$instance\"}", - "legendFormat": "GPU {{ gpu }} ({{ instance }})" - } - ] - }, - { - "id": 22, - "type": "timeseries", - "title": "GPU Power Usage", - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 19 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 10, - "gradientMode": "scheme", - "spanNulls": false, - "showPoints": "never", - "stacking": { - "mode": "none", - "group": "A" - }, - "axisPlacement": "auto", - "lineWidth": 2 - }, - "unit": "watt", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "color": { - "mode": "palette-classic" - }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "max", - "last" - ] - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "dcgm_power_usage{gpu=~\"$gpu\", instance=~\"$instance\"}", - "legendFormat": "GPU {{ gpu }} ({{ instance }})" - } - ] - }, - { - "id": 30, - "type": "row", - "title": "GPU Clocks & Per-Pod Usage", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 27 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 31, - "type": "timeseries", - "title": "GPU Memory Clock", - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 28 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 10, - "gradientMode": "none", - "spanNulls": false, - "showPoints": "never", - "stacking": { - "mode": "none", - "group": "A" - }, - "axisPlacement": "auto", - "lineWidth": 2 - }, - "unit": "hertz", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "color": { - "mode": "palette-classic" - }, - "mappings": [], - "custom.transform": "negative-Y" - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "max", - "last" - ] - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "dcgm_memory_clock{gpu=~\"$gpu\", instance=~\"$instance\"} * 1000000", - "legendFormat": "GPU {{ gpu }} mem clock" - } - ] - }, - { - "id": 32, - "type": "timeseries", - "title": "Per-Pod GPU Utilization", - "description": "GPU utilization broken down by pod. Requires DCGM exporter pod-level labels (exported_pod, exported_namespace).", - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 28 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 15, - "gradientMode": "none", - "spanNulls": false, - "showPoints": "never", - "stacking": { - "mode": "none", - "group": "A" - }, - "axisPlacement": "auto", - "lineWidth": 2 - }, - "unit": "percent", - "min": 0, - "max": 100, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "color": { - "mode": "palette-classic" - }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "max", - "last" - ] - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "dcgm_gpu_utilization{exported_namespace=~\"$namespace\", gpu=~\"$gpu\", instance=~\"$instance\"}", - "legendFormat": "{{ exported_namespace }}/{{ exported_pod }} GPU {{ gpu }}" - } - ] - }, - { - "id": 33, - "type": "timeseries", - "title": "Per-Pod GPU Memory Used", - "description": "GPU framebuffer memory used per pod. Requires DCGM exporter pod-level labels.", - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 36 - }, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 15, - "gradientMode": "none", - "spanNulls": false, - "showPoints": "never", - "stacking": { - "mode": "none", - "group": "A" - }, - "axisPlacement": "auto", - "lineWidth": 2 - }, - "unit": "decmbytes", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "color": { - "mode": "palette-classic" - }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "max", - "last" - ] - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "dcgm_fb_used{exported_namespace=~\"$namespace\", gpu=~\"$gpu\", instance=~\"$instance\"}", - "legendFormat": "{{ exported_namespace }}/{{ exported_pod }} GPU {{ gpu }}" - } - ] - } - ] -} \ No newline at end of file diff --git a/infra/k8s/monitoring/dashboards/import.sh b/infra/k8s/monitoring/dashboards/import.sh index 1f64d877f..75465b11b 100755 --- a/infra/k8s/monitoring/dashboards/import.sh +++ b/infra/k8s/monitoring/dashboards/import.sh @@ -123,16 +123,20 @@ KUBE_NAMESPACE="${KUBE_NAMESPACE:-foxhunt}" # ConfigMap grouping: name -> list of dashboard files declare -A CONFIGMAP_GROUPS=( - [grafana-dashboards-cockpits]="foxhunt-cockpit.json foxhunt-trading-cockpit.json foxhunt-training-cockpit.json" - [grafana-dashboards-operations]="foxhunt-overview.json foxhunt-services.json foxhunt-traces.json foxhunt-logs.json" - [grafana-dashboards-infra]="cluster-overview.json gpu-overview.json foxhunt-infrastructure.json foxhunt-gpu-training.json" + [grafana-dashboards-cockpit]="foxhunt-cockpit.json" + [grafana-dashboards-trading]="foxhunt-trading.json" + [grafana-dashboards-training]="foxhunt-training.json" + [grafana-dashboards-infra]="foxhunt-infrastructure.json" + [grafana-dashboards-observability]="foxhunt-observability.json" ) # Folder mapping: ConfigMap name -> Grafana folder name (used by sidecar annotation) declare -A CONFIGMAP_FOLDERS=( - [grafana-dashboards-cockpits]="Cockpits" - [grafana-dashboards-operations]="Operations" - [grafana-dashboards-infra]="Infrastructure" + [grafana-dashboards-cockpit]="Foxhunt" + [grafana-dashboards-trading]="Foxhunt" + [grafana-dashboards-training]="Foxhunt" + [grafana-dashboards-infra]="Foxhunt" + [grafana-dashboards-observability]="Foxhunt" ) deploy_configmaps() { @@ -221,17 +225,11 @@ else echo "Skipping ConfigMap deploy (kubectl not available or SKIP_CONFIGMAP=1)." echo "Falling back to API import..." - for dashboard in "${SCRIPT_DIR}/cluster-overview.json" \ - "${SCRIPT_DIR}/gpu-overview.json" \ - "${SCRIPT_DIR}/foxhunt-overview.json" \ - "${SCRIPT_DIR}/foxhunt-services.json" \ - "${SCRIPT_DIR}/foxhunt-gpu-training.json" \ + for dashboard in "${SCRIPT_DIR}/foxhunt-cockpit.json" \ + "${SCRIPT_DIR}/foxhunt-trading.json" \ + "${SCRIPT_DIR}/foxhunt-training.json" \ "${SCRIPT_DIR}/foxhunt-infrastructure.json" \ - "${SCRIPT_DIR}/foxhunt-logs.json" \ - "${SCRIPT_DIR}/foxhunt-cockpit.json" \ - "${SCRIPT_DIR}/foxhunt-trading-cockpit.json" \ - "${SCRIPT_DIR}/foxhunt-training-cockpit.json" \ - "${SCRIPT_DIR}/foxhunt-traces.json"; do + "${SCRIPT_DIR}/foxhunt-observability.json"; do if [[ -f "$dashboard" ]]; then if ! import_dashboard "$dashboard"; then failed=$((failed + 1)) diff --git a/infra/k8s/monitoring/prometheus.yaml b/infra/k8s/monitoring/prometheus.yaml index 4fc425ac2..08080e455 100644 --- a/infra/k8s/monitoring/prometheus.yaml +++ b/infra/k8s/monitoring/prometheus.yaml @@ -166,6 +166,17 @@ data: - source_labels: [__meta_kubernetes_pod_node_name] target_label: node + # Argo Workflows controller metrics (HTTPS endpoint) + - job_name: argo-workflows + scheme: https + tls_config: + insecure_skip_verify: true + static_configs: + - targets: ['argo-workflows-workflow-controller-metrics.foxhunt.svc.cluster.local:9090'] + relabel_configs: + - target_label: service + replacement: argo-workflows + # Pushgateway (training job metrics) - job_name: pushgateway honor_labels: true @@ -195,7 +206,7 @@ spec: spec: serviceAccountName: prometheus nodeSelector: - k8s.scaleway.com/pool-name: foxhunt + k8s.scaleway.com/pool-name: platform securityContext: fsGroup: 65534 runAsNonRoot: true diff --git a/infra/k8s/network-policies/prometheus.yaml b/infra/k8s/network-policies/prometheus.yaml index 995534094..36c11feaa 100644 --- a/infra/k8s/network-policies/prometheus.yaml +++ b/infra/k8s/network-policies/prometheus.yaml @@ -1,5 +1,5 @@ -# Prometheus egress: allow scraping foxhunt pods, node-exporter, kube-state-metrics, -# DCGM exporter, and kube-system components. +# Prometheus egress: K8s API (service discovery), foxhunt services, node-exporter, +# kube-state-metrics, DCGM exporter, Argo workflow controller, and pushgateway. apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: @@ -15,6 +15,17 @@ spec: policyTypes: - Egress egress: + # Kubernetes API server (required for kubernetes_sd_configs service discovery) + # ClusterIP 10.32.0.1:443 NATs to node IP 172.16.0.0/16:6443 + - to: + - ipBlock: + cidr: 10.32.0.0/16 + ports: + - protocol: TCP + port: 443 + - protocol: TCP + port: 6443 + # Scrape foxhunt service metrics ports (9091-9098) - to: - podSelector: @@ -23,6 +34,8 @@ spec: operator: In values: ["foxhunt", "foxhunt-ci"] ports: + - protocol: TCP + port: 9090 - protocol: TCP port: 9091 - protocol: TCP @@ -40,7 +53,7 @@ spec: - protocol: TCP port: 9098 - # Scrape foxhunt pods on port 8080 (kube-state-metrics sidecar) + # Scrape foxhunt pods on port 8080 (kube-state-metrics sidecar, health) - to: - podSelector: matchExpressions: @@ -51,11 +64,13 @@ spec: - protocol: TCP port: 8080 - # Node-exporter (hostNetwork) + DCGM exporter on node IPs + # Node-exporter (hostNetwork), DCGM exporter, and K8s API on node IPs - to: - ipBlock: cidr: 172.16.0.0/16 ports: + - protocol: TCP + port: 6443 - protocol: TCP port: 9100 - protocol: TCP @@ -85,6 +100,15 @@ spec: - protocol: TCP port: 9091 + # Argo Workflows controller metrics + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: argo-workflows-workflow-controller + ports: + - protocol: TCP + port: 9090 + # Alertmanager - to: - podSelector: