diff --git a/infra/k8s/argo/ci-pipeline-template.yaml b/infra/k8s/argo/ci-pipeline-template.yaml index da0d424ab..2172b7d2c 100644 --- a/infra/k8s/argo/ci-pipeline-template.yaml +++ b/infra/k8s/argo/ci-pipeline-template.yaml @@ -129,7 +129,7 @@ spec: nodeSelector: k8s.scaleway.com/pool-name: platform container: - image: localhost:30500/root/foxhunt/foxhunt-runtime:latest + image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest command: ["/bin/sh", "-c"] env: - name: COMMITS_JSON @@ -305,7 +305,7 @@ spec: nodeSelector: k8s.scaleway.com/pool-name: platform container: - image: localhost:30500/root/foxhunt/ci-builder-cpu:latest + image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/ci-builder-cpu:latest imagePullPolicy: Always command: ["/bin/sh", "-c"] env: @@ -433,7 +433,7 @@ spec: operator: Exists effect: NoSchedule container: - image: localhost:30500/root/foxhunt/ci-builder:latest + image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/ci-builder:latest imagePullPolicy: Always command: ["/bin/sh", "-c"] env: diff --git a/infra/k8s/argo/training-workflow-template.yaml b/infra/k8s/argo/training-workflow-template.yaml index 6646a3542..9e12d6bd2 100644 --- a/infra/k8s/argo/training-workflow-template.yaml +++ b/infra/k8s/argo/training-workflow-template.yaml @@ -83,7 +83,7 @@ spec: nodeSelector: k8s.scaleway.com/pool-name: platform container: - image: localhost:30500/root/foxhunt/foxhunt-runtime:latest + image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest command: ["/bin/sh", "-c"] args: - | @@ -133,7 +133,7 @@ spec: operator: Exists effect: NoSchedule container: - image: localhost:30500/root/foxhunt/foxhunt-training-runtime:latest + image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-training-runtime:latest command: ["/bin/sh", "-c"] args: - | @@ -229,7 +229,7 @@ spec: operator: Exists effect: NoSchedule container: - image: localhost:30500/root/foxhunt/foxhunt-training-runtime:latest + image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-training-runtime:latest command: ["/bin/sh", "-c"] args: - | @@ -319,7 +319,7 @@ spec: operator: Exists effect: NoSchedule container: - image: localhost:30500/root/foxhunt/foxhunt-training-runtime:latest + image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-training-runtime:latest command: ["/bin/sh", "-c"] args: - | @@ -392,7 +392,7 @@ spec: nodeSelector: k8s.scaleway.com/pool-name: platform container: - image: localhost:30500/root/foxhunt/foxhunt-runtime:latest + image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest command: ["/bin/sh", "-c"] args: - | diff --git a/infra/k8s/gitlab/values.yaml b/infra/k8s/gitlab/values.yaml index de1b6bda9..3e41aef79 100644 --- a/infra/k8s/gitlab/values.yaml +++ b/infra/k8s/gitlab/values.yaml @@ -95,7 +95,7 @@ gitlab-runner: # Registry — S3-backed, NodePort for kubelet image pulls # Token auth disabled (cluster-internal only, secured by network policies) -# Pods pull via localhost:30500; Kaniko pushes via ClusterIP gitlab-registry:5000 +# Pods pull via gitlab-registry.foxhunt.svc.cluster.local:5000 (internal DNS) registry: replicaCount: 1 hpa: diff --git a/infra/k8s/gpu-overlays/ml-training-service-gpu.yaml b/infra/k8s/gpu-overlays/ml-training-service-gpu.yaml index a1cc9527b..4e31ab100 100644 --- a/infra/k8s/gpu-overlays/ml-training-service-gpu.yaml +++ b/infra/k8s/gpu-overlays/ml-training-service-gpu.yaml @@ -49,7 +49,7 @@ spec: - name: gitlab-registry initContainers: - name: fetch-binary - image: localhost:30500/root/foxhunt/foxhunt-runtime:latest + image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest command: ["/bin/sh", "-c"] args: - | @@ -88,7 +88,7 @@ spec: memory: 128Mi containers: - name: ml-training-service - image: localhost:30500/root/foxhunt/foxhunt-runtime:latest + image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest securityContext: allowPrivilegeEscalation: false readOnlyRootFilesystem: true diff --git a/infra/k8s/gpu-overlays/trading-service-gpu.yaml b/infra/k8s/gpu-overlays/trading-service-gpu.yaml index 4637d0857..1806ac817 100644 --- a/infra/k8s/gpu-overlays/trading-service-gpu.yaml +++ b/infra/k8s/gpu-overlays/trading-service-gpu.yaml @@ -48,7 +48,7 @@ spec: - name: gitlab-registry initContainers: - name: fetch-binary - image: localhost:30500/root/foxhunt/foxhunt-runtime:latest + image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest command: ["/bin/sh", "-c"] args: - | @@ -87,7 +87,7 @@ spec: memory: 128Mi containers: - name: trading-service - image: localhost:30500/root/foxhunt/foxhunt-runtime:latest + image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest securityContext: allowPrivilegeEscalation: false readOnlyRootFilesystem: true diff --git a/infra/k8s/monitoring/dashboards/foxhunt-cockpit.json b/infra/k8s/monitoring/dashboards/foxhunt-cockpit.json index bef3b5b92..408e8ff6c 100644 --- a/infra/k8s/monitoring/dashboards/foxhunt-cockpit.json +++ b/infra/k8s/monitoring/dashboards/foxhunt-cockpit.json @@ -2,18 +2,15 @@ "id": null, "uid": "foxhunt-cockpit", "title": "Foxhunt Operations Cockpit", - "description": "Single-pane-of-glass for the foxhunt HFT trading platform. Service health, CI/CD pipelines, resource utilization, gRPC backends, and security at a glance.", - "tags": [ - "foxhunt", - "cockpit" - ], + "description": "Operations overview: service health, gRPC traffic, CI/CD pipelines, API gateway, trading metrics, and metrics pipeline health.", + "tags": ["foxhunt", "cockpit"], "timezone": "browser", "schemaVersion": 39, - "version": 1, + "version": 2, "editable": true, - "refresh": "10s", + "refresh": "30s", "time": { - "from": "now-1h", + "from": "now-6h", "to": "now" }, "fiscalYearStartMonth": 0, @@ -27,8 +24,8 @@ "query": "1m,5m,15m,30m,1h", "current": { "selected": true, - "text": "5m", - "value": "5m" + "text": "auto", + "value": "$__auto_interval_interval" }, "auto": true, "auto_count": 30, @@ -41,40 +38,23 @@ { "id": 1, "type": "row", - "title": "Health Status Bar", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, + "title": "Service Health", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "collapsed": false, "panels": [] }, { "id": 2, "type": "stat", - "title": "Services Running", - "gridPos": { - "h": 4, - "w": 4, - "x": 0, - "y": 1 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, + "title": "Services Up", + "description": "Number of foxhunt service targets currently up", + "gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "targets": [ { - "expr": "count(up{job=~\"foxhunt.*|pushgateway|argo-workflows\"} == 1)", - "legendFormat": "Up", + "expr": "count(up{job=\"foxhunt-services\"} == 1)", + "legendFormat": "up", "refId": "A" - }, - { - "expr": "count(up{job=~\"foxhunt.*|pushgateway|argo-workflows\"})", - "legendFormat": "Total", - "refId": "B" } ], "fieldConfig": { @@ -82,56 +62,36 @@ "thresholds": { "mode": "absolute", "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "orange", - "value": 3 - }, - { - "color": "green", - "value": 6 - } + { "color": "red", "value": null }, + { "color": "yellow", "value": 3 }, + { "color": "green", "value": 5 } ] }, - "unit": "none", - "mappings": [] + "color": { "mode": "thresholds" }, + "unit": "none" }, "overrides": [] }, "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", "colorMode": "background", "graphMode": "none", - "textMode": "auto", "justifyMode": "auto" - }, - "transformations": [] + } }, { "id": 3, "type": "stat", - "title": "Total Request Rate", - "gridPos": { - "h": 4, - "w": 4, - "x": 4, - "y": 1 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, + "title": "Services Down", + "description": "Number of foxhunt service targets currently down", + "gridPos": { "h": 4, "w": 4, "x": 4, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "targets": [ { - "expr": "sum(rate(api_gateway_http_requests_total[5m]))", - "legendFormat": "req/s", + "expr": "count(up{job=\"foxhunt-services\"} == 0) OR on() vector(0)", + "legendFormat": "down", "refId": "A" } ], @@ -140,48 +100,35 @@ "thresholds": { "mode": "absolute", "steps": [ - { - "color": "blue", - "value": null - } + { "color": "green", "value": null }, + { "color": "yellow", "value": 1 }, + { "color": "red", "value": 2 } ] }, - "unit": "reqps", - "decimals": 1, - "mappings": [] + "color": { "mode": "thresholds" }, + "unit": "none" }, "overrides": [] }, "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", "colorMode": "background", "graphMode": "none", - "textMode": "auto", "justifyMode": "auto" } }, { "id": 4, "type": "stat", - "title": "Error Rate", - "gridPos": { - "h": 4, - "w": 4, - "x": 8, - "y": 1 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, + "title": "Trading Service Uptime", + "gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "targets": [ { - "expr": "sum(rate(api_gateway_http_requests_total{status=~\"5..\"}[5m])) / sum(rate(api_gateway_http_requests_total[5m])) * 100", - "legendFormat": "Error %", + "expr": "trading_service_uptime_seconds", + "legendFormat": "trading", "refId": "A" } ], @@ -190,58 +137,35 @@ "thresholds": { "mode": "absolute", "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 1 - }, - { - "color": "red", - "value": 5 - } + { "color": "red", "value": null }, + { "color": "yellow", "value": 300 }, + { "color": "green", "value": 3600 } ] }, - "unit": "percent", - "decimals": 2, - "mappings": [], - "min": 0, - "max": 100 + "color": { "mode": "thresholds" }, + "unit": "s" }, "overrides": [] }, "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "colorMode": "background", - "graphMode": "none", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", "textMode": "auto", + "colorMode": "value", + "graphMode": "none", "justifyMode": "auto" } }, { "id": 5, "type": "stat", - "title": "Active Workflows", - "gridPos": { - "h": 4, - "w": 4, - "x": 12, - "y": 1 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, + "title": "Backtesting Service Uptime", + "gridPos": { "h": 4, "w": 4, "x": 12, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "targets": [ { - "expr": "argo_workflows_gauge{status=\"Running\"}", - "legendFormat": "Running", + "expr": "backtesting_service_uptime_seconds", + "legendFormat": "backtesting", "refId": "A" } ], @@ -250,51 +174,35 @@ "thresholds": { "mode": "absolute", "steps": [ - { - "color": "blue", - "value": null - }, - { - "color": "green", - "value": 1 - } + { "color": "red", "value": null }, + { "color": "yellow", "value": 300 }, + { "color": "green", "value": 3600 } ] }, - "unit": "none", - "mappings": [] + "color": { "mode": "thresholds" }, + "unit": "s" }, "overrides": [] }, "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "colorMode": "background", - "graphMode": "none", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", "textMode": "auto", + "colorMode": "value", + "graphMode": "none", "justifyMode": "auto" } }, { "id": 6, "type": "stat", - "title": "GPU Utilization", - "gridPos": { - "h": 4, - "w": 4, - "x": 16, - "y": 1 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, + "title": "ML Training Service Uptime", + "gridPos": { "h": 4, "w": 4, "x": 16, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "targets": [ { - "expr": "avg(DCGM_FI_DEV_GPU_UTIL)", - "legendFormat": "GPU %", + "expr": "ml_training_service_uptime_seconds", + "legendFormat": "ml_training", "refId": "A" } ], @@ -303,58 +211,35 @@ "thresholds": { "mode": "absolute", "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 70 - }, - { - "color": "red", - "value": 90 - } + { "color": "red", "value": null }, + { "color": "yellow", "value": 300 }, + { "color": "green", "value": 3600 } ] }, - "unit": "percent", - "decimals": 0, - "min": 0, - "max": 100, - "mappings": [] + "color": { "mode": "thresholds" }, + "unit": "s" }, "overrides": [] }, "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "colorMode": "background", - "graphMode": "none", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", "textMode": "auto", + "colorMode": "value", + "graphMode": "none", "justifyMode": "auto" } }, { "id": 7, "type": "stat", - "title": "Training Active", - "gridPos": { - "h": 4, - "w": 4, - "x": 20, - "y": 1 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, + "title": "Data Acquisition Uptime", + "gridPos": { "h": 4, "w": 4, "x": 0, "y": 5 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "targets": [ { - "expr": "ml_training_active_workers or count(foxhunt_training_current_epoch > 0)", - "legendFormat": "Workers", + "expr": "foxhunt_data_acquisition_uptime_seconds", + "legendFormat": "data_acquisition", "refId": "A" } ], @@ -363,106 +248,36 @@ "thresholds": { "mode": "absolute", "steps": [ - { - "color": "semi-dark-blue", - "value": null - }, - { - "color": "green", - "value": 1 - } + { "color": "red", "value": null }, + { "color": "yellow", "value": 300 }, + { "color": "green", "value": 3600 } ] }, - "unit": "none", - "mappings": [ - { - "type": "special", - "options": { - "match": "null", - "result": { - "text": "0", - "color": "text" - } - } - } - ] + "color": { "mode": "thresholds" }, + "unit": "s" }, "overrides": [] }, "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "colorMode": "background", - "graphMode": "none", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", "textMode": "auto", + "colorMode": "value", + "graphMode": "none", "justifyMode": "auto" } }, { "id": 8, - "type": "row", - "title": "Service Health Matrix", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 5 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 9, - "type": "table", - "title": "Service Status Table", - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 6 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, + "type": "stat", + "title": "Web Gateway Uptime", + "gridPos": { "h": 4, "w": 4, "x": 4, "y": 5 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "targets": [ { - "expr": "up{job=~\"foxhunt.*\"}", - "legendFormat": "{{job}}", - "refId": "A", - "format": "table", - "instant": true - }, - { - "expr": "time() - process_start_time_seconds{job=~\"foxhunt.*\"}", - "legendFormat": "{{job}}", - "refId": "B", - "format": "table", - "instant": true - }, - { - "expr": "sum by (job) (rate(api_gateway_http_requests_total{job=~\"foxhunt.*\"}[5m]))", - "legendFormat": "{{job}}", - "refId": "C", - "format": "table", - "instant": true - }, - { - "expr": "sum by (job) (rate(api_gateway_http_requests_total{job=~\"foxhunt.*\", status=~\"5..\"}[5m])) / sum by (job) (rate(api_gateway_http_requests_total{job=~\"foxhunt.*\"}[5m])) * 100", - "legendFormat": "{{job}}", - "refId": "D", - "format": "table", - "instant": true - }, - { - "expr": "histogram_quantile(0.99, sum by (job, le) (rate(api_gateway_http_request_duration_seconds_bucket{job=~\"foxhunt.*\"}[5m])))", - "legendFormat": "{{job}}", - "refId": "E", - "format": "table", - "instant": true + "expr": "foxhunt_web_gateway_uptime_seconds", + "legendFormat": "web_gateway", + "refId": "A" } ], "fieldConfig": { @@ -470,1308 +285,1389 @@ "thresholds": { "mode": "absolute", "steps": [ - { - "color": "green", - "value": null - } + { "color": "red", "value": null }, + { "color": "yellow", "value": 300 }, + { "color": "green", "value": 3600 } ] }, - "mappings": [] + "color": { "mode": "thresholds" }, + "unit": "s" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto" + } + }, + { + "id": 9, + "type": "stat", + "title": "Active WebSocket Connections", + "gridPos": { "h": 4, "w": 4, "x": 8, "y": 5 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "targets": [ + { + "expr": "foxhunt_web_gateway_ws_connections_active", + "legendFormat": "ws_active", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "blue", "value": null }, + { "color": "green", "value": 1 }, + { "color": "yellow", "value": 50 }, + { "color": "red", "value": 100 } + ] + }, + "color": { "mode": "thresholds" }, + "unit": "none" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto" + } + }, + { + "id": 10, + "type": "table", + "title": "Per-Service Status", + "description": "Current up/down status for each service target", + "gridPos": { "h": 4, "w": 12, "x": 12, "y": 5 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "targets": [ + { + "expr": "up{job=\"foxhunt-services\"}", + "legendFormat": "{{service}}", + "refId": "A", + "format": "table", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "displayMode": "color-background-solid" + } }, "overrides": [ { - "matcher": { - "id": "byName", - "options": "Value #A" - }, + "matcher": { "id": "byName", "options": "Value" }, "properties": [ - { - "id": "displayName", - "value": "Status" - }, { "id": "mappings", "value": [ - { - "type": "value", - "options": { - "0": { - "text": "DOWN", - "color": "red" - }, - "1": { - "text": "UP", - "color": "green" - } - } - } + { "type": "value", "options": { "1": { "text": "UP", "color": "green" } } }, + { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" } } } ] }, - { - "id": "custom.cellOptions", - "value": { - "type": "color-background" - } - } + { "id": "custom.displayMode", "value": "color-background-solid" } ] }, { - "matcher": { - "id": "byName", - "options": "Value #B" - }, - "properties": [ - { - "id": "displayName", - "value": "Uptime" - }, - { - "id": "unit", - "value": "s" - } - ] + "matcher": { "id": "byName", "options": "Time" }, + "properties": [{ "id": "custom.hidden", "value": true }] }, { - "matcher": { - "id": "byName", - "options": "Value #C" - }, - "properties": [ - { - "id": "displayName", - "value": "Request Rate" - }, - { - "id": "unit", - "value": "reqps" - }, - { - "id": "decimals", - "value": 2 - } - ] + "matcher": { "id": "byName", "options": "__name__" }, + "properties": [{ "id": "custom.hidden", "value": true }] }, { - "matcher": { - "id": "byName", - "options": "Value #D" - }, - "properties": [ - { - "id": "displayName", - "value": "Error Rate %" - }, - { - "id": "unit", - "value": "percent" - }, - { - "id": "decimals", - "value": 2 - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 1 - }, - { - "color": "red", - "value": 5 - } - ] - } - }, - { - "id": "custom.cellOptions", - "value": { - "type": "color-background" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value #E" - }, - "properties": [ - { - "id": "displayName", - "value": "P99 Latency" - }, - { - "id": "unit", - "value": "s" - }, - { - "id": "decimals", - "value": 3 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "job" - }, - "properties": [ - { - "id": "displayName", - "value": "Service" - } - ] + "matcher": { "id": "byName", "options": "job" }, + "properties": [{ "id": "custom.hidden", "value": true }] } ] }, "options": { "showHeader": true, - "footer": { - "show": false - }, - "sortBy": [ - { - "displayName": "Service", - "desc": false - } - ] + "sortBy": [{ "displayName": "service", "desc": false }] }, "transformations": [ - { - "id": "merge", - "options": {} - }, { "id": "organize", "options": { - "excludeByName": { - "Time": true, - "__name__": true, - "instance": true - } + "excludeByName": { "Time": true, "__name__": true, "job": true }, + "renameByName": { "service": "Service", "instance": "Instance", "Value": "Status" } } } ] }, - { - "id": 10, - "type": "timeseries", - "title": "Service Request Rate", - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 6 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "targets": [ - { - "expr": "sum by (service) (rate(api_gateway_http_requests_total[$interval]))", - "legendFormat": "{{service}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "reqps", - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 30, - "stacking": { - "mode": "normal", - "group": "A" - }, - "lineWidth": 1, - "pointSize": 5, - "showPoints": "never", - "gradientMode": "opacity", - "axisPlacement": "auto" - }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "max", - "lastNotNull" - ] - } - } - }, - { - "id": 11, - "type": "row", - "title": "Argo CI/CD Pipeline", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 14 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 12, - "type": "stat", - "title": "Workflow Status", - "gridPos": { - "h": 8, - "w": 6, - "x": 0, - "y": 15 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "targets": [ - { - "expr": "argo_workflows_gauge{status=\"Running\"}", - "legendFormat": "Running", - "refId": "A" - }, - { - "expr": "argo_workflows_gauge{status=\"Succeeded\"}", - "legendFormat": "Succeeded", - "refId": "B" - }, - { - "expr": "argo_workflows_gauge{status=\"Failed\"}", - "legendFormat": "Failed", - "refId": "C" - }, - { - "expr": "argo_workflows_gauge{status=\"Pending\"}", - "legendFormat": "Pending", - "refId": "D" - } - ], - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "none", - "mappings": [] - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Running" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Succeeded" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Failed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Pending" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "fixed" - } - } - ] - } - ] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "colorMode": "background", - "graphMode": "none", - "textMode": "auto", - "justifyMode": "auto", - "orientation": "vertical" - } - }, - { - "id": 13, - "type": "timeseries", - "title": "Workflow Duration", - "gridPos": { - "h": 8, - "w": 9, - "x": 6, - "y": 15 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "targets": [ - { - "expr": "histogram_quantile(0.50, sum by (le) (rate(argo_workflows_operation_duration_seconds_bucket[5m])))", - "legendFormat": "p50", - "refId": "A" - }, - { - "expr": "histogram_quantile(0.95, sum by (le) (rate(argo_workflows_operation_duration_seconds_bucket[5m])))", - "legendFormat": "p95", - "refId": "B" - } - ], - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "s", - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 10, - "lineWidth": 2, - "pointSize": 5, - "showPoints": "never", - "gradientMode": "scheme", - "axisPlacement": "auto" - }, - "mappings": [] - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "p95" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "orange", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "p50" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - } - ] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "max", - "lastNotNull" - ] - } - } - }, - { - "id": 14, - "type": "timeseries", - "title": "Workflow Errors", - "gridPos": { - "h": 8, - "w": 9, - "x": 15, - "y": 15 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "targets": [ - { - "expr": "rate(argo_workflows_error_count[5m])", - "legendFormat": "{{cause}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "ops", - "custom": { - "drawStyle": "bars", - "lineInterpolation": "linear", - "fillOpacity": 80, - "stacking": { - "mode": "normal", - "group": "A" - }, - "lineWidth": 1, - "pointSize": 5, - "showPoints": "never", - "gradientMode": "none", - "axisPlacement": "auto" - }, - "color": { - "mode": "palette-classic" - }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "sum", - "lastNotNull" - ] - } - } - }, - { - "id": 15, - "type": "row", - "title": "Resource Overview", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 23 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 16, - "type": "gauge", - "title": "CPU Usage", - "gridPos": { - "h": 8, - "w": 6, - "x": 0, - "y": 24 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "targets": [ - { - "expr": "avg(1 - rate(node_cpu_seconds_total{mode=\"idle\"}[5m])) * 100", - "legendFormat": "CPU %", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 60 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent", - "decimals": 1, - "min": 0, - "max": 100, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "orientation": "auto" - } - }, - { - "id": 17, - "type": "gauge", - "title": "Memory Usage", - "gridPos": { - "h": 8, - "w": 6, - "x": 6, - "y": 24 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "targets": [ - { - "expr": "(1 - (node_memory_AvailableBytes / node_memory_MemoryTotal_bytes)) * 100", - "legendFormat": "Memory %", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 60 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent", - "decimals": 1, - "min": 0, - "max": 100, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "orientation": "auto" - } - }, - { - "id": 18, - "type": "gauge", - "title": "Disk Usage", - "gridPos": { - "h": 8, - "w": 6, - "x": 12, - "y": 24 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "targets": [ - { - "expr": "(1 - (node_filesystem_avail_bytes{mountpoint=\"/\"} / node_filesystem_size_bytes{mountpoint=\"/\"})) * 100", - "legendFormat": "Disk %", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 60 - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent", - "decimals": 1, - "min": 0, - "max": 100, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "orientation": "auto" - } - }, - { - "id": 19, - "type": "timeseries", - "title": "Network IO", - "gridPos": { - "h": 8, - "w": 6, - "x": 18, - "y": 24 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "targets": [ - { - "expr": "rate(node_network_receive_bytes_total{device!=\"lo\"}[5m])", - "legendFormat": "Receive {{device}}", - "refId": "A" - }, - { - "expr": "rate(node_network_transmit_bytes_total{device!=\"lo\"}[5m])", - "legendFormat": "Transmit {{device}}", - "refId": "B" - } - ], - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "Bps", - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 15, - "lineWidth": 1, - "pointSize": 5, - "showPoints": "never", - "gradientMode": "opacity", - "axisPlacement": "auto" - }, - "mappings": [] - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/Transmit.*/" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "max" - ] - } - } - }, + { "id": 20, "type": "row", - "title": "gRPC & Backend", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 32 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 21, - "type": "timeseries", - "title": "gRPC Request Rate", - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 33 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "targets": [ + "title": "gRPC Overview", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 9 }, + "collapsed": true, + "panels": [ { - "expr": "sum by (grpc_service, grpc_method) (rate(grpc_server_handled_total[5m]))", - "legendFormat": "{{grpc_service}}/{{grpc_method}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "ops", - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 20, - "stacking": { - "mode": "normal", - "group": "A" - }, - "lineWidth": 1, - "pointSize": 5, - "showPoints": "never", - "gradientMode": "opacity", - "axisPlacement": "auto" - }, - "color": { - "mode": "palette-classic" - }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "max", - "lastNotNull" - ] - } - } - }, - { - "id": 22, - "type": "timeseries", - "title": "gRPC Latency", - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 33 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "targets": [ - { - "expr": "histogram_quantile(0.50, sum by (le, grpc_service) (rate(grpc_server_handling_seconds_bucket[5m])))", - "legendFormat": "p50 {{grpc_service}}", - "refId": "A" - }, - { - "expr": "histogram_quantile(0.95, sum by (le, grpc_service) (rate(grpc_server_handling_seconds_bucket[5m])))", - "legendFormat": "p95 {{grpc_service}}", - "refId": "B" - }, - { - "expr": "histogram_quantile(0.99, sum by (le, grpc_service) (rate(grpc_server_handling_seconds_bucket[5m])))", - "legendFormat": "p99 {{grpc_service}}", - "refId": "C" - } - ], - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "s", - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 0, - "lineWidth": 2, - "pointSize": 5, - "showPoints": "never", - "gradientMode": "none", - "axisPlacement": "auto" - }, - "color": { - "mode": "palette-classic" - }, - "mappings": [] - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "/p99.*/" - }, - "properties": [ - { - "id": "custom.lineStyle", - "value": { - "fill": "solid" - } - }, - { - "id": "custom.lineWidth", - "value": 2 - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "/p50.*/" - }, - "properties": [ - { - "id": "custom.lineStyle", - "value": { - "fill": "dash", - "dash": [ - 10, - 10 - ] - } - }, - { - "id": "custom.lineWidth", - "value": 1 - } - ] - } - ] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "max", - "lastNotNull" - ] - } - } - }, - { - "id": 23, - "type": "row", - "title": "Rate Limiting & Auth", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 41 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 24, - "type": "timeseries", - "title": "Rate Limit Rejections", - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 42 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "targets": [ - { - "expr": "rate(api_gateway_rate_limit_rejected_total[5m])", - "legendFormat": "{{tier}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "ops", - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 20, - "lineWidth": 2, - "pointSize": 5, - "showPoints": "never", - "gradientMode": "opacity", - "axisPlacement": "auto" - }, - "color": { - "mode": "palette-classic" - }, - "mappings": [] - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "sum", - "mean", - "lastNotNull" - ] - } - } - }, - { - "id": 25, - "type": "timeseries", - "title": "Auth Success/Fail", - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 42 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "targets": [ - { - "expr": "rate(api_gateway_auth_attempts_total{result=\"success\"}[5m])", - "legendFormat": "Success", - "refId": "A" - }, - { - "expr": "rate(api_gateway_auth_attempts_total{result=\"failure\"}[5m])", - "legendFormat": "Failure", - "refId": "B" - } - ], - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "ops", - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 20, - "lineWidth": 2, - "pointSize": 5, - "showPoints": "never", - "gradientMode": "opacity", - "axisPlacement": "auto" - }, - "mappings": [] - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Success" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Failure" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - } - ] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "sum", - "mean", - "lastNotNull" - ] - } - } - }, - { - "id": 26, - "type": "stat", - "title": "JWT Cache Hit Rate", - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 42 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "targets": [ - { - "expr": "api_gateway_jwt_cache_hit_total / (api_gateway_jwt_cache_hit_total + api_gateway_jwt_cache_miss_total) * 100", - "legendFormat": "Hit Rate", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "yellow", - "value": 50 - }, - { - "color": "green", - "value": 80 - } - ] - }, - "unit": "percent", - "decimals": 1, - "min": 0, - "max": 100, - "mappings": [ + "id": 21, + "type": "timeseries", + "title": "gRPC Request Rate (by service)", + "description": "Rate of gRPC calls started, grouped by exported_service", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 10 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "targets": [ { - "type": "special", - "options": { - "match": "null", - "result": { - "text": "N/A", - "color": "text" - } - } - }, - { - "type": "special", - "options": { - "match": "nan", - "result": { - "text": "N/A", - "color": "text" - } - } + "expr": "sum by (exported_service) (rate(grpc_server_started_total[$__rate_interval]))", + "legendFormat": "{{exported_service}}", + "refId": "A" } - ] + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 15, + "pointSize": 5, + "showPoints": "auto", + "spanNulls": false, + "stacking": { "mode": "none" }, + "axisPlacement": "auto", + "lineWidth": 2 + }, + "unit": "reqps", + "color": { "mode": "palette-classic" } + }, + "overrides": [] + }, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + } }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] + { + "id": 22, + "type": "timeseries", + "title": "gRPC Error Rate (by service)", + "description": "Rate of gRPC calls that returned a non-OK code", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 10 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "targets": [ + { + "expr": "sum by (exported_service) (rate(grpc_server_handled_total{grpc_code!=\"OK\"}[$__rate_interval]))", + "legendFormat": "{{exported_service}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 15, + "pointSize": 5, + "showPoints": "auto", + "spanNulls": false, + "stacking": { "mode": "none" }, + "axisPlacement": "auto", + "lineWidth": 2 + }, + "unit": "reqps", + "color": { "mode": "palette-classic" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 1 }, + { "color": "red", "value": 5 } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + } }, - "colorMode": "background", - "graphMode": "none", - "textMode": "auto", - "justifyMode": "auto" - } + { + "id": 23, + "type": "timeseries", + "title": "gRPC p99 Latency (by service)", + "description": "99th percentile gRPC server handling duration", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 18 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum by (exported_service, le) (rate(grpc_server_handling_seconds_bucket[$__rate_interval])))", + "legendFormat": "{{exported_service}} p99", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 10, + "pointSize": 5, + "showPoints": "auto", + "spanNulls": false, + "stacking": { "mode": "none" }, + "axisPlacement": "auto", + "lineWidth": 2 + }, + "unit": "s", + "color": { "mode": "palette-classic" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 0.5 }, + { "color": "red", "value": 2 } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + } + }, + { + "id": 24, + "type": "stat", + "title": "gRPC Error Ratio", + "description": "Percentage of gRPC calls returning non-OK status", + "gridPos": { "h": 8, "w": 6, "x": 12, "y": 18 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "targets": [ + { + "expr": "sum(rate(grpc_server_handled_total{grpc_code!=\"OK\"}[$__rate_interval])) / sum(rate(grpc_server_handled_total[$__rate_interval]))", + "legendFormat": "error ratio", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 0.01 }, + { "color": "red", "value": 0.05 } + ] + }, + "color": { "mode": "thresholds" }, + "unit": "percentunit", + "min": 0, + "max": 1 + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto" + } + }, + { + "id": 25, + "type": "stat", + "title": "gRPC Total RPS", + "description": "Total gRPC requests per second across all services", + "gridPos": { "h": 8, "w": 6, "x": 18, "y": 18 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "targets": [ + { + "expr": "sum(rate(grpc_server_started_total[$__rate_interval]))", + "legendFormat": "total rps", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "blue", "value": null }, + { "color": "green", "value": 1 }, + { "color": "yellow", "value": 500 }, + { "color": "red", "value": 1000 } + ] + }, + "color": { "mode": "thresholds" }, + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto" + } + } + ] + }, + + { + "id": 30, + "type": "row", + "title": "Argo CI/CD", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 10 }, + "collapsed": true, + "panels": [ + { + "id": 31, + "type": "gauge", + "title": "Running Workflows", + "gridPos": { "h": 6, "w": 4, "x": 0, "y": 11 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "targets": [ + { + "expr": "argo_workflows_gauge{status=\"Running\"}", + "legendFormat": "running", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 5 }, + { "color": "red", "value": 10 } + ] + }, + "color": { "mode": "thresholds" }, + "unit": "none", + "min": 0, + "max": 15 + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "orientation": "auto" + } + }, + { + "id": 32, + "type": "gauge", + "title": "Pending Workflows", + "gridPos": { "h": 6, "w": 4, "x": 4, "y": 11 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "targets": [ + { + "expr": "argo_workflows_gauge{status=\"Pending\"}", + "legendFormat": "pending", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 3 }, + { "color": "red", "value": 8 } + ] + }, + "color": { "mode": "thresholds" }, + "unit": "none", + "min": 0, + "max": 15 + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "orientation": "auto" + } + }, + { + "id": 33, + "type": "stat", + "title": "Failed Workflows", + "gridPos": { "h": 6, "w": 4, "x": 8, "y": 11 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "targets": [ + { + "expr": "argo_workflows_gauge{status=\"Failed\"}", + "legendFormat": "failed", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 1 }, + { "color": "red", "value": 3 } + ] + }, + "color": { "mode": "thresholds" }, + "unit": "none" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto" + } + }, + { + "id": 34, + "type": "stat", + "title": "Succeeded Workflows", + "gridPos": { "h": 6, "w": 4, "x": 12, "y": 11 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "targets": [ + { + "expr": "argo_workflows_gauge{status=\"Succeeded\"}", + "legendFormat": "succeeded", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "yellow", "value": 1 }, + { "color": "green", "value": 1 } + ] + }, + "color": { "mode": "thresholds" }, + "unit": "none" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto" + } + }, + { + "id": 35, + "type": "stat", + "title": "Workflow Pods", + "gridPos": { "h": 6, "w": 4, "x": 16, "y": 11 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "targets": [ + { + "expr": "argo_workflows_pods_gauge", + "legendFormat": "pods", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "blue", "value": null }, + { "color": "green", "value": 1 }, + { "color": "yellow", "value": 20 }, + { "color": "red", "value": 50 } + ] + }, + "color": { "mode": "thresholds" }, + "unit": "none" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto" + } + }, + { + "id": 36, + "type": "timeseries", + "title": "Argo Error Rate", + "description": "Rate of errors from Argo Workflows controller", + "gridPos": { "h": 6, "w": 4, "x": 20, "y": 11 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "targets": [ + { + "expr": "rate(argo_workflows_error_count[$__rate_interval])", + "legendFormat": "errors/s", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 20, + "pointSize": 5, + "showPoints": "auto", + "spanNulls": false, + "stacking": { "mode": "none" }, + "axisPlacement": "auto", + "lineWidth": 2, + "thresholdsStyle": { "mode": "area" } + }, + "unit": "ops", + "color": { "mode": "palette-classic" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 0.1 }, + { "color": "red", "value": 1 } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { "displayMode": "list", "placement": "bottom" }, + "tooltip": { "mode": "single" } + } + }, + { + "id": 37, + "type": "timeseries", + "title": "Argo Operation Duration p95", + "description": "95th percentile Argo operation duration", + "gridPos": { "h": 6, "w": 12, "x": 0, "y": 17 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum by (le) (rate(argo_workflows_operation_duration_seconds_bucket[$__rate_interval])))", + "legendFormat": "p95", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 10, + "pointSize": 5, + "showPoints": "auto", + "spanNulls": false, + "stacking": { "mode": "none" }, + "axisPlacement": "auto", + "lineWidth": 2, + "thresholdsStyle": { "mode": "line" } + }, + "unit": "s", + "color": { "fixedColor": "purple", "mode": "fixed" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 30 }, + { "color": "red", "value": 120 } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { "displayMode": "list", "placement": "bottom" }, + "tooltip": { "mode": "single" } + } + }, + { + "id": 38, + "type": "timeseries", + "title": "Argo K8s API Requests", + "description": "Rate of Kubernetes API requests from Argo controller", + "gridPos": { "h": 6, "w": 12, "x": 12, "y": 17 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "targets": [ + { + "expr": "sum(rate(argo_workflows_k8s_request_total[$__rate_interval]))", + "legendFormat": "k8s req/s", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 10, + "pointSize": 5, + "showPoints": "auto", + "spanNulls": false, + "stacking": { "mode": "none" }, + "axisPlacement": "auto", + "lineWidth": 2 + }, + "unit": "reqps", + "color": { "fixedColor": "orange", "mode": "fixed" } + }, + "overrides": [] + }, + "options": { + "legend": { "displayMode": "list", "placement": "bottom" }, + "tooltip": { "mode": "single" } + } + } + ] + }, + + { + "id": 40, + "type": "row", + "title": "API Gateway", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 11 }, + "collapsed": true, + "panels": [ + { + "id": 41, + "type": "timeseries", + "title": "Auth Request Rate", + "description": "Rate of authentication requests (total, success, failure)", + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 12 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "targets": [ + { + "expr": "rate(api_gateway_auth_requests_total[$__rate_interval])", + "legendFormat": "total", + "refId": "A" + }, + { + "expr": "rate(api_gateway_auth_requests_success[$__rate_interval])", + "legendFormat": "success", + "refId": "B" + }, + { + "expr": "rate(api_gateway_auth_requests_failure[$__rate_interval])", + "legendFormat": "failure", + "refId": "C" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 10, + "pointSize": 5, + "showPoints": "auto", + "spanNulls": false, + "stacking": { "mode": "none" }, + "axisPlacement": "auto", + "lineWidth": 2 + }, + "unit": "reqps", + "color": { "mode": "palette-classic" } + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "failure" }, + "properties": [{ "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } }] + }, + { + "matcher": { "id": "byName", "options": "success" }, + "properties": [{ "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } }] + } + ] + }, + "options": { + "legend": { "displayMode": "list", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + } + }, + { + "id": 42, + "type": "stat", + "title": "Rate Limiter Entries", + "description": "Current number of entries in the API gateway rate limiter", + "gridPos": { "h": 4, "w": 4, "x": 8, "y": 12 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "targets": [ + { + "expr": "api_gateway_rate_limiter_entries", + "legendFormat": "entries", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 100 }, + { "color": "red", "value": 500 } + ] + }, + "color": { "mode": "thresholds" }, + "unit": "none" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto" + } + }, + { + "id": 43, + "type": "stat", + "title": "JWT Cache Hit Ratio", + "description": "Ratio of JWT cache hits to total lookups", + "gridPos": { "h": 4, "w": 4, "x": 12, "y": 12 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "targets": [ + { + "expr": "api_gateway_jwt_cache_hits / (api_gateway_jwt_cache_hits + api_gateway_jwt_cache_misses)", + "legendFormat": "hit ratio", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "yellow", "value": 0.5 }, + { "color": "green", "value": 0.8 } + ] + }, + "color": { "mode": "thresholds" }, + "unit": "percentunit", + "min": 0, + "max": 1 + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto" + } + }, + { + "id": 44, + "type": "timeseries", + "title": "Routing Latency p50 / p95 / p99", + "description": "API gateway routing duration percentiles", + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 12 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "targets": [ + { + "expr": "histogram_quantile(0.50, sum by (le) (rate(api_gateway_routing_duration_microseconds_bucket[$__rate_interval])))", + "legendFormat": "p50", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, sum by (le) (rate(api_gateway_routing_duration_microseconds_bucket[$__rate_interval])))", + "legendFormat": "p95", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.99, sum by (le) (rate(api_gateway_routing_duration_microseconds_bucket[$__rate_interval])))", + "legendFormat": "p99", + "refId": "C" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 5, + "pointSize": 5, + "showPoints": "auto", + "spanNulls": false, + "stacking": { "mode": "none" }, + "axisPlacement": "auto", + "lineWidth": 2 + }, + "unit": "us", + "color": { "mode": "palette-classic" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 5000 }, + { "color": "red", "value": 50000 } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { "displayMode": "list", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + } + }, + { + "id": 45, + "type": "stat", + "title": "Auth Failure Rate", + "description": "Percentage of auth requests that fail", + "gridPos": { "h": 4, "w": 4, "x": 8, "y": 16 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "targets": [ + { + "expr": "rate(api_gateway_auth_requests_failure[$__rate_interval]) / rate(api_gateway_auth_requests_total[$__rate_interval])", + "legendFormat": "failure rate", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 0.05 }, + { "color": "red", "value": 0.2 } + ] + }, + "color": { "mode": "thresholds" }, + "unit": "percentunit", + "min": 0, + "max": 1 + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto" + } + }, + { + "id": 46, + "type": "stat", + "title": "JWT Cache Misses/s", + "description": "Rate of JWT cache misses", + "gridPos": { "h": 4, "w": 4, "x": 12, "y": 16 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "targets": [ + { + "expr": "rate(api_gateway_jwt_cache_misses[$__rate_interval])", + "legendFormat": "misses/s", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 5 }, + { "color": "red", "value": 20 } + ] + }, + "color": { "mode": "thresholds" }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto" + } + } + ] + }, + + { + "id": 50, + "type": "row", + "title": "Trading", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 12 }, + "collapsed": true, + "panels": [ + { + "id": 51, + "type": "stat", + "title": "Portfolio Value", + "description": "Current portfolio value in USD", + "gridPos": { "h": 6, "w": 6, "x": 0, "y": 13 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "targets": [ + { + "expr": "trading_agent_portfolio_value_usd", + "legendFormat": "portfolio", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "yellow", "value": 10000 }, + { "color": "green", "value": 100000 } + ] + }, + "color": { "mode": "thresholds" }, + "unit": "currencyUSD", + "decimals": 2 + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto" + } + }, + { + "id": 52, + "type": "stat", + "title": "Assets Selected", + "description": "Number of assets currently selected for trading", + "gridPos": { "h": 6, "w": 3, "x": 6, "y": 13 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "targets": [ + { + "expr": "trading_agent_assets_selected", + "legendFormat": "selected", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "blue", "value": null }, + { "color": "green", "value": 1 } + ] + }, + "color": { "mode": "thresholds" }, + "unit": "none" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto" + } + }, + { + "id": 53, + "type": "stat", + "title": "Universe Instruments", + "description": "Total instruments in the trading universe", + "gridPos": { "h": 6, "w": 3, "x": 9, "y": 13 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "targets": [ + { + "expr": "trading_agent_universe_instruments", + "legendFormat": "instruments", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "blue", "value": null } + ] + }, + "color": { "mode": "thresholds" }, + "unit": "none" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto" + } + }, + { + "id": 54, + "type": "timeseries", + "title": "Trading Measurements Rate", + "description": "Rate of trading measurements emitted", + "gridPos": { "h": 6, "w": 6, "x": 12, "y": 13 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "targets": [ + { + "expr": "rate(trading_measurements_total[$__rate_interval])", + "legendFormat": "measurements/s", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 15, + "pointSize": 5, + "showPoints": "auto", + "spanNulls": false, + "stacking": { "mode": "none" }, + "axisPlacement": "auto", + "lineWidth": 2 + }, + "unit": "ops", + "color": { "fixedColor": "blue", "mode": "fixed" } + }, + "overrides": [] + }, + "options": { + "legend": { "displayMode": "list", "placement": "bottom" }, + "tooltip": { "mode": "single" } + } + }, + { + "id": 55, + "type": "timeseries", + "title": "Total Trading Latency", + "description": "End-to-end trading latency", + "gridPos": { "h": 6, "w": 6, "x": 18, "y": 13 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "targets": [ + { + "expr": "trading_total_latency_seconds", + "legendFormat": "total latency", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 10, + "pointSize": 5, + "showPoints": "auto", + "spanNulls": false, + "stacking": { "mode": "none" }, + "axisPlacement": "auto", + "lineWidth": 2, + "thresholdsStyle": { "mode": "area" } + }, + "unit": "s", + "color": { "fixedColor": "orange", "mode": "fixed" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 0.01 }, + { "color": "red", "value": 0.1 } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { "displayMode": "list", "placement": "bottom" }, + "tooltip": { "mode": "single" } + } + }, + { + "id": 56, + "type": "timeseries", + "title": "Order Processing vs Risk Check vs Market Data", + "description": "Breakdown of trading pipeline latency components", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 19 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "targets": [ + { + "expr": "trading_order_processing_seconds", + "legendFormat": "order processing", + "refId": "A" + }, + { + "expr": "trading_risk_check_seconds", + "legendFormat": "risk check", + "refId": "B" + }, + { + "expr": "trading_market_data_seconds", + "legendFormat": "market data", + "refId": "C" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 10, + "pointSize": 5, + "showPoints": "auto", + "spanNulls": false, + "stacking": { "mode": "none" }, + "axisPlacement": "auto", + "lineWidth": 2 + }, + "unit": "s", + "color": { "mode": "palette-classic" } + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "order processing" }, + "properties": [{ "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } }] + }, + { + "matcher": { "id": "byName", "options": "risk check" }, + "properties": [{ "id": "color", "value": { "fixedColor": "orange", "mode": "fixed" } }] + }, + { + "matcher": { "id": "byName", "options": "market data" }, + "properties": [{ "id": "color", "value": { "fixedColor": "purple", "mode": "fixed" } }] + } + ] + }, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "lastNotNull"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + } + }, + { + "id": 57, + "type": "timeseries", + "title": "Portfolio Value Over Time", + "description": "Historical portfolio value trend", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 19 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "targets": [ + { + "expr": "trading_agent_portfolio_value_usd", + "legendFormat": "portfolio USD", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 20, + "pointSize": 5, + "showPoints": "never", + "spanNulls": true, + "stacking": { "mode": "none" }, + "axisPlacement": "auto", + "lineWidth": 2, + "gradientMode": "scheme" + }, + "unit": "currencyUSD", + "color": { "mode": "continuous-GrYlRd" }, + "decimals": 2 + }, + "overrides": [] + }, + "options": { + "legend": { "displayMode": "list", "placement": "bottom" }, + "tooltip": { "mode": "single" } + } + } + ] + }, + + { + "id": 60, + "type": "row", + "title": "Metrics Health", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 13 }, + "collapsed": true, + "panels": [ + { + "id": 61, + "type": "gauge", + "title": "Metrics Buffer Utilization", + "description": "Percentage of the metrics buffer currently in use", + "gridPos": { "h": 6, "w": 8, "x": 0, "y": 14 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "targets": [ + { + "expr": "metrics_buffer_utilization_percent", + "legendFormat": "buffer %", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 60 }, + { "color": "orange", "value": 80 }, + { "color": "red", "value": 95 } + ] + }, + "color": { "mode": "thresholds" }, + "unit": "percent", + "min": 0, + "max": 100 + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "orientation": "auto" + } + }, + { + "id": 62, + "type": "timeseries", + "title": "Metrics Dropped Rate", + "description": "Rate at which metrics are being dropped", + "gridPos": { "h": 6, "w": 8, "x": 8, "y": 14 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "targets": [ + { + "expr": "rate(metrics_dropped_total[$__rate_interval])", + "legendFormat": "dropped/s", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 20, + "pointSize": 5, + "showPoints": "auto", + "spanNulls": false, + "stacking": { "mode": "none" }, + "axisPlacement": "auto", + "lineWidth": 2, + "thresholdsStyle": { "mode": "area" } + }, + "unit": "ops", + "color": { "fixedColor": "red", "mode": "fixed" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 1 }, + { "color": "red", "value": 10 } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { "displayMode": "list", "placement": "bottom" }, + "tooltip": { "mode": "single" } + } + }, + { + "id": 63, + "type": "stat", + "title": "Total Metrics Dropped", + "description": "Total count of metrics dropped since service start", + "gridPos": { "h": 6, "w": 8, "x": 16, "y": 14 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "targets": [ + { + "expr": "metrics_dropped_total", + "legendFormat": "total dropped", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 100 }, + { "color": "red", "value": 1000 } + ] + }, + "color": { "mode": "thresholds" }, + "unit": "none" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto" + } + } + ] } ], "annotations": { "list": [ { "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, + "datasource": { "type": "grafana", "uid": "-- Grafana --" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", @@ -1782,4 +1678,4 @@ }, "links": [], "graphTooltip": 1 -} +} \ No newline at end of file diff --git a/infra/k8s/monitoring/dashboards/foxhunt-infrastructure.json b/infra/k8s/monitoring/dashboards/foxhunt-infrastructure.json index 925b6fe5f..83e5792de 100644 --- a/infra/k8s/monitoring/dashboards/foxhunt-infrastructure.json +++ b/infra/k8s/monitoring/dashboards/foxhunt-infrastructure.json @@ -1,22 +1,15 @@ { "id": null, "uid": "foxhunt-infrastructure", - "title": "Infrastructure & Cluster", - "description": "K8s cluster health, node pools, pods, storage, and networking for the foxhunt platform", - "tags": [ - "foxhunt", - "infrastructure", - "k8s" - ], + "title": "Foxhunt Infrastructure & Cluster", + "description": "K8s cluster health, node resources, GPU, storage, and Prometheus health", + "tags": ["foxhunt", "infrastructure", "k8s"], "timezone": "browser", "schemaVersion": 39, "version": 1, "editable": true, "refresh": "30s", - "time": { - "from": "now-3h", - "to": "now" - }, + "time": { "from": "now-3h", "to": "now" }, "fiscalYearStartMonth": 0, "liveNow": false, "weekStart": "", @@ -29,31 +22,20 @@ "auto": true, "auto_count": 30, "auto_min": "1m", - "current": { - "selected": true, - "text": "auto", - "value": "$__auto_interval_interval" - }, + "current": { "selected": true, "text": "auto", "value": "$__auto_interval_interval" }, "hide": 0, "refresh": 2 }, { "name": "node", "type": "query", - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "query": "label_values(node_uname_info, nodename)", "refresh": 2, "includeAll": true, "multi": true, "allValue": ".*", - "current": { - "selected": true, - "text": "All", - "value": "$__all" - }, + "current": { "selected": true, "text": "All", "value": "$__all" }, "sort": 1, "hide": 0 } @@ -61,92 +43,24 @@ }, "panels": [ { - "id": 100, + "id": 1, "type": "row", - "title": "Cluster Health", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, + "title": "Cluster Overview", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "collapsed": false, "panels": [] }, - { - "id": 1, - "type": "stat", - "title": "Nodes Ready", - "gridPos": { - "h": 4, - "w": 4, - "x": 0, - "y": 1 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "targets": [ - { - "expr": "sum(kube_node_status_condition{condition=\"Ready\",status=\"true\"})", - "legendFormat": "Ready Nodes", - "refId": "A", - "instant": true - } - ], - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - }, - "unit": "none", - "noValue": "0" - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "colorMode": "background", - "graphMode": "none", - "textMode": "auto", - "justifyMode": "auto" - } - }, { "id": 2, "type": "stat", - "title": "Total Pods", - "gridPos": { - "h": 4, - "w": 4, - "x": 4, - "y": 1 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, + "title": "Total Nodes", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 }, "targets": [ { - "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\",phase=\"Running\"})", - "legendFormat": "Running", - "refId": "A", - "instant": true + "expr": "count(kube_node_info)", + "legendFormat": "", + "refId": "A" } ], "fieldConfig": { @@ -154,53 +68,35 @@ "thresholds": { "mode": "absolute", "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 1 - } + { "color": "red", "value": null }, + { "color": "green", "value": 1 } ] }, - "unit": "none", - "noValue": "0" + "mappings": [], + "color": { "mode": "thresholds" } }, "overrides": [] }, "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "colorMode": "background", - "graphMode": "none", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", "textMode": "auto", + "colorMode": "value", + "graphMode": "none", "justifyMode": "auto" } }, { "id": 3, "type": "stat", - "title": "Pending Pods", - "gridPos": { - "h": 4, - "w": 4, - "x": 8, - "y": 1 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, + "title": "Nodes Ready", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { "h": 4, "w": 4, "x": 4, "y": 1 }, "targets": [ { - "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\",phase=\"Pending\"})", - "legendFormat": "Pending", - "refId": "A", - "instant": true + "expr": "sum(kube_node_status_condition{condition=\"Ready\", status=\"true\"})", + "legendFormat": "", + "refId": "A" } ], "fieldConfig": { @@ -208,53 +104,35 @@ "thresholds": { "mode": "absolute", "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } + { "color": "red", "value": null }, + { "color": "green", "value": 1 } ] }, - "unit": "none", - "noValue": "0" + "mappings": [], + "color": { "mode": "thresholds" } }, "overrides": [] }, "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "colorMode": "background", - "graphMode": "none", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", "textMode": "auto", + "colorMode": "value", + "graphMode": "none", "justifyMode": "auto" } }, { "id": 4, "type": "stat", - "title": "Restarts (1h)", - "gridPos": { - "h": 4, - "w": 4, - "x": 12, - "y": 1 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, + "title": "Pods Running", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 }, "targets": [ { - "expr": "sum(increase(kube_pod_container_status_restarts_total{namespace=\"foxhunt\"}[1h]))", - "legendFormat": "Restarts", - "refId": "A", - "instant": true + "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", phase=\"Running\"})", + "legendFormat": "", + "refId": "A" } ], "fieldConfig": { @@ -262,728 +140,925 @@ "thresholds": { "mode": "absolute", "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 3 - }, - { - "color": "red", - "value": 5 - } + { "color": "red", "value": null }, + { "color": "green", "value": 1 } ] }, - "unit": "none", - "noValue": "0", - "decimals": 0 + "mappings": [], + "color": { "mode": "thresholds" } }, "overrides": [] }, "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "colorMode": "background", - "graphMode": "none", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", "textMode": "auto", + "colorMode": "value", + "graphMode": "none", "justifyMode": "auto" } }, { "id": 5, "type": "stat", - "title": "CPU Request %", - "gridPos": { - "h": 4, - "w": 4, - "x": 16, - "y": 1 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, + "title": "Total CPU Capacity", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { "h": 4, "w": 4, "x": 12, "y": 1 }, "targets": [ { - "expr": "sum(kube_pod_container_resource_requests{namespace=\"foxhunt\",resource=\"cpu\"}) / sum(kube_node_status_allocatable{resource=\"cpu\"}) * 100", - "legendFormat": "CPU Req %", - "refId": "A", - "instant": true + "expr": "sum(kube_node_status_capacity{resource=\"cpu\"})", + "legendFormat": "", + "refId": "A" } ], "fieldConfig": { "defaults": { + "unit": "short", "thresholds": { "mode": "absolute", "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 70 - }, - { - "color": "red", - "value": 90 - } + { "color": "green", "value": null } ] }, - "unit": "percent", - "noValue": "0", - "decimals": 1 + "mappings": [], + "color": { "mode": "thresholds" } }, "overrides": [] }, "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "colorMode": "background", - "graphMode": "none", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", "textMode": "auto", - "justifyMode": "auto" + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "text": { "titleSize": 12 } } }, { "id": 6, "type": "stat", - "title": "Memory Request %", - "gridPos": { - "h": 4, - "w": 4, - "x": 20, - "y": 1 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, + "title": "Total Memory Capacity", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { "h": 4, "w": 4, "x": 16, "y": 1 }, "targets": [ { - "expr": "sum(kube_pod_container_resource_requests{namespace=\"foxhunt\",resource=\"memory\"}) / sum(kube_node_status_allocatable{resource=\"memory\"}) * 100", - "legendFormat": "Mem Req %", + "expr": "sum(kube_node_status_capacity{resource=\"memory\"})", + "legendFormat": "", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto" + } + }, + { + "id": 7, + "type": "stat", + "title": "GPUs Available", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { "h": 4, "w": 4, "x": 20, "y": 1 }, + "targets": [ + { + "expr": "sum(kube_node_status_capacity{resource=\"nvidia_com_gpu\"}) or vector(0)", + "legendFormat": "", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "yellow", "value": null }, + { "color": "green", "value": 1 } + ] + }, + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto" + } + }, + { + "id": 10, + "type": "row", + "title": "Node Resources", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, + "collapsed": false, + "panels": [] + }, + { + "id": 11, + "type": "timeseries", + "title": "CPU Usage by Node (stacked)", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 }, + "targets": [ + { + "expr": "1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[$__rate_interval]))", + "legendFormat": "{{ instance }}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "min": 0, + "max": 1, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 30, + "stacking": { "mode": "normal", "group": "A" }, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0, + "axisSoftMax": 1 + }, + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" } + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } + } + }, + { + "id": 12, + "type": "timeseries", + "title": "Memory Usage by Node", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 }, + "targets": [ + { + "expr": "1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)", + "legendFormat": "{{ instance }}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "min": 0, + "max": 1, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 20, + "stacking": { "mode": "none" }, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false + }, + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "orange", "value": 0.8 }, { "color": "red", "value": 0.95 }] }, + "color": { "mode": "palette-classic" } + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } + } + }, + { + "id": 13, + "type": "timeseries", + "title": "Disk I/O (Read / Write)", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 14 }, + "targets": [ + { + "expr": "sum by (instance) (rate(node_disk_read_bytes_total[$__rate_interval]))", + "legendFormat": "{{ instance }} read", + "refId": "A" + }, + { + "expr": "sum by (instance) (rate(node_disk_written_bytes_total[$__rate_interval]))", + "legendFormat": "{{ instance }} write", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "Bps", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 15, + "stacking": { "mode": "none" }, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false + }, + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" } + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "list", "placement": "bottom" } + } + }, + { + "id": 14, + "type": "timeseries", + "title": "Network I/O (Receive / Transmit)", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 14 }, + "targets": [ + { + "expr": "sum by (instance) (rate(node_network_receive_bytes_total{device!~\"lo|veth.*|cali.*|flannel.*\"}[$__rate_interval]))", + "legendFormat": "{{ instance }} rx", + "refId": "A" + }, + { + "expr": "sum by (instance) (rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|cali.*|flannel.*\"}[$__rate_interval]))", + "legendFormat": "{{ instance }} tx", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "Bps", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 15, + "stacking": { "mode": "none" }, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false + }, + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" } + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "list", "placement": "bottom" } + } + }, + { + "id": 15, + "type": "timeseries", + "title": "Load Average (1m / 5m / 15m)", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 14 }, + "targets": [ + { + "expr": "node_load1", + "legendFormat": "{{ instance }} load1", + "refId": "A" + }, + { + "expr": "node_load5", + "legendFormat": "{{ instance }} load5", + "refId": "B" + }, + { + "expr": "node_load15", + "legendFormat": "{{ instance }} load15", + "refId": "C" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 0, + "stacking": { "mode": "none" }, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false + }, + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" } + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "list", "placement": "bottom" } + } + }, + { + "id": 20, + "type": "row", + "title": "Kubernetes Workloads", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 22 }, + "collapsed": false, + "panels": [] + }, + { + "id": 21, + "type": "timeseries", + "title": "Deployment Replicas (Desired vs Available)", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 23 }, + "targets": [ + { + "expr": "kube_deployment_spec_replicas{namespace=\"foxhunt\"}", + "legendFormat": "{{ deployment }} desired", + "refId": "A" + }, + { + "expr": "kube_deployment_status_replicas_available{namespace=\"foxhunt\"}", + "legendFormat": "{{ deployment }} available", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "lineInterpolation": "stepAfter", + "fillOpacity": 0, + "stacking": { "mode": "none" }, + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false + }, + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" } + }, + "overrides": [ + { + "matcher": { "id": "byRegexp", "options": ".*desired" }, + "properties": [{ "id": "custom.lineStyle", "value": { "fill": "dash", "dash": [10, 10] } }] + } + ] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull"] } + } + }, + { + "id": 22, + "type": "piechart", + "title": "Pod Phase Distribution", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { "h": 8, "w": 6, "x": 12, "y": 23 }, + "targets": [ + { + "expr": "sum by (phase) (kube_pod_status_phase{namespace=\"foxhunt\"})", + "legendFormat": "{{ phase }}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } + }, + "overrides": [ + { "matcher": { "id": "byName", "options": "Running" }, "properties": [{ "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } }] }, + { "matcher": { "id": "byName", "options": "Pending" }, "properties": [{ "id": "color", "value": { "fixedColor": "yellow", "mode": "fixed" } }] }, + { "matcher": { "id": "byName", "options": "Failed" }, "properties": [{ "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } }] }, + { "matcher": { "id": "byName", "options": "Succeeded" }, "properties": [{ "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } }] } + ] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "pieType": "donut", + "tooltip": { "mode": "single" }, + "legend": { "displayMode": "list", "placement": "right" } + } + }, + { + "id": 23, + "type": "timeseries", + "title": "Container Restarts", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { "h": 8, "w": 6, "x": 18, "y": 23 }, + "targets": [ + { + "expr": "increase(kube_pod_container_status_restarts_total{namespace=\"foxhunt\"}[$__rate_interval])", + "legendFormat": "{{ pod }} / {{ container }}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "bars", + "lineInterpolation": "linear", + "fillOpacity": 80, + "stacking": { "mode": "normal", "group": "A" }, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false + }, + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "red", "value": 1 }] }, + "color": { "mode": "palette-classic" } + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "list", "placement": "bottom" } + } + }, + { + "id": 24, + "type": "table", + "title": "Pending / Failed Pods", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { "h": 6, "w": 24, "x": 0, "y": 31 }, + "targets": [ + { + "expr": "kube_pod_status_phase{namespace=\"foxhunt\", phase=~\"Pending|Failed\"} > 0", + "legendFormat": "", + "refId": "A", + "format": "table", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "thresholds" } + }, + "overrides": [ + { "matcher": { "id": "byName", "options": "Time" }, "properties": [{ "id": "custom.hidden", "value": true }] }, + { "matcher": { "id": "byName", "options": "__name__" }, "properties": [{ "id": "custom.hidden", "value": true }] }, + { "matcher": { "id": "byName", "options": "job" }, "properties": [{ "id": "custom.hidden", "value": true }] }, + { "matcher": { "id": "byName", "options": "instance" }, "properties": [{ "id": "custom.hidden", "value": true }] } + ] + }, + "options": { + "showHeader": true, + "sortBy": [{ "desc": true, "displayName": "Value" }], + "footer": { "show": false } + }, + "transformations": [ + { "id": "organize", "options": { "excludeByName": { "Time": true, "__name__": true, "job": true, "instance": true }, "indexByName": {}, "renameByName": { "pod": "Pod", "phase": "Phase", "namespace": "Namespace", "Value": "Count" } } } + ] + }, + { + "id": 30, + "type": "row", + "title": "Storage", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 37 }, + "collapsed": false, + "panels": [] + }, + { + "id": 31, + "type": "bargauge", + "title": "Disk Usage by Mountpoint", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 38 }, + "targets": [ + { + "expr": "1 - (node_filesystem_avail_bytes{fstype=~\"ext4|xfs|btrfs\"} / node_filesystem_size_bytes{fstype=~\"ext4|xfs|btrfs\"})", + "legendFormat": "{{ instance }} {{ mountpoint }}", "refId": "A", "instant": true } ], "fieldConfig": { "defaults": { + "unit": "percentunit", + "min": 0, + "max": 1, "thresholds": { "mode": "absolute", "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 70 - }, - { - "color": "red", - "value": 90 - } + { "color": "green", "value": null }, + { "color": "yellow", "value": 0.7 }, + { "color": "orange", "value": 0.85 }, + { "color": "red", "value": 0.95 } ] }, - "unit": "percent", - "noValue": "0", - "decimals": 1 + "color": { "mode": "thresholds" } }, "overrides": [] }, "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "horizontal", + "displayMode": "gradient", + "showUnfilled": true, + "minVizWidth": 0, + "minVizHeight": 10 + } + }, + { + "id": 32, + "type": "table", + "title": "PVC Status & Requested Storage", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 38 }, + "targets": [ + { + "expr": "kube_persistentvolumeclaim_status_phase{namespace=\"foxhunt\"}", + "legendFormat": "", + "refId": "A", + "format": "table", + "instant": true }, - "colorMode": "background", - "graphMode": "none", + { + "expr": "kube_persistentvolumeclaim_resource_requests_storage_bytes{namespace=\"foxhunt\"}", + "legendFormat": "", + "refId": "B", + "format": "table", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "thresholds" } + }, + "overrides": [ + { "matcher": { "id": "byName", "options": "Storage" }, "properties": [{ "id": "unit", "value": "bytes" }] } + ] + }, + "options": { + "showHeader": true, + "footer": { "show": false } + }, + "transformations": [ + { + "id": "seriesToColumns", + "options": { "byField": "persistentvolumeclaim" } + }, + { + "id": "organize", + "options": { + "excludeByName": { "Time 1": true, "Time 2": true, "__name__ 1": true, "__name__ 2": true, "job 1": true, "job 2": true, "instance 1": true, "instance 2": true, "namespace 1": true, "namespace 2": true, "phase": false }, + "renameByName": { "persistentvolumeclaim": "PVC", "phase": "Phase", "Value 1": "Phase Value", "Value 2": "Storage" } + } + } + ] + }, + { + "id": 40, + "type": "row", + "title": "GPU Resources", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 46 }, + "collapsed": false, + "panels": [] + }, + { + "id": 41, + "type": "timeseries", + "title": "GPU Utilization %", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 47 }, + "targets": [ + { + "expr": "DCGM_FI_DEV_GPU_UTIL", + "legendFormat": "GPU {{ gpu }} ({{ instance }})", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "max": 100, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 25, + "stacking": { "mode": "none" }, + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "thresholdsStyle": { "mode": "line" } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 80 }, + { "color": "red", "value": 95 } + ] + }, + "color": { "mode": "palette-classic" } + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "list", "placement": "bottom" } + } + }, + { + "id": 42, + "type": "timeseries", + "title": "VRAM Usage", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 47 }, + "targets": [ + { + "expr": "DCGM_FI_DEV_FB_USED", + "legendFormat": "GPU {{ gpu }} used", + "refId": "A" + }, + { + "expr": "DCGM_FI_DEV_FB_FREE", + "legendFormat": "GPU {{ gpu }} free", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "decmbytes", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 20, + "stacking": { "mode": "none" }, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false + }, + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" } + }, + "overrides": [ + { "matcher": { "id": "byRegexp", "options": ".*free" }, "properties": [{ "id": "custom.fillOpacity", "value": 5 }, { "id": "custom.lineStyle", "value": { "fill": "dash", "dash": [10, 10] } }] } + ] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "list", "placement": "bottom" } + } + }, + { + "id": 43, + "type": "timeseries", + "title": "GPU Temperature", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 47 }, + "targets": [ + { + "expr": "DCGM_FI_DEV_GPU_TEMP", + "legendFormat": "GPU {{ gpu }} core", + "refId": "A" + }, + { + "expr": "DCGM_FI_DEV_MEMORY_TEMP", + "legendFormat": "GPU {{ gpu }} mem", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "celsius", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 0, + "stacking": { "mode": "none" }, + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "thresholdsStyle": { "mode": "line" } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "orange", "value": 75 }, + { "color": "red", "value": 90 } + ] + }, + "color": { "mode": "palette-classic" } + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "list", "placement": "bottom" } + } + }, + { + "id": 44, + "type": "timeseries", + "title": "GPU Power Draw", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 55 }, + "targets": [ + { + "expr": "DCGM_FI_DEV_POWER_USAGE", + "legendFormat": "GPU {{ gpu }}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "watt", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 15, + "stacking": { "mode": "none" }, + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false + }, + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" } + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "list", "placement": "bottom" } + } + }, + { + "id": 45, + "type": "timeseries", + "title": "GPU PCIe Throughput", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 55 }, + "targets": [ + { + "expr": "DCGM_FI_PROF_PCIE_RX_BYTES", + "legendFormat": "GPU {{ gpu }} rx", + "refId": "A" + }, + { + "expr": "DCGM_FI_PROF_PCIE_TX_BYTES", + "legendFormat": "GPU {{ gpu }} tx", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "Bps", + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "fillOpacity": 15, + "stacking": { "mode": "none" }, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false + }, + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "color": { "mode": "palette-classic" } + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "list", "placement": "bottom" } + } + }, + { + "id": 50, + "type": "row", + "title": "Prometheus Health", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 63 }, + "collapsed": false, + "panels": [] + }, + { + "id": 51, + "type": "stat", + "title": "Active Time Series", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { "h": 4, "w": 6, "x": 0, "y": 64 }, + "targets": [ + { + "expr": "prometheus_tsdb_head_series", + "legendFormat": "", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 500000 }, + { "color": "red", "value": 1000000 } + ] + }, + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", "textMode": "auto", + "colorMode": "value", + "graphMode": "area", "justifyMode": "auto" } }, { - "id": 101, - "type": "row", - "title": "Node CPU & Memory", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 5 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 7, - "type": "timeseries", - "title": "CPU Usage by Node", - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 6 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, + "id": 52, + "type": "stat", + "title": "TSDB Storage Size", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { "h": 4, "w": 6, "x": 6, "y": 64 }, "targets": [ { - "expr": "1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[$interval]))", - "legendFormat": "{{ instance }}", + "expr": "prometheus_tsdb_storage_blocks_bytes", + "legendFormat": "", "refId": "A" } ], "fieldConfig": { "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 0.7 - }, - { - "color": "red", - "value": 0.9 - } - ] - }, - "unit": "percentunit", - "min": 0, - "max": 1, - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 15, - "lineWidth": 2, - "pointSize": 5, - "showPoints": "never", - "spanNulls": true, - "axisCenteredZero": false, - "axisColorMode": "text", - "scaleDistribution": { - "type": "linear" - }, - "thresholdsStyle": { - "mode": "off" - } - } - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "max" - ] - } - } - }, - { - "id": 8, - "type": "timeseries", - "title": "Memory Usage by Node", - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 6 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "targets": [ - { - "expr": "1 - (node_memory_AvailableBytes / node_memory_MemoryTotal_bytes)", - "legendFormat": "{{ instance }}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 0.7 - }, - { - "color": "red", - "value": 0.9 - } - ] - }, - "unit": "percentunit", - "min": 0, - "max": 1, - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 15, - "lineWidth": 2, - "pointSize": 5, - "showPoints": "never", - "spanNulls": true, - "axisCenteredZero": false, - "axisColorMode": "text", - "scaleDistribution": { - "type": "linear" - }, - "thresholdsStyle": { - "mode": "off" - } - } - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "max" - ] - } - } - }, - { - "id": 102, - "type": "row", - "title": "Pod Resources", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 14 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 9, - "type": "timeseries", - "title": "Pod CPU Usage", - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 15 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "targets": [ - { - "expr": "topk(10, sum by (pod) (rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\",container!=\"POD\",container!=\"\"}[$interval])))", - "legendFormat": "{{ pod }}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short", - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 10, - "lineWidth": 2, - "pointSize": 5, - "showPoints": "never", - "spanNulls": true, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "CPU cores", - "scaleDistribution": { - "type": "linear" - }, - "thresholdsStyle": { - "mode": "off" - } - } - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "max" - ] - } - } - }, - { - "id": 10, - "type": "timeseries", - "title": "Pod Memory Usage", - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 15 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "targets": [ - { - "expr": "topk(10, sum by (pod) (container_memory_working_set_bytes{namespace=\"foxhunt\",container!=\"POD\",container!=\"\"}))", - "legendFormat": "{{ pod }}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, "unit": "bytes", - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 10, - "lineWidth": 2, - "pointSize": 5, - "showPoints": "never", - "spanNulls": true, - "axisCenteredZero": false, - "axisColorMode": "text", - "scaleDistribution": { - "type": "linear" - }, - "thresholdsStyle": { - "mode": "off" - } - } + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 5368709120 }, + { "color": "red", "value": 10737418240 } + ] + }, + "mappings": [], + "color": { "mode": "thresholds" } }, "overrides": [] }, "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "max" - ] - } + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto" } }, { - "id": 103, - "type": "row", - "title": "Storage", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 23 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 11, - "type": "table", - "title": "PVC Usage", - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 24 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, + "id": 53, + "type": "stat", + "title": "Scrape Targets Up", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { "h": 4, "w": 6, "x": 12, "y": 64 }, "targets": [ { - "expr": "kubelet_volume_stats_used_bytes{namespace=\"foxhunt\"}", - "legendFormat": "", - "refId": "A", - "instant": true, - "format": "table" - }, - { - "expr": "kubelet_volume_stats_capacity_bytes{namespace=\"foxhunt\"}", - "legendFormat": "", - "refId": "B", - "instant": true, - "format": "table" - }, - { - "expr": "kubelet_volume_stats_used_bytes{namespace=\"foxhunt\"} / kubelet_volume_stats_capacity_bytes{namespace=\"foxhunt\"} * 100", - "legendFormat": "", - "refId": "C", - "instant": true, - "format": "table" - } - ], - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 70 - }, - { - "color": "red", - "value": 90 - } - ] - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "filterable": true - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Value #A" - }, - "properties": [ - { - "id": "displayName", - "value": "Used" - }, - { - "id": "unit", - "value": "bytes" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value #B" - }, - "properties": [ - { - "id": "displayName", - "value": "Total" - }, - { - "id": "unit", - "value": "bytes" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value #C" - }, - "properties": [ - { - "id": "displayName", - "value": "Used %" - }, - { - "id": "unit", - "value": "percent" - }, - { - "id": "custom.cellOptions", - "value": { - "mode": "gradient", - "type": "gauge" - } - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 70 - }, - { - "color": "red", - "value": 90 - } - ] - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "persistentvolumeclaim" - }, - "properties": [ - { - "id": "displayName", - "value": "PVC" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Time" - }, - "properties": [ - { - "id": "custom.hidden", - "value": true - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "__name__" - }, - "properties": [ - { - "id": "custom.hidden", - "value": true - } - ] - } - ] - }, - "options": { - "showHeader": true, - "sortBy": [ - { - "displayName": "Used %", - "desc": true - } - ], - "footer": { - "show": false - } - }, - "transformations": [ - { - "id": "merge", - "options": {} - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "__name__": true, - "endpoint": true, - "instance": true, - "job": true, - "metrics_path": true, - "node": true, - "service": true - }, - "indexByName": {}, - "renameByName": { - "persistentvolumeclaim": "PVC", - "namespace": "Namespace" - } - } - } - ] - }, - { - "id": 12, - "type": "timeseries", - "title": "Disk I/O", - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 24 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "targets": [ - { - "expr": "rate(node_disk_read_bytes_total{device=~\"sd.*|nvme.*\"}[$interval])", - "legendFormat": "{{ instance }} {{ device }} read", + "expr": "count(up == 1)", + "legendFormat": "up", "refId": "A" }, { - "expr": "rate(node_disk_written_bytes_total{device=~\"sd.*|nvme.*\"}[$interval])", - "legendFormat": "{{ instance }} {{ device }} write", + "expr": "count(up == 0)", + "legendFormat": "down", "refId": "B" } ], @@ -992,847 +1067,105 @@ "thresholds": { "mode": "absolute", "steps": [ - { - "color": "green", - "value": null - } + { "color": "green", "value": null } ] }, - "unit": "Bps", + "mappings": [], + "color": { "mode": "thresholds" } + }, + "overrides": [ + { "matcher": { "id": "byName", "options": "down" }, "properties": [{ "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } }] } + ] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "value_and_name", + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto" + } + }, + { + "id": 54, + "type": "timeseries", + "title": "Scrape Duration by Job", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 68 }, + "targets": [ + { + "expr": "scrape_duration_seconds", + "legendFormat": "{{ job }} / {{ instance }}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "fillOpacity": 10, - "lineWidth": 2, - "pointSize": 5, - "showPoints": "never", - "spanNulls": true, - "axisCenteredZero": false, - "axisColorMode": "text", - "scaleDistribution": { - "type": "linear" - }, - "thresholdsStyle": { - "mode": "off" - } - } - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*write.*" - }, - "properties": [ - { - "id": "custom.transform", - "value": "negative-Y" - } - ] - } - ] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "max" - ] - } - } - }, - { - "id": 104, - "type": "row", - "title": "Network", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 32 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 13, - "type": "timeseries", - "title": "Network Receive", - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 33 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "targets": [ - { - "expr": "sum by (instance) (rate(node_network_receive_bytes_total{device!=\"lo\"}[$interval]))", - "legendFormat": "{{ instance }}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "Bps", - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 15, - "lineWidth": 2, - "pointSize": 5, - "showPoints": "never", - "spanNulls": true, - "axisCenteredZero": false, - "axisColorMode": "text", - "scaleDistribution": { - "type": "linear" - }, - "thresholdsStyle": { - "mode": "off" - } - } - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "max" - ] - } - } - }, - { - "id": 14, - "type": "timeseries", - "title": "Network Transmit", - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 33 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "targets": [ - { - "expr": "sum by (instance) (rate(node_network_transmit_bytes_total{device!=\"lo\"}[$interval]))", - "legendFormat": "{{ instance }}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "Bps", - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 15, - "lineWidth": 2, - "pointSize": 5, - "showPoints": "never", - "spanNulls": true, - "axisCenteredZero": false, - "axisColorMode": "text", - "scaleDistribution": { - "type": "linear" - }, - "thresholdsStyle": { - "mode": "off" - } - } - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "max" - ] - } - } - }, - { - "id": 105, - "type": "row", - "title": "Pod Status Detail", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 41 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 15, - "type": "timeseries", - "title": "Pod Restarts", - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 42 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "targets": [ - { - "expr": "increase(kube_pod_container_status_restarts_total{namespace=\"foxhunt\"}[$interval])", - "legendFormat": "{{ pod }}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 1 - }, - { - "color": "red", - "value": 5 - } - ] - }, - "unit": "none", - "decimals": 0, - "custom": { - "drawStyle": "bars", - "lineInterpolation": "linear", - "fillOpacity": 80, + "stacking": { "mode": "none" }, "lineWidth": 1, "pointSize": 5, "showPoints": "never", - "spanNulls": true, - "axisCenteredZero": false, - "axisColorMode": "text", - "scaleDistribution": { - "type": "linear" - }, - "thresholdsStyle": { - "mode": "off" - }, - "stacking": { - "mode": "normal", - "group": "A" - } - } + "spanNulls": false + }, + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "red", "value": 5 }] }, + "color": { "mode": "palette-classic" } }, "overrides": [] }, "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "sum" - ] - } + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } } }, { - "id": 16, + "id": 55, "type": "table", - "title": "Container Status", - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 42 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, + "title": "Scrape Targets Status", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 68 }, "targets": [ { - "expr": "kube_pod_container_status_running{namespace=\"foxhunt\"}", + "expr": "up", "legendFormat": "", "refId": "A", - "instant": true, - "format": "table" - }, - { - "expr": "kube_pod_container_status_waiting{namespace=\"foxhunt\"}", - "legendFormat": "", - "refId": "B", - "instant": true, - "format": "table" - }, - { - "expr": "kube_pod_container_status_terminated{namespace=\"foxhunt\"}", - "legendFormat": "", - "refId": "C", - "instant": true, - "format": "table" + "format": "table", + "instant": true } ], "fieldConfig": { "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "filterable": true - } + "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] }, + "color": { "mode": "thresholds" }, + "mappings": [ + { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } } + ] }, "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Value #A" - }, - "properties": [ - { - "id": "displayName", - "value": "Running" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value #B" - }, - "properties": [ - { - "id": "displayName", - "value": "Waiting" - }, - { - "id": "custom.cellOptions", - "value": { - "mode": "basic", - "type": "color-background" - } - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "transparent", - "value": null - }, - { - "color": "orange", - "value": 1 - } - ] - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value #C" - }, - "properties": [ - { - "id": "displayName", - "value": "Terminated" - }, - { - "id": "custom.cellOptions", - "value": { - "mode": "basic", - "type": "color-background" - } - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "transparent", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Time" - }, - "properties": [ - { - "id": "custom.hidden", - "value": true - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "__name__" - }, - "properties": [ - { - "id": "custom.hidden", - "value": true - } - ] - } + { "matcher": { "id": "byName", "options": "Time" }, "properties": [{ "id": "custom.hidden", "value": true }] }, + { "matcher": { "id": "byName", "options": "__name__" }, "properties": [{ "id": "custom.hidden", "value": true }] }, + { "matcher": { "id": "byName", "options": "Value" }, "properties": [{ "id": "custom.cellOptions", "value": { "type": "color-background" } }] } ] }, "options": { "showHeader": true, - "sortBy": [ - { - "displayName": "Waiting", - "desc": true - } - ], - "footer": { - "show": false - } + "sortBy": [{ "desc": false, "displayName": "Value" }], + "footer": { "show": false } }, "transformations": [ - { - "id": "merge", - "options": {} - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "__name__": true, - "endpoint": true, - "instance": true, - "job": true, - "metrics_path": true, - "node": true, - "service": true, - "uid": true - }, - "indexByName": {}, - "renameByName": { - "pod": "Pod", - "container": "Container", - "namespace": "Namespace" - } - } - } + { "id": "organize", "options": { "excludeByName": { "Time": true, "__name__": true }, "renameByName": { "job": "Job", "instance": "Instance", "Value": "Status" } } } ] - }, - { - "id": 106, - "type": "row", - "title": "kube-state-metrics", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 50 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 17, - "type": "table", - "title": "Deployment Replicas", - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 51 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "targets": [ - { - "expr": "kube_deployment_spec_replicas{namespace=\"foxhunt\"}", - "legendFormat": "", - "refId": "A", - "instant": true, - "format": "table" - }, - { - "expr": "kube_deployment_status_replicas_available{namespace=\"foxhunt\"}", - "legendFormat": "", - "refId": "B", - "instant": true, - "format": "table" - }, - { - "expr": "kube_deployment_status_replicas_unavailable{namespace=\"foxhunt\"}", - "legendFormat": "", - "refId": "C", - "instant": true, - "format": "table" - } - ], - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "filterable": true - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Value #A" - }, - "properties": [ - { - "id": "displayName", - "value": "Desired" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value #B" - }, - "properties": [ - { - "id": "displayName", - "value": "Available" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Value #C" - }, - "properties": [ - { - "id": "displayName", - "value": "Unavailable" - }, - { - "id": "custom.cellOptions", - "value": { - "mode": "basic", - "type": "color-background" - } - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "transparent", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Time" - }, - "properties": [ - { - "id": "custom.hidden", - "value": true - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "__name__" - }, - "properties": [ - { - "id": "custom.hidden", - "value": true - } - ] - } - ] - }, - "options": { - "showHeader": true, - "sortBy": [ - { - "displayName": "Unavailable", - "desc": true - } - ], - "footer": { - "show": false - } - }, - "transformations": [ - { - "id": "merge", - "options": {} - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "__name__": true, - "endpoint": true, - "instance": true, - "job": true, - "metrics_path": true, - "node": true, - "service": true, - "uid": true - }, - "indexByName": {}, - "renameByName": { - "deployment": "Deployment", - "namespace": "Namespace" - } - } - } - ] - }, - { - "id": 18, - "type": "timeseries", - "title": "HPA Status", - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 51 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "targets": [ - { - "expr": "kube_horizontalpodautoscaler_status_current_replicas{namespace=\"foxhunt\"}", - "legendFormat": "{{ horizontalpodautoscaler }} current", - "refId": "A" - }, - { - "expr": "kube_horizontalpodautoscaler_spec_min_replicas{namespace=\"foxhunt\"}", - "legendFormat": "{{ horizontalpodautoscaler }} min", - "refId": "B" - }, - { - "expr": "kube_horizontalpodautoscaler_spec_max_replicas{namespace=\"foxhunt\"}", - "legendFormat": "{{ horizontalpodautoscaler }} max", - "refId": "C" - }, - { - "expr": "kube_horizontalpodautoscaler_status_desired_replicas{namespace=\"foxhunt\"}", - "legendFormat": "{{ horizontalpodautoscaler }} desired", - "refId": "D" - } - ], - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "none", - "decimals": 0, - "custom": { - "drawStyle": "line", - "lineInterpolation": "stepAfter", - "fillOpacity": 0, - "lineWidth": 2, - "pointSize": 5, - "showPoints": "never", - "spanNulls": true, - "axisCenteredZero": false, - "axisColorMode": "text", - "scaleDistribution": { - "type": "linear" - }, - "thresholdsStyle": { - "mode": "off" - } - } - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*min.*" - }, - "properties": [ - { - "id": "custom.lineStyle", - "value": { - "fill": "dash", - "dash": [ - 10, - 10 - ] - } - }, - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "orange" - } - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*max.*" - }, - "properties": [ - { - "id": "custom.lineStyle", - "value": { - "fill": "dash", - "dash": [ - 10, - 10 - ] - } - }, - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "red" - } - } - ] - } - ] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "lastNotNull" - ] - } - } } ], "annotations": { "list": [ { "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, + "datasource": { "type": "grafana", "uid": "-- Grafana --" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", diff --git a/infra/k8s/monitoring/dashboards/foxhunt-observability.json b/infra/k8s/monitoring/dashboards/foxhunt-observability.json index 7d288b579..2220204e4 100644 --- a/infra/k8s/monitoring/dashboards/foxhunt-observability.json +++ b/infra/k8s/monitoring/dashboards/foxhunt-observability.json @@ -10,550 +10,250 @@ "panels": [ { "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 100, - "title": "API Gateway Overview", + "title": "API Gateway Auth", "type": "row" }, { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { - "color": { - "mode": "thresholds" + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisLabel": "", + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "auto", + "stacking": { "mode": "none" } }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } + "unit": "ops" + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "failure" }, + "properties": [ + { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } ] }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 0, - "y": 1 + { + "matcher": { "id": "byName", "options": "success" }, + "properties": [ + { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } + ] + } + ] }, + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 1 }, "id": 1, "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "colorMode": "background", - "graphMode": "none" + "tooltip": { "mode": "multi" }, + "legend": { "displayMode": "table", "placement": "bottom" } }, - "title": "Total Request Rate", - "type": "stat", + "title": "Auth Request Rate (Success / Failure)", + "type": "timeseries", "targets": [ { - "expr": "sum(rate(api_gateway_http_requests_total[$interval]))", - "legendFormat": "", + "expr": "rate(api_gateway_auth_requests_success[$interval])", + "legendFormat": "success", "refId": "A" + }, + { + "expr": "rate(api_gateway_auth_requests_failure[$interval])", + "legendFormat": "failure", + "refId": "B" + }, + { + "expr": "rate(api_gateway_auth_requests_total[$interval])", + "legendFormat": "total", + "refId": "C" } ] }, { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { - "color": { - "mode": "thresholds" + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisLabel": "", + "drawStyle": "bars", + "fillOpacity": 50, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "stacking": { "mode": "normal" } }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 1 - }, - { - "color": "red", - "value": 5 - } - ] - }, - "unit": "percent" + "unit": "ops" }, "overrides": [] }, - "gridPos": { - "h": 4, - "w": 4, - "x": 4, - "y": 1 - }, + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 1 }, "id": 2, "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "colorMode": "background", - "graphMode": "none" + "tooltip": { "mode": "multi" }, + "legend": { "displayMode": "table", "placement": "bottom" } }, - "title": "Error Rate %", - "type": "stat", + "title": "Auth Error Breakdown", + "type": "timeseries", "targets": [ { - "expr": "sum(rate(api_gateway_http_requests_total{status=~\"5..\"}[$interval])) / sum(rate(api_gateway_http_requests_total[$interval])) * 100", - "legendFormat": "", + "expr": "rate(api_gateway_auth_errors_expired_jwt[$interval])", + "legendFormat": "expired_jwt", "refId": "A" + }, + { + "expr": "rate(api_gateway_auth_errors_invalid_jwt[$interval])", + "legendFormat": "invalid_jwt", + "refId": "B" + }, + { + "expr": "rate(api_gateway_auth_errors_missing_jwt[$interval])", + "legendFormat": "missing_jwt", + "refId": "C" + }, + { + "expr": "rate(api_gateway_auth_errors_permission_denied[$interval])", + "legendFormat": "permission_denied", + "refId": "D" + }, + { + "expr": "rate(api_gateway_auth_errors_rate_limited[$interval])", + "legendFormat": "rate_limited", + "refId": "E" } ] }, { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { - "color": { - "mode": "thresholds" + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisLabel": "", + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "auto", + "stacking": { "mode": "none" } }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.5 - }, - { - "color": "red", - "value": 1 - } - ] - }, - "unit": "s" + "unit": "\u00b5s" }, "overrides": [] }, - "gridPos": { - "h": 4, - "w": 4, - "x": 8, - "y": 1 - }, + "gridPos": { "h": 8, "w": 4, "x": 16, "y": 1 }, "id": 3, "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "colorMode": "background", - "graphMode": "none" + "tooltip": { "mode": "multi" }, + "legend": { "displayMode": "list", "placement": "bottom" } }, - "title": "P99 Latency", - "type": "stat", - "targets": [ - { - "expr": "histogram_quantile(0.99, sum(rate(api_gateway_request_duration_seconds_bucket[$interval])) by (le))", - "legendFormat": "", - "refId": "A" - } - ] - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 12, - "y": 1 - }, - "id": 4, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "colorMode": "background", - "graphMode": "none" - }, - "title": "Active WebSockets", - "type": "stat", - "targets": [ - { - "expr": "api_gateway_websocket_connections_active or vector(0)", - "legendFormat": "", - "refId": "A" - } - ] - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 1 - }, - { - "color": "red", - "value": 10 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 16, - "y": 1 - }, - "id": 5, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "colorMode": "background", - "graphMode": "none" - }, - "title": "Rate Limit Rejections/s", - "type": "stat", - "targets": [ - { - "expr": "sum(rate(api_gateway_rate_limit_rejected_total[$interval]))", - "legendFormat": "", - "refId": "A" - } - ] - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 0.5 - }, - { - "color": "red", - "value": 2 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 20, - "y": 1 - }, - "id": 6, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "colorMode": "background", - "graphMode": "none" - }, - "title": "Auth Failures/s", - "type": "stat", - "targets": [ - { - "expr": "sum(rate(api_gateway_auth_attempts_total{result=\"failure\"}[$interval]))", - "legendFormat": "", - "refId": "A" - } - ] - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 5 - }, - "id": 101, - "title": "Request Patterns", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisLabel": "", - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "pointSize": 5, - "showPoints": "auto", - "stacking": { - "mode": "none" - } - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 6 - }, - "id": 7, - "options": { - "tooltip": { - "mode": "multi" - }, - "legend": { - "displayMode": "table", - "placement": "bottom" - } - }, - "title": "Request Rate by Handler", + "title": "JWT Validation Latency", "type": "timeseries", "targets": [ { - "expr": "topk(10, sum by (handler) (rate(api_gateway_http_requests_total[$interval])))", - "legendFormat": "{{handler}}", - "refId": "A" - } - ] - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisLabel": "", - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "pointSize": 5, - "showPoints": "auto", - "stacking": { - "mode": "none" - } - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 6 - }, - "id": 8, - "options": { - "tooltip": { - "mode": "multi" - }, - "legend": { - "displayMode": "table", - "placement": "bottom" - } - }, - "title": "Latency by Handler", - "type": "timeseries", - "targets": [ - { - "expr": "histogram_quantile(0.50, sum by (le, handler) (rate(api_gateway_request_duration_seconds_bucket[$interval])))", - "legendFormat": "p50 {{handler}}", - "refId": "A" - }, - { - "expr": "histogram_quantile(0.95, sum by (le, handler) (rate(api_gateway_request_duration_seconds_bucket[$interval])))", - "legendFormat": "p95 {{handler}}", - "refId": "B" - } - ] - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 14 - }, - "id": 102, - "title": "Auth & Security", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisLabel": "", - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 1, - "pointSize": 5, - "showPoints": "auto", - "stacking": { - "mode": "none" - } - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 15 - }, - "id": 9, - "options": { - "tooltip": { - "mode": "multi" - }, - "legend": { - "displayMode": "table", - "placement": "bottom" - } - }, - "title": "Auth Timing", - "type": "timeseries", - "targets": [ - { - "expr": "histogram_quantile(0.50, rate(api_gateway_auth_duration_seconds_bucket[$interval]))", + "expr": "histogram_quantile(0.50, rate(api_gateway_jwt_validation_duration_microseconds_bucket[$interval]))", "legendFormat": "p50", "refId": "A" }, { - "expr": "histogram_quantile(0.95, rate(api_gateway_auth_duration_seconds_bucket[$interval]))", + "expr": "histogram_quantile(0.95, rate(api_gateway_jwt_validation_duration_microseconds_bucket[$interval]))", "legendFormat": "p95", "refId": "B" }, { - "expr": "histogram_quantile(0.99, rate(api_gateway_auth_duration_seconds_bucket[$interval]))", + "expr": "histogram_quantile(0.99, rate(api_gateway_jwt_validation_duration_microseconds_bucket[$interval]))", "legendFormat": "p99", "refId": "C" } ] }, { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { - "color": { - "mode": "palette-classic" + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "blue", "value": null } + ] }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 4, "x": 20, "y": 1 }, + "id": 4, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"] }, + "colorMode": "background", + "graphMode": "area" + }, + "title": "Active JWT Tokens", + "type": "stat", + "targets": [ + { + "expr": "api_gateway_active_jwt_tokens", + "legendFormat": "", + "refId": "A" + } + ] + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 9 }, + "id": 101, + "title": "API Gateway Security", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "orange", "value": 500 }, + { "color": "red", "value": 1000 } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 6, "w": 6, "x": 0, "y": 10 }, + "id": 5, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"] }, + "colorMode": "background", + "graphMode": "area" + }, + "title": "Rate Limiter Entries", + "type": "stat", + "targets": [ + { + "expr": "api_gateway_rate_limiter_entries", + "legendFormat": "", + "refId": "A" + } + ] + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -563,50 +263,168 @@ "lineWidth": 1, "pointSize": 5, "showPoints": "auto", - "stacking": { - "mode": "none" - } + "stacking": { "mode": "none" } }, - "unit": "ops" + "unit": "percentunit", + "min": 0, + "max": 1 }, "overrides": [] }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 15 - }, - "id": 10, + "gridPos": { "h": 6, "w": 6, "x": 6, "y": 10 }, + "id": 6, "options": { - "tooltip": { - "mode": "multi" - }, - "legend": { - "displayMode": "table", - "placement": "bottom" - } + "tooltip": { "mode": "multi" }, + "legend": { "displayMode": "list", "placement": "bottom" } }, - "title": "RBAC Checks", + "title": "RBAC Cache Hit Ratio", "type": "timeseries", "targets": [ { - "expr": "sum by (result) (rate(api_gateway_rbac_checks_total[$interval]))", - "legendFormat": "{{result}}", + "expr": "api_gateway_rbac_cache_hits / (api_gateway_rbac_cache_hits + api_gateway_rbac_cache_misses)", + "legendFormat": "hit ratio", "refId": "A" } ] }, { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { - "color": { - "mode": "palette-classic" + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "orange", "value": 10 }, + { "color": "red", "value": 50 } + ] }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 6, "w": 6, "x": 12, "y": 10 }, + "id": 7, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"] }, + "colorMode": "background", + "graphMode": "area" + }, + "title": "Revoked Tokens Cached", + "type": "stat", + "targets": [ + { + "expr": "api_gateway_revoked_tokens_cached", + "legendFormat": "", + "refId": "A" + } + ] + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisLabel": "", + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "auto", + "stacking": { "mode": "none" } + }, + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { "h": 6, "w": 6, "x": 18, "y": 10 }, + "id": 8, + "options": { + "tooltip": { "mode": "multi" }, + "legend": { "displayMode": "list", "placement": "bottom" } + }, + "title": "MFA Verification Latency", + "type": "timeseries", + "targets": [ + { + "expr": "histogram_quantile(0.50, rate(api_gateway_mfa_verification_duration_microseconds_bucket[$interval]))", + "legendFormat": "p50", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, rate(api_gateway_mfa_verification_duration_microseconds_bucket[$interval]))", + "legendFormat": "p95", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.99, rate(api_gateway_mfa_verification_duration_microseconds_bucket[$interval]))", + "legendFormat": "p99", + "refId": "C" + } + ] + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 16 }, + "id": 102, + "title": "API Gateway Routing", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisLabel": "", + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "auto", + "stacking": { "mode": "none" } + }, + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { "h": 7, "w": 8, "x": 0, "y": 17 }, + "id": 9, + "options": { + "tooltip": { "mode": "multi" }, + "legend": { "displayMode": "list", "placement": "bottom" } + }, + "title": "Routing Duration p50/p95/p99", + "type": "timeseries", + "targets": [ + { + "expr": "histogram_quantile(0.50, rate(api_gateway_routing_duration_microseconds_bucket[$interval]))", + "legendFormat": "p50", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, rate(api_gateway_routing_duration_microseconds_bucket[$interval]))", + "legendFormat": "p95", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.99, rate(api_gateway_routing_duration_microseconds_bucket[$interval]))", + "legendFormat": "p99", + "refId": "C" + } + ] + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -616,221 +434,250 @@ "lineWidth": 1, "pointSize": 5, "showPoints": "never", - "stacking": { - "mode": "normal" - } + "stacking": { "mode": "normal" } }, - "unit": "reqps" + "unit": "ops" }, "overrides": [] }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 15 - }, - "id": 11, + "gridPos": { "h": 7, "w": 8, "x": 8, "y": 17 }, + "id": 10, "options": { - "tooltip": { - "mode": "multi" - }, - "legend": { - "displayMode": "table", - "placement": "bottom" - } + "tooltip": { "mode": "multi" }, + "legend": { "displayMode": "table", "placement": "bottom" } }, - "title": "Rate Limiting by Tier", + "title": "Config Updates Rate", "type": "timeseries", "targets": [ { - "expr": "sum by (tier) (rate(api_gateway_rate_limit_allowed_total[$interval]))", - "legendFormat": "allowed {{tier}}", + "expr": "rate(api_gateway_config_updates_auth[$interval])", + "legendFormat": "auth", "refId": "A" }, { - "expr": "sum by (tier) (rate(api_gateway_rate_limit_rejected_total[$interval]))", - "legendFormat": "rejected {{tier}}", + "expr": "rate(api_gateway_config_updates_backend[$interval])", + "legendFormat": "backend", + "refId": "B" + }, + { + "expr": "rate(api_gateway_config_updates_rate_limit[$interval])", + "legendFormat": "rate_limit", + "refId": "C" + }, + { + "expr": "rate(api_gateway_config_updates_routing[$interval])", + "legendFormat": "routing", + "refId": "D" + } + ] + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisLabel": "", + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "auto", + "stacking": { "mode": "none" } + }, + "unit": "percentunit", + "min": 0, + "max": 1 + }, + "overrides": [] + }, + "gridPos": { "h": 7, "w": 8, "x": 16, "y": 17 }, + "id": 11, + "options": { + "tooltip": { "mode": "multi" }, + "legend": { "displayMode": "list", "placement": "bottom" } + }, + "title": "Config Cache Hit Ratio", + "type": "timeseries", + "targets": [ + { + "expr": "api_gateway_config_cache_hits / (api_gateway_config_cache_hits + api_gateway_config_cache_misses)", + "legendFormat": "config hit ratio", + "refId": "A" + }, + { + "expr": "api_gateway_jwt_cache_hits / (api_gateway_jwt_cache_hits + api_gateway_jwt_cache_misses)", + "legendFormat": "jwt hit ratio", "refId": "B" } ] }, { "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 23 - }, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 24 }, "id": 103, - "title": "Log Volume", + "title": "API Gateway Events", "type": "row" }, { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { - "color": { - "mode": "palette-classic" - }, + "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisLabel": "", "drawStyle": "line", - "fillOpacity": 30, + "fillOpacity": 10, "lineWidth": 1, "pointSize": 5, - "showPoints": "never", - "stacking": { - "mode": "normal" - } + "showPoints": "auto", + "stacking": { "mode": "none" } + }, + "unit": "ops" + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "success" }, + "properties": [ + { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } + ] + }, + { + "matcher": { "id": "byName", "options": "failure" }, + "properties": [ + { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } + ] + } + ] + }, + "gridPos": { "h": 7, "w": 8, "x": 0, "y": 25 }, + "id": 12, + "options": { + "tooltip": { "mode": "multi" }, + "legend": { "displayMode": "table", "placement": "bottom" } + }, + "title": "Notify Events Rate", + "type": "timeseries", + "targets": [ + { + "expr": "rate(api_gateway_notify_events_total[$interval])", + "legendFormat": "total", + "refId": "A" + }, + { + "expr": "rate(api_gateway_notify_events_success[$interval])", + "legendFormat": "success", + "refId": "B" + }, + { + "expr": "rate(api_gateway_notify_events_failure[$interval])", + "legendFormat": "failure", + "refId": "C" + } + ] + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 7, "w": 8, "x": 8, "y": 25 }, + "id": 13, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"] }, + "colorMode": "background", + "graphMode": "area" + }, + "title": "Notify Listener Connected", + "type": "stat", + "targets": [ + { + "expr": "api_gateway_notify_listener_connected", + "legendFormat": "", + "refId": "A" + } + ] + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisLabel": "", + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "auto", + "stacking": { "mode": "none" } }, "unit": "short" }, "overrides": [ { - "matcher": { - "id": "byName", - "options": "error" - }, + "matcher": { "id": "byName", "options": "errors" }, "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } + { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } ] }, { - "matcher": { - "id": "byName", - "options": "warn" - }, + "matcher": { "id": "byName", "options": "reconnections" }, "properties": [ - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "info" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "debug" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - } + { "id": "color", "value": { "fixedColor": "orange", "mode": "fixed" } } ] } ] }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 24 - }, - "id": 12, + "gridPos": { "h": 7, "w": 8, "x": 16, "y": 25 }, + "id": 14, "options": { - "tooltip": { - "mode": "multi" - }, - "legend": { - "displayMode": "table", - "placement": "bottom" - } + "tooltip": { "mode": "multi" }, + "legend": { "displayMode": "list", "placement": "bottom" } }, - "title": "Log Rate by Level", + "title": "Listener Errors & Reconnections", "type": "timeseries", "targets": [ { - "expr": "sum by (level) (count_over_time({namespace=\"foxhunt\"} | json | level=~\"$level\" [$interval]))", - "legendFormat": "{{level}}", + "expr": "rate(api_gateway_notify_listener_errors[$interval])", + "legendFormat": "errors", "refId": "A" - } - ] - }, - { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 24 - }, - "id": 13, - "options": { - "showTime": true, - "showLabels": true, - "showCommonLabels": false, - "wrapLogMessage": true, - "enableLogDetails": true - }, - "title": "Error Logs", - "type": "logs", - "targets": [ + }, { - "expr": "{namespace=\"foxhunt\"} | json | level=\"error\"", - "refId": "A" + "expr": "rate(api_gateway_notify_listener_reconnections[$interval])", + "legendFormat": "reconnections", + "refId": "B" } ] }, { "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 32 - }, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 32 }, "id": 104, - "title": "Log Detail", + "title": "Logs (Loki)", "type": "row" }, { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, + "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, "fieldConfig": { "defaults": { - "color": { - "mode": "palette-classic" - }, + "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -840,52 +687,100 @@ "lineWidth": 1, "pointSize": 5, "showPoints": "never", - "stacking": { - "mode": "normal" - } + "stacking": { "mode": "normal" } }, "unit": "short" }, - "overrides": [] + "overrides": [ + { + "matcher": { "id": "byName", "options": "error" }, + "properties": [ + { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } + ] + }, + { + "matcher": { "id": "byName", "options": "warn" }, + "properties": [ + { "id": "color", "value": { "fixedColor": "yellow", "mode": "fixed" } } + ] + }, + { + "matcher": { "id": "byName", "options": "info" }, + "properties": [ + { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } + ] + }, + { + "matcher": { "id": "byName", "options": "debug" }, + "properties": [ + { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } + ] + } + ] }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 33 - }, - "id": 14, + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 33 }, + "id": 15, "options": { - "tooltip": { - "mode": "multi" - }, - "legend": { - "displayMode": "table", - "placement": "bottom" - } + "tooltip": { "mode": "multi" }, + "legend": { "displayMode": "table", "placement": "bottom" } }, - "title": "Logs by Service", + "title": "Log Volume by Level", "type": "timeseries", "targets": [ { - "expr": "sum by (app) (count_over_time({namespace=\"foxhunt\", app=~\".+\"} [$interval]))", + "expr": "sum by (level) (count_over_time({namespace=\"foxhunt\"} | json [$__interval]))", + "legendFormat": "{{level}}", + "refId": "A" + } + ] + }, + { + "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisLabel": "", + "drawStyle": "line", + "fillOpacity": 20, + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "stacking": { "mode": "none" } + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { "id": "byFrameRefID", "options": "A" }, + "properties": [ + { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } + ] + } + ] + }, + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 33 }, + "id": 16, + "options": { + "tooltip": { "mode": "multi" }, + "legend": { "displayMode": "list", "placement": "bottom" } + }, + "title": "Error Log Rate by App", + "type": "timeseries", + "targets": [ + { + "expr": "sum by (app) (count_over_time({namespace=\"foxhunt\"} |= \"error\" [$__interval]))", "legendFormat": "{{app}}", "refId": "A" } ] }, { - "datasource": { - "type": "loki", - "uid": "P8E80F9AEF21F6940" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 33 - }, - "id": 15, + "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 33 }, + "id": 17, "options": { "showTime": true, "showLabels": true, @@ -893,39 +788,26 @@ "wrapLogMessage": true, "enableLogDetails": true }, - "title": "Log Stream", + "title": "Recent Errors", "type": "logs", "targets": [ { - "expr": "{namespace=\"foxhunt\", app=~\"$service\"} | json | level=~\"$level\"", + "expr": "{namespace=\"foxhunt\"} |= \"error\"", "refId": "A" } ] }, { "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 41 - }, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 41 }, "id": 105, - "title": "Traces", + "title": "Traces (Tempo)", "type": "row" }, { - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 42 - }, - "id": 16, + "datasource": { "type": "tempo", "uid": "tempo" }, + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 42 }, + "id": 18, "options": {}, "title": "Trace Search", "type": "traces", @@ -938,26 +820,92 @@ }, { "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 50 - }, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 50 }, "id": 106, - "title": "Prometheus Self-Monitoring", + "title": "Prometheus Health", "type": "row" }, { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { - "color": { - "mode": "palette-classic" + "color": { "mode": "fixed", "fixedColor": "text" }, + "custom": { + "align": "auto", + "cellOptions": { "type": "color-text" }, + "inspect": false }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "green", "value": 1 } + ] + }, + "mappings": [ + { + "options": { + "0": { "text": "DOWN", "color": "red" }, + "1": { "text": "UP", "color": "green" } + }, + "type": "value" + } + ] + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "Value" }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { "type": "color-background" } + }, + { "id": "custom.width", "value": 80 } + ] + } + ] + }, + "gridPos": { "h": 8, "w": 6, "x": 0, "y": 51 }, + "id": 19, + "options": { + "showHeader": true, + "sortBy": [ + { "desc": false, "displayName": "Value" } + ] + }, + "title": "Scrape Targets Up/Down", + "type": "table", + "targets": [ + { + "expr": "up", + "legendFormat": "{{job}}", + "refId": "A", + "instant": true, + "format": "table" + } + ], + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "__name__": true + }, + "renameByName": { + "job": "Job", + "instance": "Instance", + "Value": "Status" + } + } + } + ] + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -967,31 +915,19 @@ "lineWidth": 1, "pointSize": 5, "showPoints": "auto", - "stacking": { - "mode": "none" - } + "stacking": { "mode": "none" } }, "unit": "s" }, "overrides": [] }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 51 - }, - "id": 17, + "gridPos": { "h": 8, "w": 6, "x": 6, "y": 51 }, + "id": 20, "options": { - "tooltip": { - "mode": "multi" - }, - "legend": { - "displayMode": "table", - "placement": "bottom" - } + "tooltip": { "mode": "multi" }, + "legend": { "displayMode": "table", "placement": "bottom" } }, - "title": "Prometheus Scrape Duration", + "title": "Scrape Duration by Job", "type": "timeseries", "targets": [ { @@ -1002,71 +938,69 @@ ] }, { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { - "color": { - "mode": "palette-classic" + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 50000 }, + { "color": "red", "value": 100000 } + ] }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisLabel": "", - "drawStyle": "line", - "fillOpacity": 10, - "lineWidth": 2, - "pointSize": 5, - "showPoints": "never", - "stacking": { - "mode": "none" - } - }, - "mappings": [ - { - "options": { - "0": { - "text": "DOWN", - "color": "red" - }, - "1": { - "text": "UP", - "color": "green" - } - }, - "type": "value" - } - ], - "unit": "short", - "min": 0, - "max": 1 + "unit": "short" }, "overrides": [] }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 51 - }, - "id": 18, + "gridPos": { "h": 8, "w": 6, "x": 12, "y": 51 }, + "id": 21, "options": { - "tooltip": { - "mode": "multi" - }, - "legend": { - "displayMode": "table", - "placement": "bottom" - } + "reduceOptions": { "calcs": ["lastNotNull"] }, + "colorMode": "background", + "graphMode": "area" }, - "title": "Targets Up/Down", - "type": "timeseries", + "title": "TSDB Head Series", + "type": "stat", "targets": [ { - "expr": "up", - "legendFormat": "{{job}}", + "expr": "prometheus_tsdb_head_series", + "legendFormat": "", + "refId": "A" + } + ] + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 1073741824 }, + { "color": "red", "value": 5368709120 } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 6, "x": 18, "y": 51 }, + "id": 22, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"] }, + "colorMode": "background", + "graphMode": "area" + }, + "title": "TSDB Storage Bytes", + "type": "stat", + "targets": [ + { + "expr": "prometheus_tsdb_storage_blocks_bytes", + "legendFormat": "", "refId": "A" } ] @@ -1077,6 +1011,7 @@ "tags": [ "foxhunt", "observability", + "api-gateway", "logs", "traces" ], @@ -1095,104 +1030,14 @@ "label": "Interval", "name": "interval", "options": [ - { - "selected": false, - "text": "1m", - "value": "1m" - }, - { - "selected": false, - "text": "5m", - "value": "5m" - }, - { - "selected": false, - "text": "15m", - "value": "15m" - }, - { - "selected": false, - "text": "30m", - "value": "30m" - }, - { - "selected": false, - "text": "1h", - "value": "1h" - } + { "selected": false, "text": "1m", "value": "1m" }, + { "selected": false, "text": "5m", "value": "5m" }, + { "selected": false, "text": "15m", "value": "15m" }, + { "selected": false, "text": "30m", "value": "30m" }, + { "selected": false, "text": "1h", "value": "1h" } ], "query": "1m,5m,15m,30m,1h", "type": "interval" - }, - { - "allValue": ".*", - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "definition": "label_values(up{job=~\"foxhunt.*\"}, job)", - "hide": 0, - "includeAll": true, - "label": "Service", - "multi": true, - "name": "service", - "options": [], - "query": { - "qryType": 1, - "query": "label_values(up{job=~\"foxhunt.*\"}, job)" - }, - "refresh": 2, - "type": "query" - }, - { - "allValue": ".*", - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "hide": 0, - "includeAll": true, - "label": "Level", - "multi": true, - "name": "level", - "options": [ - { - "selected": false, - "text": "error", - "value": "error" - }, - { - "selected": false, - "text": "warn", - "value": "warn" - }, - { - "selected": false, - "text": "info", - "value": "info" - }, - { - "selected": false, - "text": "debug", - "value": "debug" - } - ], - "query": "error,warn,info,debug", - "type": "custom" } ] }, @@ -1202,7 +1047,7 @@ }, "timepicker": {}, "timezone": "browser", - "title": "Observability & API Gateway", + "title": "Foxhunt Observability & API Gateway", "uid": "foxhunt-observability", "version": 1 } diff --git a/infra/k8s/monitoring/dashboards/foxhunt-trading.json b/infra/k8s/monitoring/dashboards/foxhunt-trading.json index 882911f99..f745e2fb7 100644 --- a/infra/k8s/monitoring/dashboards/foxhunt-trading.json +++ b/infra/k8s/monitoring/dashboards/foxhunt-trading.json @@ -1,8 +1,8 @@ { "id": null, "uid": "foxhunt-trading", - "title": "Trading & ML Intelligence", - "description": "ML signal quality, execution latency, risk checks, and gRPC service health for the foxhunt HFT platform", + "title": "Foxhunt Trading & ML Intelligence", + "description": "Trading overview, execution latency, gRPC pipeline health, data acquisition, and backtesting for the foxhunt HFT platform", "tags": [ "foxhunt", "trading", @@ -10,7 +10,7 @@ ], "timezone": "browser", "schemaVersion": 39, - "version": 1, + "version": 2, "editable": true, "refresh": "10s", "time": { @@ -28,42 +28,12 @@ "query": "1m,5m,15m,30m,1h", "current": { "selected": true, - "text": "5m", - "value": "5m" + "text": "auto", + "value": "$__auto_interval_interval" }, - "auto": false - }, - { - "name": "model", - "type": "query", - "query": "label_values(foxhunt_predictions_confidence_bucket, model)", - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "refresh": 2, - "includeAll": true, - "multi": true, - "current": { - "text": "All", - "value": "$__all" - } - }, - { - "name": "symbol", - "type": "query", - "query": "label_values(foxhunt_predictions_confidence_bucket, symbol)", - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "refresh": 2, - "includeAll": true, - "multi": true, - "current": { - "text": "All", - "value": "$__all" - } + "auto": true, + "auto_count": 30, + "auto_min": "10s" } ] }, @@ -71,133 +41,77 @@ { "id": 1, "type": "row", - "title": "ML Signal Quality", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, + "title": "Trading Overview", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "collapsed": false, "panels": [] }, { "id": 2, "type": "stat", - "title": "Ensemble Agreement", - "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 1 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, + "title": "Portfolio Value", + "description": "Current portfolio value in USD from the trading agent", + "gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "yellow", - "value": 0.5 - }, - { - "color": "green", - "value": 0.7 - } + { "color": "red", "value": null }, + { "color": "yellow", "value": 10000 }, + { "color": "green", "value": 100000 } ] }, - "unit": "percentunit", - "color": { - "mode": "thresholds" - }, - "noValue": "No data" + "unit": "currencyUSD", + "color": { "mode": "thresholds" }, + "noValue": "N/A" }, "overrides": [] }, "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "orientation": "auto", "textMode": "auto", "colorMode": "background", - "graphMode": "none", + "graphMode": "area", "justifyMode": "center" }, "targets": [ { "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "foxhunt_ensemble_agreement_rate{symbol=~\"$symbol\"}", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "trading_agent_portfolio_value_usd", "instant": true, - "legendFormat": "{{symbol}}" + "legendFormat": "portfolio" } ] }, { "id": 3, "type": "stat", - "title": "Prediction Confidence", - "gridPos": { - "h": 4, - "w": 6, - "x": 6, - "y": 1 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, + "title": "Assets Selected", + "description": "Number of assets currently selected by the trading agent", + "gridPos": { "h": 4, "w": 6, "x": 6, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "yellow", - "value": 0.5 - }, - { - "color": "green", - "value": 0.7 - } + { "color": "blue", "value": null }, + { "color": "green", "value": 1 }, + { "color": "yellow", "value": 10 } ] }, - "unit": "percentunit", - "color": { - "mode": "thresholds" - }, - "noValue": "No data" + "unit": "none", + "color": { "mode": "thresholds" }, + "noValue": "0" }, "overrides": [] }, "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -207,65 +121,37 @@ "targets": [ { "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "histogram_quantile(0.5, rate(foxhunt_predictions_confidence_bucket{model=~\"$model\"}[5m]))", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "trading_agent_assets_selected", "instant": true, - "legendFormat": "p50" + "legendFormat": "selected" } ] }, { "id": 4, "type": "stat", - "title": "Model Predictions/s", - "gridPos": { - "h": 4, - "w": 6, - "x": 12, - "y": 1 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, + "title": "Universe Instruments", + "description": "Total instruments in the trading universe", + "gridPos": { "h": 4, "w": 6, "x": 12, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "yellow", - "value": 1 - }, - { - "color": "green", - "value": 10 - } + { "color": "blue", "value": null }, + { "color": "green", "value": 1 } ] }, - "unit": "ops", - "color": { - "mode": "thresholds" - }, + "unit": "none", + "color": { "mode": "thresholds" }, "noValue": "0" }, "overrides": [] }, "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -275,65 +161,38 @@ "targets": [ { "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "sum(rate(foxhunt_predictions_total{model=~\"$model\"}[5m]))", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "trading_agent_universe_instruments", "instant": true, - "legendFormat": "predictions/s" + "legendFormat": "instruments" } ] }, { "id": 5, "type": "stat", - "title": "Active Models", - "gridPos": { - "h": 4, - "w": 6, - "x": 18, - "y": 1 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, + "title": "Trading Service Uptime", + "description": "How long the trading service has been running", + "gridPos": { "h": 4, "w": 6, "x": 18, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "yellow", - "value": 1 - }, - { - "color": "green", - "value": 5 - } + { "color": "red", "value": null }, + { "color": "yellow", "value": 3600 }, + { "color": "green", "value": 86400 } ] }, - "unit": "none", - "color": { - "mode": "thresholds" - }, - "noValue": "0" + "unit": "s", + "color": { "mode": "thresholds" }, + "noValue": "Down" }, "overrides": [] }, "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -343,49 +202,110 @@ "targets": [ { "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "count(foxhunt_model_loaded{model=~\"$model\"} == 1)", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "trading_service_uptime_seconds", "instant": true, - "legendFormat": "models" + "legendFormat": "uptime" } ] }, { "id": 6, - "type": "row", - "title": "Prediction Detail", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 5 + "type": "timeseries", + "title": "Trading Measurements Rate", + "description": "Rate of trading measurements over time", + "gridPos": { "h": 6, "w": 12, "x": 0, "y": 5 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "color": { "mode": "palette-classic" }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15, + "gradientMode": "scheme", + "showPoints": "never", + "spanNulls": false, + "axisCenteredZero": false, + "axisLabel": "", + "scaleDistribution": { "type": "linear" } + } + }, + "overrides": [] }, - "collapsed": false, - "panels": [] + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "list", "placement": "bottom", "calcs": ["lastNotNull", "mean"] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "rate(trading_measurements_total[$__rate_interval])", + "legendFormat": "measurements/s" + } + ] }, { "id": 7, "type": "timeseries", - "title": "Confidence Distribution", - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 6 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, + "title": "Portfolio Value Over Time", + "description": "Trading agent portfolio value trend", + "gridPos": { "h": 6, "w": 12, "x": 12, "y": 5 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { - "unit": "percentunit", - "color": { - "mode": "palette-classic" - }, + "unit": "currencyUSD", + "color": { "mode": "palette-classic" }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 10, + "gradientMode": "scheme", + "showPoints": "never", + "spanNulls": false, + "axisCenteredZero": false, + "axisLabel": "", + "scaleDistribution": { "type": "linear" } + } + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "single", "sort": "none" }, + "legend": { "displayMode": "list", "placement": "bottom", "calcs": ["lastNotNull", "min", "max"] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "trading_agent_portfolio_value_usd", + "legendFormat": "portfolio value" + } + ] + }, + { + "id": 10, + "type": "row", + "title": "Trading Latency", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 11 }, + "collapsed": false, + "panels": [] + }, + { + "id": 11, + "type": "timeseries", + "title": "Total Trading Latency (p50 / p95 / p99)", + "description": "End-to-end trading latency percentiles from trading_total_latency_seconds histogram", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "unit": "s", + "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "smooth", @@ -397,392 +317,462 @@ "spanNulls": false, "axisCenteredZero": false, "axisLabel": "", - "scaleDistribution": { - "type": "linear" - } + "scaleDistribution": { "type": "linear" } } }, - "overrides": [] + "overrides": [ + { + "matcher": { "id": "byName", "options": "p99" }, + "properties": [ + { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } + ] + }, + { + "matcher": { "id": "byName", "options": "p95" }, + "properties": [ + { "id": "color", "value": { "fixedColor": "yellow", "mode": "fixed" } } + ] + }, + { + "matcher": { "id": "byName", "options": "p50" }, + "properties": [ + { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } + ] + } + ] }, "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "lastNotNull", - "mean" - ] - } + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "mean", "max"] } }, "targets": [ { "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "histogram_quantile(0.1, rate(foxhunt_predictions_confidence_bucket{model=~\"$model\"}[$interval]))", - "legendFormat": "p10" - }, - { - "refId": "B", - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "histogram_quantile(0.5, rate(foxhunt_predictions_confidence_bucket{model=~\"$model\"}[$interval]))", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "histogram_quantile(0.5, rate(trading_total_latency_seconds_bucket[$__rate_interval]))", "legendFormat": "p50" }, + { + "refId": "B", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "histogram_quantile(0.95, rate(trading_total_latency_seconds_bucket[$__rate_interval]))", + "legendFormat": "p95" + }, { "refId": "C", - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "histogram_quantile(0.9, rate(foxhunt_predictions_confidence_bucket{model=~\"$model\"}[$interval]))", - "legendFormat": "p90" - } - ] - }, - { - "id": 8, - "type": "timeseries", - "title": "Prediction Rate by Model", - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 6 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "unit": "ops", - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "bars", - "lineWidth": 1, - "fillOpacity": 80, - "gradientMode": "none", - "showPoints": "never", - "stacking": { - "mode": "normal", - "group": "A" - }, - "axisCenteredZero": false, - "axisLabel": "", - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "lastNotNull", - "mean" - ] - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "sum by (model) (rate(foxhunt_predictions_total{model=~\"$model\"}[$interval]))", - "legendFormat": "{{model}}" - } - ] - }, - { - "id": 9, - "type": "row", - "title": "Order Execution", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 14 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 10, - "type": "stat", - "title": "Orders/s", - "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 15 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - }, - { - "color": "green", - "value": 1 - }, - { - "color": "yellow", - "value": 100 - }, - { - "color": "red", - "value": 500 - } - ] - }, - "unit": "ops", - "color": { - "mode": "thresholds" - }, - "noValue": "0" - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "sum(rate(trading_orders_total[5m]))", - "instant": true, - "legendFormat": "orders/s" - } - ] - }, - { - "id": 11, - "type": "stat", - "title": "Order Latency P99", - "gridPos": { - "h": 4, - "w": 6, - "x": 6, - "y": 15 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.01 - }, - { - "color": "red", - "value": 0.1 - } - ] - }, - "unit": "ms", - "color": { - "mode": "thresholds" - }, - "noValue": "N/A" - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "histogram_quantile(0.99, rate(trading_order_processing_seconds_bucket[5m]))", - "instant": true, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "histogram_quantile(0.99, rate(trading_total_latency_seconds_bucket[$__rate_interval]))", "legendFormat": "p99" } ] }, { "id": 12, - "type": "stat", - "title": "Risk Check Latency", - "gridPos": { - "h": 4, - "w": 6, - "x": 12, - "y": 15 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, + "type": "timeseries", + "title": "Order Processing Latency (p50 / p95 / p99)", + "description": "Order processing latency percentiles from trading_order_processing_seconds histogram", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.005 - }, - { - "color": "red", - "value": 0.05 - } + "unit": "s", + "color": { "mode": "palette-classic" }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 10, + "gradientMode": "none", + "showPoints": "auto", + "pointSize": 5, + "spanNulls": false, + "axisCenteredZero": false, + "axisLabel": "", + "scaleDistribution": { "type": "linear" } + } + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "p99" }, + "properties": [ + { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } ] }, - "unit": "ms", - "color": { - "mode": "thresholds" + { + "matcher": { "id": "byName", "options": "p95" }, + "properties": [ + { "id": "color", "value": { "fixedColor": "yellow", "mode": "fixed" } } + ] }, - "noValue": "N/A" - }, - "overrides": [] + { + "matcher": { "id": "byName", "options": "p50" }, + "properties": [ + { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } + ] + } + ] }, "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center" + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "mean", "max"] } }, "targets": [ { "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "histogram_quantile(0.99, rate(trading_risk_check_seconds_bucket[5m]))", - "instant": true, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "histogram_quantile(0.5, rate(trading_order_processing_seconds_bucket[$__rate_interval]))", + "legendFormat": "p50" + }, + { + "refId": "B", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "histogram_quantile(0.95, rate(trading_order_processing_seconds_bucket[$__rate_interval]))", + "legendFormat": "p95" + }, + { + "refId": "C", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "histogram_quantile(0.99, rate(trading_order_processing_seconds_bucket[$__rate_interval]))", "legendFormat": "p99" } ] }, { "id": 13, + "type": "timeseries", + "title": "Risk Check Latency (p50 / p95 / p99)", + "description": "Risk check latency percentiles from trading_risk_check_seconds histogram", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 20 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "unit": "s", + "color": { "mode": "palette-classic" }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 10, + "gradientMode": "none", + "showPoints": "auto", + "pointSize": 5, + "spanNulls": false, + "axisCenteredZero": false, + "axisLabel": "", + "scaleDistribution": { "type": "linear" } + } + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "p99" }, + "properties": [ + { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } + ] + }, + { + "matcher": { "id": "byName", "options": "p95" }, + "properties": [ + { "id": "color", "value": { "fixedColor": "yellow", "mode": "fixed" } } + ] + }, + { + "matcher": { "id": "byName", "options": "p50" }, + "properties": [ + { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } + ] + } + ] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "mean", "max"] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "histogram_quantile(0.5, rate(trading_risk_check_seconds_bucket[$__rate_interval]))", + "legendFormat": "p50" + }, + { + "refId": "B", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "histogram_quantile(0.95, rate(trading_risk_check_seconds_bucket[$__rate_interval]))", + "legendFormat": "p95" + }, + { + "refId": "C", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "histogram_quantile(0.99, rate(trading_risk_check_seconds_bucket[$__rate_interval]))", + "legendFormat": "p99" + } + ] + }, + { + "id": 14, + "type": "timeseries", + "title": "Market Data Latency (p50 / p95 / p99)", + "description": "Market data fetch latency percentiles from trading_market_data_seconds histogram", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 20 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "unit": "s", + "color": { "mode": "palette-classic" }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 10, + "gradientMode": "none", + "showPoints": "auto", + "pointSize": 5, + "spanNulls": false, + "axisCenteredZero": false, + "axisLabel": "", + "scaleDistribution": { "type": "linear" } + } + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "p99" }, + "properties": [ + { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } + ] + }, + { + "matcher": { "id": "byName", "options": "p95" }, + "properties": [ + { "id": "color", "value": { "fixedColor": "yellow", "mode": "fixed" } } + ] + }, + { + "matcher": { "id": "byName", "options": "p50" }, + "properties": [ + { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } + ] + } + ] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "mean", "max"] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "histogram_quantile(0.5, rate(trading_market_data_seconds_bucket[$__rate_interval]))", + "legendFormat": "p50" + }, + { + "refId": "B", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "histogram_quantile(0.95, rate(trading_market_data_seconds_bucket[$__rate_interval]))", + "legendFormat": "p95" + }, + { + "refId": "C", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "histogram_quantile(0.99, rate(trading_market_data_seconds_bucket[$__rate_interval]))", + "legendFormat": "p99" + } + ] + }, + { + "id": 20, + "type": "row", + "title": "gRPC Trading Pipeline", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 28 }, + "collapsed": false, + "panels": [] + }, + { + "id": 21, + "type": "timeseries", + "title": "gRPC Request Rate by Service", + "description": "Rate of gRPC calls handled per service (trading-service, trading-agent-service, broker-gateway)", + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 29 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "color": { "mode": "palette-classic" }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15, + "gradientMode": "none", + "showPoints": "never", + "spanNulls": false, + "stacking": { "mode": "none", "group": "A" }, + "axisCenteredZero": false, + "axisLabel": "", + "scaleDistribution": { "type": "linear" } + } + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "mean"] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "sum by (exported_service) (rate(grpc_server_handled_total{exported_service=~\"trading-service|trading-agent-service|broker-gateway\"}[$__rate_interval]))", + "legendFormat": "{{exported_service}}" + } + ] + }, + { + "id": 22, + "type": "timeseries", + "title": "gRPC Error Rate by Service", + "description": "Rate of non-OK gRPC responses across trading services", + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 29 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "color": { "mode": "palette-classic" }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15, + "gradientMode": "none", + "showPoints": "never", + "spanNulls": false, + "axisCenteredZero": false, + "axisLabel": "", + "scaleDistribution": { "type": "linear" } + } + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "sum"] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "sum by (exported_service, grpc_code) (rate(grpc_server_handled_total{exported_service=~\"trading-service|trading-agent-service|broker-gateway\", grpc_code!=\"OK\"}[$__rate_interval]))", + "legendFormat": "{{exported_service}} {{grpc_code}}" + } + ] + }, + { + "id": 23, + "type": "timeseries", + "title": "gRPC p99 Latency by Service", + "description": "99th percentile gRPC handling latency per trading service", + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 29 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "unit": "s", + "color": { "mode": "palette-classic" }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 10, + "gradientMode": "none", + "showPoints": "auto", + "pointSize": 5, + "spanNulls": false, + "axisCenteredZero": false, + "axisLabel": "", + "scaleDistribution": { "type": "linear" } + } + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "mean", "max"] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "histogram_quantile(0.99, sum by (exported_service, le) (rate(grpc_server_handling_seconds_bucket{exported_service=~\"trading-service|trading-agent-service|broker-gateway\"}[$__rate_interval])))", + "legendFormat": "{{exported_service}} p99" + } + ] + }, + { + "id": 24, "type": "stat", - "title": "Total Latency P99", - "gridPos": { - "h": 4, - "w": 6, - "x": 18, - "y": 15 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, + "title": "gRPC Total Requests Started", + "description": "Total gRPC requests started across all trading services", + "gridPos": { "h": 4, "w": 8, "x": 0, "y": 37 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.05 - }, - { - "color": "red", - "value": 0.5 - } + { "color": "blue", "value": null }, + { "color": "green", "value": 1 } ] }, - "unit": "ms", - "color": { - "mode": "thresholds" - }, - "noValue": "N/A" + "unit": "short", + "color": { "mode": "thresholds" }, + "noValue": "0" }, "overrides": [] }, "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "area", + "justifyMode": "center" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "sum(grpc_server_started_total{exported_service=~\"trading-service|trading-agent-service|broker-gateway\"})", + "instant": true, + "legendFormat": "total started" + } + ] + }, + { + "id": 25, + "type": "stat", + "title": "gRPC Error Ratio", + "description": "Ratio of non-OK to total handled gRPC requests across trading services", + "gridPos": { "h": 4, "w": 8, "x": 8, "y": 37 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 0.01 }, + { "color": "red", "value": 0.05 } + ] + }, + "unit": "percentunit", + "color": { "mode": "thresholds" }, + "noValue": "0%", + "min": 0, + "max": 1 }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -792,515 +782,487 @@ "targets": [ { "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "histogram_quantile(0.99, rate(trading_total_latency_seconds_bucket[5m]))", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "sum(rate(grpc_server_handled_total{exported_service=~\"trading-service|trading-agent-service|broker-gateway\", grpc_code!=\"OK\"}[$__rate_interval])) / sum(rate(grpc_server_handled_total{exported_service=~\"trading-service|trading-agent-service|broker-gateway\"}[$__rate_interval]))", "instant": true, - "legendFormat": "p99" + "legendFormat": "error ratio" } ] }, { - "id": 14, - "type": "row", - "title": "Execution Quality", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 19 + "id": 26, + "type": "stat", + "title": "Broker Gateway Requests", + "description": "Total handled requests on broker-gateway (highest volume service)", + "gridPos": { "h": 4, "w": 8, "x": 16, "y": 37 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "blue", "value": null }, + { "color": "green", "value": 100 } + ] + }, + "unit": "short", + "color": { "mode": "thresholds" }, + "noValue": "0" + }, + "overrides": [] }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "area", + "justifyMode": "center" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "sum(grpc_server_handled_total{exported_service=\"broker-gateway\"})", + "instant": true, + "legendFormat": "broker-gateway total" + } + ] + }, + { + "id": 30, + "type": "row", + "title": "Data Acquisition", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 41 }, "collapsed": false, "panels": [] }, { - "id": 15, - "type": "timeseries", - "title": "Order Processing Latency", - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 20 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, + "id": 31, + "type": "stat", + "title": "Active Feeds", + "description": "Number of currently active data acquisition feeds", + "gridPos": { "h": 5, "w": 8, "x": 0, "y": 42 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { - "unit": "s", - "color": { - "mode": "palette-classic" + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "yellow", "value": 1 }, + { "color": "green", "value": 2 } + ] }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 10, - "gradientMode": "none", - "showPoints": "auto", - "pointSize": 5, - "spanNulls": false, - "axisCenteredZero": false, - "axisLabel": "", - "scaleDistribution": { - "type": "linear" - } - } + "unit": "none", + "color": { "mode": "thresholds" }, + "noValue": "0" }, "overrides": [] }, "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "lastNotNull", - "mean", - "max" - ] - } + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "area", + "justifyMode": "center" }, "targets": [ { "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "histogram_quantile(0.5, rate(trading_order_processing_seconds_bucket[$interval]))", - "legendFormat": "p50" - }, - { - "refId": "B", - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "histogram_quantile(0.95, rate(trading_order_processing_seconds_bucket[$interval]))", - "legendFormat": "p95" - }, - { - "refId": "C", - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "histogram_quantile(0.99, rate(trading_order_processing_seconds_bucket[$interval]))", - "legendFormat": "p99" + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "foxhunt_data_acquisition_feeds_active", + "instant": true, + "legendFormat": "active feeds" } ] }, { - "id": 16, + "id": 32, "type": "timeseries", - "title": "Risk Check Latency", - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 20 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "unit": "s", - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 10, - "gradientMode": "none", - "showPoints": "auto", - "pointSize": 5, - "spanNulls": false, - "axisCenteredZero": false, - "axisLabel": "", - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "lastNotNull", - "mean", - "max" - ] - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "histogram_quantile(0.5, rate(trading_risk_check_seconds_bucket[$interval]))", - "legendFormat": "p50" - }, - { - "refId": "B", - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "histogram_quantile(0.95, rate(trading_risk_check_seconds_bucket[$interval]))", - "legendFormat": "p95" - }, - { - "refId": "C", - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "histogram_quantile(0.99, rate(trading_risk_check_seconds_bucket[$interval]))", - "legendFormat": "p99" - } - ] - }, - { - "id": 17, - "type": "row", - "title": "API Gateway Trading Path", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 28 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 18, - "type": "timeseries", - "title": "Trading Endpoint Latency", - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 29 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "unit": "s", - "color": { - "mode": "palette-classic" - }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 10, - "gradientMode": "none", - "showPoints": "auto", - "pointSize": 5, - "spanNulls": false, - "axisCenteredZero": false, - "axisLabel": "", - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "lastNotNull", - "mean", - "max" - ] - } - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "histogram_quantile(0.99, rate(api_gateway_request_duration_seconds_bucket{handler=~\".*trade.*|.*order.*\"}[$interval]))", - "legendFormat": "{{handler}} p99" - } - ] - }, - { - "id": 19, - "type": "timeseries", - "title": "Trading Rate Limit", - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 29 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, + "title": "Data Acquisition Error Rate", + "description": "Rate of data acquisition errors over time", + "gridPos": { "h": 5, "w": 8, "x": 8, "y": 42 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "unit": "ops", - "color": { - "mode": "palette-classic" + "color": { "mode": "fixed", "fixedColor": "red" }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 20, + "gradientMode": "none", + "showPoints": "never", + "spanNulls": false, + "axisCenteredZero": false, + "axisLabel": "", + "scaleDistribution": { "type": "linear" } + } + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "single", "sort": "none" }, + "legend": { "displayMode": "list", "placement": "bottom", "calcs": ["lastNotNull", "mean"] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "rate(foxhunt_data_acquisition_errors_total[$__rate_interval])", + "legendFormat": "errors/s" + } + ] + }, + { + "id": 33, + "type": "stat", + "title": "Data Acquisition Uptime", + "description": "How long the data acquisition service has been running", + "gridPos": { "h": 5, "w": 8, "x": 16, "y": 42 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "yellow", "value": 3600 }, + { "color": "green", "value": 86400 } + ] }, + "unit": "s", + "color": { "mode": "thresholds" }, + "noValue": "Down" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "foxhunt_data_acquisition_uptime_seconds", + "instant": true, + "legendFormat": "uptime" + } + ] + }, + { + "id": 40, + "type": "row", + "title": "Backtesting", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 47 }, + "collapsed": false, + "panels": [] + }, + { + "id": 41, + "type": "stat", + "title": "Backtests Completed", + "description": "Total backtests completed", + "gridPos": { "h": 5, "w": 6, "x": 0, "y": 48 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "blue", "value": null }, + { "color": "green", "value": 1 } + ] + }, + "unit": "short", + "color": { "mode": "thresholds" }, + "noValue": "0" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "area", + "justifyMode": "center" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "backtesting_backtests_completed_total", + "instant": true, + "legendFormat": "completed" + } + ] + }, + { + "id": 42, + "type": "stat", + "title": "Backtests Started", + "description": "Total backtests started", + "gridPos": { "h": 5, "w": 6, "x": 6, "y": 48 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "blue", "value": null }, + { "color": "green", "value": 1 } + ] + }, + "unit": "short", + "color": { "mode": "thresholds" }, + "noValue": "0" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "area", + "justifyMode": "center" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "backtesting_backtests_started_total", + "instant": true, + "legendFormat": "started" + } + ] + }, + { + "id": 43, + "type": "gauge", + "title": "Backtest Completion Ratio", + "description": "Ratio of completed to started backtests (1.0 = all finished)", + "gridPos": { "h": 5, "w": 6, "x": 12, "y": 48 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "yellow", "value": 0.5 }, + { "color": "green", "value": 0.9 } + ] + }, + "unit": "percentunit", + "color": { "mode": "thresholds" }, + "min": 0, + "max": 1, + "noValue": "N/A" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "orientation": "auto" + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "backtesting_backtests_completed_total / backtesting_backtests_started_total", + "instant": true, + "legendFormat": "completion ratio" + } + ] + }, + { + "id": 44, + "type": "timeseries", + "title": "Backtest Error Rate", + "description": "Rate of backtesting errors over time", + "gridPos": { "h": 5, "w": 6, "x": 18, "y": 48 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "color": { "mode": "fixed", "fixedColor": "red" }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 20, + "gradientMode": "none", + "showPoints": "never", + "spanNulls": false, + "axisCenteredZero": false, + "axisLabel": "", + "scaleDistribution": { "type": "linear" } + } + }, + "overrides": [] + }, + "options": { + "tooltip": { "mode": "single", "sort": "none" }, + "legend": { "displayMode": "list", "placement": "bottom", "calcs": ["lastNotNull", "mean"] } + }, + "targets": [ + { + "refId": "A", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "rate(backtesting_errors_total[$__rate_interval])", + "legendFormat": "errors/s" + } + ] + }, + { + "id": 45, + "type": "timeseries", + "title": "Backtests Started vs Completed Over Time", + "description": "Rate of backtests started and completed, showing active backtest throughput", + "gridPos": { "h": 6, "w": 12, "x": 0, "y": 53 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "lineInterpolation": "smooth", "lineWidth": 2, "fillOpacity": 15, "gradientMode": "none", - "showPoints": "auto", - "pointSize": 5, + "showPoints": "never", "spanNulls": false, "axisCenteredZero": false, "axisLabel": "", - "scaleDistribution": { - "type": "linear" - } + "scaleDistribution": { "type": "linear" } } }, "overrides": [ { - "matcher": { - "id": "byName", - "options": "rejected" - }, + "matcher": { "id": "byName", "options": "started" }, "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } + { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] }, { - "matcher": { - "id": "byName", - "options": "allowed" - }, + "matcher": { "id": "byName", "options": "completed" }, "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } + { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] } ] }, "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "lastNotNull", - "mean" - ] - } + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "mean"] } }, "targets": [ { "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "rate(api_gateway_rate_limit_rejected_total{tier=\"trading\"}[$interval])", - "legendFormat": "rejected" + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "rate(backtesting_backtests_started_total[$__rate_interval])", + "legendFormat": "started" }, { "refId": "B", - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "rate(api_gateway_rate_limit_allowed_total{tier=\"trading\"}[$interval])", - "legendFormat": "allowed" + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "rate(backtesting_backtests_completed_total[$__rate_interval])", + "legendFormat": "completed" } ] }, { - "id": 20, - "type": "row", - "title": "gRPC Services", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 37 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 21, - "type": "timeseries", - "title": "gRPC Trading Calls", - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 38 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, + "id": 46, + "type": "stat", + "title": "Active Backtests (est.)", + "description": "Estimated active backtests: started minus completed", + "gridPos": { "h": 6, "w": 6, "x": 12, "y": 53 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { - "unit": "ops", - "color": { - "mode": "palette-classic" + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 5 }, + { "color": "red", "value": 10 } + ] }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 10, - "gradientMode": "none", - "showPoints": "auto", - "pointSize": 5, - "spanNulls": false, - "axisCenteredZero": false, - "axisLabel": "", - "scaleDistribution": { - "type": "linear" - } - } + "unit": "none", + "color": { "mode": "thresholds" }, + "noValue": "0" }, "overrides": [] }, "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "lastNotNull", - "mean" - ] - } + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "area", + "justifyMode": "center" }, "targets": [ { "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "sum by (grpc_method) (rate(grpc_server_handled_total{grpc_service=~\".*trading.*|.*ml.*\"}[$interval]))", - "legendFormat": "{{grpc_method}}" + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "backtesting_backtests_started_total - backtesting_backtests_completed_total", + "instant": true, + "legendFormat": "active" } ] }, { - "id": 22, - "type": "timeseries", - "title": "gRPC Error Rate", - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 38 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, + "id": 47, + "type": "stat", + "title": "Backtesting Uptime", + "description": "How long the backtesting service has been running", + "gridPos": { "h": 6, "w": 6, "x": 18, "y": 53 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { - "unit": "ops", - "color": { - "mode": "palette-classic" + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "yellow", "value": 3600 }, + { "color": "green", "value": 86400 } + ] }, - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "lineWidth": 2, - "fillOpacity": 15, - "gradientMode": "none", - "showPoints": "auto", - "pointSize": 5, - "spanNulls": false, - "axisCenteredZero": false, - "axisLabel": "", - "scaleDistribution": { - "type": "linear" - } - } + "unit": "s", + "color": { "mode": "thresholds" }, + "noValue": "Down" }, "overrides": [] }, "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "lastNotNull", - "sum" - ] - } + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center" }, "targets": [ { "refId": "A", - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "expr": "sum by (grpc_service, grpc_code) (rate(grpc_server_handled_total{grpc_code!=\"OK\"}[$interval]))", - "legendFormat": "{{grpc_service}} {{grpc_code}}" + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "backtesting_service_uptime_seconds", + "instant": true, + "legendFormat": "uptime" } ] } diff --git a/infra/k8s/monitoring/dashboards/foxhunt-training.json b/infra/k8s/monitoring/dashboards/foxhunt-training.json index 167147171..285066d9f 100644 --- a/infra/k8s/monitoring/dashboards/foxhunt-training.json +++ b/infra/k8s/monitoring/dashboards/foxhunt-training.json @@ -1,21 +1,13 @@ { "uid": "foxhunt-training", - "title": "Training & CI/CD", - "description": "ML model training, hyperopt, GPU utilization, and Argo CI pipelines for the foxhunt HFT platform", - "tags": [ - "foxhunt", - "training", - "gpu", - "ci" - ], + "title": "Foxhunt Training & CI/CD", + "description": "ML model training, hyperopt, GPU utilization, and Argo CI pipelines", + "tags": ["foxhunt", "training", "gpu", "ci"], "editable": true, "graphTooltip": 1, "refresh": "15s", "schemaVersion": 39, - "time": { - "from": "now-6h", - "to": "now" - }, + "time": { "from": "now-6h", "to": "now" }, "timepicker": {}, "timezone": "browser", "templating": { @@ -27,39 +19,41 @@ "auto": true, "auto_count": 30, "auto_min": "1m", - "current": { - "selected": true, - "text": "auto", - "value": "$__auto_interval_interval" - }, + "current": { "selected": true, "text": "auto", "value": "$__auto_interval_interval" }, "options": [ - { - "selected": false, - "text": "1m", - "value": "1m" - }, - { - "selected": false, - "text": "5m", - "value": "5m" - }, - { - "selected": false, - "text": "15m", - "value": "15m" - }, - { - "selected": false, - "text": "30m", - "value": "30m" - }, - { - "selected": false, - "text": "1h", - "value": "1h" - } + { "selected": false, "text": "1m", "value": "1m" }, + { "selected": false, "text": "5m", "value": "5m" }, + { "selected": false, "text": "15m", "value": "15m" }, + { "selected": false, "text": "30m", "value": "30m" }, + { "selected": false, "text": "1h", "value": "1h" } ], "query": "1m,5m,15m,30m,1h" + }, + { + "name": "model", + "label": "Model", + "type": "query", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "query": "label_values(foxhunt_training_current_epoch, model)", + "refresh": 2, + "includeAll": true, + "multi": true, + "allValue": ".*", + "current": { "selected": true, "text": "All", "value": "$__all" }, + "sort": 1 + }, + { + "name": "fold", + "label": "Fold", + "type": "query", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "query": "label_values(foxhunt_training_current_epoch, fold)", + "refresh": 2, + "includeAll": true, + "multi": true, + "allValue": ".*", + "current": { "selected": true, "text": "All", "value": "$__all" }, + "sort": 1 } ] }, @@ -67,1957 +61,1486 @@ { "type": "row", "title": "Training Status", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "collapsed": false, - "panels": [] - }, - { "id": 1, - "type": "stat", - "title": "Active Workers", - "gridPos": { - "h": 4, - "w": 4, - "x": 0, - "y": 1 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "targets": [ - { - "expr": "ml_training_active_workers", - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - }, - "mappings": [], - "unit": "none" - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "colorMode": "background", - "graphMode": "none" - } + "panels": [] }, { "id": 2, "type": "stat", - "title": "Jobs Started", - "gridPos": { - "h": 4, - "w": 4, - "x": 4, - "y": 1 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "targets": [ - { - "expr": "ml_training_jobs_started_total", - "legendFormat": "started", - "refId": "A" - } - ], + "title": "Active Workers", + "gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ - { - "color": "blue", - "value": null - } + { "color": "red", "value": null }, + { "color": "yellow", "value": 1 }, + { "color": "green", "value": 2 } ] }, - "mappings": [], - "unit": "none" + "color": { "mode": "thresholds" } }, "overrides": [] }, "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "colorMode": "background", - "graphMode": "none" - } + "graphMode": "none", + "justifyMode": "center", + "textMode": "auto" + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "foxhunt_training_active_workers", + "legendFormat": "Workers", + "refId": "A" + } + ] }, { "id": 3, "type": "stat", - "title": "Jobs Completed", - "gridPos": { - "h": 4, - "w": 4, - "x": 8, - "y": 1 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "targets": [ - { - "expr": "ml_training_jobs_completed_total", - "legendFormat": "completed", - "refId": "A" - } - ], + "title": "Current Epoch", + "gridPos": { "h": 4, "w": 4, "x": 4, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ - { - "color": "green", - "value": null - } + { "color": "blue", "value": null } ] }, - "mappings": [], - "unit": "none" + "color": { "mode": "thresholds" } }, "overrides": [] }, "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "colorMode": "background", - "graphMode": "none" - } + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "textMode": "auto" + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "foxhunt_training_current_epoch{model=~\"$model\", fold=~\"$fold\"}", + "legendFormat": "{{model}} / fold {{fold}}", + "refId": "A" + } + ] }, { "id": 4, "type": "stat", - "title": "Training Errors", - "gridPos": { - "h": 4, - "w": 4, - "x": 12, - "y": 1 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "targets": [ - { - "expr": "ml_training_errors_total", - "legendFormat": "errors", - "refId": "A" - } - ], + "title": "Mode (Training / Hyperopt)", + "description": "1 = hyperopt, 0 = training", + "gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { + "mappings": [ + { "type": "value", "options": { "0": { "text": "Training", "color": "green", "index": 0 }, "1": { "text": "Hyperopt", "color": "orange", "index": 1 } } } + ], "thresholds": { "mode": "absolute", "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } + { "color": "green", "value": null }, + { "color": "orange", "value": 1 } ] }, - "mappings": [], - "unit": "none" + "color": { "mode": "thresholds" } }, "overrides": [] }, "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "colorMode": "background", - "graphMode": "none" - } + "graphMode": "none", + "justifyMode": "center", + "textMode": "auto" + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "foxhunt_hyperopt_mode{model=~\"$model\"}", + "legendFormat": "{{model}}", + "refId": "A" + } + ] }, { "id": 5, - "type": "stat", - "title": "Current Epoch", - "gridPos": { - "h": 4, - "w": 4, - "x": 16, - "y": 1 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "targets": [ - { - "expr": "foxhunt_training_current_epoch", - "legendFormat": "epoch", - "refId": "A" - } - ], + "type": "gauge", + "title": "Hyperopt Trial Progress", + "gridPos": { "h": 4, "w": 4, "x": 12, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { + "min": 0, + "max": 1, + "unit": "percentunit", "thresholds": { "mode": "absolute", "steps": [ - { - "color": "purple", - "value": null - } + { "color": "red", "value": null }, + { "color": "yellow", "value": 0.5 }, + { "color": "green", "value": 0.9 } ] }, - "mappings": [], - "unit": "none" + "color": { "mode": "thresholds" } }, "overrides": [] }, "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "colorMode": "background", - "graphMode": "none" - } + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "foxhunt_hyperopt_trial_current{model=~\"$model\"} / foxhunt_hyperopt_trial_total{model=~\"$model\"}", + "legendFormat": "{{model}}", + "refId": "A" + } + ] }, { "id": 6, "type": "stat", - "title": "Hyperopt Mode", - "gridPos": { - "h": 4, - "w": 4, - "x": 20, - "y": 1 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "targets": [ - { - "expr": "foxhunt_hyperopt_mode", - "legendFormat": "{{mode}}", - "refId": "A" - } - ], + "title": "ML Service Jobs Started", + "gridPos": { "h": 4, "w": 4, "x": 16, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ - { - "color": "orange", - "value": null - } + { "color": "blue", "value": null } ] }, - "mappings": [ - { - "type": "value", - "options": { - "0": { - "text": "Idle", - "color": "text" - }, - "1": { - "text": "PSO", - "color": "orange" - }, - "2": { - "text": "Grid", - "color": "blue" - } - } - } - ], - "unit": "none" + "color": { "mode": "thresholds" } }, "overrides": [] }, "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "textMode": "auto" + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "ml_training_jobs_started_total", + "legendFormat": "Started", + "refId": "A" + } + ] + }, + { + "id": 7, + "type": "stat", + "title": "ML Service Errors", + "gridPos": { "h": 4, "w": 4, "x": 20, "y": 1 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "red", "value": 1 } + ] + }, + "color": { "mode": "thresholds" } }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "colorMode": "background", - "graphMode": "none", - "textMode": "value_and_name" - } + "graphMode": "area", + "justifyMode": "center", + "textMode": "auto" + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "ml_training_errors_total", + "legendFormat": "Errors", + "refId": "A" + } + ] }, { "type": "row", "title": "Training Curves", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 5 - }, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, "collapsed": false, - "panels": [] - }, - { - "id": 7, - "type": "timeseries", - "title": "Epoch Loss", - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 6 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "targets": [ - { - "expr": "foxhunt_training_epoch_loss", - "legendFormat": "{{model}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 10, - "gradientMode": "scheme", - "spanNulls": false, - "lineWidth": 2, - "pointSize": 5, - "showPoints": "auto", - "axisLabel": "Loss", - "scaleDistribution": { - "type": "linear" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "none", - "color": { - "mode": "palette-classic" - } - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "min", - "max", - "last" - ] - } - } - }, - { - "id": 8, - "type": "timeseries", - "title": "Epoch Sharpe Ratio", - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 6 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "targets": [ - { - "expr": "foxhunt_training_epoch_sharpe", - "legendFormat": "{{model}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 10, - "gradientMode": "scheme", - "spanNulls": false, - "lineWidth": 2, - "pointSize": 5, - "showPoints": "auto", - "axisLabel": "Sharpe", - "scaleDistribution": { - "type": "linear" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "none", - "color": { - "mode": "palette-classic" - } - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "min", - "max", - "last" - ] - } - } - }, - { - "type": "row", - "title": "Training Quality", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 13 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 9, - "type": "timeseries", - "title": "Win Rate", - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 14 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "targets": [ - { - "expr": "foxhunt_training_epoch_win_rate", - "legendFormat": "{{model}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 10, - "gradientMode": "scheme", - "spanNulls": false, - "lineWidth": 2, - "pointSize": 5, - "showPoints": "auto", - "axisLabel": "Win Rate", - "scaleDistribution": { - "type": "linear" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "yellow", - "value": 40 - }, - { - "color": "green", - "value": 55 - } - ] - }, - "unit": "percent", - "min": 0, - "max": 100, - "color": { - "mode": "palette-classic" - } - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "last" - ] - } - } - }, - { "id": 10, - "type": "timeseries", - "title": "Learning Rate", - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 14 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "targets": [ - { - "expr": "foxhunt_training_learning_rate", - "legendFormat": "{{model}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 5, - "spanNulls": false, - "lineWidth": 2, - "pointSize": 5, - "showPoints": "auto", - "axisLabel": "LR", - "scaleDistribution": { - "type": "log", - "log": 10 - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - } - ] - }, - "unit": "none", - "color": { - "mode": "palette-classic" - } - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "last" - ] - } - } + "panels": [] }, { "id": 11, "type": "timeseries", - "title": "Gradient Norm", - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 14 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "targets": [ - { - "expr": "foxhunt_training_gradient_norm", - "legendFormat": "{{model}}", - "refId": "A" - } - ], + "title": "Training & Validation Loss", + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 6 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "lineInterpolation": "smooth", - "fillOpacity": 10, - "spanNulls": false, "lineWidth": 2, - "pointSize": 5, + "fillOpacity": 10, + "gradientMode": "scheme", "showPoints": "auto", - "axisLabel": "Grad Norm", - "scaleDistribution": { - "type": "linear" - }, - "thresholdsStyle": { - "mode": "line+area" - } + "pointSize": 5, + "axisCenteredZero": false, + "axisLabel": "Loss" }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 10 - }, - { - "color": "red", - "value": 100 - } - ] - }, - "unit": "none", - "color": { - "mode": "palette-classic" - } + "color": { "mode": "palette-classic" } }, "overrides": [] }, "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "max", - "last" - ] - } - } - }, - { - "type": "row", - "title": "Hyperopt Progress", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 21 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 12, - "type": "gauge", - "title": "Trial Progress", - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 22 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "min"] }, + "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { - "expr": "foxhunt_hyperopt_trial_current / foxhunt_hyperopt_trial_total * 100", - "legendFormat": "progress", + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "foxhunt_training_epoch_loss{model=~\"$model\", fold=~\"$fold\"}", + "legendFormat": "train {{model}}/{{fold}}", "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "foxhunt_training_validation_loss{model=~\"$model\", fold=~\"$fold\"}", + "legendFormat": "val {{model}}/{{fold}}", + "refId": "B" } - ], + ] + }, + { + "id": 12, + "type": "timeseries", + "title": "Learning Rate", + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 6 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "orange", - "value": 25 - }, - { - "color": "yellow", - "value": 50 - }, - { - "color": "light-green", - "value": 75 - }, - { - "color": "green", - "value": 95 - } - ] + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 5, + "showPoints": "auto", + "pointSize": 5, + "scaleDistribution": { "type": "log", "log": 10 }, + "axisLabel": "LR" }, - "unit": "percent", - "min": 0, - "max": 100 + "color": { "mode": "palette-classic" }, + "unit": "short" }, "overrides": [] }, "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - } + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "foxhunt_training_learning_rate{model=~\"$model\", fold=~\"$fold\"}", + "legendFormat": "{{model}}/{{fold}}", + "refId": "A" + } + ] }, { "id": 13, "type": "timeseries", - "title": "Best Objective", - "gridPos": { - "h": 8, - "w": 16, - "x": 8, - "y": 22 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "targets": [ - { - "expr": "foxhunt_training_best_objective", - "legendFormat": "{{model}}", - "refId": "A" - } - ], + "title": "Gradient Norm", + "gridPos": { "h": 8, "w": 4, "x": 16, "y": 6 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", - "lineInterpolation": "stepAfter", - "fillOpacity": 15, - "gradientMode": "scheme", - "spanNulls": true, - "lineWidth": 2, - "pointSize": 5, - "showPoints": "auto", - "axisLabel": "Objective", - "scaleDistribution": { - "type": "linear" - } + "lineInterpolation": "smooth", + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never", + "axisLabel": "Norm" }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "none", - "color": { - "mode": "palette-classic" - } + "color": { "mode": "palette-classic" } }, "overrides": [] }, "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "min", - "max", - "last" - ] - } - } - }, - { - "type": "row", - "title": "GPU Metrics", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 29 + "legend": { "displayMode": "list", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } }, - "collapsed": false, - "panels": [] + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "foxhunt_training_gradient_norm{model=~\"$model\", fold=~\"$fold\"}", + "legendFormat": "{{model}}/{{fold}}", + "refId": "A" + } + ] }, { "id": 14, "type": "timeseries", - "title": "GPU Utilization", - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 30 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "targets": [ - { - "expr": "DCGM_FI_DEV_GPU_UTIL", - "legendFormat": "{{Hostname}} gpu{{gpu}}", - "refId": "A" - } - ], + "title": "Batches/sec & Epoch Duration", + "gridPos": { "h": 8, "w": 4, "x": 20, "y": 6 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "lineInterpolation": "smooth", - "fillOpacity": 20, - "gradientMode": "scheme", - "spanNulls": false, "lineWidth": 2, - "pointSize": 5, - "showPoints": "never", - "axisLabel": "Utilization", - "scaleDistribution": { - "type": "linear" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "yellow", - "value": 50 - }, - { - "color": "green", - "value": 80 - } - ] - }, - "unit": "percent", - "min": 0, - "max": 100, - "color": { - "mode": "palette-classic" - } - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "max", - "last" - ] - } - } - }, - { - "id": 15, - "type": "timeseries", - "title": "GPU Memory Used", - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 30 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "targets": [ - { - "expr": "DCGM_FI_DEV_FB_USED / (DCGM_FI_DEV_FB_USED + DCGM_FI_DEV_FB_FREE) * 100", - "legendFormat": "{{Hostname}} gpu{{gpu}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 20, - "gradientMode": "scheme", - "spanNulls": false, - "lineWidth": 2, - "pointSize": 5, - "showPoints": "never", - "axisLabel": "Memory %", - "scaleDistribution": { - "type": "linear" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 75 - }, - { - "color": "red", - "value": 90 - } - ] - }, - "unit": "percent", - "min": 0, - "max": 100, - "color": { - "mode": "palette-classic" - } - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "max", - "last" - ] - } - } - }, - { - "id": 16, - "type": "timeseries", - "title": "GPU Temperature", - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 30 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "targets": [ - { - "expr": "DCGM_FI_DEV_GPU_TEMP", - "legendFormat": "{{Hostname}} gpu{{gpu}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", "fillOpacity": 10, - "spanNulls": false, - "lineWidth": 2, - "pointSize": 5, - "showPoints": "never", - "axisLabel": "Temp", - "scaleDistribution": { - "type": "linear" - }, - "thresholdsStyle": { - "mode": "line+area" - } + "showPoints": "never" }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "orange", - "value": 80 - }, - { - "color": "red", - "value": 90 - } - ] - }, - "unit": "celsius", - "color": { - "mode": "palette-classic" - } - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "max", - "last" - ] - } - } - }, - { - "type": "row", - "title": "GPU Detail", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 37 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 17, - "type": "timeseries", - "title": "GPU Power Draw", - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 38 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "targets": [ - { - "expr": "DCGM_FI_DEV_POWER_USAGE", - "legendFormat": "{{Hostname}} gpu{{gpu}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 15, - "gradientMode": "scheme", - "spanNulls": false, - "lineWidth": 2, - "pointSize": 5, - "showPoints": "never", - "axisLabel": "Power", - "scaleDistribution": { - "type": "linear" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "watt", - "color": { - "mode": "palette-classic" - } - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "max", - "last" - ] - } - } - }, - { - "id": 18, - "type": "timeseries", - "title": "SM Clock", - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 38 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "targets": [ - { - "expr": "DCGM_FI_DEV_SM_CLOCK", - "legendFormat": "{{Hostname}} gpu{{gpu}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 10, - "spanNulls": false, - "lineWidth": 2, - "pointSize": 5, - "showPoints": "never", - "axisLabel": "Clock", - "scaleDistribution": { - "type": "linear" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - } - ] - }, - "unit": "clockHz", - "color": { - "mode": "palette-classic" - } - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "last" - ] - } - } - }, - { - "id": 19, - "type": "timeseries", - "title": "PCIe TX/RX", - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 38 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "targets": [ - { - "expr": "DCGM_FI_DEV_PCIE_TX_THROUGHPUT", - "legendFormat": "TX {{Hostname}} gpu{{gpu}}", - "refId": "A" - }, - { - "expr": "DCGM_FI_DEV_PCIE_RX_THROUGHPUT", - "legendFormat": "RX {{Hostname}} gpu{{gpu}}", - "refId": "B" - } - ], - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 10, - "spanNulls": false, - "lineWidth": 2, - "pointSize": 5, - "showPoints": "never", - "axisLabel": "Throughput", - "scaleDistribution": { - "type": "linear" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "Bps", - "color": { - "mode": "palette-classic" - } - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "max", - "last" - ] - } - } - }, - { - "type": "row", - "title": "Argo CI Pipelines", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 45 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 20, - "type": "stat", - "title": "Active Workflows", - "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 46 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "targets": [ - { - "expr": "argo_workflows_gauge{status=\"Running\"}", - "legendFormat": "running", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": null - }, - { - "color": "green", - "value": 1 - } - ] - }, - "mappings": [], - "unit": "none" - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "colorMode": "background", - "graphMode": "none" - } - }, - { - "id": 21, - "type": "stat", - "title": "Succeeded", - "gridPos": { - "h": 4, - "w": 6, - "x": 6, - "y": 46 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "targets": [ - { - "expr": "argo_workflows_gauge{status=\"Succeeded\"}", - "legendFormat": "succeeded", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "mappings": [], - "unit": "none" - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "colorMode": "background", - "graphMode": "none" - } - }, - { - "id": 22, - "type": "stat", - "title": "Failed", - "gridPos": { - "h": 4, - "w": 6, - "x": 12, - "y": 46 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "targets": [ - { - "expr": "argo_workflows_gauge{status=\"Failed\"}", - "legendFormat": "failed", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - }, - "mappings": [], - "unit": "none" - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "colorMode": "background", - "graphMode": "none" - } - }, - { - "id": 23, - "type": "stat", - "title": "Pending", - "gridPos": { - "h": 4, - "w": 6, - "x": 18, - "y": 46 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "targets": [ - { - "expr": "argo_workflows_gauge{status=\"Pending\"}", - "legendFormat": "pending", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 1 - } - ] - }, - "mappings": [], - "unit": "none" - }, - "overrides": [] - }, - "options": { - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "colorMode": "background", - "graphMode": "none" - } - }, - { - "type": "row", - "title": "Argo Performance", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 50 - }, - "collapsed": false, - "panels": [] - }, - { - "id": 24, - "type": "timeseries", - "title": "Operation Duration", - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 51 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "targets": [ - { - "expr": "histogram_quantile(0.50, rate(argo_workflows_operation_duration_seconds_bucket[$interval]))", - "legendFormat": "p50", - "refId": "A" - }, - { - "expr": "histogram_quantile(0.95, rate(argo_workflows_operation_duration_seconds_bucket[$interval]))", - "legendFormat": "p95", - "refId": "B" - } - ], - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 10, - "gradientMode": "scheme", - "spanNulls": false, - "lineWidth": 2, - "pointSize": 5, - "showPoints": "auto", - "axisLabel": "Duration", - "scaleDistribution": { - "type": "linear" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 30 - }, - { - "color": "red", - "value": 60 - } - ] - }, - "unit": "s", - "color": { - "mode": "palette-classic" - } + "color": { "mode": "palette-classic" } }, "overrides": [ { - "matcher": { - "id": "byName", - "options": "p95" - }, + "matcher": { "id": "byFrameRefID", "options": "B" }, "properties": [ - { - "id": "custom.lineStyle", - "value": { - "fill": "dash", - "dash": [ - 10, - 10 - ] - } - } + { "id": "custom.axisPlacement", "value": "right" }, + { "id": "unit", "value": "s" } ] } ] }, "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" + "legend": { "displayMode": "list", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "foxhunt_training_batches_per_second{model=~\"$model\", fold=~\"$fold\"}", + "legendFormat": "batch/s {{model}}", + "refId": "A" }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "max", - "last" - ] + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "foxhunt_training_epoch_duration_seconds{model=~\"$model\", fold=~\"$fold\"}", + "legendFormat": "epoch dur {{model}}", + "refId": "B" } - } + ] + }, + { + "type": "row", + "title": "Training Quality", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 }, + "collapsed": false, + "id": 20, + "panels": [] + }, + { + "id": 21, + "type": "timeseries", + "title": "Epoch Sharpe Ratio", + "gridPos": { "h": 7, "w": 6, "x": 0, "y": 15 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 10, + "gradientMode": "scheme", + "showPoints": "auto", + "pointSize": 4, + "axisLabel": "Sharpe" + }, + "color": { "mode": "palette-classic" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "yellow", "value": 1 }, + { "color": "green", "value": 2 } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "max"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "foxhunt_training_epoch_sharpe{model=~\"$model\", fold=~\"$fold\"}", + "legendFormat": "{{model}}/{{fold}}", + "refId": "A" + } + ] + }, + { + "id": 22, + "type": "timeseries", + "title": "Epoch Sortino Ratio", + "gridPos": { "h": 7, "w": 6, "x": 6, "y": 15 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 10, + "showPoints": "auto", + "pointSize": 4, + "axisLabel": "Sortino" + }, + "color": { "mode": "palette-classic" } + }, + "overrides": [] + }, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "max"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "foxhunt_training_epoch_sortino{model=~\"$model\", fold=~\"$fold\"}", + "legendFormat": "{{model}}/{{fold}}", + "refId": "A" + } + ] + }, + { + "id": 23, + "type": "timeseries", + "title": "Win Rate", + "gridPos": { "h": 7, "w": 4, "x": 12, "y": 15 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 10, + "showPoints": "auto", + "pointSize": 4, + "axisLabel": "Win Rate" + }, + "color": { "mode": "palette-classic" }, + "unit": "percentunit", + "min": 0, + "max": 1 + }, + "overrides": [] + }, + "options": { + "legend": { "displayMode": "list", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "foxhunt_training_epoch_win_rate{model=~\"$model\", fold=~\"$fold\"}", + "legendFormat": "{{model}}/{{fold}}", + "refId": "A" + } + ] + }, + { + "id": 24, + "type": "timeseries", + "title": "Max Drawdown", + "gridPos": { "h": 7, "w": 4, "x": 16, "y": 15 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 10, + "showPoints": "auto", + "pointSize": 4, + "axisLabel": "Drawdown" + }, + "color": { "mode": "palette-classic" }, + "unit": "percentunit" + }, + "overrides": [] + }, + "options": { + "legend": { "displayMode": "list", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "foxhunt_training_epoch_max_drawdown{model=~\"$model\", fold=~\"$fold\"}", + "legendFormat": "{{model}}/{{fold}}", + "refId": "A" + } + ] }, { "id": 25, "type": "timeseries", - "title": "K8s API Requests", - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 51 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "targets": [ - { - "expr": "topk(10, sum by (verb, kind) (rate(argo_workflows_k8s_request_total[$interval])))", - "legendFormat": "{{verb}} {{kind}}", - "refId": "A" - } - ], + "title": "Profit Factor & Total Trades", + "gridPos": { "h": 7, "w": 4, "x": 20, "y": 15 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "lineInterpolation": "smooth", - "fillOpacity": 10, - "gradientMode": "scheme", - "spanNulls": false, "lineWidth": 2, - "pointSize": 5, - "showPoints": "never", - "axisLabel": "req/s", - "scaleDistribution": { - "type": "linear" - } + "fillOpacity": 5, + "showPoints": "auto", + "pointSize": 4 }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "reqps", - "color": { - "mode": "palette-classic" - } + "color": { "mode": "palette-classic" } }, - "overrides": [] + "overrides": [ + { + "matcher": { "id": "byFrameRefID", "options": "B" }, + "properties": [ + { "id": "custom.axisPlacement", "value": "right" }, + { "id": "unit", "value": "short" }, + { "id": "custom.drawStyle", "value": "bars" }, + { "id": "custom.fillOpacity", "value": 30 } + ] + } + ] }, "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" + "legend": { "displayMode": "list", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "foxhunt_training_epoch_profit_factor{model=~\"$model\", fold=~\"$fold\"}", + "legendFormat": "PF {{model}}", + "refId": "A" }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "max", - "last" - ] + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "foxhunt_training_epoch_total_trades{model=~\"$model\", fold=~\"$fold\"}", + "legendFormat": "trades {{model}}", + "refId": "B" } - } + ] }, { "type": "row", - "title": "Argo Internals", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 58 - }, + "title": "RL Diagnostics", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 22 }, "collapsed": false, + "id": 30, "panels": [] }, { - "id": 26, + "id": 31, "type": "timeseries", - "title": "Queue Depth", - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 59 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "targets": [ - { - "expr": "argo_workflows_queue_depth_gauge", - "legendFormat": "{{queue_name}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "stepAfter", - "fillOpacity": 20, - "gradientMode": "scheme", - "spanNulls": false, - "lineWidth": 2, - "pointSize": 5, - "showPoints": "never", - "axisLabel": "Depth", - "scaleDistribution": { - "type": "linear" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 50 - }, - { - "color": "red", - "value": 100 - } - ] - }, - "unit": "none", - "min": 0, - "color": { - "mode": "palette-classic" - } - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "max", - "last" - ] - } - } - }, - { - "id": 27, - "type": "timeseries", - "title": "Worker Busy", - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 59 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "targets": [ - { - "expr": "argo_workflows_workers_busy_count", - "legendFormat": "{{worker_type}}", - "refId": "A" - } - ], - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "stepAfter", - "fillOpacity": 20, - "gradientMode": "scheme", - "spanNulls": false, - "lineWidth": 2, - "pointSize": 5, - "showPoints": "never", - "axisLabel": "Busy", - "scaleDistribution": { - "type": "linear" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "none", - "min": 0, - "color": { - "mode": "palette-classic" - } - }, - "overrides": [] - }, - "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "mean", - "max", - "last" - ] - } - } - }, - { - "id": 28, - "type": "timeseries", - "title": "Error Count", - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 59 - }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "targets": [ - { - "expr": "rate(argo_workflows_error_count[$interval])", - "legendFormat": "{{cause}}", - "refId": "A" - } - ], + "title": "Q-Value (Mean & Max)", + "gridPos": { "h": 7, "w": 6, "x": 0, "y": 23 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "lineInterpolation": "smooth", - "fillOpacity": 15, - "gradientMode": "scheme", - "spanNulls": false, "lineWidth": 2, - "pointSize": 5, + "fillOpacity": 10, "showPoints": "never", - "axisLabel": "Errors/s", - "scaleDistribution": { - "type": "linear" - } + "axisLabel": "Q-Value" }, + "color": { "mode": "palette-classic" } + }, + "overrides": [] + }, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "max", "min"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "foxhunt_training_q_value_mean{model=~\"$model\", fold=~\"$fold\"}", + "legendFormat": "mean {{model}}/{{fold}}", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "foxhunt_training_q_value_max{model=~\"$model\", fold=~\"$fold\"}", + "legendFormat": "max {{model}}/{{fold}}", + "refId": "B" + } + ] + }, + { + "id": 32, + "type": "timeseries", + "title": "Replay Buffer Size", + "gridPos": { "h": 7, "w": 4, "x": 6, "y": 23 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "stepAfter", + "lineWidth": 2, + "fillOpacity": 15, + "showPoints": "never", + "axisLabel": "Transitions" + }, + "color": { "mode": "palette-classic" }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { "displayMode": "list", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "foxhunt_training_replay_buffer_size{model=~\"$model\", fold=~\"$fold\"}", + "legendFormat": "{{model}}/{{fold}}", + "refId": "A" + } + ] + }, + { + "id": 33, + "type": "timeseries", + "title": "Action Entropy", + "description": "Normalized entropy of policy actions. 0 = collapsed to single action, 1 = uniform distribution.", + "gridPos": { "h": 7, "w": 4, "x": 10, "y": 23 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15, + "showPoints": "never", + "axisLabel": "Entropy" + }, + "color": { "mode": "palette-classic" }, + "min": 0, + "max": 1, "thresholds": { "mode": "absolute", "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 0.1 - } + { "color": "red", "value": null }, + { "color": "yellow", "value": 0.1 }, + { "color": "green", "value": 0.3 } ] }, - "unit": "none", + "custom.thresholdsStyle": { "mode": "line" } + }, + "overrides": [] + }, + "options": { + "legend": { "displayMode": "list", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "foxhunt_training_action_entropy{model=~\"$model\", fold=~\"$fold\"}", + "legendFormat": "{{model}}/{{fold}}", + "refId": "A" + } + ] + }, + { + "id": 34, + "type": "timeseries", + "title": "Action Diversity", + "description": "Fraction of 45 factored actions actually used by the policy.", + "gridPos": { "h": 7, "w": 4, "x": 14, "y": 23 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15, + "showPoints": "never", + "axisLabel": "Diversity" + }, + "color": { "mode": "palette-classic" }, + "unit": "percentunit", "min": 0, - "color": { - "mode": "palette-classic" + "max": 1 + }, + "overrides": [] + }, + "options": { + "legend": { "displayMode": "list", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "foxhunt_training_action_diversity{model=~\"$model\", fold=~\"$fold\"}", + "legendFormat": "{{model}}/{{fold}}", + "refId": "A" + } + ] + }, + { + "id": 35, + "type": "timeseries", + "title": "Action Distribution (Buy/Sell/Hold %)", + "gridPos": { "h": 7, "w": 6, "x": 18, "y": 23 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 20, + "showPoints": "never", + "stacking": { "mode": "normal" }, + "axisLabel": "Pct" + }, + "color": { "mode": "palette-classic" }, + "unit": "percentunit", + "min": 0, + "max": 1 + }, + "overrides": [ + { + "matcher": { "id": "byRegexp", "options": "buy.*" }, + "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] + }, + { + "matcher": { "id": "byRegexp", "options": "sell.*" }, + "properties": [ { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } ] + }, + { + "matcher": { "id": "byRegexp", "options": "hold.*" }, + "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] + } + ] + }, + "options": { + "legend": { "displayMode": "list", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "foxhunt_training_epoch_action_buy_pct{model=~\"$model\", fold=~\"$fold\"}", + "legendFormat": "buy {{model}}", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "foxhunt_training_epoch_action_sell_pct{model=~\"$model\", fold=~\"$fold\"}", + "legendFormat": "sell {{model}}", + "refId": "B" + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "foxhunt_training_epoch_action_hold_pct{model=~\"$model\", fold=~\"$fold\"}", + "legendFormat": "hold {{model}}", + "refId": "C" + } + ] + }, + { + "type": "row", + "title": "Hyperopt", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 30 }, + "collapsed": false, + "id": 40, + "panels": [] + }, + { + "id": 41, + "type": "gauge", + "title": "Trial Progress (current / total)", + "gridPos": { "h": 6, "w": 4, "x": 0, "y": 31 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "min": 0, + "max": 1, + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "orange", "value": 0.25 }, + { "color": "yellow", "value": 0.5 }, + { "color": "green", "value": 0.75 } + ] + }, + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "foxhunt_hyperopt_trial_current{model=~\"$model\"} / foxhunt_hyperopt_trial_total{model=~\"$model\"}", + "legendFormat": "{{model}}", + "refId": "A" + } + ] + }, + { + "id": 42, + "type": "timeseries", + "title": "Best Objective Over Time", + "gridPos": { "h": 6, "w": 6, "x": 4, "y": 31 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "stepAfter", + "lineWidth": 2, + "fillOpacity": 10, + "showPoints": "auto", + "pointSize": 4, + "axisLabel": "Objective" + }, + "color": { "mode": "palette-classic" } + }, + "overrides": [] + }, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "min"] }, + "tooltip": { "mode": "multi", "sort": "asc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "foxhunt_hyperopt_best_objective{model=~\"$model\"}", + "legendFormat": "best {{model}}", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "foxhunt_hyperopt_trial_best_loss{model=~\"$model\"}", + "legendFormat": "trial loss {{model}}", + "refId": "B" + } + ] + }, + { + "id": 43, + "type": "timeseries", + "title": "Trial Duration (p50/p95/p99)", + "gridPos": { "h": 6, "w": 6, "x": 10, "y": 31 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 10, + "showPoints": "never", + "axisLabel": "Duration" + }, + "color": { "mode": "palette-classic" }, + "unit": "s" + }, + "overrides": [] + }, + "options": { + "legend": { "displayMode": "list", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "histogram_quantile(0.50, rate(foxhunt_hyperopt_trial_duration_seconds_bucket{model=~\"$model\"}[$__rate_interval]))", + "legendFormat": "p50 {{model}}", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "histogram_quantile(0.95, rate(foxhunt_hyperopt_trial_duration_seconds_bucket{model=~\"$model\"}[$__rate_interval]))", + "legendFormat": "p95 {{model}}", + "refId": "B" + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "histogram_quantile(0.99, rate(foxhunt_hyperopt_trial_duration_seconds_bucket{model=~\"$model\"}[$__rate_interval]))", + "legendFormat": "p99 {{model}}", + "refId": "C" + } + ] + }, + { + "id": 44, + "type": "stat", + "title": "Elapsed Time", + "gridPos": { "h": 6, "w": 4, "x": 16, "y": 31 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "unit": "s", + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 3600 }, + { "color": "red", "value": 14400 } + ] + }, + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "textMode": "auto" + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "foxhunt_hyperopt_elapsed_seconds{model=~\"$model\"}", + "legendFormat": "{{model}}", + "refId": "A" + } + ] + }, + { + "id": 45, + "type": "stat", + "title": "Current Trial / Total", + "gridPos": { "h": 6, "w": 4, "x": 20, "y": 31 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "blue", "value": null } + ] + }, + "color": { "mode": "thresholds" } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "textMode": "auto" + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "foxhunt_hyperopt_trial_current{model=~\"$model\"}", + "legendFormat": "trial {{model}}", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "foxhunt_hyperopt_trial_total{model=~\"$model\"}", + "legendFormat": "total {{model}}", + "refId": "B" + } + ] + }, + { + "type": "row", + "title": "GPU Utilization", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 37 }, + "collapsed": false, + "id": 50, + "panels": [] + }, + { + "id": 51, + "type": "timeseries", + "title": "GPU Utilization %", + "gridPos": { "h": 7, "w": 6, "x": 0, "y": 38 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 20, + "gradientMode": "scheme", + "showPoints": "never", + "axisLabel": "Util %" + }, + "color": { "mode": "palette-classic" }, + "unit": "percent", + "min": 0, + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "yellow", "value": 50 }, + { "color": "green", "value": 80 } + ] } }, "overrides": [] }, "options": { - "tooltip": { - "mode": "multi", - "sort": "desc" - }, - "legend": { - "displayMode": "table", - "placement": "bottom", - "calcs": [ - "sum", - "max", - "last" - ] + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "DCGM_FI_DEV_GPU_UTIL", + "legendFormat": "GPU {{gpu}}", + "refId": "A" } - } + ] + }, + { + "id": 52, + "type": "timeseries", + "title": "VRAM Used / Free (MiB)", + "gridPos": { "h": 7, "w": 6, "x": 6, "y": 38 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15, + "showPoints": "never", + "axisLabel": "MiB" + }, + "color": { "mode": "palette-classic" }, + "unit": "decmbytes" + }, + "overrides": [ + { + "matcher": { "id": "byRegexp", "options": "free.*" }, + "properties": [ + { "id": "custom.fillOpacity", "value": 5 }, + { "id": "custom.lineStyle", "value": { "fill": "dash", "dash": [10, 5] } } + ] + } + ] + }, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull"] }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "DCGM_FI_DEV_FB_USED", + "legendFormat": "used GPU {{gpu}}", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "DCGM_FI_DEV_FB_FREE", + "legendFormat": "free GPU {{gpu}}", + "refId": "B" + } + ] + }, + { + "id": 53, + "type": "timeseries", + "title": "GPU / Memory Temperature", + "gridPos": { "h": 7, "w": 4, "x": 12, "y": 38 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 10, + "showPoints": "never", + "axisLabel": "Temp" + }, + "color": { "mode": "palette-classic" }, + "unit": "celsius", + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 75 }, + { "color": "red", "value": 85 } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { "displayMode": "list", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "DCGM_FI_DEV_GPU_TEMP", + "legendFormat": "GPU temp {{gpu}}", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "DCGM_FI_DEV_MEMORY_TEMP", + "legendFormat": "mem temp {{gpu}}", + "refId": "B" + } + ] + }, + { + "id": 54, + "type": "timeseries", + "title": "Power Usage (W)", + "gridPos": { "h": 7, "w": 4, "x": 16, "y": 38 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15, + "showPoints": "never", + "axisLabel": "Watts" + }, + "color": { "mode": "palette-classic" }, + "unit": "watt" + }, + "overrides": [] + }, + "options": { + "legend": { "displayMode": "list", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "DCGM_FI_DEV_POWER_USAGE", + "legendFormat": "GPU {{gpu}}", + "refId": "A" + } + ] + }, + { + "id": 55, + "type": "timeseries", + "title": "Tensor Core & DRAM Activity", + "gridPos": { "h": 7, "w": 4, "x": 20, "y": 38 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 15, + "showPoints": "never", + "axisLabel": "Activity" + }, + "color": { "mode": "palette-classic" }, + "unit": "percentunit", + "min": 0, + "max": 1 + }, + "overrides": [] + }, + "options": { + "legend": { "displayMode": "list", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "DCGM_FI_PROF_PIPE_TENSOR_ACTIVE", + "legendFormat": "tensor {{gpu}}", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "DCGM_FI_PROF_DRAM_ACTIVE", + "legendFormat": "DRAM {{gpu}}", + "refId": "B" + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "DCGM_FI_PROF_GR_ENGINE_ACTIVE", + "legendFormat": "engine {{gpu}}", + "refId": "C" + } + ] + }, + { + "type": "row", + "title": "Argo CI/CD", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 45 }, + "collapsed": false, + "id": 60, + "panels": [] + }, + { + "id": 61, + "type": "stat", + "title": "Workflows by Status", + "gridPos": { "h": 5, "w": 6, "x": 0, "y": 46 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "blue", "value": null } + ] + }, + "color": { "mode": "thresholds" } + }, + "overrides": [ + { + "matcher": { "id": "byRegexp", "options": "Running" }, + "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] + }, + { + "matcher": { "id": "byRegexp", "options": "Pending" }, + "properties": [ { "id": "color", "value": { "fixedColor": "yellow", "mode": "fixed" } } ] + }, + { + "matcher": { "id": "byRegexp", "options": "Failed|Error" }, + "properties": [ { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } ] + }, + { + "matcher": { "id": "byRegexp", "options": "Succeeded" }, + "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] + } + ] + }, + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "textMode": "auto", + "orientation": "horizontal" + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "argo_workflows_gauge{status=\"Running\"}", + "legendFormat": "Running", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "argo_workflows_gauge{status=\"Pending\"}", + "legendFormat": "Pending", + "refId": "B" + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "argo_workflows_gauge{status=\"Failed\"}", + "legendFormat": "Failed", + "refId": "C" + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "argo_workflows_gauge{status=\"Succeeded\"}", + "legendFormat": "Succeeded", + "refId": "D" + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "argo_workflows_gauge{status=\"Error\"}", + "legendFormat": "Error", + "refId": "E" + } + ] + }, + { + "id": 62, + "type": "timeseries", + "title": "Workflow Pods", + "gridPos": { "h": 5, "w": 4, "x": 6, "y": 46 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "stepAfter", + "lineWidth": 2, + "fillOpacity": 20, + "showPoints": "never", + "axisLabel": "Pods" + }, + "color": { "mode": "palette-classic" }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { "displayMode": "list", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "argo_workflows_pods_gauge", + "legendFormat": "pods", + "refId": "A" + } + ] + }, + { + "id": 63, + "type": "timeseries", + "title": "Error Rate", + "gridPos": { "h": 5, "w": 4, "x": 10, "y": 46 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "bars", + "lineWidth": 1, + "fillOpacity": 50, + "showPoints": "never", + "axisLabel": "Errors/s" + }, + "color": { "fixedColor": "red", "mode": "fixed" }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { "displayMode": "list", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "rate(argo_workflows_error_count[$__rate_interval])", + "legendFormat": "errors/s", + "refId": "A" + } + ] + }, + { + "id": 64, + "type": "timeseries", + "title": "Operation Duration (p50/p95)", + "gridPos": { "h": 5, "w": 5, "x": 14, "y": 46 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 10, + "showPoints": "never", + "axisLabel": "Duration" + }, + "color": { "mode": "palette-classic" }, + "unit": "s" + }, + "overrides": [] + }, + "options": { + "legend": { "displayMode": "list", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "histogram_quantile(0.50, rate(argo_workflows_operation_duration_seconds_bucket[$__rate_interval]))", + "legendFormat": "p50", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "histogram_quantile(0.95, rate(argo_workflows_operation_duration_seconds_bucket[$__rate_interval]))", + "legendFormat": "p95", + "refId": "B" + } + ] + }, + { + "id": 65, + "type": "timeseries", + "title": "Template Trigger Rate", + "gridPos": { "h": 5, "w": 5, "x": 19, "y": 46 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "bars", + "lineWidth": 1, + "fillOpacity": 40, + "showPoints": "never", + "stacking": { "mode": "normal" }, + "axisLabel": "Triggers/s" + }, + "color": { "mode": "palette-classic" }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { "displayMode": "list", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "expr": "rate(argo_workflows_workflowtemplate_triggered_total[$__rate_interval])", + "legendFormat": "triggers/s", + "refId": "A" + } + ] } ] -} +} \ No newline at end of file diff --git a/infra/k8s/monitoring/dashboards/import.sh b/infra/k8s/monitoring/dashboards/import.sh index 75465b11b..6cacdab63 100755 --- a/infra/k8s/monitoring/dashboards/import.sh +++ b/infra/k8s/monitoring/dashboards/import.sh @@ -126,7 +126,7 @@ declare -A CONFIGMAP_GROUPS=( [grafana-dashboards-cockpit]="foxhunt-cockpit.json" [grafana-dashboards-trading]="foxhunt-trading.json" [grafana-dashboards-training]="foxhunt-training.json" - [grafana-dashboards-infra]="foxhunt-infrastructure.json" + [grafana-dashboards-infrastructure]="foxhunt-infrastructure.json" [grafana-dashboards-observability]="foxhunt-observability.json" ) @@ -135,7 +135,7 @@ declare -A CONFIGMAP_FOLDERS=( [grafana-dashboards-cockpit]="Foxhunt" [grafana-dashboards-trading]="Foxhunt" [grafana-dashboards-training]="Foxhunt" - [grafana-dashboards-infra]="Foxhunt" + [grafana-dashboards-infrastructure]="Foxhunt" [grafana-dashboards-observability]="Foxhunt" ) diff --git a/infra/k8s/monitoring/kube-state-metrics.yaml b/infra/k8s/monitoring/kube-state-metrics.yaml index f21919770..361cb095d 100644 --- a/infra/k8s/monitoring/kube-state-metrics.yaml +++ b/infra/k8s/monitoring/kube-state-metrics.yaml @@ -106,16 +106,11 @@ spec: metadata: labels: app.kubernetes.io/name: kube-state-metrics - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "8080" - gitlab.com/prometheus_scrape: "true" - gitlab.com/prometheus_port: "8080" - gitlab.com/prometheus_path: "/metrics" + app.kubernetes.io/part-of: foxhunt spec: serviceAccountName: kube-state-metrics nodeSelector: - k8s.scaleway.com/pool-name: infra + k8s.scaleway.com/pool-name: platform containers: - name: kube-state-metrics image: registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.14.0 @@ -127,10 +122,10 @@ spec: resources: requests: cpu: 25m - memory: 64Mi - limits: - cpu: 100m memory: 128Mi + limits: + cpu: 200m + memory: 256Mi securityContext: readOnlyRootFilesystem: true allowPrivilegeEscalation: false @@ -156,9 +151,6 @@ metadata: namespace: foxhunt labels: app.kubernetes.io/name: kube-state-metrics - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "8080" spec: selector: app.kubernetes.io/name: kube-state-metrics diff --git a/infra/k8s/monitoring/prometheus.yaml b/infra/k8s/monitoring/prometheus.yaml index 7de4685b6..3ca177c0b 100644 --- a/infra/k8s/monitoring/prometheus.yaml +++ b/infra/k8s/monitoring/prometheus.yaml @@ -128,6 +128,24 @@ data: - target_label: service replacement: argo-workflows + # Training workflow pods (ephemeral Argo pods exposing metrics on :9094) + - job_name: training-pods + kubernetes_sd_configs: + - role: pod + namespaces: + names: [foxhunt] + relabel_configs: + - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_component] + action: keep + regex: training-workflow + - source_labels: [__meta_kubernetes_pod_ip] + target_label: __address__ + replacement: ${1}:9094 + - source_labels: [__meta_kubernetes_pod_name] + target_label: pod + - source_labels: [__meta_kubernetes_pod_label_workflows_argoproj_io_workflow] + target_label: workflow + # Pushgateway (training job metrics) - job_name: pushgateway honor_labels: true diff --git a/infra/k8s/network-policies/allow-monitoring.yaml b/infra/k8s/network-policies/allow-monitoring.yaml index e083fd97f..7c74da48c 100644 --- a/infra/k8s/network-policies/allow-monitoring.yaml +++ b/infra/k8s/network-policies/allow-monitoring.yaml @@ -27,6 +27,8 @@ spec: matchLabels: app.kubernetes.io/name: prometheus ports: + - protocol: TCP + port: 8080 - protocol: TCP port: 9091 - protocol: TCP @@ -55,3 +57,18 @@ spec: ports: - protocol: TCP port: 4317 + # Kubernetes API (for kube-state-metrics list/watch) + - to: + - ipBlock: + cidr: 10.32.0.0/16 + ports: + - protocol: TCP + port: 443 + - protocol: TCP + port: 6443 + - to: + - ipBlock: + cidr: 172.16.0.0/16 + ports: + - protocol: TCP + port: 6443 diff --git a/infra/k8s/services/api.yaml b/infra/k8s/services/api.yaml index eac68b206..9d6dda1f0 100644 --- a/infra/k8s/services/api.yaml +++ b/infra/k8s/services/api.yaml @@ -40,7 +40,7 @@ spec: k8s.scaleway.com/pool-name: foxhunt initContainers: - name: fetch-binary - image: localhost:30500/root/foxhunt/foxhunt-runtime:latest + image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest command: ["/bin/sh", "-c"] args: - | @@ -79,7 +79,7 @@ spec: memory: 128Mi containers: - name: api - image: localhost:30500/root/foxhunt/foxhunt-runtime:latest + image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest securityContext: allowPrivilegeEscalation: false readOnlyRootFilesystem: true diff --git a/infra/k8s/services/backtesting-service.yaml b/infra/k8s/services/backtesting-service.yaml index e8e4b8877..e0c1a0377 100644 --- a/infra/k8s/services/backtesting-service.yaml +++ b/infra/k8s/services/backtesting-service.yaml @@ -39,7 +39,7 @@ spec: k8s.scaleway.com/pool-name: foxhunt initContainers: - name: fetch-binary - image: localhost:30500/root/foxhunt/foxhunt-runtime:latest + image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest command: ["/bin/sh", "-c"] args: - | @@ -78,7 +78,7 @@ spec: memory: 128Mi containers: - name: backtesting-service - image: localhost:30500/root/foxhunt/foxhunt-runtime:latest + image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest securityContext: allowPrivilegeEscalation: false readOnlyRootFilesystem: true diff --git a/infra/k8s/services/broker-gateway.yaml b/infra/k8s/services/broker-gateway.yaml index ef45a631d..d629ae9ca 100644 --- a/infra/k8s/services/broker-gateway.yaml +++ b/infra/k8s/services/broker-gateway.yaml @@ -39,7 +39,7 @@ spec: k8s.scaleway.com/pool-name: foxhunt initContainers: - name: fetch-binary - image: localhost:30500/root/foxhunt/foxhunt-runtime:latest + image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest command: ["/bin/sh", "-c"] args: - | @@ -78,7 +78,7 @@ spec: memory: 128Mi containers: - name: broker-gateway - image: localhost:30500/root/foxhunt/foxhunt-runtime:latest + image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest securityContext: allowPrivilegeEscalation: false readOnlyRootFilesystem: true diff --git a/infra/k8s/services/data-acquisition-service.yaml b/infra/k8s/services/data-acquisition-service.yaml index a056b10b1..682d34445 100644 --- a/infra/k8s/services/data-acquisition-service.yaml +++ b/infra/k8s/services/data-acquisition-service.yaml @@ -39,7 +39,7 @@ spec: k8s.scaleway.com/pool-name: foxhunt initContainers: - name: fetch-binary - image: localhost:30500/root/foxhunt/foxhunt-runtime:latest + image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest command: ["/bin/sh", "-c"] args: - | @@ -78,7 +78,7 @@ spec: memory: 128Mi containers: - name: data-acquisition-service - image: localhost:30500/root/foxhunt/foxhunt-runtime:latest + image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest securityContext: allowPrivilegeEscalation: false readOnlyRootFilesystem: true diff --git a/infra/k8s/services/ml-training-service.yaml b/infra/k8s/services/ml-training-service.yaml index 482fd5281..f7a314758 100644 --- a/infra/k8s/services/ml-training-service.yaml +++ b/infra/k8s/services/ml-training-service.yaml @@ -82,7 +82,7 @@ spec: k8s.scaleway.com/pool-name: foxhunt initContainers: - name: fetch-binary - image: localhost:30500/root/foxhunt/foxhunt-runtime:latest + image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest command: ["/bin/sh", "-c"] args: - | @@ -121,7 +121,7 @@ spec: memory: 128Mi containers: - name: ml-training-service - image: localhost:30500/root/foxhunt/foxhunt-runtime:latest + image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest securityContext: allowPrivilegeEscalation: false readOnlyRootFilesystem: true @@ -169,7 +169,7 @@ spec: name: minio-credentials key: secret-key - name: TRAINING_RUNTIME_IMAGE - value: "localhost:30500/root/foxhunt/foxhunt-training-runtime:latest" + value: "gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-training-runtime:latest" - name: RUST_LOG value: "info,opentelemetry=warn,h2=warn,tonic=warn,hyper=warn" - name: OTEL_EXPORTER_OTLP_ENDPOINT diff --git a/infra/k8s/services/trading-agent-service.yaml b/infra/k8s/services/trading-agent-service.yaml index 73df5930a..6f0327375 100644 --- a/infra/k8s/services/trading-agent-service.yaml +++ b/infra/k8s/services/trading-agent-service.yaml @@ -39,7 +39,7 @@ spec: k8s.scaleway.com/pool-name: foxhunt initContainers: - name: fetch-binary - image: localhost:30500/root/foxhunt/foxhunt-runtime:latest + image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest command: ["/bin/sh", "-c"] args: - | @@ -78,7 +78,7 @@ spec: memory: 128Mi containers: - name: trading-agent-service - image: localhost:30500/root/foxhunt/foxhunt-runtime:latest + image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest securityContext: allowPrivilegeEscalation: false readOnlyRootFilesystem: true diff --git a/infra/k8s/services/trading-service.yaml b/infra/k8s/services/trading-service.yaml index 20128a299..20a247bd5 100644 --- a/infra/k8s/services/trading-service.yaml +++ b/infra/k8s/services/trading-service.yaml @@ -39,7 +39,7 @@ spec: k8s.scaleway.com/pool-name: foxhunt initContainers: - name: fetch-binary - image: localhost:30500/root/foxhunt/foxhunt-runtime:latest + image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest command: ["/bin/sh", "-c"] args: - | @@ -78,7 +78,7 @@ spec: memory: 128Mi containers: - name: trading-service - image: localhost:30500/root/foxhunt/foxhunt-runtime:latest + image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest securityContext: allowPrivilegeEscalation: false readOnlyRootFilesystem: true diff --git a/infra/k8s/services/web-dashboard.yaml b/infra/k8s/services/web-dashboard.yaml index abdc2d1df..8031227c9 100644 --- a/infra/k8s/services/web-dashboard.yaml +++ b/infra/k8s/services/web-dashboard.yaml @@ -28,7 +28,7 @@ spec: - name: gitlab-registry initContainers: - name: fetch-assets - image: localhost:30500/root/foxhunt/foxhunt-runtime:latest + image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest command: ["/bin/sh", "-c"] args: - | diff --git a/infra/k8s/training/image-prepuller.yaml b/infra/k8s/training/image-prepuller.yaml index 20ca586cc..01c6e9bc9 100644 --- a/infra/k8s/training/image-prepuller.yaml +++ b/infra/k8s/training/image-prepuller.yaml @@ -39,7 +39,7 @@ spec: - name: gitlab-registry initContainers: - name: pull-training-runtime - image: localhost:30500/root/foxhunt/foxhunt-training-runtime:latest + image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-training-runtime:latest command: ["echo", "foxhunt-training-runtime image pulled"] resources: requests: diff --git a/infra/k8s/training/job-template.yaml b/infra/k8s/training/job-template.yaml index 3ff2e84ec..118e8aa69 100644 --- a/infra/k8s/training/job-template.yaml +++ b/infra/k8s/training/job-template.yaml @@ -38,7 +38,7 @@ spec: # Replaces PVC-based copy — no node affinity constraint, no L40S autoscale for writes. initContainers: - name: fetch-binary - image: localhost:30500/root/foxhunt/foxhunt-training-runtime:latest + image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-training-runtime:latest command: ["/bin/sh", "-c"] args: - | @@ -79,7 +79,7 @@ spec: memory: 128Mi containers: - name: training - image: localhost:30500/root/foxhunt/foxhunt-training-runtime:latest + image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-training-runtime:latest # Available binaries (copied by initContainer): # train_baseline_rl (for dqn, ppo) # train_baseline_supervised (for tft, mamba2, tggn, tlob, liquid, kan, xlstm, diffusion)