From 0070a0117e1d1b11c743ce1e7602f21550b3d59a Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Thu, 5 Mar 2026 21:27:07 +0100 Subject: [PATCH] feat(api): add api_uptime_seconds metric, replace web-gateway panels in cockpit - Add api_uptime_seconds gauge to API service (5s update interval), matching the pattern used by trading/backtesting/ml-training services - Remove obsolete Web Gateway Uptime and Active WebSocket Connections panels from cockpit (web-gateway is not deployed) - Add API Gateway Uptime (api_uptime_seconds) and API Auth Rate (api_auth_requests_total) panels in their place Co-Authored-By: Claude Opus 4.6 --- .../dashboards/foxhunt-cockpit.json | 1704 ++++++++++++++--- services/api/src/main.rs | 16 + 2 files changed, 1401 insertions(+), 319 deletions(-) diff --git a/infra/k8s/monitoring/dashboards/foxhunt-cockpit.json b/infra/k8s/monitoring/dashboards/foxhunt-cockpit.json index 3f5c133b6..0f270abb1 100644 --- a/infra/k8s/monitoring/dashboards/foxhunt-cockpit.json +++ b/infra/k8s/monitoring/dashboards/foxhunt-cockpit.json @@ -3,7 +3,10 @@ "uid": "foxhunt-cockpit", "title": "Foxhunt Operations Cockpit", "description": "Operations overview: service health, gRPC traffic, CI/CD pipelines, API gateway, trading metrics, and metrics pipeline health.", - "tags": ["foxhunt", "cockpit"], + "tags": [ + "foxhunt", + "cockpit" + ], "timezone": "browser", "schemaVersion": 39, "version": 2, @@ -39,7 +42,12 @@ "id": 1, "type": "row", "title": "Service Health", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, "collapsed": false, "panels": [] }, @@ -48,8 +56,16 @@ "type": "stat", "title": "Services Up", "description": "Number of foxhunt service targets currently up", - "gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 1 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "count(up{job=\"foxhunt-services\"} == 1)", @@ -62,18 +78,35 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "red", "value": null }, - { "color": "yellow", "value": 3 }, - { "color": "green", "value": 5 } + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 3 + }, + { + "color": "green", + "value": 5 + } ] }, - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "unit": "none" }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -86,8 +119,16 @@ "type": "stat", "title": "Services Down", "description": "Number of foxhunt service targets currently down", - "gridPos": { "h": 4, "w": 4, "x": 4, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 1 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "count(up{job=\"foxhunt-services\"} == 0) OR on() vector(0)", @@ -100,18 +141,35 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 1 }, - { "color": "red", "value": 2 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 2 + } ] }, - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "unit": "none" }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -123,8 +181,16 @@ "id": 4, "type": "stat", "title": "Trading Service Uptime", - "gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 1 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "trading_service_uptime_seconds", @@ -137,18 +203,35 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "red", "value": null }, - { "color": "yellow", "value": 300 }, - { "color": "green", "value": 3600 } + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 300 + }, + { + "color": "green", + "value": 3600 + } ] }, - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "unit": "s" }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "value", @@ -160,8 +243,16 @@ "id": 5, "type": "stat", "title": "Backtesting Service Uptime", - "gridPos": { "h": 4, "w": 4, "x": 12, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 1 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "backtesting_service_uptime_seconds", @@ -174,18 +265,35 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "red", "value": null }, - { "color": "yellow", "value": 300 }, - { "color": "green", "value": 3600 } + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 300 + }, + { + "color": "green", + "value": 3600 + } ] }, - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "unit": "s" }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "value", @@ -197,8 +305,16 @@ "id": 6, "type": "stat", "title": "ML Training Service Uptime", - "gridPos": { "h": 4, "w": 4, "x": 16, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 1 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "ml_training_service_uptime_seconds", @@ -211,18 +327,35 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "red", "value": null }, - { "color": "yellow", "value": 300 }, - { "color": "green", "value": 3600 } + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 300 + }, + { + "color": "green", + "value": 3600 + } ] }, - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "unit": "s" }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "value", @@ -234,8 +367,16 @@ "id": 7, "type": "stat", "title": "Data Acquisition Uptime", - "gridPos": { "h": 4, "w": 4, "x": 0, "y": 5 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 5 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "foxhunt_data_acquisition_uptime_seconds", @@ -248,18 +389,35 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "red", "value": null }, - { "color": "yellow", "value": 300 }, - { "color": "green", "value": 3600 } + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 300 + }, + { + "color": "green", + "value": 3600 + } ] }, - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "unit": "s" }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "value", @@ -270,33 +428,59 @@ { "id": 8, "type": "stat", - "title": "Web Gateway Uptime", - "gridPos": { "h": 4, "w": 4, "x": 4, "y": 5 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "title": "API Gateway Uptime", + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 5 + }, "targets": [ { - "expr": "foxhunt_web_gateway_uptime_seconds", - "legendFormat": "web_gateway", + "expr": "api_uptime_seconds", + "legendFormat": "api", "refId": "A" } ], "fieldConfig": { "defaults": { + "unit": "s", "thresholds": { "mode": "absolute", "steps": [ - { "color": "red", "value": null }, - { "color": "yellow", "value": 300 }, - { "color": "green", "value": 3600 } + { + "color": "red", + "value": null + }, + { + "color": "orange", + "value": 300 + }, + { + "color": "green", + "value": 3600 + } ] }, - "color": { "mode": "thresholds" }, - "unit": "s" + "mappings": [], + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "value", @@ -307,34 +491,51 @@ { "id": 9, "type": "stat", - "title": "Active WebSocket Connections", - "gridPos": { "h": 4, "w": 4, "x": 8, "y": 5 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "title": "API Auth Rate", + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 5 + }, "targets": [ { - "expr": "foxhunt_web_gateway_ws_connections_active", - "legendFormat": "ws_active", + "expr": "sum(rate(api_auth_requests_total[$__rate_interval]))", + "legendFormat": "req/s", "refId": "A" } ], "fieldConfig": { "defaults": { + "unit": "reqps", "thresholds": { "mode": "absolute", "steps": [ - { "color": "blue", "value": null }, - { "color": "green", "value": 1 }, - { "color": "yellow", "value": 50 }, - { "color": "red", "value": 100 } + { + "color": "green", + "value": null + } ] }, - "color": { "mode": "thresholds" }, - "unit": "none" + "mappings": [], + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "value", @@ -347,8 +548,16 @@ "type": "table", "title": "Per-Service Status", "description": "Current up/down status for each service target", - "gridPos": { "h": 4, "w": 12, "x": 12, "y": 5 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 5 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "up{job=\"foxhunt-services\"}", @@ -367,52 +576,115 @@ }, "overrides": [ { - "matcher": { "id": "byName", "options": "Value" }, + "matcher": { + "id": "byName", + "options": "Value" + }, "properties": [ { "id": "mappings", "value": [ - { "type": "value", "options": { "1": { "text": "UP", "color": "green" } } }, - { "type": "value", "options": { "0": { "text": "DOWN", "color": "red" } } } + { + "type": "value", + "options": { + "1": { + "text": "UP", + "color": "green" + } + } + }, + { + "type": "value", + "options": { + "0": { + "text": "DOWN", + "color": "red" + } + } + } ] }, - { "id": "custom.displayMode", "value": "color-background-solid" } + { + "id": "custom.displayMode", + "value": "color-background-solid" + } ] }, { - "matcher": { "id": "byName", "options": "Time" }, - "properties": [{ "id": "custom.hidden", "value": true }] + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] }, { - "matcher": { "id": "byName", "options": "__name__" }, - "properties": [{ "id": "custom.hidden", "value": true }] + "matcher": { + "id": "byName", + "options": "__name__" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] }, { - "matcher": { "id": "byName", "options": "job" }, - "properties": [{ "id": "custom.hidden", "value": true }] + "matcher": { + "id": "byName", + "options": "job" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] } ] }, "options": { "showHeader": true, - "sortBy": [{ "displayName": "service", "desc": false }] + "sortBy": [ + { + "displayName": "service", + "desc": false + } + ] }, "transformations": [ { "id": "organize", "options": { - "excludeByName": { "Time": true, "__name__": true, "job": true }, - "renameByName": { "service": "Service", "instance": "Instance", "Value": "Status" } + "excludeByName": { + "Time": true, + "__name__": true, + "job": true + }, + "renameByName": { + "service": "Service", + "instance": "Instance", + "Value": "Status" + } } } ] }, - { "id": 20, "type": "row", "title": "gRPC Overview", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 9 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, "collapsed": true, "panels": [ { @@ -420,8 +692,16 @@ "type": "timeseries", "title": "gRPC Request Rate (by service)", "description": "Rate of gRPC calls started, grouped by exported_service", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 10 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 10 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "sum by (exported_service) (rate(grpc_server_started_total[$__rate_interval]))", @@ -438,18 +718,32 @@ "pointSize": 5, "showPoints": "auto", "spanNulls": false, - "stacking": { "mode": "none" }, + "stacking": { + "mode": "none" + }, "axisPlacement": "auto", "lineWidth": 2 }, "unit": "reqps", - "color": { "mode": "palette-classic" } + "color": { + "mode": "palette-classic" + } }, "overrides": [] }, "options": { - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] }, - "tooltip": { "mode": "multi", "sort": "desc" } + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } } }, { @@ -457,8 +751,16 @@ "type": "timeseries", "title": "gRPC Error Rate (by service)", "description": "Rate of gRPC calls that returned a non-OK code", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 10 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 10 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "sum by (exported_service) (rate(grpc_server_handled_total{grpc_code!=\"0\"}[$__rate_interval]))", @@ -475,26 +777,49 @@ "pointSize": 5, "showPoints": "auto", "spanNulls": false, - "stacking": { "mode": "none" }, + "stacking": { + "mode": "none" + }, "axisPlacement": "auto", "lineWidth": 2 }, "unit": "reqps", - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 1 }, - { "color": "red", "value": 5 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 5 + } ] } }, "overrides": [] }, "options": { - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] }, - "tooltip": { "mode": "multi", "sort": "desc" } + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } } }, { @@ -502,8 +827,16 @@ "type": "timeseries", "title": "gRPC p99 Latency (by service)", "description": "99th percentile gRPC server handling duration", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 18 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 18 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "histogram_quantile(0.99, sum by (exported_service, le) (rate(grpc_server_handling_seconds_bucket[$__rate_interval])))", @@ -520,26 +853,49 @@ "pointSize": 5, "showPoints": "auto", "spanNulls": false, - "stacking": { "mode": "none" }, + "stacking": { + "mode": "none" + }, "axisPlacement": "auto", "lineWidth": 2 }, "unit": "s", - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 0.5 }, - { "color": "red", "value": 2 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.5 + }, + { + "color": "red", + "value": 2 + } ] } }, "overrides": [] }, "options": { - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] }, - "tooltip": { "mode": "multi", "sort": "desc" } + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } } }, { @@ -547,8 +903,16 @@ "type": "stat", "title": "gRPC Error Ratio", "description": "Percentage of gRPC calls returning non-OK status", - "gridPos": { "h": 8, "w": 6, "x": 12, "y": 18 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 18 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "sum(rate(grpc_server_handled_total{grpc_code!=\"0\"}[$__rate_interval])) / sum(rate(grpc_server_handled_total[$__rate_interval]))", @@ -561,12 +925,23 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 0.01 }, - { "color": "red", "value": 0.05 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.01 + }, + { + "color": "red", + "value": 0.05 + } ] }, - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "unit": "percentunit", "min": 0, "max": 1 @@ -574,7 +949,13 @@ "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -587,8 +968,16 @@ "type": "stat", "title": "gRPC Total RPS", "description": "Total gRPC requests per second across all services", - "gridPos": { "h": 8, "w": 6, "x": 18, "y": 18 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 18 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "sum(rate(grpc_server_started_total[$__rate_interval]))", @@ -601,19 +990,39 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "blue", "value": null }, - { "color": "green", "value": 1 }, - { "color": "yellow", "value": 500 }, - { "color": "red", "value": 1000 } + { + "color": "blue", + "value": null + }, + { + "color": "green", + "value": 1 + }, + { + "color": "yellow", + "value": 500 + }, + { + "color": "red", + "value": 1000 + } ] }, - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "unit": "reqps" }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "value", @@ -623,20 +1032,32 @@ } ] }, - { "id": 30, "type": "row", "title": "Argo CI/CD", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 10 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, "collapsed": true, "panels": [ { "id": 31, "type": "gauge", "title": "Running Workflows", - "gridPos": { "h": 6, "w": 4, "x": 0, "y": 11 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 11 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "argo_workflows_gauge{status=\"Running\"}", @@ -649,12 +1070,23 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 5 }, - { "color": "red", "value": 10 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 5 + }, + { + "color": "red", + "value": 10 + } ] }, - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "unit": "none", "min": 0, "max": 15 @@ -662,7 +1094,13 @@ "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "showThresholdLabels": false, "showThresholdMarkers": true, "orientation": "auto" @@ -672,8 +1110,16 @@ "id": 32, "type": "gauge", "title": "Pending Workflows", - "gridPos": { "h": 6, "w": 4, "x": 4, "y": 11 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 6, + "w": 4, + "x": 4, + "y": 11 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "argo_workflows_gauge{status=\"Pending\"}", @@ -686,12 +1132,23 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 3 }, - { "color": "red", "value": 8 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 3 + }, + { + "color": "red", + "value": 8 + } ] }, - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "unit": "none", "min": 0, "max": 15 @@ -699,7 +1156,13 @@ "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "showThresholdLabels": false, "showThresholdMarkers": true, "orientation": "auto" @@ -709,8 +1172,16 @@ "id": 33, "type": "stat", "title": "Failed Workflows", - "gridPos": { "h": 6, "w": 4, "x": 8, "y": 11 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 6, + "w": 4, + "x": 8, + "y": 11 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "argo_workflows_gauge{status=\"Failed\"}", @@ -723,18 +1194,35 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 1 }, - { "color": "red", "value": 3 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 3 + } ] }, - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "unit": "none" }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -746,8 +1234,16 @@ "id": 34, "type": "stat", "title": "Succeeded Workflows", - "gridPos": { "h": 6, "w": 4, "x": 12, "y": 11 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 6, + "w": 4, + "x": 12, + "y": 11 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "argo_workflows_gauge{status=\"Succeeded\"}", @@ -760,18 +1256,35 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "red", "value": null }, - { "color": "yellow", "value": 1 }, - { "color": "green", "value": 1 } + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "green", + "value": 1 + } ] }, - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "unit": "none" }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "value", @@ -783,8 +1296,16 @@ "id": 35, "type": "stat", "title": "Workflow Pods", - "gridPos": { "h": 6, "w": 4, "x": 16, "y": 11 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 6, + "w": 4, + "x": 16, + "y": 11 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "argo_workflows_pods_gauge", @@ -797,19 +1318,39 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "blue", "value": null }, - { "color": "green", "value": 1 }, - { "color": "yellow", "value": 20 }, - { "color": "red", "value": 50 } + { + "color": "blue", + "value": null + }, + { + "color": "green", + "value": 1 + }, + { + "color": "yellow", + "value": 20 + }, + { + "color": "red", + "value": 50 + } ] }, - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "unit": "none" }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "value", @@ -822,8 +1363,16 @@ "type": "timeseries", "title": "Argo Error Rate", "description": "Rate of errors from Argo Workflows controller", - "gridPos": { "h": 6, "w": 4, "x": 20, "y": 11 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 6, + "w": 4, + "x": 20, + "y": 11 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "rate(argo_workflows_error_count[$__rate_interval])", @@ -840,27 +1389,47 @@ "pointSize": 5, "showPoints": "auto", "spanNulls": false, - "stacking": { "mode": "none" }, + "stacking": { + "mode": "none" + }, "axisPlacement": "auto", "lineWidth": 2, - "thresholdsStyle": { "mode": "area" } + "thresholdsStyle": { + "mode": "area" + } }, "unit": "ops", - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 0.1 }, - { "color": "red", "value": 1 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.1 + }, + { + "color": "red", + "value": 1 + } ] } }, "overrides": [] }, "options": { - "legend": { "displayMode": "list", "placement": "bottom" }, - "tooltip": { "mode": "single" } + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } } }, { @@ -868,8 +1437,16 @@ "type": "timeseries", "title": "Argo Operation Duration p95", "description": "95th percentile Argo operation duration", - "gridPos": { "h": 6, "w": 12, "x": 0, "y": 17 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 17 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "histogram_quantile(0.95, sum by (le) (rate(argo_workflows_operation_duration_seconds_bucket[$__rate_interval])))", @@ -886,27 +1463,48 @@ "pointSize": 5, "showPoints": "auto", "spanNulls": false, - "stacking": { "mode": "none" }, + "stacking": { + "mode": "none" + }, "axisPlacement": "auto", "lineWidth": 2, - "thresholdsStyle": { "mode": "line" } + "thresholdsStyle": { + "mode": "line" + } }, "unit": "s", - "color": { "fixedColor": "purple", "mode": "fixed" }, + "color": { + "fixedColor": "purple", + "mode": "fixed" + }, "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 30 }, - { "color": "red", "value": 120 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 30 + }, + { + "color": "red", + "value": 120 + } ] } }, "overrides": [] }, "options": { - "legend": { "displayMode": "list", "placement": "bottom" }, - "tooltip": { "mode": "single" } + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } } }, { @@ -914,8 +1512,16 @@ "type": "timeseries", "title": "Argo K8s API Requests", "description": "Rate of Kubernetes API requests from Argo controller", - "gridPos": { "h": 6, "w": 12, "x": 12, "y": 17 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 17 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "sum(rate(argo_workflows_k8s_request_total[$__rate_interval]))", @@ -932,28 +1538,42 @@ "pointSize": 5, "showPoints": "auto", "spanNulls": false, - "stacking": { "mode": "none" }, + "stacking": { + "mode": "none" + }, "axisPlacement": "auto", "lineWidth": 2 }, "unit": "reqps", - "color": { "fixedColor": "orange", "mode": "fixed" } + "color": { + "fixedColor": "orange", + "mode": "fixed" + } }, "overrides": [] }, "options": { - "legend": { "displayMode": "list", "placement": "bottom" }, - "tooltip": { "mode": "single" } + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } } } ] }, - { "id": 40, "type": "row", "title": "API Gateway", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 11 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, "collapsed": true, "panels": [ { @@ -961,8 +1581,16 @@ "type": "timeseries", "title": "Auth Request Rate", "description": "Rate of authentication requests (total, success, failure)", - "gridPos": { "h": 8, "w": 8, "x": 0, "y": 12 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 12 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "rate(api_auth_requests_total[$__rate_interval])", @@ -989,27 +1617,59 @@ "pointSize": 5, "showPoints": "auto", "spanNulls": false, - "stacking": { "mode": "none" }, + "stacking": { + "mode": "none" + }, "axisPlacement": "auto", "lineWidth": 2 }, "unit": "reqps", - "color": { "mode": "palette-classic" } + "color": { + "mode": "palette-classic" + } }, "overrides": [ { - "matcher": { "id": "byName", "options": "failure" }, - "properties": [{ "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } }] + "matcher": { + "id": "byName", + "options": "failure" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] }, { - "matcher": { "id": "byName", "options": "success" }, - "properties": [{ "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } }] + "matcher": { + "id": "byName", + "options": "success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] } ] }, "options": { - "legend": { "displayMode": "list", "placement": "bottom" }, - "tooltip": { "mode": "multi", "sort": "desc" } + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } } }, { @@ -1017,8 +1677,16 @@ "type": "stat", "title": "Rate Limiter Entries", "description": "Current number of entries in the API gateway rate limiter", - "gridPos": { "h": 4, "w": 4, "x": 8, "y": 12 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 12 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "api_rate_limiter_entries", @@ -1031,18 +1699,35 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 100 }, - { "color": "red", "value": 500 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 100 + }, + { + "color": "red", + "value": 500 + } ] }, - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "unit": "none" }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -1055,8 +1740,16 @@ "type": "stat", "title": "JWT Cache Hit Ratio", "description": "Ratio of JWT cache hits to total lookups", - "gridPos": { "h": 4, "w": 4, "x": 12, "y": 12 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 12 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "api_jwt_cache_hits / (api_jwt_cache_hits + api_jwt_cache_misses)", @@ -1069,12 +1762,23 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "red", "value": null }, - { "color": "yellow", "value": 0.5 }, - { "color": "green", "value": 0.8 } + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 0.5 + }, + { + "color": "green", + "value": 0.8 + } ] }, - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "unit": "percentunit", "min": 0, "max": 1 @@ -1082,7 +1786,13 @@ "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -1095,8 +1805,16 @@ "type": "timeseries", "title": "Routing Latency p50 / p95 / p99", "description": "API gateway routing duration percentiles", - "gridPos": { "h": 8, "w": 8, "x": 16, "y": 12 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 12 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "histogram_quantile(0.50, sum by (le) (rate(api_routing_duration_microseconds_bucket[$__rate_interval])))", @@ -1123,26 +1841,45 @@ "pointSize": 5, "showPoints": "auto", "spanNulls": false, - "stacking": { "mode": "none" }, + "stacking": { + "mode": "none" + }, "axisPlacement": "auto", "lineWidth": 2 }, "unit": "us", - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 5000 }, - { "color": "red", "value": 50000 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 5000 + }, + { + "color": "red", + "value": 50000 + } ] } }, "overrides": [] }, "options": { - "legend": { "displayMode": "list", "placement": "bottom" }, - "tooltip": { "mode": "multi", "sort": "desc" } + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } } }, { @@ -1150,8 +1887,16 @@ "type": "stat", "title": "Auth Failure Rate", "description": "Percentage of auth requests that fail", - "gridPos": { "h": 4, "w": 4, "x": 8, "y": 16 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 16 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "rate(api_auth_requests_failure[$__rate_interval]) / rate(api_auth_requests_total[$__rate_interval])", @@ -1164,12 +1909,23 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 0.05 }, - { "color": "red", "value": 0.2 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.05 + }, + { + "color": "red", + "value": 0.2 + } ] }, - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "unit": "percentunit", "min": 0, "max": 1 @@ -1177,7 +1933,13 @@ "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -1190,8 +1952,16 @@ "type": "stat", "title": "JWT Cache Misses/s", "description": "Rate of JWT cache misses", - "gridPos": { "h": 4, "w": 4, "x": 12, "y": 16 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 16 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "rate(api_jwt_cache_misses[$__rate_interval])", @@ -1204,18 +1974,35 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 5 }, - { "color": "red", "value": 20 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 5 + }, + { + "color": "red", + "value": 20 + } ] }, - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "unit": "ops" }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "value", @@ -1225,12 +2012,16 @@ } ] }, - { "id": 50, "type": "row", "title": "Trading", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 12 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 12 + }, "collapsed": true, "panels": [ { @@ -1238,8 +2029,16 @@ "type": "stat", "title": "Portfolio Value", "description": "Current portfolio value in USD", - "gridPos": { "h": 6, "w": 6, "x": 0, "y": 13 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 13 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "trading_agent_portfolio_value_usd", @@ -1252,19 +2051,36 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "red", "value": null }, - { "color": "yellow", "value": 10000 }, - { "color": "green", "value": 100000 } + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 10000 + }, + { + "color": "green", + "value": 100000 + } ] }, - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "unit": "currencyUSD", "decimals": 2 }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "value", @@ -1277,8 +2093,16 @@ "type": "stat", "title": "Assets Selected", "description": "Number of assets currently selected for trading", - "gridPos": { "h": 6, "w": 3, "x": 6, "y": 13 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 6, + "w": 3, + "x": 6, + "y": 13 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "trading_agent_assets_selected", @@ -1291,17 +2115,31 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "blue", "value": null }, - { "color": "green", "value": 1 } + { + "color": "blue", + "value": null + }, + { + "color": "green", + "value": 1 + } ] }, - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "unit": "none" }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "value", @@ -1314,8 +2152,16 @@ "type": "stat", "title": "Universe Instruments", "description": "Total instruments in the trading universe", - "gridPos": { "h": 6, "w": 3, "x": 9, "y": 13 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 6, + "w": 3, + "x": 9, + "y": 13 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "trading_agent_universe_instruments", @@ -1328,16 +2174,27 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "blue", "value": null } + { + "color": "blue", + "value": null + } ] }, - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "unit": "none" }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "value", @@ -1350,8 +2207,16 @@ "type": "timeseries", "title": "Trading Measurements Rate", "description": "Rate of trading measurements emitted", - "gridPos": { "h": 6, "w": 6, "x": 12, "y": 13 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 13 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "rate(trading_measurements_total[$__rate_interval])", @@ -1368,18 +2233,28 @@ "pointSize": 5, "showPoints": "auto", "spanNulls": false, - "stacking": { "mode": "none" }, + "stacking": { + "mode": "none" + }, "axisPlacement": "auto", "lineWidth": 2 }, "unit": "ops", - "color": { "fixedColor": "blue", "mode": "fixed" } + "color": { + "fixedColor": "blue", + "mode": "fixed" + } }, "overrides": [] }, "options": { - "legend": { "displayMode": "list", "placement": "bottom" }, - "tooltip": { "mode": "single" } + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } } }, { @@ -1387,8 +2262,16 @@ "type": "timeseries", "title": "Total Trading Latency", "description": "End-to-end trading latency", - "gridPos": { "h": 6, "w": 6, "x": 18, "y": 13 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 13 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "trading_total_latency_seconds", @@ -1405,27 +2288,48 @@ "pointSize": 5, "showPoints": "auto", "spanNulls": false, - "stacking": { "mode": "none" }, + "stacking": { + "mode": "none" + }, "axisPlacement": "auto", "lineWidth": 2, - "thresholdsStyle": { "mode": "area" } + "thresholdsStyle": { + "mode": "area" + } }, "unit": "s", - "color": { "fixedColor": "orange", "mode": "fixed" }, + "color": { + "fixedColor": "orange", + "mode": "fixed" + }, "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 0.01 }, - { "color": "red", "value": 0.1 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.01 + }, + { + "color": "red", + "value": 0.1 + } ] } }, "overrides": [] }, "options": { - "legend": { "displayMode": "list", "placement": "bottom" }, - "tooltip": { "mode": "single" } + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } } }, { @@ -1433,8 +2337,16 @@ "type": "timeseries", "title": "Order Processing vs Risk Check vs Market Data", "description": "Breakdown of trading pipeline latency components", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 19 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 19 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "trading_order_processing_seconds", @@ -1461,31 +2373,79 @@ "pointSize": 5, "showPoints": "auto", "spanNulls": false, - "stacking": { "mode": "none" }, + "stacking": { + "mode": "none" + }, "axisPlacement": "auto", "lineWidth": 2 }, "unit": "s", - "color": { "mode": "palette-classic" } + "color": { + "mode": "palette-classic" + } }, "overrides": [ { - "matcher": { "id": "byName", "options": "order processing" }, - "properties": [{ "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } }] + "matcher": { + "id": "byName", + "options": "order processing" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] }, { - "matcher": { "id": "byName", "options": "risk check" }, - "properties": [{ "id": "color", "value": { "fixedColor": "orange", "mode": "fixed" } }] + "matcher": { + "id": "byName", + "options": "risk check" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] }, { - "matcher": { "id": "byName", "options": "market data" }, - "properties": [{ "id": "color", "value": { "fixedColor": "purple", "mode": "fixed" } }] + "matcher": { + "id": "byName", + "options": "market data" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + } + ] } ] }, "options": { - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "lastNotNull"] }, - "tooltip": { "mode": "multi", "sort": "desc" } + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } } }, { @@ -1493,8 +2453,16 @@ "type": "timeseries", "title": "Portfolio Value Over Time", "description": "Historical portfolio value trend", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 19 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 19 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "trading_agent_portfolio_value_usd", @@ -1511,30 +2479,43 @@ "pointSize": 5, "showPoints": "never", "spanNulls": true, - "stacking": { "mode": "none" }, + "stacking": { + "mode": "none" + }, "axisPlacement": "auto", "lineWidth": 2, "gradientMode": "scheme" }, "unit": "currencyUSD", - "color": { "mode": "continuous-GrYlRd" }, + "color": { + "mode": "continuous-GrYlRd" + }, "decimals": 2 }, "overrides": [] }, "options": { - "legend": { "displayMode": "list", "placement": "bottom" }, - "tooltip": { "mode": "single" } + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } } } ] }, - { "id": 60, "type": "row", "title": "Metrics Health", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 13 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 13 + }, "collapsed": true, "panels": [ { @@ -1542,8 +2523,16 @@ "type": "gauge", "title": "Metrics Buffer Utilization", "description": "Percentage of the metrics buffer currently in use", - "gridPos": { "h": 6, "w": 8, "x": 0, "y": 14 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 14 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "metrics_buffer_utilization_percent", @@ -1556,13 +2545,27 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 60 }, - { "color": "orange", "value": 80 }, - { "color": "red", "value": 95 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 60 + }, + { + "color": "orange", + "value": 80 + }, + { + "color": "red", + "value": 95 + } ] }, - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "unit": "percent", "min": 0, "max": 100 @@ -1570,7 +2573,13 @@ "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "showThresholdLabels": false, "showThresholdMarkers": true, "orientation": "auto" @@ -1581,8 +2590,16 @@ "type": "timeseries", "title": "Metrics Dropped Rate", "description": "Rate at which metrics are being dropped", - "gridPos": { "h": 6, "w": 8, "x": 8, "y": 14 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 14 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "rate(metrics_dropped_total[$__rate_interval])", @@ -1599,27 +2616,48 @@ "pointSize": 5, "showPoints": "auto", "spanNulls": false, - "stacking": { "mode": "none" }, + "stacking": { + "mode": "none" + }, "axisPlacement": "auto", "lineWidth": 2, - "thresholdsStyle": { "mode": "area" } + "thresholdsStyle": { + "mode": "area" + } }, "unit": "ops", - "color": { "fixedColor": "red", "mode": "fixed" }, + "color": { + "fixedColor": "red", + "mode": "fixed" + }, "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 1 }, - { "color": "red", "value": 10 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 10 + } ] } }, "overrides": [] }, "options": { - "legend": { "displayMode": "list", "placement": "bottom" }, - "tooltip": { "mode": "single" } + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } } }, { @@ -1627,8 +2665,16 @@ "type": "stat", "title": "Total Metrics Dropped", "description": "Total count of metrics dropped since service start", - "gridPos": { "h": 6, "w": 8, "x": 16, "y": 14 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 14 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "targets": [ { "expr": "metrics_dropped_total", @@ -1641,18 +2687,35 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 100 }, - { "color": "red", "value": 1000 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 100 + }, + { + "color": "red", + "value": 1000 + } ] }, - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "unit": "none" }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -1667,7 +2730,10 @@ "list": [ { "builtIn": 1, - "datasource": { "type": "grafana", "uid": "-- Grafana --" }, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", @@ -1678,4 +2744,4 @@ }, "links": [], "graphTooltip": 1 -} \ No newline at end of file +} diff --git a/services/api/src/main.rs b/services/api/src/main.rs index bb516b5d8..a7c65320a 100644 --- a/services/api/src/main.rs +++ b/services/api/src/main.rs @@ -543,6 +543,22 @@ async fn main() -> Result<()> { .map_err(|e| anyhow::anyhow!("Failed to register service info: {}", e))?; service_info.set(1.0); + // Add uptime gauge (matches pattern of other services) + let uptime_gauge = register_gauge_with_registry!( + Opts::new("api_uptime_seconds", "API service uptime in seconds"), + gateway_metrics.registry().as_ref() + ) + .map_err(|e| anyhow::anyhow!("Failed to register uptime gauge: {}", e))?; + let start_instant = std::time::Instant::now(); + let uptime_clone = uptime_gauge.clone(); + tokio::spawn(async move { + let mut tick = tokio::time::interval(Duration::from_secs(5)); + loop { + tick.tick().await; + uptime_clone.set(start_instant.elapsed().as_secs_f64()); + } + }); + let metrics_registry = gateway_metrics.registry(); tokio::spawn(async move {