{ "annotations": { "list": [ { "builtIn": 1, "datasource": { "type": "grafana", "uid": "-- Grafana --" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard" } ] }, "description": "Operations overview: service health, gRPC traffic, CI/CD pipelines, API gateway, trading metrics, and metrics pipeline health.", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, "links": [], "liveNow": false, "panels": [ { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 1, "panels": [], "title": "Service Health", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Number of foxhunt service targets currently up", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "yellow", "value": 3 }, { "color": "green", "value": 5 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 }, "id": 2, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "targets": [ { "expr": "count(up{job=\"foxhunt-services\"} == 1)", "legendFormat": "up", "refId": "A" } ], "title": "Services Up", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Number of foxhunt service targets currently down", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 1 }, { "color": "red", "value": 2 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 4, "y": 1 }, "id": 3, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "targets": [ { "expr": "count(up{job=\"foxhunt-services\"} == 0) OR on() vector(0)", "legendFormat": "down", "refId": "A" } ], "title": "Services Down", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "yellow", "value": 300 }, { "color": "green", "value": 3600 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 }, "id": 4, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "targets": [ { "expr": "trading_service_uptime_seconds", "legendFormat": "trading", "refId": "A" } ], "title": "Trading Service Uptime", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "yellow", "value": 300 }, { "color": "green", "value": 3600 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 12, "y": 1 }, "id": 5, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "targets": [ { "expr": "backtesting_service_uptime_seconds", "legendFormat": "backtesting", "refId": "A" } ], "title": "Backtesting Service Uptime", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "yellow", "value": 300 }, { "color": "green", "value": 3600 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 16, "y": 1 }, "id": 6, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "targets": [ { "expr": "ml_training_service_uptime_seconds", "legendFormat": "ml_training", "refId": "A" } ], "title": "ML Training Service Uptime", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "yellow", "value": 300 }, { "color": "green", "value": 3600 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 0, "y": 5 }, "id": 7, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "targets": [ { "expr": "foxhunt_data_acquisition_uptime_seconds", "legendFormat": "data_acquisition", "refId": "A" } ], "title": "Data Acquisition Uptime", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "orange", "value": 300 }, { "color": "green", "value": 3600 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 4, "y": 5 }, "id": 8, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "targets": [ { "expr": "max(time() - process_start_time_seconds{job=\"foxhunt-services\", instance=~\".*:9091\"})", "legendFormat": "api", "refId": "A" } ], "title": "API Gateway Uptime", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "reqps" }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 8, "y": 5 }, "id": 9, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "targets": [ { "expr": "sum(rate(api_auth_requests_total[$__rate_interval]))", "legendFormat": "req/s", "refId": "A" } ], "title": "API Auth Rate", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Current up/down status for each service target", "fieldConfig": { "defaults": { "custom": { "align": "auto", "displayMode": "color-background-solid" } }, "overrides": [ { "matcher": { "id": "byName", "options": "Value" }, "properties": [ { "id": "mappings", "value": [ { "options": { "1": { "color": "green", "text": "UP" } }, "type": "value" }, { "options": { "0": { "color": "red", "text": "DOWN" } }, "type": "value" } ] }, { "id": "custom.displayMode", "value": "color-background-solid" } ] }, { "matcher": { "id": "byName", "options": "Time" }, "properties": [ { "id": "custom.hidden", "value": true } ] }, { "matcher": { "id": "byName", "options": "__name__" }, "properties": [ { "id": "custom.hidden", "value": true } ] }, { "matcher": { "id": "byName", "options": "job" }, "properties": [ { "id": "custom.hidden", "value": true } ] } ] }, "gridPos": { "h": 4, "w": 12, "x": 12, "y": 5 }, "id": 10, "options": { "showHeader": true, "sortBy": [ { "desc": false, "displayName": "service" } ] }, "targets": [ { "expr": "up{job=\"foxhunt-services\"}", "format": "table", "instant": true, "legendFormat": "{{service}}", "refId": "A" } ], "title": "Per-Service Status", "transformations": [ { "id": "organize", "options": { "excludeByName": { "Time": true, "__name__": true, "job": true }, "renameByName": { "Value": "Status", "instance": "Instance", "service": "Service" } } } ], "type": "table" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 9 }, "id": 20, "panels": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Rate of gRPC calls started, grouped by exported_service", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 15, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "showPoints": "auto", "spanNulls": false, "stacking": { "mode": "none" } }, "unit": "reqps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 10 }, "id": 21, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "expr": "sum by (exported_service) (rate(grpc_server_started_total[$__rate_interval]))", "legendFormat": "{{exported_service}}", "refId": "A" } ], "title": "gRPC Request Rate (by service)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Rate of gRPC calls that returned a non-OK code", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 15, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "showPoints": "auto", "spanNulls": false, "stacking": { "mode": "none" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 1 }, { "color": "red", "value": 5 } ] }, "unit": "reqps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 10 }, "id": 22, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "expr": "sum by (exported_service) (rate(grpc_server_handled_total{grpc_code!=\"0\"}[$__rate_interval]))", "legendFormat": "{{exported_service}}", "refId": "A" } ], "title": "gRPC Error Rate (by service)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "99th percentile gRPC server handling duration", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 10, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "showPoints": "auto", "spanNulls": false, "stacking": { "mode": "none" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 0.5 }, { "color": "red", "value": 2 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 18 }, "id": 23, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "expr": "histogram_quantile(0.99, sum by (exported_service, le) (rate(grpc_server_handling_seconds_bucket[$__rate_interval])))", "legendFormat": "{{exported_service}} p99", "refId": "A" } ], "title": "gRPC p99 Latency (by service)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Percentage of gRPC calls returning non-OK status", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 0.01 }, { "color": "red", "value": 0.05 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 6, "x": 12, "y": 18 }, "id": 24, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "targets": [ { "expr": "sum(rate(grpc_server_handled_total{grpc_code!=\"0\"}[$__rate_interval])) / sum(rate(grpc_server_handled_total[$__rate_interval]))", "legendFormat": "error ratio", "refId": "A" } ], "title": "gRPC Error Ratio", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Total gRPC requests per second across all services", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "blue", "value": null }, { "color": "green", "value": 1 }, { "color": "yellow", "value": 500 }, { "color": "red", "value": 1000 } ] }, "unit": "reqps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 6, "x": 18, "y": 18 }, "id": 25, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "targets": [ { "expr": "sum(rate(grpc_server_started_total[$__rate_interval]))", "legendFormat": "total rps", "refId": "A" } ], "title": "gRPC Total RPS", "type": "stat" } ], "title": "gRPC Overview", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 10 }, "id": 30, "panels": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "max": 15, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 5 }, { "color": "red", "value": 10 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 6, "w": 4, "x": 0, "y": 11 }, "id": 31, "options": { "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true }, "targets": [ { "expr": "argo_workflows_gauge{status=\"Running\"}", "legendFormat": "running", "refId": "A" } ], "title": "Running Workflows", "type": "gauge" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "max": 15, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 3 }, { "color": "red", "value": 8 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 6, "w": 4, "x": 4, "y": 11 }, "id": 32, "options": { "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true }, "targets": [ { "expr": "argo_workflows_gauge{status=\"Pending\"}", "legendFormat": "pending", "refId": "A" } ], "title": "Pending Workflows", "type": "gauge" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 1 }, { "color": "red", "value": 3 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 6, "w": 4, "x": 8, "y": 11 }, "id": 33, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "targets": [ { "expr": "argo_workflows_gauge{status=\"Failed\"}", "legendFormat": "failed", "refId": "A" } ], "title": "Failed Workflows", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "yellow", "value": 1 }, { "color": "green", "value": 1 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 6, "w": 4, "x": 12, "y": 11 }, "id": 34, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "targets": [ { "expr": "argo_workflows_gauge{status=\"Succeeded\"}", "legendFormat": "succeeded", "refId": "A" } ], "title": "Succeeded Workflows", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "blue", "value": null }, { "color": "green", "value": 1 }, { "color": "yellow", "value": 20 }, { "color": "red", "value": 50 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 6, "w": 4, "x": 16, "y": 11 }, "id": 35, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "targets": [ { "expr": "argo_workflows_pods_gauge", "legendFormat": "pods", "refId": "A" } ], "title": "Workflow Pods", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Rate of errors from Argo Workflows controller", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 20, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "showPoints": "auto", "spanNulls": false, "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "area" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 0.1 }, { "color": "red", "value": 1 } ] }, "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 6, "w": 4, "x": 20, "y": 11 }, "id": 36, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "single" } }, "targets": [ { "expr": "rate(argo_workflows_error_count[$__rate_interval])", "legendFormat": "errors/s", "refId": "A" } ], "title": "Argo Error Rate", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "95th percentile Argo operation duration", "fieldConfig": { "defaults": { "color": { "fixedColor": "purple", "mode": "fixed" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 10, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "showPoints": "auto", "spanNulls": false, "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "line" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 30 }, { "color": "red", "value": 120 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 6, "w": 12, "x": 0, "y": 17 }, "id": 37, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "single" } }, "targets": [ { "expr": "histogram_quantile(0.95, sum by (le) (rate(argo_workflows_operation_duration_seconds_bucket[$__rate_interval])))", "legendFormat": "p95", "refId": "A" } ], "title": "Argo Operation Duration p95", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Rate of Kubernetes API requests from Argo controller", "fieldConfig": { "defaults": { "color": { "fixedColor": "orange", "mode": "fixed" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 10, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "showPoints": "auto", "spanNulls": false, "stacking": { "mode": "none" } }, "unit": "reqps" }, "overrides": [] }, "gridPos": { "h": 6, "w": 12, "x": 12, "y": 17 }, "id": 38, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "single" } }, "targets": [ { "expr": "sum(rate(argo_workflows_k8s_request_total[$__rate_interval]))", "legendFormat": "k8s req/s", "refId": "A" } ], "title": "Argo K8s API Requests", "type": "timeseries" } ], "title": "Argo CI/CD", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 11 }, "id": 40, "panels": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Rate of authentication requests (total, success, failure)", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 10, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "showPoints": "auto", "spanNulls": false, "stacking": { "mode": "none" } }, "unit": "reqps" }, "overrides": [ { "matcher": { "id": "byName", "options": "failure" }, "properties": [ { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "success" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 12 }, "id": 41, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "expr": "rate(api_auth_requests_total[$__rate_interval])", "legendFormat": "total", "refId": "A" }, { "expr": "rate(api_auth_requests_success[$__rate_interval])", "legendFormat": "success", "refId": "B" }, { "expr": "rate(api_auth_requests_failure[$__rate_interval])", "legendFormat": "failure", "refId": "C" } ], "title": "Auth Request Rate", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Current number of entries in the API gateway rate limiter", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 100 }, { "color": "red", "value": 500 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 8, "y": 12 }, "id": 42, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "targets": [ { "expr": "api_rate_limiter_entries", "legendFormat": "entries", "refId": "A" } ], "title": "Rate Limiter Entries", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Ratio of JWT cache hits to total lookups", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "yellow", "value": 0.5 }, { "color": "green", "value": 0.8 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 12, "y": 12 }, "id": 43, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "targets": [ { "expr": "api_jwt_cache_hits / (api_jwt_cache_hits + api_jwt_cache_misses)", "legendFormat": "hit ratio", "refId": "A" } ], "title": "JWT Cache Hit Ratio", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "API gateway routing duration percentiles", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 5, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "showPoints": "auto", "spanNulls": false, "stacking": { "mode": "none" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 5000 }, { "color": "red", "value": 50000 } ] }, "unit": "us" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 12 }, "id": 44, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "expr": "histogram_quantile(0.50, sum by (le) (rate(api_routing_duration_microseconds_bucket[$__rate_interval])))", "legendFormat": "p50", "refId": "A" }, { "expr": "histogram_quantile(0.95, sum by (le) (rate(api_routing_duration_microseconds_bucket[$__rate_interval])))", "legendFormat": "p95", "refId": "B" }, { "expr": "histogram_quantile(0.99, sum by (le) (rate(api_routing_duration_microseconds_bucket[$__rate_interval])))", "legendFormat": "p99", "refId": "C" } ], "title": "Routing Latency p50 / p95 / p99", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Percentage of auth requests that fail", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 0.05 }, { "color": "red", "value": 0.2 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 8, "y": 16 }, "id": 45, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "targets": [ { "expr": "rate(api_auth_requests_failure[$__rate_interval]) / rate(api_auth_requests_total[$__rate_interval])", "legendFormat": "failure rate", "refId": "A" } ], "title": "Auth Failure Rate", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Rate of JWT cache misses", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 5 }, { "color": "red", "value": 20 } ] }, "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 12, "y": 16 }, "id": 46, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "targets": [ { "expr": "rate(api_jwt_cache_misses[$__rate_interval])", "legendFormat": "misses/s", "refId": "A" } ], "title": "JWT Cache Misses/s", "type": "stat" } ], "title": "API Gateway", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 12 }, "id": 50, "panels": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Current portfolio value in USD", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 2, "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "yellow", "value": 10000 }, { "color": "green", "value": 100000 } ] }, "unit": "currencyUSD" }, "overrides": [] }, "gridPos": { "h": 6, "w": 6, "x": 0, "y": 13 }, "id": 51, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "targets": [ { "expr": "trading_agent_portfolio_value_usd", "legendFormat": "portfolio", "refId": "A" } ], "title": "Portfolio Value", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Number of assets currently selected for trading", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "blue", "value": null }, { "color": "green", "value": 1 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 6, "w": 3, "x": 6, "y": 13 }, "id": 52, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "targets": [ { "expr": "trading_agent_assets_selected", "legendFormat": "selected", "refId": "A" } ], "title": "Assets Selected", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Total instruments in the trading universe", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "blue", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 6, "w": 3, "x": 9, "y": 13 }, "id": 53, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "targets": [ { "expr": "trading_agent_universe_instruments", "legendFormat": "instruments", "refId": "A" } ], "title": "Universe Instruments", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Rate of trading measurements emitted", "fieldConfig": { "defaults": { "color": { "fixedColor": "blue", "mode": "fixed" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 15, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "showPoints": "auto", "spanNulls": false, "stacking": { "mode": "none" } }, "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 6, "w": 6, "x": 12, "y": 13 }, "id": 54, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "single" } }, "targets": [ { "expr": "rate(trading_measurements_total[$__rate_interval])", "legendFormat": "measurements/s", "refId": "A" } ], "title": "Trading Measurements Rate", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "End-to-end trading latency", "fieldConfig": { "defaults": { "color": { "fixedColor": "orange", "mode": "fixed" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 10, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "showPoints": "auto", "spanNulls": false, "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "area" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 0.01 }, { "color": "red", "value": 0.1 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 6, "w": 6, "x": 18, "y": 13 }, "id": 55, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "single" } }, "targets": [ { "expr": "trading_total_latency_seconds", "legendFormat": "total latency", "refId": "A" } ], "title": "Total Trading Latency", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Breakdown of trading pipeline latency components", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 10, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "showPoints": "auto", "spanNulls": false, "stacking": { "mode": "none" } }, "unit": "s" }, "overrides": [ { "matcher": { "id": "byName", "options": "order processing" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "risk check" }, "properties": [ { "id": "color", "value": { "fixedColor": "orange", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "market data" }, "properties": [ { "id": "color", "value": { "fixedColor": "purple", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 19 }, "id": 56, "options": { "legend": { "calcs": [ "mean", "max", "lastNotNull" ], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "expr": "trading_order_processing_seconds", "legendFormat": "order processing", "refId": "A" }, { "expr": "trading_risk_check_seconds", "legendFormat": "risk check", "refId": "B" }, { "expr": "trading_market_data_seconds", "legendFormat": "market data", "refId": "C" } ], "title": "Order Processing vs Risk Check vs Market Data", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Historical portfolio value trend", "fieldConfig": { "defaults": { "color": { "mode": "continuous-GrYlRd" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 20, "gradientMode": "scheme", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "showPoints": "never", "spanNulls": true, "stacking": { "mode": "none" } }, "decimals": 2, "unit": "currencyUSD" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 19 }, "id": 57, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "single" } }, "targets": [ { "expr": "trading_agent_portfolio_value_usd", "legendFormat": "portfolio USD", "refId": "A" } ], "title": "Portfolio Value Over Time", "type": "timeseries" } ], "title": "Trading", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 13 }, "id": 60, "panels": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Percentage of the metrics buffer currently in use", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "max": 100, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 60 }, { "color": "orange", "value": 80 }, { "color": "red", "value": 95 } ] }, "unit": "percent" }, "overrides": [] }, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 14 }, "id": 61, "options": { "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true }, "targets": [ { "expr": "metrics_buffer_utilization_percent", "legendFormat": "buffer %", "refId": "A" } ], "title": "Metrics Buffer Utilization", "type": "gauge" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Rate at which metrics are being dropped", "fieldConfig": { "defaults": { "color": { "fixedColor": "red", "mode": "fixed" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 20, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "showPoints": "auto", "spanNulls": false, "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "area" } }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 1 }, { "color": "red", "value": 10 } ] }, "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 14 }, "id": 62, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "single" } }, "targets": [ { "expr": "rate(metrics_dropped_total[$__rate_interval])", "legendFormat": "dropped/s", "refId": "A" } ], "title": "Metrics Dropped Rate", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Total count of metrics dropped since service start", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 100 }, { "color": "red", "value": 1000 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 14 }, "id": 63, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "targets": [ { "expr": "metrics_dropped_total", "legendFormat": "total dropped", "refId": "A" } ], "title": "Total Metrics Dropped", "type": "stat" } ], "title": "Metrics Health", "type": "row" } ], "refresh": "30s", "schemaVersion": 39, "tags": [ "foxhunt", "cockpit" ], "templating": { "list": [ { "auto": true, "auto_count": 30, "auto_min": "1m", "current": { "selected": true, "text": "auto", "value": "$__auto_interval_interval" }, "label": "Interval", "name": "interval", "query": "1m,5m,15m,30m,1h", "type": "interval" } ] }, "time": { "from": "now-6h", "to": "now" }, "timezone": "browser", "title": "Operations Cockpit", "uid": "foxhunt-cockpit", "weekStart": "", "__inputs": [ { "name": "DS_PROMETHEUS", "type": "datasource", "pluginId": "prometheus" }, { "name": "DS_LOKI", "type": "datasource", "pluginId": "loki" }, { "name": "DS_TEMPO", "type": "datasource", "pluginId": "${DS_TEMPO}" } ] }