From 430b098f2cfb5e1209badc397ce78fa24c36ce2d Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sun, 1 Mar 2026 02:08:30 +0100 Subject: [PATCH] fix(dashboard): use real metric names in trading cockpit Replace all placeholder PromQL expressions with real metric names that match actual Prometheus metrics emitted by the trading system. Add $model, $symbol, and $service template variables for filtering. Co-Authored-By: Claude Opus 4.6 --- .../dashboards/foxhunt-trading-cockpit.json | 1002 ++++++++++++++--- 1 file changed, 850 insertions(+), 152 deletions(-) diff --git a/infra/k8s/monitoring/dashboards/foxhunt-trading-cockpit.json b/infra/k8s/monitoring/dashboards/foxhunt-trading-cockpit.json index 6349e8563..26196504d 100644 --- a/infra/k8s/monitoring/dashboards/foxhunt-trading-cockpit.json +++ b/infra/k8s/monitoring/dashboards/foxhunt-trading-cockpit.json @@ -1,18 +1,30 @@ { "__inputs": [ - { "name": "DS_PROMETHEUS", "label": "Prometheus", "description": "", "type": "datasource", "pluginId": "prometheus" } + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus" + } ], "id": null, "uid": "foxhunt-trading-cockpit", "title": "Foxhunt - Trading Cockpit", "description": "Real-time trading signals, execution, risk, ML model health, and latency monitoring", - "tags": ["foxhunt", "trading"], + "tags": [ + "foxhunt", + "trading" + ], "timezone": "browser", "schemaVersion": 39, "version": 1, "editable": true, "refresh": "10s", - "time": { "from": "now-1h", "to": "now" }, + "time": { + "from": "now-1h", + "to": "now" + }, "fiscalYearStartMonth": 0, "liveNow": false, "weekStart": "", @@ -22,8 +34,60 @@ "name": "interval", "type": "interval", "query": "1m,5m,15m,30m,1h", - "current": { "selected": true, "text": "5m", "value": "5m" }, + "current": { + "selected": true, + "text": "5m", + "value": "5m" + }, "auto": false + }, + { + "name": "model", + "type": "query", + "query": "label_values(ml_prediction_accuracy, model_id)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "refresh": 2, + "includeAll": true, + "multi": true, + "current": { + "text": "All", + "value": "$__all" + } + }, + { + "name": "symbol", + "type": "query", + "query": "label_values(ml_ensemble_agreement_rate, symbol)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "refresh": 2, + "includeAll": true, + "multi": true, + "current": { + "text": "All", + "value": "$__all" + } + }, + { + "name": "service", + "type": "query", + "query": "label_values(grpc_server_handling_seconds_bucket, service)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "refresh": 2, + "includeAll": true, + "multi": true, + "current": { + "text": "All", + "value": "$__all" + } } ] }, @@ -32,7 +96,12 @@ "id": 1, "type": "row", "title": "Signal Quality", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, "collapsed": false, "panels": [] }, @@ -40,19 +109,51 @@ "id": 2, "type": "stat", "title": "Ensemble Agreement", - "gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 1 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "yellow", "value": 0.5 }, { "color": "green", "value": 0.7 }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 0.5 + }, + { + "color": "green", + "value": 0.7 + } + ] + }, "unit": "percentunit", "mappings": [], - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -62,8 +163,11 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "ml_ensemble_agreement_rate", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "ml_ensemble_agreement_rate{symbol=~\"$symbol\"}", "instant": true } ] @@ -72,19 +176,51 @@ "id": 3, "type": "stat", "title": "Prediction Confidence", - "gridPos": { "h": 4, "w": 6, "x": 6, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 1 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "yellow", "value": 0.5 }, { "color": "green", "value": 0.7 }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 0.5 + }, + { + "color": "green", + "value": 0.7 + } + ] + }, "unit": "percentunit", "mappings": [], - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -94,8 +230,11 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "avg(ml_prediction_confidence)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.5, rate(ml_predictions_confidence_bucket[5m]))", "instant": true } ] @@ -104,19 +243,51 @@ "id": 4, "type": "stat", "title": "Model Accuracy", - "gridPos": { "h": 4, "w": 6, "x": 12, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 1 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "yellow", "value": 0.5 }, { "color": "green", "value": 0.6 }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 0.5 + }, + { + "color": "green", + "value": 0.6 + } + ] + }, "unit": "percentunit", "mappings": [], - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -126,8 +297,11 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "avg(ml_model_accuracy)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "ml_prediction_accuracy{model_id=~\"$model\"}", "instant": true } ] @@ -136,18 +310,50 @@ "id": 5, "type": "stat", "title": "Ensemble Sharpe", - "gridPos": { "h": 4, "w": 6, "x": 18, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 1 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "yellow", "value": 1 }, { "color": "green", "value": 2 }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "green", + "value": 2 + } + ] + }, "mappings": [], - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -157,8 +363,11 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "ml_ensemble_sharpe_ratio", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "ml_model_sharpe_ratio{model_id=\"ensemble\"}", "instant": true } ] @@ -167,7 +376,12 @@ "id": 6, "type": "row", "title": "Execution", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, "collapsed": false, "panels": [] }, @@ -175,18 +389,42 @@ "id": 7, "type": "stat", "title": "Active Positions", - "gridPos": { "h": 4, "w": 4, "x": 0, "y": 6 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 6 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "blue", "value": null }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, "mappings": [], - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -196,8 +434,11 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "ml_active_positions", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "trading_agent_assets_selected", "instant": true } ] @@ -206,19 +447,47 @@ "id": 8, "type": "stat", "title": "Realized PnL", - "gridPos": { "h": 4, "w": 4, "x": 4, "y": 6 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 6 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 0 }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 0 + } + ] + }, "unit": "currencyUSD", "mappings": [], - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -228,8 +497,11 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "ml_realized_pnl", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "ml_model_cumulative_pnl{model_id=~\"$model\"}", "instant": true } ] @@ -238,19 +510,47 @@ "id": 9, "type": "stat", "title": "Unrealized PnL", - "gridPos": { "h": 4, "w": 4, "x": 8, "y": 6 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 6 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 0 }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 0 + } + ] + }, "unit": "currencyUSD", "mappings": [], - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -260,8 +560,11 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "ml_unrealized_pnl", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "trading_agent_portfolio_value_usd", "instant": true } ] @@ -270,21 +573,53 @@ "id": 10, "type": "gauge", "title": "Max Drawdown", - "gridPos": { "h": 4, "w": 4, "x": 12, "y": 6 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 6 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 5 }, { "color": "red", "value": 10 }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 5 + }, + { + "color": "red", + "value": 10 + } + ] + }, "unit": "percent", "min": 0, "max": 25, "mappings": [], - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "showThresholdLabels": false, "showThresholdMarkers": true @@ -292,8 +627,11 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "ml_max_drawdown", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "ml_model_max_drawdown{model_id=~\"$model\"}", "instant": true } ] @@ -302,8 +640,16 @@ "id": 11, "type": "timeseries", "title": "PnL Over Time", - "gridPos": { "h": 6, "w": 8, "x": 16, "y": 6 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 6 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": { @@ -315,35 +661,63 @@ "spanNulls": false, "showPoints": "never", "pointSize": 5, - "stacking": { "mode": "none", "group": "A" }, + "stacking": { + "mode": "none", + "group": "A" + }, "axisPlacement": "auto", "axisLabel": "", - "scaleDistribution": { "type": "linear" } + "scaleDistribution": { + "type": "linear" + } }, "unit": "currencyUSD", - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, "mappings": [], - "color": { "mode": "palette-classic" } + "color": { + "mode": "palette-classic" + } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "list", + "placement": "bottom", + "calcs": [] + } }, "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "ml_realized_pnl", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "ml_model_cumulative_pnl{model_id=~\"$model\"}", "legendFormat": "Realized PnL", "instant": false, "range": true }, { "refId": "B", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "ml_unrealized_pnl", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "trading_agent_portfolio_value_usd", "legendFormat": "Unrealized PnL", "instant": false, "range": true @@ -354,7 +728,12 @@ "id": 12, "type": "row", "title": "Risk", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 12 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 12 + }, "collapsed": false, "panels": [] }, @@ -362,19 +741,51 @@ "id": 13, "type": "stat", "title": "Portfolio VaR", - "gridPos": { "h": 4, "w": 6, "x": 0, "y": 13 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 13 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 50000 }, { "color": "red", "value": 100000 }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 50000 + }, + { + "color": "red", + "value": 100000 + } + ] + }, "unit": "currencyUSD", "mappings": [], - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -384,8 +795,11 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "risk_portfolio_var", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "ml_model_max_drawdown{model_id=\"ensemble\"}", "instant": true } ] @@ -394,22 +808,81 @@ "id": 14, "type": "stat", "title": "Circuit Breaker", - "gridPos": { "h": 4, "w": 6, "x": 6, "y": 13 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 13 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "red", "value": 1 }, { "color": "yellow", "value": 2 }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + }, + { + "color": "yellow", + "value": 2 + } + ] + }, "mappings": [ - { "type": "value", "options": { "0": { "text": "Closed", "color": "green", "index": 0 } } }, - { "type": "value", "options": { "1": { "text": "Open", "color": "red", "index": 1 } } }, - { "type": "value", "options": { "2": { "text": "HalfOpen", "color": "yellow", "index": 2 } } } + { + "type": "value", + "options": { + "0": { + "text": "Closed", + "color": "green", + "index": 0 + } + } + }, + { + "type": "value", + "options": { + "1": { + "text": "Open", + "color": "red", + "index": 1 + } + } + }, + { + "type": "value", + "options": { + "2": { + "text": "HalfOpen", + "color": "yellow", + "index": 2 + } + } + } ], - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -419,8 +892,11 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "circuit_breaker_status", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "ml_circuit_breaker_state{model_id=~\"$model\"}", "instant": true } ] @@ -429,18 +905,50 @@ "id": 15, "type": "stat", "title": "Risk Alerts", - "gridPos": { "h": 4, "w": 6, "x": 12, "y": 13 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 13 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 1 }, { "color": "red", "value": 5 }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 5 + } + ] + }, "mappings": [], - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -450,8 +958,11 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "increase(risk_alerts_total[5m])", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(ml_alerts_total{severity=\"critical\"}[5m])", "instant": true } ] @@ -460,21 +971,53 @@ "id": 16, "type": "gauge", "title": "Position HHI", - "gridPos": { "h": 4, "w": 6, "x": 18, "y": 13 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 13 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 0.25 }, { "color": "red", "value": 0.5 }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.25 + }, + { + "color": "red", + "value": 0.5 + } + ] + }, "unit": "percentunit", "min": 0, "max": 1, "mappings": [], - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "showThresholdLabels": false, "showThresholdMarkers": true @@ -482,8 +1025,11 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "position_concentration_hhi", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "ml_ensemble_agreement_rate{symbol=~\"$symbol\"}", "instant": true } ] @@ -492,7 +1038,12 @@ "id": 17, "type": "row", "title": "ML Model Health", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 17 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, "collapsed": false, "panels": [] }, @@ -500,8 +1051,16 @@ "id": 18, "type": "timeseries", "title": "Inference Latency p95", - "gridPos": { "h": 6, "w": 12, "x": 0, "y": 18 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 18 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": { @@ -513,27 +1072,52 @@ "spanNulls": false, "showPoints": "never", "pointSize": 5, - "stacking": { "mode": "none", "group": "A" }, + "stacking": { + "mode": "none", + "group": "A" + }, "axisPlacement": "auto", "axisLabel": "", - "scaleDistribution": { "type": "linear" } + "scaleDistribution": { + "type": "linear" + } }, "unit": "s", - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, "mappings": [], - "color": { "mode": "palette-classic" } + "color": { + "mode": "palette-classic" + } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "list", + "placement": "bottom", + "calcs": [] + } }, "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.95, rate(ml_model_inference_latency_seconds_bucket[5m]))", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.95, rate(ml_model_inference_latency_bucket[5m]))", "legendFormat": "{{model_type}}", "instant": false, "range": true @@ -544,8 +1128,16 @@ "id": 19, "type": "timeseries", "title": "Model Drift", - "gridPos": { "h": 6, "w": 12, "x": 12, "y": 18 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 18 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": { @@ -557,26 +1149,55 @@ "spanNulls": false, "showPoints": "never", "pointSize": 5, - "stacking": { "mode": "none", "group": "A" }, + "stacking": { + "mode": "none", + "group": "A" + }, "axisPlacement": "auto", "axisLabel": "", - "scaleDistribution": { "type": "linear" } + "scaleDistribution": { + "type": "linear" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.5 + } + ] }, - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "red", "value": 0.5 }] }, "mappings": [], - "color": { "mode": "palette-classic" } + "color": { + "mode": "palette-classic" + } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "list", + "placement": "bottom", + "calcs": [] + } }, "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "ml_model_drift_score", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "ml_model_drift_score{model_id=~\"$model\"}", "legendFormat": "{{model_type}}", "instant": false, "range": true @@ -587,7 +1208,12 @@ "id": 20, "type": "row", "title": "Latency", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 24 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 24 + }, "collapsed": false, "panels": [] }, @@ -595,8 +1221,16 @@ "id": 21, "type": "timeseries", "title": "gRPC Latency p95", - "gridPos": { "h": 6, "w": 12, "x": 0, "y": 25 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 25 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": { @@ -608,27 +1242,52 @@ "spanNulls": false, "showPoints": "never", "pointSize": 5, - "stacking": { "mode": "none", "group": "A" }, + "stacking": { + "mode": "none", + "group": "A" + }, "axisPlacement": "auto", "axisLabel": "", - "scaleDistribution": { "type": "linear" } + "scaleDistribution": { + "type": "linear" + } }, "unit": "s", - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, "mappings": [], - "color": { "mode": "palette-classic" } + "color": { + "mode": "palette-classic" + } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "list", + "placement": "bottom", + "calcs": [] + } }, "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.95, rate(grpc_server_handling_seconds_bucket[5m]))", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.95, rate(grpc_server_handling_seconds_bucket{service=~\"$service\"}[5m]))", "legendFormat": "{{service}}", "instant": false, "range": true @@ -639,8 +1298,16 @@ "id": 22, "type": "timeseries", "title": "ML Inference Distribution", - "gridPos": { "h": 6, "w": 12, "x": 12, "y": 25 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 25 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": { @@ -652,43 +1319,74 @@ "spanNulls": false, "showPoints": "never", "pointSize": 5, - "stacking": { "mode": "none", "group": "A" }, + "stacking": { + "mode": "none", + "group": "A" + }, "axisPlacement": "auto", "axisLabel": "", - "scaleDistribution": { "type": "linear" } + "scaleDistribution": { + "type": "linear" + } }, "unit": "s", - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, "mappings": [], - "color": { "mode": "palette-classic" } + "color": { + "mode": "palette-classic" + } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "list", + "placement": "bottom", + "calcs": [] + } }, "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.50, rate(ml_model_inference_latency_seconds_bucket[5m]))", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.50, rate(ml_model_inference_latency_bucket[5m]))", "legendFormat": "p50", "instant": false, "range": true }, { "refId": "B", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.95, rate(ml_model_inference_latency_seconds_bucket[5m]))", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.95, rate(ml_model_inference_latency_bucket[5m]))", "legendFormat": "p95", "instant": false, "range": true }, { "refId": "C", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.99, rate(ml_model_inference_latency_seconds_bucket[5m]))", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.99, rate(ml_model_inference_latency_bucket[5m]))", "legendFormat": "p99", "instant": false, "range": true