fix(infra): resolve dashboard datasource UIDs, fix Grafana→Prometheus netpol
- Remove duplicate annotated-pods and gitlab-annotated-pods scrape jobs (all targets already covered by dedicated jobs) - Drop Argo controller from foxhunt-services job (dedicated HTTPS job exists) - Resolve datasource variable placeholders to actual UIDs in dashboard JSON - Add Ingress rule to Prometheus netpol allowing Grafana on port 9090 Targets: 49 → 18 (zero duplicates) Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,13 +1,4 @@
|
||||
{
|
||||
"__inputs": [
|
||||
{
|
||||
"name": "DS_PROMETHEUS",
|
||||
"label": "Prometheus",
|
||||
"description": "",
|
||||
"type": "datasource",
|
||||
"pluginId": "prometheus"
|
||||
}
|
||||
],
|
||||
"id": null,
|
||||
"uid": "foxhunt-trading",
|
||||
"title": "Trading & ML Intelligence",
|
||||
@@ -48,7 +39,7 @@
|
||||
"query": "label_values(foxhunt_predictions_confidence_bucket, model)",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"refresh": 2,
|
||||
"includeAll": true,
|
||||
@@ -64,7 +55,7 @@
|
||||
"query": "label_values(foxhunt_predictions_confidence_bucket, symbol)",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"refresh": 2,
|
||||
"includeAll": true,
|
||||
@@ -102,7 +93,7 @@
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
@@ -150,7 +141,7 @@
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"expr": "foxhunt_ensemble_agreement_rate{symbol=~\"$symbol\"}",
|
||||
"instant": true,
|
||||
@@ -170,7 +161,7 @@
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
@@ -218,7 +209,7 @@
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"expr": "histogram_quantile(0.5, rate(foxhunt_predictions_confidence_bucket{model=~\"$model\"}[5m]))",
|
||||
"instant": true,
|
||||
@@ -238,7 +229,7 @@
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
@@ -286,7 +277,7 @@
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"expr": "sum(rate(foxhunt_predictions_total{model=~\"$model\"}[5m]))",
|
||||
"instant": true,
|
||||
@@ -306,7 +297,7 @@
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
@@ -354,7 +345,7 @@
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"expr": "count(foxhunt_model_loaded{model=~\"$model\"} == 1)",
|
||||
"instant": true,
|
||||
@@ -387,7 +378,7 @@
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
@@ -432,7 +423,7 @@
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"expr": "histogram_quantile(0.1, rate(foxhunt_predictions_confidence_bucket{model=~\"$model\"}[$interval]))",
|
||||
"legendFormat": "p10"
|
||||
@@ -441,7 +432,7 @@
|
||||
"refId": "B",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"expr": "histogram_quantile(0.5, rate(foxhunt_predictions_confidence_bucket{model=~\"$model\"}[$interval]))",
|
||||
"legendFormat": "p50"
|
||||
@@ -450,7 +441,7 @@
|
||||
"refId": "C",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"expr": "histogram_quantile(0.9, rate(foxhunt_predictions_confidence_bucket{model=~\"$model\"}[$interval]))",
|
||||
"legendFormat": "p90"
|
||||
@@ -469,7 +460,7 @@
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
@@ -515,7 +506,7 @@
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"expr": "sum by (model) (rate(foxhunt_predictions_total{model=~\"$model\"}[$interval]))",
|
||||
"legendFormat": "{{model}}"
|
||||
@@ -547,7 +538,7 @@
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
@@ -599,7 +590,7 @@
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"expr": "sum(rate(trading_orders_total[5m]))",
|
||||
"instant": true,
|
||||
@@ -619,7 +610,7 @@
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
@@ -667,7 +658,7 @@
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"expr": "histogram_quantile(0.99, rate(trading_order_processing_seconds_bucket[5m]))",
|
||||
"instant": true,
|
||||
@@ -687,7 +678,7 @@
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
@@ -735,7 +726,7 @@
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"expr": "histogram_quantile(0.99, rate(trading_risk_check_seconds_bucket[5m]))",
|
||||
"instant": true,
|
||||
@@ -755,7 +746,7 @@
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
@@ -803,7 +794,7 @@
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"expr": "histogram_quantile(0.99, rate(trading_total_latency_seconds_bucket[5m]))",
|
||||
"instant": true,
|
||||
@@ -836,7 +827,7 @@
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
@@ -882,7 +873,7 @@
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"expr": "histogram_quantile(0.5, rate(trading_order_processing_seconds_bucket[$interval]))",
|
||||
"legendFormat": "p50"
|
||||
@@ -891,7 +882,7 @@
|
||||
"refId": "B",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"expr": "histogram_quantile(0.95, rate(trading_order_processing_seconds_bucket[$interval]))",
|
||||
"legendFormat": "p95"
|
||||
@@ -900,7 +891,7 @@
|
||||
"refId": "C",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"expr": "histogram_quantile(0.99, rate(trading_order_processing_seconds_bucket[$interval]))",
|
||||
"legendFormat": "p99"
|
||||
@@ -919,7 +910,7 @@
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
@@ -965,7 +956,7 @@
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"expr": "histogram_quantile(0.5, rate(trading_risk_check_seconds_bucket[$interval]))",
|
||||
"legendFormat": "p50"
|
||||
@@ -974,7 +965,7 @@
|
||||
"refId": "B",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"expr": "histogram_quantile(0.95, rate(trading_risk_check_seconds_bucket[$interval]))",
|
||||
"legendFormat": "p95"
|
||||
@@ -983,7 +974,7 @@
|
||||
"refId": "C",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"expr": "histogram_quantile(0.99, rate(trading_risk_check_seconds_bucket[$interval]))",
|
||||
"legendFormat": "p99"
|
||||
@@ -1015,7 +1006,7 @@
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
@@ -1061,7 +1052,7 @@
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"expr": "histogram_quantile(0.99, rate(api_gateway_request_duration_seconds_bucket{handler=~\".*trade.*|.*order.*\"}[$interval]))",
|
||||
"legendFormat": "{{handler}} p99"
|
||||
@@ -1080,7 +1071,7 @@
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
@@ -1156,7 +1147,7 @@
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"expr": "rate(api_gateway_rate_limit_rejected_total{tier=\"trading\"}[$interval])",
|
||||
"legendFormat": "rejected"
|
||||
@@ -1165,7 +1156,7 @@
|
||||
"refId": "B",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"expr": "rate(api_gateway_rate_limit_allowed_total{tier=\"trading\"}[$interval])",
|
||||
"legendFormat": "allowed"
|
||||
@@ -1197,7 +1188,7 @@
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
@@ -1242,7 +1233,7 @@
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"expr": "sum by (grpc_method) (rate(grpc_server_handled_total{grpc_service=~\".*trading.*|.*ml.*\"}[$interval]))",
|
||||
"legendFormat": "{{grpc_method}}"
|
||||
@@ -1261,7 +1252,7 @@
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
@@ -1306,7 +1297,7 @@
|
||||
"refId": "A",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
"uid": "PBFA97CFB590B2093"
|
||||
},
|
||||
"expr": "sum by (grpc_service, grpc_code) (rate(grpc_server_handled_total{grpc_code!=\"OK\"}[$interval]))",
|
||||
"legendFormat": "{{grpc_service}} {{grpc_code}}"
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -63,6 +63,9 @@ data:
|
||||
- source_labels: [__meta_kubernetes_service_label_app_kubernetes_io_part_of]
|
||||
action: keep
|
||||
regex: foxhunt
|
||||
- source_labels: [__meta_kubernetes_service_name]
|
||||
action: drop
|
||||
regex: argo-workflows-workflow-controller-metrics
|
||||
- source_labels: [__meta_kubernetes_endpoint_port_name]
|
||||
action: keep
|
||||
regex: metrics
|
||||
@@ -71,58 +74,6 @@ data:
|
||||
- source_labels: [__meta_kubernetes_namespace]
|
||||
target_label: namespace
|
||||
|
||||
# Pod annotation-based scraping (prometheus.io/scrape: "true")
|
||||
- job_name: annotated-pods
|
||||
kubernetes_sd_configs:
|
||||
- role: pod
|
||||
namespaces:
|
||||
names: [foxhunt]
|
||||
relabel_configs:
|
||||
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
|
||||
action: keep
|
||||
regex: "true"
|
||||
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
|
||||
action: replace
|
||||
regex: (.+)
|
||||
target_label: __metrics_path__
|
||||
- source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
|
||||
action: replace
|
||||
regex: ([^:]+)(?::\d+)?;(\d+)
|
||||
replacement: $1:$2
|
||||
target_label: __address__
|
||||
- action: labelmap
|
||||
regex: __meta_kubernetes_pod_label_(.+)
|
||||
- source_labels: [__meta_kubernetes_namespace]
|
||||
target_label: namespace
|
||||
- source_labels: [__meta_kubernetes_pod_name]
|
||||
target_label: pod
|
||||
|
||||
# GitLab annotation-based scraping (gitlab.com/prometheus_scrape: "true")
|
||||
- job_name: gitlab-annotated-pods
|
||||
kubernetes_sd_configs:
|
||||
- role: pod
|
||||
namespaces:
|
||||
names: [foxhunt]
|
||||
relabel_configs:
|
||||
- source_labels: [__meta_kubernetes_pod_annotation_gitlab_com_prometheus_scrape]
|
||||
action: keep
|
||||
regex: "true"
|
||||
- source_labels: [__meta_kubernetes_pod_annotation_gitlab_com_prometheus_path]
|
||||
action: replace
|
||||
regex: (.+)
|
||||
target_label: __metrics_path__
|
||||
- source_labels: [__address__, __meta_kubernetes_pod_annotation_gitlab_com_prometheus_port]
|
||||
action: replace
|
||||
regex: ([^:]+)(?::\d+)?;(\d+)
|
||||
replacement: $1:$2
|
||||
target_label: __address__
|
||||
- action: labelmap
|
||||
regex: __meta_kubernetes_pod_label_(.+)
|
||||
- source_labels: [__meta_kubernetes_namespace]
|
||||
target_label: namespace
|
||||
- source_labels: [__meta_kubernetes_pod_name]
|
||||
target_label: kubernetes_pod_name
|
||||
|
||||
# Node-exporter (host metrics)
|
||||
- job_name: node-exporter
|
||||
kubernetes_sd_configs:
|
||||
|
||||
@@ -13,7 +13,17 @@ spec:
|
||||
app.kubernetes.io/name: prometheus
|
||||
app.kubernetes.io/part-of: foxhunt
|
||||
policyTypes:
|
||||
- Ingress
|
||||
- Egress
|
||||
ingress:
|
||||
# Grafana queries Prometheus on port 9090
|
||||
- from:
|
||||
- podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: grafana
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: 9090
|
||||
egress:
|
||||
# Kubernetes API server (required for kubernetes_sd_configs service discovery)
|
||||
# ClusterIP 10.32.0.1:443 NATs to node IP 172.16.0.0/16:6443
|
||||
|
||||
Reference in New Issue
Block a user