fix(infra): replace gitlab.com/prometheus annotations with prometheus.io
Prometheus was configured to scrape pods via prometheus.io/scrape annotations, but all services and training jobs used gitlab.com/ prefix from legacy GitLab-managed Prometheus — causing Prometheus to never discover any foxhunt pods. This resulted in stale/missing metrics on the Grafana training dashboard. 12 files updated across services/, gpu-overlays/, training/, and monitoring/ (node-exporter, dcgm-exporter cleanup). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -24,9 +24,9 @@ spec:
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
gitlab.com/prometheus_scrape: "true"
|
||||
gitlab.com/prometheus_port: "9094"
|
||||
gitlab.com/prometheus_path: "/metrics"
|
||||
prometheus.io/scrape: "true"
|
||||
prometheus.io/port: "9094"
|
||||
prometheus.io/path: "/metrics"
|
||||
labels:
|
||||
app.kubernetes.io/name: ml-training-service
|
||||
app.kubernetes.io/part-of: foxhunt
|
||||
|
||||
@@ -24,9 +24,9 @@ spec:
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
gitlab.com/prometheus_scrape: "true"
|
||||
gitlab.com/prometheus_port: "9092"
|
||||
gitlab.com/prometheus_path: "/metrics"
|
||||
prometheus.io/scrape: "true"
|
||||
prometheus.io/port: "9092"
|
||||
prometheus.io/path: "/metrics"
|
||||
labels:
|
||||
app.kubernetes.io/name: trading-service
|
||||
app.kubernetes.io/part-of: foxhunt
|
||||
|
||||
@@ -51,9 +51,6 @@ spec:
|
||||
prometheus.io/scrape: "true"
|
||||
prometheus.io/port: "9400"
|
||||
prometheus.io/path: "/metrics"
|
||||
gitlab.com/prometheus_scrape: "true"
|
||||
gitlab.com/prometheus_port: "9400"
|
||||
gitlab.com/prometheus_path: "/metrics"
|
||||
spec:
|
||||
# Only schedule on nodes that have GPUs
|
||||
affinity:
|
||||
@@ -118,8 +115,6 @@ metadata:
|
||||
annotations:
|
||||
prometheus.io/scrape: "true"
|
||||
prometheus.io/port: "9400"
|
||||
gitlab.com/prometheus_scrape: "true"
|
||||
gitlab.com/prometheus_port: "9400"
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
|
||||
@@ -30,9 +30,7 @@ spec:
|
||||
annotations:
|
||||
prometheus.io/scrape: "true"
|
||||
prometheus.io/port: "9100"
|
||||
gitlab.com/prometheus_scrape: "true"
|
||||
gitlab.com/prometheus_port: "9100"
|
||||
gitlab.com/prometheus_path: "/metrics"
|
||||
prometheus.io/path: "/metrics"
|
||||
spec:
|
||||
serviceAccountName: node-exporter
|
||||
hostPID: true
|
||||
|
||||
@@ -19,9 +19,9 @@ spec:
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
gitlab.com/prometheus_scrape: "true"
|
||||
gitlab.com/prometheus_port: "9091"
|
||||
gitlab.com/prometheus_path: "/metrics"
|
||||
prometheus.io/scrape: "true"
|
||||
prometheus.io/port: "9091"
|
||||
prometheus.io/path: "/metrics"
|
||||
labels:
|
||||
app.kubernetes.io/name: api
|
||||
app.kubernetes.io/part-of: foxhunt
|
||||
|
||||
@@ -19,9 +19,9 @@ spec:
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
gitlab.com/prometheus_scrape: "true"
|
||||
gitlab.com/prometheus_port: "9093"
|
||||
gitlab.com/prometheus_path: "/metrics"
|
||||
prometheus.io/scrape: "true"
|
||||
prometheus.io/port: "9093"
|
||||
prometheus.io/path: "/metrics"
|
||||
labels:
|
||||
app.kubernetes.io/name: backtesting-service
|
||||
app.kubernetes.io/part-of: foxhunt
|
||||
|
||||
@@ -19,9 +19,9 @@ spec:
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
gitlab.com/prometheus_scrape: "true"
|
||||
gitlab.com/prometheus_port: "9096"
|
||||
gitlab.com/prometheus_path: "/metrics"
|
||||
prometheus.io/scrape: "true"
|
||||
prometheus.io/port: "9096"
|
||||
prometheus.io/path: "/metrics"
|
||||
labels:
|
||||
app.kubernetes.io/name: broker-gateway
|
||||
app.kubernetes.io/part-of: foxhunt
|
||||
|
||||
@@ -19,9 +19,9 @@ spec:
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
gitlab.com/prometheus_scrape: "true"
|
||||
gitlab.com/prometheus_port: "9097"
|
||||
gitlab.com/prometheus_path: "/metrics"
|
||||
prometheus.io/scrape: "true"
|
||||
prometheus.io/port: "9097"
|
||||
prometheus.io/path: "/metrics"
|
||||
labels:
|
||||
app.kubernetes.io/name: data-acquisition-service
|
||||
app.kubernetes.io/part-of: foxhunt
|
||||
|
||||
@@ -61,9 +61,9 @@ spec:
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
gitlab.com/prometheus_scrape: "true"
|
||||
gitlab.com/prometheus_port: "9094"
|
||||
gitlab.com/prometheus_path: "/metrics"
|
||||
prometheus.io/scrape: "true"
|
||||
prometheus.io/port: "9094"
|
||||
prometheus.io/path: "/metrics"
|
||||
labels:
|
||||
app.kubernetes.io/name: ml-training-service
|
||||
app.kubernetes.io/part-of: foxhunt
|
||||
|
||||
@@ -19,9 +19,9 @@ spec:
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
gitlab.com/prometheus_scrape: "true"
|
||||
gitlab.com/prometheus_port: "9095"
|
||||
gitlab.com/prometheus_path: "/metrics"
|
||||
prometheus.io/scrape: "true"
|
||||
prometheus.io/port: "9095"
|
||||
prometheus.io/path: "/metrics"
|
||||
labels:
|
||||
app.kubernetes.io/name: trading-agent-service
|
||||
app.kubernetes.io/part-of: foxhunt
|
||||
|
||||
@@ -19,9 +19,9 @@ spec:
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
gitlab.com/prometheus_scrape: "true"
|
||||
gitlab.com/prometheus_port: "9092"
|
||||
gitlab.com/prometheus_path: "/metrics"
|
||||
prometheus.io/scrape: "true"
|
||||
prometheus.io/port: "9092"
|
||||
prometheus.io/path: "/metrics"
|
||||
labels:
|
||||
app.kubernetes.io/name: trading-service
|
||||
app.kubernetes.io/part-of: foxhunt
|
||||
|
||||
@@ -14,9 +14,9 @@ spec:
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
gitlab.com/prometheus_scrape: "true"
|
||||
gitlab.com/prometheus_port: "9094"
|
||||
gitlab.com/prometheus_path: "/metrics"
|
||||
prometheus.io/scrape: "true"
|
||||
prometheus.io/port: "9094"
|
||||
prometheus.io/path: "/metrics"
|
||||
labels:
|
||||
app.kubernetes.io/name: training
|
||||
app.kubernetes.io/component: training-workflow
|
||||
|
||||
Reference in New Issue
Block a user