fix(infra): replace gitlab.com/prometheus annotations with prometheus.io

Prometheus was configured to scrape pods via prometheus.io/scrape
annotations, but all services and training jobs used gitlab.com/
prefix from legacy GitLab-managed Prometheus — causing Prometheus
to never discover any foxhunt pods. This resulted in stale/missing
metrics on the Grafana training dashboard.

12 files updated across services/, gpu-overlays/, training/, and
monitoring/ (node-exporter, dcgm-exporter cleanup).

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-03-08 18:39:24 +01:00
parent 9c249e2a27
commit c384ab4b43
12 changed files with 31 additions and 38 deletions

View File

@@ -24,9 +24,9 @@ spec:
template:
metadata:
annotations:
gitlab.com/prometheus_scrape: "true"
gitlab.com/prometheus_port: "9094"
gitlab.com/prometheus_path: "/metrics"
prometheus.io/scrape: "true"
prometheus.io/port: "9094"
prometheus.io/path: "/metrics"
labels:
app.kubernetes.io/name: ml-training-service
app.kubernetes.io/part-of: foxhunt

View File

@@ -24,9 +24,9 @@ spec:
template:
metadata:
annotations:
gitlab.com/prometheus_scrape: "true"
gitlab.com/prometheus_port: "9092"
gitlab.com/prometheus_path: "/metrics"
prometheus.io/scrape: "true"
prometheus.io/port: "9092"
prometheus.io/path: "/metrics"
labels:
app.kubernetes.io/name: trading-service
app.kubernetes.io/part-of: foxhunt

View File

@@ -51,9 +51,6 @@ spec:
prometheus.io/scrape: "true"
prometheus.io/port: "9400"
prometheus.io/path: "/metrics"
gitlab.com/prometheus_scrape: "true"
gitlab.com/prometheus_port: "9400"
gitlab.com/prometheus_path: "/metrics"
spec:
# Only schedule on nodes that have GPUs
affinity:
@@ -118,8 +115,6 @@ metadata:
annotations:
prometheus.io/scrape: "true"
prometheus.io/port: "9400"
gitlab.com/prometheus_scrape: "true"
gitlab.com/prometheus_port: "9400"
spec:
type: ClusterIP
selector:

View File

@@ -30,9 +30,7 @@ spec:
annotations:
prometheus.io/scrape: "true"
prometheus.io/port: "9100"
gitlab.com/prometheus_scrape: "true"
gitlab.com/prometheus_port: "9100"
gitlab.com/prometheus_path: "/metrics"
prometheus.io/path: "/metrics"
spec:
serviceAccountName: node-exporter
hostPID: true

View File

@@ -19,9 +19,9 @@ spec:
template:
metadata:
annotations:
gitlab.com/prometheus_scrape: "true"
gitlab.com/prometheus_port: "9091"
gitlab.com/prometheus_path: "/metrics"
prometheus.io/scrape: "true"
prometheus.io/port: "9091"
prometheus.io/path: "/metrics"
labels:
app.kubernetes.io/name: api
app.kubernetes.io/part-of: foxhunt

View File

@@ -19,9 +19,9 @@ spec:
template:
metadata:
annotations:
gitlab.com/prometheus_scrape: "true"
gitlab.com/prometheus_port: "9093"
gitlab.com/prometheus_path: "/metrics"
prometheus.io/scrape: "true"
prometheus.io/port: "9093"
prometheus.io/path: "/metrics"
labels:
app.kubernetes.io/name: backtesting-service
app.kubernetes.io/part-of: foxhunt

View File

@@ -19,9 +19,9 @@ spec:
template:
metadata:
annotations:
gitlab.com/prometheus_scrape: "true"
gitlab.com/prometheus_port: "9096"
gitlab.com/prometheus_path: "/metrics"
prometheus.io/scrape: "true"
prometheus.io/port: "9096"
prometheus.io/path: "/metrics"
labels:
app.kubernetes.io/name: broker-gateway
app.kubernetes.io/part-of: foxhunt

View File

@@ -19,9 +19,9 @@ spec:
template:
metadata:
annotations:
gitlab.com/prometheus_scrape: "true"
gitlab.com/prometheus_port: "9097"
gitlab.com/prometheus_path: "/metrics"
prometheus.io/scrape: "true"
prometheus.io/port: "9097"
prometheus.io/path: "/metrics"
labels:
app.kubernetes.io/name: data-acquisition-service
app.kubernetes.io/part-of: foxhunt

View File

@@ -61,9 +61,9 @@ spec:
template:
metadata:
annotations:
gitlab.com/prometheus_scrape: "true"
gitlab.com/prometheus_port: "9094"
gitlab.com/prometheus_path: "/metrics"
prometheus.io/scrape: "true"
prometheus.io/port: "9094"
prometheus.io/path: "/metrics"
labels:
app.kubernetes.io/name: ml-training-service
app.kubernetes.io/part-of: foxhunt

View File

@@ -19,9 +19,9 @@ spec:
template:
metadata:
annotations:
gitlab.com/prometheus_scrape: "true"
gitlab.com/prometheus_port: "9095"
gitlab.com/prometheus_path: "/metrics"
prometheus.io/scrape: "true"
prometheus.io/port: "9095"
prometheus.io/path: "/metrics"
labels:
app.kubernetes.io/name: trading-agent-service
app.kubernetes.io/part-of: foxhunt

View File

@@ -19,9 +19,9 @@ spec:
template:
metadata:
annotations:
gitlab.com/prometheus_scrape: "true"
gitlab.com/prometheus_port: "9092"
gitlab.com/prometheus_path: "/metrics"
prometheus.io/scrape: "true"
prometheus.io/port: "9092"
prometheus.io/path: "/metrics"
labels:
app.kubernetes.io/name: trading-service
app.kubernetes.io/part-of: foxhunt

View File

@@ -14,9 +14,9 @@ spec:
template:
metadata:
annotations:
gitlab.com/prometheus_scrape: "true"
gitlab.com/prometheus_port: "9094"
gitlab.com/prometheus_path: "/metrics"
prometheus.io/scrape: "true"
prometheus.io/port: "9094"
prometheus.io/path: "/metrics"
labels:
app.kubernetes.io/name: training
app.kubernetes.io/component: training-workflow