gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(infra): add Prometheus scrape annotations to all service deployments

Browse Source 
Enable GitLab Prometheus to auto-discover and scrape metrics from all
foxhunt pods by adding gitlab.com/prometheus_scrape annotations to the
pod template metadata of 9 service deployments and the training job
template.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-03-01 00:42:17 +01:00
parent 2ef89ceb02
commit 033eed1526
10 changed files with 101 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
infra/k8s/services/api-gateway.yaml
View File

@@ -18,6 +18,10 @@ spec:
app.kubernetes.io/name: api-gateway
template:
metadata:
annotations:
gitlab.com/prometheus_scrape: "true"
gitlab.com/prometheus_port: "9091"
gitlab.com/prometheus_path: "/metrics"
labels:
app.kubernetes.io/name: api-gateway
spec:
@@ -49,20 +53,20 @@ spec:
fi
env:
- name: RCLONE_S3_PROVIDER
value: Scaleway
value: Minio
- name: RCLONE_S3_ENDPOINT
value: s3.fr-par.scw.cloud
value: minio.foxhunt.svc.cluster.local:9000
- name: RCLONE_S3_REGION
value: fr-par
value: us-east-1
- name: RCLONE_S3_ACCESS_KEY_ID
valueFrom:
secretKeyRef:
name: s3-credentials
name: minio-credentials
key: access-key
- name: RCLONE_S3_SECRET_ACCESS_KEY
valueFrom:
secretKeyRef:
name: s3-credentials
name: minio-credentials
key: secret-key
volumeMounts:
- name: binaries

14
infra/k8s/services/backtesting-service.yaml
View File

@@ -18,6 +18,10 @@ spec:
app.kubernetes.io/name: backtesting-service
template:
metadata:
annotations:
gitlab.com/prometheus_scrape: "true"
gitlab.com/prometheus_port: "9093"
gitlab.com/prometheus_path: "/metrics"
labels:
app.kubernetes.io/name: backtesting-service
spec:
@@ -49,20 +53,20 @@ spec:
fi
env:
- name: RCLONE_S3_PROVIDER
value: Scaleway
value: Minio
- name: RCLONE_S3_ENDPOINT
value: s3.fr-par.scw.cloud
value: minio.foxhunt.svc.cluster.local:9000
- name: RCLONE_S3_REGION
value: fr-par
value: us-east-1
- name: RCLONE_S3_ACCESS_KEY_ID
valueFrom:
secretKeyRef:
name: s3-credentials
name: minio-credentials
key: access-key
- name: RCLONE_S3_SECRET_ACCESS_KEY
valueFrom:
secretKeyRef:
name: s3-credentials
name: minio-credentials
key: secret-key
volumeMounts:
- name: binaries

14
infra/k8s/services/broker-gateway.yaml
View File

@@ -18,6 +18,10 @@ spec:
app.kubernetes.io/name: broker-gateway
template:
metadata:
annotations:
gitlab.com/prometheus_scrape: "true"
gitlab.com/prometheus_port: "9096"
gitlab.com/prometheus_path: "/metrics"
labels:
app.kubernetes.io/name: broker-gateway
spec:
@@ -49,20 +53,20 @@ spec:
fi
env:
- name: RCLONE_S3_PROVIDER
value: Scaleway
value: Minio
- name: RCLONE_S3_ENDPOINT
value: s3.fr-par.scw.cloud
value: minio.foxhunt.svc.cluster.local:9000
- name: RCLONE_S3_REGION
value: fr-par
value: us-east-1
- name: RCLONE_S3_ACCESS_KEY_ID
valueFrom:
secretKeyRef:
name: s3-credentials
name: minio-credentials
key: access-key
- name: RCLONE_S3_SECRET_ACCESS_KEY
valueFrom:
secretKeyRef:
name: s3-credentials
name: minio-credentials
key: secret-key
volumeMounts:
- name: binaries

14
infra/k8s/services/data-acquisition-service.yaml
View File

@@ -18,6 +18,10 @@ spec:
app.kubernetes.io/name: data-acquisition-service
template:
metadata:
annotations:
gitlab.com/prometheus_scrape: "true"
gitlab.com/prometheus_port: "9097"
gitlab.com/prometheus_path: "/metrics"
labels:
app.kubernetes.io/name: data-acquisition-service
spec:
@@ -49,20 +53,20 @@ spec:
fi
env:
- name: RCLONE_S3_PROVIDER
value: Scaleway
value: Minio
- name: RCLONE_S3_ENDPOINT
value: s3.fr-par.scw.cloud
value: minio.foxhunt.svc.cluster.local:9000
- name: RCLONE_S3_REGION
value: fr-par
value: us-east-1
- name: RCLONE_S3_ACCESS_KEY_ID
valueFrom:
secretKeyRef:
name: s3-credentials
name: minio-credentials
key: access-key
- name: RCLONE_S3_SECRET_ACCESS_KEY
valueFrom:
secretKeyRef:
name: s3-credentials
name: minio-credentials
key: secret-key
volumeMounts:
- name: binaries

18
infra/k8s/services/ml-training-service-gpu.yaml
View File

@@ -21,6 +21,10 @@ spec:
app.kubernetes.io/name: ml-training-service
template:
metadata:
annotations:
gitlab.com/prometheus_scrape: "true"
gitlab.com/prometheus_port: "9094"
gitlab.com/prometheus_path: "/metrics"
labels:
app.kubernetes.io/name: ml-training-service
spec:
@@ -57,20 +61,20 @@ spec:
fi
env:
- name: RCLONE_S3_PROVIDER
value: Scaleway
value: Minio
- name: RCLONE_S3_ENDPOINT
value: s3.fr-par.scw.cloud
value: minio.foxhunt.svc.cluster.local:9000
- name: RCLONE_S3_REGION
value: fr-par
value: us-east-1
- name: RCLONE_S3_ACCESS_KEY_ID
valueFrom:
secretKeyRef:
name: s3-credentials
name: minio-credentials
key: access-key
- name: RCLONE_S3_SECRET_ACCESS_KEY
valueFrom:
secretKeyRef:
name: s3-credentials
name: minio-credentials
key: secret-key
volumeMounts:
- name: binaries
@@ -113,9 +117,9 @@ spec:
- name: JWT_AUDIENCE
value: foxhunt-services
- name: S3_ENDPOINT
value: "https://s3.fr-par.scw.cloud"
value: "http://minio.foxhunt.svc.cluster.local:9000"
- name: S3_BUCKET
value: foxhunt-artifacts
value: foxhunt-models
- name: ENABLE_GPU
value: "true"
- name: RUST_LOG

28
infra/k8s/services/ml-training-service.yaml
View File

@@ -60,6 +60,10 @@ spec:
app.kubernetes.io/name: ml-training-service
template:
metadata:
annotations:
gitlab.com/prometheus_scrape: "true"
gitlab.com/prometheus_port: "9094"
gitlab.com/prometheus_path: "/metrics"
labels:
app.kubernetes.io/name: ml-training-service
spec:
@@ -92,20 +96,20 @@ spec:
fi
env:
- name: RCLONE_S3_PROVIDER
value: Scaleway
value: Minio
- name: RCLONE_S3_ENDPOINT
value: s3.fr-par.scw.cloud
value: minio.foxhunt.svc.cluster.local:9000
- name: RCLONE_S3_REGION
value: fr-par
value: us-east-1
- name: RCLONE_S3_ACCESS_KEY_ID
valueFrom:
secretKeyRef:
name: s3-credentials
name: minio-credentials
key: access-key
- name: RCLONE_S3_SECRET_ACCESS_KEY
valueFrom:
secretKeyRef:
name: s3-credentials
name: minio-credentials
key: secret-key
volumeMounts:
- name: binaries
@@ -148,21 +152,19 @@ spec:
- name: JWT_AUDIENCE
value: foxhunt-services
- name: S3_ENDPOINT
value: "https://s3.fr-par.scw.cloud"
value: "http://minio.foxhunt.svc.cluster.local:9000"
- name: S3_BUCKET
value: foxhunt-artifacts
value: foxhunt-models
- name: AWS_ACCESS_KEY_ID
valueFrom:
secretKeyRef:
name: foxhunt-secrets
key: s3-access-key
optional: true
name: minio-credentials
key: access-key
- name: AWS_SECRET_ACCESS_KEY
valueFrom:
secretKeyRef:
name: foxhunt-secrets
key: s3-secret-key
optional: true
name: minio-credentials
key: secret-key
- name: RUST_LOG
value: info
- name: OTEL_EXPORTER_OTLP_ENDPOINT

14
infra/k8s/services/trading-agent-service.yaml
View File

@@ -18,6 +18,10 @@ spec:
app.kubernetes.io/name: trading-agent-service
template:
metadata:
annotations:
gitlab.com/prometheus_scrape: "true"
gitlab.com/prometheus_port: "9095"
gitlab.com/prometheus_path: "/metrics"
labels:
app.kubernetes.io/name: trading-agent-service
spec:
@@ -49,20 +53,20 @@ spec:
fi
env:
- name: RCLONE_S3_PROVIDER
value: Scaleway
value: Minio
- name: RCLONE_S3_ENDPOINT
value: s3.fr-par.scw.cloud
value: minio.foxhunt.svc.cluster.local:9000
- name: RCLONE_S3_REGION
value: fr-par
value: us-east-1
- name: RCLONE_S3_ACCESS_KEY_ID
valueFrom:
secretKeyRef:
name: s3-credentials
name: minio-credentials
key: access-key
- name: RCLONE_S3_SECRET_ACCESS_KEY
valueFrom:
secretKeyRef:
name: s3-credentials
name: minio-credentials
key: secret-key
volumeMounts:
- name: binaries

14
infra/k8s/services/trading-service-gpu.yaml
View File

@@ -21,6 +21,10 @@ spec:
app.kubernetes.io/name: trading-service
template:
metadata:
annotations:
gitlab.com/prometheus_scrape: "true"
gitlab.com/prometheus_port: "9092"
gitlab.com/prometheus_path: "/metrics"
labels:
app.kubernetes.io/name: trading-service
spec:
@@ -56,20 +60,20 @@ spec:
fi
env:
- name: RCLONE_S3_PROVIDER
value: Scaleway
value: Minio
- name: RCLONE_S3_ENDPOINT
value: s3.fr-par.scw.cloud
value: minio.foxhunt.svc.cluster.local:9000
- name: RCLONE_S3_REGION
value: fr-par
value: us-east-1
- name: RCLONE_S3_ACCESS_KEY_ID
valueFrom:
secretKeyRef:
name: s3-credentials
name: minio-credentials
key: access-key
- name: RCLONE_S3_SECRET_ACCESS_KEY
valueFrom:
secretKeyRef:
name: s3-credentials
name: minio-credentials
key: secret-key
volumeMounts:
- name: binaries

14
infra/k8s/services/trading-service.yaml
View File

@@ -18,6 +18,10 @@ spec:
app.kubernetes.io/name: trading-service
template:
metadata:
annotations:
gitlab.com/prometheus_scrape: "true"
gitlab.com/prometheus_port: "9092"
gitlab.com/prometheus_path: "/metrics"
labels:
app.kubernetes.io/name: trading-service
spec:
@@ -49,20 +53,20 @@ spec:
fi
env:
- name: RCLONE_S3_PROVIDER
value: Scaleway
value: Minio
- name: RCLONE_S3_ENDPOINT
value: s3.fr-par.scw.cloud
value: minio.foxhunt.svc.cluster.local:9000
- name: RCLONE_S3_REGION
value: fr-par
value: us-east-1
- name: RCLONE_S3_ACCESS_KEY_ID
valueFrom:
secretKeyRef:
name: s3-credentials
name: minio-credentials
key: access-key
- name: RCLONE_S3_SECRET_ACCESS_KEY
valueFrom:
secretKeyRef:
name: s3-credentials
name: minio-credentials
key: secret-key
volumeMounts:
- name: binaries

20
infra/k8s/training/job-template.yaml
View File

@@ -13,6 +13,10 @@ spec:
ttlSecondsAfterFinished: 600
template:
metadata:
annotations:
gitlab.com/prometheus_scrape: "true"
gitlab.com/prometheus_port: "9094"
gitlab.com/prometheus_path: "/metrics"
labels:
app.kubernetes.io/name: training
foxhunt/job-type: training
@@ -41,21 +45,21 @@ spec:
args:
- |
set -e
export RCLONE_S3_PROVIDER=Scaleway RCLONE_S3_ENDPOINT=s3.fr-par.scw.cloud RCLONE_S3_REGION=fr-par
export RCLONE_S3_PROVIDER=Minio RCLONE_S3_ENDPOINT=minio.foxhunt.svc.cluster.local:9000 RCLONE_S3_REGION=us-east-1
rclone sync :s3:foxhunt-binaries/latest/training/ /binaries/
chmod +x /binaries/*
ls -lh /binaries/
echo "Fetched training binaries from S3"
echo "Fetched training binaries from MinIO"
env:
- name: RCLONE_S3_ACCESS_KEY_ID
valueFrom:
secretKeyRef:
name: s3-credentials
name: minio-credentials
key: access-key
- name: RCLONE_S3_SECRET_ACCESS_KEY
valueFrom:
secretKeyRef:
name: s3-credentials
name: minio-credentials
key: secret-key
volumeMounts:
- name: binaries
@@ -77,9 +81,9 @@ spec:
- name: S3_BUCKET
value: foxhunt-models
- name: S3_ENDPOINT
value: https://s3.fr-par.scw.cloud
value: http://minio.foxhunt.svc.cluster.local:9000
- name: S3_REGION
value: fr-par
value: us-east-1
- name: S3_PREFIX
value: "models/PLACEHOLDER"
- name: CALLBACK_ENDPOINT
@@ -89,12 +93,12 @@ spec:
- name: AWS_ACCESS_KEY_ID
valueFrom:
secretKeyRef:
name: s3-credentials
name: minio-credentials
key: access-key
- name: AWS_SECRET_ACCESS_KEY
valueFrom:
secretKeyRef:
name: s3-credentials
name: minio-credentials
key: secret-key
volumeMounts:
- name: output