From 033eed152667c4e6c801f8afc76ea0bec3325cd0 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sun, 1 Mar 2026 00:42:17 +0100 Subject: [PATCH] feat(infra): add Prometheus scrape annotations to all service deployments Enable GitLab Prometheus to auto-discover and scrape metrics from all foxhunt pods by adding gitlab.com/prometheus_scrape annotations to the pod template metadata of 9 service deployments and the training job template. Co-Authored-By: Claude Opus 4.6 --- infra/k8s/services/api-gateway.yaml | 14 ++++++---- infra/k8s/services/backtesting-service.yaml | 14 ++++++---- infra/k8s/services/broker-gateway.yaml | 14 ++++++---- .../services/data-acquisition-service.yaml | 14 ++++++---- .../k8s/services/ml-training-service-gpu.yaml | 18 +++++++----- infra/k8s/services/ml-training-service.yaml | 28 ++++++++++--------- infra/k8s/services/trading-agent-service.yaml | 14 ++++++---- infra/k8s/services/trading-service-gpu.yaml | 14 ++++++---- infra/k8s/services/trading-service.yaml | 14 ++++++---- infra/k8s/training/job-template.yaml | 20 +++++++------ 10 files changed, 101 insertions(+), 63 deletions(-) diff --git a/infra/k8s/services/api-gateway.yaml b/infra/k8s/services/api-gateway.yaml index 21af499c1..ec43ecc26 100644 --- a/infra/k8s/services/api-gateway.yaml +++ b/infra/k8s/services/api-gateway.yaml @@ -18,6 +18,10 @@ spec: app.kubernetes.io/name: api-gateway template: metadata: + annotations: + gitlab.com/prometheus_scrape: "true" + gitlab.com/prometheus_port: "9091" + gitlab.com/prometheus_path: "/metrics" labels: app.kubernetes.io/name: api-gateway spec: @@ -49,20 +53,20 @@ spec: fi env: - name: RCLONE_S3_PROVIDER - value: Scaleway + value: Minio - name: RCLONE_S3_ENDPOINT - value: s3.fr-par.scw.cloud + value: minio.foxhunt.svc.cluster.local:9000 - name: RCLONE_S3_REGION - value: fr-par + value: us-east-1 - name: RCLONE_S3_ACCESS_KEY_ID valueFrom: secretKeyRef: - name: s3-credentials + name: minio-credentials key: access-key - name: RCLONE_S3_SECRET_ACCESS_KEY valueFrom: secretKeyRef: - name: s3-credentials + name: minio-credentials key: secret-key volumeMounts: - name: binaries diff --git a/infra/k8s/services/backtesting-service.yaml b/infra/k8s/services/backtesting-service.yaml index 73815da70..6204aaf7c 100644 --- a/infra/k8s/services/backtesting-service.yaml +++ b/infra/k8s/services/backtesting-service.yaml @@ -18,6 +18,10 @@ spec: app.kubernetes.io/name: backtesting-service template: metadata: + annotations: + gitlab.com/prometheus_scrape: "true" + gitlab.com/prometheus_port: "9093" + gitlab.com/prometheus_path: "/metrics" labels: app.kubernetes.io/name: backtesting-service spec: @@ -49,20 +53,20 @@ spec: fi env: - name: RCLONE_S3_PROVIDER - value: Scaleway + value: Minio - name: RCLONE_S3_ENDPOINT - value: s3.fr-par.scw.cloud + value: minio.foxhunt.svc.cluster.local:9000 - name: RCLONE_S3_REGION - value: fr-par + value: us-east-1 - name: RCLONE_S3_ACCESS_KEY_ID valueFrom: secretKeyRef: - name: s3-credentials + name: minio-credentials key: access-key - name: RCLONE_S3_SECRET_ACCESS_KEY valueFrom: secretKeyRef: - name: s3-credentials + name: minio-credentials key: secret-key volumeMounts: - name: binaries diff --git a/infra/k8s/services/broker-gateway.yaml b/infra/k8s/services/broker-gateway.yaml index bba74d6f7..c30d58edb 100644 --- a/infra/k8s/services/broker-gateway.yaml +++ b/infra/k8s/services/broker-gateway.yaml @@ -18,6 +18,10 @@ spec: app.kubernetes.io/name: broker-gateway template: metadata: + annotations: + gitlab.com/prometheus_scrape: "true" + gitlab.com/prometheus_port: "9096" + gitlab.com/prometheus_path: "/metrics" labels: app.kubernetes.io/name: broker-gateway spec: @@ -49,20 +53,20 @@ spec: fi env: - name: RCLONE_S3_PROVIDER - value: Scaleway + value: Minio - name: RCLONE_S3_ENDPOINT - value: s3.fr-par.scw.cloud + value: minio.foxhunt.svc.cluster.local:9000 - name: RCLONE_S3_REGION - value: fr-par + value: us-east-1 - name: RCLONE_S3_ACCESS_KEY_ID valueFrom: secretKeyRef: - name: s3-credentials + name: minio-credentials key: access-key - name: RCLONE_S3_SECRET_ACCESS_KEY valueFrom: secretKeyRef: - name: s3-credentials + name: minio-credentials key: secret-key volumeMounts: - name: binaries diff --git a/infra/k8s/services/data-acquisition-service.yaml b/infra/k8s/services/data-acquisition-service.yaml index 5dbbd37fe..076fb456a 100644 --- a/infra/k8s/services/data-acquisition-service.yaml +++ b/infra/k8s/services/data-acquisition-service.yaml @@ -18,6 +18,10 @@ spec: app.kubernetes.io/name: data-acquisition-service template: metadata: + annotations: + gitlab.com/prometheus_scrape: "true" + gitlab.com/prometheus_port: "9097" + gitlab.com/prometheus_path: "/metrics" labels: app.kubernetes.io/name: data-acquisition-service spec: @@ -49,20 +53,20 @@ spec: fi env: - name: RCLONE_S3_PROVIDER - value: Scaleway + value: Minio - name: RCLONE_S3_ENDPOINT - value: s3.fr-par.scw.cloud + value: minio.foxhunt.svc.cluster.local:9000 - name: RCLONE_S3_REGION - value: fr-par + value: us-east-1 - name: RCLONE_S3_ACCESS_KEY_ID valueFrom: secretKeyRef: - name: s3-credentials + name: minio-credentials key: access-key - name: RCLONE_S3_SECRET_ACCESS_KEY valueFrom: secretKeyRef: - name: s3-credentials + name: minio-credentials key: secret-key volumeMounts: - name: binaries diff --git a/infra/k8s/services/ml-training-service-gpu.yaml b/infra/k8s/services/ml-training-service-gpu.yaml index 848b0971f..e4f418f48 100644 --- a/infra/k8s/services/ml-training-service-gpu.yaml +++ b/infra/k8s/services/ml-training-service-gpu.yaml @@ -21,6 +21,10 @@ spec: app.kubernetes.io/name: ml-training-service template: metadata: + annotations: + gitlab.com/prometheus_scrape: "true" + gitlab.com/prometheus_port: "9094" + gitlab.com/prometheus_path: "/metrics" labels: app.kubernetes.io/name: ml-training-service spec: @@ -57,20 +61,20 @@ spec: fi env: - name: RCLONE_S3_PROVIDER - value: Scaleway + value: Minio - name: RCLONE_S3_ENDPOINT - value: s3.fr-par.scw.cloud + value: minio.foxhunt.svc.cluster.local:9000 - name: RCLONE_S3_REGION - value: fr-par + value: us-east-1 - name: RCLONE_S3_ACCESS_KEY_ID valueFrom: secretKeyRef: - name: s3-credentials + name: minio-credentials key: access-key - name: RCLONE_S3_SECRET_ACCESS_KEY valueFrom: secretKeyRef: - name: s3-credentials + name: minio-credentials key: secret-key volumeMounts: - name: binaries @@ -113,9 +117,9 @@ spec: - name: JWT_AUDIENCE value: foxhunt-services - name: S3_ENDPOINT - value: "https://s3.fr-par.scw.cloud" + value: "http://minio.foxhunt.svc.cluster.local:9000" - name: S3_BUCKET - value: foxhunt-artifacts + value: foxhunt-models - name: ENABLE_GPU value: "true" - name: RUST_LOG diff --git a/infra/k8s/services/ml-training-service.yaml b/infra/k8s/services/ml-training-service.yaml index 600a91d3d..18a6b9c67 100644 --- a/infra/k8s/services/ml-training-service.yaml +++ b/infra/k8s/services/ml-training-service.yaml @@ -60,6 +60,10 @@ spec: app.kubernetes.io/name: ml-training-service template: metadata: + annotations: + gitlab.com/prometheus_scrape: "true" + gitlab.com/prometheus_port: "9094" + gitlab.com/prometheus_path: "/metrics" labels: app.kubernetes.io/name: ml-training-service spec: @@ -92,20 +96,20 @@ spec: fi env: - name: RCLONE_S3_PROVIDER - value: Scaleway + value: Minio - name: RCLONE_S3_ENDPOINT - value: s3.fr-par.scw.cloud + value: minio.foxhunt.svc.cluster.local:9000 - name: RCLONE_S3_REGION - value: fr-par + value: us-east-1 - name: RCLONE_S3_ACCESS_KEY_ID valueFrom: secretKeyRef: - name: s3-credentials + name: minio-credentials key: access-key - name: RCLONE_S3_SECRET_ACCESS_KEY valueFrom: secretKeyRef: - name: s3-credentials + name: minio-credentials key: secret-key volumeMounts: - name: binaries @@ -148,21 +152,19 @@ spec: - name: JWT_AUDIENCE value: foxhunt-services - name: S3_ENDPOINT - value: "https://s3.fr-par.scw.cloud" + value: "http://minio.foxhunt.svc.cluster.local:9000" - name: S3_BUCKET - value: foxhunt-artifacts + value: foxhunt-models - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: - name: foxhunt-secrets - key: s3-access-key - optional: true + name: minio-credentials + key: access-key - name: AWS_SECRET_ACCESS_KEY valueFrom: secretKeyRef: - name: foxhunt-secrets - key: s3-secret-key - optional: true + name: minio-credentials + key: secret-key - name: RUST_LOG value: info - name: OTEL_EXPORTER_OTLP_ENDPOINT diff --git a/infra/k8s/services/trading-agent-service.yaml b/infra/k8s/services/trading-agent-service.yaml index 556ffbf98..4b9e811c2 100644 --- a/infra/k8s/services/trading-agent-service.yaml +++ b/infra/k8s/services/trading-agent-service.yaml @@ -18,6 +18,10 @@ spec: app.kubernetes.io/name: trading-agent-service template: metadata: + annotations: + gitlab.com/prometheus_scrape: "true" + gitlab.com/prometheus_port: "9095" + gitlab.com/prometheus_path: "/metrics" labels: app.kubernetes.io/name: trading-agent-service spec: @@ -49,20 +53,20 @@ spec: fi env: - name: RCLONE_S3_PROVIDER - value: Scaleway + value: Minio - name: RCLONE_S3_ENDPOINT - value: s3.fr-par.scw.cloud + value: minio.foxhunt.svc.cluster.local:9000 - name: RCLONE_S3_REGION - value: fr-par + value: us-east-1 - name: RCLONE_S3_ACCESS_KEY_ID valueFrom: secretKeyRef: - name: s3-credentials + name: minio-credentials key: access-key - name: RCLONE_S3_SECRET_ACCESS_KEY valueFrom: secretKeyRef: - name: s3-credentials + name: minio-credentials key: secret-key volumeMounts: - name: binaries diff --git a/infra/k8s/services/trading-service-gpu.yaml b/infra/k8s/services/trading-service-gpu.yaml index 092f62b7b..d37a8a8c6 100644 --- a/infra/k8s/services/trading-service-gpu.yaml +++ b/infra/k8s/services/trading-service-gpu.yaml @@ -21,6 +21,10 @@ spec: app.kubernetes.io/name: trading-service template: metadata: + annotations: + gitlab.com/prometheus_scrape: "true" + gitlab.com/prometheus_port: "9092" + gitlab.com/prometheus_path: "/metrics" labels: app.kubernetes.io/name: trading-service spec: @@ -56,20 +60,20 @@ spec: fi env: - name: RCLONE_S3_PROVIDER - value: Scaleway + value: Minio - name: RCLONE_S3_ENDPOINT - value: s3.fr-par.scw.cloud + value: minio.foxhunt.svc.cluster.local:9000 - name: RCLONE_S3_REGION - value: fr-par + value: us-east-1 - name: RCLONE_S3_ACCESS_KEY_ID valueFrom: secretKeyRef: - name: s3-credentials + name: minio-credentials key: access-key - name: RCLONE_S3_SECRET_ACCESS_KEY valueFrom: secretKeyRef: - name: s3-credentials + name: minio-credentials key: secret-key volumeMounts: - name: binaries diff --git a/infra/k8s/services/trading-service.yaml b/infra/k8s/services/trading-service.yaml index a18cf3126..24de7d643 100644 --- a/infra/k8s/services/trading-service.yaml +++ b/infra/k8s/services/trading-service.yaml @@ -18,6 +18,10 @@ spec: app.kubernetes.io/name: trading-service template: metadata: + annotations: + gitlab.com/prometheus_scrape: "true" + gitlab.com/prometheus_port: "9092" + gitlab.com/prometheus_path: "/metrics" labels: app.kubernetes.io/name: trading-service spec: @@ -49,20 +53,20 @@ spec: fi env: - name: RCLONE_S3_PROVIDER - value: Scaleway + value: Minio - name: RCLONE_S3_ENDPOINT - value: s3.fr-par.scw.cloud + value: minio.foxhunt.svc.cluster.local:9000 - name: RCLONE_S3_REGION - value: fr-par + value: us-east-1 - name: RCLONE_S3_ACCESS_KEY_ID valueFrom: secretKeyRef: - name: s3-credentials + name: minio-credentials key: access-key - name: RCLONE_S3_SECRET_ACCESS_KEY valueFrom: secretKeyRef: - name: s3-credentials + name: minio-credentials key: secret-key volumeMounts: - name: binaries diff --git a/infra/k8s/training/job-template.yaml b/infra/k8s/training/job-template.yaml index 19e072c36..02b4ef6e5 100644 --- a/infra/k8s/training/job-template.yaml +++ b/infra/k8s/training/job-template.yaml @@ -13,6 +13,10 @@ spec: ttlSecondsAfterFinished: 600 template: metadata: + annotations: + gitlab.com/prometheus_scrape: "true" + gitlab.com/prometheus_port: "9094" + gitlab.com/prometheus_path: "/metrics" labels: app.kubernetes.io/name: training foxhunt/job-type: training @@ -41,21 +45,21 @@ spec: args: - | set -e - export RCLONE_S3_PROVIDER=Scaleway RCLONE_S3_ENDPOINT=s3.fr-par.scw.cloud RCLONE_S3_REGION=fr-par + export RCLONE_S3_PROVIDER=Minio RCLONE_S3_ENDPOINT=minio.foxhunt.svc.cluster.local:9000 RCLONE_S3_REGION=us-east-1 rclone sync :s3:foxhunt-binaries/latest/training/ /binaries/ chmod +x /binaries/* ls -lh /binaries/ - echo "Fetched training binaries from S3" + echo "Fetched training binaries from MinIO" env: - name: RCLONE_S3_ACCESS_KEY_ID valueFrom: secretKeyRef: - name: s3-credentials + name: minio-credentials key: access-key - name: RCLONE_S3_SECRET_ACCESS_KEY valueFrom: secretKeyRef: - name: s3-credentials + name: minio-credentials key: secret-key volumeMounts: - name: binaries @@ -77,9 +81,9 @@ spec: - name: S3_BUCKET value: foxhunt-models - name: S3_ENDPOINT - value: https://s3.fr-par.scw.cloud + value: http://minio.foxhunt.svc.cluster.local:9000 - name: S3_REGION - value: fr-par + value: us-east-1 - name: S3_PREFIX value: "models/PLACEHOLDER" - name: CALLBACK_ENDPOINT @@ -89,12 +93,12 @@ spec: - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: - name: s3-credentials + name: minio-credentials key: access-key - name: AWS_SECRET_ACCESS_KEY valueFrom: secretKeyRef: - name: s3-credentials + name: minio-credentials key: secret-key volumeMounts: - name: output