From 2f97402d5d5854c3ced1a80b5169e6e83e8823d4 Mon Sep 17 00:00:00 2001
From: jgrusewski <jgrusewski@localhost>
Date: Mon, 2 Mar 2026 23:21:15 +0100
Subject: [PATCH] fix(monitoring): use active_workers metric instead of
 kube_job query

CI training jobs run as GitLab runner pods, not K8s Jobs, so the
kube_job_status_active query always returned 0. Switch to
foxhunt_training_active_workers which is emitted by all training
binaries regardless of deployment method.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 services/monitoring_service/src/prometheus_client.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/services/monitoring_service/src/prometheus_client.rs b/services/monitoring_service/src/prometheus_client.rs
index 6c7eb4938..333c3eb40 100644
--- a/services/monitoring_service/src/prometheus_client.rs
+++ b/services/monitoring_service/src/prometheus_client.rs
@@ -89,10 +89,10 @@ impl PrometheusClient {
         Ok(parse_samples(results))
     }
 
-    /// Fetch active K8s training job count
+    /// Fetch active training worker count (covers both K8s Jobs and CI runner pods)
     pub async fn fetch_active_jobs(&self) -> Result<u32> {
         let results = self
-            .query(r#"kube_job_status_active{namespace="foxhunt",job_name=~"training-.*"}"#)
+            .query("foxhunt_training_active_workers")
             .await?;
         let count: f64 = results
             .iter()