From df772ff985578c7ff6d9c816fcad15b32263e52e Mon Sep 17 00:00:00 2001
From: jgrusewski <jgrusewski@localhost>
Date: Thu, 12 Mar 2026 09:29:29 +0100
Subject: [PATCH] fix(metrics): reduce training log noise, fix Prometheus
 scraping for hyperopt pods
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- Drawdown circuit breaker: warn! → debug! (per-bar flooding in portfolio_tracker)
- Volatility epsilon adjustment: info! → debug! (per-step noise in dqn/risk)
- Prometheus: expand training-pods scrape regex to include compile-and-train pods

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 crates/ml-core/src/portfolio_tracker.rs | 4 ++--
 crates/ml/src/trainers/dqn/risk.rs      | 4 ++--
 infra/k8s/monitoring/prometheus.yaml    | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/crates/ml-core/src/portfolio_tracker.rs b/crates/ml-core/src/portfolio_tracker.rs
index 6e2c82a80..2c8ce24cd 100644
--- a/crates/ml-core/src/portfolio_tracker.rs
+++ b/crates/ml-core/src/portfolio_tracker.rs
@@ -243,14 +243,14 @@ impl PortfolioTracker {
             if drawdown > 0.20 {
                 if self.position_size.abs() < f32::EPSILON {
                     // Already flat, refuse any new trades
-                    warn!(
+                    debug!(
                         "Drawdown circuit breaker: {:.1}% drawdown, refusing new position. Peak: ${:.2}, Current: ${:.2}",
                         drawdown * 100.0, self.peak_value, current_value
                     );
                     return;
                 }
                 // Have an open position: force close to flat
-                warn!(
+                debug!(
                     "Drawdown circuit breaker: {:.1}% drawdown, force-closing position {:.2}. Peak: ${:.2}, Current: ${:.2}",
                     drawdown * 100.0, self.position_size, self.peak_value, current_value
                 );
diff --git a/crates/ml/src/trainers/dqn/risk.rs b/crates/ml/src/trainers/dqn/risk.rs
index 9d54a37e8..a84c8b8be 100644
--- a/crates/ml/src/trainers/dqn/risk.rs
+++ b/crates/ml/src/trainers/dqn/risk.rs
@@ -3,7 +3,7 @@
 //! Helper methods for volatility-adjusted exploration, risk-adjusted rewards,
 //! Kelly criterion position sizing, and risk tracker updates.
 
-use tracing::info;
+use tracing::debug;
 
 use super::trainer::DQNTrainer;
 
@@ -40,7 +40,7 @@ impl DQNTrainer {
 
         let adjusted_epsilon = (base_epsilon * multiplier).clamp(0.01, 1.0);
 
-        info!(
+        debug!(
             "Volatility-adjusted epsilon: base={:.4} × {:.2} = {:.4} (vol={:.4})",
             base_epsilon, multiplier, adjusted_epsilon, volatility
         );
diff --git a/infra/k8s/monitoring/prometheus.yaml b/infra/k8s/monitoring/prometheus.yaml
index 109699ead..dad9890fb 100644
--- a/infra/k8s/monitoring/prometheus.yaml
+++ b/infra/k8s/monitoring/prometheus.yaml
@@ -137,7 +137,7 @@ data:
         relabel_configs:
           - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_component]
             action: keep
-            regex: training-workflow
+            regex: training-workflow|compile-and-train
           # Only scrape Running pods — completed/failed Argo steps no longer serve metrics
           - source_labels: [__meta_kubernetes_pod_phase]
             action: keep