fix(metrics): reduce training log noise, fix Prometheus scraping for hyperopt pods

- Drawdown circuit breaker: warn! → debug! (per-bar flooding in portfolio_tracker)
- Volatility epsilon adjustment: info! → debug! (per-step noise in dqn/risk)
- Prometheus: expand training-pods scrape regex to include compile-and-train pods

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-03-12 09:29:29 +01:00
parent ebf7b39288
commit df772ff985
3 changed files with 5 additions and 5 deletions

View File

@@ -243,14 +243,14 @@ impl PortfolioTracker {
if drawdown > 0.20 {
if self.position_size.abs() < f32::EPSILON {
// Already flat, refuse any new trades
warn!(
debug!(
"Drawdown circuit breaker: {:.1}% drawdown, refusing new position. Peak: ${:.2}, Current: ${:.2}",
drawdown * 100.0, self.peak_value, current_value
);
return;
}
// Have an open position: force close to flat
warn!(
debug!(
"Drawdown circuit breaker: {:.1}% drawdown, force-closing position {:.2}. Peak: ${:.2}, Current: ${:.2}",
drawdown * 100.0, self.position_size, self.peak_value, current_value
);

View File

@@ -3,7 +3,7 @@
//! Helper methods for volatility-adjusted exploration, risk-adjusted rewards,
//! Kelly criterion position sizing, and risk tracker updates.
use tracing::info;
use tracing::debug;
use super::trainer::DQNTrainer;
@@ -40,7 +40,7 @@ impl DQNTrainer {
let adjusted_epsilon = (base_epsilon * multiplier).clamp(0.01, 1.0);
info!(
debug!(
"Volatility-adjusted epsilon: base={:.4} × {:.2} = {:.4} (vol={:.4})",
base_epsilon, multiplier, adjusted_epsilon, volatility
);

View File

@@ -137,7 +137,7 @@ data:
relabel_configs:
- source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_component]
action: keep
regex: training-workflow
regex: training-workflow|compile-and-train
# Only scrape Running pods — completed/failed Argo steps no longer serve metrics
- source_labels: [__meta_kubernetes_pod_phase]
action: keep