fix(metrics): reduce training log noise, fix Prometheus scraping for hyperopt pods
- Drawdown circuit breaker: warn! → debug! (per-bar flooding in portfolio_tracker) - Volatility epsilon adjustment: info! → debug! (per-step noise in dqn/risk) - Prometheus: expand training-pods scrape regex to include compile-and-train pods Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -243,14 +243,14 @@ impl PortfolioTracker {
|
||||
if drawdown > 0.20 {
|
||||
if self.position_size.abs() < f32::EPSILON {
|
||||
// Already flat, refuse any new trades
|
||||
warn!(
|
||||
debug!(
|
||||
"Drawdown circuit breaker: {:.1}% drawdown, refusing new position. Peak: ${:.2}, Current: ${:.2}",
|
||||
drawdown * 100.0, self.peak_value, current_value
|
||||
);
|
||||
return;
|
||||
}
|
||||
// Have an open position: force close to flat
|
||||
warn!(
|
||||
debug!(
|
||||
"Drawdown circuit breaker: {:.1}% drawdown, force-closing position {:.2}. Peak: ${:.2}, Current: ${:.2}",
|
||||
drawdown * 100.0, self.position_size, self.peak_value, current_value
|
||||
);
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
//! Helper methods for volatility-adjusted exploration, risk-adjusted rewards,
|
||||
//! Kelly criterion position sizing, and risk tracker updates.
|
||||
|
||||
use tracing::info;
|
||||
use tracing::debug;
|
||||
|
||||
use super::trainer::DQNTrainer;
|
||||
|
||||
@@ -40,7 +40,7 @@ impl DQNTrainer {
|
||||
|
||||
let adjusted_epsilon = (base_epsilon * multiplier).clamp(0.01, 1.0);
|
||||
|
||||
info!(
|
||||
debug!(
|
||||
"Volatility-adjusted epsilon: base={:.4} × {:.2} = {:.4} (vol={:.4})",
|
||||
base_epsilon, multiplier, adjusted_epsilon, volatility
|
||||
);
|
||||
|
||||
@@ -137,7 +137,7 @@ data:
|
||||
relabel_configs:
|
||||
- source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_component]
|
||||
action: keep
|
||||
regex: training-workflow
|
||||
regex: training-workflow|compile-and-train
|
||||
# Only scrape Running pods — completed/failed Argo steps no longer serve metrics
|
||||
- source_labels: [__meta_kubernetes_pod_phase]
|
||||
action: keep
|
||||
|
||||
Reference in New Issue
Block a user