feat(metrics): add 6 hyperopt Prometheus metrics to training_metrics
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
//! Prometheus metrics for ML training binaries.
|
||||
//!
|
||||
//! Registers the 18 metrics expected by the Training Cockpit Grafana dashboard
|
||||
//! Registers the 24 metrics expected by the Training Cockpit Grafana dashboard
|
||||
//! and provides typed helper functions so callers never mis-spell metric names.
|
||||
//!
|
||||
//! # Usage
|
||||
@@ -26,7 +26,7 @@ use super::{
|
||||
// Registration
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Register all 18 training metrics with the global Prometheus registry.
|
||||
/// Register all 24 training metrics with the global Prometheus registry.
|
||||
///
|
||||
/// Safe to call multiple times — the registry silently ignores duplicates.
|
||||
pub fn init() {
|
||||
@@ -128,6 +128,43 @@ pub fn init() {
|
||||
"foxhunt_training_active_workers",
|
||||
"Number of active training workers",
|
||||
);
|
||||
|
||||
// Hyperopt gauges (model only)
|
||||
_ = register_gauge_vec(
|
||||
"foxhunt_hyperopt_trial_current",
|
||||
"Current trial number (1-indexed)",
|
||||
m,
|
||||
);
|
||||
_ = register_gauge_vec(
|
||||
"foxhunt_hyperopt_trial_total",
|
||||
"Total trials configured",
|
||||
m,
|
||||
);
|
||||
_ = register_gauge_vec(
|
||||
"foxhunt_hyperopt_best_objective",
|
||||
"Best objective value so far (Sharpe)",
|
||||
m,
|
||||
);
|
||||
_ = register_gauge_vec(
|
||||
"foxhunt_hyperopt_mode",
|
||||
"1=hyperopt active, 0=training only",
|
||||
m,
|
||||
);
|
||||
|
||||
// Hyperopt counters (model only)
|
||||
_ = register_counter_vec(
|
||||
"foxhunt_hyperopt_trials_failed_total",
|
||||
"Failed trials count",
|
||||
m,
|
||||
);
|
||||
|
||||
// Hyperopt histogram (model only)
|
||||
_ = register_histogram_vec(
|
||||
"foxhunt_hyperopt_trial_duration_seconds",
|
||||
"Duration of each trial",
|
||||
m,
|
||||
vec![5.0, 15.0, 30.0, 60.0, 120.0, 300.0, 600.0],
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -233,3 +270,71 @@ pub fn record_data_load(model: &str, duration_secs: f64) {
|
||||
pub fn set_active_workers(value: f64) {
|
||||
set_gauge("foxhunt_training_active_workers", value);
|
||||
}
|
||||
|
||||
pub fn set_hyperopt_trial(model: &str, current: f64, total: f64) {
|
||||
set_gauge_vec("foxhunt_hyperopt_trial_current", &[model], current);
|
||||
set_gauge_vec("foxhunt_hyperopt_trial_total", &[model], total);
|
||||
}
|
||||
|
||||
pub fn set_hyperopt_best_objective(model: &str, value: f64) {
|
||||
set_gauge_vec("foxhunt_hyperopt_best_objective", &[model], value);
|
||||
}
|
||||
|
||||
pub fn record_hyperopt_trial_duration(model: &str, secs: f64) {
|
||||
observe_histogram_vec("foxhunt_hyperopt_trial_duration_seconds", &[model], secs);
|
||||
}
|
||||
|
||||
pub fn record_hyperopt_trial_failure(model: &str) {
|
||||
increment_counter_vec("foxhunt_hyperopt_trials_failed_total", &[model], 1.0);
|
||||
}
|
||||
|
||||
pub fn set_hyperopt_mode(model: &str, active: bool) {
|
||||
set_gauge_vec(
|
||||
"foxhunt_hyperopt_mode",
|
||||
&[model],
|
||||
if active { 1.0 } else { 0.0 },
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_init_is_idempotent() {
|
||||
init();
|
||||
init(); // calling twice must not panic
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_set_hyperopt_trial() {
|
||||
init();
|
||||
set_hyperopt_trial("dqn", 5.0, 20.0);
|
||||
// No panic = success (gauge_vec values not directly readable without label matching)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_set_hyperopt_mode() {
|
||||
init();
|
||||
set_hyperopt_mode("dqn", true);
|
||||
set_hyperopt_mode("dqn", false);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_record_hyperopt_trial_failure() {
|
||||
init();
|
||||
record_hyperopt_trial_failure("ppo");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_set_hyperopt_best_objective() {
|
||||
init();
|
||||
set_hyperopt_best_objective("tft", 2.34);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_record_hyperopt_trial_duration() {
|
||||
init();
|
||||
record_hyperopt_trial_duration("dqn", 45.3);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user