feat(metrics): add 6 hyperopt Prometheus metrics to training_metrics

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-03-02 21:18:50 +01:00
parent 4b2f9e8133
commit 354cd5c116

View File

@@ -1,6 +1,6 @@
//! Prometheus metrics for ML training binaries.
//!
//! Registers the 18 metrics expected by the Training Cockpit Grafana dashboard
//! Registers the 24 metrics expected by the Training Cockpit Grafana dashboard
//! and provides typed helper functions so callers never mis-spell metric names.
//!
//! # Usage
@@ -26,7 +26,7 @@ use super::{
// Registration
// ---------------------------------------------------------------------------
/// Register all 18 training metrics with the global Prometheus registry.
/// Register all 24 training metrics with the global Prometheus registry.
///
/// Safe to call multiple times — the registry silently ignores duplicates.
pub fn init() {
@@ -128,6 +128,43 @@ pub fn init() {
"foxhunt_training_active_workers",
"Number of active training workers",
);
// Hyperopt gauges (model only)
_ = register_gauge_vec(
"foxhunt_hyperopt_trial_current",
"Current trial number (1-indexed)",
m,
);
_ = register_gauge_vec(
"foxhunt_hyperopt_trial_total",
"Total trials configured",
m,
);
_ = register_gauge_vec(
"foxhunt_hyperopt_best_objective",
"Best objective value so far (Sharpe)",
m,
);
_ = register_gauge_vec(
"foxhunt_hyperopt_mode",
"1=hyperopt active, 0=training only",
m,
);
// Hyperopt counters (model only)
_ = register_counter_vec(
"foxhunt_hyperopt_trials_failed_total",
"Failed trials count",
m,
);
// Hyperopt histogram (model only)
_ = register_histogram_vec(
"foxhunt_hyperopt_trial_duration_seconds",
"Duration of each trial",
m,
vec![5.0, 15.0, 30.0, 60.0, 120.0, 300.0, 600.0],
);
}
// ---------------------------------------------------------------------------
@@ -233,3 +270,71 @@ pub fn record_data_load(model: &str, duration_secs: f64) {
pub fn set_active_workers(value: f64) {
set_gauge("foxhunt_training_active_workers", value);
}
pub fn set_hyperopt_trial(model: &str, current: f64, total: f64) {
set_gauge_vec("foxhunt_hyperopt_trial_current", &[model], current);
set_gauge_vec("foxhunt_hyperopt_trial_total", &[model], total);
}
pub fn set_hyperopt_best_objective(model: &str, value: f64) {
set_gauge_vec("foxhunt_hyperopt_best_objective", &[model], value);
}
pub fn record_hyperopt_trial_duration(model: &str, secs: f64) {
observe_histogram_vec("foxhunt_hyperopt_trial_duration_seconds", &[model], secs);
}
pub fn record_hyperopt_trial_failure(model: &str) {
increment_counter_vec("foxhunt_hyperopt_trials_failed_total", &[model], 1.0);
}
pub fn set_hyperopt_mode(model: &str, active: bool) {
set_gauge_vec(
"foxhunt_hyperopt_mode",
&[model],
if active { 1.0 } else { 0.0 },
);
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_init_is_idempotent() {
init();
init(); // calling twice must not panic
}
#[test]
fn test_set_hyperopt_trial() {
init();
set_hyperopt_trial("dqn", 5.0, 20.0);
// No panic = success (gauge_vec values not directly readable without label matching)
}
#[test]
fn test_set_hyperopt_mode() {
init();
set_hyperopt_mode("dqn", true);
set_hyperopt_mode("dqn", false);
}
#[test]
fn test_record_hyperopt_trial_failure() {
init();
record_hyperopt_trial_failure("ppo");
}
#[test]
fn test_set_hyperopt_best_objective() {
init();
set_hyperopt_best_objective("tft", 2.34);
}
#[test]
fn test_record_hyperopt_trial_duration() {
init();
record_hyperopt_trial_duration("dqn", 45.3);
}
}