diff --git a/crates/common/src/metrics/training_metrics.rs b/crates/common/src/metrics/training_metrics.rs index fbcd18488..168740625 100644 --- a/crates/common/src/metrics/training_metrics.rs +++ b/crates/common/src/metrics/training_metrics.rs @@ -1,6 +1,6 @@ //! Prometheus metrics for ML training binaries. //! -//! Registers the 18 metrics expected by the Training Cockpit Grafana dashboard +//! Registers the 24 metrics expected by the Training Cockpit Grafana dashboard //! and provides typed helper functions so callers never mis-spell metric names. //! //! # Usage @@ -26,7 +26,7 @@ use super::{ // Registration // --------------------------------------------------------------------------- -/// Register all 18 training metrics with the global Prometheus registry. +/// Register all 24 training metrics with the global Prometheus registry. /// /// Safe to call multiple times — the registry silently ignores duplicates. pub fn init() { @@ -128,6 +128,43 @@ pub fn init() { "foxhunt_training_active_workers", "Number of active training workers", ); + + // Hyperopt gauges (model only) + _ = register_gauge_vec( + "foxhunt_hyperopt_trial_current", + "Current trial number (1-indexed)", + m, + ); + _ = register_gauge_vec( + "foxhunt_hyperopt_trial_total", + "Total trials configured", + m, + ); + _ = register_gauge_vec( + "foxhunt_hyperopt_best_objective", + "Best objective value so far (Sharpe)", + m, + ); + _ = register_gauge_vec( + "foxhunt_hyperopt_mode", + "1=hyperopt active, 0=training only", + m, + ); + + // Hyperopt counters (model only) + _ = register_counter_vec( + "foxhunt_hyperopt_trials_failed_total", + "Failed trials count", + m, + ); + + // Hyperopt histogram (model only) + _ = register_histogram_vec( + "foxhunt_hyperopt_trial_duration_seconds", + "Duration of each trial", + m, + vec![5.0, 15.0, 30.0, 60.0, 120.0, 300.0, 600.0], + ); } // --------------------------------------------------------------------------- @@ -233,3 +270,71 @@ pub fn record_data_load(model: &str, duration_secs: f64) { pub fn set_active_workers(value: f64) { set_gauge("foxhunt_training_active_workers", value); } + +pub fn set_hyperopt_trial(model: &str, current: f64, total: f64) { + set_gauge_vec("foxhunt_hyperopt_trial_current", &[model], current); + set_gauge_vec("foxhunt_hyperopt_trial_total", &[model], total); +} + +pub fn set_hyperopt_best_objective(model: &str, value: f64) { + set_gauge_vec("foxhunt_hyperopt_best_objective", &[model], value); +} + +pub fn record_hyperopt_trial_duration(model: &str, secs: f64) { + observe_histogram_vec("foxhunt_hyperopt_trial_duration_seconds", &[model], secs); +} + +pub fn record_hyperopt_trial_failure(model: &str) { + increment_counter_vec("foxhunt_hyperopt_trials_failed_total", &[model], 1.0); +} + +pub fn set_hyperopt_mode(model: &str, active: bool) { + set_gauge_vec( + "foxhunt_hyperopt_mode", + &[model], + if active { 1.0 } else { 0.0 }, + ); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_init_is_idempotent() { + init(); + init(); // calling twice must not panic + } + + #[test] + fn test_set_hyperopt_trial() { + init(); + set_hyperopt_trial("dqn", 5.0, 20.0); + // No panic = success (gauge_vec values not directly readable without label matching) + } + + #[test] + fn test_set_hyperopt_mode() { + init(); + set_hyperopt_mode("dqn", true); + set_hyperopt_mode("dqn", false); + } + + #[test] + fn test_record_hyperopt_trial_failure() { + init(); + record_hyperopt_trial_failure("ppo"); + } + + #[test] + fn test_set_hyperopt_best_objective() { + init(); + set_hyperopt_best_objective("tft", 2.34); + } + + #[test] + fn test_record_hyperopt_trial_duration() { + init(); + record_hyperopt_trial_duration("dqn", 45.3); + } +}