From ab218121b671e0d06bb9c8bf36d0956c68bbca19 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Tue, 3 Mar 2026 00:09:54 +0100 Subject: [PATCH] =?UTF-8?q?feat(monitoring):=20wire=2012=20new=20metric?= =?UTF-8?q?=E2=86=92proto=20field=20mappings=20in=20group=5Finto=5Fsession?= =?UTF-8?q?s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.6 --- services/monitoring_service/src/service.rs | 148 +++++++++++++++++++++ 1 file changed, 148 insertions(+) diff --git a/services/monitoring_service/src/service.rs b/services/monitoring_service/src/service.rs index a66c65909..85404dd49 100644 --- a/services/monitoring_service/src/service.rs +++ b/services/monitoring_service/src/service.rs @@ -155,6 +155,29 @@ fn group_into_sessions(samples: &[MetricSample], model_filter: &str) -> Vec session.q_value_mean = s.value as f32, + "foxhunt_training_q_value_max" => session.q_value_max = s.value as f32, + "foxhunt_training_policy_entropy" => session.policy_entropy = s.value as f32, + "foxhunt_training_kl_divergence" => session.kl_divergence = s.value as f32, + "foxhunt_training_advantage_mean" => session.advantage_mean = s.value as f32, + "foxhunt_training_replay_buffer_size" => { + session.replay_buffer_size = s.value as u32; + } + // Gradient & training health + "foxhunt_training_gradient_norm" => session.gradient_norm = s.value as f32, + "foxhunt_training_learning_rate" => session.learning_rate = s.value as f32, + "foxhunt_training_epoch_duration_seconds" => { + session.epoch_duration_seconds = s.value as f32; + } + // Hyperopt intra-trial + "foxhunt_hyperopt_trial_epoch" => session.hyperopt_trial_epoch = s.value as u32, + "foxhunt_hyperopt_trial_best_loss" => { + session.hyperopt_trial_best_loss = s.value as f32; + } + "foxhunt_hyperopt_elapsed_seconds" => { + session.hyperopt_elapsed_seconds = s.value as f32; + } _ => {} } } @@ -227,6 +250,131 @@ mod tests { assert!(sessions.first().map(|s| s.is_hyperopt).unwrap_or(false)); } + #[test] + fn test_group_new_diagnostic_metrics() { + let samples = vec![ + MetricSample { + name: "foxhunt_training_current_epoch".to_owned(), + model: "dqn".to_owned(), + fold: "0".to_owned(), + value: 25.0, + }, + MetricSample { + name: "foxhunt_training_q_value_mean".to_owned(), + model: "dqn".to_owned(), + fold: "0".to_owned(), + value: 12.3, + }, + MetricSample { + name: "foxhunt_training_q_value_max".to_owned(), + model: "dqn".to_owned(), + fold: "0".to_owned(), + value: 45.6, + }, + MetricSample { + name: "foxhunt_training_gradient_norm".to_owned(), + model: "dqn".to_owned(), + fold: "0".to_owned(), + value: 1.23, + }, + MetricSample { + name: "foxhunt_training_learning_rate".to_owned(), + model: "dqn".to_owned(), + fold: "0".to_owned(), + value: 0.001, + }, + MetricSample { + name: "foxhunt_training_epoch_duration_seconds".to_owned(), + model: "dqn".to_owned(), + fold: "0".to_owned(), + value: 12.5, + }, + MetricSample { + name: "foxhunt_training_replay_buffer_size".to_owned(), + model: "dqn".to_owned(), + fold: "0".to_owned(), + value: 50000.0, + }, + ]; + let sessions = group_into_sessions(&samples, ""); + assert_eq!(sessions.len(), 1); + let s = &sessions[0]; + assert_eq!(s.current_epoch, 25.0); + assert!((s.q_value_mean - 12.3).abs() < 0.01); + assert!((s.q_value_max - 45.6).abs() < 0.01); + assert!((s.gradient_norm - 1.23).abs() < 0.01); + assert!((s.learning_rate - 0.001).abs() < 0.0001); + assert!((s.epoch_duration_seconds - 12.5).abs() < 0.01); + assert_eq!(s.replay_buffer_size, 50000); + } + + #[test] + fn test_group_ppo_diagnostics() { + let samples = vec![ + MetricSample { + name: "foxhunt_training_policy_entropy".to_owned(), + model: "ppo".to_owned(), + fold: "0".to_owned(), + value: 1.23, + }, + MetricSample { + name: "foxhunt_training_kl_divergence".to_owned(), + model: "ppo".to_owned(), + fold: "0".to_owned(), + value: 0.008, + }, + MetricSample { + name: "foxhunt_training_advantage_mean".to_owned(), + model: "ppo".to_owned(), + fold: "0".to_owned(), + value: 0.001, + }, + ]; + let sessions = group_into_sessions(&samples, ""); + assert_eq!(sessions.len(), 1); + let s = &sessions[0]; + assert!((s.policy_entropy - 1.23).abs() < 0.01); + assert!((s.kl_divergence - 0.008).abs() < 0.001); + assert!((s.advantage_mean - 0.001).abs() < 0.001); + } + + #[test] + fn test_group_hyperopt_intra_trial() { + let samples = vec![ + MetricSample { + name: "foxhunt_hyperopt_mode".to_owned(), + model: "dqn".to_owned(), + fold: "".to_owned(), + value: 1.0, + }, + MetricSample { + name: "foxhunt_hyperopt_trial_epoch".to_owned(), + model: "dqn".to_owned(), + fold: "".to_owned(), + value: 12.0, + }, + MetricSample { + name: "foxhunt_hyperopt_trial_best_loss".to_owned(), + model: "dqn".to_owned(), + fold: "".to_owned(), + value: 0.042, + }, + MetricSample { + name: "foxhunt_hyperopt_elapsed_seconds".to_owned(), + model: "dqn".to_owned(), + fold: "".to_owned(), + value: 123.4, + }, + ]; + let sessions = group_into_sessions(&samples, ""); + assert_eq!(sessions.len(), 1); + let s = &sessions[0]; + assert!(s.is_hyperopt); + assert_eq!(s.hyperopt_trial_epoch, 12); + assert!((s.hyperopt_trial_best_loss - 0.042).abs() < 0.001); + assert!((s.hyperopt_elapsed_seconds - 123.4).abs() < 0.1); + } + #[test] fn test_build_gpu_snapshot() { let samples = vec![