diff --git a/bin/fxt/proto/monitoring.proto b/bin/fxt/proto/monitoring.proto index c3f90f4ab..d7233faf6 100644 --- a/bin/fxt/proto/monitoring.proto +++ b/bin/fxt/proto/monitoring.proto @@ -63,6 +63,24 @@ message TrainingSession { uint32 hyperopt_trial_total = 21; float hyperopt_best_objective = 22; uint32 hyperopt_trials_failed = 23; + + // RL diagnostics + float q_value_mean = 24; + float q_value_max = 25; + float policy_entropy = 26; + float kl_divergence = 27; + float advantage_mean = 28; + uint32 replay_buffer_size = 29; + + // Gradient & training health + float gradient_norm = 30; + float learning_rate = 31; + float epoch_duration_seconds = 32; + + // Hyperopt intra-trial + uint32 hyperopt_trial_epoch = 33; + float hyperopt_trial_best_loss = 34; + float hyperopt_elapsed_seconds = 35; } message GpuSnapshot { diff --git a/bin/fxt/src/commands/train/monitor.rs b/bin/fxt/src/commands/train/monitor.rs index 410fa4784..b8a13e00f 100644 --- a/bin/fxt/src/commands/train/monitor.rs +++ b/bin/fxt/src/commands/train/monitor.rs @@ -98,6 +98,7 @@ fn render_snapshot(resp: &GetLiveTrainingMetricsResponse) { } print_session_table(&resp.sessions); + print_rl_diagnostics(&resp.sessions); print_hyperopt_summary(&resp.sessions); print_health_summary(&resp.sessions); } @@ -126,16 +127,17 @@ fn render_tui(resp: &GetLiveTrainingMetricsResponse) { fn print_session_table(sessions: &[TrainingSession]) { println!( - "{:<10} {:<6} {:<7} {:<10} {:<10} {:<9} {:<10}", + "{:<10} {:<6} {:<7} {:<10} {:<10} {:<9} {:<10} {:<10}", "Model".bright_cyan(), "Fold".bright_cyan(), "Epoch".bright_cyan(), "Loss".bright_cyan(), "Val Loss".bright_cyan(), "Batch/s".bright_cyan(), + "Grad Norm".bright_cyan(), "Eval Acc".bright_cyan(), ); - println!("{}", "-".repeat(72).bright_black()); + println!("{}", "-".repeat(82).bright_black()); for s in sessions { let acc = if s.eval_accuracy > 0.0 { @@ -143,14 +145,51 @@ fn print_session_table(sessions: &[TrainingSession]) { } else { "-".to_owned() }; + let grad = if s.gradient_norm > 0.0 { + format!("{:.4}", s.gradient_norm) + } else { + "-".to_owned() + }; println!( - "{:<10} {:<6} {:<7.0} {:<10.4} {:<10.4} {:<9.1} {:<10}", + "{:<10} {:<6} {:<7.0} {:<10.4} {:<10.4} {:<9.1} {:<10} {:<10}", s.model, s.fold, s.current_epoch, s.epoch_loss, s.validation_loss, - s.batches_per_second, acc, + s.batches_per_second, grad, acc, ); } } +fn print_rl_diagnostics(sessions: &[TrainingSession]) { + let rl_sessions: Vec<_> = sessions + .iter() + .filter(|s| s.q_value_mean != 0.0 || s.policy_entropy != 0.0) + .collect(); + if rl_sessions.is_empty() { + return; + } + println!(); + println!("{}", "RL Diagnostics:".bright_cyan()); + for s in &rl_sessions { + if s.q_value_mean != 0.0 || s.q_value_max != 0.0 { + println!( + " {}: Q-mean={:.2} Q-max={:.2} | buffer={}", + s.model.bright_white(), + s.q_value_mean, + s.q_value_max, + s.replay_buffer_size, + ); + } + if s.policy_entropy != 0.0 || s.kl_divergence != 0.0 { + println!( + " {}: entropy={:.3} KL={:.4} adv-mean={:.4}", + s.model.bright_white(), + s.policy_entropy, + s.kl_divergence, + s.advantage_mean, + ); + } + } +} + fn print_hyperopt_summary(sessions: &[TrainingSession]) { let hyperopt: Vec<_> = sessions.iter().filter(|s| s.is_hyperopt).collect(); if hyperopt.is_empty() { @@ -158,13 +197,28 @@ fn print_hyperopt_summary(sessions: &[TrainingSession]) { } println!(); for s in &hyperopt { + let trial_detail = if s.hyperopt_trial_epoch > 0 { + format!( + " (epoch {}, loss {:.4})", + s.hyperopt_trial_epoch, s.hyperopt_trial_best_loss + ) + } else { + String::new() + }; + let elapsed = if s.hyperopt_elapsed_seconds > 0.0 { + format!(" | {:.0}s elapsed", s.hyperopt_elapsed_seconds) + } else { + String::new() + }; println!( - "Hyperopt ({}): trial {}/{} | best Sharpe {:.2} | {} failures", + "Hyperopt ({}): trial {}/{}{} | best Sharpe {:.2} | {} failures{}", s.model.bright_magenta(), s.hyperopt_trial_current, s.hyperopt_trial_total, + trial_detail, s.hyperopt_best_objective, s.hyperopt_trials_failed, + elapsed, ); } }