feat(fxt): add grad norm, RL diagnostics, and hyperopt trial detail to monitor

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-03-03 00:09:56 +01:00
parent ab218121b6
commit 6c7a7275de
2 changed files with 77 additions and 5 deletions

View File

@@ -63,6 +63,24 @@ message TrainingSession {
uint32 hyperopt_trial_total = 21;
float hyperopt_best_objective = 22;
uint32 hyperopt_trials_failed = 23;
// RL diagnostics
float q_value_mean = 24;
float q_value_max = 25;
float policy_entropy = 26;
float kl_divergence = 27;
float advantage_mean = 28;
uint32 replay_buffer_size = 29;
// Gradient & training health
float gradient_norm = 30;
float learning_rate = 31;
float epoch_duration_seconds = 32;
// Hyperopt intra-trial
uint32 hyperopt_trial_epoch = 33;
float hyperopt_trial_best_loss = 34;
float hyperopt_elapsed_seconds = 35;
}
message GpuSnapshot {

View File

@@ -98,6 +98,7 @@ fn render_snapshot(resp: &GetLiveTrainingMetricsResponse) {
}
print_session_table(&resp.sessions);
print_rl_diagnostics(&resp.sessions);
print_hyperopt_summary(&resp.sessions);
print_health_summary(&resp.sessions);
}
@@ -126,16 +127,17 @@ fn render_tui(resp: &GetLiveTrainingMetricsResponse) {
fn print_session_table(sessions: &[TrainingSession]) {
println!(
"{:<10} {:<6} {:<7} {:<10} {:<10} {:<9} {:<10}",
"{:<10} {:<6} {:<7} {:<10} {:<10} {:<9} {:<10} {:<10}",
"Model".bright_cyan(),
"Fold".bright_cyan(),
"Epoch".bright_cyan(),
"Loss".bright_cyan(),
"Val Loss".bright_cyan(),
"Batch/s".bright_cyan(),
"Grad Norm".bright_cyan(),
"Eval Acc".bright_cyan(),
);
println!("{}", "-".repeat(72).bright_black());
println!("{}", "-".repeat(82).bright_black());
for s in sessions {
let acc = if s.eval_accuracy > 0.0 {
@@ -143,14 +145,51 @@ fn print_session_table(sessions: &[TrainingSession]) {
} else {
"-".to_owned()
};
let grad = if s.gradient_norm > 0.0 {
format!("{:.4}", s.gradient_norm)
} else {
"-".to_owned()
};
println!(
"{:<10} {:<6} {:<7.0} {:<10.4} {:<10.4} {:<9.1} {:<10}",
"{:<10} {:<6} {:<7.0} {:<10.4} {:<10.4} {:<9.1} {:<10} {:<10}",
s.model, s.fold, s.current_epoch, s.epoch_loss, s.validation_loss,
s.batches_per_second, acc,
s.batches_per_second, grad, acc,
);
}
}
fn print_rl_diagnostics(sessions: &[TrainingSession]) {
let rl_sessions: Vec<_> = sessions
.iter()
.filter(|s| s.q_value_mean != 0.0 || s.policy_entropy != 0.0)
.collect();
if rl_sessions.is_empty() {
return;
}
println!();
println!("{}", "RL Diagnostics:".bright_cyan());
for s in &rl_sessions {
if s.q_value_mean != 0.0 || s.q_value_max != 0.0 {
println!(
" {}: Q-mean={:.2} Q-max={:.2} | buffer={}",
s.model.bright_white(),
s.q_value_mean,
s.q_value_max,
s.replay_buffer_size,
);
}
if s.policy_entropy != 0.0 || s.kl_divergence != 0.0 {
println!(
" {}: entropy={:.3} KL={:.4} adv-mean={:.4}",
s.model.bright_white(),
s.policy_entropy,
s.kl_divergence,
s.advantage_mean,
);
}
}
}
fn print_hyperopt_summary(sessions: &[TrainingSession]) {
let hyperopt: Vec<_> = sessions.iter().filter(|s| s.is_hyperopt).collect();
if hyperopt.is_empty() {
@@ -158,13 +197,28 @@ fn print_hyperopt_summary(sessions: &[TrainingSession]) {
}
println!();
for s in &hyperopt {
let trial_detail = if s.hyperopt_trial_epoch > 0 {
format!(
" (epoch {}, loss {:.4})",
s.hyperopt_trial_epoch, s.hyperopt_trial_best_loss
)
} else {
String::new()
};
let elapsed = if s.hyperopt_elapsed_seconds > 0.0 {
format!(" | {:.0}s elapsed", s.hyperopt_elapsed_seconds)
} else {
String::new()
};
println!(
"Hyperopt ({}): trial {}/{} | best Sharpe {:.2} | {} failures",
"Hyperopt ({}): trial {}/{}{} | best Sharpe {:.2} | {} failures{}",
s.model.bright_magenta(),
s.hyperopt_trial_current,
s.hyperopt_trial_total,
trial_detail,
s.hyperopt_best_objective,
s.hyperopt_trials_failed,
elapsed,
);
}
}