feat(fxt): add grad norm, RL diagnostics, and hyperopt trial detail to monitor
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -63,6 +63,24 @@ message TrainingSession {
|
||||
uint32 hyperopt_trial_total = 21;
|
||||
float hyperopt_best_objective = 22;
|
||||
uint32 hyperopt_trials_failed = 23;
|
||||
|
||||
// RL diagnostics
|
||||
float q_value_mean = 24;
|
||||
float q_value_max = 25;
|
||||
float policy_entropy = 26;
|
||||
float kl_divergence = 27;
|
||||
float advantage_mean = 28;
|
||||
uint32 replay_buffer_size = 29;
|
||||
|
||||
// Gradient & training health
|
||||
float gradient_norm = 30;
|
||||
float learning_rate = 31;
|
||||
float epoch_duration_seconds = 32;
|
||||
|
||||
// Hyperopt intra-trial
|
||||
uint32 hyperopt_trial_epoch = 33;
|
||||
float hyperopt_trial_best_loss = 34;
|
||||
float hyperopt_elapsed_seconds = 35;
|
||||
}
|
||||
|
||||
message GpuSnapshot {
|
||||
|
||||
@@ -98,6 +98,7 @@ fn render_snapshot(resp: &GetLiveTrainingMetricsResponse) {
|
||||
}
|
||||
|
||||
print_session_table(&resp.sessions);
|
||||
print_rl_diagnostics(&resp.sessions);
|
||||
print_hyperopt_summary(&resp.sessions);
|
||||
print_health_summary(&resp.sessions);
|
||||
}
|
||||
@@ -126,16 +127,17 @@ fn render_tui(resp: &GetLiveTrainingMetricsResponse) {
|
||||
|
||||
fn print_session_table(sessions: &[TrainingSession]) {
|
||||
println!(
|
||||
"{:<10} {:<6} {:<7} {:<10} {:<10} {:<9} {:<10}",
|
||||
"{:<10} {:<6} {:<7} {:<10} {:<10} {:<9} {:<10} {:<10}",
|
||||
"Model".bright_cyan(),
|
||||
"Fold".bright_cyan(),
|
||||
"Epoch".bright_cyan(),
|
||||
"Loss".bright_cyan(),
|
||||
"Val Loss".bright_cyan(),
|
||||
"Batch/s".bright_cyan(),
|
||||
"Grad Norm".bright_cyan(),
|
||||
"Eval Acc".bright_cyan(),
|
||||
);
|
||||
println!("{}", "-".repeat(72).bright_black());
|
||||
println!("{}", "-".repeat(82).bright_black());
|
||||
|
||||
for s in sessions {
|
||||
let acc = if s.eval_accuracy > 0.0 {
|
||||
@@ -143,14 +145,51 @@ fn print_session_table(sessions: &[TrainingSession]) {
|
||||
} else {
|
||||
"-".to_owned()
|
||||
};
|
||||
let grad = if s.gradient_norm > 0.0 {
|
||||
format!("{:.4}", s.gradient_norm)
|
||||
} else {
|
||||
"-".to_owned()
|
||||
};
|
||||
println!(
|
||||
"{:<10} {:<6} {:<7.0} {:<10.4} {:<10.4} {:<9.1} {:<10}",
|
||||
"{:<10} {:<6} {:<7.0} {:<10.4} {:<10.4} {:<9.1} {:<10} {:<10}",
|
||||
s.model, s.fold, s.current_epoch, s.epoch_loss, s.validation_loss,
|
||||
s.batches_per_second, acc,
|
||||
s.batches_per_second, grad, acc,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn print_rl_diagnostics(sessions: &[TrainingSession]) {
|
||||
let rl_sessions: Vec<_> = sessions
|
||||
.iter()
|
||||
.filter(|s| s.q_value_mean != 0.0 || s.policy_entropy != 0.0)
|
||||
.collect();
|
||||
if rl_sessions.is_empty() {
|
||||
return;
|
||||
}
|
||||
println!();
|
||||
println!("{}", "RL Diagnostics:".bright_cyan());
|
||||
for s in &rl_sessions {
|
||||
if s.q_value_mean != 0.0 || s.q_value_max != 0.0 {
|
||||
println!(
|
||||
" {}: Q-mean={:.2} Q-max={:.2} | buffer={}",
|
||||
s.model.bright_white(),
|
||||
s.q_value_mean,
|
||||
s.q_value_max,
|
||||
s.replay_buffer_size,
|
||||
);
|
||||
}
|
||||
if s.policy_entropy != 0.0 || s.kl_divergence != 0.0 {
|
||||
println!(
|
||||
" {}: entropy={:.3} KL={:.4} adv-mean={:.4}",
|
||||
s.model.bright_white(),
|
||||
s.policy_entropy,
|
||||
s.kl_divergence,
|
||||
s.advantage_mean,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn print_hyperopt_summary(sessions: &[TrainingSession]) {
|
||||
let hyperopt: Vec<_> = sessions.iter().filter(|s| s.is_hyperopt).collect();
|
||||
if hyperopt.is_empty() {
|
||||
@@ -158,13 +197,28 @@ fn print_hyperopt_summary(sessions: &[TrainingSession]) {
|
||||
}
|
||||
println!();
|
||||
for s in &hyperopt {
|
||||
let trial_detail = if s.hyperopt_trial_epoch > 0 {
|
||||
format!(
|
||||
" (epoch {}, loss {:.4})",
|
||||
s.hyperopt_trial_epoch, s.hyperopt_trial_best_loss
|
||||
)
|
||||
} else {
|
||||
String::new()
|
||||
};
|
||||
let elapsed = if s.hyperopt_elapsed_seconds > 0.0 {
|
||||
format!(" | {:.0}s elapsed", s.hyperopt_elapsed_seconds)
|
||||
} else {
|
||||
String::new()
|
||||
};
|
||||
println!(
|
||||
"Hyperopt ({}): trial {}/{} | best Sharpe {:.2} | {} failures",
|
||||
"Hyperopt ({}): trial {}/{}{} | best Sharpe {:.2} | {} failures{}",
|
||||
s.model.bright_magenta(),
|
||||
s.hyperopt_trial_current,
|
||||
s.hyperopt_trial_total,
|
||||
trial_detail,
|
||||
s.hyperopt_best_objective,
|
||||
s.hyperopt_trials_failed,
|
||||
elapsed,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user