06cf6bf39e
Adds a new CLI subcommand that connects to the monitoring service via gRPC to display live training metrics. Supports single-snapshot mode (--once), model filtering (--model), and configurable streaming interval. Renders per-model epoch/loss table, GPU telemetry, hyperopt progress, and health counters. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
75 lines
1.8 KiB
Protocol Buffer
75 lines
1.8 KiB
Protocol Buffer
syntax = "proto3";
|
|
package monitoring;
|
|
|
|
service MonitoringService {
|
|
// Single snapshot of all active training sessions
|
|
rpc GetLiveTrainingMetrics(GetLiveTrainingMetricsRequest)
|
|
returns (GetLiveTrainingMetricsResponse);
|
|
|
|
// Server-streaming: pushes updates every N seconds
|
|
rpc StreamTrainingMetrics(StreamTrainingMetricsRequest)
|
|
returns (stream GetLiveTrainingMetricsResponse);
|
|
}
|
|
|
|
message GetLiveTrainingMetricsRequest {
|
|
string model_filter = 1; // optional: "dqn", "ppo", etc.
|
|
}
|
|
|
|
message StreamTrainingMetricsRequest {
|
|
string model_filter = 1;
|
|
uint32 interval_seconds = 2; // 0 = server default (3s)
|
|
}
|
|
|
|
message GetLiveTrainingMetricsResponse {
|
|
repeated TrainingSession sessions = 1;
|
|
GpuSnapshot gpu = 2;
|
|
uint32 active_k8s_jobs = 3;
|
|
int64 timestamp = 4;
|
|
}
|
|
|
|
message TrainingSession {
|
|
string model = 1;
|
|
string fold = 2;
|
|
bool is_hyperopt = 3;
|
|
|
|
// Epoch/progress
|
|
float current_epoch = 4;
|
|
float epoch_loss = 5;
|
|
float validation_loss = 6;
|
|
|
|
// Throughput
|
|
float batches_per_second = 7;
|
|
float batches_processed = 8;
|
|
float iteration_seconds = 9;
|
|
|
|
// Eval metrics
|
|
float eval_accuracy = 10;
|
|
float eval_precision = 11;
|
|
float eval_recall = 12;
|
|
float eval_f1 = 13;
|
|
|
|
// Checkpoint
|
|
float checkpoint_size_bytes = 14;
|
|
uint32 checkpoint_saves = 15;
|
|
uint32 checkpoint_failures = 16;
|
|
|
|
// Health counters
|
|
uint32 nan_detected = 17;
|
|
uint32 gradient_explosions = 18;
|
|
uint32 feature_errors = 19;
|
|
|
|
// Hyperopt fields (populated when is_hyperopt=true)
|
|
uint32 hyperopt_trial_current = 20;
|
|
uint32 hyperopt_trial_total = 21;
|
|
float hyperopt_best_objective = 22;
|
|
uint32 hyperopt_trials_failed = 23;
|
|
}
|
|
|
|
message GpuSnapshot {
|
|
float utilization_percent = 1;
|
|
float memory_used_mb = 2;
|
|
float memory_total_mb = 3;
|
|
float temperature_celsius = 4;
|
|
float power_watts = 5;
|
|
}
|