93 lines
2.2 KiB
Protocol Buffer
93 lines
2.2 KiB
Protocol Buffer
syntax = "proto3";
|
|
package monitoring;
|
|
|
|
service MonitoringService {
|
|
// Single snapshot of all active training sessions
|
|
rpc GetLiveTrainingMetrics(GetLiveTrainingMetricsRequest)
|
|
returns (GetLiveTrainingMetricsResponse);
|
|
|
|
// Server-streaming: pushes updates every N seconds
|
|
rpc StreamTrainingMetrics(StreamTrainingMetricsRequest)
|
|
returns (stream GetLiveTrainingMetricsResponse);
|
|
}
|
|
|
|
message GetLiveTrainingMetricsRequest {
|
|
string model_filter = 1; // optional: "dqn", "ppo", etc.
|
|
}
|
|
|
|
message StreamTrainingMetricsRequest {
|
|
string model_filter = 1;
|
|
uint32 interval_seconds = 2; // 0 = server default (3s)
|
|
}
|
|
|
|
message GetLiveTrainingMetricsResponse {
|
|
repeated TrainingSession sessions = 1;
|
|
GpuSnapshot gpu = 2;
|
|
uint32 active_k8s_jobs = 3;
|
|
int64 timestamp = 4;
|
|
}
|
|
|
|
message TrainingSession {
|
|
string model = 1;
|
|
string fold = 2;
|
|
bool is_hyperopt = 3;
|
|
|
|
// Epoch/progress
|
|
float current_epoch = 4;
|
|
float epoch_loss = 5;
|
|
float validation_loss = 6;
|
|
|
|
// Throughput
|
|
float batches_per_second = 7;
|
|
float batches_processed = 8;
|
|
float iteration_seconds = 9;
|
|
|
|
// Eval metrics
|
|
float eval_accuracy = 10;
|
|
float eval_precision = 11;
|
|
float eval_recall = 12;
|
|
float eval_f1 = 13;
|
|
|
|
// Checkpoint
|
|
float checkpoint_size_bytes = 14;
|
|
uint32 checkpoint_saves = 15;
|
|
uint32 checkpoint_failures = 16;
|
|
|
|
// Health counters
|
|
uint32 nan_detected = 17;
|
|
uint32 gradient_explosions = 18;
|
|
uint32 feature_errors = 19;
|
|
|
|
// Hyperopt fields (populated when is_hyperopt=true)
|
|
uint32 hyperopt_trial_current = 20;
|
|
uint32 hyperopt_trial_total = 21;
|
|
float hyperopt_best_objective = 22;
|
|
uint32 hyperopt_trials_failed = 23;
|
|
|
|
// RL diagnostics
|
|
float q_value_mean = 24;
|
|
float q_value_max = 25;
|
|
float policy_entropy = 26;
|
|
float kl_divergence = 27;
|
|
float advantage_mean = 28;
|
|
uint32 replay_buffer_size = 29;
|
|
|
|
// Gradient & training health
|
|
float gradient_norm = 30;
|
|
float learning_rate = 31;
|
|
float epoch_duration_seconds = 32;
|
|
|
|
// Hyperopt intra-trial
|
|
uint32 hyperopt_trial_epoch = 33;
|
|
float hyperopt_trial_best_loss = 34;
|
|
float hyperopt_elapsed_seconds = 35;
|
|
}
|
|
|
|
message GpuSnapshot {
|
|
float utilization_percent = 1;
|
|
float memory_used_mb = 2;
|
|
float memory_total_mb = 3;
|
|
float temperature_celsius = 4;
|
|
float power_watts = 5;
|
|
}
|