From 0c227bb7d0a08dc18ca3ed3426c41dc770f217f9 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Mon, 2 Mar 2026 21:05:48 +0100 Subject: [PATCH] =?UTF-8?q?docs:=20add=20monitoring=20service=20design=20?= =?UTF-8?q?=E2=80=94=20live=20training=20metrics=20over=20gRPC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New standalone monitoring_service that bridges Prometheus → gRPC, enabling `fxt train monitor` and web-gateway WS streaming for all training jobs (CI and service-managed). Co-Authored-By: Claude Opus 4.6 --- .../2026-03-02-monitoring-service-design.md | 222 ++++++++++++++++++ 1 file changed, 222 insertions(+) create mode 100644 docs/plans/2026-03-02-monitoring-service-design.md diff --git a/docs/plans/2026-03-02-monitoring-service-design.md b/docs/plans/2026-03-02-monitoring-service-design.md new file mode 100644 index 000000000..39106b8fd --- /dev/null +++ b/docs/plans/2026-03-02-monitoring-service-design.md @@ -0,0 +1,222 @@ +# Monitoring Service Design — Live Training Metrics over gRPC + +**Date**: 2026-03-02 +**Status**: Draft + +## Problem + +CI training jobs (hyperopt, train_baseline) run as standalone K8s pods managed by GitLab CI. They expose Prometheus metrics on port 9094 but are invisible to `fxt train status` which requires the ml_training_service gRPC chain. There is no native way to monitor live training metrics from the CLI or web dashboard for CI-triggered jobs. + +## Architecture + +``` +fxt CLI ─────gRPC──────▶ monitoring_service ──HTTP──▶ Prometheus API + (port 50057) (in-cluster) + │ +web-gateway ──gRPC─────────────┘ ┌── foxhunt_training_* (training binaries) + (WS: training_progress topic) ├── foxhunt_hyperopt_* (hyperopt binaries) + ├── dcgm_* (GPU exporter) + └── kube_job_* (kube-state-metrics) +``` + +- `fxt` stays 100% gRPC — no Prometheus dependency on the client +- `monitoring_service` is the single Prometheus-to-gRPC bridge +- `web-gateway` connects as a gRPC client and forwards to WS `training_progress` topic + +## Components + +### 1. New Prometheus metrics: hyperopt (`common::metrics::training_metrics`) + +Add 6 new metrics to the existing 18: + +| Metric | Type | Labels | Description | +|--------|------|--------|-------------| +| `foxhunt_hyperopt_trial_current` | Gauge | model | Current trial number (1-indexed) | +| `foxhunt_hyperopt_trial_total` | Gauge | model | Total trials configured | +| `foxhunt_hyperopt_best_objective` | Gauge | model | Best objective value so far (Sharpe) | +| `foxhunt_hyperopt_trial_duration_seconds` | Histogram | model | Duration of each trial | +| `foxhunt_hyperopt_trials_failed` | Counter | model | Failed trials count | +| `foxhunt_hyperopt_mode` | Gauge | model | 1=hyperopt active, 0=training | + +Typed helpers in `common::metrics::training_metrics`: +```rust +pub fn set_hyperopt_trial(model: &str, current: f64, total: f64) { ... } +pub fn set_hyperopt_best_objective(model: &str, value: f64) { ... } +pub fn record_hyperopt_trial_duration(model: &str, secs: f64) { ... } +pub fn record_hyperopt_trial_failure(model: &str) { ... } +pub fn set_hyperopt_mode(model: &str, active: bool) { ... } +``` + +Wire into: `hyperopt_baseline_rl.rs`, `hyperopt_baseline_supervised.rs` + +### 2. New service: `services/monitoring_service/` + +Lightweight Rust service (~500 lines): + +**Dependencies**: axum, tonic, reqwest, serde_json, prometheus (for self-metrics) + +**Modules**: +- `prometheus_client.rs` — HTTP client for Prometheus query API +- `service.rs` — gRPC MonitoringService implementation +- `main.rs` — startup, config + +**Prometheus queries** (single batch per call): +```promql +{__name__=~"foxhunt_training_.*|foxhunt_hyperopt_.*"} +{__name__=~"dcgm_gpu_utilization|dcgm_fb_used|dcgm_fb_free|dcgm_gpu_temp|dcgm_power_usage"} +kube_job_status_active{namespace="foxhunt",job_name=~"training-.*"} +``` + +3 HTTP calls → parse JSON → group by `{model, fold}` → build proto response. + +**Config (env vars)**: +- `PROMETHEUS_URL` — default `http://gitlab-prometheus-server.foxhunt.svc:80` +- `GRPC_PORT` — default `50057` +- `METRICS_PORT` — default `9099` +- `STREAM_INTERVAL_SECS` — default `3` + +### 3. Proto: `services/monitoring_service/proto/monitoring.proto` + +```protobuf +syntax = "proto3"; +package monitoring; + +service MonitoringService { + // Single snapshot of all active training sessions + rpc GetLiveTrainingMetrics(GetLiveTrainingMetricsRequest) + returns (GetLiveTrainingMetricsResponse); + + // Server-streaming: pushes updates every N seconds + rpc StreamTrainingMetrics(StreamTrainingMetricsRequest) + returns (stream GetLiveTrainingMetricsResponse); +} + +message GetLiveTrainingMetricsRequest { + string model_filter = 1; // optional: "dqn", "ppo", etc. +} + +message StreamTrainingMetricsRequest { + string model_filter = 1; + uint32 interval_seconds = 2; // 0 = server default (3s) +} + +message GetLiveTrainingMetricsResponse { + repeated TrainingSession sessions = 1; + GpuSnapshot gpu = 2; + uint32 active_k8s_jobs = 3; + int64 timestamp = 4; +} + +message TrainingSession { + string model = 1; + string fold = 2; + bool is_hyperopt = 3; + + // Epoch/progress + float current_epoch = 4; + float epoch_loss = 5; + float validation_loss = 6; + + // Throughput + float batches_per_second = 7; + float batches_processed = 8; + float iteration_seconds = 9; + + // Eval metrics + float eval_accuracy = 10; + float eval_precision = 11; + float eval_recall = 12; + float eval_f1 = 13; + + // Checkpoint + float checkpoint_size_bytes = 14; + uint32 checkpoint_saves = 15; + uint32 checkpoint_failures = 16; + + // Health counters + uint32 nan_detected = 17; + uint32 gradient_explosions = 18; + uint32 feature_errors = 19; + + // Hyperopt fields (populated when is_hyperopt=true) + uint32 hyperopt_trial_current = 20; + uint32 hyperopt_trial_total = 21; + float hyperopt_best_objective = 22; + uint32 hyperopt_trials_failed = 23; +} + +message GpuSnapshot { + float utilization_percent = 1; + float memory_used_mb = 2; + float memory_total_mb = 3; + float temperature_celsius = 4; + float power_watts = 5; +} +``` + +### 4. CLI: `fxt train monitor` + +New subcommand in `bin/fxt/src/commands/train/monitor.rs`: + +``` +fxt train monitor # live TUI (streaming RPC), Ctrl+C to exit +fxt train monitor --once # single snapshot, print and exit +fxt train monitor --model dqn # filter to DQN sessions only +``` + +**Live TUI layout**: +``` +╔═══════════════════════════════════════════════════════════════════════╗ +║ Foxhunt Training Monitor ║ +╚═══════════════════════════════════════════════════════════════════════╝ + +GPU: 87% util │ 38.2/48.0 GB VRAM │ 62°C │ 245W +Active K8s training jobs: 1 + +┌─────────┬──────┬───────┬──────────┬──────────┬─────────┬──────────┐ +│ Model │ Fold │ Epoch │ Loss │ Val Loss │ Batch/s │ Eval Acc │ +├─────────┼──────┼───────┼──────────┼──────────┼─────────┼──────────┤ +│ dqn │ 0 │ 12 │ 0.0034 │ 0.0041 │ 847.2 │ — │ +│ dqn │ 1 │ 8 │ 0.0052 │ 0.0058 │ 812.5 │ — │ +└─────────┴──────┴───────┴──────────┴──────────┴─────────┴──────────┘ + +Hyperopt: trial 7/20 │ best Sharpe 2.34 │ 0 failures +Health: 0 NaN │ 0 grad explosions │ 2 checkpoints +Updated 2s ago │ Ctrl+C to exit +``` + +Uses `StreamTrainingMetrics` RPC for live mode, `GetLiveTrainingMetrics` for `--once`. + +### 5. Web Gateway integration + +**gRPC client**: Add `monitoring_service` client to `crates/web-gateway/src/grpc/clients.rs` + +**WS broadcaster**: Background task polls `GetLiveTrainingMetrics` every 3s, broadcasts `training_progress` topic to subscribed WS clients. The `ServerMessage::TrainingProgress` variant already exists. + +**REST endpoint**: `GET /api/v1/training/live-metrics` — proxies single `GetLiveTrainingMetrics` call. + +**Config**: `MONITORING_SERVICE_URL` env var (default `http://monitoring-service.foxhunt.svc:50057`) + +### 6. Deployment + +**K8s manifest**: `infra/k8s/services/monitoring-service.yaml` +- Deployment: 1 replica, CPU-only (no GPU needed) +- Service: ClusterIP, port 50057 (gRPC) + 9099 (metrics) +- Prometheus annotations: `gitlab.com/prometheus_scrape: "true"`, port 9099 + +**CI**: Binary compiled in existing `compile-services` job, uploaded to S3. + +**fxt config**: Add `monitoring_service` endpoint to `bin/fxt/src/client/mod.rs` ServiceEndpoints. + +## Non-goals + +- Historical metric storage (Prometheus handles retention) +- Replacing Grafana dashboards (complementary, not replacement) +- Managing training jobs (that's ml_training_service's job) + +## Migration + +- Existing `fxt train status` (gRPC to ml_training_service) continues to work unchanged +- `fxt train monitor` is additive — new subcommand +- Web dashboard gets live metrics for free via the new WS data +- Hyperopt metrics are additive (new foxhunt_hyperopt_* gauges)