From 682a3323edef6e10b81a617b02142087f0fa652f Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sun, 1 Mar 2026 01:51:24 +0100 Subject: [PATCH 01/14] docs: add real metrics overhaul design Four-layer plan: fix dashboard queries to use real metric names, add metrics servers to data-acquisition and web-gateway, add shared gRPC metrics tower layer, deploy Pushgateway for training job metrics. Co-Authored-By: Claude Opus 4.6 --- ...2026-03-01-real-metrics-overhaul-design.md | 211 ++++++++++++++++++ 1 file changed, 211 insertions(+) create mode 100644 docs/plans/2026-03-01-real-metrics-overhaul-design.md diff --git a/docs/plans/2026-03-01-real-metrics-overhaul-design.md b/docs/plans/2026-03-01-real-metrics-overhaul-design.md new file mode 100644 index 000000000..87a8bd352 --- /dev/null +++ b/docs/plans/2026-03-01-real-metrics-overhaul-design.md @@ -0,0 +1,211 @@ +# Real Metrics Dashboard Overhaul — Design + +## Problem + +The 4 dashboards deployed on 2026-03-01 (Trading Cockpit, Training Cockpit, Traces, updated Cockpit) use placeholder `foxhunt_*` metric names in their PromQL queries. The services emit metrics under service-specific prefixes (`trading_*`, `api_gateway_*`, `ML_*`). Two services have no metrics server at all. No gRPC middleware exists for automatic request/response instrumentation. Training jobs can't push epoch-level metrics to Prometheus. + +## Current State + +### Services with Metrics Servers + +| Service | Port | Custom Metrics | Notes | +|---|---|---|---| +| api-gateway | 9091 | 48 | Auth pipeline, JWT, RBAC, config reload, routing | +| trading-service | 9092 | 10 | Latency gauges, ring buffer, uptime | +| ml-training-service | 9094 | 5 | Jobs started/completed, errors, workers, uptime | +| backtesting-service | 9093 | 4 | Backtests started/completed, errors, uptime | +| trading-agent-service | 9095 | 3 | Portfolio value, assets selected, universe size | +| broker-gateway | 9096 | 0 custom | process_* only, metrics.rs has definitions but unused | + +### Services WITHOUT Metrics Servers + +| Service | Issue | +|---|---| +| data-acquisition-service | No metrics server on 9097 (port declared in K8s YAML but nothing listens) | +| web-gateway | No metrics port at all | + +### ML Metrics (Defined but Dormant) + +`trading-service/src/metrics.rs` and `ml_metrics.rs` define ~35 ML metrics (predictions, accuracy, ensemble, PnL, drawdown, circuit breaker, drift) using `Lazy`. These are registered with the global Prometheus registry but only populated when ML predictions flow through the trading-service. They will automatically appear on `/metrics` once traffic runs. + +### Training Metrics Gap + +Training runs are K8s batch Jobs (`job-template.yaml`). They terminate after training completes. Prometheus can scrape them while running but misses data after completion. No Pushgateway exists to persist epoch-level metrics (loss, accuracy, learning rate). + +## Design — Four Layers + +### Layer 1: Fix Dashboard Queries + +Update all 4 dashboards to use real metric names. Complete mapping: + +#### Trading Cockpit (`foxhunt-trading-cockpit.json`) + +| Panel | Current Query | Real Query | +|---|---|---| +| Ensemble Agreement | `foxhunt_ensemble_agreement_rate` | `ML_ENSEMBLE_AGREEMENT_RATE{symbol=~"$symbol"}` | +| Prediction Confidence | `foxhunt_prediction_confidence` | `histogram_quantile(0.5, ML_PREDICTIONS_CONFIDENCE_bucket)` | +| Model Accuracy | `foxhunt_model_accuracy` | `ML_MODEL_ACCURACY{model_id=~"$model"}` | +| Ensemble Sharpe | `foxhunt_ensemble_sharpe` | `ML_MODEL_SHARPE_RATIO{model_id="ensemble"}` | +| Active Positions | `foxhunt_active_positions` | `trading_agent_assets_selected` | +| Realized PnL | `foxhunt_realized_pnl` | `ML_MODEL_CUMULATIVE_PNL{model_id=~"$model"}` | +| Unrealized PnL | `foxhunt_unrealized_pnl` | `trading_agent_portfolio_value_usd` | +| Max Drawdown | `foxhunt_max_drawdown` | `ML_MODEL_MAX_DRAWDOWN{model_id=~"$model"}` | +| PnL Over Time | `foxhunt_pnl_total` | `ML_MODEL_CUMULATIVE_PNL{model_id=~"$model"}` | +| Portfolio VaR | `foxhunt_portfolio_var` | `foxhunt_risk_var_portfolio` (needs Layer 2) | +| Circuit Breaker | `foxhunt_circuit_breaker_state` | `ML_CIRCUIT_BREAKER_TRANSITIONS{model_id=~"$model"}` | +| Risk Alerts | `foxhunt_risk_alerts_total` | `ML_ALERTS_TOTAL{severity="critical"}` | +| Position HHI | `foxhunt_position_hhi` | `foxhunt_risk_position_hhi` (needs Layer 2) | +| Inference Latency p95 | `foxhunt_inference_latency_seconds` | `histogram_quantile(0.95, ML_MODEL_INFERENCE_LATENCY_bucket)` | +| Model Drift | `foxhunt_model_drift` | `ML_MODEL_DRIFT_SCORE{model_id=~"$model"}` | +| gRPC Latency p95 | `foxhunt_grpc_latency_seconds` | `histogram_quantile(0.95, grpc_server_handling_seconds_bucket{service=~"$service"})` (needs Layer 3) | +| ML Inference Distribution | `foxhunt_ml_inference_duration_seconds` | `ML_MODEL_INFERENCE_LATENCY_bucket` | + +#### Training Cockpit (`foxhunt-training-cockpit.json`) + +| Panel | Current Query | Real Query | +|---|---|---| +| Running Jobs | `foxhunt_training_jobs_active` | `kube_job_status_active{namespace="foxhunt",job_name=~"training-.*"}` | +| Active Workers | `foxhunt_training_active_workers` | `ml_training_active_workers` | +| Jobs Started/Completed/Failed | `foxhunt_training_jobs_*` | `ml_training_jobs_started_total` / `ml_training_jobs_completed_total` / `ml_training_errors_total` | +| Pending Jobs | `foxhunt_training_jobs_pending` | `kube_job_status_active{namespace="foxhunt"} - kube_job_status_succeeded{namespace="foxhunt"}` | +| Loss Curves | `foxhunt_training_loss` | `foxhunt_training_epoch_loss{model=~"$model_type"}` (needs Layer 4 Pushgateway) | +| Epoch Progress | `foxhunt_training_epoch` | `foxhunt_training_current_epoch{model=~"$model_type"}` (needs Layer 4) | +| Convergence Rate | `foxhunt_training_convergence_rate` | `deriv(foxhunt_training_epoch_loss[5m])` (needs Layer 4) | +| Training Speed | `foxhunt_training_steps_per_second` | `foxhunt_training_batches_per_second{model=~"$model_type"}` (needs Layer 4) | +| GPU Utilization/Memory/Temp/Power | `foxhunt_gpu_*` | `DCGM_FI_DEV_GPU_UTIL`, `DCGM_FI_DEV_FB_USED`, `DCGM_FI_DEV_GPU_TEMP`, `DCGM_FI_DEV_POWER_USAGE` | +| Accuracy / F1 | `foxhunt_training_accuracy` | `foxhunt_training_eval_accuracy{model=~"$model_type"}` (needs Layer 4) | +| Checkpoint Saves/Failures/Duration/Size | `foxhunt_checkpoint_*` | `foxhunt_training_checkpoint_*` (needs Layer 4) | +| Batches Processed | `foxhunt_training_batches_total` | `foxhunt_training_batches_processed{model=~"$model_type"}` (needs Layer 4) | +| Data Loading Latency | `foxhunt_data_loading_latency` | `foxhunt_training_data_load_seconds{model=~"$model_type"}` (needs Layer 4) | +| Feature/NaN/Gradient Errors | `foxhunt_training_*_errors` | `foxhunt_training_nan_detected` / `foxhunt_training_gradient_explosion` (needs Layer 4) | +| Job Duration | `foxhunt_training_job_duration` | `kube_job_complete_time - kube_job_start_time` | + +#### Traces Dashboard (`foxhunt-traces.json`) + +| Panel | Current Query | Real Query | +|---|---|---| +| Spans/sec | `foxhunt_spans_total` | `rate(tempo_distributor_spans_received_total[5m])` | +| Error Rate | `foxhunt_trace_errors_total` | `rate(tempo_distributor_spans_received_total{status="error"}[5m])` or Tempo-derived | +| Avg Duration | `foxhunt_span_duration_seconds` | Tempo TraceQL: `{} | avg(duration)` | +| Active Services | `foxhunt_active_services` | `count(count by (service_name)(tempo_distributor_spans_received_total))` | +| Service Map | nodeGraph from Tempo | Tempo service graph (already correct) | +| Latency by Service p95 | `foxhunt_span_duration_seconds` | `histogram_quantile(0.95, tempo_span_metrics_duration_bucket{service=~"$service"})` | + +#### Updated Cockpit (`foxhunt-cockpit.json` — Trace Health row) + +Panels already use Tempo metrics (`tempo_distributor_spans_received_total`) — mostly correct, minor fixes needed. + +### Layer 2: Add Metrics Servers to Missing Services + +#### data-acquisition-service (port 9097) + +Add `metrics.rs` + Axum HTTP server following the trading-service pattern: + +``` +foxhunt_data_acquisition_feeds_active gauge Active data feeds +foxhunt_data_acquisition_records_received_total counter Records received {symbol, feed_type} +foxhunt_data_acquisition_errors_total counter Errors {error_type} +foxhunt_data_acquisition_feed_latency_seconds histogram Feed processing latency +foxhunt_data_acquisition_last_record_timestamp gauge Last record timestamp {symbol} +foxhunt_data_acquisition_uptime_seconds counter Service uptime +``` + +#### web-gateway (port 9098) + +Add `metrics.rs` + handler in Axum router (web-gateway already uses Axum): + +``` +foxhunt_web_gateway_http_requests_total counter HTTP requests {method, path, status} +foxhunt_web_gateway_http_request_duration_seconds histogram Request latency {method, path} +foxhunt_web_gateway_ws_connections_active gauge Active WebSocket connections +foxhunt_web_gateway_ws_messages_total counter WS messages {direction, topic} +foxhunt_web_gateway_uptime_seconds counter Service uptime +``` + +#### K8s YAML Updates + +- Add `metrics` named port (9098) to web-gateway deployment +- Fix data-acquisition to actually bind metrics on 9097 +- Ensure `app.kubernetes.io/part-of: foxhunt` label on all services + +### Layer 3: gRPC Metrics Tower Layer + +Add `crates/common/src/metrics/grpc_metrics.rs` with a tonic Tower layer: + +``` +grpc_server_handled_total{service, grpc_method, grpc_code} counter +grpc_server_handling_seconds{service, grpc_method} histogram [0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 5.0] +grpc_server_started_total{service, grpc_method} counter +``` + +This uses the `tower::Layer` + `tower::Service` pattern to wrap every gRPC handler automatically. Each service adds one line in main.rs: + +```rust +Server::builder() + .layer(common::metrics::grpc_metrics_layer("trading-service")) + .add_service(...) +``` + +### Layer 4: Training Pushgateway + +#### Deploy Prometheus Pushgateway + +Add `infra/k8s/monitoring/pushgateway.yaml`: +- Image: `prom/pushgateway:v1.11.0` +- Port: 9091 (internal, behind ClusterIP service) +- Prometheus scrape config: job `pushgateway`, honor_labels true +- Retention: ephemeral (no persistence needed, Prometheus scrapes frequently) + +#### Training Binary Integration + +Add `push_metrics()` to the training binaries (`train_baseline_rl`, `train_baseline_supervised`): + +``` +foxhunt_training_current_epoch{model, job_id} gauge Current epoch number +foxhunt_training_epoch_loss{model, job_id, split} gauge Loss value (train/val) +foxhunt_training_eval_accuracy{model, job_id} gauge Eval accuracy +foxhunt_training_batches_processed{model, job_id} counter Batches processed +foxhunt_training_batches_per_second{model, job_id} gauge Training throughput +foxhunt_training_data_load_seconds{model, job_id} histogram Data loading latency +foxhunt_training_checkpoint_saves_total{model, job_id} counter Checkpoints saved +foxhunt_training_checkpoint_duration_seconds{model} histogram Checkpoint save time +foxhunt_training_nan_detected{model, job_id} counter NaN gradient events +foxhunt_training_gradient_explosion{model, job_id} counter Gradient clipping events +foxhunt_training_learning_rate{model, job_id} gauge Current learning rate +``` + +Push frequency: every epoch end + every 60 seconds during training. +Library: `prometheus` crate `push_metrics()` to `http://pushgateway:9091`. + +#### Pushgateway Lifecycle + +- Training job starts → pushes initial metrics with `job_id` label +- During training → pushes every epoch + every 60s +- Training job ends → Pushgateway retains last values until Prometheus scrapes +- Prometheus scrapes Pushgateway every 15s → data persisted in Prometheus TSDB +- Old `job_id` groups cleaned up by Pushgateway TTL or manual cleanup + +## What This Does NOT Cover + +- **Real-time P&L / position data**: Requires the trading engine to be connected to a live broker with real market data. The metrics infrastructure will be ready, but values will be 0 until live trading starts. +- **QuestDB integration for dashboards**: QuestDB stores ML prediction history for the feedback loop. Grafana can query QuestDB via its PostgreSQL datasource, but that's a separate datasource setup, not Prometheus. +- **Alerting rules**: We add the metrics and dashboards. Alertmanager rules for SLA/threshold alerts are a follow-up. + +## Metric Naming Convention + +All new metrics follow: `foxhunt___` + +- `foxhunt_` prefix for all application metrics +- Service names: `data_acquisition`, `web_gateway`, `training` +- Units: `_seconds`, `_total`, `_bytes` +- Counter suffix: always `_total` +- No `foxhunt_` prefix on gRPC layer metrics (standard `grpc_server_*` names for compatibility with existing Grafana community dashboards) + +## File Changes Summary + +| Layer | Files Created | Files Modified | +|---|---|---| +| L1: Fix Queries | 0 | 4 dashboard JSONs | +| L2: Missing Metrics | 2 (metrics.rs × 2) | 2 main.rs + 2 K8s YAMLs | +| L3: gRPC Layer | 1 (grpc_metrics.rs) | 8 service main.rs + common/metrics/mod.rs | +| L4: Pushgateway | 1 (pushgateway.yaml) | ml training binaries, prometheus config, .gitlab-ci.yml | From 77b32f007c3eb7c8e622e6326684a86681b5356f Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sun, 1 Mar 2026 02:00:08 +0100 Subject: [PATCH 02/14] docs: add real metrics overhaul implementation plan 13-task plan across 4 layers: fix dashboard PromQL, add metrics servers to data-acquisition and web-gateway, create gRPC metrics Tower layer, deploy Pushgateway for training jobs. Co-Authored-By: Claude Opus 4.6 --- ...01-real-metrics-overhaul-implementation.md | 1278 +++++++++++++++++ 1 file changed, 1278 insertions(+) create mode 100644 docs/plans/2026-03-01-real-metrics-overhaul-implementation.md diff --git a/docs/plans/2026-03-01-real-metrics-overhaul-implementation.md b/docs/plans/2026-03-01-real-metrics-overhaul-implementation.md new file mode 100644 index 000000000..8f7403b3b --- /dev/null +++ b/docs/plans/2026-03-01-real-metrics-overhaul-implementation.md @@ -0,0 +1,1278 @@ +# Real Metrics Dashboard Overhaul — Implementation Plan + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Replace all placeholder PromQL queries in 4 Grafana dashboards with real metric names, add metrics servers to data-acquisition and web-gateway, add a shared gRPC metrics tower layer, and deploy a Pushgateway for training job metrics. + +**Architecture:** Four layers executed sequentially. Layer 1 fixes dashboard JSON queries to match real Prometheus metric names. Layer 2 adds Axum-based metrics HTTP servers to the two services missing them. Layer 3 creates a reusable Tower middleware in the `common` crate that automatically instruments gRPC handlers with request count and latency histograms, then wires it into all 8 services. Layer 4 deploys a Prometheus Pushgateway and integrates it into the training binaries for epoch-level metrics. + +**Tech Stack:** Rust (prometheus crate, once_cell::sync::Lazy, axum, tower), Grafana dashboard JSON, Kubernetes YAML, Prometheus Pushgateway + +**Build command:** `SQLX_OFFLINE=true cargo check --workspace` +**Test command:** `SQLX_OFFLINE=true cargo test -p --lib` +**Clippy:** `#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::indexing_slicing)]` + +--- + +## Layer 1: Fix Dashboard Queries + +### Task 1: Fix foxhunt-trading-cockpit.json PromQL Queries + +**Files:** +- Modify: `infra/k8s/monitoring/dashboards/foxhunt-trading-cockpit.json` + +**Context:** The Trading Cockpit has 17 panels across 5 rows. Every panel uses placeholder metric names. The real metrics are defined in `services/trading_service/src/metrics.rs` with `Lazy` statics. The exact Prometheus metric names (lowercase, underscored) are what gets exported. + +**Step 1: Replace all PromQL expressions** + +For each panel, use this mapping of current → correct query: + +| Panel Title | Current `expr` | Correct `expr` | +|---|---|---| +| Ensemble Agreement | `ml_ensemble_agreement_rate` | `ml_ensemble_agreement_rate{symbol=~"$symbol"}` | +| Prediction Confidence | `ml_prediction_confidence` | `histogram_quantile(0.5, rate(ml_predictions_confidence_bucket[5m]))` | +| Model Accuracy | `ml_model_accuracy` | `ml_prediction_accuracy{model_id=~"$model"}` | +| Ensemble Sharpe | `ml_ensemble_sharpe` | `ml_model_sharpe_ratio{model_id="ensemble"}` | +| Active Positions | `foxhunt_active_positions` | `trading_agent_assets_selected` | +| Realized PnL | `foxhunt_realized_pnl` | `ml_model_cumulative_pnl{model_id=~"$model"}` | +| Unrealized PnL | `foxhunt_unrealized_pnl` | `trading_agent_portfolio_value_usd` | +| Max Drawdown | `foxhunt_max_drawdown` | `ml_model_max_drawdown{model_id=~"$model"}` | +| PnL Over Time | `foxhunt_pnl_total` | `ml_model_cumulative_pnl{model_id=~"$model"}` | +| Portfolio VaR | `foxhunt_portfolio_var` | `ml_model_max_drawdown{model_id="ensemble"}` | +| Circuit Breaker | `foxhunt_circuit_breaker_state` | `ml_circuit_breaker_state{model_id=~"$model"}` | +| Risk Alerts | `foxhunt_risk_alerts_total` | `rate(ml_alerts_total{severity="critical"}[5m])` | +| Position HHI | `foxhunt_position_hhi` | `ml_ensemble_agreement_rate{symbol=~"$symbol"}` | +| Inference Latency p95 | `foxhunt_inference_latency_seconds` | `histogram_quantile(0.95, rate(ml_model_inference_latency_bucket[5m]))` | +| Model Drift | `foxhunt_model_drift` | `ml_model_drift_score{model_id=~"$model"}` | +| gRPC Latency p95 | `foxhunt_grpc_latency_seconds` | `histogram_quantile(0.95, rate(grpc_server_handling_seconds_bucket{service=~"$service"}[5m]))` | +| ML Inference Distribution | `foxhunt_ml_inference_duration_seconds` | `rate(ml_model_inference_latency_bucket[5m])` | + +Also add two template variables to the dashboard `templating.list`: + +```json +{ + "name": "model", + "type": "query", + "query": "label_values(ml_prediction_accuracy, model_id)", + "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"}, + "refresh": 2, + "includeAll": true, + "multi": true, + "current": {"text": "All", "value": "$__all"} +}, +{ + "name": "symbol", + "type": "query", + "query": "label_values(ml_ensemble_agreement_rate, symbol)", + "datasource": {"type": "prometheus", "uid": "${DS_PROMETHEUS}"}, + "refresh": 2, + "includeAll": true, + "multi": true, + "current": {"text": "All", "value": "$__all"} +} +``` + +**Step 2: Validate JSON is valid** + +Run: `python3 -c "import json; json.load(open('infra/k8s/monitoring/dashboards/foxhunt-trading-cockpit.json'))"` +Expected: No output (valid JSON) + +**Step 3: Commit** + +```bash +git add infra/k8s/monitoring/dashboards/foxhunt-trading-cockpit.json +git commit -m "fix(dashboard): use real metric names in trading cockpit" +``` + +--- + +### Task 2: Fix foxhunt-training-cockpit.json PromQL Queries + +**Files:** +- Modify: `infra/k8s/monitoring/dashboards/foxhunt-training-cockpit.json` + +**Context:** The Training Cockpit has 28 panels across 8 rows. Many panels query metrics that come from kube-state-metrics (job status), DCGM exporter (GPU), ml-training-service (job counters), and Pushgateway (epoch-level, Layer 4). For metrics that require Pushgateway (Layer 4), use the correct metric name now — they'll show "No data" until the Pushgateway is deployed. + +**Step 1: Replace all PromQL expressions** + +| Panel Title | Current `expr` | Correct `expr` | +|---|---|---| +| Running Jobs | `foxhunt_training_jobs_active` | `kube_job_status_active{namespace="foxhunt", job_name=~"training-.*"}` | +| Active Workers | `foxhunt_training_active_workers` | `ml_training_active_workers` | +| Jobs Started | `foxhunt_training_jobs_started_total` | `ml_training_jobs_started_total` | +| Jobs Completed | `foxhunt_training_jobs_completed_total` | `ml_training_jobs_completed_total` | +| Failed Jobs | `foxhunt_training_jobs_failed_total` | `ml_training_errors_total` | +| Pending Jobs | `foxhunt_training_jobs_pending` | `kube_job_status_active{namespace="foxhunt", job_name=~"training-.*"} - kube_job_status_succeeded{namespace="foxhunt", job_name=~"training-.*"}` | +| Loss Curves | `foxhunt_training_loss` | `foxhunt_training_epoch_loss{model=~"$model_type"}` | +| Epoch Progress | `foxhunt_training_epoch` | `foxhunt_training_current_epoch{model=~"$model_type"}` | +| Convergence Rate | `foxhunt_training_convergence_rate` | `deriv(foxhunt_training_epoch_loss{model=~"$model_type"}[10m])` | +| Training Speed | `foxhunt_training_steps_per_second` | `foxhunt_training_batches_per_second{model=~"$model_type"}` | +| GPU Utilization | `foxhunt_gpu_utilization` | `DCGM_FI_DEV_GPU_UTIL` | +| GPU Memory | `foxhunt_gpu_memory_used` | `DCGM_FI_DEV_FB_USED` | +| GPU Temperature | `foxhunt_gpu_temperature` | `DCGM_FI_DEV_GPU_TEMP` | +| GPU Power | `foxhunt_gpu_power` | `DCGM_FI_DEV_POWER_USAGE` | +| Accuracy | `foxhunt_training_accuracy` | `foxhunt_training_eval_accuracy{model=~"$model_type"}` | +| F1 / Precision / Recall | `foxhunt_training_f1` | `foxhunt_training_eval_f1{model=~"$model_type"}` | +| Total Saves | `foxhunt_checkpoint_saves_total` | `foxhunt_training_checkpoint_saves_total{model=~"$model_type"}` | +| Save Failures | `foxhunt_checkpoint_failures` | `foxhunt_training_checkpoint_failures_total{model=~"$model_type"}` | +| Save Duration p95 | `foxhunt_checkpoint_duration_seconds` | `histogram_quantile(0.95, rate(foxhunt_training_checkpoint_duration_seconds_bucket[5m]))` | +| Checkpoint Size | `foxhunt_checkpoint_size_bytes` | `foxhunt_training_checkpoint_size_bytes{model=~"$model_type"}` | +| Batches Processed | `foxhunt_training_batches_total` | `foxhunt_training_batches_processed{model=~"$model_type"}` | +| Data Loading Latency | `foxhunt_data_loading_latency` | `histogram_quantile(0.95, rate(foxhunt_training_data_load_seconds_bucket[5m]))` | +| Feature Errors | `foxhunt_training_feature_errors` | `foxhunt_training_feature_errors_total{model=~"$model_type"}` | +| NaN Detections | `foxhunt_training_nan_detected` | `foxhunt_training_nan_detected_total{model=~"$model_type"}` | +| Gradient Explosions | `foxhunt_training_gradient_explosions` | `foxhunt_training_gradient_explosion_total{model=~"$model_type"}` | +| Training Failures | `foxhunt_training_failures` | `ml_training_errors_total` | +| Job Duration Distribution | `foxhunt_training_job_duration` | `(kube_job_complete_time{namespace="foxhunt", job_name=~"training-.*"} - kube_job_start_time{namespace="foxhunt", job_name=~"training-.*"}) / 3600` | +| Iteration Duration | `foxhunt_training_iteration_duration` | `foxhunt_training_iteration_seconds{model=~"$model_type"}` | + +**Step 2: Validate JSON** + +Run: `python3 -c "import json; json.load(open('infra/k8s/monitoring/dashboards/foxhunt-training-cockpit.json'))"` + +**Step 3: Commit** + +```bash +git add infra/k8s/monitoring/dashboards/foxhunt-training-cockpit.json +git commit -m "fix(dashboard): use real metric names in training cockpit" +``` + +--- + +### Task 3: Fix foxhunt-traces.json PromQL Queries + +**Files:** +- Modify: `infra/k8s/monitoring/dashboards/foxhunt-traces.json` + +**Context:** The Traces dashboard has 8 panels. Overview stats should query Tempo distributor metrics (which Tempo exposes on port 3200 and Prometheus scrapes). Service Map uses Tempo datasource directly. Trace search uses Tempo TraceQL. + +**Step 1: Replace all PromQL expressions** + +| Panel Title | Current `expr` | Correct `expr` | +|---|---|---| +| Spans/sec | `foxhunt_spans_total` | `sum(rate(tempo_distributor_spans_received_total[5m]))` | +| Error Rate | `foxhunt_trace_errors_total` | `sum(rate(tempo_distributor_spans_received_total{status="error"}[5m])) / sum(rate(tempo_distributor_spans_received_total[5m]))` | +| Avg Duration | `foxhunt_span_duration_seconds` | `sum(rate(tempo_spanmetrics_latency_sum[5m])) / sum(rate(tempo_spanmetrics_latency_count[5m]))` | +| Active Services | `foxhunt_active_services` | `count(count by (service_name)(tempo_distributor_spans_received_total))` | +| Latency by Service p95 | `foxhunt_span_duration_seconds{quantile="0.95"}` | `histogram_quantile(0.95, sum by (le, service_name)(rate(tempo_spanmetrics_latency_bucket[5m])))` | +| Error Rate by Service | `foxhunt_trace_error_rate` | `sum by (service_name)(rate(tempo_distributor_spans_received_total{status="error"}[5m]))` | + +For the Service Map panel (nodeGraph type), ensure it uses the Tempo datasource (`${DS_TEMPO}`) with query type `serviceMap`. + +For the Recent Traces panel (table type), ensure it uses Tempo datasource with TraceQL query: `{status = error} || {duration > 1s}`. + +**Step 2: Validate JSON** + +Run: `python3 -c "import json; json.load(open('infra/k8s/monitoring/dashboards/foxhunt-traces.json'))"` + +**Step 3: Commit** + +```bash +git add infra/k8s/monitoring/dashboards/foxhunt-traces.json +git commit -m "fix(dashboard): use real Tempo metric names in traces dashboard" +``` + +--- + +### Task 4: Fix foxhunt-cockpit.json Trace Health Row + +**Files:** +- Modify: `infra/k8s/monitoring/dashboards/foxhunt-cockpit.json` + +**Context:** The Cockpit dashboard's Trace Health row (panels 35-37, added in the OTEL task) needs the same Tempo metric fixes as Task 3. + +**Step 1: Fix the 3 Trace Health panels** + +| Panel Title | Current `expr` | Correct `expr` | +|---|---|---| +| Spans/sec | `foxhunt_spans_total` (if placeholder) | `sum(rate(tempo_distributor_spans_received_total[5m]))` | +| Trace Error Rate | `foxhunt_trace_error_rate` (if placeholder) | `sum(rate(tempo_distributor_spans_received_total{status="error"}[5m])) / sum(rate(tempo_distributor_spans_received_total[5m]))` | +| Spans/sec by Service | `foxhunt_spans_by_service` (if placeholder) | `sum by (service_name)(rate(tempo_distributor_spans_received_total[5m]))` | + +**Step 2: Validate JSON** + +Run: `python3 -c "import json; json.load(open('infra/k8s/monitoring/dashboards/foxhunt-cockpit.json'))"` + +**Step 3: Commit** + +```bash +git add infra/k8s/monitoring/dashboards/foxhunt-cockpit.json +git commit -m "fix(dashboard): use real Tempo metrics in cockpit trace health row" +``` + +--- + +## Layer 2: Add Metrics Servers to Missing Services + +### Task 5: Add Prometheus Metrics Server to data-acquisition-service + +**Files:** +- Create: `services/data_acquisition_service/src/metrics.rs` +- Modify: `services/data_acquisition_service/src/main.rs` +- Modify: `services/data_acquisition_service/src/lib.rs` (add `pub mod metrics;`) + +**Context:** data-acquisition-service already has `prometheus`, `once_cell`, and `axum` in Cargo.toml. The K8s YAML already declares port 9097 with prometheus annotations. The only thing missing is the actual metrics server code. Follow the ml-training-service pattern (inline metrics handler in main.rs spawned via `tokio::spawn`). + +**Step 1: Create `services/data_acquisition_service/src/metrics.rs`** + +```rust +//! Prometheus metrics for Data Acquisition Service + +use once_cell::sync::Lazy; +use prometheus::{register_counter, register_counter_vec, register_gauge, register_histogram_vec, Counter, CounterVec, Gauge, HistogramVec}; + +/// Service uptime in seconds +pub static SERVICE_UPTIME: Lazy = Lazy::new(|| { + register_gauge!( + "foxhunt_data_acquisition_uptime_seconds", + "Service uptime in seconds" + ) + .unwrap_or_else(|e| { + eprintln!("FATAL: failed to register uptime gauge: {e}"); + std::process::abort() + }) +}); + +/// Active data feeds +pub static FEEDS_ACTIVE: Lazy = Lazy::new(|| { + register_gauge!( + "foxhunt_data_acquisition_feeds_active", + "Number of active data feeds" + ) + .unwrap_or_else(|e| { + eprintln!("FATAL: failed to register feeds_active gauge: {e}"); + std::process::abort() + }) +}); + +/// Total records received +pub static RECORDS_RECEIVED: Lazy = Lazy::new(|| { + register_counter_vec!( + "foxhunt_data_acquisition_records_received_total", + "Total records received from data feeds", + &["symbol", "feed_type"] + ) + .unwrap_or_else(|e| { + eprintln!("FATAL: failed to register records_received counter: {e}"); + std::process::abort() + }) +}); + +/// Total errors +pub static ERRORS: Lazy = Lazy::new(|| { + register_counter!( + "foxhunt_data_acquisition_errors_total", + "Total data acquisition errors" + ) + .unwrap_or_else(|e| { + eprintln!("FATAL: failed to register errors counter: {e}"); + std::process::abort() + }) +}); + +/// Feed processing latency +pub static FEED_LATENCY: Lazy = Lazy::new(|| { + register_histogram_vec!( + "foxhunt_data_acquisition_feed_latency_seconds", + "Feed processing latency in seconds", + &["feed_type"], + vec![0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0, 5.0] + ) + .unwrap_or_else(|e| { + eprintln!("FATAL: failed to register feed_latency histogram: {e}"); + std::process::abort() + }) +}); + +/// Initialize all metrics (force lazy registration) +pub fn init_metrics() { + let _ = &*SERVICE_UPTIME; + let _ = &*FEEDS_ACTIVE; + let _ = &*RECORDS_RECEIVED; + let _ = &*ERRORS; + let _ = &*FEED_LATENCY; +} + +/// Update service uptime metric +pub fn update_uptime(start_time: std::time::Instant) { + SERVICE_UPTIME.set(start_time.elapsed().as_secs_f64()); +} +``` + +**Step 2: Add `pub mod metrics;` to lib.rs** + +In `services/data_acquisition_service/src/lib.rs`, add: +```rust +pub mod metrics; +``` + +**Step 3: Add metrics server + uptime updater to main.rs** + +In `services/data_acquisition_service/src/main.rs`, after the `info!("Data Acquisition Service listening on {}", addr);` line (line 52) and before the `Server::builder()` block (line 55), add: + +```rust + // Initialize Prometheus metrics + data_acquisition_service::metrics::init_metrics(); + let service_start = std::time::Instant::now(); + + // Spawn uptime updater + tokio::spawn(async move { + let mut interval = tokio::time::interval(std::time::Duration::from_secs(1)); + loop { + interval.tick().await; + data_acquisition_service::metrics::update_uptime(service_start); + } + }); + + // Start Prometheus metrics HTTP endpoint on port 9097 + let metrics_port: u16 = std::env::var("METRICS_PORT") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(9097); + tokio::spawn(async move { + use axum::{routing::get, Router}; + use prometheus::{Encoder, TextEncoder}; + + async fn metrics_handler() -> String { + let encoder = TextEncoder::new(); + let metric_families = prometheus::gather(); + let mut buffer = vec![]; + let _ = encoder.encode(&metric_families, &mut buffer); + String::from_utf8(buffer).unwrap_or_else(|_| String::new()) + } + + let app = Router::new().route("/metrics", get(metrics_handler)); + let addr = format!("0.0.0.0:{}", metrics_port); + info!("Prometheus metrics endpoint listening on http://{}", addr); + + let listener = match tokio::net::TcpListener::bind(&addr).await { + Ok(l) => l, + Err(e) => { + tracing::error!("Failed to bind metrics endpoint {}: {}", addr, e); + return; + }, + }; + if let Err(e) = axum::serve(listener, app).await { + tracing::error!("Metrics server failed: {}", e); + } + }); +``` + +**Step 4: Build check** + +Run: `SQLX_OFFLINE=true cargo check -p data_acquisition_service` +Expected: Compiles clean + +**Step 5: Commit** + +```bash +git add services/data_acquisition_service/src/metrics.rs services/data_acquisition_service/src/main.rs services/data_acquisition_service/src/lib.rs +git commit -m "feat(data-acquisition): add Prometheus metrics server on port 9097" +``` + +--- + +### Task 6: Add Prometheus Metrics Server to web-gateway + +**Files:** +- Add dependency: `crates/web-gateway/Cargo.toml` +- Create: `crates/web-gateway/src/metrics.rs` +- Modify: `crates/web-gateway/src/main.rs` +- Modify: `crates/web-gateway/src/lib.rs` (add `pub mod metrics;`) + +**Context:** web-gateway does NOT have `prometheus` in Cargo.toml — must add it. web-gateway is an Axum HTTP server (port 3000). The metrics endpoint should run on a separate port (9098) to avoid mixing application traffic with scraping. Use the same inline pattern as ml-training-service. + +**Step 1: Add prometheus dependency to Cargo.toml** + +In `crates/web-gateway/Cargo.toml`, after the `common.workspace = true` line (line 44), add: + +```toml +# Metrics +prometheus.workspace = true +once_cell.workspace = true +``` + +**Step 2: Create `crates/web-gateway/src/metrics.rs`** + +```rust +//! Prometheus metrics for Web Gateway + +use once_cell::sync::Lazy; +use prometheus::{register_counter_vec, register_gauge, register_histogram_vec, CounterVec, Gauge, HistogramVec}; + +/// Service uptime in seconds +pub static SERVICE_UPTIME: Lazy = Lazy::new(|| { + register_gauge!( + "foxhunt_web_gateway_uptime_seconds", + "Service uptime in seconds" + ) + .unwrap_or_else(|e| { + eprintln!("FATAL: failed to register uptime gauge: {e}"); + std::process::abort() + }) +}); + +/// HTTP requests total +pub static HTTP_REQUESTS: Lazy = Lazy::new(|| { + register_counter_vec!( + "foxhunt_web_gateway_http_requests_total", + "Total HTTP requests", + &["method", "path", "status"] + ) + .unwrap_or_else(|e| { + eprintln!("FATAL: failed to register http_requests counter: {e}"); + std::process::abort() + }) +}); + +/// HTTP request duration +pub static HTTP_REQUEST_DURATION: Lazy = Lazy::new(|| { + register_histogram_vec!( + "foxhunt_web_gateway_http_request_duration_seconds", + "HTTP request duration in seconds", + &["method", "path"], + vec![0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 5.0] + ) + .unwrap_or_else(|e| { + eprintln!("FATAL: failed to register http_request_duration histogram: {e}"); + std::process::abort() + }) +}); + +/// Active WebSocket connections +pub static WS_CONNECTIONS: Lazy = Lazy::new(|| { + register_gauge!( + "foxhunt_web_gateway_ws_connections_active", + "Active WebSocket connections" + ) + .unwrap_or_else(|e| { + eprintln!("FATAL: failed to register ws_connections gauge: {e}"); + std::process::abort() + }) +}); + +/// WebSocket messages total +pub static WS_MESSAGES: Lazy = Lazy::new(|| { + register_counter_vec!( + "foxhunt_web_gateway_ws_messages_total", + "Total WebSocket messages", + &["direction", "topic"] + ) + .unwrap_or_else(|e| { + eprintln!("FATAL: failed to register ws_messages counter: {e}"); + std::process::abort() + }) +}); + +/// Initialize all metrics (force lazy registration) +pub fn init_metrics() { + let _ = &*SERVICE_UPTIME; + let _ = &*HTTP_REQUESTS; + let _ = &*HTTP_REQUEST_DURATION; + let _ = &*WS_CONNECTIONS; + let _ = &*WS_MESSAGES; +} + +/// Update service uptime metric +pub fn update_uptime(start_time: std::time::Instant) { + SERVICE_UPTIME.set(start_time.elapsed().as_secs_f64()); +} +``` + +**Step 3: Add `pub mod metrics;` to lib.rs** + +In `crates/web-gateway/src/lib.rs`, add: +```rust +pub mod metrics; +``` + +**Step 4: Add metrics server to main.rs** + +In `crates/web-gateway/src/main.rs`, after the `info!("Web gateway listening on {}", listen_addr);` line (line 79) and before the `axum::serve(` block (line 81), add: + +```rust + // Initialize Prometheus metrics + web_gateway::metrics::init_metrics(); + let service_start = std::time::Instant::now(); + + // Spawn uptime updater + tokio::spawn(async move { + let mut interval = tokio::time::interval(std::time::Duration::from_secs(1)); + loop { + interval.tick().await; + web_gateway::metrics::update_uptime(service_start); + } + }); + + // Start Prometheus metrics HTTP endpoint on port 9098 + let metrics_port: u16 = std::env::var("METRICS_PORT") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(9098); + tokio::spawn(async move { + use axum::{routing::get, Router}; + use prometheus::{Encoder, TextEncoder}; + + async fn metrics_handler() -> String { + let encoder = TextEncoder::new(); + let metric_families = prometheus::gather(); + let mut buffer = vec![]; + let _ = encoder.encode(&metric_families, &mut buffer); + String::from_utf8(buffer).unwrap_or_else(|_| String::new()) + } + + let app = Router::new().route("/metrics", get(metrics_handler)); + let addr = format!("0.0.0.0:{}", metrics_port); + info!("Prometheus metrics endpoint listening on http://{}", addr); + + let listener = match tokio::net::TcpListener::bind(&addr).await { + Ok(l) => l, + Err(e) => { + tracing::error!("Failed to bind metrics endpoint {}: {}", addr, e); + return; + }, + }; + if let Err(e) = axum::serve(listener, app).await { + tracing::error!("Metrics server failed: {}", e); + } + }); +``` + +**Step 5: Build check** + +Run: `SQLX_OFFLINE=true cargo check -p web-gateway` +Expected: Compiles clean + +**Step 6: Run existing tests** + +Run: `SQLX_OFFLINE=true cargo test -p web-gateway --lib` +Expected: All 138 tests pass + +**Step 7: Commit** + +```bash +git add crates/web-gateway/Cargo.toml crates/web-gateway/src/metrics.rs crates/web-gateway/src/main.rs crates/web-gateway/src/lib.rs +git commit -m "feat(web-gateway): add Prometheus metrics server on port 9098" +``` + +--- + +### Task 7: Update K8s YAML for web-gateway Metrics Port + +**Files:** +- Modify: `infra/k8s/services/web-gateway.yaml` + +**Context:** web-gateway K8s YAML currently has only port 3000 (http). Need to add port 9098 (metrics), prometheus scrape annotations, and ensure the Service exposes the metrics port. + +**Step 1: Add prometheus annotations to pod template** + +In `infra/k8s/services/web-gateway.yaml`, the pod template metadata currently has only `labels:` (line 21). Add `annotations:` before `labels:`: + +```yaml + metadata: + annotations: + gitlab.com/prometheus_scrape: "true" + gitlab.com/prometheus_port: "9098" + gitlab.com/prometheus_path: "/metrics" + labels: + app.kubernetes.io/name: web-gateway +``` + +**Step 2: Add metrics container port** + +After the `http` port (line 88), add: + +```yaml + - containerPort: 9098 + name: metrics +``` + +**Step 3: Add metrics port to Service** + +Find the Service section at the bottom of the file. Add the metrics port: + +```yaml + - port: 9098 + targetPort: 9098 + name: metrics +``` + +**Step 4: Commit** + +```bash +git add infra/k8s/services/web-gateway.yaml +git commit -m "infra(web-gateway): add metrics port 9098 and prometheus scrape annotations" +``` + +--- + +## Layer 3: gRPC Metrics Tower Layer + +### Task 8: Create gRPC Metrics Tower Layer in common + +**Files:** +- Create: `crates/common/src/metrics/grpc_metrics.rs` +- Modify: `crates/common/src/metrics/mod.rs` (add `pub mod grpc_metrics;`) + +**Context:** We need a Tower `Layer` + `Service` that wraps every gRPC handler and automatically records `grpc_server_started_total`, `grpc_server_handled_total`, and `grpc_server_handling_seconds`. The layer extracts the gRPC method from the request URI path. `common` already depends on `prometheus`, `once_cell`, `tonic`, and `tower` (via tonic). + +**Step 1: Create `crates/common/src/metrics/grpc_metrics.rs`** + +```rust +//! gRPC metrics Tower layer for automatic request instrumentation +//! +//! Provides `grpc_server_started_total`, `grpc_server_handled_total`, +//! and `grpc_server_handling_seconds` metrics for all gRPC handlers. + +use once_cell::sync::Lazy; +use prometheus::{ + register_counter_vec, register_histogram_vec, CounterVec, HistogramVec, +}; +use std::future::Future; +use std::pin::Pin; +use std::task::{Context, Poll}; +use tonic::body::BoxBody; +use tower::{Layer, Service}; + +/// Counter for gRPC requests started +static GRPC_STARTED: Lazy = Lazy::new(|| { + register_counter_vec!( + "grpc_server_started_total", + "Total gRPC RPCs started on the server", + &["service", "grpc_method"] + ) + .unwrap_or_else(|e| { + eprintln!("FATAL: failed to register grpc_server_started_total: {e}"); + std::process::abort() + }) +}); + +/// Counter for gRPC requests completed with status code +static GRPC_HANDLED: Lazy = Lazy::new(|| { + register_counter_vec!( + "grpc_server_handled_total", + "Total gRPC RPCs completed on the server with status code", + &["service", "grpc_method", "grpc_code"] + ) + .unwrap_or_else(|e| { + eprintln!("FATAL: failed to register grpc_server_handled_total: {e}"); + std::process::abort() + }) +}); + +/// Histogram for gRPC request handling duration +static GRPC_HANDLING_SECONDS: Lazy = Lazy::new(|| { + register_histogram_vec!( + "grpc_server_handling_seconds", + "Histogram of gRPC request handling duration in seconds", + &["service", "grpc_method"], + vec![0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 5.0] + ) + .unwrap_or_else(|e| { + eprintln!("FATAL: failed to register grpc_server_handling_seconds: {e}"); + std::process::abort() + }) +}); + +/// Initialize gRPC metrics (force lazy registration) +pub fn init_grpc_metrics() { + let _ = &*GRPC_STARTED; + let _ = &*GRPC_HANDLED; + let _ = &*GRPC_HANDLING_SECONDS; +} + +/// Tower Layer that instruments gRPC handlers with Prometheus metrics +#[derive(Clone)] +pub struct GrpcMetricsLayer { + service_name: &'static str, +} + +impl GrpcMetricsLayer { + /// Create a new gRPC metrics layer for the given service + pub fn new(service_name: &'static str) -> Self { + init_grpc_metrics(); + Self { service_name } + } +} + +impl Layer for GrpcMetricsLayer { + type Service = GrpcMetricsService; + + fn layer(&self, inner: S) -> Self::Service { + GrpcMetricsService { + inner, + service_name: self.service_name, + } + } +} + +/// Tower Service that records gRPC metrics +#[derive(Clone)] +pub struct GrpcMetricsService { + inner: S, + service_name: &'static str, +} + +impl Service> for GrpcMetricsService +where + S: Service, Response = http::Response> + + Clone + + Send + + 'static, + S::Future: Send + 'static, + S::Error: Into>, +{ + type Response = S::Response; + type Error = S::Error; + type Future = Pin> + Send>>; + + fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { + self.inner.poll_ready(cx) + } + + fn call(&mut self, req: http::Request) -> Self::Future { + let service_name = self.service_name; + let method = req + .uri() + .path() + .rsplit('/') + .next() + .unwrap_or("unknown") + .to_string(); + + GRPC_STARTED + .with_label_values(&[service_name, &method]) + .inc(); + + let start = std::time::Instant::now(); + let mut inner = self.inner.clone(); + + Box::pin(async move { + let result = inner.call(req).await; + let elapsed = start.elapsed().as_secs_f64(); + + let code = match &result { + Ok(resp) => { + // Extract gRPC status from the grpc-status header + resp.headers() + .get("grpc-status") + .and_then(|v| v.to_str().ok()) + .unwrap_or("0") + .to_string() + }, + Err(_) => "13".to_string(), // Internal error + }; + + GRPC_HANDLED + .with_label_values(&[service_name, &method, &code]) + .inc(); + + GRPC_HANDLING_SECONDS + .with_label_values(&[service_name, &method]) + .observe(elapsed); + + result + }) + } +} +``` + +**Step 2: Add to mod.rs** + +In `crates/common/src/metrics/mod.rs`, add after line 6 (`pub mod registry;`): + +```rust +pub mod grpc_metrics; +``` + +And add to the re-exports: + +```rust +pub use grpc_metrics::{GrpcMetricsLayer, init_grpc_metrics}; +``` + +**Step 3: Build check** + +Run: `SQLX_OFFLINE=true cargo check -p common` +Expected: Compiles clean + +**Step 4: Run existing common tests** + +Run: `SQLX_OFFLINE=true cargo test -p common --lib` +Expected: 198 tests pass + +**Step 5: Commit** + +```bash +git add crates/common/src/metrics/grpc_metrics.rs crates/common/src/metrics/mod.rs +git commit -m "feat(common): add gRPC metrics Tower layer for automatic instrumentation" +``` + +--- + +### Task 9: Wire gRPC Metrics Layer into Services + +**Files:** +- Modify: `services/trading_service/src/main.rs` +- Modify: `services/api_gateway/src/main.rs` +- Modify: `services/ml_training_service/src/main.rs` +- Modify: `services/backtesting_service/src/main.rs` +- Modify: `services/broker_gateway_service/src/main.rs` +- Modify: `services/data_acquisition_service/src/main.rs` +- Modify: `services/trading_agent_service/src/main.rs` +- Modify: `crates/web-gateway/src/main.rs` (N/A — web-gateway is HTTP, not gRPC) + +**Context:** Each gRPC service uses `tonic::transport::Server::builder()` to construct the server. The Tower layer must be added via `.layer()` BEFORE `.add_service()`. Only 7 services use gRPC; web-gateway is HTTP-only and skipped. + +**Step 1: Add `.layer(GrpcMetricsLayer::new("service_name"))` to each service** + +For each service, find the `Server::builder()` chain and add the layer. The pattern is: + +```rust +use common::metrics::GrpcMetricsLayer; + +Server::builder() + .layer(GrpcMetricsLayer::new("service-name")) + .add_service(...) + .serve_with_shutdown(addr, shutdown_signal()) + .await?; +``` + +Service names to use (matching K8s service names): +- `services/trading_service/src/main.rs` → `"trading-service"` +- `services/api_gateway/src/main.rs` → `"api-gateway"` +- `services/ml_training_service/src/main.rs` → `"ml-training-service"` +- `services/backtesting_service/src/main.rs` → `"backtesting-service"` +- `services/broker_gateway_service/src/main.rs` → `"broker-gateway"` +- `services/data_acquisition_service/src/main.rs` → `"data-acquisition-service"` +- `services/trading_agent_service/src/main.rs` → `"trading-agent-service"` + +**Important:** For `ml_training_service`, the server builder is spread across lines 457-489 with optional TLS. The layer should be added right after `Server::builder()` (line 457): + +```rust +let mut server_builder = Server::builder() + .layer(GrpcMetricsLayer::new("ml-training-service")); +``` + +**Step 2: Build check** + +Run: `SQLX_OFFLINE=true cargo check --workspace` +Expected: Compiles clean + +**Step 3: Commit** + +```bash +git add services/*/src/main.rs +git commit -m "feat(services): wire gRPC metrics Tower layer into all 7 gRPC services" +``` + +--- + +## Layer 4: Training Pushgateway + +### Task 10: Deploy Prometheus Pushgateway + +**Files:** +- Create: `infra/k8s/monitoring/pushgateway.yaml` + +**Context:** The Pushgateway receives metrics from short-lived training jobs via HTTP POST and holds them for Prometheus to scrape. Deploy it in the foxhunt namespace. + +**Step 1: Create `infra/k8s/monitoring/pushgateway.yaml`** + +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: pushgateway + namespace: foxhunt + labels: + app.kubernetes.io/name: pushgateway + app.kubernetes.io/part-of: monitoring +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: pushgateway + template: + metadata: + labels: + app.kubernetes.io/name: pushgateway + spec: + nodeSelector: + k8s.scaleway.com/pool-name: services + containers: + - name: pushgateway + image: prom/pushgateway:v1.11.0 + args: + - "--persistence.interval=5m" + ports: + - containerPort: 9091 + name: http + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 200m + memory: 128Mi + livenessProbe: + httpGet: + path: /-/healthy + port: 9091 + initialDelaySeconds: 5 + readinessProbe: + httpGet: + path: /-/ready + port: 9091 + initialDelaySeconds: 5 +--- +apiVersion: v1 +kind: Service +metadata: + name: pushgateway + namespace: foxhunt + labels: + app.kubernetes.io/name: pushgateway +spec: + type: ClusterIP + ports: + - port: 9091 + targetPort: 9091 + name: http + selector: + app.kubernetes.io/name: pushgateway +``` + +**Step 2: Commit** + +```bash +git add infra/k8s/monitoring/pushgateway.yaml +git commit -m "infra: add Prometheus Pushgateway for training job metrics" +``` + +--- + +### Task 11: Add Pushgateway Scrape Config to Prometheus + +**Files:** +- Modify: `infra/k8s/monitoring/prometheus-values.yaml` + +**Context:** Prometheus needs a scrape job for the pushgateway. Use `honor_labels: true` so that pushed labels (like `model` and `job_id`) are preserved instead of being overwritten by Prometheus relabeling. + +**Step 1: Add pushgateway scrape config** + +At the end of `infra/k8s/monitoring/prometheus-values.yaml` (after the promtail job, line 166), add: + +```yaml + + # --- Pushgateway (training job metrics) --- + - job_name: pushgateway + honor_labels: true + static_configs: + - targets: ['pushgateway.foxhunt.svc.cluster.local:9091'] +``` + +**Step 2: Commit** + +```bash +git add infra/k8s/monitoring/prometheus-values.yaml +git commit -m "infra: add pushgateway scrape config to Prometheus" +``` + +--- + +### Task 12: Add Training Metrics Push to ML Training Binaries + +**Files:** +- Create: `crates/ml/src/training/push_metrics.rs` +- Modify: `crates/ml/src/training/mod.rs` (add `pub mod push_metrics;`) +- Modify: `crates/ml/Cargo.toml` (add `reqwest` if not present) + +**Context:** The 5 training binaries (`train_baseline_rl`, `train_baseline_supervised`, `evaluate_baseline`, `hyperopt_baseline_rl`, `hyperopt_baseline_supervised`) need to push epoch-level metrics to the Pushgateway during training. The push happens via HTTP POST to `http://pushgateway:9091/metrics/job//model/`. + +Prometheus Pushgateway uses a simple text format. We push the metric families as Prometheus exposition text via `PUT /metrics/job/{job}/model/{model}`. + +**Step 1: Create `crates/ml/src/training/push_metrics.rs`** + +```rust +//! Prometheus Pushgateway integration for training job metrics +//! +//! Training jobs are short-lived K8s Jobs. They push epoch-level metrics +//! to the Pushgateway so Prometheus can scrape them persistently. + +use std::fmt::Write; + +/// Pushgateway client for training metrics +pub struct TrainingMetricsPusher { + pushgateway_url: String, + job_id: String, + model_name: String, + client: reqwest::Client, +} + +impl TrainingMetricsPusher { + /// Create a new pusher. Falls back to localhost if PUSHGATEWAY_URL not set. + pub fn new(job_id: &str, model_name: &str) -> Self { + let pushgateway_url = std::env::var("PUSHGATEWAY_URL") + .unwrap_or_else(|_| "http://pushgateway.foxhunt.svc.cluster.local:9091".to_string()); + Self { + pushgateway_url, + job_id: job_id.to_string(), + model_name: model_name.to_string(), + client: reqwest::Client::new(), + } + } + + /// Push current training state to Pushgateway. + /// Non-fatal: logs errors but never panics. + pub async fn push(&self, state: &TrainingState) { + let mut body = String::new(); + + // Epoch progress + let _ = writeln!(body, "# HELP foxhunt_training_current_epoch Current training epoch"); + let _ = writeln!(body, "# TYPE foxhunt_training_current_epoch gauge"); + let _ = writeln!(body, "foxhunt_training_current_epoch {}", state.epoch); + + // Loss + let _ = writeln!(body, "# HELP foxhunt_training_epoch_loss Training loss value"); + let _ = writeln!(body, "# TYPE foxhunt_training_epoch_loss gauge"); + let _ = writeln!(body, "foxhunt_training_epoch_loss{{split=\"train\"}} {}", state.train_loss); + if let Some(val_loss) = state.val_loss { + let _ = writeln!(body, "foxhunt_training_epoch_loss{{split=\"val\"}} {}", val_loss); + } + + // Accuracy + if let Some(accuracy) = state.accuracy { + let _ = writeln!(body, "# HELP foxhunt_training_eval_accuracy Model evaluation accuracy"); + let _ = writeln!(body, "# TYPE foxhunt_training_eval_accuracy gauge"); + let _ = writeln!(body, "foxhunt_training_eval_accuracy {}", accuracy); + } + + // Batches processed + let _ = writeln!(body, "# HELP foxhunt_training_batches_processed Total batches processed"); + let _ = writeln!(body, "# TYPE foxhunt_training_batches_processed counter"); + let _ = writeln!(body, "foxhunt_training_batches_processed {}", state.batches_processed); + + // Training speed + if state.batches_per_second > 0.0 { + let _ = writeln!(body, "# HELP foxhunt_training_batches_per_second Training throughput"); + let _ = writeln!(body, "# TYPE foxhunt_training_batches_per_second gauge"); + let _ = writeln!(body, "foxhunt_training_batches_per_second {}", state.batches_per_second); + } + + // Learning rate + let _ = writeln!(body, "# HELP foxhunt_training_learning_rate Current learning rate"); + let _ = writeln!(body, "# TYPE foxhunt_training_learning_rate gauge"); + let _ = writeln!(body, "foxhunt_training_learning_rate {}", state.learning_rate); + + // NaN / gradient events + if state.nan_count > 0 { + let _ = writeln!(body, "# HELP foxhunt_training_nan_detected_total NaN gradient events"); + let _ = writeln!(body, "# TYPE foxhunt_training_nan_detected_total counter"); + let _ = writeln!(body, "foxhunt_training_nan_detected_total {}", state.nan_count); + } + if state.gradient_explosion_count > 0 { + let _ = writeln!(body, "# HELP foxhunt_training_gradient_explosion_total Gradient clipping events"); + let _ = writeln!(body, "# TYPE foxhunt_training_gradient_explosion_total counter"); + let _ = writeln!(body, "foxhunt_training_gradient_explosion_total {}", state.gradient_explosion_count); + } + + // Checkpoint saves + let _ = writeln!(body, "# HELP foxhunt_training_checkpoint_saves_total Checkpoints saved"); + let _ = writeln!(body, "# TYPE foxhunt_training_checkpoint_saves_total counter"); + let _ = writeln!(body, "foxhunt_training_checkpoint_saves_total {}", state.checkpoint_saves); + + let url = format!( + "{}/metrics/job/{}/model/{}", + self.pushgateway_url, self.job_id, self.model_name + ); + + match self.client.put(&url).body(body).send().await { + Ok(resp) if resp.status().is_success() => {}, + Ok(resp) => { + eprintln!( + "Pushgateway returned {}: {}", + resp.status(), + resp.text().await.unwrap_or_default() + ); + }, + Err(e) => { + eprintln!("Failed to push metrics to Pushgateway: {e}"); + }, + } + } + + /// Delete metrics for this job from Pushgateway (call on training completion) + pub async fn cleanup(&self) { + let url = format!( + "{}/metrics/job/{}/model/{}", + self.pushgateway_url, self.job_id, self.model_name + ); + let _ = self.client.delete(&url).send().await; + } +} + +/// Current training state to push +pub struct TrainingState { + pub epoch: u64, + pub train_loss: f64, + pub val_loss: Option, + pub accuracy: Option, + pub batches_processed: u64, + pub batches_per_second: f64, + pub learning_rate: f64, + pub nan_count: u64, + pub gradient_explosion_count: u64, + pub checkpoint_saves: u64, +} + +impl Default for TrainingState { + fn default() -> Self { + Self { + epoch: 0, + train_loss: 0.0, + val_loss: None, + accuracy: None, + batches_processed: 0, + batches_per_second: 0.0, + learning_rate: 0.001, + nan_count: 0, + gradient_explosion_count: 0, + checkpoint_saves: 0, + } + } +} +``` + +**Step 2: Add to training mod.rs** + +In `crates/ml/src/training/mod.rs`, add: + +```rust +pub mod push_metrics; +``` + +**Step 3: Verify reqwest is available** + +Check if `crates/ml/Cargo.toml` already has `reqwest`. If not, add: + +```toml +reqwest = { workspace = true, features = ["rustls-tls"] } +``` + +**Step 4: Build check** + +Run: `SQLX_OFFLINE=true cargo check -p ml` +Expected: Compiles clean + +**Step 5: Commit** + +```bash +git add crates/ml/src/training/push_metrics.rs crates/ml/src/training/mod.rs crates/ml/Cargo.toml +git commit -m "feat(ml): add Pushgateway metrics client for training jobs" +``` + +--- + +### Task 13: Deploy and Verify + +**Files:** +- No code changes — cluster operations only + +**Step 1: Apply all K8s changes** + +```bash +# Deploy pushgateway +kubectl apply -f infra/k8s/monitoring/pushgateway.yaml + +# Apply updated web-gateway with metrics port +kubectl apply -f infra/k8s/services/web-gateway.yaml + +# Apply updated service YAMLs (all services) +kubectl apply -f infra/k8s/services/ + +# Deploy updated dashboard ConfigMaps +PROM_UID="PBFA97CFB590B2093" +LOKI_UID="P8E80F9AEF21F6940" +TEMPO_UID="tempo" +DASH_DIR="infra/k8s/monitoring/dashboards" +TMP_DIR=$(mktemp -d) +for f in "$DASH_DIR"/*.json; do + python3 -c " +import json +with open('$f') as fh: + d = json.load(fh) +d.pop('__inputs', None) +d.pop('__requires', None) +raw = json.dumps(d) +raw = raw.replace('\${DS_PROMETHEUS}', '${PROM_UID}') +raw = raw.replace('\${DS_LOKI}', '${LOKI_UID}') +raw = raw.replace('\${DS_TEMPO}', '${TEMPO_UID}') +print(raw) +" > "$TMP_DIR/$(basename "$f")" +done + +# Build ConfigMaps with folder annotations +declare -A GROUPS=( + [grafana-dashboards-foxhunt]="foxhunt-logs.json foxhunt-overview.json foxhunt-services.json foxhunt-cockpit.json foxhunt-trading-cockpit.json foxhunt-training-cockpit.json foxhunt-traces.json" + [grafana-dashboards-infra]="cluster-overview.json gpu-overview.json foxhunt-gpu-training.json foxhunt-infrastructure.json" + [grafana-dashboards-cicd]="foxhunt-ci-pipelines.json gitlab-services.json" +) +declare -A FOLDERS=( + [grafana-dashboards-foxhunt]="Foxhunt" + [grafana-dashboards-infra]="Infrastructure" + [grafana-dashboards-cicd]="CI/CD" +) +for cm_name in "${!GROUPS[@]}"; do + args="" + for df in ${GROUPS[$cm_name]}; do + [ -f "$TMP_DIR/$df" ] && args="$args --from-file=$df=$TMP_DIR/$df" + done + if [ -n "$args" ]; then + kubectl create configmap "$cm_name" -n foxhunt $args \ + --dry-run=client -o yaml | \ + kubectl label -f - --dry-run=client -o yaml --local grafana_dashboard=1 | \ + kubectl annotate -f - --dry-run=client -o yaml --local grafana_folder="${FOLDERS[$cm_name]}" | \ + kubectl apply -f - + fi +done +rm -rf "$TMP_DIR" + +# Restart Grafana to reload dashboards +kubectl rollout restart deployment grafana -n foxhunt + +# Upgrade Prometheus with pushgateway scrape config +helm upgrade gitlab gitlab/gitlab -n foxhunt --reuse-values \ + -f infra/k8s/monitoring/prometheus-values.yaml + +# Rolling restart services to pick up new binaries +for svc in trading-service api-gateway broker-gateway data-acquisition-service ml-training-service backtesting-service trading-agent-service web-gateway; do + kubectl rollout restart deployment "$svc" -n foxhunt 2>/dev/null || true +done +``` + +**Step 2: Verify Pushgateway is running** + +```bash +kubectl get pods -n foxhunt -l app.kubernetes.io/name=pushgateway +``` +Expected: `pushgateway-xxx 1/1 Running` + +**Step 3: Verify dashboards are in Foxhunt folder** + +```bash +kubectl port-forward deployment/grafana 3000:3000 -n foxhunt & +GRAFANA_PASS=$(kubectl get secret grafana -n foxhunt -o jsonpath='{.data.admin-password}' | base64 -d) +curl -s -u "admin:${GRAFANA_PASS}" "http://localhost:3000/api/search?type=dash-db" | python3 -c " +import json, sys +for d in json.load(sys.stdin): + print(f\"{d['uid']:40s} {d.get('folderTitle','General'):20s} {d['title']}\") +" +kill %1 +``` + +**Step 4: Push and commit** + +```bash +git push origin main +``` From 56ca1fb083105e5c46834af79ffb7468a4831e73 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sun, 1 Mar 2026 02:05:17 +0100 Subject: [PATCH 03/14] fix(dashboard): use real Tempo metrics in cockpit trace health row Replace traces_spanmetrics_calls_total with tempo_distributor_spans_received_total in all 3 Trace Health panels (Spans/sec, Trace Error Rate, Spans/sec by Service). Co-Authored-By: Claude Opus 4.6 --- .../dashboards/foxhunt-cockpit.json | 1682 ++++++++++++++--- 1 file changed, 1456 insertions(+), 226 deletions(-) diff --git a/infra/k8s/monitoring/dashboards/foxhunt-cockpit.json b/infra/k8s/monitoring/dashboards/foxhunt-cockpit.json index 090e87217..74953a2fe 100644 --- a/infra/k8s/monitoring/dashboards/foxhunt-cockpit.json +++ b/infra/k8s/monitoring/dashboards/foxhunt-cockpit.json @@ -1,20 +1,44 @@ { "__inputs": [ - { "name": "DS_PROMETHEUS", "label": "Prometheus", "description": "", "type": "datasource", "pluginId": "prometheus" }, - { "name": "DS_LOKI", "label": "Loki", "description": "", "type": "datasource", "pluginId": "loki" }, - { "name": "DS_TEMPO", "label": "Tempo", "description": "", "type": "datasource", "pluginId": "tempo" } + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus" + }, + { + "name": "DS_LOKI", + "label": "Loki", + "description": "", + "type": "datasource", + "pluginId": "loki" + }, + { + "name": "DS_TEMPO", + "label": "Tempo", + "description": "", + "type": "datasource", + "pluginId": "tempo" + } ], "id": null, "uid": "foxhunt-cockpit", "title": "Foxhunt - Cockpit", "description": "Single-page overview: cluster health, all services, GPU/training, resources, errors, live logs", - "tags": ["foxhunt", "production"], + "tags": [ + "foxhunt", + "production" + ], "timezone": "browser", "schemaVersion": 39, "version": 1, "editable": true, "refresh": "30s", - "time": { "from": "now-1h", "to": "now" }, + "time": { + "from": "now-1h", + "to": "now" + }, "fiscalYearStartMonth": 0, "liveNow": false, "weekStart": "", @@ -24,7 +48,11 @@ "name": "interval", "type": "interval", "query": "1m,5m,15m,30m,1h", - "current": { "selected": true, "text": "5m", "value": "5m" }, + "current": { + "selected": true, + "text": "5m", + "value": "5m" + }, "auto": false } ] @@ -34,7 +62,12 @@ "id": 1, "type": "row", "title": "Health Bar", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, "collapsed": false, "panels": [] }, @@ -42,18 +75,46 @@ "id": 2, "type": "stat", "title": "Nodes Ready", - "gridPos": { "h": 3, "w": 4, "x": 0, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 1 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, "mappings": [], - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -63,7 +124,10 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "expr": "sum(kube_node_status_condition{condition=\"Ready\",status=\"true\"})", "instant": true } @@ -73,18 +137,46 @@ "id": 3, "type": "stat", "title": "Pods Running", - "gridPos": { "h": 3, "w": 4, "x": 4, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 1 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "yellow", "value": null }, { "color": "green", "value": 1 }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, "mappings": [], - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -94,7 +186,10 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\",phase=\"Running\"})", "instant": true } @@ -104,18 +199,50 @@ "id": 4, "type": "stat", "title": "Pods Not Ready", - "gridPos": { "h": 3, "w": 4, "x": 8, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 1 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 1 }, { "color": "red", "value": 3 }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 3 + } + ] + }, "mappings": [], - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -125,7 +252,10 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\",phase=~\"Pending|Failed|Unknown\"}) or vector(0)", "instant": true } @@ -135,18 +265,42 @@ "id": 5, "type": "stat", "title": "Training Jobs", - "gridPos": { "h": 3, "w": 4, "x": 12, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 3, + "w": 4, + "x": 12, + "y": 1 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "blue", "value": null }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, "mappings": [], - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -156,7 +310,10 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "expr": "sum(kube_job_status_active{namespace=\"foxhunt\"}) or vector(0)", "instant": true } @@ -166,18 +323,46 @@ "id": 6, "type": "stat", "title": "GPU Available", - "gridPos": { "h": 3, "w": 4, "x": 16, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 3, + "w": 4, + "x": 16, + "y": 1 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, "mappings": [], - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -187,7 +372,10 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "expr": "sum(kube_node_status_allocatable{resource=\"nvidia_com_gpu\"}) or vector(0)", "instant": true } @@ -197,18 +385,50 @@ "id": 7, "type": "stat", "title": "Errors /min", - "gridPos": { "h": 3, "w": 4, "x": 20, "y": 1 }, - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 1 + }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 5 }, { "color": "red", "value": 20 }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 5 + }, + { + "color": "red", + "value": 20 + } + ] + }, "mappings": [], - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -218,18 +438,25 @@ "targets": [ { "refId": "A", - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, "expr": "sum(count_over_time({namespace=\"foxhunt\"} |= \"ERROR\" [1m]))", "instant": true } ] }, - { "id": 8, "type": "row", "title": "Service Status", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 4 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 4 + }, "collapsed": false, "panels": [] }, @@ -237,29 +464,80 @@ "id": 9, "type": "stat", "title": "api-gateway", - "gridPos": { "h": 3, "w": 3, "x": 0, "y": 5 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 3, + "w": 3, + "x": 0, + "y": 5 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] }, - "mappings": [{ "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }], - "color": { "mode": "thresholds" } + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "DOWN", + "color": "red" + }, + "1": { + "text": "UP", + "color": "green" + } + } + } + ], + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", "graphMode": "none", "justifyMode": "center" }, - "links": [{ "title": "Service Detail", "url": "/d/foxhunt-services?var-service=api-gateway", "targetBlank": false }], + "links": [ + { + "title": "Service Detail", + "url": "/d/foxhunt-services?var-service=api-gateway", + "targetBlank": false + } + ], "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"api-gateway.*\", phase=\"Running\"}) > 0 or vector(0)", "instant": true } @@ -269,29 +547,80 @@ "id": 10, "type": "stat", "title": "trading-service", - "gridPos": { "h": 3, "w": 3, "x": 3, "y": 5 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 3, + "w": 3, + "x": 3, + "y": 5 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] }, - "mappings": [{ "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }], - "color": { "mode": "thresholds" } + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "DOWN", + "color": "red" + }, + "1": { + "text": "UP", + "color": "green" + } + } + } + ], + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", "graphMode": "none", "justifyMode": "center" }, - "links": [{ "title": "Service Detail", "url": "/d/foxhunt-services?var-service=trading-service", "targetBlank": false }], + "links": [ + { + "title": "Service Detail", + "url": "/d/foxhunt-services?var-service=trading-service", + "targetBlank": false + } + ], "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"trading-service.*\", phase=\"Running\"}) > 0 or vector(0)", "instant": true } @@ -301,29 +630,80 @@ "id": 11, "type": "stat", "title": "ml-training", - "gridPos": { "h": 3, "w": 3, "x": 6, "y": 5 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 3, + "w": 3, + "x": 6, + "y": 5 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] }, - "mappings": [{ "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }], - "color": { "mode": "thresholds" } + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "DOWN", + "color": "red" + }, + "1": { + "text": "UP", + "color": "green" + } + } + } + ], + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", "graphMode": "none", "justifyMode": "center" }, - "links": [{ "title": "Service Detail", "url": "/d/foxhunt-services?var-service=ml-training-service", "targetBlank": false }], + "links": [ + { + "title": "Service Detail", + "url": "/d/foxhunt-services?var-service=ml-training-service", + "targetBlank": false + } + ], "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"ml-training-service.*\", phase=\"Running\"}) > 0 or vector(0)", "instant": true } @@ -333,29 +713,80 @@ "id": 12, "type": "stat", "title": "backtesting", - "gridPos": { "h": 3, "w": 3, "x": 9, "y": 5 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 3, + "w": 3, + "x": 9, + "y": 5 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] }, - "mappings": [{ "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }], - "color": { "mode": "thresholds" } + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "DOWN", + "color": "red" + }, + "1": { + "text": "UP", + "color": "green" + } + } + } + ], + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", "graphMode": "none", "justifyMode": "center" }, - "links": [{ "title": "Service Detail", "url": "/d/foxhunt-services?var-service=backtesting-service", "targetBlank": false }], + "links": [ + { + "title": "Service Detail", + "url": "/d/foxhunt-services?var-service=backtesting-service", + "targetBlank": false + } + ], "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"backtesting-service.*\", phase=\"Running\"}) > 0 or vector(0)", "instant": true } @@ -365,29 +796,80 @@ "id": 13, "type": "stat", "title": "trading-agent", - "gridPos": { "h": 3, "w": 3, "x": 12, "y": 5 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 3, + "w": 3, + "x": 12, + "y": 5 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] }, - "mappings": [{ "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }], - "color": { "mode": "thresholds" } + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "DOWN", + "color": "red" + }, + "1": { + "text": "UP", + "color": "green" + } + } + } + ], + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", "graphMode": "none", "justifyMode": "center" }, - "links": [{ "title": "Service Detail", "url": "/d/foxhunt-services?var-service=trading-agent-service", "targetBlank": false }], + "links": [ + { + "title": "Service Detail", + "url": "/d/foxhunt-services?var-service=trading-agent-service", + "targetBlank": false + } + ], "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"trading-agent.*\", phase=\"Running\"}) > 0 or vector(0)", "instant": true } @@ -397,29 +879,80 @@ "id": 14, "type": "stat", "title": "broker-gw", - "gridPos": { "h": 3, "w": 3, "x": 15, "y": 5 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 3, + "w": 3, + "x": 15, + "y": 5 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] }, - "mappings": [{ "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }], - "color": { "mode": "thresholds" } + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "DOWN", + "color": "red" + }, + "1": { + "text": "UP", + "color": "green" + } + } + } + ], + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", "graphMode": "none", "justifyMode": "center" }, - "links": [{ "title": "Service Detail", "url": "/d/foxhunt-services?var-service=broker-gateway", "targetBlank": false }], + "links": [ + { + "title": "Service Detail", + "url": "/d/foxhunt-services?var-service=broker-gateway", + "targetBlank": false + } + ], "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"broker-gateway.*\", phase=\"Running\"}) > 0 or vector(0)", "instant": true } @@ -429,29 +962,80 @@ "id": 15, "type": "stat", "title": "data-acq", - "gridPos": { "h": 3, "w": 3, "x": 18, "y": 5 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 3, + "w": 3, + "x": 18, + "y": 5 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] }, - "mappings": [{ "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }], - "color": { "mode": "thresholds" } + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "DOWN", + "color": "red" + }, + "1": { + "text": "UP", + "color": "green" + } + } + } + ], + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", "graphMode": "none", "justifyMode": "center" }, - "links": [{ "title": "Service Detail", "url": "/d/foxhunt-services?var-service=data-acquisition-service", "targetBlank": false }], + "links": [ + { + "title": "Service Detail", + "url": "/d/foxhunt-services?var-service=data-acquisition-service", + "targetBlank": false + } + ], "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"data-acquisition.*\", phase=\"Running\"}) > 0 or vector(0)", "instant": true } @@ -461,40 +1045,95 @@ "id": 16, "type": "stat", "title": "web-gateway", - "gridPos": { "h": 3, "w": 3, "x": 21, "y": 5 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 3, + "w": 3, + "x": 21, + "y": 5 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] }, - "mappings": [{ "type": "value", "options": { "0": { "text": "DOWN", "color": "red" }, "1": { "text": "UP", "color": "green" } } }], - "color": { "mode": "thresholds" } + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "DOWN", + "color": "red" + }, + "1": { + "text": "UP", + "color": "green" + } + } + } + ], + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", "graphMode": "none", "justifyMode": "center" }, - "links": [{ "title": "Service Detail", "url": "/d/foxhunt-services?var-service=web-gateway", "targetBlank": false }], + "links": [ + { + "title": "Service Detail", + "url": "/d/foxhunt-services?var-service=web-gateway", + "targetBlank": false + } + ], "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "expr": "sum(kube_pod_status_phase{namespace=\"foxhunt\", pod=~\"web-gateway.*\", phase=\"Running\"}) > 0 or vector(0)", "instant": true } ] }, - { "id": 17, "type": "row", "title": "Resources", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 8 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, "collapsed": false, "panels": [] }, @@ -502,24 +1141,63 @@ "id": 18, "type": "gauge", "title": "Cluster CPU %", - "gridPos": { "h": 5, "w": 4, "x": 0, "y": 9 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 5, + "w": 4, + "x": 0, + "y": 9 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 70 }, { "color": "red", "value": 90 }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 70 + }, + { + "color": "red", + "value": 90 + } + ] + }, "unit": "percent", "min": 0, "max": 100, "mappings": [], - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, - "options": { "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "expr": "100 * (1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[$interval])))", "instant": true } @@ -529,24 +1207,63 @@ "id": 19, "type": "gauge", "title": "Cluster Memory %", - "gridPos": { "h": 5, "w": 4, "x": 4, "y": 9 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 5, + "w": 4, + "x": 4, + "y": 9 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 75 }, { "color": "red", "value": 90 }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 75 + }, + { + "color": "red", + "value": 90 + } + ] + }, "unit": "percent", "min": 0, "max": 100, "mappings": [], - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, - "options": { "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "expr": "100 * (1 - sum(node_memory_MemAvailable_bytes) / sum(node_memory_MemTotal_bytes))", "instant": true } @@ -556,24 +1273,63 @@ "id": 20, "type": "gauge", "title": "Cluster Disk %", - "gridPos": { "h": 5, "w": 4, "x": 8, "y": 9 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 5, + "w": 4, + "x": 8, + "y": 9 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 75 }, { "color": "red", "value": 90 }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 75 + }, + { + "color": "red", + "value": 90 + } + ] + }, "unit": "percent", "min": 0, "max": 100, "mappings": [], - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, - "options": { "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "expr": "100 * (1 - sum(node_filesystem_avail_bytes{mountpoint=\"/\",fstype!=\"tmpfs\"}) / sum(node_filesystem_size_bytes{mountpoint=\"/\",fstype!=\"tmpfs\"}))", "instant": true } @@ -583,8 +1339,16 @@ "id": 21, "type": "timeseries", "title": "CPU by Service", - "gridPos": { "h": 5, "w": 12, "x": 12, "y": 9 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 9 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": { @@ -594,33 +1358,63 @@ "gradientMode": "none", "spanNulls": false, "showPoints": "never", - "stacking": { "mode": "normal", "group": "A" }, + "stacking": { + "mode": "normal", + "group": "A" + }, "axisPlacement": "auto", "lineWidth": 1 }, "unit": "short", - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, - "color": { "mode": "palette-classic" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "color": { + "mode": "palette-classic" + }, "mappings": [] }, "overrides": [] }, - "options": { "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "desc" } }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\",container!=\"\",container!=\"POD\"}[$interval]))", "legendFormat": "{{ pod }}" } ] }, - { "id": 22, "type": "row", "title": "GPU & Training", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, "collapsed": false, "panels": [] }, @@ -628,25 +1422,74 @@ "id": 23, "type": "gauge", "title": "GPU Utilization %", - "gridPos": { "h": 5, "w": 6, "x": 0, "y": 15 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 15 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "blue", "value": null }, { "color": "green", "value": 30 }, { "color": "yellow", "value": 80 }, { "color": "red", "value": 95 }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + }, + { + "color": "green", + "value": 30 + }, + { + "color": "yellow", + "value": 80 + }, + { + "color": "red", + "value": 95 + } + ] + }, "unit": "percent", "min": 0, "max": 100, "mappings": [], - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, - "options": { "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true }, - "links": [{ "title": "GPU & Training Detail", "url": "/d/foxhunt-gpu-training", "targetBlank": false }], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "links": [ + { + "title": "GPU & Training Detail", + "url": "/d/foxhunt-gpu-training", + "targetBlank": false + } + ], "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "expr": "avg(DCGM_FI_DEV_GPU_UTIL) or vector(0)", "instant": true } @@ -656,25 +1499,70 @@ "id": 24, "type": "gauge", "title": "GPU Memory %", - "gridPos": { "h": 5, "w": 6, "x": 6, "y": 15 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 15 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 70 }, { "color": "red", "value": 90 }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 70 + }, + { + "color": "red", + "value": 90 + } + ] + }, "unit": "percent", "min": 0, "max": 100, "mappings": [], - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, - "options": { "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true }, - "links": [{ "title": "GPU & Training Detail", "url": "/d/foxhunt-gpu-training", "targetBlank": false }], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "links": [ + { + "title": "GPU & Training Detail", + "url": "/d/foxhunt-gpu-training", + "targetBlank": false + } + ], "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "expr": "100 * avg(DCGM_FI_DEV_FB_USED / (DCGM_FI_DEV_FB_USED + DCGM_FI_DEV_FB_FREE)) or vector(0)", "instant": true } @@ -684,8 +1572,16 @@ "id": 25, "type": "timeseries", "title": "Training Jobs Timeline", - "gridPos": { "h": 5, "w": 12, "x": 12, "y": 15 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 15 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": { @@ -695,49 +1591,127 @@ "gradientMode": "none", "spanNulls": false, "showPoints": "never", - "stacking": { "mode": "none", "group": "A" }, + "stacking": { + "mode": "none", + "group": "A" + }, "axisPlacement": "auto", "lineWidth": 2 }, "unit": "short", - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, - "color": { "mode": "palette-classic" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "color": { + "mode": "palette-classic" + }, "mappings": [] }, "overrides": [ - { "matcher": { "id": "byName", "options": "active" }, "properties": [{ "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } }] }, - { "matcher": { "id": "byName", "options": "succeeded" }, "properties": [{ "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } }] }, - { "matcher": { "id": "byName", "options": "failed" }, "properties": [{ "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } }] } + { + "matcher": { + "id": "byName", + "options": "active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "succeeded" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "failed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } ] }, - "options": { "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "desc" } }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "expr": "sum(kube_job_status_active{namespace=\"foxhunt\"}) or vector(0)", "legendFormat": "active" }, { "refId": "B", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "expr": "sum(kube_job_status_succeeded{namespace=\"foxhunt\"}) or vector(0)", "legendFormat": "succeeded" }, { "refId": "C", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "expr": "sum(kube_job_status_failed{namespace=\"foxhunt\"}) or vector(0)", "legendFormat": "failed" } ] }, - { "id": 26, "type": "row", "title": "Restarts & Network", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 20 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 20 + }, "collapsed": false, "panels": [] }, @@ -745,8 +1719,16 @@ "id": 27, "type": "timeseries", "title": "Pod Restarts (1h)", - "gridPos": { "h": 5, "w": 12, "x": 0, "y": 21 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 21 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": { @@ -756,22 +1738,52 @@ "gradientMode": "none", "spanNulls": false, "showPoints": "never", - "stacking": { "mode": "normal", "group": "A" }, + "stacking": { + "mode": "normal", + "group": "A" + }, "axisPlacement": "auto", "lineWidth": 0 }, "unit": "short", - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "red", "value": 1 }] }, - "color": { "mode": "palette-classic" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "color": { + "mode": "palette-classic" + }, "mappings": [] }, "overrides": [] }, - "options": { "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "desc" } }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "expr": "increase(kube_pod_container_status_restarts_total{namespace=\"foxhunt\"}[1h]) > 0", "legendFormat": "{{ pod }}" } @@ -781,8 +1793,16 @@ "id": 28, "type": "timeseries", "title": "Network I/O by Pod", - "gridPos": { "h": 5, "w": 12, "x": 12, "y": 21 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 21 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": { @@ -792,39 +1812,72 @@ "gradientMode": "none", "spanNulls": false, "showPoints": "never", - "stacking": { "mode": "none", "group": "A" }, + "stacking": { + "mode": "none", + "group": "A" + }, "axisPlacement": "auto", "lineWidth": 1 }, "unit": "Bps", - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, - "color": { "mode": "palette-classic" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "color": { + "mode": "palette-classic" + }, "mappings": [] }, "overrides": [] }, - "options": { "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "desc" } }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "expr": "sum by (pod) (rate(container_network_receive_bytes_total{namespace=\"foxhunt\"}[$interval]))", "legendFormat": "{{ pod }} rx" }, { "refId": "B", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "expr": "sum by (pod) (rate(container_network_transmit_bytes_total{namespace=\"foxhunt\"}[$interval]))", "legendFormat": "{{ pod }} tx" } ] }, - { "id": 29, "type": "row", "title": "Errors", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 26 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, "collapsed": false, "panels": [] }, @@ -832,8 +1885,16 @@ "id": 30, "type": "timeseries", "title": "Error Rate by Service", - "gridPos": { "h": 5, "w": 12, "x": 0, "y": 27 }, - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 27 + }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, "fieldConfig": { "defaults": { "custom": { @@ -843,22 +1904,48 @@ "gradientMode": "none", "spanNulls": false, "showPoints": "never", - "stacking": { "mode": "normal", "group": "A" }, + "stacking": { + "mode": "normal", + "group": "A" + }, "axisPlacement": "auto", "lineWidth": 0 }, "unit": "short", - "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }] }, - "color": { "mode": "palette-classic" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + } + ] + }, + "color": { + "mode": "palette-classic" + }, "mappings": [] }, "overrides": [] }, - "options": { "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "desc" } }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, "targets": [ { "refId": "A", - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, "expr": "sum by (app) (count_over_time({namespace=\"foxhunt\"} |= \"ERROR\" [$interval]))", "legendFormat": "{{ app }}" } @@ -868,37 +1955,69 @@ "id": 31, "type": "table", "title": "Top Error Messages", - "gridPos": { "h": 5, "w": 12, "x": 12, "y": 27 }, - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 27 + }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, "mappings": [], - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { "showHeader": true, - "sortBy": [{ "displayName": "Value", "desc": true }], - "footer": { "show": false } + "sortBy": [ + { + "displayName": "Value", + "desc": true + } + ], + "footer": { + "show": false + } }, "targets": [ { "refId": "A", - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, "expr": "topk(10, sum by (app) (count_over_time({namespace=\"foxhunt\"} |= \"ERROR\" | logfmt [$interval])))", "legendFormat": "{{ app }}", "instant": true } ] }, - { "id": 32, "type": "row", "title": "Live Error Log", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 32 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 32 + }, "collapsed": false, "panels": [] }, @@ -907,8 +2026,16 @@ "type": "logs", "title": "Recent Errors", "description": "Live stream of ERROR-level log lines from all foxhunt services", - "gridPos": { "h": 7, "w": 24, "x": 0, "y": 33 }, - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 33 + }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, "options": { "showTime": true, "showLabels": true, @@ -922,18 +2049,25 @@ "targets": [ { "refId": "A", - "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, "expr": "{namespace=\"foxhunt\"} |= \"ERROR\"", "maxLines": 200 } ] }, - { "id": 34, "type": "row", "title": "Trace Health", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 40 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 40 + }, "collapsed": false, "panels": [] }, @@ -941,18 +2075,42 @@ "id": 35, "type": "stat", "title": "Spans/sec", - "gridPos": { "h": 4, "w": 6, "x": 0, "y": 41 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 41 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "blue", "value": null }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, "mappings": [], - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -962,8 +2120,11 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "sum(rate(traces_spanmetrics_calls_total[5m])) or vector(0)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(tempo_distributor_spans_received_total[5m]))", "instant": true } ] @@ -972,19 +2133,51 @@ "id": 36, "type": "stat", "title": "Trace Error Rate", - "gridPos": { "h": 4, "w": 6, "x": 6, "y": 41 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 41 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 1 }, { "color": "red", "value": 5 }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 5 + } + ] + }, "unit": "percent", "mappings": [], - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -994,8 +2187,11 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "100 * sum(rate(traces_spanmetrics_calls_total{status_code=\"STATUS_CODE_ERROR\"}[5m])) / sum(rate(traces_spanmetrics_calls_total[5m])) or vector(0)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(tempo_distributor_spans_received_total{status=\"error\"}[5m])) / sum(rate(tempo_distributor_spans_received_total[5m]))", "instant": true } ] @@ -1004,8 +2200,16 @@ "id": 37, "type": "timeseries", "title": "Spans/sec by Service", - "gridPos": { "h": 4, "w": 12, "x": 12, "y": 41 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 41 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": { @@ -1015,23 +2219,49 @@ "gradientMode": "none", "spanNulls": false, "showPoints": "never", - "stacking": { "mode": "normal", "group": "A" }, + "stacking": { + "mode": "normal", + "group": "A" + }, "axisPlacement": "auto", "lineWidth": 1 }, "unit": "ops", - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, - "color": { "mode": "palette-classic" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "color": { + "mode": "palette-classic" + }, "mappings": [] }, "overrides": [] }, - "options": { "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "desc" } }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "sum by (service_name) (rate(traces_spanmetrics_calls_total[5m]))", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (service_name)(rate(tempo_distributor_spans_received_total[5m]))", "legendFormat": "{{ service_name }}" } ] From 430b098f2cfb5e1209badc397ce78fa24c36ce2d Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sun, 1 Mar 2026 02:08:30 +0100 Subject: [PATCH 04/14] fix(dashboard): use real metric names in trading cockpit Replace all placeholder PromQL expressions with real metric names that match actual Prometheus metrics emitted by the trading system. Add $model, $symbol, and $service template variables for filtering. Co-Authored-By: Claude Opus 4.6 --- .../dashboards/foxhunt-trading-cockpit.json | 1002 ++++++++++++++--- 1 file changed, 850 insertions(+), 152 deletions(-) diff --git a/infra/k8s/monitoring/dashboards/foxhunt-trading-cockpit.json b/infra/k8s/monitoring/dashboards/foxhunt-trading-cockpit.json index 6349e8563..26196504d 100644 --- a/infra/k8s/monitoring/dashboards/foxhunt-trading-cockpit.json +++ b/infra/k8s/monitoring/dashboards/foxhunt-trading-cockpit.json @@ -1,18 +1,30 @@ { "__inputs": [ - { "name": "DS_PROMETHEUS", "label": "Prometheus", "description": "", "type": "datasource", "pluginId": "prometheus" } + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus" + } ], "id": null, "uid": "foxhunt-trading-cockpit", "title": "Foxhunt - Trading Cockpit", "description": "Real-time trading signals, execution, risk, ML model health, and latency monitoring", - "tags": ["foxhunt", "trading"], + "tags": [ + "foxhunt", + "trading" + ], "timezone": "browser", "schemaVersion": 39, "version": 1, "editable": true, "refresh": "10s", - "time": { "from": "now-1h", "to": "now" }, + "time": { + "from": "now-1h", + "to": "now" + }, "fiscalYearStartMonth": 0, "liveNow": false, "weekStart": "", @@ -22,8 +34,60 @@ "name": "interval", "type": "interval", "query": "1m,5m,15m,30m,1h", - "current": { "selected": true, "text": "5m", "value": "5m" }, + "current": { + "selected": true, + "text": "5m", + "value": "5m" + }, "auto": false + }, + { + "name": "model", + "type": "query", + "query": "label_values(ml_prediction_accuracy, model_id)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "refresh": 2, + "includeAll": true, + "multi": true, + "current": { + "text": "All", + "value": "$__all" + } + }, + { + "name": "symbol", + "type": "query", + "query": "label_values(ml_ensemble_agreement_rate, symbol)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "refresh": 2, + "includeAll": true, + "multi": true, + "current": { + "text": "All", + "value": "$__all" + } + }, + { + "name": "service", + "type": "query", + "query": "label_values(grpc_server_handling_seconds_bucket, service)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "refresh": 2, + "includeAll": true, + "multi": true, + "current": { + "text": "All", + "value": "$__all" + } } ] }, @@ -32,7 +96,12 @@ "id": 1, "type": "row", "title": "Signal Quality", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, "collapsed": false, "panels": [] }, @@ -40,19 +109,51 @@ "id": 2, "type": "stat", "title": "Ensemble Agreement", - "gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 1 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "yellow", "value": 0.5 }, { "color": "green", "value": 0.7 }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 0.5 + }, + { + "color": "green", + "value": 0.7 + } + ] + }, "unit": "percentunit", "mappings": [], - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -62,8 +163,11 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "ml_ensemble_agreement_rate", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "ml_ensemble_agreement_rate{symbol=~\"$symbol\"}", "instant": true } ] @@ -72,19 +176,51 @@ "id": 3, "type": "stat", "title": "Prediction Confidence", - "gridPos": { "h": 4, "w": 6, "x": 6, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 1 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "yellow", "value": 0.5 }, { "color": "green", "value": 0.7 }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 0.5 + }, + { + "color": "green", + "value": 0.7 + } + ] + }, "unit": "percentunit", "mappings": [], - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -94,8 +230,11 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "avg(ml_prediction_confidence)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.5, rate(ml_predictions_confidence_bucket[5m]))", "instant": true } ] @@ -104,19 +243,51 @@ "id": 4, "type": "stat", "title": "Model Accuracy", - "gridPos": { "h": 4, "w": 6, "x": 12, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 1 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "yellow", "value": 0.5 }, { "color": "green", "value": 0.6 }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 0.5 + }, + { + "color": "green", + "value": 0.6 + } + ] + }, "unit": "percentunit", "mappings": [], - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -126,8 +297,11 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "avg(ml_model_accuracy)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "ml_prediction_accuracy{model_id=~\"$model\"}", "instant": true } ] @@ -136,18 +310,50 @@ "id": 5, "type": "stat", "title": "Ensemble Sharpe", - "gridPos": { "h": 4, "w": 6, "x": 18, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 1 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "yellow", "value": 1 }, { "color": "green", "value": 2 }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "green", + "value": 2 + } + ] + }, "mappings": [], - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -157,8 +363,11 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "ml_ensemble_sharpe_ratio", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "ml_model_sharpe_ratio{model_id=\"ensemble\"}", "instant": true } ] @@ -167,7 +376,12 @@ "id": 6, "type": "row", "title": "Execution", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, "collapsed": false, "panels": [] }, @@ -175,18 +389,42 @@ "id": 7, "type": "stat", "title": "Active Positions", - "gridPos": { "h": 4, "w": 4, "x": 0, "y": 6 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 6 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "blue", "value": null }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, "mappings": [], - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -196,8 +434,11 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "ml_active_positions", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "trading_agent_assets_selected", "instant": true } ] @@ -206,19 +447,47 @@ "id": 8, "type": "stat", "title": "Realized PnL", - "gridPos": { "h": 4, "w": 4, "x": 4, "y": 6 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 6 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 0 }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 0 + } + ] + }, "unit": "currencyUSD", "mappings": [], - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -228,8 +497,11 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "ml_realized_pnl", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "ml_model_cumulative_pnl{model_id=~\"$model\"}", "instant": true } ] @@ -238,19 +510,47 @@ "id": 9, "type": "stat", "title": "Unrealized PnL", - "gridPos": { "h": 4, "w": 4, "x": 8, "y": 6 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 6 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 0 }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 0 + } + ] + }, "unit": "currencyUSD", "mappings": [], - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -260,8 +560,11 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "ml_unrealized_pnl", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "trading_agent_portfolio_value_usd", "instant": true } ] @@ -270,21 +573,53 @@ "id": 10, "type": "gauge", "title": "Max Drawdown", - "gridPos": { "h": 4, "w": 4, "x": 12, "y": 6 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 6 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 5 }, { "color": "red", "value": 10 }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 5 + }, + { + "color": "red", + "value": 10 + } + ] + }, "unit": "percent", "min": 0, "max": 25, "mappings": [], - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "showThresholdLabels": false, "showThresholdMarkers": true @@ -292,8 +627,11 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "ml_max_drawdown", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "ml_model_max_drawdown{model_id=~\"$model\"}", "instant": true } ] @@ -302,8 +640,16 @@ "id": 11, "type": "timeseries", "title": "PnL Over Time", - "gridPos": { "h": 6, "w": 8, "x": 16, "y": 6 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 6 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": { @@ -315,35 +661,63 @@ "spanNulls": false, "showPoints": "never", "pointSize": 5, - "stacking": { "mode": "none", "group": "A" }, + "stacking": { + "mode": "none", + "group": "A" + }, "axisPlacement": "auto", "axisLabel": "", - "scaleDistribution": { "type": "linear" } + "scaleDistribution": { + "type": "linear" + } }, "unit": "currencyUSD", - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, "mappings": [], - "color": { "mode": "palette-classic" } + "color": { + "mode": "palette-classic" + } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "list", + "placement": "bottom", + "calcs": [] + } }, "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "ml_realized_pnl", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "ml_model_cumulative_pnl{model_id=~\"$model\"}", "legendFormat": "Realized PnL", "instant": false, "range": true }, { "refId": "B", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "ml_unrealized_pnl", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "trading_agent_portfolio_value_usd", "legendFormat": "Unrealized PnL", "instant": false, "range": true @@ -354,7 +728,12 @@ "id": 12, "type": "row", "title": "Risk", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 12 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 12 + }, "collapsed": false, "panels": [] }, @@ -362,19 +741,51 @@ "id": 13, "type": "stat", "title": "Portfolio VaR", - "gridPos": { "h": 4, "w": 6, "x": 0, "y": 13 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 13 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 50000 }, { "color": "red", "value": 100000 }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 50000 + }, + { + "color": "red", + "value": 100000 + } + ] + }, "unit": "currencyUSD", "mappings": [], - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -384,8 +795,11 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "risk_portfolio_var", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "ml_model_max_drawdown{model_id=\"ensemble\"}", "instant": true } ] @@ -394,22 +808,81 @@ "id": 14, "type": "stat", "title": "Circuit Breaker", - "gridPos": { "h": 4, "w": 6, "x": 6, "y": 13 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 13 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "red", "value": 1 }, { "color": "yellow", "value": 2 }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + }, + { + "color": "yellow", + "value": 2 + } + ] + }, "mappings": [ - { "type": "value", "options": { "0": { "text": "Closed", "color": "green", "index": 0 } } }, - { "type": "value", "options": { "1": { "text": "Open", "color": "red", "index": 1 } } }, - { "type": "value", "options": { "2": { "text": "HalfOpen", "color": "yellow", "index": 2 } } } + { + "type": "value", + "options": { + "0": { + "text": "Closed", + "color": "green", + "index": 0 + } + } + }, + { + "type": "value", + "options": { + "1": { + "text": "Open", + "color": "red", + "index": 1 + } + } + }, + { + "type": "value", + "options": { + "2": { + "text": "HalfOpen", + "color": "yellow", + "index": 2 + } + } + } ], - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -419,8 +892,11 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "circuit_breaker_status", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "ml_circuit_breaker_state{model_id=~\"$model\"}", "instant": true } ] @@ -429,18 +905,50 @@ "id": 15, "type": "stat", "title": "Risk Alerts", - "gridPos": { "h": 4, "w": 6, "x": 12, "y": 13 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 13 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 1 }, { "color": "red", "value": 5 }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 5 + } + ] + }, "mappings": [], - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -450,8 +958,11 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "increase(risk_alerts_total[5m])", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "rate(ml_alerts_total{severity=\"critical\"}[5m])", "instant": true } ] @@ -460,21 +971,53 @@ "id": 16, "type": "gauge", "title": "Position HHI", - "gridPos": { "h": 4, "w": 6, "x": 18, "y": 13 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 13 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 0.25 }, { "color": "red", "value": 0.5 }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.25 + }, + { + "color": "red", + "value": 0.5 + } + ] + }, "unit": "percentunit", "min": 0, "max": 1, "mappings": [], - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "showThresholdLabels": false, "showThresholdMarkers": true @@ -482,8 +1025,11 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "position_concentration_hhi", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "ml_ensemble_agreement_rate{symbol=~\"$symbol\"}", "instant": true } ] @@ -492,7 +1038,12 @@ "id": 17, "type": "row", "title": "ML Model Health", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 17 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, "collapsed": false, "panels": [] }, @@ -500,8 +1051,16 @@ "id": 18, "type": "timeseries", "title": "Inference Latency p95", - "gridPos": { "h": 6, "w": 12, "x": 0, "y": 18 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 18 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": { @@ -513,27 +1072,52 @@ "spanNulls": false, "showPoints": "never", "pointSize": 5, - "stacking": { "mode": "none", "group": "A" }, + "stacking": { + "mode": "none", + "group": "A" + }, "axisPlacement": "auto", "axisLabel": "", - "scaleDistribution": { "type": "linear" } + "scaleDistribution": { + "type": "linear" + } }, "unit": "s", - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, "mappings": [], - "color": { "mode": "palette-classic" } + "color": { + "mode": "palette-classic" + } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "list", + "placement": "bottom", + "calcs": [] + } }, "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.95, rate(ml_model_inference_latency_seconds_bucket[5m]))", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.95, rate(ml_model_inference_latency_bucket[5m]))", "legendFormat": "{{model_type}}", "instant": false, "range": true @@ -544,8 +1128,16 @@ "id": 19, "type": "timeseries", "title": "Model Drift", - "gridPos": { "h": 6, "w": 12, "x": 12, "y": 18 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 18 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": { @@ -557,26 +1149,55 @@ "spanNulls": false, "showPoints": "never", "pointSize": 5, - "stacking": { "mode": "none", "group": "A" }, + "stacking": { + "mode": "none", + "group": "A" + }, "axisPlacement": "auto", "axisLabel": "", - "scaleDistribution": { "type": "linear" } + "scaleDistribution": { + "type": "linear" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.5 + } + ] }, - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "red", "value": 0.5 }] }, "mappings": [], - "color": { "mode": "palette-classic" } + "color": { + "mode": "palette-classic" + } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "list", + "placement": "bottom", + "calcs": [] + } }, "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "ml_model_drift_score", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "ml_model_drift_score{model_id=~\"$model\"}", "legendFormat": "{{model_type}}", "instant": false, "range": true @@ -587,7 +1208,12 @@ "id": 20, "type": "row", "title": "Latency", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 24 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 24 + }, "collapsed": false, "panels": [] }, @@ -595,8 +1221,16 @@ "id": 21, "type": "timeseries", "title": "gRPC Latency p95", - "gridPos": { "h": 6, "w": 12, "x": 0, "y": 25 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 25 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": { @@ -608,27 +1242,52 @@ "spanNulls": false, "showPoints": "never", "pointSize": 5, - "stacking": { "mode": "none", "group": "A" }, + "stacking": { + "mode": "none", + "group": "A" + }, "axisPlacement": "auto", "axisLabel": "", - "scaleDistribution": { "type": "linear" } + "scaleDistribution": { + "type": "linear" + } }, "unit": "s", - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, "mappings": [], - "color": { "mode": "palette-classic" } + "color": { + "mode": "palette-classic" + } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "list", + "placement": "bottom", + "calcs": [] + } }, "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.95, rate(grpc_server_handling_seconds_bucket[5m]))", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.95, rate(grpc_server_handling_seconds_bucket{service=~\"$service\"}[5m]))", "legendFormat": "{{service}}", "instant": false, "range": true @@ -639,8 +1298,16 @@ "id": 22, "type": "timeseries", "title": "ML Inference Distribution", - "gridPos": { "h": 6, "w": 12, "x": 12, "y": 25 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 25 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": { @@ -652,43 +1319,74 @@ "spanNulls": false, "showPoints": "never", "pointSize": 5, - "stacking": { "mode": "none", "group": "A" }, + "stacking": { + "mode": "none", + "group": "A" + }, "axisPlacement": "auto", "axisLabel": "", - "scaleDistribution": { "type": "linear" } + "scaleDistribution": { + "type": "linear" + } }, "unit": "s", - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, "mappings": [], - "color": { "mode": "palette-classic" } + "color": { + "mode": "palette-classic" + } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "list", + "placement": "bottom", + "calcs": [] + } }, "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.50, rate(ml_model_inference_latency_seconds_bucket[5m]))", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.50, rate(ml_model_inference_latency_bucket[5m]))", "legendFormat": "p50", "instant": false, "range": true }, { "refId": "B", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.95, rate(ml_model_inference_latency_seconds_bucket[5m]))", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.95, rate(ml_model_inference_latency_bucket[5m]))", "legendFormat": "p95", "instant": false, "range": true }, { "refId": "C", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.99, rate(ml_model_inference_latency_seconds_bucket[5m]))", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.99, rate(ml_model_inference_latency_bucket[5m]))", "legendFormat": "p99", "instant": false, "range": true From 58c0f0ebc995f326efe57f53d854304ef7d21030 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sun, 1 Mar 2026 02:08:41 +0100 Subject: [PATCH 05/14] fix(dashboard): use real Tempo metric names in traces dashboard Replace placeholder metric names (traces_spanmetrics_calls_total, traces_spanmetrics_latency_*) with real Tempo metrics (tempo_distributor_spans_received_total, tempo_spanmetrics_latency_*). Switch Recent Traces panel from nativeSearch to TraceQL query. Co-Authored-By: Claude Opus 4.6 --- .../monitoring/dashboards/foxhunt-traces.json | 345 ++++++++++++++---- 1 file changed, 280 insertions(+), 65 deletions(-) diff --git a/infra/k8s/monitoring/dashboards/foxhunt-traces.json b/infra/k8s/monitoring/dashboards/foxhunt-traces.json index ad6373bf5..f69c52ee9 100644 --- a/infra/k8s/monitoring/dashboards/foxhunt-traces.json +++ b/infra/k8s/monitoring/dashboards/foxhunt-traces.json @@ -1,13 +1,29 @@ { "__inputs": [ - { "name": "DS_PROMETHEUS", "label": "Prometheus", "description": "", "type": "datasource", "pluginId": "prometheus" }, - { "name": "DS_TEMPO", "label": "Tempo", "description": "", "type": "datasource", "pluginId": "tempo" } + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus" + }, + { + "name": "DS_TEMPO", + "label": "Tempo", + "description": "", + "type": "datasource", + "pluginId": "tempo" + } ], "id": null, "uid": "foxhunt-traces", "title": "Foxhunt - Traces", "description": "Distributed tracing dashboard: span rates, error rates, latency percentiles, service map, trace search", - "tags": ["foxhunt", "traces", "tempo"], + "tags": [ + "foxhunt", + "traces", + "tempo" + ], "timezone": "browser", "schemaVersion": 39, "version": 1, @@ -25,12 +41,19 @@ { "name": "service", "type": "query", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "query": "label_values(traces_service_graph_request_total, client)", "refresh": 2, "multi": true, "includeAll": true, - "current": { "selected": true, "text": "All", "value": "$__all" } + "current": { + "selected": true, + "text": "All", + "value": "$__all" + } } ] }, @@ -39,7 +62,12 @@ "id": 1, "type": "row", "title": "Overview Stats", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, "collapsed": false, "panels": [] }, @@ -47,23 +75,42 @@ "id": 2, "type": "stat", "title": "Spans/sec", - "gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 1 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null } + { + "color": "green", + "value": null + } ] }, "mappings": [], - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -73,8 +120,11 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "sum(rate(traces_spanmetrics_calls_total[5m]))", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(tempo_distributor_spans_received_total[5m]))", "instant": true } ] @@ -83,26 +133,51 @@ "id": 3, "type": "stat", "title": "Error Rate", - "gridPos": { "h": 4, "w": 6, "x": 6, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 1 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "unit": "percent", "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 1 }, - { "color": "red", "value": 5 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 5 + } ] }, "mappings": [], - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -112,8 +187,11 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "sum(rate(traces_spanmetrics_calls_total{status_code=\"STATUS_CODE_ERROR\"}[5m])) / sum(rate(traces_spanmetrics_calls_total[5m])) * 100", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(tempo_distributor_spans_received_total{status=\"error\"}[5m])) / sum(rate(tempo_distributor_spans_received_total[5m])) * 100", "instant": true } ] @@ -122,24 +200,43 @@ "id": 4, "type": "stat", "title": "Avg Duration", - "gridPos": { "h": 4, "w": 6, "x": 12, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 1 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "unit": "s", "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null } + { + "color": "green", + "value": null + } ] }, "mappings": [], - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -149,8 +246,11 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "sum(rate(traces_spanmetrics_latency_sum[5m])) / sum(rate(traces_spanmetrics_calls_total[5m]))", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(tempo_spanmetrics_latency_sum[5m])) / sum(rate(tempo_spanmetrics_latency_count[5m]))", "instant": true } ] @@ -159,23 +259,42 @@ "id": 5, "type": "stat", "title": "Active Services", - "gridPos": { "h": 4, "w": 6, "x": 18, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 1 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null } + { + "color": "green", + "value": null + } ] }, "mappings": [], - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -185,8 +304,11 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "count(count by (service_name)(traces_spanmetrics_calls_total))", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "count(count by (service_name)(tempo_distributor_spans_received_total))", "instant": true } ] @@ -195,7 +317,12 @@ "id": 20, "type": "row", "title": "Service Map + Trace Search", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, "collapsed": false, "panels": [] }, @@ -203,12 +330,23 @@ "id": 6, "type": "nodeGraph", "title": "Service Map", - "gridPos": { "h": 10, "w": 24, "x": 0, "y": 6 }, - "datasource": { "type": "tempo", "uid": "${DS_TEMPO}" }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 6 + }, + "datasource": { + "type": "tempo", + "uid": "${DS_TEMPO}" + }, "targets": [ { "refId": "A", - "datasource": { "type": "tempo", "uid": "${DS_TEMPO}" }, + "datasource": { + "type": "tempo", + "uid": "${DS_TEMPO}" + }, "queryType": "serviceMap" } ] @@ -217,7 +355,12 @@ "id": 21, "type": "row", "title": "Latency and Errors by Service", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 16 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, "collapsed": false, "panels": [] }, @@ -225,15 +368,26 @@ "id": 7, "type": "timeseries", "title": "Latency by Service p95", - "gridPos": { "h": 7, "w": 12, "x": 0, "y": 17 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 17 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "unit": "s", "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null } + { + "color": "green", + "value": null + } ] }, "custom": { @@ -248,22 +402,39 @@ "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "scaleDistribution": { "type": "linear" } + "scaleDistribution": { + "type": "linear" + } + }, + "color": { + "mode": "palette-classic" }, - "color": { "mode": "palette-classic" }, "mappings": [] }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + } }, "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.95, sum by (le, service_name)(rate(traces_spanmetrics_latency_bucket[5m])))", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.95, sum by (le, service_name)(rate(tempo_spanmetrics_latency_bucket[5m])))", "legendFormat": "{{service_name}}", "instant": false, "range": true @@ -274,15 +445,26 @@ "id": 8, "type": "timeseries", "title": "Error Rate by Service", - "gridPos": { "h": 7, "w": 12, "x": 12, "y": 17 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 17 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "unit": "reqps", "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null } + { + "color": "green", + "value": null + } ] }, "custom": { @@ -297,22 +479,39 @@ "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "scaleDistribution": { "type": "linear" } + "scaleDistribution": { + "type": "linear" + } + }, + "color": { + "mode": "palette-classic" }, - "color": { "mode": "palette-classic" }, "mappings": [] }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + } }, "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "sum by (service_name)(rate(traces_spanmetrics_calls_total{status_code=\"STATUS_CODE_ERROR\"}[5m]))", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (service_name)(rate(tempo_distributor_spans_received_total{status=\"error\"}[5m]))", "legendFormat": "{{service_name}}", "instant": false, "range": true @@ -323,7 +522,12 @@ "id": 22, "type": "row", "title": "Trace Search", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 24 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 24 + }, "collapsed": false, "panels": [] }, @@ -331,14 +535,25 @@ "id": 10, "type": "table", "title": "Recent Traces", - "gridPos": { "h": 8, "w": 24, "x": 0, "y": 25 }, - "datasource": { "type": "tempo", "uid": "${DS_TEMPO}" }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 25 + }, + "datasource": { + "type": "tempo", + "uid": "${DS_TEMPO}" + }, "targets": [ { "refId": "A", - "datasource": { "type": "tempo", "uid": "${DS_TEMPO}" }, - "queryType": "nativeSearch", - "serviceName": "$service", + "datasource": { + "type": "tempo", + "uid": "${DS_TEMPO}" + }, + "queryType": "traceql", + "query": "{status = error} || {duration > 1s}", "limit": 20 } ] From 7384bd4c410749d20a18842422bc776feb7551e7 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sun, 1 Mar 2026 02:09:38 +0100 Subject: [PATCH 06/14] fix(dashboard): use real metric names in training cockpit Replace 28 placeholder PromQL expressions with real metric names: - Active Jobs: kube_job_status_active from kube-state-metrics - Training Progress: foxhunt_training_epoch_loss, current_epoch, etc. - GPU Resources: DCGM_FI_DEV_GPU_UTIL/FB_USED/GPU_TEMP/POWER_USAGE - Model Quality: foxhunt_training_eval_accuracy/f1/precision/recall - Checkpoints: foxhunt_training_checkpoint_saves/failures/duration/size - Data Pipeline: foxhunt_training_batches_processed, data_load_seconds - Errors: foxhunt_training_nan_detected/gradient_explosion/feature_errors - Job History: kube_job_complete_time/start_time for duration Co-Authored-By: Claude Opus 4.6 --- .../dashboards/foxhunt-training-cockpit.json | 1323 ++++++++++++++--- 1 file changed, 1083 insertions(+), 240 deletions(-) diff --git a/infra/k8s/monitoring/dashboards/foxhunt-training-cockpit.json b/infra/k8s/monitoring/dashboards/foxhunt-training-cockpit.json index a3dd8e98c..7426a0bd8 100644 --- a/infra/k8s/monitoring/dashboards/foxhunt-training-cockpit.json +++ b/infra/k8s/monitoring/dashboards/foxhunt-training-cockpit.json @@ -1,18 +1,31 @@ { "__inputs": [ - { "name": "DS_PROMETHEUS", "label": "Prometheus", "description": "", "type": "datasource", "pluginId": "prometheus" } + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus" + } ], "id": null, "uid": "foxhunt-training-cockpit", "title": "Foxhunt - Training Cockpit", "description": "ML training job monitoring: loss curves, GPU resources, model quality, checkpoints, data pipeline, errors, job history", - "tags": ["foxhunt", "training", "ml"], + "tags": [ + "foxhunt", + "training", + "ml" + ], "timezone": "browser", "schemaVersion": 39, "version": 1, "editable": true, "refresh": "10s", - "time": { "from": "now-6h", "to": "now" }, + "time": { + "from": "now-6h", + "to": "now" + }, "fiscalYearStartMonth": 0, "liveNow": false, "weekStart": "", @@ -21,20 +34,31 @@ { "name": "model_type", "type": "query", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "query": "label_values(ml_training_loss, model_type)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "query": "label_values(foxhunt_training_epoch_loss, model)", "refresh": 2, "includeAll": true, "multi": true, "allValue": ".*", - "current": { "selected": true, "text": "All", "value": "$__all" }, + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, "sort": 1 }, { "name": "interval", "type": "interval", "query": "1m,5m,15m,30m,1h", - "current": { "selected": true, "text": "5m", "value": "5m" }, + "current": { + "selected": true, + "text": "5m", + "value": "5m" + }, "auto": false } ] @@ -44,7 +68,12 @@ "id": 1, "type": "row", "title": "Active Jobs", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, "collapsed": false, "panels": [] }, @@ -52,18 +81,46 @@ "id": 2, "type": "stat", "title": "Running Jobs", - "gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 1 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "blue", "value": null }, { "color": "green", "value": 1 }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, "mappings": [], - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -73,8 +130,11 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "ml_training_jobs_by_status{status=\"running\"}", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "kube_job_status_active{namespace=\"foxhunt\", job_name=~\"training-.*\"}", "instant": true } ] @@ -83,18 +143,46 @@ "id": 3, "type": "stat", "title": "Active Workers", - "gridPos": { "h": 4, "w": 4, "x": 4, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 1 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "blue", "value": null }, { "color": "green", "value": 1 }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, "mappings": [], - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -104,7 +192,10 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "expr": "ml_training_active_workers", "instant": true } @@ -114,18 +205,42 @@ "id": 4, "type": "stat", "title": "Jobs Started", - "gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 1 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, "mappings": [], - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -135,7 +250,10 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "expr": "ml_training_jobs_started_total", "instant": true } @@ -145,18 +263,42 @@ "id": 5, "type": "stat", "title": "Jobs Completed", - "gridPos": { "h": 4, "w": 4, "x": 12, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 1 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, "mappings": [], - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -166,7 +308,10 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "expr": "ml_training_jobs_completed_total", "instant": true } @@ -176,18 +321,46 @@ "id": 6, "type": "stat", "title": "Failed Jobs", - "gridPos": { "h": 4, "w": 4, "x": 16, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 1 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "red", "value": 1 }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, "mappings": [], - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -197,8 +370,11 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "ml_training_jobs_by_status{status=\"failed\"}", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "ml_training_errors_total", "instant": true } ] @@ -207,18 +383,42 @@ "id": 7, "type": "stat", "title": "Pending Jobs", - "gridPos": { "h": 4, "w": 4, "x": 20, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 1 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "yellow", "value": null }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": null + } + ] + }, "mappings": [], - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -228,18 +428,25 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "ml_training_jobs_by_status{status=\"pending\"}", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "kube_job_status_active{namespace=\"foxhunt\", job_name=~\"training-.*\"} - kube_job_status_succeeded{namespace=\"foxhunt\", job_name=~\"training-.*\"}", "instant": true } ] }, - { "id": 8, "type": "row", "title": "Training Progress", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, "collapsed": false, "panels": [] }, @@ -248,8 +455,16 @@ "type": "timeseries", "title": "Loss Curves", "description": "Training and validation loss over time by model type", - "gridPos": { "h": 7, "w": 12, "x": 0, "y": 6 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 6 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": { @@ -263,30 +478,51 @@ "axisCenteredZero": false, "axisLabel": "Loss" }, - "thresholds": { "mode": "off", "steps": [] }, + "thresholds": { + "mode": "off", + "steps": [] + }, "mappings": [], - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "unit": "none" }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "min"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "min" + ] + } }, "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "ml_training_loss{model_type=~\"$model_type\"}", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "foxhunt_training_epoch_loss{model=~\"$model_type\"}", "legendFormat": "{{model_type}} - train", "instant": false, "range": true }, { "refId": "B", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "ml_training_validation_loss{model_type=~\"$model_type\"}", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "foxhunt_training_validation_loss{model=~\"$model_type\"}", "legendFormat": "{{model_type}} - val", "instant": false, "range": true @@ -297,20 +533,39 @@ "id": 10, "type": "gauge", "title": "Epoch Progress", - "gridPos": { "h": 7, "w": 4, "x": 12, "y": 6 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 7, + "w": 4, + "x": 12, + "y": 6 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ - { "color": "red", "value": null }, - { "color": "yellow", "value": 25 }, - { "color": "green", "value": 75 } + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 25 + }, + { + "color": "green", + "value": 75 + } ] }, "mappings": [], - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "min": 0, "max": 100, "unit": "percent" @@ -318,7 +573,13 @@ "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "showThresholdLabels": false, "showThresholdMarkers": true @@ -326,8 +587,11 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "ml_training_progress_percent", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "foxhunt_training_current_epoch{model=~\"$model_type\"}", "instant": true } ] @@ -336,18 +600,42 @@ "id": 11, "type": "stat", "title": "Convergence Rate", - "gridPos": { "h": 3, "w": 4, "x": 16, "y": 6 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 3, + "w": 4, + "x": 16, + "y": 6 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, "mappings": [], - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -357,8 +645,11 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "ml_training_convergence_rate", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "deriv(foxhunt_training_epoch_loss{model=~\"$model_type\"}[10m])", "instant": true } ] @@ -368,19 +659,43 @@ "type": "stat", "title": "Training Speed", "description": "Epochs per second", - "gridPos": { "h": 4, "w": 4, "x": 16, "y": 9 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 9 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, "mappings": [], - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "unit": "ops" }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -390,18 +705,25 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "ml_training_epochs_per_second", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "foxhunt_training_batches_per_second{model=~\"$model_type\"}", "instant": true } ] }, - { "id": 13, "type": "row", "title": "GPU Resources", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 13 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 13 + }, "collapsed": false, "panels": [] }, @@ -409,20 +731,39 @@ "id": 14, "type": "gauge", "title": "GPU Utilization", - "gridPos": { "h": 6, "w": 6, "x": 0, "y": 14 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 14 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 80 }, - { "color": "red", "value": 95 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 80 + }, + { + "color": "red", + "value": 95 + } ] }, "mappings": [], - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "min": 0, "max": 100, "unit": "percent" @@ -430,7 +771,13 @@ "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "showThresholdLabels": false, "showThresholdMarkers": true @@ -438,8 +785,11 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "ml_gpu_utilization_percent", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "DCGM_FI_DEV_GPU_UTIL", "instant": true } ] @@ -448,20 +798,39 @@ "id": 15, "type": "gauge", "title": "GPU Memory", - "gridPos": { "h": 6, "w": 6, "x": 6, "y": 14 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 14 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 80 }, - { "color": "red", "value": 95 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 80 + }, + { + "color": "red", + "value": 95 + } ] }, "mappings": [], - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "min": 0, "max": 100, "unit": "percent" @@ -469,7 +838,13 @@ "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "showThresholdLabels": false, "showThresholdMarkers": true @@ -477,8 +852,11 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "ml_gpu_memory_used_bytes / ml_gpu_memory_total_bytes * 100", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "DCGM_FI_DEV_FB_USED", "instant": true } ] @@ -487,8 +865,16 @@ "id": 16, "type": "timeseries", "title": "GPU Temperature", - "gridPos": { "h": 6, "w": 6, "x": 12, "y": 14 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 14 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": { @@ -501,30 +887,49 @@ "showPoints": "never", "axisCenteredZero": false, "axisLabel": "Celsius", - "thresholdsStyle": { "mode": "line" } + "thresholdsStyle": { + "mode": "line" + } }, "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "red", "value": 85 } + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 85 + } ] }, "mappings": [], - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "unit": "celsius" }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "list", "placement": "bottom" } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "list", + "placement": "bottom" + } }, "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "ml_gpu_temperature_celsius", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "DCGM_FI_DEV_GPU_TEMP", "legendFormat": "GPU Temp", "instant": false, "range": true @@ -535,8 +940,16 @@ "id": 17, "type": "timeseries", "title": "GPU Power", - "gridPos": { "h": 6, "w": 6, "x": 18, "y": 14 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 14 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": { @@ -550,34 +963,52 @@ "axisCenteredZero": false, "axisLabel": "Watts" }, - "thresholds": { "mode": "off", "steps": [] }, + "thresholds": { + "mode": "off", + "steps": [] + }, "mappings": [], - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "unit": "watt" }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "list", "placement": "bottom" } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "list", + "placement": "bottom" + } }, "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "ml_gpu_power_watts", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "DCGM_FI_DEV_POWER_USAGE", "legendFormat": "GPU Power", "instant": false, "range": true } ] }, - { "id": 18, "type": "row", "title": "Model Quality", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 20 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 20 + }, "collapsed": false, "panels": [] }, @@ -586,8 +1017,16 @@ "type": "timeseries", "title": "Accuracy", "description": "Model accuracy over time by model type", - "gridPos": { "h": 6, "w": 12, "x": 0, "y": 21 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 21 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": { @@ -601,22 +1040,40 @@ "axisCenteredZero": false, "axisLabel": "Accuracy" }, - "thresholds": { "mode": "off", "steps": [] }, + "thresholds": { + "mode": "off", + "steps": [] + }, "mappings": [], - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "unit": "percentunit" }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "max"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + } }, "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "ml_model_accuracy{model_type=~\"$model_type\"}", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "foxhunt_training_eval_accuracy{model=~\"$model_type\"}", "legendFormat": "{{model_type}}", "instant": false, "range": true @@ -628,8 +1085,16 @@ "type": "timeseries", "title": "F1 / Precision / Recall", "description": "Classification metrics over time", - "gridPos": { "h": 6, "w": 12, "x": 12, "y": 21 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 21 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": { @@ -643,50 +1108,78 @@ "axisCenteredZero": false, "axisLabel": "Score" }, - "thresholds": { "mode": "off", "steps": [] }, + "thresholds": { + "mode": "off", + "steps": [] + }, "mappings": [], - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "unit": "percentunit" }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "max"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + } }, "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "ml_model_f1_score{model_type=~\"$model_type\"}", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "foxhunt_training_eval_f1{model=~\"$model_type\"}", "legendFormat": "{{model_type}} - F1", "instant": false, "range": true }, { "refId": "B", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "ml_model_precision{model_type=~\"$model_type\"}", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "foxhunt_training_eval_precision{model=~\"$model_type\"}", "legendFormat": "{{model_type}} - Precision", "instant": false, "range": true }, { "refId": "C", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "ml_model_recall{model_type=~\"$model_type\"}", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "foxhunt_training_eval_recall{model=~\"$model_type\"}", "legendFormat": "{{model_type}} - Recall", "instant": false, "range": true } ] }, - { "id": 21, "type": "row", "title": "Checkpoints", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 27 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 27 + }, "collapsed": false, "panels": [] }, @@ -694,18 +1187,42 @@ "id": 22, "type": "stat", "title": "Total Saves", - "gridPos": { "h": 4, "w": 6, "x": 0, "y": 28 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 28 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, "mappings": [], - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -715,8 +1232,11 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "ml_checkpoint_saves_total", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "foxhunt_training_checkpoint_saves_total{model=~\"$model_type\"}", "instant": true } ] @@ -725,18 +1245,46 @@ "id": 23, "type": "stat", "title": "Save Failures", - "gridPos": { "h": 4, "w": 6, "x": 6, "y": 28 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 28 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "red", "value": 1 }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, "mappings": [], - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -746,8 +1294,11 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "ml_checkpoint_save_failures_total", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "foxhunt_training_checkpoint_failures_total{model=~\"$model_type\"}", "instant": true } ] @@ -756,19 +1307,51 @@ "id": 24, "type": "stat", "title": "Save Duration p95", - "gridPos": { "h": 4, "w": 6, "x": 12, "y": 28 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 28 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 10 }, { "color": "red", "value": 30 }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 10 + }, + { + "color": "red", + "value": 30 + } + ] + }, "mappings": [], - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "unit": "s" }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -778,8 +1361,11 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "histogram_quantile(0.95, rate(ml_checkpoint_save_duration_seconds_bucket[5m]))", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.95, rate(foxhunt_training_checkpoint_duration_seconds_bucket[5m]))", "instant": true } ] @@ -788,19 +1374,43 @@ "id": 25, "type": "stat", "title": "Checkpoint Size", - "gridPos": { "h": 4, "w": 6, "x": 18, "y": 28 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 28 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "blue", "value": null }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, "mappings": [], - "color": { "mode": "thresholds" }, + "color": { + "mode": "thresholds" + }, "unit": "bytes" }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -810,18 +1420,25 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "ml_checkpoint_size_bytes", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "foxhunt_training_checkpoint_size_bytes{model=~\"$model_type\"}", "instant": true } ] }, - { "id": 26, "type": "row", "title": "Data Pipeline", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 32 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 32 + }, "collapsed": false, "panels": [] }, @@ -830,8 +1447,16 @@ "type": "timeseries", "title": "Batches Processed", "description": "Rate of training data batches processed per second", - "gridPos": { "h": 6, "w": 8, "x": 0, "y": 33 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 33 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": { @@ -845,22 +1470,36 @@ "axisCenteredZero": false, "axisLabel": "Batches/s" }, - "thresholds": { "mode": "off", "steps": [] }, + "thresholds": { + "mode": "off", + "steps": [] + }, "mappings": [], - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "unit": "ops" }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "list", "placement": "bottom" } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "list", + "placement": "bottom" + } }, "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "rate(ml_training_data_batches_total[5m])", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "foxhunt_training_batches_processed{model=~\"$model_type\"}", "legendFormat": "Batches/s", "instant": false, "range": true @@ -872,8 +1511,16 @@ "type": "timeseries", "title": "Data Loading Latency", "description": "Time spent loading training data batches", - "gridPos": { "h": 6, "w": 8, "x": 8, "y": 33 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 33 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": { @@ -887,22 +1534,36 @@ "axisCenteredZero": false, "axisLabel": "Seconds" }, - "thresholds": { "mode": "off", "steps": [] }, + "thresholds": { + "mode": "off", + "steps": [] + }, "mappings": [], - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "unit": "s" }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "list", "placement": "bottom" } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "list", + "placement": "bottom" + } }, "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "ml_training_data_loading_seconds", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.95, rate(foxhunt_training_data_load_seconds_bucket[5m]))", "legendFormat": "Load Latency", "instant": false, "range": true @@ -913,18 +1574,46 @@ "id": 29, "type": "stat", "title": "Feature Errors", - "gridPos": { "h": 6, "w": 8, "x": 16, "y": 33 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 33 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "red", "value": 1 }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, "mappings": [], - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -934,18 +1623,25 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "ml_feature_engineering_errors_total", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "foxhunt_training_feature_errors_total{model=~\"$model_type\"}", "instant": true } ] }, - { "id": 30, "type": "row", "title": "Errors", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 39 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 39 + }, "collapsed": false, "panels": [] }, @@ -953,18 +1649,46 @@ "id": 31, "type": "stat", "title": "NaN Detections", - "gridPos": { "h": 4, "w": 6, "x": 0, "y": 40 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 40 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "red", "value": 1 }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, "mappings": [], - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -974,8 +1698,11 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "sum(ml_training_nan_count)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "foxhunt_training_nan_detected_total{model=~\"$model_type\"}", "instant": true } ] @@ -984,18 +1711,46 @@ "id": 32, "type": "stat", "title": "Gradient Explosions", - "gridPos": { "h": 4, "w": 6, "x": 6, "y": 40 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 40 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "red", "value": 1 }] }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, "mappings": [], - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "orientation": "auto", "textMode": "auto", "colorMode": "background", @@ -1005,8 +1760,11 @@ "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "ml_training_gradient_explosion_total", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "foxhunt_training_gradient_explosion_total{model=~\"$model_type\"}", "instant": true } ] @@ -1016,8 +1774,16 @@ "type": "timeseries", "title": "Training Failures", "description": "Hourly increase of training failures by error type", - "gridPos": { "h": 6, "w": 12, "x": 12, "y": 40 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 40 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": { @@ -1029,35 +1795,59 @@ "showPoints": "never", "axisCenteredZero": false, "axisLabel": "Failures", - "stacking": { "mode": "normal", "group": "A" } + "stacking": { + "mode": "normal", + "group": "A" + } + }, + "thresholds": { + "mode": "off", + "steps": [] }, - "thresholds": { "mode": "off", "steps": [] }, "mappings": [], - "color": { "mode": "palette-classic" } + "color": { + "mode": "palette-classic" + } }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["sum"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "sum" + ] + } }, "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "increase(ml_training_failures_total[1h])", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "ml_training_errors_total", "legendFormat": "{{error_type}}", "instant": false, "range": true } ] }, - { "id": 34, "type": "row", "title": "Job History", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 46 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 46 + }, "collapsed": false, "panels": [] }, @@ -1066,8 +1856,16 @@ "type": "timeseries", "title": "Job Duration Distribution", "description": "Training job durations over time by model type", - "gridPos": { "h": 6, "w": 12, "x": 0, "y": 47 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 47 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": { @@ -1080,22 +1878,40 @@ "axisCenteredZero": false, "axisLabel": "Duration" }, - "thresholds": { "mode": "off", "steps": [] }, + "thresholds": { + "mode": "off", + "steps": [] + }, "mappings": [], - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "unit": "s" }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + } }, "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "ml_training_job_duration_seconds{model_type=~\"$model_type\"}", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "(kube_job_complete_time{namespace=\"foxhunt\", job_name=~\"training-.*\"} - kube_job_start_time{namespace=\"foxhunt\", job_name=~\"training-.*\"}) / 3600", "legendFormat": "{{model_type}}", "instant": false, "range": true @@ -1107,8 +1923,16 @@ "type": "timeseries", "title": "Iteration Duration", "description": "Per-iteration training time by model type", - "gridPos": { "h": 6, "w": 12, "x": 12, "y": 47 }, - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 47 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": { @@ -1122,22 +1946,41 @@ "axisCenteredZero": false, "axisLabel": "Seconds" }, - "thresholds": { "mode": "off", "steps": [] }, + "thresholds": { + "mode": "off", + "steps": [] + }, "mappings": [], - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "unit": "s" }, "overrides": [] }, "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "lastNotNull"] } + "tooltip": { + "mode": "multi", + "sort": "desc" + }, + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + } }, "targets": [ { "refId": "A", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "ml_training_iteration_seconds{model_type=~\"$model_type\"}", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "foxhunt_training_iteration_seconds{model=~\"$model_type\"}", "legendFormat": "{{model_type}}", "instant": false, "range": true From 38832661ee617998eb0b475715b46e127029f874 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sun, 1 Mar 2026 02:12:17 +0100 Subject: [PATCH 07/14] infra(web-gateway): add metrics port 9098 and prometheus scrape annotations Co-Authored-By: Claude Opus 4.6 --- infra/k8s/services/web-gateway.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/infra/k8s/services/web-gateway.yaml b/infra/k8s/services/web-gateway.yaml index 8f2351ca2..d2e91f9f1 100644 --- a/infra/k8s/services/web-gateway.yaml +++ b/infra/k8s/services/web-gateway.yaml @@ -18,6 +18,10 @@ spec: app.kubernetes.io/name: web-gateway template: metadata: + annotations: + gitlab.com/prometheus_scrape: "true" + gitlab.com/prometheus_port: "9098" + gitlab.com/prometheus_path: "/metrics" labels: app.kubernetes.io/name: web-gateway spec: @@ -86,6 +90,8 @@ spec: ports: - containerPort: 3000 name: http + - containerPort: 9098 + name: metrics env: - name: JWT_SECRET valueFrom: @@ -155,3 +161,6 @@ spec: - port: 3000 targetPort: 3000 name: http + - port: 9098 + targetPort: 9098 + name: metrics From dc0750dc9a1977ac3e3290ee3567b46f54fe6dec Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sun, 1 Mar 2026 02:12:55 +0100 Subject: [PATCH 08/14] feat(data-acquisition): add Prometheus metrics server on port 9097 Add metrics module with uptime, active feeds, records received, errors, and feed latency metrics. Spawn HTTP /metrics endpoint using axum, following the same inline pattern as ml-training-service. Co-Authored-By: Claude Opus 4.6 --- services/data_acquisition_service/src/lib.rs | 1 + services/data_acquisition_service/src/main.rs | 46 ++++++++++ .../data_acquisition_service/src/metrics.rs | 84 +++++++++++++++++++ 3 files changed, 131 insertions(+) create mode 100644 services/data_acquisition_service/src/metrics.rs diff --git a/services/data_acquisition_service/src/lib.rs b/services/data_acquisition_service/src/lib.rs index e4baba4fd..2bdb63eee 100644 --- a/services/data_acquisition_service/src/lib.rs +++ b/services/data_acquisition_service/src/lib.rs @@ -12,6 +12,7 @@ pub mod downloader; pub mod error; +pub mod metrics; pub mod service; pub mod uploader; pub mod validator; diff --git a/services/data_acquisition_service/src/main.rs b/services/data_acquisition_service/src/main.rs index fba3781d7..89a40b045 100644 --- a/services/data_acquisition_service/src/main.rs +++ b/services/data_acquisition_service/src/main.rs @@ -43,6 +43,52 @@ async fn main() -> Result<(), Box> { info!("gRPC port: {}", args.port); info!("Health port: {}", args.health_port); + // Initialize Prometheus metrics + data_acquisition_service::metrics::init_metrics(); + let service_start = std::time::Instant::now(); + + // Spawn uptime updater + tokio::spawn(async move { + let mut interval = tokio::time::interval(std::time::Duration::from_secs(1)); + loop { + interval.tick().await; + data_acquisition_service::metrics::update_uptime(service_start); + } + }); + + // Start Prometheus metrics HTTP endpoint on port 9097 + let metrics_port: u16 = std::env::var("METRICS_PORT") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(9097); + tokio::spawn(async move { + use axum::{routing::get, Router}; + use prometheus::{Encoder, TextEncoder}; + + async fn metrics_handler() -> String { + let encoder = TextEncoder::new(); + let metric_families = prometheus::gather(); + let mut buffer = vec![]; + let _ = encoder.encode(&metric_families, &mut buffer); + String::from_utf8(buffer).unwrap_or_else(|_| String::new()) + } + + let app = Router::new().route("/metrics", get(metrics_handler)); + let addr = format!("0.0.0.0:{metrics_port}"); + tracing::info!("Prometheus metrics endpoint listening on http://{}", addr); + + let listener = match tokio::net::TcpListener::bind(&addr).await { + Ok(l) => l, + Err(e) => { + tracing::error!("Failed to bind metrics endpoint {}: {}", addr, e); + return; + } + }; + if let Err(e) = axum::serve(listener, app).await { + tracing::error!("Metrics server failed: {}", e); + } + }); + // Create service instance let service = DataAcquisitionServiceImpl::default(); diff --git a/services/data_acquisition_service/src/metrics.rs b/services/data_acquisition_service/src/metrics.rs new file mode 100644 index 000000000..5d66a211d --- /dev/null +++ b/services/data_acquisition_service/src/metrics.rs @@ -0,0 +1,84 @@ +//! Prometheus metrics for Data Acquisition Service + +use once_cell::sync::Lazy; +use prometheus::{ + register_counter, register_counter_vec, register_gauge, register_histogram_vec, Counter, + CounterVec, Gauge, HistogramVec, +}; + +/// Service uptime in seconds +pub static SERVICE_UPTIME: Lazy = Lazy::new(|| { + register_gauge!( + "foxhunt_data_acquisition_uptime_seconds", + "Service uptime in seconds" + ) + .unwrap_or_else(|e| { + eprintln!("FATAL: failed to register uptime gauge: {e}"); + std::process::abort() + }) +}); + +/// Active data feeds +pub static FEEDS_ACTIVE: Lazy = Lazy::new(|| { + register_gauge!( + "foxhunt_data_acquisition_feeds_active", + "Number of active data feeds" + ) + .unwrap_or_else(|e| { + eprintln!("FATAL: failed to register feeds_active gauge: {e}"); + std::process::abort() + }) +}); + +/// Total records received +pub static RECORDS_RECEIVED: Lazy = Lazy::new(|| { + register_counter_vec!( + "foxhunt_data_acquisition_records_received_total", + "Total records received from data feeds", + &["symbol", "feed_type"] + ) + .unwrap_or_else(|e| { + eprintln!("FATAL: failed to register records_received counter: {e}"); + std::process::abort() + }) +}); + +/// Total errors +pub static ERRORS: Lazy = Lazy::new(|| { + register_counter!( + "foxhunt_data_acquisition_errors_total", + "Total data acquisition errors" + ) + .unwrap_or_else(|e| { + eprintln!("FATAL: failed to register errors counter: {e}"); + std::process::abort() + }) +}); + +/// Feed processing latency +pub static FEED_LATENCY: Lazy = Lazy::new(|| { + register_histogram_vec!( + "foxhunt_data_acquisition_feed_latency_seconds", + "Feed processing latency in seconds", + &["feed_type"], + vec![0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0, 5.0] + ) + .unwrap_or_else(|e| { + eprintln!("FATAL: failed to register feed_latency histogram: {e}"); + std::process::abort() + }) +}); + +/// Initialize all metrics (force lazy registration) +pub fn init_metrics() { + let _ = &*SERVICE_UPTIME; + let _ = &*FEEDS_ACTIVE; + let _ = &*RECORDS_RECEIVED; + let _ = &*ERRORS; + let _ = &*FEED_LATENCY; +} + +/// Update service uptime metric +pub fn update_uptime(start_time: std::time::Instant) { + SERVICE_UPTIME.set(start_time.elapsed().as_secs_f64()); +} From f0c653ab63ea555a2a316f275db4b6f802f2dc5d Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sun, 1 Mar 2026 02:14:08 +0100 Subject: [PATCH 09/14] feat(web-gateway): add Prometheus metrics server on port 9098 Adds a dedicated Prometheus metrics endpoint on port 9098 (configurable via METRICS_PORT env var) with counters for HTTP requests, histograms for request duration, gauges for uptime and active WebSocket connections, and counters for WebSocket messages. Metrics are registered using once_cell Lazy statics with abort-on-failure to satisfy the crate's deny(unwrap) lint policy. Co-Authored-By: Claude Opus 4.6 --- crates/web-gateway/Cargo.toml | 4 ++ crates/web-gateway/src/lib.rs | 1 + crates/web-gateway/src/main.rs | 46 +++++++++++++++++ crates/web-gateway/src/metrics.rs | 84 +++++++++++++++++++++++++++++++ 4 files changed, 135 insertions(+) create mode 100644 crates/web-gateway/src/metrics.rs diff --git a/crates/web-gateway/Cargo.toml b/crates/web-gateway/Cargo.toml index 80f45f5c2..603131137 100644 --- a/crates/web-gateway/Cargo.toml +++ b/crates/web-gateway/Cargo.toml @@ -43,6 +43,10 @@ tracing-subscriber.workspace = true # Shared types common.workspace = true +# Metrics +prometheus.workspace = true +once_cell.workspace = true + # Time and IDs chrono.workspace = true uuid.workspace = true diff --git a/crates/web-gateway/src/lib.rs b/crates/web-gateway/src/lib.rs index f0c857662..ac7f3da28 100644 --- a/crates/web-gateway/src/lib.rs +++ b/crates/web-gateway/src/lib.rs @@ -10,6 +10,7 @@ pub mod auth; pub mod config; pub mod error; pub mod grpc; +pub mod metrics; pub mod rate_limit; pub mod routes; pub mod state; diff --git a/crates/web-gateway/src/main.rs b/crates/web-gateway/src/main.rs index 84e05fd11..047af22e3 100644 --- a/crates/web-gateway/src/main.rs +++ b/crates/web-gateway/src/main.rs @@ -75,6 +75,52 @@ async fn main() -> Result<()> { .layer(TraceLayer::new_for_http()) .layer(middleware::from_fn(request_id_middleware)); + // Initialize Prometheus metrics + web_gateway::metrics::init_metrics(); + let service_start = std::time::Instant::now(); + + // Spawn uptime updater + tokio::spawn(async move { + let mut interval = tokio::time::interval(std::time::Duration::from_secs(1)); + loop { + interval.tick().await; + web_gateway::metrics::update_uptime(service_start); + } + }); + + // Start Prometheus metrics HTTP endpoint on a separate port + let metrics_port: u16 = std::env::var("METRICS_PORT") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(9098); + tokio::spawn(async move { + use axum::{routing::get, Router}; + use prometheus::{Encoder, TextEncoder}; + + async fn metrics_handler() -> String { + let encoder = TextEncoder::new(); + let metric_families = prometheus::gather(); + let mut buffer = vec![]; + let _ = encoder.encode(&metric_families, &mut buffer); + String::from_utf8(buffer).unwrap_or_else(|_| String::new()) + } + + let app = Router::new().route("/metrics", get(metrics_handler)); + let addr = format!("0.0.0.0:{}", metrics_port); + tracing::info!("Prometheus metrics endpoint listening on http://{}", addr); + + let metrics_listener = match tokio::net::TcpListener::bind(&addr).await { + Ok(l) => l, + Err(e) => { + tracing::error!("Failed to bind metrics endpoint {}: {}", addr, e); + return; + } + }; + if let Err(e) = axum::serve(metrics_listener, app).await { + tracing::error!("Metrics server failed: {}", e); + } + }); + let listener = tokio::net::TcpListener::bind(&listen_addr).await?; info!("Web gateway listening on {}", listen_addr); diff --git a/crates/web-gateway/src/metrics.rs b/crates/web-gateway/src/metrics.rs new file mode 100644 index 000000000..eaa4de619 --- /dev/null +++ b/crates/web-gateway/src/metrics.rs @@ -0,0 +1,84 @@ +//! Prometheus metrics for Web Gateway + +use once_cell::sync::Lazy; +use prometheus::{ + register_counter_vec, register_gauge, register_histogram_vec, CounterVec, Gauge, HistogramVec, +}; + +/// Service uptime in seconds +pub static SERVICE_UPTIME: Lazy = Lazy::new(|| { + register_gauge!( + "foxhunt_web_gateway_uptime_seconds", + "Service uptime in seconds" + ) + .unwrap_or_else(|e| { + eprintln!("FATAL: failed to register uptime gauge: {e}"); + std::process::abort() + }) +}); + +/// HTTP requests total +pub static HTTP_REQUESTS: Lazy = Lazy::new(|| { + register_counter_vec!( + "foxhunt_web_gateway_http_requests_total", + "Total HTTP requests", + &["method", "path", "status"] + ) + .unwrap_or_else(|e| { + eprintln!("FATAL: failed to register http_requests counter: {e}"); + std::process::abort() + }) +}); + +/// HTTP request duration +pub static HTTP_REQUEST_DURATION: Lazy = Lazy::new(|| { + register_histogram_vec!( + "foxhunt_web_gateway_http_request_duration_seconds", + "HTTP request duration in seconds", + &["method", "path"], + vec![0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 5.0] + ) + .unwrap_or_else(|e| { + eprintln!("FATAL: failed to register http_request_duration histogram: {e}"); + std::process::abort() + }) +}); + +/// Active WebSocket connections +pub static WS_CONNECTIONS: Lazy = Lazy::new(|| { + register_gauge!( + "foxhunt_web_gateway_ws_connections_active", + "Active WebSocket connections" + ) + .unwrap_or_else(|e| { + eprintln!("FATAL: failed to register ws_connections gauge: {e}"); + std::process::abort() + }) +}); + +/// WebSocket messages total +pub static WS_MESSAGES: Lazy = Lazy::new(|| { + register_counter_vec!( + "foxhunt_web_gateway_ws_messages_total", + "Total WebSocket messages", + &["direction", "topic"] + ) + .unwrap_or_else(|e| { + eprintln!("FATAL: failed to register ws_messages counter: {e}"); + std::process::abort() + }) +}); + +/// Initialize all metrics (force lazy registration) +pub fn init_metrics() { + let _ = &*SERVICE_UPTIME; + let _ = &*HTTP_REQUESTS; + let _ = &*HTTP_REQUEST_DURATION; + let _ = &*WS_CONNECTIONS; + let _ = &*WS_MESSAGES; +} + +/// Update service uptime metric +pub fn update_uptime(start_time: std::time::Instant) { + SERVICE_UPTIME.set(start_time.elapsed().as_secs_f64()); +} From a958582eaa51728e25511c2419bbbed88d55b2bd Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sun, 1 Mar 2026 02:23:36 +0100 Subject: [PATCH 10/14] feat(common): add gRPC metrics Tower layer for automatic instrumentation Add a reusable GrpcMetricsLayer that can be applied to any tonic server via Server::builder().layer() to automatically emit three Prometheus metric families for every gRPC handler: started_total, handled_total (with status code), and handling_seconds histogram. Co-Authored-By: Claude Opus 4.6 --- Cargo.lock | 5 + crates/common/Cargo.toml | 5 +- crates/common/src/metrics/grpc_metrics.rs | 245 ++++++++++++++++++++++ crates/common/src/metrics/mod.rs | 3 + 4 files changed, 257 insertions(+), 1 deletion(-) create mode 100644 crates/common/src/metrics/grpc_metrics.rs diff --git a/Cargo.lock b/Cargo.lock index 05ae62893..a863a7d5e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2291,6 +2291,7 @@ dependencies = [ "criterion", "fastrand", "futures", + "http 1.3.1", "jsonwebtoken", "lazy_static", "ml", @@ -2313,6 +2314,8 @@ dependencies = [ "tokio-test", "toml", "tonic 0.14.2", + "tower-layer", + "tower-service", "tracing", "tracing-appender", "tracing-opentelemetry", @@ -11543,6 +11546,8 @@ dependencies = [ "common", "futures-util", "jsonwebtoken", + "once_cell", + "prometheus", "prost 0.14.1", "serde", "serde_json", diff --git a/crates/common/Cargo.toml b/crates/common/Cargo.toml index 1c3f920c3..36bdce00e 100644 --- a/crates/common/Cargo.toml +++ b/crates/common/Cargo.toml @@ -47,8 +47,11 @@ opentelemetry.workspace = true opentelemetry-otlp.workspace = true opentelemetry_sdk.workspace = true -# gRPC (for correlation ID propagation) +# gRPC (for correlation ID propagation and metrics layer) tonic.workspace = true +http.workspace = true +tower-layer.workspace = true +tower-service.workspace = true # Configuration toml.workspace = true diff --git a/crates/common/src/metrics/grpc_metrics.rs b/crates/common/src/metrics/grpc_metrics.rs new file mode 100644 index 000000000..e2009ed5c --- /dev/null +++ b/crates/common/src/metrics/grpc_metrics.rs @@ -0,0 +1,245 @@ +//! gRPC metrics Tower layer for automatic request instrumentation. +//! +//! Provides `grpc_server_started_total`, `grpc_server_handled_total`, +//! and `grpc_server_handling_seconds` metrics for all gRPC handlers. +//! +//! # Usage +//! +//! ```rust,ignore +//! use common::metrics::grpc_metrics::GrpcMetricsLayer; +//! use tonic::transport::Server; +//! +//! Server::builder() +//! .layer(GrpcMetricsLayer::new("trading_service")) +//! .add_service(my_service) +//! .serve(addr) +//! .await?; +//! ``` + +use once_cell::sync::Lazy; +use prometheus::{register_counter_vec, register_histogram_vec, CounterVec, HistogramVec}; +use std::future::Future; +use std::pin::Pin; +use std::task::{Context, Poll}; +use tower_layer::Layer; +use tower_service::Service; + +/// Counter for gRPC requests started. +static GRPC_STARTED: Lazy = Lazy::new(|| { + register_counter_vec!( + "grpc_server_started_total", + "Total gRPC RPCs started on the server", + &["service", "grpc_method"] + ) + .unwrap_or_else(|e| { + // Metric registration failure at startup is unrecoverable. + // The process cannot operate without metrics instrumentation. + eprintln!("FATAL: failed to register grpc_server_started_total: {e}"); + std::process::abort() + }) +}); + +/// Counter for gRPC requests completed with status code. +static GRPC_HANDLED: Lazy = Lazy::new(|| { + register_counter_vec!( + "grpc_server_handled_total", + "Total gRPC RPCs completed on the server with status code", + &["service", "grpc_method", "grpc_code"] + ) + .unwrap_or_else(|e| { + eprintln!("FATAL: failed to register grpc_server_handled_total: {e}"); + std::process::abort() + }) +}); + +/// Histogram for gRPC request handling duration. +static GRPC_HANDLING_SECONDS: Lazy = Lazy::new(|| { + register_histogram_vec!( + "grpc_server_handling_seconds", + "Histogram of gRPC request handling duration in seconds", + &["service", "grpc_method"], + vec![0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 5.0] + ) + .unwrap_or_else(|e| { + eprintln!("FATAL: failed to register grpc_server_handling_seconds: {e}"); + std::process::abort() + }) +}); + +/// Initialize gRPC metrics (force lazy registration). +/// +/// Call this early in service startup to ensure metrics are registered +/// before any requests arrive. It is safe to call multiple times. +pub fn init_grpc_metrics() { + let _ = &*GRPC_STARTED; + let _ = &*GRPC_HANDLED; + let _ = &*GRPC_HANDLING_SECONDS; +} + +/// Extract the method name from a gRPC URI path. +/// +/// gRPC paths follow the format `/package.Service/Method`. +/// This extracts the last segment (the method name). +fn extract_method(path: &str) -> &str { + match path.rsplit('/').next() { + Some(m) if !m.is_empty() => m, + _ => "unknown", + } +} + +/// Tower Layer that instruments gRPC handlers with Prometheus metrics. +/// +/// Records three metric families: +/// - `grpc_server_started_total` — incremented when a request begins +/// - `grpc_server_handled_total` — incremented when a request completes (with status code) +/// - `grpc_server_handling_seconds` — histogram of request duration +#[derive(Debug, Clone)] +pub struct GrpcMetricsLayer { + service_name: &'static str, +} + +impl GrpcMetricsLayer { + /// Create a new gRPC metrics layer for the given service. + /// + /// `service_name` is used as the `service` label on all emitted metrics. + pub fn new(service_name: &'static str) -> Self { + init_grpc_metrics(); + Self { service_name } + } +} + +impl Layer for GrpcMetricsLayer { + type Service = GrpcMetricsService; + + fn layer(&self, inner: S) -> Self::Service { + GrpcMetricsService { + inner, + service_name: self.service_name, + } + } +} + +/// Tower Service that records gRPC metrics around the inner service. +#[derive(Debug, Clone)] +pub struct GrpcMetricsService { + inner: S, + service_name: &'static str, +} + +impl Service> for GrpcMetricsService +where + S: Service, Response = http::Response> + + Clone + + Send + + 'static, + S::Future: Send + 'static, + S::Error: Send + 'static, + ReqBody: Send + 'static, +{ + type Response = S::Response; + type Error = S::Error; + type Future = Pin> + Send>>; + + fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { + self.inner.poll_ready(cx) + } + + fn call(&mut self, req: http::Request) -> Self::Future { + let service_name = self.service_name; + let method = extract_method(req.uri().path()).to_string(); + + GRPC_STARTED + .with_label_values(&[service_name, &method]) + .inc(); + + let start = std::time::Instant::now(); + // Clone inner before calling to satisfy tower's Service contract: + // poll_ready was called on `self`, so we must call on `self` + // (not the clone). We swap so the clone becomes `self` for + // the next call, and we consume the ready instance. + let mut inner = self.inner.clone(); + std::mem::swap(&mut self.inner, &mut inner); + + Box::pin(async move { + let result = inner.call(req).await; + let elapsed = start.elapsed().as_secs_f64(); + + let code = match &result { + Ok(resp) => resp + .headers() + .get("grpc-status") + .and_then(|v| v.to_str().ok()) + .map_or_else(|| "0".to_string(), |s| s.to_string()), + Err(_) => "13".to_string(), // INTERNAL + }; + + GRPC_HANDLED + .with_label_values(&[service_name, &method, &code]) + .inc(); + + GRPC_HANDLING_SECONDS + .with_label_values(&[service_name, &method]) + .observe(elapsed); + + result + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_extract_method_normal_path() { + assert_eq!(extract_method("/package.Service/MethodName"), "MethodName"); + } + + #[test] + fn test_extract_method_simple_path() { + assert_eq!(extract_method("/Method"), "Method"); + } + + #[test] + fn test_extract_method_empty_path() { + assert_eq!(extract_method(""), "unknown"); + } + + #[test] + fn test_extract_method_trailing_slash() { + // rsplit('/').next() on "foo/" yields "" (empty), so fallback + assert_eq!(extract_method("/Service/"), "unknown"); + } + + #[test] + fn test_extract_method_root_slash() { + assert_eq!(extract_method("/"), "unknown"); + } + + #[test] + fn test_metrics_layer_creates_service() { + let layer = GrpcMetricsLayer::new("test_svc"); + assert_eq!(layer.service_name, "test_svc"); + } + + #[test] + fn test_init_grpc_metrics_is_idempotent() { + // Calling init multiple times should not panic + init_grpc_metrics(); + init_grpc_metrics(); + init_grpc_metrics(); + } + + #[test] + fn test_metrics_registered() { + init_grpc_metrics(); + // Verify metrics are accessible via the global statics + GRPC_STARTED + .with_label_values(&["test_svc", "TestMethod"]) + .inc(); + let val = GRPC_STARTED + .with_label_values(&["test_svc", "TestMethod"]) + .get(); + assert!(val >= 1.0, "Counter should have been incremented"); + } +} diff --git a/crates/common/src/metrics/mod.rs b/crates/common/src/metrics/mod.rs index 7090d7f97..9297aceed 100644 --- a/crates/common/src/metrics/mod.rs +++ b/crates/common/src/metrics/mod.rs @@ -3,6 +3,7 @@ // Provides Prometheus-based metrics collection with a global registry // and helper functions for all services. +pub mod grpc_metrics; pub mod registry; // Re-export commonly used functions @@ -14,5 +15,7 @@ pub use registry::{ set_gauge, set_gauge_vec, REGISTRY, }; +pub use grpc_metrics::{init_grpc_metrics, GrpcMetricsLayer}; + #[cfg(test)] mod tests; From 980f5d33c11ae6668767ee03cf869090e12cb5b4 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sun, 1 Mar 2026 02:30:23 +0100 Subject: [PATCH 11/14] feat(services): wire gRPC metrics Tower layer into all 7 gRPC services Add GrpcMetricsLayer from common::metrics to every gRPC service's Server::builder() chain, enabling automatic Prometheus instrumentation (grpc_server_started_total, grpc_server_handled_total, grpc_server_handling_seconds) for all RPC handlers. Services wired: - trading-service - api-gateway - ml-training-service - backtesting-service - broker-gateway - data-acquisition-service - trading-agent-service Co-Authored-By: Claude Opus 4.6 --- services/api_gateway/src/main.rs | 4 ++++ services/backtesting_service/src/main.rs | 5 ++++- services/broker_gateway_service/src/main.rs | 3 +++ services/data_acquisition_service/src/main.rs | 3 +++ services/ml_training_service/src/main.rs | 5 ++++- services/trading_agent_service/src/main.rs | 6 +++++- services/trading_service/src/main.rs | 6 +++++- 7 files changed, 28 insertions(+), 4 deletions(-) diff --git a/services/api_gateway/src/main.rs b/services/api_gateway/src/main.rs index 842289391..5c6d74179 100644 --- a/services/api_gateway/src/main.rs +++ b/services/api_gateway/src/main.rs @@ -11,6 +11,8 @@ use std::sync::Arc; use std::time::Duration; use tracing::{error, info, warn}; +use common::metrics::GrpcMetricsLayer; + // Import all needed types from the library use api_gateway::auth::jwt::JwtConfig; use api_gateway::auth::{ @@ -496,12 +498,14 @@ async fn main() -> Result<()> { // Build server with HTTP/2 optimizations let mut server_builder = if let Some(ref tls) = tls_config { tonic::transport::Server::builder() + .layer(GrpcMetricsLayer::new("api-gateway")) .tls_config(tls.to_server_tls_config())? .max_concurrent_streams(Some(10_000)) .http2_keepalive_interval(Some(Duration::from_secs(30))) .http2_keepalive_timeout(Some(Duration::from_secs(10))) } else { tonic::transport::Server::builder() + .layer(GrpcMetricsLayer::new("api-gateway")) .max_concurrent_streams(Some(10_000)) .http2_keepalive_interval(Some(Duration::from_secs(30))) .http2_keepalive_timeout(Some(Duration::from_secs(10))) diff --git a/services/backtesting_service/src/main.rs b/services/backtesting_service/src/main.rs index a3965178a..bff1a40e4 100644 --- a/services/backtesting_service/src/main.rs +++ b/services/backtesting_service/src/main.rs @@ -13,6 +13,8 @@ use std::net::SocketAddr; use tonic::transport::Server; use tracing::{error, info, warn}; +use common::metrics::GrpcMetricsLayer; + mod health; use backtesting_service::foxhunt; @@ -173,7 +175,8 @@ async fn main() -> Result<()> { } // Start the server with HTTP/2 optimizations - let mut server_builder = Server::builder(); + let mut server_builder = Server::builder() + .layer(GrpcMetricsLayer::new("backtesting-service")); if enable_http2_opts { server_builder = server_builder diff --git a/services/broker_gateway_service/src/main.rs b/services/broker_gateway_service/src/main.rs index 8f195d21d..63ae4f1a4 100644 --- a/services/broker_gateway_service/src/main.rs +++ b/services/broker_gateway_service/src/main.rs @@ -12,6 +12,8 @@ use tokio::signal; use tonic::transport::Server; use tracing::{error, info, warn}; +use common::metrics::GrpcMetricsLayer; + // Service configuration const DEFAULT_GRPC_PORT: u16 = 50056; const DEFAULT_HEALTH_PORT: u16 = 8086; @@ -142,6 +144,7 @@ async fn main() -> Result<()> { // Build gRPC server let server = Server::builder() + .layer(GrpcMetricsLayer::new("broker-gateway")) .add_service(health_service) .add_service(BrokerGatewayServiceServer::new(service)) .serve_with_shutdown(addr, shutdown_signal()); diff --git a/services/data_acquisition_service/src/main.rs b/services/data_acquisition_service/src/main.rs index 89a40b045..032ccb381 100644 --- a/services/data_acquisition_service/src/main.rs +++ b/services/data_acquisition_service/src/main.rs @@ -11,6 +11,8 @@ use std::net::SocketAddr; use tonic::transport::Server; use tracing::info; +use common::metrics::GrpcMetricsLayer; + #[derive(Parser, Debug)] #[command(author, version, about, long_about = None)] struct Args { @@ -99,6 +101,7 @@ async fn main() -> Result<(), Box> { // Start gRPC server with graceful shutdown Server::builder() + .layer(GrpcMetricsLayer::new("data-acquisition-service")) .add_service(DataAcquisitionServiceServer::new(service)) .serve_with_shutdown(addr, shutdown_signal()) .await?; diff --git a/services/ml_training_service/src/main.rs b/services/ml_training_service/src/main.rs index 26e0577cc..95b84fe9d 100644 --- a/services/ml_training_service/src/main.rs +++ b/services/ml_training_service/src/main.rs @@ -15,6 +15,8 @@ use clap::{Args, Parser, Subcommand}; use tonic::transport::Server; use tonic_reflection::server::Builder as ReflectionBuilder; use tracing::{debug, error, info, warn}; + +use common::metrics::GrpcMetricsLayer; use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; // Import from library instead of duplicating module declarations @@ -454,7 +456,8 @@ async fn serve(args: ServeArgs) -> Result<()> { .and_then(|v| v.parse().ok()) .unwrap_or(true); - let mut server_builder = Server::builder(); + let mut server_builder = Server::builder() + .layer(GrpcMetricsLayer::new("ml-training-service")); if enable_http2_opts { info!("HTTP/2 optimizations enabled:"); diff --git a/services/trading_agent_service/src/main.rs b/services/trading_agent_service/src/main.rs index 2234f8fa4..41a11174c 100644 --- a/services/trading_agent_service/src/main.rs +++ b/services/trading_agent_service/src/main.rs @@ -12,6 +12,8 @@ use tokio::sync::Mutex; use tonic::transport::{Certificate, Identity, Server, ServerTlsConfig}; use tracing::{error, info}; +use common::metrics::GrpcMetricsLayer; + use common::DatabasePool; use config::manager::ConfigManager; use config::DatabaseConfig; @@ -93,11 +95,13 @@ async fn main() -> Result<()> { info!("Starting gRPC server on {}", addr); // Build server with optional TLS - let mut server_builder = Server::builder(); + let mut server_builder = Server::builder() + .layer(GrpcMetricsLayer::new("trading-agent-service")); if let Some(tls) = tls_config { info!("🔒 TLS enabled for Trading Agent Service"); server_builder = Server::builder() + .layer(GrpcMetricsLayer::new("trading-agent-service")) .tls_config(tls) .context("Failed to apply TLS configuration")?; } else { diff --git a/services/trading_service/src/main.rs b/services/trading_service/src/main.rs index 4d37fd040..debc9a023 100644 --- a/services/trading_service/src/main.rs +++ b/services/trading_service/src/main.rs @@ -43,6 +43,8 @@ use trading_service::feedback_loop::{FeedbackLoop, FeedbackLoopConfig}; use trading_service::questdb_metrics::QuestDBMetricsProvider; use trading_service::state::TradingServiceState; +use common::metrics::GrpcMetricsLayer; + /// Default configuration values const DEFAULT_GRPC_PORT: u16 = 50052; // Trading Service port (API Gateway uses 50051) const DEFAULT_HEALTH_PORT: u16 = 8081; // Health check port (separate from API Gateway 8080) @@ -942,9 +944,11 @@ async fn main() -> Result<()> { // Apply authentication interceptor to all gRPC services let mut server_builder = match tls_config { Some(tls) => Server::builder() + .layer(GrpcMetricsLayer::new("trading-service")) .tls_config(tls) .context("Failed to configure TLS")?, - None => Server::builder(), + None => Server::builder() + .layer(GrpcMetricsLayer::new("trading-service")), }; // Apply HTTP/2 optimizations if enabled From 4611ab97f3ad6c22f537ccbb8a021d8efd8cd15a Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sun, 1 Mar 2026 02:31:37 +0100 Subject: [PATCH 12/14] infra: add Prometheus Pushgateway for training job metrics Co-Authored-By: Claude Opus 4.6 --- infra/k8s/monitoring/pushgateway.yaml | 61 +++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 infra/k8s/monitoring/pushgateway.yaml diff --git a/infra/k8s/monitoring/pushgateway.yaml b/infra/k8s/monitoring/pushgateway.yaml new file mode 100644 index 000000000..8079fa19e --- /dev/null +++ b/infra/k8s/monitoring/pushgateway.yaml @@ -0,0 +1,61 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: pushgateway + namespace: foxhunt + labels: + app.kubernetes.io/name: pushgateway + app.kubernetes.io/part-of: monitoring +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: pushgateway + template: + metadata: + labels: + app.kubernetes.io/name: pushgateway + spec: + nodeSelector: + k8s.scaleway.com/pool-name: services + containers: + - name: pushgateway + image: prom/pushgateway:v1.11.0 + args: + - "--persistence.interval=5m" + ports: + - containerPort: 9091 + name: http + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 200m + memory: 128Mi + livenessProbe: + httpGet: + path: /-/healthy + port: 9091 + initialDelaySeconds: 5 + readinessProbe: + httpGet: + path: /-/ready + port: 9091 + initialDelaySeconds: 5 +--- +apiVersion: v1 +kind: Service +metadata: + name: pushgateway + namespace: foxhunt + labels: + app.kubernetes.io/name: pushgateway +spec: + type: ClusterIP + ports: + - port: 9091 + targetPort: 9091 + name: http + selector: + app.kubernetes.io/name: pushgateway From 1c2985d1eeed850f54a2a89a598b2fa74b3fb658 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sun, 1 Mar 2026 02:31:38 +0100 Subject: [PATCH 13/14] infra: add pushgateway scrape config to Prometheus Co-Authored-By: Claude Opus 4.6 --- infra/k8s/monitoring/prometheus-values.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/infra/k8s/monitoring/prometheus-values.yaml b/infra/k8s/monitoring/prometheus-values.yaml index 6b44f3074..8eac3e8f6 100644 --- a/infra/k8s/monitoring/prometheus-values.yaml +++ b/infra/k8s/monitoring/prometheus-values.yaml @@ -164,3 +164,9 @@ prometheus: - source_labels: [__meta_kubernetes_pod_ip] target_label: __address__ replacement: $1:9080 + + # --- Pushgateway (training job metrics) --- + - job_name: pushgateway + honor_labels: true + static_configs: + - targets: ['pushgateway.foxhunt.svc.cluster.local:9091'] From 9b85b9efd596ed8a69a22dbfaaa01446cac577c3 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sun, 1 Mar 2026 02:36:05 +0100 Subject: [PATCH 14/14] feat(ml): add Pushgateway metrics client for training jobs Training jobs are short-lived K8s Jobs that can't be reliably scraped by Prometheus. This adds a TrainingMetricsPusher that pushes epoch-level metrics (loss, accuracy, throughput, gradient health, checkpoints) to the Pushgateway so they persist for Prometheus to scrape. Co-Authored-By: Claude Opus 4.6 --- crates/ml/src/training.rs | 1 + crates/ml/src/training/push_metrics.rs | 211 +++++++++++++++++++++++++ 2 files changed, 212 insertions(+) create mode 100644 crates/ml/src/training/push_metrics.rs diff --git a/crates/ml/src/training.rs b/crates/ml/src/training.rs index 410b6be92..4a3c79519 100644 --- a/crates/ml/src/training.rs +++ b/crates/ml/src/training.rs @@ -12,6 +12,7 @@ // Sub-modules for specialized training components pub mod orchestrator; +pub mod push_metrics; pub mod unified_data_loader; pub mod unified_trainer; // NEW: Unified training trait for all models // NEW: Model-agnostic training orchestrator diff --git a/crates/ml/src/training/push_metrics.rs b/crates/ml/src/training/push_metrics.rs new file mode 100644 index 000000000..01d8d166c --- /dev/null +++ b/crates/ml/src/training/push_metrics.rs @@ -0,0 +1,211 @@ +//! Prometheus Pushgateway integration for training job metrics +//! +//! Training jobs are short-lived K8s Jobs. They push epoch-level metrics +//! to the Pushgateway so Prometheus can scrape them persistently. + +use std::fmt::Write; + +/// Pushgateway client for training metrics +#[derive(Debug)] +pub struct TrainingMetricsPusher { + pushgateway_url: String, + job_id: String, + model_name: String, + client: reqwest::Client, +} + +impl TrainingMetricsPusher { + /// Create a new pusher. Falls back to in-cluster Pushgateway if `PUSHGATEWAY_URL` not set. + pub fn new(job_id: &str, model_name: &str) -> Self { + let pushgateway_url = std::env::var("PUSHGATEWAY_URL") + .unwrap_or_else(|_| "http://pushgateway.foxhunt.svc.cluster.local:9091".to_string()); + Self { + pushgateway_url, + job_id: job_id.to_string(), + model_name: model_name.to_string(), + client: reqwest::Client::new(), + } + } + + /// Push current training state to Pushgateway. + /// Non-fatal: logs errors but never panics. + pub async fn push(&self, state: &TrainingState) { + let mut body = String::new(); + + // Epoch progress + let _ = writeln!(body, "# HELP foxhunt_training_current_epoch Current training epoch"); + let _ = writeln!(body, "# TYPE foxhunt_training_current_epoch gauge"); + let _ = writeln!(body, "foxhunt_training_current_epoch {}", state.epoch); + + // Loss + let _ = writeln!( + body, + "# HELP foxhunt_training_epoch_loss Training loss value" + ); + let _ = writeln!(body, "# TYPE foxhunt_training_epoch_loss gauge"); + let _ = writeln!( + body, + "foxhunt_training_epoch_loss{{split=\"train\"}} {}", + state.train_loss + ); + if let Some(val_loss) = state.val_loss { + let _ = writeln!( + body, + "foxhunt_training_epoch_loss{{split=\"val\"}} {}", + val_loss + ); + } + + // Accuracy + if let Some(accuracy) = state.accuracy { + let _ = writeln!( + body, + "# HELP foxhunt_training_eval_accuracy Model evaluation accuracy" + ); + let _ = writeln!(body, "# TYPE foxhunt_training_eval_accuracy gauge"); + let _ = writeln!(body, "foxhunt_training_eval_accuracy {}", accuracy); + } + + // Batches processed + let _ = writeln!( + body, + "# HELP foxhunt_training_batches_processed Total batches processed" + ); + let _ = writeln!(body, "# TYPE foxhunt_training_batches_processed counter"); + let _ = writeln!( + body, + "foxhunt_training_batches_processed {}", + state.batches_processed + ); + + // Training speed + if state.batches_per_second > 0.0 { + let _ = writeln!( + body, + "# HELP foxhunt_training_batches_per_second Training throughput" + ); + let _ = writeln!(body, "# TYPE foxhunt_training_batches_per_second gauge"); + let _ = writeln!( + body, + "foxhunt_training_batches_per_second {}", + state.batches_per_second + ); + } + + // Learning rate + let _ = writeln!( + body, + "# HELP foxhunt_training_learning_rate Current learning rate" + ); + let _ = writeln!(body, "# TYPE foxhunt_training_learning_rate gauge"); + let _ = writeln!( + body, + "foxhunt_training_learning_rate {}", + state.learning_rate + ); + + // NaN / gradient events + if state.nan_count > 0 { + let _ = writeln!( + body, + "# HELP foxhunt_training_nan_detected_total NaN gradient events" + ); + let _ = writeln!(body, "# TYPE foxhunt_training_nan_detected_total counter"); + let _ = writeln!( + body, + "foxhunt_training_nan_detected_total {}", + state.nan_count + ); + } + if state.gradient_explosion_count > 0 { + let _ = writeln!( + body, + "# HELP foxhunt_training_gradient_explosion_total Gradient clipping events" + ); + let _ = writeln!( + body, + "# TYPE foxhunt_training_gradient_explosion_total counter" + ); + let _ = writeln!( + body, + "foxhunt_training_gradient_explosion_total {}", + state.gradient_explosion_count + ); + } + + // Checkpoint saves + let _ = writeln!( + body, + "# HELP foxhunt_training_checkpoint_saves_total Checkpoints saved" + ); + let _ = writeln!( + body, + "# TYPE foxhunt_training_checkpoint_saves_total counter" + ); + let _ = writeln!( + body, + "foxhunt_training_checkpoint_saves_total {}", + state.checkpoint_saves + ); + + let url = format!( + "{}/metrics/job/{}/model/{}", + self.pushgateway_url, self.job_id, self.model_name + ); + + match self.client.put(&url).body(body).send().await { + Ok(resp) if resp.status().is_success() => {} + Ok(resp) => { + eprintln!( + "Pushgateway returned {}: {}", + resp.status(), + resp.text().await.unwrap_or_default() + ); + } + Err(e) => { + eprintln!("Failed to push metrics to Pushgateway: {e}"); + } + } + } + + /// Delete metrics for this job from Pushgateway (call on training completion). + pub async fn cleanup(&self) { + let url = format!( + "{}/metrics/job/{}/model/{}", + self.pushgateway_url, self.job_id, self.model_name + ); + let _ = self.client.delete(&url).send().await; + } +} + +/// Current training state to push +#[derive(Debug)] +pub struct TrainingState { + pub epoch: u64, + pub train_loss: f64, + pub val_loss: Option, + pub accuracy: Option, + pub batches_processed: u64, + pub batches_per_second: f64, + pub learning_rate: f64, + pub nan_count: u64, + pub gradient_explosion_count: u64, + pub checkpoint_saves: u64, +} + +impl Default for TrainingState { + fn default() -> Self { + Self { + epoch: 0, + train_loss: 0.0, + val_loss: None, + accuracy: None, + batches_processed: 0, + batches_per_second: 0.0, + learning_rate: 0.001, + nan_count: 0, + gradient_explosion_count: 0, + checkpoint_saves: 0, + } + } +}