{ "dashboard": { "id": null, "uid": "ml-training-monitoring", "title": "ML Training Service - Production Monitoring", "tags": ["ml", "training", "gpu", "monitoring"], "timezone": "browser", "schemaVersion": 16, "version": 0, "refresh": "30s", "time": { "from": "now-6h", "to": "now" }, "panels": [ { "id": 1, "title": "Training Jobs by Status", "type": "stat", "gridPos": { "x": 0, "y": 0, "w": 6, "h": 4 }, "targets": [ { "expr": "ml_training_jobs_by_status", "legendFormat": "{{status}}", "refId": "A" } ], "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } } } }, { "id": 2, "title": "GPU Memory Usage", "type": "gauge", "gridPos": { "x": 6, "y": 0, "w": 6, "h": 4 }, "targets": [ { "expr": "100 * ml_gpu_memory_used_bytes / ml_gpu_memory_total_bytes", "legendFormat": "GPU {{gpu_id}}", "refId": "A" } ], "fieldConfig": { "defaults": { "min": 0, "max": 100, "unit": "percent", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "yellow", "value": 80 }, { "color": "red", "value": 90 } ] } } } }, { "id": 3, "title": "GPU Temperature", "type": "gauge", "gridPos": { "x": 12, "y": 0, "w": 6, "h": 4 }, "targets": [ { "expr": "ml_gpu_temperature_celsius", "legendFormat": "GPU {{gpu_id}}", "refId": "A" } ], "fieldConfig": { "defaults": { "min": 0, "max": 100, "unit": "celsius", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "yellow", "value": 75 }, { "color": "red", "value": 85 } ] } } } }, { "id": 4, "title": "GPU Utilization", "type": "gauge", "gridPos": { "x": 18, "y": 0, "w": 6, "h": 4 }, "targets": [ { "expr": "ml_gpu_utilization_percent", "legendFormat": "GPU {{gpu_id}}", "refId": "A" } ], "fieldConfig": { "defaults": { "min": 0, "max": 100, "unit": "percent", "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": 0 }, { "color": "yellow", "value": 30 }, { "color": "green", "value": 60 } ] } } } }, { "id": 5, "title": "Training Loss (All Models)", "type": "graph", "gridPos": { "x": 0, "y": 4, "w": 12, "h": 8 }, "targets": [ { "expr": "ml_training_loss", "legendFormat": "{{model_type}} - {{job_id}}", "refId": "A" } ], "yaxes": [ { "format": "short", "logBase": 1, "show": true } ], "xaxis": { "show": true, "mode": "time" } }, { "id": 6, "title": "Validation Loss (All Models)", "type": "graph", "gridPos": { "x": 12, "y": 4, "w": 12, "h": 8 }, "targets": [ { "expr": "ml_training_validation_loss", "legendFormat": "{{model_type}} - {{job_id}}", "refId": "A" } ], "yaxes": [ { "format": "short", "logBase": 1, "show": true } ], "xaxis": { "show": true, "mode": "time" } }, { "id": 7, "title": "Training Speed (Epochs/sec)", "type": "graph", "gridPos": { "x": 0, "y": 12, "w": 12, "h": 6 }, "targets": [ { "expr": "ml_training_epochs_per_second", "legendFormat": "{{model_type}} - {{job_id}}", "refId": "A" } ], "yaxes": [ { "format": "short", "logBase": 1, "show": true, "min": 0 } ], "alert": { "conditions": [ { "evaluator": { "params": [0.1], "type": "lt" }, "operator": { "type": "and" }, "query": { "params": ["A", "5m", "now"] }, "reducer": { "params": [], "type": "avg" }, "type": "query" } ], "executionErrorState": "alerting", "for": "5m", "frequency": "1m", "name": "Training Speed Degraded", "noDataState": "no_data", "notifications": [] } }, { "id": 8, "title": "Training Progress (%)", "type": "graph", "gridPos": { "x": 12, "y": 12, "w": 12, "h": 6 }, "targets": [ { "expr": "ml_training_progress_percent", "legendFormat": "{{model_type}} - {{job_id}}", "refId": "A" } ], "yaxes": [ { "format": "percent", "logBase": 1, "show": true, "min": 0, "max": 100 } ] }, { "id": 9, "title": "Checkpoint Save Duration (P95)", "type": "graph", "gridPos": { "x": 0, "y": 18, "w": 12, "h": 6 }, "targets": [ { "expr": "histogram_quantile(0.95, rate(ml_checkpoint_save_duration_seconds_bucket[5m]))", "legendFormat": "{{model_type}} - P95", "refId": "A" } ], "yaxes": [ { "format": "s", "logBase": 1, "show": true } ] }, { "id": 10, "title": "NaN Detection Events", "type": "graph", "gridPos": { "x": 12, "y": 18, "w": 12, "h": 6 }, "targets": [ { "expr": "rate(ml_training_nan_count[5m])", "legendFormat": "{{model_type}} - {{tensor_type}}", "refId": "A" } ], "yaxes": [ { "format": "short", "logBase": 1, "show": true, "min": 0 } ] }, { "id": 11, "title": "Model Accuracy (Validation)", "type": "graph", "gridPos": { "x": 0, "y": 24, "w": 12, "h": 6 }, "targets": [ { "expr": "ml_model_accuracy", "legendFormat": "{{model_type}}", "refId": "A" } ], "yaxes": [ { "format": "percentunit", "logBase": 1, "show": true, "min": 0, "max": 1 } ] }, { "id": 12, "title": "Data Loading Duration (P95)", "type": "graph", "gridPos": { "x": 12, "y": 24, "w": 12, "h": 6 }, "targets": [ { "expr": "histogram_quantile(0.95, rate(ml_training_data_loading_seconds_bucket[5m]))", "legendFormat": "{{model_type}} - P95", "refId": "A" } ], "yaxes": [ { "format": "s", "logBase": 1, "show": true } ] }, { "id": 13, "title": "Training Failures by Type", "type": "piechart", "gridPos": { "x": 0, "y": 30, "w": 8, "h": 6 }, "targets": [ { "expr": "sum by (error_type) (rate(ml_training_failures_total[1h]))", "legendFormat": "{{error_type}}", "refId": "A" } ] }, { "id": 14, "title": "S3 Request Errors", "type": "stat", "gridPos": { "x": 8, "y": 30, "w": 8, "h": 6 }, "targets": [ { "expr": "rate(ml_s3_request_errors_total[5m])", "legendFormat": "Errors/sec", "refId": "A" } ], "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 0.5 }, { "color": "red", "value": 1 } ] } } } }, { "id": 15, "title": "Model Storage Usage", "type": "graph", "gridPos": { "x": 16, "y": 30, "w": 8, "h": 6 }, "targets": [ { "expr": "100 * ml_model_storage_used_bytes / ml_model_storage_limit_bytes", "legendFormat": "Storage Usage %", "refId": "A" } ], "yaxes": [ { "format": "percent", "logBase": 1, "show": true, "min": 0, "max": 100 } ] }, { "id": 16, "title": "Data Drift Score (All Features)", "type": "graph", "gridPos": { "x": 0, "y": 36, "w": 12, "h": 6 }, "targets": [ { "expr": "ml_model_drift_score", "legendFormat": "{{feature}}", "refId": "A" } ], "yaxes": [ { "format": "short", "logBase": 1, "show": true, "min": 0, "max": 1 } ], "alert": { "conditions": [ { "evaluator": { "params": [0.15], "type": "gt" }, "operator": { "type": "and" }, "query": { "params": ["A", "5m", "now"] }, "reducer": { "params": [], "type": "avg" }, "type": "query" } ], "executionErrorState": "alerting", "for": "5m", "frequency": "1m", "name": "Data Drift Detected", "noDataState": "no_data", "notifications": [] } }, { "id": 17, "title": "Cost Tracking (Monthly Projection)", "type": "stat", "gridPos": { "x": 12, "y": 36, "w": 12, "h": 6 }, "targets": [ { "expr": "ml_monthly_cost_projection_dollars", "legendFormat": "Projected Monthly Cost", "refId": "A" } ], "fieldConfig": { "defaults": { "unit": "currencyUSD", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 800 }, { "color": "red", "value": 1000 } ] } } } } ], "annotations": { "list": [ { "datasource": "Prometheus", "enable": true, "expr": "ALERTS{alertname=~\".*Training.*|.*GPU.*\"}", "iconColor": "red", "name": "Training Alerts", "step": "60s", "tagKeys": "alertname,severity", "titleFormat": "Alert: {{alertname}}", "type": "tags" } ] } } }