diff --git a/infra/k8s/monitoring/dashboards/foxhunt-training.json b/infra/k8s/monitoring/dashboards/foxhunt-training.json index 285066d9f..927f04d49 100644 --- a/infra/k8s/monitoring/dashboards/foxhunt-training.json +++ b/infra/k8s/monitoring/dashboards/foxhunt-training.json @@ -2,12 +2,20 @@ "uid": "foxhunt-training", "title": "Foxhunt Training & CI/CD", "description": "ML model training, hyperopt, GPU utilization, and Argo CI pipelines", - "tags": ["foxhunt", "training", "gpu", "ci"], + "tags": [ + "foxhunt", + "training", + "gpu", + "ci" + ], "editable": true, "graphTooltip": 1, "refresh": "15s", "schemaVersion": 39, - "time": { "from": "now-6h", "to": "now" }, + "time": { + "from": "now-6h", + "to": "now" + }, "timepicker": {}, "timezone": "browser", "templating": { @@ -19,13 +27,37 @@ "auto": true, "auto_count": 30, "auto_min": "1m", - "current": { "selected": true, "text": "auto", "value": "$__auto_interval_interval" }, + "current": { + "selected": true, + "text": "auto", + "value": "$__auto_interval_interval" + }, "options": [ - { "selected": false, "text": "1m", "value": "1m" }, - { "selected": false, "text": "5m", "value": "5m" }, - { "selected": false, "text": "15m", "value": "15m" }, - { "selected": false, "text": "30m", "value": "30m" }, - { "selected": false, "text": "1h", "value": "1h" } + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "15m", + "value": "15m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } ], "query": "1m,5m,15m,30m,1h" }, @@ -33,26 +65,40 @@ "name": "model", "label": "Model", "type": "query", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "query": "label_values(foxhunt_training_current_epoch, model)", "refresh": 2, "includeAll": true, "multi": true, "allValue": ".*", - "current": { "selected": true, "text": "All", "value": "$__all" }, + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, "sort": 1 }, { "name": "fold", "label": "Fold", "type": "query", - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "query": "label_values(foxhunt_training_current_epoch, fold)", "refresh": 2, "includeAll": true, "multi": true, "allValue": ".*", - "current": { "selected": true, "text": "All", "value": "$__all" }, + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, "sort": 1 } ] @@ -61,7 +107,12 @@ { "type": "row", "title": "Training Status", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, "collapsed": false, "id": 1, "panels": [] @@ -70,24 +121,49 @@ "id": 2, "type": "stat", "title": "Active Workers", - "gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 1 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ - { "color": "red", "value": null }, - { "color": "yellow", "value": 1 }, - { "color": "green", "value": 2 } + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "green", + "value": 2 + } ] }, - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "colorMode": "background", "graphMode": "none", "justifyMode": "center", @@ -95,7 +171,10 @@ }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "expr": "foxhunt_training_active_workers", "legendFormat": "Workers", "refId": "A" @@ -106,22 +185,41 @@ "id": 3, "type": "stat", "title": "Current Epoch", - "gridPos": { "h": 4, "w": 4, "x": 4, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 1 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ - { "color": "blue", "value": null } + { + "color": "blue", + "value": null + } ] }, - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "colorMode": "value", "graphMode": "area", "justifyMode": "center", @@ -129,7 +227,10 @@ }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "expr": "foxhunt_training_current_epoch{model=~\"$model\", fold=~\"$fold\"}", "legendFormat": "{{model}} / fold {{fold}}", "refId": "A" @@ -141,26 +242,62 @@ "type": "stat", "title": "Mode (Training / Hyperopt)", "description": "1 = hyperopt, 0 = training", - "gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 1 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "mappings": [ - { "type": "value", "options": { "0": { "text": "Training", "color": "green", "index": 0 }, "1": { "text": "Hyperopt", "color": "orange", "index": 1 } } } + { + "type": "value", + "options": { + "0": { + "text": "Training", + "color": "green", + "index": 0 + }, + "1": { + "text": "Hyperopt", + "color": "orange", + "index": 1 + } + } + } ], "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "orange", "value": 1 } + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 1 + } ] }, - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "colorMode": "background", "graphMode": "none", "justifyMode": "center", @@ -168,7 +305,10 @@ }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "expr": "foxhunt_hyperopt_mode{model=~\"$model\"}", "legendFormat": "{{model}}", "refId": "A" @@ -179,8 +319,16 @@ "id": 5, "type": "gauge", "title": "Hyperopt Trial Progress", - "gridPos": { "h": 4, "w": 4, "x": 12, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 1 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "min": 0, @@ -189,23 +337,43 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "red", "value": null }, - { "color": "yellow", "value": 0.5 }, - { "color": "green", "value": 0.9 } + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 0.5 + }, + { + "color": "green", + "value": 0.9 + } ] }, - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "showThresholdLabels": false, "showThresholdMarkers": true }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "expr": "foxhunt_hyperopt_trial_current{model=~\"$model\"} / foxhunt_hyperopt_trial_total{model=~\"$model\"}", "legendFormat": "{{model}}", "refId": "A" @@ -216,22 +384,41 @@ "id": 6, "type": "stat", "title": "ML Service Jobs Started", - "gridPos": { "h": 4, "w": 4, "x": 16, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 1 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ - { "color": "blue", "value": null } + { + "color": "blue", + "value": null + } ] }, - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "colorMode": "value", "graphMode": "area", "justifyMode": "center", @@ -239,7 +426,10 @@ }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "expr": "ml_training_jobs_started_total", "legendFormat": "Started", "refId": "A" @@ -250,23 +440,45 @@ "id": 7, "type": "stat", "title": "ML Service Errors", - "gridPos": { "h": 4, "w": 4, "x": 20, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 1 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "red", "value": 1 } + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } ] }, - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "colorMode": "background", "graphMode": "area", "justifyMode": "center", @@ -274,17 +486,408 @@ }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "expr": "ml_training_errors_total", "legendFormat": "Errors", "refId": "A" } ] }, + { + "type": "row", + "title": "Run Summary", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 66, + "panels": [] + }, + { + "type": "stat", + "title": "Best Val Loss", + "id": 67, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 6 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.1 + }, + { + "color": "red", + "value": 0.5 + } + ] + }, + "decimals": 6, + "unit": "none" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "min" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto" + }, + "targets": [ + { + "expr": "foxhunt_training_validation_loss{model=~\"$model\", fold=~\"$fold\"}", + "legendFormat": "{{model}}/{{fold}}", + "refId": "A" + } + ] + }, + { + "type": "stat", + "title": "Best Sharpe", + "id": 68, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 6 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "green", + "value": 3 + } + ] + }, + "decimals": 2, + "unit": "none" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "max" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto" + }, + "targets": [ + { + "expr": "foxhunt_training_epoch_sharpe{model=~\"$model\", fold=~\"$fold\"}", + "legendFormat": "{{model}}/{{fold}}", + "refId": "A" + } + ] + }, + { + "type": "stat", + "title": "Best Win Rate", + "id": 69, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 6 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 0.4 + }, + { + "color": "green", + "value": 0.55 + } + ] + }, + "decimals": 1, + "unit": "percentunit" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "max" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto" + }, + "targets": [ + { + "expr": "foxhunt_training_epoch_win_rate{model=~\"$model\", fold=~\"$fold\"}", + "legendFormat": "{{model}}/{{fold}}", + "refId": "A" + } + ] + }, + { + "type": "stat", + "title": "Min Max Drawdown", + "id": 70, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 6 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.1 + }, + { + "color": "red", + "value": 0.25 + } + ] + }, + "decimals": 1, + "unit": "percentunit" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "min" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto" + }, + "targets": [ + { + "expr": "foxhunt_training_epoch_max_drawdown{model=~\"$model\", fold=~\"$fold\"}", + "legendFormat": "{{model}}/{{fold}}", + "refId": "A" + } + ] + }, + { + "type": "stat", + "title": "Total Epochs", + "id": 71, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 6 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "decimals": 0, + "unit": "none" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "max" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto" + }, + "targets": [ + { + "expr": "foxhunt_training_current_epoch{model=~\"$model\", fold=~\"$fold\"}", + "legendFormat": "{{model}}/{{fold}}", + "refId": "A" + } + ] + }, + { + "type": "stat", + "title": "Best Profit Factor", + "id": 72, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 6 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 1.0 + }, + { + "color": "green", + "value": 1.5 + } + ] + }, + "decimals": 2, + "unit": "none" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "max" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto" + }, + "targets": [ + { + "expr": "foxhunt_training_epoch_profit_factor{model=~\"$model\", fold=~\"$fold\"}", + "legendFormat": "{{model}}/{{fold}}", + "refId": "A" + } + ] + }, { "type": "row", "title": "Training Curves", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, "collapsed": false, "id": 10, "panels": [] @@ -293,8 +896,16 @@ "id": 11, "type": "timeseries", "title": "Training & Validation Loss", - "gridPos": { "h": 8, "w": 8, "x": 0, "y": 6 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 11 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "custom": { @@ -308,23 +919,41 @@ "axisCenteredZero": false, "axisLabel": "Loss" }, - "color": { "mode": "palette-classic" } + "color": { + "mode": "palette-classic" + } }, "overrides": [] }, "options": { - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "min"] }, - "tooltip": { "mode": "multi", "sort": "desc" } + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "min" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "expr": "foxhunt_training_epoch_loss{model=~\"$model\", fold=~\"$fold\"}", "legendFormat": "train {{model}}/{{fold}}", "refId": "A" }, { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "expr": "foxhunt_training_validation_loss{model=~\"$model\", fold=~\"$fold\"}", "legendFormat": "val {{model}}/{{fold}}", "refId": "B" @@ -335,8 +964,16 @@ "id": 12, "type": "timeseries", "title": "Learning Rate", - "gridPos": { "h": 8, "w": 8, "x": 8, "y": 6 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 11 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "custom": { @@ -346,21 +983,38 @@ "fillOpacity": 5, "showPoints": "auto", "pointSize": 5, - "scaleDistribution": { "type": "log", "log": 10 }, + "scaleDistribution": { + "type": "log", + "log": 10 + }, "axisLabel": "LR" }, - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "unit": "short" }, "overrides": [] }, "options": { - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull"] }, - "tooltip": { "mode": "multi", "sort": "desc" } + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "expr": "foxhunt_training_learning_rate{model=~\"$model\", fold=~\"$fold\"}", "legendFormat": "{{model}}/{{fold}}", "refId": "A" @@ -371,8 +1025,16 @@ "id": 13, "type": "timeseries", "title": "Gradient Norm", - "gridPos": { "h": 8, "w": 4, "x": 16, "y": 6 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 8, + "w": 4, + "x": 16, + "y": 11 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "custom": { @@ -383,17 +1045,28 @@ "showPoints": "never", "axisLabel": "Norm" }, - "color": { "mode": "palette-classic" } + "color": { + "mode": "palette-classic" + } }, "overrides": [] }, "options": { - "legend": { "displayMode": "list", "placement": "bottom" }, - "tooltip": { "mode": "multi", "sort": "desc" } + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "expr": "foxhunt_training_gradient_norm{model=~\"$model\", fold=~\"$fold\"}", "legendFormat": "{{model}}/{{fold}}", "refId": "A" @@ -404,8 +1077,16 @@ "id": 14, "type": "timeseries", "title": "Batches/sec & Epoch Duration", - "gridPos": { "h": 8, "w": 4, "x": 20, "y": 6 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 8, + "w": 4, + "x": 20, + "y": 11 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "custom": { @@ -415,31 +1096,54 @@ "fillOpacity": 10, "showPoints": "never" }, - "color": { "mode": "palette-classic" } + "color": { + "mode": "palette-classic" + } }, "overrides": [ { - "matcher": { "id": "byFrameRefID", "options": "B" }, + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, "properties": [ - { "id": "custom.axisPlacement", "value": "right" }, - { "id": "unit", "value": "s" } + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "s" + } ] } ] }, "options": { - "legend": { "displayMode": "list", "placement": "bottom" }, - "tooltip": { "mode": "multi", "sort": "desc" } + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "expr": "foxhunt_training_batches_per_second{model=~\"$model\", fold=~\"$fold\"}", "legendFormat": "batch/s {{model}}", "refId": "A" }, { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "expr": "foxhunt_training_epoch_duration_seconds{model=~\"$model\", fold=~\"$fold\"}", "legendFormat": "epoch dur {{model}}", "refId": "B" @@ -449,7 +1153,12 @@ { "type": "row", "title": "Training Quality", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 19 + }, "collapsed": false, "id": 20, "panels": [] @@ -458,8 +1167,16 @@ "id": 21, "type": "timeseries", "title": "Epoch Sharpe Ratio", - "gridPos": { "h": 7, "w": 6, "x": 0, "y": 15 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 20 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "custom": { @@ -472,25 +1189,49 @@ "pointSize": 4, "axisLabel": "Sharpe" }, - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "thresholds": { "mode": "absolute", "steps": [ - { "color": "red", "value": null }, - { "color": "yellow", "value": 1 }, - { "color": "green", "value": 2 } + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "green", + "value": 2 + } ] } }, "overrides": [] }, "options": { - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "max"] }, - "tooltip": { "mode": "multi", "sort": "desc" } + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "expr": "foxhunt_training_epoch_sharpe{model=~\"$model\", fold=~\"$fold\"}", "legendFormat": "{{model}}/{{fold}}", "refId": "A" @@ -501,8 +1242,16 @@ "id": 22, "type": "timeseries", "title": "Epoch Sortino Ratio", - "gridPos": { "h": 7, "w": 6, "x": 6, "y": 15 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 20 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "custom": { @@ -514,17 +1263,32 @@ "pointSize": 4, "axisLabel": "Sortino" }, - "color": { "mode": "palette-classic" } + "color": { + "mode": "palette-classic" + } }, "overrides": [] }, "options": { - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "max"] }, - "tooltip": { "mode": "multi", "sort": "desc" } + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "expr": "foxhunt_training_epoch_sortino{model=~\"$model\", fold=~\"$fold\"}", "legendFormat": "{{model}}/{{fold}}", "refId": "A" @@ -535,8 +1299,16 @@ "id": 23, "type": "timeseries", "title": "Win Rate", - "gridPos": { "h": 7, "w": 4, "x": 12, "y": 15 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 7, + "w": 4, + "x": 12, + "y": 20 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "custom": { @@ -548,7 +1320,9 @@ "pointSize": 4, "axisLabel": "Win Rate" }, - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "unit": "percentunit", "min": 0, "max": 1 @@ -556,12 +1330,21 @@ "overrides": [] }, "options": { - "legend": { "displayMode": "list", "placement": "bottom" }, - "tooltip": { "mode": "multi", "sort": "desc" } + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "expr": "foxhunt_training_epoch_win_rate{model=~\"$model\", fold=~\"$fold\"}", "legendFormat": "{{model}}/{{fold}}", "refId": "A" @@ -572,8 +1355,16 @@ "id": 24, "type": "timeseries", "title": "Max Drawdown", - "gridPos": { "h": 7, "w": 4, "x": 16, "y": 15 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 7, + "w": 4, + "x": 16, + "y": 20 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "custom": { @@ -585,18 +1376,29 @@ "pointSize": 4, "axisLabel": "Drawdown" }, - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "unit": "percentunit" }, "overrides": [] }, "options": { - "legend": { "displayMode": "list", "placement": "bottom" }, - "tooltip": { "mode": "multi", "sort": "desc" } + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "expr": "foxhunt_training_epoch_max_drawdown{model=~\"$model\", fold=~\"$fold\"}", "legendFormat": "{{model}}/{{fold}}", "refId": "A" @@ -607,8 +1409,16 @@ "id": 25, "type": "timeseries", "title": "Profit Factor & Total Trades", - "gridPos": { "h": 7, "w": 4, "x": 20, "y": 15 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 7, + "w": 4, + "x": 20, + "y": 20 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "custom": { @@ -619,33 +1429,62 @@ "showPoints": "auto", "pointSize": 4 }, - "color": { "mode": "palette-classic" } + "color": { + "mode": "palette-classic" + } }, "overrides": [ { - "matcher": { "id": "byFrameRefID", "options": "B" }, + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, "properties": [ - { "id": "custom.axisPlacement", "value": "right" }, - { "id": "unit", "value": "short" }, - { "id": "custom.drawStyle", "value": "bars" }, - { "id": "custom.fillOpacity", "value": 30 } + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "short" + }, + { + "id": "custom.drawStyle", + "value": "bars" + }, + { + "id": "custom.fillOpacity", + "value": 30 + } ] } ] }, "options": { - "legend": { "displayMode": "list", "placement": "bottom" }, - "tooltip": { "mode": "multi", "sort": "desc" } + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "expr": "foxhunt_training_epoch_profit_factor{model=~\"$model\", fold=~\"$fold\"}", "legendFormat": "PF {{model}}", "refId": "A" }, { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "expr": "foxhunt_training_epoch_total_trades{model=~\"$model\", fold=~\"$fold\"}", "legendFormat": "trades {{model}}", "refId": "B" @@ -655,7 +1494,12 @@ { "type": "row", "title": "RL Diagnostics", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 22 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 27 + }, "collapsed": false, "id": 30, "panels": [] @@ -664,8 +1508,16 @@ "id": 31, "type": "timeseries", "title": "Q-Value (Mean & Max)", - "gridPos": { "h": 7, "w": 6, "x": 0, "y": 23 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 28 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "custom": { @@ -676,23 +1528,42 @@ "showPoints": "never", "axisLabel": "Q-Value" }, - "color": { "mode": "palette-classic" } + "color": { + "mode": "palette-classic" + } }, "overrides": [] }, "options": { - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "max", "min"] }, - "tooltip": { "mode": "multi", "sort": "desc" } + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max", + "min" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "expr": "foxhunt_training_q_value_mean{model=~\"$model\", fold=~\"$fold\"}", "legendFormat": "mean {{model}}/{{fold}}", "refId": "A" }, { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "expr": "foxhunt_training_q_value_max{model=~\"$model\", fold=~\"$fold\"}", "legendFormat": "max {{model}}/{{fold}}", "refId": "B" @@ -703,8 +1574,16 @@ "id": 32, "type": "timeseries", "title": "Replay Buffer Size", - "gridPos": { "h": 7, "w": 4, "x": 6, "y": 23 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 7, + "w": 4, + "x": 6, + "y": 28 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "custom": { @@ -715,18 +1594,29 @@ "showPoints": "never", "axisLabel": "Transitions" }, - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "unit": "short" }, "overrides": [] }, "options": { - "legend": { "displayMode": "list", "placement": "bottom" }, - "tooltip": { "mode": "multi", "sort": "desc" } + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "expr": "foxhunt_training_replay_buffer_size{model=~\"$model\", fold=~\"$fold\"}", "legendFormat": "{{model}}/{{fold}}", "refId": "A" @@ -738,8 +1628,16 @@ "type": "timeseries", "title": "Action Entropy", "description": "Normalized entropy of policy actions. 0 = collapsed to single action, 1 = uniform distribution.", - "gridPos": { "h": 7, "w": 4, "x": 10, "y": 23 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 7, + "w": 4, + "x": 10, + "y": 28 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "custom": { @@ -750,28 +1648,50 @@ "showPoints": "never", "axisLabel": "Entropy" }, - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "min": 0, "max": 1, "thresholds": { "mode": "absolute", "steps": [ - { "color": "red", "value": null }, - { "color": "yellow", "value": 0.1 }, - { "color": "green", "value": 0.3 } + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 0.1 + }, + { + "color": "green", + "value": 0.3 + } ] }, - "custom.thresholdsStyle": { "mode": "line" } + "custom.thresholdsStyle": { + "mode": "line" + } }, "overrides": [] }, "options": { - "legend": { "displayMode": "list", "placement": "bottom" }, - "tooltip": { "mode": "multi", "sort": "desc" } + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "expr": "foxhunt_training_action_entropy{model=~\"$model\", fold=~\"$fold\"}", "legendFormat": "{{model}}/{{fold}}", "refId": "A" @@ -783,8 +1703,16 @@ "type": "timeseries", "title": "Action Diversity", "description": "Fraction of 45 factored actions actually used by the policy.", - "gridPos": { "h": 7, "w": 4, "x": 14, "y": 23 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 7, + "w": 4, + "x": 14, + "y": 28 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "custom": { @@ -795,7 +1723,9 @@ "showPoints": "never", "axisLabel": "Diversity" }, - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "unit": "percentunit", "min": 0, "max": 1 @@ -803,12 +1733,21 @@ "overrides": [] }, "options": { - "legend": { "displayMode": "list", "placement": "bottom" }, - "tooltip": { "mode": "multi", "sort": "desc" } + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "expr": "foxhunt_training_action_diversity{model=~\"$model\", fold=~\"$fold\"}", "legendFormat": "{{model}}/{{fold}}", "refId": "A" @@ -819,8 +1758,16 @@ "id": 35, "type": "timeseries", "title": "Action Distribution (Buy/Sell/Hold %)", - "gridPos": { "h": 7, "w": 6, "x": 18, "y": 23 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 28 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "custom": { @@ -829,48 +1776,100 @@ "lineWidth": 2, "fillOpacity": 20, "showPoints": "never", - "stacking": { "mode": "normal" }, + "stacking": { + "mode": "normal" + }, "axisLabel": "Pct" }, - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "unit": "percentunit", "min": 0, "max": 1 }, "overrides": [ { - "matcher": { "id": "byRegexp", "options": "buy.*" }, - "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] + "matcher": { + "id": "byRegexp", + "options": "buy.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] }, { - "matcher": { "id": "byRegexp", "options": "sell.*" }, - "properties": [ { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } ] + "matcher": { + "id": "byRegexp", + "options": "sell.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] }, { - "matcher": { "id": "byRegexp", "options": "hold.*" }, - "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] + "matcher": { + "id": "byRegexp", + "options": "hold.*" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] } ] }, "options": { - "legend": { "displayMode": "list", "placement": "bottom" }, - "tooltip": { "mode": "multi", "sort": "desc" } + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "expr": "foxhunt_training_epoch_action_buy_pct{model=~\"$model\", fold=~\"$fold\"}", "legendFormat": "buy {{model}}", "refId": "A" }, { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "expr": "foxhunt_training_epoch_action_sell_pct{model=~\"$model\", fold=~\"$fold\"}", "legendFormat": "sell {{model}}", "refId": "B" }, { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "expr": "foxhunt_training_epoch_action_hold_pct{model=~\"$model\", fold=~\"$fold\"}", "legendFormat": "hold {{model}}", "refId": "C" @@ -880,7 +1879,12 @@ { "type": "row", "title": "Hyperopt", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 30 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 35 + }, "collapsed": false, "id": 40, "panels": [] @@ -889,8 +1893,16 @@ "id": 41, "type": "gauge", "title": "Trial Progress (current / total)", - "gridPos": { "h": 6, "w": 4, "x": 0, "y": 31 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 36 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "min": 0, @@ -899,24 +1911,47 @@ "thresholds": { "mode": "absolute", "steps": [ - { "color": "red", "value": null }, - { "color": "orange", "value": 0.25 }, - { "color": "yellow", "value": 0.5 }, - { "color": "green", "value": 0.75 } + { + "color": "red", + "value": null + }, + { + "color": "orange", + "value": 0.25 + }, + { + "color": "yellow", + "value": 0.5 + }, + { + "color": "green", + "value": 0.75 + } ] }, - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "showThresholdLabels": false, "showThresholdMarkers": true }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "expr": "foxhunt_hyperopt_trial_current{model=~\"$model\"} / foxhunt_hyperopt_trial_total{model=~\"$model\"}", "legendFormat": "{{model}}", "refId": "A" @@ -927,8 +1962,16 @@ "id": 42, "type": "timeseries", "title": "Best Objective Over Time", - "gridPos": { "h": 6, "w": 6, "x": 4, "y": 31 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 6, + "w": 6, + "x": 4, + "y": 36 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "custom": { @@ -940,23 +1983,41 @@ "pointSize": 4, "axisLabel": "Objective" }, - "color": { "mode": "palette-classic" } + "color": { + "mode": "palette-classic" + } }, "overrides": [] }, "options": { - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "min"] }, - "tooltip": { "mode": "multi", "sort": "asc" } + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "min" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "expr": "foxhunt_hyperopt_best_objective{model=~\"$model\"}", "legendFormat": "best {{model}}", "refId": "A" }, { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "expr": "foxhunt_hyperopt_trial_best_loss{model=~\"$model\"}", "legendFormat": "trial loss {{model}}", "refId": "B" @@ -967,8 +2028,16 @@ "id": 43, "type": "timeseries", "title": "Trial Duration (p50/p95/p99)", - "gridPos": { "h": 6, "w": 6, "x": 10, "y": 31 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 6, + "w": 6, + "x": 10, + "y": 36 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "custom": { @@ -979,30 +2048,47 @@ "showPoints": "never", "axisLabel": "Duration" }, - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "unit": "s" }, "overrides": [] }, "options": { - "legend": { "displayMode": "list", "placement": "bottom" }, - "tooltip": { "mode": "multi", "sort": "desc" } + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "expr": "histogram_quantile(0.50, rate(foxhunt_hyperopt_trial_duration_seconds_bucket{model=~\"$model\"}[$__rate_interval]))", "legendFormat": "p50 {{model}}", "refId": "A" }, { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "expr": "histogram_quantile(0.95, rate(foxhunt_hyperopt_trial_duration_seconds_bucket{model=~\"$model\"}[$__rate_interval]))", "legendFormat": "p95 {{model}}", "refId": "B" }, { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "expr": "histogram_quantile(0.99, rate(foxhunt_hyperopt_trial_duration_seconds_bucket{model=~\"$model\"}[$__rate_interval]))", "legendFormat": "p99 {{model}}", "refId": "C" @@ -1013,25 +2099,50 @@ "id": 44, "type": "stat", "title": "Elapsed Time", - "gridPos": { "h": 6, "w": 4, "x": 16, "y": 31 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 6, + "w": 4, + "x": 16, + "y": 36 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "unit": "s", "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 3600 }, - { "color": "red", "value": 14400 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 3600 + }, + { + "color": "red", + "value": 14400 + } ] }, - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "colorMode": "value", "graphMode": "area", "justifyMode": "center", @@ -1039,7 +2150,10 @@ }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "expr": "foxhunt_hyperopt_elapsed_seconds{model=~\"$model\"}", "legendFormat": "{{model}}", "refId": "A" @@ -1050,22 +2164,41 @@ "id": 45, "type": "stat", "title": "Current Trial / Total", - "gridPos": { "h": 6, "w": 4, "x": 20, "y": 31 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 6, + "w": 4, + "x": 20, + "y": 36 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ - { "color": "blue", "value": null } + { + "color": "blue", + "value": null + } ] }, - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "colorMode": "value", "graphMode": "area", "justifyMode": "center", @@ -1073,13 +2206,19 @@ }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "expr": "foxhunt_hyperopt_trial_current{model=~\"$model\"}", "legendFormat": "trial {{model}}", "refId": "A" }, { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "expr": "foxhunt_hyperopt_trial_total{model=~\"$model\"}", "legendFormat": "total {{model}}", "refId": "B" @@ -1089,7 +2228,12 @@ { "type": "row", "title": "GPU Utilization", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 37 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 42 + }, "collapsed": false, "id": 50, "panels": [] @@ -1098,8 +2242,16 @@ "id": 51, "type": "timeseries", "title": "GPU Utilization %", - "gridPos": { "h": 7, "w": 6, "x": 0, "y": 38 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 43 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "custom": { @@ -1111,28 +2263,52 @@ "showPoints": "never", "axisLabel": "Util %" }, - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "unit": "percent", "min": 0, "max": 100, "thresholds": { "mode": "absolute", "steps": [ - { "color": "red", "value": null }, - { "color": "yellow", "value": 50 }, - { "color": "green", "value": 80 } + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 50 + }, + { + "color": "green", + "value": 80 + } ] } }, "overrides": [] }, "options": { - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] }, - "tooltip": { "mode": "multi", "sort": "desc" } + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "mean", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "expr": "DCGM_FI_DEV_GPU_UTIL", "legendFormat": "GPU {{gpu}}", "refId": "A" @@ -1143,8 +2319,16 @@ "id": 52, "type": "timeseries", "title": "VRAM Used / Free (MiB)", - "gridPos": { "h": 7, "w": 6, "x": 6, "y": 38 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 43 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "custom": { @@ -1155,32 +2339,64 @@ "showPoints": "never", "axisLabel": "MiB" }, - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "unit": "decmbytes" }, "overrides": [ { - "matcher": { "id": "byRegexp", "options": "free.*" }, + "matcher": { + "id": "byRegexp", + "options": "free.*" + }, "properties": [ - { "id": "custom.fillOpacity", "value": 5 }, - { "id": "custom.lineStyle", "value": { "fill": "dash", "dash": [10, 5] } } + { + "id": "custom.fillOpacity", + "value": 5 + }, + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 10, + 5 + ] + } + } ] } ] }, "options": { - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull"] }, - "tooltip": { "mode": "multi", "sort": "desc" } + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "expr": "DCGM_FI_DEV_FB_USED", "legendFormat": "used GPU {{gpu}}", "refId": "A" }, { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "expr": "DCGM_FI_DEV_FB_FREE", "legendFormat": "free GPU {{gpu}}", "refId": "B" @@ -1191,8 +2407,16 @@ "id": 53, "type": "timeseries", "title": "GPU / Memory Temperature", - "gridPos": { "h": 7, "w": 4, "x": 12, "y": 38 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 7, + "w": 4, + "x": 12, + "y": 43 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "custom": { @@ -1203,32 +2427,55 @@ "showPoints": "never", "axisLabel": "Temp" }, - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "unit": "celsius", "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 75 }, - { "color": "red", "value": 85 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 75 + }, + { + "color": "red", + "value": 85 + } ] } }, "overrides": [] }, "options": { - "legend": { "displayMode": "list", "placement": "bottom" }, - "tooltip": { "mode": "multi", "sort": "desc" } + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "expr": "DCGM_FI_DEV_GPU_TEMP", "legendFormat": "GPU temp {{gpu}}", "refId": "A" }, { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "expr": "DCGM_FI_DEV_MEMORY_TEMP", "legendFormat": "mem temp {{gpu}}", "refId": "B" @@ -1239,8 +2486,16 @@ "id": 54, "type": "timeseries", "title": "Power Usage (W)", - "gridPos": { "h": 7, "w": 4, "x": 16, "y": 38 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 7, + "w": 4, + "x": 16, + "y": 43 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "custom": { @@ -1251,18 +2506,29 @@ "showPoints": "never", "axisLabel": "Watts" }, - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "unit": "watt" }, "overrides": [] }, "options": { - "legend": { "displayMode": "list", "placement": "bottom" }, - "tooltip": { "mode": "multi", "sort": "desc" } + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "expr": "DCGM_FI_DEV_POWER_USAGE", "legendFormat": "GPU {{gpu}}", "refId": "A" @@ -1273,8 +2539,16 @@ "id": 55, "type": "timeseries", "title": "Tensor Core & DRAM Activity", - "gridPos": { "h": 7, "w": 4, "x": 20, "y": 38 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 7, + "w": 4, + "x": 20, + "y": 43 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "custom": { @@ -1285,7 +2559,9 @@ "showPoints": "never", "axisLabel": "Activity" }, - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "unit": "percentunit", "min": 0, "max": 1 @@ -1293,24 +2569,39 @@ "overrides": [] }, "options": { - "legend": { "displayMode": "list", "placement": "bottom" }, - "tooltip": { "mode": "multi", "sort": "desc" } + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "expr": "DCGM_FI_PROF_PIPE_TENSOR_ACTIVE", "legendFormat": "tensor {{gpu}}", "refId": "A" }, { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "expr": "DCGM_FI_PROF_DRAM_ACTIVE", "legendFormat": "DRAM {{gpu}}", "refId": "B" }, { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "expr": "DCGM_FI_PROF_GR_ENGINE_ACTIVE", "legendFormat": "engine {{gpu}}", "refId": "C" @@ -1320,7 +2611,12 @@ { "type": "row", "title": "Argo CI/CD", - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 45 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 50 + }, "collapsed": false, "id": 60, "panels": [] @@ -1329,39 +2625,102 @@ "id": 61, "type": "stat", "title": "Workflows by Status", - "gridPos": { "h": 5, "w": 6, "x": 0, "y": 46 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 51 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ - { "color": "blue", "value": null } + { + "color": "blue", + "value": null + } ] }, - "color": { "mode": "thresholds" } + "color": { + "mode": "thresholds" + } }, "overrides": [ { - "matcher": { "id": "byRegexp", "options": "Running" }, - "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] + "matcher": { + "id": "byRegexp", + "options": "Running" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] }, { - "matcher": { "id": "byRegexp", "options": "Pending" }, - "properties": [ { "id": "color", "value": { "fixedColor": "yellow", "mode": "fixed" } } ] + "matcher": { + "id": "byRegexp", + "options": "Pending" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] }, { - "matcher": { "id": "byRegexp", "options": "Failed|Error" }, - "properties": [ { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } ] + "matcher": { + "id": "byRegexp", + "options": "Failed|Error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] }, { - "matcher": { "id": "byRegexp", "options": "Succeeded" }, - "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] + "matcher": { + "id": "byRegexp", + "options": "Succeeded" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] } ] }, "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "colorMode": "background", "graphMode": "none", "justifyMode": "center", @@ -1370,31 +2729,46 @@ }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "expr": "argo_workflows_gauge{status=\"Running\"}", "legendFormat": "Running", "refId": "A" }, { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "expr": "argo_workflows_gauge{status=\"Pending\"}", "legendFormat": "Pending", "refId": "B" }, { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "expr": "argo_workflows_gauge{status=\"Failed\"}", "legendFormat": "Failed", "refId": "C" }, { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "expr": "argo_workflows_gauge{status=\"Succeeded\"}", "legendFormat": "Succeeded", "refId": "D" }, { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "expr": "argo_workflows_gauge{status=\"Error\"}", "legendFormat": "Error", "refId": "E" @@ -1405,8 +2779,16 @@ "id": 62, "type": "timeseries", "title": "Workflow Pods", - "gridPos": { "h": 5, "w": 4, "x": 6, "y": 46 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 5, + "w": 4, + "x": 6, + "y": 51 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "custom": { @@ -1417,18 +2799,29 @@ "showPoints": "never", "axisLabel": "Pods" }, - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "unit": "short" }, "overrides": [] }, "options": { - "legend": { "displayMode": "list", "placement": "bottom" }, - "tooltip": { "mode": "multi", "sort": "desc" } + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "expr": "argo_workflows_pods_gauge", "legendFormat": "pods", "refId": "A" @@ -1439,8 +2832,16 @@ "id": 63, "type": "timeseries", "title": "Error Rate", - "gridPos": { "h": 5, "w": 4, "x": 10, "y": 46 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 5, + "w": 4, + "x": 10, + "y": 51 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "custom": { @@ -1450,18 +2851,30 @@ "showPoints": "never", "axisLabel": "Errors/s" }, - "color": { "fixedColor": "red", "mode": "fixed" }, + "color": { + "fixedColor": "red", + "mode": "fixed" + }, "unit": "ops" }, "overrides": [] }, "options": { - "legend": { "displayMode": "list", "placement": "bottom" }, - "tooltip": { "mode": "multi", "sort": "desc" } + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "expr": "rate(argo_workflows_error_count[$__rate_interval])", "legendFormat": "errors/s", "refId": "A" @@ -1472,8 +2885,16 @@ "id": 64, "type": "timeseries", "title": "Operation Duration (p50/p95)", - "gridPos": { "h": 5, "w": 5, "x": 14, "y": 46 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 5, + "w": 5, + "x": 14, + "y": 51 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "custom": { @@ -1484,24 +2905,38 @@ "showPoints": "never", "axisLabel": "Duration" }, - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "unit": "s" }, "overrides": [] }, "options": { - "legend": { "displayMode": "list", "placement": "bottom" }, - "tooltip": { "mode": "multi", "sort": "desc" } + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "expr": "histogram_quantile(0.50, rate(argo_workflows_operation_duration_seconds_bucket[$__rate_interval]))", "legendFormat": "p50", "refId": "A" }, { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "expr": "histogram_quantile(0.95, rate(argo_workflows_operation_duration_seconds_bucket[$__rate_interval]))", "legendFormat": "p95", "refId": "B" @@ -1512,8 +2947,16 @@ "id": 65, "type": "timeseries", "title": "Template Trigger Rate", - "gridPos": { "h": 5, "w": 5, "x": 19, "y": 46 }, - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "gridPos": { + "h": 5, + "w": 5, + "x": 19, + "y": 51 + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "fieldConfig": { "defaults": { "custom": { @@ -1521,21 +2964,34 @@ "lineWidth": 1, "fillOpacity": 40, "showPoints": "never", - "stacking": { "mode": "normal" }, + "stacking": { + "mode": "normal" + }, "axisLabel": "Triggers/s" }, - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "unit": "ops" }, "overrides": [] }, "options": { - "legend": { "displayMode": "list", "placement": "bottom" }, - "tooltip": { "mode": "multi", "sort": "desc" } + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } }, "targets": [ { - "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, "expr": "rate(argo_workflows_workflowtemplate_triggered_total[$__rate_interval])", "legendFormat": "triggers/s", "refId": "A"