{ "description": "ML model training, hyperopt, GPU utilization, Argo CI pipelines, and full metrics coverage", "editable": true, "graphTooltip": 2, "panels": [ { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 1, "panels": [], "title": "Training Status", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "green", "value": 1 } ] } }, "overrides": [] }, "gridPos": { "h": 5, "w": 3, "x": 0, "y": 1 }, "id": 2, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "center", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "value" }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum(foxhunt_training_active_workers{job=\"training-pods\"}) or vector(0)", "legendFormat": "Workers", "refId": "A" } ], "title": "Active Workers", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Current epoch and step within epoch", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "decimals": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "blue", "value": null } ] } }, "overrides": [] }, "gridPos": { "h": 5, "w": 4, "x": 7, "y": 1 }, "id": 3, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "center", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "value" }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_current_epoch{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "legendFormat": "{{model}} / fold {{fold}}", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_step{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "legendFormat": "step {{model}}/{{fold}}", "refId": "B" } ], "title": "Current Epoch", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "decimals": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "purple", "value": null } ] } }, "overrides": [] }, "gridPos": { "h": 5, "w": 4, "x": 3, "y": 1 }, "id": 101, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "center", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "value" }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_hyperopt_trial_current{model=~\"$model\", job=~\"$source\"}", "legendFormat": "{{model}}", "refId": "A" } ], "title": "Current Trial", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Rate of improvement in best hyperopt objective (negative = improving for loss minimization)", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "decimals": 4, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 0 } ] } }, "overrides": [] }, "gridPos": { "h": 5, "w": 3, "x": 11, "y": 1 }, "id": 4, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "center", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "value" }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "deriv(foxhunt_hyperopt_best_objective{model=~\"$model\", job=~\"$source\"}[10m])", "legendFormat": "\u0394 objective", "refId": "A" } ], "title": "Best Objective \u0394", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "yellow", "value": 0.3 }, { "color": "green", "value": 0.7 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 5, "w": 3, "x": 14, "y": 1 }, "id": 5, "options": { "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_hyperopt_trial_current{model=~\"$model\", job=~\"$source\"} / foxhunt_hyperopt_trial_total{model=~\"$model\", job=~\"$source\"}", "legendFormat": "{{model}}", "refId": "A" } ], "title": "Hyperopt Trial Progress", "type": "gauge" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "Errors" }, "properties": [ { "id": "thresholds", "value": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] } } ] } ] }, "gridPos": { "h": 5, "w": 3, "x": 17, "y": 1 }, "id": 6, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "center", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "value" }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "ml_training_jobs_started_total", "legendFormat": "Jobs", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "ml_training_errors_total", "legendFormat": "Errors", "refId": "B" } ], "title": "ML Jobs / Errors", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "blue", "value": null } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 5, "w": 4, "x": 20, "y": 1 }, "id": 99, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "center", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "value" }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_hyperopt_elapsed_seconds{model=~\"$model\", job=~\"$source\"}", "legendFormat": "Elapsed", "refId": "A" } ], "title": "Hyperopt Elapsed", "type": "stat" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 6 }, "id": 66, "panels": [], "title": "Run Summary", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 15, "gradientMode": "opacity", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "showPoints": "auto", "spanNulls": true, "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 4, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 0.1 }, { "color": "red", "value": 0.5 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 6, "w": 6, "x": 0, "y": 6 }, "id": 67, "options": { "legend": { "calcs": [ "lastNotNull", "max", "min" ], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "clamp_max(min_over_time(foxhunt_training_validation_loss{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}[$__range]), 5)", "instant": false, "legendFormat": "{{model}}/{{fold}}", "refId": "A" } ], "title": "Best Val Loss", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 15, "gradientMode": "opacity", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "showPoints": "auto", "spanNulls": true, "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 2, "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "yellow", "value": 1 }, { "color": "green", "value": 3 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 6, "w": 6, "x": 6, "y": 6 }, "id": 68, "options": { "legend": { "calcs": [ "lastNotNull", "max", "min" ], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "clamp_max(max_over_time(foxhunt_training_epoch_sharpe{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}[$__range]), 5)", "instant": false, "legendFormat": "{{model}}/{{fold}}", "refId": "A" } ], "title": "Best Sharpe", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 15, "gradientMode": "opacity", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "showPoints": "auto", "spanNulls": true, "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 1, "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "yellow", "value": 0.4 }, { "color": "green", "value": 0.55 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 6, "w": 6, "x": 12, "y": 6 }, "id": 69, "options": { "legend": { "calcs": [ "lastNotNull", "max", "min" ], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "max_over_time(foxhunt_training_epoch_win_rate{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}[$__range])", "instant": false, "legendFormat": "{{model}}/{{fold}}", "refId": "A" } ], "title": "Best Win Rate", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "Objective", "drawStyle": "line", "fillOpacity": 10, "gradientMode": "opacity", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 4, "showPoints": "auto" } }, "overrides": [] }, "gridPos": { "h": 6, "w": 6, "x": 18, "y": 6 }, "id": 42, "options": { "legend": { "calcs": [ "lastNotNull", "min" ], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_hyperopt_best_objective{model=~\"$model\", job=~\"$source\"}", "legendFormat": "best {{model}}", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_hyperopt_trial_best_loss{model=~\"$model\", job=~\"$source\"}", "legendFormat": "trial loss {{model}}", "refId": "B" } ], "title": "Best Objective Over Time", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 15, "gradientMode": "opacity", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "showPoints": "auto", "spanNulls": true, "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 1, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 0.1 }, { "color": "red", "value": 0.25 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 6, "w": 6, "x": 0, "y": 12 }, "id": 70, "options": { "legend": { "calcs": [ "lastNotNull", "max", "min" ], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "min_over_time(foxhunt_training_epoch_max_drawdown{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}[$__range])", "instant": false, "legendFormat": "{{model}}/{{fold}}", "refId": "A" } ], "title": "Min Max Drawdown", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 15, "gradientMode": "opacity", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "showPoints": "auto", "spanNulls": true, "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "blue", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 6, "w": 6, "x": 6, "y": 12 }, "id": 71, "options": { "legend": { "calcs": [ "lastNotNull", "max", "min" ], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_current_epoch{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "instant": false, "legendFormat": "{{model}}/{{fold}}", "refId": "A" } ], "title": "Total Epochs", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 15, "gradientMode": "opacity", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "showPoints": "auto", "spanNulls": true, "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 2, "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "yellow", "value": 1 }, { "color": "green", "value": 1.5 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 6, "w": 6, "x": 12, "y": 12 }, "id": 72, "options": { "legend": { "calcs": [ "lastNotNull", "max", "min" ], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "clamp_max(max_over_time(foxhunt_training_epoch_profit_factor{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}[$__range]), 20)", "instant": false, "legendFormat": "{{model}}/{{fold}}", "refId": "A" } ], "title": "Best Profit Factor", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "Duration", "drawStyle": "line", "fillOpacity": 10, "gradientMode": "opacity", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 4, "showPoints": "never" }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 6, "w": 6, "x": 18, "y": 12 }, "id": 43, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.50, rate(foxhunt_hyperopt_trial_duration_seconds_bucket{model=~\"$model\", job=~\"$source\"}[$__rate_interval]))", "legendFormat": "p50 {{model}}", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.95, rate(foxhunt_hyperopt_trial_duration_seconds_bucket{model=~\"$model\", job=~\"$source\"}[$__rate_interval]))", "legendFormat": "p95 {{model}}", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.99, rate(foxhunt_hyperopt_trial_duration_seconds_bucket{model=~\"$model\", job=~\"$source\"}[$__rate_interval]))", "legendFormat": "p99 {{model}}", "refId": "C" } ], "title": "Trial Duration (p50/p95/p99)", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 18 }, "id": 10, "panels": [], "title": "Training Curves", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisLabel": "Loss", "drawStyle": "line", "fillOpacity": 10, "gradientMode": "opacity", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "showPoints": "never" } }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 19 }, "id": 11, "options": { "legend": { "calcs": [ "lastNotNull", "min" ], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_epoch_loss{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "legendFormat": "train {{model}}/{{fold}}", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_validation_loss{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "legendFormat": "val {{model}}/{{fold}}", "refId": "B" } ], "title": "Training & Validation Loss", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "LR", "drawStyle": "line", "fillOpacity": 15, "gradientMode": "opacity", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "scaleDistribution": { "log": 10, "type": "log" }, "showPoints": "never" }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 19 }, "id": 12, "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_learning_rate{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "legendFormat": "{{model}}/{{fold}}", "refId": "A" } ], "title": "Learning Rate", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "Norm", "drawStyle": "line", "fillOpacity": 10, "gradientMode": "opacity", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 4, "showPoints": "never" } }, "overrides": [] }, "gridPos": { "h": 8, "w": 4, "x": 16, "y": 19 }, "id": 13, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_gradient_norm{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "legendFormat": "{{model}}/{{fold}}", "refId": "A" } ], "title": "Gradient Norm", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "fillOpacity": 10, "gradientMode": "opacity", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 4, "showPoints": "never" } }, "overrides": [ { "matcher": { "id": "byFrameRefID", "options": "B" }, "properties": [ { "id": "custom.axisPlacement", "value": "right" }, { "id": "unit", "value": "s" } ] } ] }, "gridPos": { "h": 8, "w": 4, "x": 20, "y": 19 }, "id": 14, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_batches_per_second{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "legendFormat": "batch/s {{model}}", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_epoch_duration_seconds{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "legendFormat": "epoch dur {{model}}", "refId": "B" } ], "title": "Batches/sec & Epoch Duration", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 27 }, "id": 20, "panels": [], "title": "Training Quality", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "Sharpe", "drawStyle": "line", "fillOpacity": 10, "gradientMode": "opacity", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 4, "showPoints": "auto" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "yellow", "value": 1 }, { "color": "green", "value": 2 } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 0, "y": 28 }, "id": 21, "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "clamp_max(foxhunt_training_epoch_sharpe{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}, 5)", "legendFormat": "{{model}}/{{fold}}", "refId": "A" } ], "title": "Epoch Sharpe Ratio", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "Sortino", "drawStyle": "line", "fillOpacity": 10, "gradientMode": "opacity", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 4, "showPoints": "auto" } }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 6, "y": 28 }, "id": 22, "options": { "legend": { "calcs": [ "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "clamp_max(foxhunt_training_epoch_sortino{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}, 10)", "legendFormat": "{{model}}/{{fold}}", "refId": "A" } ], "title": "Epoch Sortino Ratio", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "Win Rate", "drawStyle": "line", "fillOpacity": 10, "gradientMode": "opacity", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 4, "showPoints": "auto" }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 7, "w": 4, "x": 12, "y": 28 }, "id": 23, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_epoch_win_rate{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "legendFormat": "{{model}}/{{fold}}", "refId": "A" } ], "title": "Win Rate", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "Drawdown", "drawStyle": "line", "fillOpacity": 10, "gradientMode": "opacity", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 4, "showPoints": "auto" }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 7, "w": 4, "x": 16, "y": 28 }, "id": 24, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_epoch_max_drawdown{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "legendFormat": "{{model}}/{{fold}}", "refId": "A" } ], "title": "Max Drawdown", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "fillOpacity": 15, "gradientMode": "opacity", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 4, "showPoints": "auto" } }, "overrides": [ { "matcher": { "id": "byFrameRefID", "options": "B" }, "properties": [ { "id": "custom.axisPlacement", "value": "right" }, { "id": "unit", "value": "short" }, { "id": "custom.drawStyle", "value": "bars" }, { "id": "custom.fillOpacity", "value": 30 } ] }, { "matcher": { "id": "byName", "options": "foxhunt_training_epoch_profit_factor" }, "properties": [ { "id": "max", "value": 20 } ] } ] }, "gridPos": { "h": 7, "w": 4, "x": 20, "y": 28 }, "id": 25, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "clamp_max(foxhunt_training_epoch_profit_factor{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}, 20)", "legendFormat": "PF {{model}}", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_epoch_total_trades{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "legendFormat": "trades {{model}}", "refId": "B" } ], "title": "Profit Factor & Total Trades", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 35 }, "id": 30, "panels": [], "title": "RL Diagnostics", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "Q-Value", "drawStyle": "line", "fillOpacity": 10, "gradientMode": "opacity", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 4, "showPoints": "never" } }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 0, "y": 36 }, "id": 31, "options": { "legend": { "calcs": [ "lastNotNull", "max", "min" ], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_q_value_mean{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "legendFormat": "mean {{model}}/{{fold}}", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_q_value_max{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "legendFormat": "max {{model}}/{{fold}}", "refId": "B" } ], "title": "Q-Value (Mean & Max)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "continuous-blues" }, "custom": { "axisLabel": "Transitions", "drawStyle": "line", "fillOpacity": 30, "gradientMode": "scheme", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 4, "showPoints": "never" }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 7, "w": 4, "x": 6, "y": 36 }, "id": 32, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_replay_buffer_size{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "legendFormat": "{{model}}/{{fold}}", "refId": "A" } ], "title": "Replay Buffer Size", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Normalized entropy of policy actions. 0 = collapsed to single action, 1 = uniform distribution.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "Entropy", "drawStyle": "line", "fillOpacity": 15, "gradientMode": "opacity", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 4, "showPoints": "never" }, "custom.thresholdsStyle": { "mode": "line" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "yellow", "value": 0.1 }, { "color": "green", "value": 0.3 } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 4, "x": 10, "y": 36 }, "id": 33, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_action_entropy{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "legendFormat": "{{model}}/{{fold}}", "refId": "A" } ], "title": "Action Entropy", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Fraction of 45 factored actions actually used by the policy.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "Diversity", "drawStyle": "line", "fillOpacity": 15, "gradientMode": "opacity", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 4, "showPoints": "never" }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 7, "w": 4, "x": 14, "y": 36 }, "id": 34, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_action_diversity{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "legendFormat": "{{model}}/{{fold}}", "refId": "A" } ], "title": "Action Diversity", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "Pct", "drawStyle": "line", "fillOpacity": 50, "gradientMode": "opacity", "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 4, "showPoints": "never", "stacking": { "group": "A", "mode": "normal" } }, "unit": "percentunit" }, "overrides": [ { "matcher": { "id": "byRegexp", "options": "buy.*" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": "sell.*" }, "properties": [ { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": "hold.*" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 7, "w": 6, "x": 18, "y": 36 }, "id": 35, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_epoch_action_buy_pct{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "legendFormat": "buy {{model}}", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_epoch_action_sell_pct{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "legendFormat": "sell {{model}}", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_epoch_action_hold_pct{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "legendFormat": "hold {{model}}", "refId": "C" } ], "title": "Action Distribution (Buy/Sell/Hold %)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Q-max / |Q-mean| \u2014 spikes above 10 indicate DQN overestimation", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 15, "gradientMode": "opacity", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "showPoints": "auto", "spanNulls": true, "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 1, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 5 }, { "color": "red", "value": 10 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 43 }, "id": 102, "options": { "legend": { "calcs": [ "lastNotNull", "max", "min" ], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_q_overestimation_ratio{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "legendFormat": "{{model}}/{{fold}}", "refId": "A" } ], "title": "Q-Overestimation Ratio", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Policy entropy (exploration health) and KL divergence (policy drift)", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 15, "gradientMode": "opacity", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "showPoints": "auto", "spanNulls": true, "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 4, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 43 }, "id": 103, "options": { "legend": { "calcs": [ "lastNotNull", "max", "min" ], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_policy_entropy{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "legendFormat": "entropy {{model}}/{{fold}}", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_kl_divergence{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "legendFormat": "KL {{model}}/{{fold}}", "refId": "B" } ], "title": "Policy Entropy & KL Divergence (PPO)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Mean advantage estimate \u2014 should hover near zero", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 15, "gradientMode": "opacity", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "showPoints": "auto", "spanNulls": true, "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 4, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 43 }, "id": 104, "options": { "legend": { "calcs": [ "lastNotNull", "max", "min" ], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_advantage_mean{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "legendFormat": "{{model}}/{{fold}}", "refId": "A" } ], "title": "Advantage Mean (PPO)", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 51 }, "id": 105, "panels": [], "title": "Training Health", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "NaN loss events, gradient explosions, and feature extraction failures", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 15, "gradientMode": "opacity", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "showPoints": "auto", "spanNulls": true, "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 52 }, "id": 106, "options": { "legend": { "calcs": [ "lastNotNull", "max", "min" ], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "increase(foxhunt_training_nan_detected_total{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}[$__rate_interval])", "legendFormat": "NaN {{model}}/{{fold}}", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "increase(foxhunt_training_gradient_explosion_total{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}[$__rate_interval])", "legendFormat": "grad explosion {{model}}/{{fold}}", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "increase(foxhunt_training_feature_errors_total{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}[$__rate_interval])", "legendFormat": "feature err {{model}}/{{fold}}", "refId": "C" } ], "title": "Training Anomalies", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Checkpoint save/failure rate", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 15, "gradientMode": "opacity", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "showPoints": "auto", "spanNulls": true, "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 52 }, "id": 107, "options": { "legend": { "calcs": [ "lastNotNull", "max", "min" ], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "increase(foxhunt_training_checkpoint_saves_total{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}[$__rate_interval])", "legendFormat": "saves {{model}}/{{fold}}", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "increase(foxhunt_training_checkpoint_failures_total{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}[$__rate_interval])", "legendFormat": "failures {{model}}/{{fold}}", "refId": "B" } ], "title": "Checkpoint Saves & Failures", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "p95 checkpoint save and data loading latency", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 15, "gradientMode": "opacity", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "showPoints": "auto", "spanNulls": true, "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 1, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 52 }, "id": 108, "options": { "legend": { "calcs": [ "lastNotNull", "max", "min" ], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.95, rate(foxhunt_training_checkpoint_duration_seconds_bucket{model=~\"$model\", job=~\"$source\"}[$__rate_interval]))", "legendFormat": "ckpt p95 {{model}}", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.95, rate(foxhunt_training_data_load_seconds_bucket{model=~\"$model\", job=~\"$source\"}[$__rate_interval]))", "legendFormat": "data load p95 {{model}}", "refId": "B" } ], "title": "Checkpoint & Data Load Latency", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Size of last saved checkpoint", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 15, "gradientMode": "opacity", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "showPoints": "auto", "spanNulls": true, "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 60 }, "id": 109, "options": { "legend": { "calcs": [ "lastNotNull", "max", "min" ], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_checkpoint_size_bytes{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "legendFormat": "{{model}}/{{fold}}", "refId": "A" } ], "title": "Checkpoint Size", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Total batches processed this run", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 15, "gradientMode": "opacity", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "showPoints": "auto", "spanNulls": true, "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 60 }, "id": 110, "options": { "legend": { "calcs": [ "lastNotNull", "max", "min" ], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_batches_processed{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "legendFormat": "{{model}}/{{fold}}", "refId": "A" } ], "title": "Batches Processed", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Last epoch wall time", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 15, "gradientMode": "opacity", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "showPoints": "auto", "spanNulls": true, "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 1, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 60 }, "id": 111, "options": { "legend": { "calcs": [ "lastNotNull", "max", "min" ], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_iteration_seconds{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "legendFormat": "{{model}}/{{fold}}", "refId": "A" } ], "title": "Iteration Time", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 68 }, "id": 112, "panels": [], "title": "Epoch Returns", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Epoch total return (fractional)", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 15, "gradientMode": "opacity", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "showPoints": "auto", "spanNulls": true, "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 2, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 69 }, "id": 113, "options": { "legend": { "calcs": [ "lastNotNull", "max", "min" ], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_epoch_total_return{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "legendFormat": "{{model}}/{{fold}}", "refId": "A" } ], "title": "Total Return per Epoch", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Average return per trade", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 15, "gradientMode": "opacity", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "showPoints": "auto", "spanNulls": true, "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 4, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 69 }, "id": 114, "options": { "legend": { "calcs": [ "lastNotNull", "max", "min" ], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_epoch_avg_return{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "legendFormat": "{{model}}/{{fold}}", "refId": "A" } ], "title": "Avg Return per Trade", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 77 }, "id": 115, "panels": [], "title": "Supervised Model Eval", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Supervised eval accuracy and F1 score", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 15, "gradientMode": "opacity", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "showPoints": "auto", "spanNulls": true, "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 2, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 78 }, "id": 116, "options": { "legend": { "calcs": [ "lastNotNull", "max", "min" ], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_eval_accuracy{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "legendFormat": "accuracy {{model}}/{{fold}}", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_eval_f1{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "legendFormat": "F1 {{model}}/{{fold}}", "refId": "B" } ], "title": "Accuracy & F1", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Supervised eval precision and recall", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 15, "gradientMode": "opacity", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "showPoints": "auto", "spanNulls": true, "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 2, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 78 }, "id": 117, "options": { "legend": { "calcs": [ "lastNotNull", "max", "min" ], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_eval_precision{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "legendFormat": "precision {{model}}/{{fold}}", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_training_eval_recall{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}", "legendFormat": "recall {{model}}/{{fold}}", "refId": "B" } ], "title": "Precision & Recall", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 86 }, "id": 118, "panels": [], "title": "Hyperopt Details", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "1 = hyperopt active, 0 = training only", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "decimals": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "blue", "value": null }, { "color": "green", "value": 1 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 5, "w": 4, "x": 0, "y": 87 }, "id": 119, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "center", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "value" }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_hyperopt_mode{model=~\"$model\", job=~\"$source\"}", "legendFormat": "{{model}}", "refId": "A" } ], "title": "Hyperopt Mode", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Current epoch within the active trial", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 15, "gradientMode": "opacity", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "showPoints": "auto", "spanNulls": true, "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 10, "x": 4, "y": 87 }, "id": 120, "options": { "legend": { "calcs": [ "lastNotNull", "max", "min" ], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_hyperopt_trial_epoch{model=~\"$model\", job=~\"$source\"}", "legendFormat": "{{model}}", "refId": "A" } ], "title": "Hyperopt Trial Epoch", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Cumulative failed trial count", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "drawStyle": "line", "fillOpacity": 15, "gradientMode": "opacity", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "showPoints": "auto", "spanNulls": true, "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 3 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 10, "x": 14, "y": 87 }, "id": 121, "options": { "legend": { "calcs": [ "lastNotNull", "max", "min" ], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "foxhunt_hyperopt_trials_failed_total{model=~\"$model\", job=~\"$source\"}", "legendFormat": "{{model}}", "refId": "A" } ], "title": "Hyperopt Failed Trials", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 95 }, "id": 50, "panels": [], "title": "GPU Utilization", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "continuous-GrYlRd" }, "custom": { "axisLabel": "Util %", "drawStyle": "line", "fillOpacity": 20, "gradientMode": "opacity", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 4, "showPoints": "never" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "yellow", "value": 50 }, { "color": "green", "value": 80 } ] }, "unit": "percent" }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 0, "y": 96 }, "id": 51, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "DCGM_FI_DEV_GPU_UTIL", "legendFormat": "GPU {{gpu}}", "refId": "A" } ], "title": "GPU Utilization %", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "MiB", "drawStyle": "line", "fillOpacity": 15, "gradientMode": "opacity", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 4, "showPoints": "never" }, "unit": "decmbytes" }, "overrides": [ { "matcher": { "id": "byRegexp", "options": "free.*" }, "properties": [ { "id": "custom.fillOpacity", "value": 5 }, { "id": "custom.lineStyle", "value": { "dash": [ 10, 5 ], "fill": "dash" } } ] } ] }, "gridPos": { "h": 7, "w": 6, "x": 6, "y": 96 }, "id": 52, "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "DCGM_FI_DEV_FB_USED", "legendFormat": "used GPU {{gpu}}", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "DCGM_FI_DEV_FB_FREE", "legendFormat": "free GPU {{gpu}}", "refId": "B" } ], "title": "VRAM Used / Free (MiB)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "Temp", "drawStyle": "line", "fillOpacity": 10, "gradientMode": "opacity", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 4, "showPoints": "never" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 75 }, { "color": "red", "value": 85 } ] }, "unit": "celsius" }, "overrides": [] }, "gridPos": { "h": 7, "w": 4, "x": 12, "y": 96 }, "id": 53, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "DCGM_FI_DEV_GPU_TEMP", "legendFormat": "GPU temp {{gpu}}", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "DCGM_FI_DEV_MEMORY_TEMP", "legendFormat": "mem temp {{gpu}}", "refId": "B" } ], "title": "GPU / Memory Temperature", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "Watts", "drawStyle": "line", "fillOpacity": 15, "gradientMode": "opacity", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 4, "showPoints": "never" }, "unit": "watt" }, "overrides": [] }, "gridPos": { "h": 7, "w": 4, "x": 16, "y": 96 }, "id": 54, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "DCGM_FI_DEV_POWER_USAGE", "legendFormat": "GPU {{gpu}}", "refId": "A" } ], "title": "Power Usage (W)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "Activity", "drawStyle": "line", "fillOpacity": 15, "gradientMode": "opacity", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 4, "showPoints": "never" }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 7, "w": 4, "x": 20, "y": 96 }, "id": 55, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "DCGM_FI_PROF_PIPE_TENSOR_ACTIVE", "legendFormat": "tensor {{gpu}}", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "DCGM_FI_PROF_DRAM_ACTIVE", "legendFormat": "DRAM {{gpu}}", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "DCGM_FI_PROF_GR_ENGINE_ACTIVE", "legendFormat": "engine {{gpu}}", "refId": "C" } ], "title": "Tensor Core & DRAM Activity", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 103 }, "id": 60, "panels": [], "title": "Argo CI/CD", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "blue", "value": null } ] } }, "overrides": [ { "matcher": { "id": "byRegexp", "options": "Running" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": "Pending" }, "properties": [ { "id": "color", "value": { "fixedColor": "yellow", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": "Failed|Error" }, "properties": [ { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } ] }, { "matcher": { "id": "byRegexp", "options": "Succeeded" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 5, "w": 6, "x": 0, "y": 104 }, "id": 61, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "center", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "argo_workflows_gauge{status=\"Running\"}", "legendFormat": "Running", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "argo_workflows_gauge{status=\"Pending\"}", "legendFormat": "Pending", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "argo_workflows_gauge{status=\"Failed\"}", "legendFormat": "Failed", "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "argo_workflows_gauge{status=\"Succeeded\"}", "legendFormat": "Succeeded", "refId": "D" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "argo_workflows_gauge{status=\"Error\"}", "legendFormat": "Error", "refId": "E" } ], "title": "Workflows by Status", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "Pods", "drawStyle": "line", "fillOpacity": 20, "gradientMode": "opacity", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 4, "showPoints": "never" }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 5, "w": 4, "x": 6, "y": 104 }, "id": 62, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "argo_workflows_pods_gauge", "legendFormat": "pods", "refId": "A" } ], "title": "Workflow Pods", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "fixedColor": "red", "mode": "fixed" }, "custom": { "axisLabel": "Errors/s", "drawStyle": "bars", "fillOpacity": 50, "gradientMode": "opacity", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 4, "showPoints": "never" }, "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 5, "w": 4, "x": 10, "y": 104 }, "id": 63, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(argo_workflows_error_count[$__rate_interval])", "legendFormat": "errors/s", "refId": "A" } ], "title": "Error Rate", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "Duration", "drawStyle": "line", "fillOpacity": 10, "gradientMode": "opacity", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 4, "showPoints": "never" }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 5, "w": 5, "x": 14, "y": 104 }, "id": 64, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.50, rate(argo_workflows_operation_duration_seconds_bucket[$__rate_interval]))", "legendFormat": "p50", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.95, rate(argo_workflows_operation_duration_seconds_bucket[$__rate_interval]))", "legendFormat": "p95", "refId": "B" } ], "title": "Operation Duration (p50/p95)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "Triggers/s", "drawStyle": "bars", "fillOpacity": 40, "gradientMode": "opacity", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 4, "showPoints": "never", "stacking": { "mode": "normal" } }, "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 5, "w": 5, "x": 19, "y": 104 }, "id": 65, "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "rate(argo_workflows_workflowtemplate_triggered_total[$__rate_interval])", "legendFormat": "triggers/s", "refId": "A" } ], "title": "Template Trigger Rate", "type": "timeseries" } ], "refresh": "15s", "schemaVersion": 39, "style": "dark", "tags": [ "foxhunt", "training", "gpu", "ci" ], "templating": { "list": [ { "auto": true, "auto_count": 30, "auto_min": "1m", "current": { "selected": true, "text": "auto", "value": "$__auto_interval_interval" }, "label": "Interval", "name": "interval", "options": [ { "selected": false, "text": "1m", "value": "1m" }, { "selected": false, "text": "5m", "value": "5m" }, { "selected": false, "text": "15m", "value": "15m" }, { "selected": false, "text": "30m", "value": "30m" }, { "selected": false, "text": "1h", "value": "1h" } ], "query": "1m,5m,15m,30m,1h", "type": "interval" }, { "current": { "selected": true, "text": "Live", "value": "training-pods" }, "description": "Live = running training pods, CI History = completed CI runs (pushgateway), All = both", "label": "Source", "name": "source", "options": [ { "selected": true, "text": "Live", "value": "training-pods" }, { "selected": false, "text": "CI History", "value": ".+_baseline.*" }, { "selected": false, "text": "All", "value": "training-pods|.+_baseline.*" } ], "query": "Live : training-pods, CI History : .+_baseline.*, All : training-pods|.+_baseline.*", "type": "custom" }, { "allValue": ".*", "current": { "selected": true, "text": "All", "value": "$__all" }, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "includeAll": true, "label": "Model", "multi": true, "name": "model", "query": "label_values(foxhunt_training_step{job=~\"$source\"}, model)", "refresh": 2, "sort": 1, "type": "query" }, { "allValue": ".*", "current": { "selected": true, "text": "All", "value": "$__all" }, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "includeAll": true, "label": "Fold", "multi": true, "name": "fold", "query": "label_values(foxhunt_training_step{job=~\"$source\"}, fold)", "refresh": 2, "sort": 1, "type": "query" } ] }, "time": { "from": "now-6h", "to": "now" }, "timepicker": {}, "timezone": "browser", "title": "Training & CI/CD", "uid": "foxhunt-training", "__inputs": [ { "name": "DS_PROMETHEUS", "type": "datasource", "pluginId": "prometheus" }, { "name": "DS_LOKI", "type": "datasource", "pluginId": "loki" }, { "name": "DS_TEMPO", "type": "datasource", "pluginId": "${DS_TEMPO}" } ] }