gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
ffc83da6439ab76da19f2769a8ad9d34ee5f66a7
foxhunt/infra/k8s/monitoring/dashboards/foxhunt-training.json
jgrusewski ddf697641a feat(monitoring): add per-service and platform dashboards, reorganize folder structure 
- 7 application service dashboards (Services folder): API Gateway, Trading
  Service, Backtesting, Data Acquisition, ML Training, Trading Agent, Web Gateway
- 9 platform dashboards (Platform folder): Prometheus, Grafana, Loki & Promtail,
  Tempo, Redis, PostgreSQL, Argo Workflows, QuestDB, Cluster & Costs
- Cluster & Costs dashboard includes Scaleway node cost estimates (€/hr, €/month)
  for platform, GPU (H100), and CI compile nodes
- Reorganized existing dashboards into 7 folders: CI, Infrastructure, Operations,
  Platform, Services, Trading, Training (dropped "Foxhunt" prefix from titles)
- Rewrote import.sh: removed ConfigMap deployment, switched to API-only imports
  with automatic folder creation and per-dashboard folder mapping
- All 24 dashboards deployed via Grafana API (Postgres-backed)

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-14 02:30:52 +01:00

4463 lines
105 KiB
JSON
Raw Blame History

{
"description": "ML model training, hyperopt, GPU utilization, Argo CI pipelines, and full metrics coverage",
"editable": true,
"graphTooltip": 2,
"panels": [
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 0
},
"id": 1,
"panels": [],
"title": "Training Status",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "green",
"value": 1
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 3,
"x": 0,
"y": 1
},
"id": 2,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "center",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "value"
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "sum(foxhunt_training_active_workers{job=\"training-pods\"}) or vector(0)",
"legendFormat": "Workers",
"refId": "A"
}
],
"title": "Active Workers",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "Current epoch and step within epoch",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"decimals": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "blue",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 4,
"x": 7,
"y": 1
},
"id": 3,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "center",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "value"
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "foxhunt_training_current_epoch{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
"legendFormat": "{{model}} / fold {{fold}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "foxhunt_training_step{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
"legendFormat": "step {{model}}/{{fold}}",
"refId": "B"
}
],
"title": "Current Epoch",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"decimals": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "purple",
"value": null
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 4,
"x": 3,
"y": 1
},
"id": 101,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "center",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "value"
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "foxhunt_hyperopt_trial_current{model=~\"$model\", job=~\"$source\"}",
"legendFormat": "{{model}}",
"refId": "A"
}
],
"title": "Current Trial",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "Rate of improvement in best hyperopt objective (negative = improving for loss minimization)",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"decimals": 4,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 0
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 3,
"x": 11,
"y": 1
},
"id": 4,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "center",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "value"
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "deriv(foxhunt_hyperopt_best_objective{model=~\"$model\", job=~\"$source\"}[10m])",
"legendFormat": "\u0394 objective",
"refId": "A"
}
],
"title": "Best Objective \u0394",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "yellow",
"value": 0.3
},
{
"color": "green",
"value": 0.7
}
]
},
"unit": "percentunit"
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 3,
"x": 14,
"y": 1
},
"id": 5,
"options": {
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showThresholdLabels": false,
"showThresholdMarkers": true
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "foxhunt_hyperopt_trial_current{model=~\"$model\", job=~\"$source\"} / foxhunt_hyperopt_trial_total{model=~\"$model\", job=~\"$source\"}",
"legendFormat": "{{model}}",
"refId": "A"
}
],
"title": "Hyperopt Trial Progress",
"type": "gauge"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "Errors"
},
"properties": [
{
"id": "thresholds",
"value": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 1
}
]
}
}
]
}
]
},
"gridPos": {
"h": 5,
"w": 3,
"x": 17,
"y": 1
},
"id": 6,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "center",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "value"
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "ml_training_jobs_started_total",
"legendFormat": "Jobs",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "ml_training_errors_total",
"legendFormat": "Errors",
"refId": "B"
}
],
"title": "ML Jobs / Errors",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "blue",
"value": null
}
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 4,
"x": 20,
"y": 1
},
"id": 99,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "center",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "value"
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "foxhunt_hyperopt_elapsed_seconds{model=~\"$model\", job=~\"$source\"}",
"legendFormat": "Elapsed",
"refId": "A"
}
],
"title": "Hyperopt Elapsed",
"type": "stat"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 6
},
"id": 66,
"panels": [],
"title": "Run Summary",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisPlacement": "auto",
"drawStyle": "line",
"fillOpacity": 15,
"gradientMode": "opacity",
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 5,
"showPoints": "auto",
"spanNulls": true,
"stacking": {
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"decimals": 4,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 0.1
},
{
"color": "red",
"value": 0.5
}
]
},
"unit": "none"
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 6,
"x": 0,
"y": 6
},
"id": 67,
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max",
"min"
],
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "clamp_max(min_over_time(foxhunt_training_validation_loss{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}[$__range]), 5)",
"instant": false,
"legendFormat": "{{model}}/{{fold}}",
"refId": "A"
}
],
"title": "Best Val Loss",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisPlacement": "auto",
"drawStyle": "line",
"fillOpacity": 15,
"gradientMode": "opacity",
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 5,
"showPoints": "auto",
"spanNulls": true,
"stacking": {
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"decimals": 2,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "yellow",
"value": 1
},
{
"color": "green",
"value": 3
}
]
},
"unit": "none"
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 6,
"x": 6,
"y": 6
},
"id": 68,
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max",
"min"
],
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "clamp_max(max_over_time(foxhunt_training_epoch_sharpe{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}[$__range]), 5)",
"instant": false,
"legendFormat": "{{model}}/{{fold}}",
"refId": "A"
}
],
"title": "Best Sharpe",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisPlacement": "auto",
"drawStyle": "line",
"fillOpacity": 15,
"gradientMode": "opacity",
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 5,
"showPoints": "auto",
"spanNulls": true,
"stacking": {
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"decimals": 1,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "yellow",
"value": 0.4
},
{
"color": "green",
"value": 0.55
}
]
},
"unit": "percentunit"
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 6,
"x": 12,
"y": 6
},
"id": 69,
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max",
"min"
],
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "max_over_time(foxhunt_training_epoch_win_rate{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}[$__range])",
"instant": false,
"legendFormat": "{{model}}/{{fold}}",
"refId": "A"
}
],
"title": "Best Win Rate",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisLabel": "Objective",
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "opacity",
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 4,
"showPoints": "auto"
}
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 6,
"x": 18,
"y": 6
},
"id": 42,
"options": {
"legend": {
"calcs": [
"lastNotNull",
"min"
],
"displayMode": "table",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "foxhunt_hyperopt_best_objective{model=~\"$model\", job=~\"$source\"}",
"legendFormat": "best {{model}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "foxhunt_hyperopt_trial_best_loss{model=~\"$model\", job=~\"$source\"}",
"legendFormat": "trial loss {{model}}",
"refId": "B"
}
],
"title": "Best Objective Over Time",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisPlacement": "auto",
"drawStyle": "line",
"fillOpacity": 15,
"gradientMode": "opacity",
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 5,
"showPoints": "auto",
"spanNulls": true,
"stacking": {
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"decimals": 1,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 0.1
},
{
"color": "red",
"value": 0.25
}
]
},
"unit": "percentunit"
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 6,
"x": 0,
"y": 12
},
"id": 70,
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max",
"min"
],
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "min_over_time(foxhunt_training_epoch_max_drawdown{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}[$__range])",
"instant": false,
"legendFormat": "{{model}}/{{fold}}",
"refId": "A"
}
],
"title": "Min Max Drawdown",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisPlacement": "auto",
"drawStyle": "line",
"fillOpacity": 15,
"gradientMode": "opacity",
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 5,
"showPoints": "auto",
"spanNulls": true,
"stacking": {
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"decimals": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "blue",
"value": null
}
]
},
"unit": "none"
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 6,
"x": 6,
"y": 12
},
"id": 71,
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max",
"min"
],
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "foxhunt_training_current_epoch{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
"instant": false,
"legendFormat": "{{model}}/{{fold}}",
"refId": "A"
}
],
"title": "Total Epochs",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisPlacement": "auto",
"drawStyle": "line",
"fillOpacity": 15,
"gradientMode": "opacity",
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 5,
"showPoints": "auto",
"spanNulls": true,
"stacking": {
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"decimals": 2,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "yellow",
"value": 1
},
{
"color": "green",
"value": 1.5
}
]
},
"unit": "none"
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 6,
"x": 12,
"y": 12
},
"id": 72,
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max",
"min"
],
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "clamp_max(max_over_time(foxhunt_training_epoch_profit_factor{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}[$__range]), 20)",
"instant": false,
"legendFormat": "{{model}}/{{fold}}",
"refId": "A"
}
],
"title": "Best Profit Factor",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisLabel": "Duration",
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "opacity",
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 4,
"showPoints": "never"
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 6,
"x": 18,
"y": 12
},
"id": 43,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.50, rate(foxhunt_hyperopt_trial_duration_seconds_bucket{model=~\"$model\", job=~\"$source\"}[$__rate_interval]))",
"legendFormat": "p50 {{model}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.95, rate(foxhunt_hyperopt_trial_duration_seconds_bucket{model=~\"$model\", job=~\"$source\"}[$__rate_interval]))",
"legendFormat": "p95 {{model}}",
"refId": "B"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.99, rate(foxhunt_hyperopt_trial_duration_seconds_bucket{model=~\"$model\", job=~\"$source\"}[$__rate_interval]))",
"legendFormat": "p99 {{model}}",
"refId": "C"
}
],
"title": "Trial Duration (p50/p95/p99)",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 18
},
"id": 10,
"panels": [],
"title": "Training Curves",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisLabel": "Loss",
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "opacity",
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 5,
"showPoints": "never"
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 0,
"y": 19
},
"id": 11,
"options": {
"legend": {
"calcs": [
"lastNotNull",
"min"
],
"displayMode": "table",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "foxhunt_training_epoch_loss{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
"legendFormat": "train {{model}}/{{fold}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "foxhunt_training_validation_loss{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
"legendFormat": "val {{model}}/{{fold}}",
"refId": "B"
}
],
"title": "Training & Validation Loss",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisLabel": "LR",
"drawStyle": "line",
"fillOpacity": 15,
"gradientMode": "opacity",
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 5,
"scaleDistribution": {
"log": 10,
"type": "log"
},
"showPoints": "never"
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 8,
"y": 19
},
"id": 12,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "foxhunt_training_learning_rate{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
"legendFormat": "{{model}}/{{fold}}",
"refId": "A"
}
],
"title": "Learning Rate",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisLabel": "Norm",
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "opacity",
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 4,
"showPoints": "never"
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 4,
"x": 16,
"y": 19
},
"id": 13,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "foxhunt_training_gradient_norm{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
"legendFormat": "{{model}}/{{fold}}",
"refId": "A"
}
],
"title": "Gradient Norm",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "opacity",
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 4,
"showPoints": "never"
}
},
"overrides": [
{
"matcher": {
"id": "byFrameRefID",
"options": "B"
},
"properties": [
{
"id": "custom.axisPlacement",
"value": "right"
},
{
"id": "unit",
"value": "s"
}
]
}
]
},
"gridPos": {
"h": 8,
"w": 4,
"x": 20,
"y": 19
},
"id": 14,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "foxhunt_training_batches_per_second{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
"legendFormat": "batch/s {{model}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "foxhunt_training_epoch_duration_seconds{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
"legendFormat": "epoch dur {{model}}",
"refId": "B"
}
],
"title": "Batches/sec & Epoch Duration",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 27
},
"id": 20,
"panels": [],
"title": "Training Quality",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisLabel": "Sharpe",
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "opacity",
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 4,
"showPoints": "auto"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "yellow",
"value": 1
},
{
"color": "green",
"value": 2
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 6,
"x": 0,
"y": 28
},
"id": 21,
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "clamp_max(foxhunt_training_epoch_sharpe{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}, 5)",
"legendFormat": "{{model}}/{{fold}}",
"refId": "A"
}
],
"title": "Epoch Sharpe Ratio",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisLabel": "Sortino",
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "opacity",
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 4,
"showPoints": "auto"
}
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 6,
"x": 6,
"y": 28
},
"id": 22,
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "clamp_max(foxhunt_training_epoch_sortino{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}, 10)",
"legendFormat": "{{model}}/{{fold}}",
"refId": "A"
}
],
"title": "Epoch Sortino Ratio",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisLabel": "Win Rate",
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "opacity",
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 4,
"showPoints": "auto"
},
"unit": "percentunit"
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 4,
"x": 12,
"y": 28
},
"id": 23,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "foxhunt_training_epoch_win_rate{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
"legendFormat": "{{model}}/{{fold}}",
"refId": "A"
}
],
"title": "Win Rate",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisLabel": "Drawdown",
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "opacity",
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 4,
"showPoints": "auto"
},
"unit": "percentunit"
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 4,
"x": 16,
"y": 28
},
"id": 24,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "foxhunt_training_epoch_max_drawdown{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
"legendFormat": "{{model}}/{{fold}}",
"refId": "A"
}
],
"title": "Max Drawdown",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"drawStyle": "line",
"fillOpacity": 15,
"gradientMode": "opacity",
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 4,
"showPoints": "auto"
}
},
"overrides": [
{
"matcher": {
"id": "byFrameRefID",
"options": "B"
},
"properties": [
{
"id": "custom.axisPlacement",
"value": "right"
},
{
"id": "unit",
"value": "short"
},
{
"id": "custom.drawStyle",
"value": "bars"
},
{
"id": "custom.fillOpacity",
"value": 30
}
]
},
{
"matcher": {
"id": "byName",
"options": "foxhunt_training_epoch_profit_factor"
},
"properties": [
{
"id": "max",
"value": 20
}
]
}
]
},
"gridPos": {
"h": 7,
"w": 4,
"x": 20,
"y": 28
},
"id": 25,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "clamp_max(foxhunt_training_epoch_profit_factor{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}, 20)",
"legendFormat": "PF {{model}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "foxhunt_training_epoch_total_trades{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
"legendFormat": "trades {{model}}",
"refId": "B"
}
],
"title": "Profit Factor & Total Trades",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 35
},
"id": 30,
"panels": [],
"title": "RL Diagnostics",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisLabel": "Q-Value",
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "opacity",
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 4,
"showPoints": "never"
}
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 6,
"x": 0,
"y": 36
},
"id": 31,
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max",
"min"
],
"displayMode": "table",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "foxhunt_training_q_value_mean{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
"legendFormat": "mean {{model}}/{{fold}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "foxhunt_training_q_value_max{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
"legendFormat": "max {{model}}/{{fold}}",
"refId": "B"
}
],
"title": "Q-Value (Mean & Max)",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "continuous-blues"
},
"custom": {
"axisLabel": "Transitions",
"drawStyle": "line",
"fillOpacity": 30,
"gradientMode": "scheme",
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 4,
"showPoints": "never"
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 4,
"x": 6,
"y": 36
},
"id": 32,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "foxhunt_training_replay_buffer_size{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
"legendFormat": "{{model}}/{{fold}}",
"refId": "A"
}
],
"title": "Replay Buffer Size",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "Normalized entropy of policy actions. 0 = collapsed to single action, 1 = uniform distribution.",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisLabel": "Entropy",
"drawStyle": "line",
"fillOpacity": 15,
"gradientMode": "opacity",
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 4,
"showPoints": "never"
},
"custom.thresholdsStyle": {
"mode": "line"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "yellow",
"value": 0.1
},
{
"color": "green",
"value": 0.3
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 4,
"x": 10,
"y": 36
},
"id": 33,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "foxhunt_training_action_entropy{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
"legendFormat": "{{model}}/{{fold}}",
"refId": "A"
}
],
"title": "Action Entropy",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "Fraction of 45 factored actions actually used by the policy.",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisLabel": "Diversity",
"drawStyle": "line",
"fillOpacity": 15,
"gradientMode": "opacity",
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 4,
"showPoints": "never"
},
"unit": "percentunit"
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 4,
"x": 14,
"y": 36
},
"id": 34,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "foxhunt_training_action_diversity{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
"legendFormat": "{{model}}/{{fold}}",
"refId": "A"
}
],
"title": "Action Diversity",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisLabel": "Pct",
"drawStyle": "line",
"fillOpacity": 50,
"gradientMode": "opacity",
"lineInterpolation": "smooth",
"lineWidth": 1,
"pointSize": 4,
"showPoints": "never",
"stacking": {
"group": "A",
"mode": "normal"
}
},
"unit": "percentunit"
},
"overrides": [
{
"matcher": {
"id": "byRegexp",
"options": "buy.*"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "green",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byRegexp",
"options": "sell.*"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "red",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byRegexp",
"options": "hold.*"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "blue",
"mode": "fixed"
}
}
]
}
]
},
"gridPos": {
"h": 7,
"w": 6,
"x": 18,
"y": 36
},
"id": 35,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "foxhunt_training_epoch_action_buy_pct{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
"legendFormat": "buy {{model}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "foxhunt_training_epoch_action_sell_pct{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
"legendFormat": "sell {{model}}",
"refId": "B"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "foxhunt_training_epoch_action_hold_pct{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
"legendFormat": "hold {{model}}",
"refId": "C"
}
],
"title": "Action Distribution (Buy/Sell/Hold %)",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "Q-max / |Q-mean| \u2014 spikes above 10 indicate DQN overestimation",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisPlacement": "auto",
"drawStyle": "line",
"fillOpacity": 15,
"gradientMode": "opacity",
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 5,
"showPoints": "auto",
"spanNulls": true,
"stacking": {
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"decimals": 1,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 5
},
{
"color": "red",
"value": 10
}
]
},
"unit": "none"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 0,
"y": 43
},
"id": 102,
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max",
"min"
],
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "foxhunt_training_q_overestimation_ratio{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
"legendFormat": "{{model}}/{{fold}}",
"refId": "A"
}
],
"title": "Q-Overestimation Ratio",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "Policy entropy (exploration health) and KL divergence (policy drift)",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisPlacement": "auto",
"drawStyle": "line",
"fillOpacity": 15,
"gradientMode": "opacity",
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 5,
"showPoints": "auto",
"spanNulls": true,
"stacking": {
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"decimals": 4,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "none"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 8,
"y": 43
},
"id": 103,
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max",
"min"
],
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "foxhunt_training_policy_entropy{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
"legendFormat": "entropy {{model}}/{{fold}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "foxhunt_training_kl_divergence{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
"legendFormat": "KL {{model}}/{{fold}}",
"refId": "B"
}
],
"title": "Policy Entropy & KL Divergence (PPO)",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "Mean advantage estimate \u2014 should hover near zero",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisPlacement": "auto",
"drawStyle": "line",
"fillOpacity": 15,
"gradientMode": "opacity",
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 5,
"showPoints": "auto",
"spanNulls": true,
"stacking": {
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"decimals": 4,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "none"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 16,
"y": 43
},
"id": 104,
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max",
"min"
],
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "foxhunt_training_advantage_mean{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
"legendFormat": "{{model}}/{{fold}}",
"refId": "A"
}
],
"title": "Advantage Mean (PPO)",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 51
},
"id": 105,
"panels": [],
"title": "Training Health",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "NaN loss events, gradient explosions, and feature extraction failures",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisPlacement": "auto",
"drawStyle": "line",
"fillOpacity": 15,
"gradientMode": "opacity",
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 5,
"showPoints": "auto",
"spanNulls": true,
"stacking": {
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"decimals": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 1
}
]
},
"unit": "none"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 0,
"y": 52
},
"id": 106,
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max",
"min"
],
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "increase(foxhunt_training_nan_detected_total{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}[$__rate_interval])",
"legendFormat": "NaN {{model}}/{{fold}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "increase(foxhunt_training_gradient_explosion_total{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}[$__rate_interval])",
"legendFormat": "grad explosion {{model}}/{{fold}}",
"refId": "B"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "increase(foxhunt_training_feature_errors_total{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}[$__rate_interval])",
"legendFormat": "feature err {{model}}/{{fold}}",
"refId": "C"
}
],
"title": "Training Anomalies",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "Checkpoint save/failure rate",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisPlacement": "auto",
"drawStyle": "line",
"fillOpacity": 15,
"gradientMode": "opacity",
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 5,
"showPoints": "auto",
"spanNulls": true,
"stacking": {
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"decimals": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "none"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 8,
"y": 52
},
"id": 107,
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max",
"min"
],
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "increase(foxhunt_training_checkpoint_saves_total{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}[$__rate_interval])",
"legendFormat": "saves {{model}}/{{fold}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "increase(foxhunt_training_checkpoint_failures_total{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}[$__rate_interval])",
"legendFormat": "failures {{model}}/{{fold}}",
"refId": "B"
}
],
"title": "Checkpoint Saves & Failures",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "p95 checkpoint save and data loading latency",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisPlacement": "auto",
"drawStyle": "line",
"fillOpacity": 15,
"gradientMode": "opacity",
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 5,
"showPoints": "auto",
"spanNulls": true,
"stacking": {
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"decimals": 1,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 16,
"y": 52
},
"id": 108,
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max",
"min"
],
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.95, rate(foxhunt_training_checkpoint_duration_seconds_bucket{model=~\"$model\", job=~\"$source\"}[$__rate_interval]))",
"legendFormat": "ckpt p95 {{model}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.95, rate(foxhunt_training_data_load_seconds_bucket{model=~\"$model\", job=~\"$source\"}[$__rate_interval]))",
"legendFormat": "data load p95 {{model}}",
"refId": "B"
}
],
"title": "Checkpoint & Data Load Latency",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "Size of last saved checkpoint",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisPlacement": "auto",
"drawStyle": "line",
"fillOpacity": 15,
"gradientMode": "opacity",
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 5,
"showPoints": "auto",
"spanNulls": true,
"stacking": {
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"decimals": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "bytes"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 0,
"y": 60
},
"id": 109,
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max",
"min"
],
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "foxhunt_training_checkpoint_size_bytes{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
"legendFormat": "{{model}}/{{fold}}",
"refId": "A"
}
],
"title": "Checkpoint Size",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "Total batches processed this run",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisPlacement": "auto",
"drawStyle": "line",
"fillOpacity": 15,
"gradientMode": "opacity",
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 5,
"showPoints": "auto",
"spanNulls": true,
"stacking": {
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"decimals": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "none"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 8,
"y": 60
},
"id": 110,
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max",
"min"
],
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "foxhunt_training_batches_processed{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
"legendFormat": "{{model}}/{{fold}}",
"refId": "A"
}
],
"title": "Batches Processed",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "Last epoch wall time",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisPlacement": "auto",
"drawStyle": "line",
"fillOpacity": 15,
"gradientMode": "opacity",
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 5,
"showPoints": "auto",
"spanNulls": true,
"stacking": {
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"decimals": 1,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 16,
"y": 60
},
"id": 111,
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max",
"min"
],
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "foxhunt_training_iteration_seconds{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
"legendFormat": "{{model}}/{{fold}}",
"refId": "A"
}
],
"title": "Iteration Time",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 68
},
"id": 112,
"panels": [],
"title": "Epoch Returns",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "Epoch total return (fractional)",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisPlacement": "auto",
"drawStyle": "line",
"fillOpacity": 15,
"gradientMode": "opacity",
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 5,
"showPoints": "auto",
"spanNulls": true,
"stacking": {
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"decimals": 2,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "percentunit"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 69
},
"id": 113,
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max",
"min"
],
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "foxhunt_training_epoch_total_return{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
"legendFormat": "{{model}}/{{fold}}",
"refId": "A"
}
],
"title": "Total Return per Epoch",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "Average return per trade",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisPlacement": "auto",
"drawStyle": "line",
"fillOpacity": 15,
"gradientMode": "opacity",
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 5,
"showPoints": "auto",
"spanNulls": true,
"stacking": {
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"decimals": 4,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "none"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 69
},
"id": 114,
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max",
"min"
],
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "foxhunt_training_epoch_avg_return{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
"legendFormat": "{{model}}/{{fold}}",
"refId": "A"
}
],
"title": "Avg Return per Trade",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 77
},
"id": 115,
"panels": [],
"title": "Supervised Model Eval",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "Supervised eval accuracy and F1 score",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisPlacement": "auto",
"drawStyle": "line",
"fillOpacity": 15,
"gradientMode": "opacity",
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 5,
"showPoints": "auto",
"spanNulls": true,
"stacking": {
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"decimals": 2,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "percentunit"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 78
},
"id": 116,
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max",
"min"
],
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "foxhunt_training_eval_accuracy{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
"legendFormat": "accuracy {{model}}/{{fold}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "foxhunt_training_eval_f1{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
"legendFormat": "F1 {{model}}/{{fold}}",
"refId": "B"
}
],
"title": "Accuracy & F1",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "Supervised eval precision and recall",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisPlacement": "auto",
"drawStyle": "line",
"fillOpacity": 15,
"gradientMode": "opacity",
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 5,
"showPoints": "auto",
"spanNulls": true,
"stacking": {
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"decimals": 2,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "percentunit"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 78
},
"id": 117,
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max",
"min"
],
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "foxhunt_training_eval_precision{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
"legendFormat": "precision {{model}}/{{fold}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "foxhunt_training_eval_recall{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
"legendFormat": "recall {{model}}/{{fold}}",
"refId": "B"
}
],
"title": "Precision & Recall",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 86
},
"id": 118,
"panels": [],
"title": "Hyperopt Details",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "1 = hyperopt active, 0 = training only",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"decimals": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "blue",
"value": null
},
{
"color": "green",
"value": 1
}
]
},
"unit": "none"
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 4,
"x": 0,
"y": 87
},
"id": 119,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "center",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "value"
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "foxhunt_hyperopt_mode{model=~\"$model\", job=~\"$source\"}",
"legendFormat": "{{model}}",
"refId": "A"
}
],
"title": "Hyperopt Mode",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "Current epoch within the active trial",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisPlacement": "auto",
"drawStyle": "line",
"fillOpacity": 15,
"gradientMode": "opacity",
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 5,
"showPoints": "auto",
"spanNulls": true,
"stacking": {
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"decimals": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "none"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 10,
"x": 4,
"y": 87
},
"id": 120,
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max",
"min"
],
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "foxhunt_hyperopt_trial_epoch{model=~\"$model\", job=~\"$source\"}",
"legendFormat": "{{model}}",
"refId": "A"
}
],
"title": "Hyperopt Trial Epoch",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "Cumulative failed trial count",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisPlacement": "auto",
"drawStyle": "line",
"fillOpacity": 15,
"gradientMode": "opacity",
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 5,
"showPoints": "auto",
"spanNulls": true,
"stacking": {
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"decimals": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 3
}
]
},
"unit": "none"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 10,
"x": 14,
"y": 87
},
"id": 121,
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max",
"min"
],
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "foxhunt_hyperopt_trials_failed_total{model=~\"$model\", job=~\"$source\"}",
"legendFormat": "{{model}}",
"refId": "A"
}
],
"title": "Hyperopt Failed Trials",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 95
},
"id": 50,
"panels": [],
"title": "GPU Utilization",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "continuous-GrYlRd"
},
"custom": {
"axisLabel": "Util %",
"drawStyle": "line",
"fillOpacity": 20,
"gradientMode": "opacity",
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 4,
"showPoints": "never"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "yellow",
"value": 50
},
{
"color": "green",
"value": 80
}
]
},
"unit": "percent"
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 6,
"x": 0,
"y": 96
},
"id": 51,
"options": {
"legend": {
"calcs": [
"mean",
"max"
],
"displayMode": "table",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "DCGM_FI_DEV_GPU_UTIL",
"legendFormat": "GPU {{gpu}}",
"refId": "A"
}
],
"title": "GPU Utilization %",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisLabel": "MiB",
"drawStyle": "line",
"fillOpacity": 15,
"gradientMode": "opacity",
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 4,
"showPoints": "never"
},
"unit": "decmbytes"
},
"overrides": [
{
"matcher": {
"id": "byRegexp",
"options": "free.*"
},
"properties": [
{
"id": "custom.fillOpacity",
"value": 5
},
{
"id": "custom.lineStyle",
"value": {
"dash": [
10,
5
],
"fill": "dash"
}
}
]
}
]
},
"gridPos": {
"h": 7,
"w": 6,
"x": 6,
"y": 96
},
"id": 52,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "DCGM_FI_DEV_FB_USED",
"legendFormat": "used GPU {{gpu}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "DCGM_FI_DEV_FB_FREE",
"legendFormat": "free GPU {{gpu}}",
"refId": "B"
}
],
"title": "VRAM Used / Free (MiB)",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisLabel": "Temp",
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "opacity",
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 4,
"showPoints": "never"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 75
},
{
"color": "red",
"value": 85
}
]
},
"unit": "celsius"
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 4,
"x": 12,
"y": 96
},
"id": 53,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "DCGM_FI_DEV_GPU_TEMP",
"legendFormat": "GPU temp {{gpu}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "DCGM_FI_DEV_MEMORY_TEMP",
"legendFormat": "mem temp {{gpu}}",
"refId": "B"
}
],
"title": "GPU / Memory Temperature",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisLabel": "Watts",
"drawStyle": "line",
"fillOpacity": 15,
"gradientMode": "opacity",
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 4,
"showPoints": "never"
},
"unit": "watt"
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 4,
"x": 16,
"y": 96
},
"id": 54,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "DCGM_FI_DEV_POWER_USAGE",
"legendFormat": "GPU {{gpu}}",
"refId": "A"
}
],
"title": "Power Usage (W)",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisLabel": "Activity",
"drawStyle": "line",
"fillOpacity": 15,
"gradientMode": "opacity",
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 4,
"showPoints": "never"
},
"unit": "percentunit"
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 4,
"x": 20,
"y": 96
},
"id": 55,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "DCGM_FI_PROF_PIPE_TENSOR_ACTIVE",
"legendFormat": "tensor {{gpu}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "DCGM_FI_PROF_DRAM_ACTIVE",
"legendFormat": "DRAM {{gpu}}",
"refId": "B"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "DCGM_FI_PROF_GR_ENGINE_ACTIVE",
"legendFormat": "engine {{gpu}}",
"refId": "C"
}
],
"title": "Tensor Core & DRAM Activity",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 103
},
"id": 60,
"panels": [],
"title": "Argo CI/CD",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "blue",
"value": null
}
]
}
},
"overrides": [
{
"matcher": {
"id": "byRegexp",
"options": "Running"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "green",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byRegexp",
"options": "Pending"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "yellow",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byRegexp",
"options": "Failed|Error"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "red",
"mode": "fixed"
}
}
]
},
{
"matcher": {
"id": "byRegexp",
"options": "Succeeded"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "blue",
"mode": "fixed"
}
}
]
}
]
},
"gridPos": {
"h": 5,
"w": 6,
"x": 0,
"y": 104
},
"id": 61,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "center",
"orientation": "horizontal",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "argo_workflows_gauge{status=\"Running\"}",
"legendFormat": "Running",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "argo_workflows_gauge{status=\"Pending\"}",
"legendFormat": "Pending",
"refId": "B"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "argo_workflows_gauge{status=\"Failed\"}",
"legendFormat": "Failed",
"refId": "C"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "argo_workflows_gauge{status=\"Succeeded\"}",
"legendFormat": "Succeeded",
"refId": "D"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "argo_workflows_gauge{status=\"Error\"}",
"legendFormat": "Error",
"refId": "E"
}
],
"title": "Workflows by Status",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisLabel": "Pods",
"drawStyle": "line",
"fillOpacity": 20,
"gradientMode": "opacity",
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 4,
"showPoints": "never"
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 4,
"x": 6,
"y": 104
},
"id": 62,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "argo_workflows_pods_gauge",
"legendFormat": "pods",
"refId": "A"
}
],
"title": "Workflow Pods",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"fixedColor": "red",
"mode": "fixed"
},
"custom": {
"axisLabel": "Errors/s",
"drawStyle": "bars",
"fillOpacity": 50,
"gradientMode": "opacity",
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 4,
"showPoints": "never"
},
"unit": "ops"
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 4,
"x": 10,
"y": 104
},
"id": 63,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(argo_workflows_error_count[$__rate_interval])",
"legendFormat": "errors/s",
"refId": "A"
}
],
"title": "Error Rate",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisLabel": "Duration",
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "opacity",
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 4,
"showPoints": "never"
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 5,
"x": 14,
"y": 104
},
"id": 64,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.50, rate(argo_workflows_operation_duration_seconds_bucket[$__rate_interval]))",
"legendFormat": "p50",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "histogram_quantile(0.95, rate(argo_workflows_operation_duration_seconds_bucket[$__rate_interval]))",
"legendFormat": "p95",
"refId": "B"
}
],
"title": "Operation Duration (p50/p95)",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisLabel": "Triggers/s",
"drawStyle": "bars",
"fillOpacity": 40,
"gradientMode": "opacity",
"lineInterpolation": "smooth",
"lineWidth": 2,
"pointSize": 4,
"showPoints": "never",
"stacking": {
"mode": "normal"
}
},
"unit": "ops"
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 5,
"x": 19,
"y": 104
},
"id": 65,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"expr": "rate(argo_workflows_workflowtemplate_triggered_total[$__rate_interval])",
"legendFormat": "triggers/s",
"refId": "A"
}
],
"title": "Template Trigger Rate",
"type": "timeseries"
}
],
"refresh": "15s",
"schemaVersion": 39,
"style": "dark",
"tags": [
"foxhunt",
"training",
"gpu",
"ci"
],
"templating": {
"list": [
{
"auto": true,
"auto_count": 30,
"auto_min": "1m",
"current": {
"selected": true,
"text": "auto",
"value": "$__auto_interval_interval"
},
"label": "Interval",
"name": "interval",
"options": [
{
"selected": false,
"text": "1m",
"value": "1m"
},
{
"selected": false,
"text": "5m",
"value": "5m"
},
{
"selected": false,
"text": "15m",
"value": "15m"
},
{
"selected": false,
"text": "30m",
"value": "30m"
},
{
"selected": false,
"text": "1h",
"value": "1h"
}
],
"query": "1m,5m,15m,30m,1h",
"type": "interval"
},
{
"current": {
"selected": true,
"text": "Live",
"value": "training-pods"
},
"description": "Live = running training pods, CI History = completed CI runs (pushgateway), All = both",
"label": "Source",
"name": "source",
"options": [
{
"selected": true,
"text": "Live",
"value": "training-pods"
},
{
"selected": false,
"text": "CI History",
"value": ".+_baseline.*"
},
{
"selected": false,
"text": "All",
"value": "training-pods|.+_baseline.*"
}
],
"query": "Live : training-pods, CI History : .+_baseline.*, All : training-pods|.+_baseline.*",
"type": "custom"
},
{
"allValue": ".*",
"current": {
"selected": true,
"text": "All",
"value": "$__all"
},
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"includeAll": true,
"label": "Model",
"multi": true,
"name": "model",
"query": "label_values(foxhunt_training_step{job=~\"$source\"}, model)",
"refresh": 2,
"sort": 1,
"type": "query"
},
{
"allValue": ".*",
"current": {
"selected": true,
"text": "All",
"value": "$__all"
},
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"includeAll": true,
"label": "Fold",
"multi": true,
"name": "fold",
"query": "label_values(foxhunt_training_step{job=~\"$source\"}, fold)",
"refresh": 2,
"sort": 1,
"type": "query"
}
]
},
"time": {
"from": "now-6h",
"to": "now"
},
"timepicker": {},
"timezone": "browser",
"title": "Training & CI/CD",
"uid": "foxhunt-training",
"__inputs": [
{
"name": "DS_PROMETHEUS",
"type": "datasource",
"pluginId": "prometheus"
},
{
"name": "DS_LOKI",
"type": "datasource",
"pluginId": "loki"
},
{
"name": "DS_TEMPO",
"type": "datasource",
"pluginId": "${DS_TEMPO}"
}
]
}
Reference in New Issue View Git Blame Copy Permalink