{
  "description": "ML model training, hyperopt, GPU utilization, Argo CI pipelines, and full metrics coverage",
  "editable": true,
  "graphTooltip": 2,
  "panels": [
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 0
      },
      "id": 1,
      "panels": [],
      "title": "Training Status",
      "type": "row"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "red",
                "value": null
              },
              {
                "color": "green",
                "value": 1
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 5,
        "w": 3,
        "x": 0,
        "y": 1
      },
      "id": 2,
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "center",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "value"
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "sum(foxhunt_training_active_workers{job=\"training-pods\"}) or vector(0)",
          "legendFormat": "Workers",
          "refId": "A"
        }
      ],
      "title": "Active Workers",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Current epoch and step within epoch",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "decimals": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "blue",
                "value": null
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 5,
        "w": 4,
        "x": 7,
        "y": 1
      },
      "id": 3,
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "center",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "value"
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "foxhunt_training_current_epoch{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
          "legendFormat": "{{model}} / fold {{fold}}",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "foxhunt_training_step{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
          "legendFormat": "step {{model}}/{{fold}}",
          "refId": "B"
        }
      ],
      "title": "Current Epoch",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "decimals": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "purple",
                "value": null
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 5,
        "w": 4,
        "x": 3,
        "y": 1
      },
      "id": 101,
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "center",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "value"
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "foxhunt_hyperopt_trial_current{model=~\"$model\", job=~\"$source\"}",
          "legendFormat": "{{model}}",
          "refId": "A"
        }
      ],
      "title": "Current Trial",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Rate of improvement in best hyperopt objective (negative = improving for loss minimization)",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "decimals": 4,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 0
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 5,
        "w": 3,
        "x": 11,
        "y": 1
      },
      "id": 4,
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "center",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "value"
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "deriv(foxhunt_hyperopt_best_objective{model=~\"$model\", job=~\"$source\"}[10m])",
          "legendFormat": "\u0394 objective",
          "refId": "A"
        }
      ],
      "title": "Best Objective \u0394",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "red",
                "value": null
              },
              {
                "color": "yellow",
                "value": 0.3
              },
              {
                "color": "green",
                "value": 0.7
              }
            ]
          },
          "unit": "percentunit"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 5,
        "w": 3,
        "x": 14,
        "y": 1
      },
      "id": 5,
      "options": {
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "showThresholdLabels": false,
        "showThresholdMarkers": true
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "foxhunt_hyperopt_trial_current{model=~\"$model\", job=~\"$source\"} / foxhunt_hyperopt_trial_total{model=~\"$model\", job=~\"$source\"}",
          "legendFormat": "{{model}}",
          "refId": "A"
        }
      ],
      "title": "Hyperopt Trial Progress",
      "type": "gauge"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          }
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Errors"
            },
            "properties": [
              {
                "id": "thresholds",
                "value": {
                  "mode": "absolute",
                  "steps": [
                    {
                      "color": "green",
                      "value": null
                    },
                    {
                      "color": "red",
                      "value": 1
                    }
                  ]
                }
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 5,
        "w": 3,
        "x": 17,
        "y": 1
      },
      "id": 6,
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "center",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "value"
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "ml_training_jobs_started_total",
          "legendFormat": "Jobs",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "ml_training_errors_total",
          "legendFormat": "Errors",
          "refId": "B"
        }
      ],
      "title": "ML Jobs / Errors",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "blue",
                "value": null
              }
            ]
          },
          "unit": "s"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 5,
        "w": 4,
        "x": 20,
        "y": 1
      },
      "id": 99,
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "center",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "value"
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "foxhunt_hyperopt_elapsed_seconds{model=~\"$model\", job=~\"$source\"}",
          "legendFormat": "Elapsed",
          "refId": "A"
        }
      ],
      "title": "Hyperopt Elapsed",
      "type": "stat"
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 6
      },
      "id": 66,
      "panels": [],
      "title": "Run Summary",
      "type": "row"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisPlacement": "auto",
            "drawStyle": "line",
            "fillOpacity": 15,
            "gradientMode": "opacity",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "pointSize": 5,
            "showPoints": "auto",
            "spanNulls": true,
            "stacking": {
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "decimals": 4,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "yellow",
                "value": 0.1
              },
              {
                "color": "red",
                "value": 0.5
              }
            ]
          },
          "unit": "none"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 6,
        "w": 6,
        "x": 0,
        "y": 6
      },
      "id": 67,
      "options": {
        "legend": {
          "calcs": [
            "lastNotNull",
            "max",
            "min"
          ],
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "clamp_max(min_over_time(foxhunt_training_validation_loss{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}[$__range]), 5)",
          "instant": false,
          "legendFormat": "{{model}}/{{fold}}",
          "refId": "A"
        }
      ],
      "title": "Best Val Loss",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisPlacement": "auto",
            "drawStyle": "line",
            "fillOpacity": 15,
            "gradientMode": "opacity",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "pointSize": 5,
            "showPoints": "auto",
            "spanNulls": true,
            "stacking": {
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "decimals": 2,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "red",
                "value": null
              },
              {
                "color": "yellow",
                "value": 1
              },
              {
                "color": "green",
                "value": 3
              }
            ]
          },
          "unit": "none"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 6,
        "w": 6,
        "x": 6,
        "y": 6
      },
      "id": 68,
      "options": {
        "legend": {
          "calcs": [
            "lastNotNull",
            "max",
            "min"
          ],
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "clamp_max(max_over_time(foxhunt_training_epoch_sharpe{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}[$__range]), 5)",
          "instant": false,
          "legendFormat": "{{model}}/{{fold}}",
          "refId": "A"
        }
      ],
      "title": "Best Sharpe",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisPlacement": "auto",
            "drawStyle": "line",
            "fillOpacity": 15,
            "gradientMode": "opacity",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "pointSize": 5,
            "showPoints": "auto",
            "spanNulls": true,
            "stacking": {
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "decimals": 1,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "red",
                "value": null
              },
              {
                "color": "yellow",
                "value": 0.4
              },
              {
                "color": "green",
                "value": 0.55
              }
            ]
          },
          "unit": "percentunit"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 6,
        "w": 6,
        "x": 12,
        "y": 6
      },
      "id": 69,
      "options": {
        "legend": {
          "calcs": [
            "lastNotNull",
            "max",
            "min"
          ],
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "max_over_time(foxhunt_training_epoch_win_rate{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}[$__range])",
          "instant": false,
          "legendFormat": "{{model}}/{{fold}}",
          "refId": "A"
        }
      ],
      "title": "Best Win Rate",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "Objective",
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "opacity",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "pointSize": 4,
            "showPoints": "auto"
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 6,
        "w": 6,
        "x": 18,
        "y": 6
      },
      "id": 42,
      "options": {
        "legend": {
          "calcs": [
            "lastNotNull",
            "min"
          ],
          "displayMode": "table",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "foxhunt_hyperopt_best_objective{model=~\"$model\", job=~\"$source\"}",
          "legendFormat": "best {{model}}",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "foxhunt_hyperopt_trial_best_loss{model=~\"$model\", job=~\"$source\"}",
          "legendFormat": "trial loss {{model}}",
          "refId": "B"
        }
      ],
      "title": "Best Objective Over Time",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisPlacement": "auto",
            "drawStyle": "line",
            "fillOpacity": 15,
            "gradientMode": "opacity",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "pointSize": 5,
            "showPoints": "auto",
            "spanNulls": true,
            "stacking": {
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "decimals": 1,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "yellow",
                "value": 0.1
              },
              {
                "color": "red",
                "value": 0.25
              }
            ]
          },
          "unit": "percentunit"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 6,
        "w": 6,
        "x": 0,
        "y": 12
      },
      "id": 70,
      "options": {
        "legend": {
          "calcs": [
            "lastNotNull",
            "max",
            "min"
          ],
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "min_over_time(foxhunt_training_epoch_max_drawdown{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}[$__range])",
          "instant": false,
          "legendFormat": "{{model}}/{{fold}}",
          "refId": "A"
        }
      ],
      "title": "Min Max Drawdown",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisPlacement": "auto",
            "drawStyle": "line",
            "fillOpacity": 15,
            "gradientMode": "opacity",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "pointSize": 5,
            "showPoints": "auto",
            "spanNulls": true,
            "stacking": {
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "decimals": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "blue",
                "value": null
              }
            ]
          },
          "unit": "none"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 6,
        "w": 6,
        "x": 6,
        "y": 12
      },
      "id": 71,
      "options": {
        "legend": {
          "calcs": [
            "lastNotNull",
            "max",
            "min"
          ],
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "foxhunt_training_current_epoch{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
          "instant": false,
          "legendFormat": "{{model}}/{{fold}}",
          "refId": "A"
        }
      ],
      "title": "Total Epochs",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisPlacement": "auto",
            "drawStyle": "line",
            "fillOpacity": 15,
            "gradientMode": "opacity",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "pointSize": 5,
            "showPoints": "auto",
            "spanNulls": true,
            "stacking": {
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "decimals": 2,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "red",
                "value": null
              },
              {
                "color": "yellow",
                "value": 1
              },
              {
                "color": "green",
                "value": 1.5
              }
            ]
          },
          "unit": "none"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 6,
        "w": 6,
        "x": 12,
        "y": 12
      },
      "id": 72,
      "options": {
        "legend": {
          "calcs": [
            "lastNotNull",
            "max",
            "min"
          ],
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "clamp_max(max_over_time(foxhunt_training_epoch_profit_factor{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}[$__range]), 20)",
          "instant": false,
          "legendFormat": "{{model}}/{{fold}}",
          "refId": "A"
        }
      ],
      "title": "Best Profit Factor",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "Duration",
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "opacity",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "pointSize": 4,
            "showPoints": "never"
          },
          "unit": "s"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 6,
        "w": 6,
        "x": 18,
        "y": 12
      },
      "id": 43,
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "histogram_quantile(0.50, rate(foxhunt_hyperopt_trial_duration_seconds_bucket{model=~\"$model\", job=~\"$source\"}[$__rate_interval]))",
          "legendFormat": "p50 {{model}}",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "histogram_quantile(0.95, rate(foxhunt_hyperopt_trial_duration_seconds_bucket{model=~\"$model\", job=~\"$source\"}[$__rate_interval]))",
          "legendFormat": "p95 {{model}}",
          "refId": "B"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "histogram_quantile(0.99, rate(foxhunt_hyperopt_trial_duration_seconds_bucket{model=~\"$model\", job=~\"$source\"}[$__rate_interval]))",
          "legendFormat": "p99 {{model}}",
          "refId": "C"
        }
      ],
      "title": "Trial Duration (p50/p95/p99)",
      "type": "timeseries"
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 18
      },
      "id": 10,
      "panels": [],
      "title": "Training Curves",
      "type": "row"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisCenteredZero": false,
            "axisLabel": "Loss",
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "opacity",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "pointSize": 5,
            "showPoints": "never"
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 0,
        "y": 19
      },
      "id": 11,
      "options": {
        "legend": {
          "calcs": [
            "lastNotNull",
            "min"
          ],
          "displayMode": "table",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "foxhunt_training_epoch_loss{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
          "legendFormat": "train {{model}}/{{fold}}",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "foxhunt_training_validation_loss{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
          "legendFormat": "val {{model}}/{{fold}}",
          "refId": "B"
        }
      ],
      "title": "Training & Validation Loss",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "LR",
            "drawStyle": "line",
            "fillOpacity": 15,
            "gradientMode": "opacity",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "pointSize": 5,
            "scaleDistribution": {
              "log": 10,
              "type": "log"
            },
            "showPoints": "never"
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 8,
        "y": 19
      },
      "id": 12,
      "options": {
        "legend": {
          "calcs": [
            "lastNotNull"
          ],
          "displayMode": "table",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "foxhunt_training_learning_rate{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
          "legendFormat": "{{model}}/{{fold}}",
          "refId": "A"
        }
      ],
      "title": "Learning Rate",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "Norm",
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "opacity",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "pointSize": 4,
            "showPoints": "never"
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 4,
        "x": 16,
        "y": 19
      },
      "id": 13,
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "foxhunt_training_gradient_norm{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
          "legendFormat": "{{model}}/{{fold}}",
          "refId": "A"
        }
      ],
      "title": "Gradient Norm",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "opacity",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "pointSize": 4,
            "showPoints": "never"
          }
        },
        "overrides": [
          {
            "matcher": {
              "id": "byFrameRefID",
              "options": "B"
            },
            "properties": [
              {
                "id": "custom.axisPlacement",
                "value": "right"
              },
              {
                "id": "unit",
                "value": "s"
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 8,
        "w": 4,
        "x": 20,
        "y": 19
      },
      "id": 14,
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "foxhunt_training_batches_per_second{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
          "legendFormat": "batch/s {{model}}",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "foxhunt_training_epoch_duration_seconds{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
          "legendFormat": "epoch dur {{model}}",
          "refId": "B"
        }
      ],
      "title": "Batches/sec & Epoch Duration",
      "type": "timeseries"
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 27
      },
      "id": 20,
      "panels": [],
      "title": "Training Quality",
      "type": "row"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "Sharpe",
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "opacity",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "pointSize": 4,
            "showPoints": "auto"
          },
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "red",
                "value": null
              },
              {
                "color": "yellow",
                "value": 1
              },
              {
                "color": "green",
                "value": 2
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 6,
        "x": 0,
        "y": 28
      },
      "id": 21,
      "options": {
        "legend": {
          "calcs": [
            "lastNotNull",
            "max"
          ],
          "displayMode": "table",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "clamp_max(foxhunt_training_epoch_sharpe{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}, 5)",
          "legendFormat": "{{model}}/{{fold}}",
          "refId": "A"
        }
      ],
      "title": "Epoch Sharpe Ratio",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "Sortino",
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "opacity",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "pointSize": 4,
            "showPoints": "auto"
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 6,
        "x": 6,
        "y": 28
      },
      "id": 22,
      "options": {
        "legend": {
          "calcs": [
            "lastNotNull",
            "max"
          ],
          "displayMode": "table",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "clamp_max(foxhunt_training_epoch_sortino{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}, 10)",
          "legendFormat": "{{model}}/{{fold}}",
          "refId": "A"
        }
      ],
      "title": "Epoch Sortino Ratio",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "Win Rate",
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "opacity",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "pointSize": 4,
            "showPoints": "auto"
          },
          "unit": "percentunit"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 4,
        "x": 12,
        "y": 28
      },
      "id": 23,
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "foxhunt_training_epoch_win_rate{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
          "legendFormat": "{{model}}/{{fold}}",
          "refId": "A"
        }
      ],
      "title": "Win Rate",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "Drawdown",
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "opacity",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "pointSize": 4,
            "showPoints": "auto"
          },
          "unit": "percentunit"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 4,
        "x": 16,
        "y": 28
      },
      "id": 24,
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "foxhunt_training_epoch_max_drawdown{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
          "legendFormat": "{{model}}/{{fold}}",
          "refId": "A"
        }
      ],
      "title": "Max Drawdown",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "drawStyle": "line",
            "fillOpacity": 15,
            "gradientMode": "opacity",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "pointSize": 4,
            "showPoints": "auto"
          }
        },
        "overrides": [
          {
            "matcher": {
              "id": "byFrameRefID",
              "options": "B"
            },
            "properties": [
              {
                "id": "custom.axisPlacement",
                "value": "right"
              },
              {
                "id": "unit",
                "value": "short"
              },
              {
                "id": "custom.drawStyle",
                "value": "bars"
              },
              {
                "id": "custom.fillOpacity",
                "value": 30
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "foxhunt_training_epoch_profit_factor"
            },
            "properties": [
              {
                "id": "max",
                "value": 20
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 7,
        "w": 4,
        "x": 20,
        "y": 28
      },
      "id": 25,
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "clamp_max(foxhunt_training_epoch_profit_factor{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}, 20)",
          "legendFormat": "PF {{model}}",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "foxhunt_training_epoch_total_trades{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
          "legendFormat": "trades {{model}}",
          "refId": "B"
        }
      ],
      "title": "Profit Factor & Total Trades",
      "type": "timeseries"
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 35
      },
      "id": 30,
      "panels": [],
      "title": "RL Diagnostics",
      "type": "row"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "Q-Value",
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "opacity",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "pointSize": 4,
            "showPoints": "never"
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 6,
        "x": 0,
        "y": 36
      },
      "id": 31,
      "options": {
        "legend": {
          "calcs": [
            "lastNotNull",
            "max",
            "min"
          ],
          "displayMode": "table",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "foxhunt_training_q_value_mean{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
          "legendFormat": "mean {{model}}/{{fold}}",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "foxhunt_training_q_value_max{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
          "legendFormat": "max {{model}}/{{fold}}",
          "refId": "B"
        }
      ],
      "title": "Q-Value (Mean & Max)",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "continuous-blues"
          },
          "custom": {
            "axisLabel": "Transitions",
            "drawStyle": "line",
            "fillOpacity": 30,
            "gradientMode": "scheme",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "pointSize": 4,
            "showPoints": "never"
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 4,
        "x": 6,
        "y": 36
      },
      "id": 32,
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "foxhunt_training_replay_buffer_size{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
          "legendFormat": "{{model}}/{{fold}}",
          "refId": "A"
        }
      ],
      "title": "Replay Buffer Size",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Normalized entropy of policy actions. 0 = collapsed to single action, 1 = uniform distribution.",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "Entropy",
            "drawStyle": "line",
            "fillOpacity": 15,
            "gradientMode": "opacity",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "pointSize": 4,
            "showPoints": "never"
          },
          "custom.thresholdsStyle": {
            "mode": "line"
          },
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "red",
                "value": null
              },
              {
                "color": "yellow",
                "value": 0.1
              },
              {
                "color": "green",
                "value": 0.3
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 4,
        "x": 10,
        "y": 36
      },
      "id": 33,
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "foxhunt_training_action_entropy{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
          "legendFormat": "{{model}}/{{fold}}",
          "refId": "A"
        }
      ],
      "title": "Action Entropy",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Fraction of 45 factored actions actually used by the policy.",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "Diversity",
            "drawStyle": "line",
            "fillOpacity": 15,
            "gradientMode": "opacity",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "pointSize": 4,
            "showPoints": "never"
          },
          "unit": "percentunit"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 4,
        "x": 14,
        "y": 36
      },
      "id": 34,
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "foxhunt_training_action_diversity{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
          "legendFormat": "{{model}}/{{fold}}",
          "refId": "A"
        }
      ],
      "title": "Action Diversity",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "Pct",
            "drawStyle": "line",
            "fillOpacity": 50,
            "gradientMode": "opacity",
            "lineInterpolation": "smooth",
            "lineWidth": 1,
            "pointSize": 4,
            "showPoints": "never",
            "stacking": {
              "group": "A",
              "mode": "normal"
            }
          },
          "unit": "percentunit"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byRegexp",
              "options": "buy.*"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "green",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byRegexp",
              "options": "sell.*"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "red",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byRegexp",
              "options": "hold.*"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "blue",
                  "mode": "fixed"
                }
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 7,
        "w": 6,
        "x": 18,
        "y": 36
      },
      "id": 35,
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "foxhunt_training_epoch_action_buy_pct{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
          "legendFormat": "buy {{model}}",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "foxhunt_training_epoch_action_sell_pct{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
          "legendFormat": "sell {{model}}",
          "refId": "B"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "foxhunt_training_epoch_action_hold_pct{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
          "legendFormat": "hold {{model}}",
          "refId": "C"
        }
      ],
      "title": "Action Distribution (Buy/Sell/Hold %)",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Q-max / |Q-mean| \u2014 spikes above 10 indicate DQN overestimation",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisPlacement": "auto",
            "drawStyle": "line",
            "fillOpacity": 15,
            "gradientMode": "opacity",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "pointSize": 5,
            "showPoints": "auto",
            "spanNulls": true,
            "stacking": {
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "decimals": 1,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "yellow",
                "value": 5
              },
              {
                "color": "red",
                "value": 10
              }
            ]
          },
          "unit": "none"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 0,
        "y": 43
      },
      "id": 102,
      "options": {
        "legend": {
          "calcs": [
            "lastNotNull",
            "max",
            "min"
          ],
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "foxhunt_training_q_overestimation_ratio{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
          "legendFormat": "{{model}}/{{fold}}",
          "refId": "A"
        }
      ],
      "title": "Q-Overestimation Ratio",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Policy entropy (exploration health) and KL divergence (policy drift)",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisPlacement": "auto",
            "drawStyle": "line",
            "fillOpacity": 15,
            "gradientMode": "opacity",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "pointSize": 5,
            "showPoints": "auto",
            "spanNulls": true,
            "stacking": {
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "decimals": 4,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          },
          "unit": "none"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 8,
        "y": 43
      },
      "id": 103,
      "options": {
        "legend": {
          "calcs": [
            "lastNotNull",
            "max",
            "min"
          ],
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "foxhunt_training_policy_entropy{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
          "legendFormat": "entropy {{model}}/{{fold}}",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "foxhunt_training_kl_divergence{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
          "legendFormat": "KL {{model}}/{{fold}}",
          "refId": "B"
        }
      ],
      "title": "Policy Entropy & KL Divergence (PPO)",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Mean advantage estimate \u2014 should hover near zero",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisPlacement": "auto",
            "drawStyle": "line",
            "fillOpacity": 15,
            "gradientMode": "opacity",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "pointSize": 5,
            "showPoints": "auto",
            "spanNulls": true,
            "stacking": {
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "decimals": 4,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          },
          "unit": "none"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 16,
        "y": 43
      },
      "id": 104,
      "options": {
        "legend": {
          "calcs": [
            "lastNotNull",
            "max",
            "min"
          ],
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "foxhunt_training_advantage_mean{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
          "legendFormat": "{{model}}/{{fold}}",
          "refId": "A"
        }
      ],
      "title": "Advantage Mean (PPO)",
      "type": "timeseries"
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 51
      },
      "id": 105,
      "panels": [],
      "title": "Training Health",
      "type": "row"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "NaN loss events, gradient explosions, and feature extraction failures",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisPlacement": "auto",
            "drawStyle": "line",
            "fillOpacity": 15,
            "gradientMode": "opacity",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "pointSize": 5,
            "showPoints": "auto",
            "spanNulls": true,
            "stacking": {
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "decimals": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 1
              }
            ]
          },
          "unit": "none"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 0,
        "y": 52
      },
      "id": 106,
      "options": {
        "legend": {
          "calcs": [
            "lastNotNull",
            "max",
            "min"
          ],
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "increase(foxhunt_training_nan_detected_total{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}[$__rate_interval])",
          "legendFormat": "NaN {{model}}/{{fold}}",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "increase(foxhunt_training_gradient_explosion_total{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}[$__rate_interval])",
          "legendFormat": "grad explosion {{model}}/{{fold}}",
          "refId": "B"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "increase(foxhunt_training_feature_errors_total{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}[$__rate_interval])",
          "legendFormat": "feature err {{model}}/{{fold}}",
          "refId": "C"
        }
      ],
      "title": "Training Anomalies",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Checkpoint save/failure rate",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisPlacement": "auto",
            "drawStyle": "line",
            "fillOpacity": 15,
            "gradientMode": "opacity",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "pointSize": 5,
            "showPoints": "auto",
            "spanNulls": true,
            "stacking": {
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "decimals": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          },
          "unit": "none"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 8,
        "y": 52
      },
      "id": 107,
      "options": {
        "legend": {
          "calcs": [
            "lastNotNull",
            "max",
            "min"
          ],
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "increase(foxhunt_training_checkpoint_saves_total{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}[$__rate_interval])",
          "legendFormat": "saves {{model}}/{{fold}}",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "increase(foxhunt_training_checkpoint_failures_total{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}[$__rate_interval])",
          "legendFormat": "failures {{model}}/{{fold}}",
          "refId": "B"
        }
      ],
      "title": "Checkpoint Saves & Failures",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "p95 checkpoint save and data loading latency",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisPlacement": "auto",
            "drawStyle": "line",
            "fillOpacity": 15,
            "gradientMode": "opacity",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "pointSize": 5,
            "showPoints": "auto",
            "spanNulls": true,
            "stacking": {
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "decimals": 1,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          },
          "unit": "s"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 16,
        "y": 52
      },
      "id": 108,
      "options": {
        "legend": {
          "calcs": [
            "lastNotNull",
            "max",
            "min"
          ],
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "histogram_quantile(0.95, rate(foxhunt_training_checkpoint_duration_seconds_bucket{model=~\"$model\", job=~\"$source\"}[$__rate_interval]))",
          "legendFormat": "ckpt p95 {{model}}",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "histogram_quantile(0.95, rate(foxhunt_training_data_load_seconds_bucket{model=~\"$model\", job=~\"$source\"}[$__rate_interval]))",
          "legendFormat": "data load p95 {{model}}",
          "refId": "B"
        }
      ],
      "title": "Checkpoint & Data Load Latency",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Size of last saved checkpoint",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisPlacement": "auto",
            "drawStyle": "line",
            "fillOpacity": 15,
            "gradientMode": "opacity",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "pointSize": 5,
            "showPoints": "auto",
            "spanNulls": true,
            "stacking": {
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "decimals": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          },
          "unit": "bytes"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 0,
        "y": 60
      },
      "id": 109,
      "options": {
        "legend": {
          "calcs": [
            "lastNotNull",
            "max",
            "min"
          ],
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "foxhunt_training_checkpoint_size_bytes{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
          "legendFormat": "{{model}}/{{fold}}",
          "refId": "A"
        }
      ],
      "title": "Checkpoint Size",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Total batches processed this run",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisPlacement": "auto",
            "drawStyle": "line",
            "fillOpacity": 15,
            "gradientMode": "opacity",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "pointSize": 5,
            "showPoints": "auto",
            "spanNulls": true,
            "stacking": {
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "decimals": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          },
          "unit": "none"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 8,
        "y": 60
      },
      "id": 110,
      "options": {
        "legend": {
          "calcs": [
            "lastNotNull",
            "max",
            "min"
          ],
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "foxhunt_training_batches_processed{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
          "legendFormat": "{{model}}/{{fold}}",
          "refId": "A"
        }
      ],
      "title": "Batches Processed",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Last epoch wall time",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisPlacement": "auto",
            "drawStyle": "line",
            "fillOpacity": 15,
            "gradientMode": "opacity",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "pointSize": 5,
            "showPoints": "auto",
            "spanNulls": true,
            "stacking": {
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "decimals": 1,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          },
          "unit": "s"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 16,
        "y": 60
      },
      "id": 111,
      "options": {
        "legend": {
          "calcs": [
            "lastNotNull",
            "max",
            "min"
          ],
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "foxhunt_training_iteration_seconds{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
          "legendFormat": "{{model}}/{{fold}}",
          "refId": "A"
        }
      ],
      "title": "Iteration Time",
      "type": "timeseries"
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 68
      },
      "id": 112,
      "panels": [],
      "title": "Epoch Returns",
      "type": "row"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Epoch total return (fractional)",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisPlacement": "auto",
            "drawStyle": "line",
            "fillOpacity": 15,
            "gradientMode": "opacity",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "pointSize": 5,
            "showPoints": "auto",
            "spanNulls": true,
            "stacking": {
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "decimals": 2,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          },
          "unit": "percentunit"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 69
      },
      "id": 113,
      "options": {
        "legend": {
          "calcs": [
            "lastNotNull",
            "max",
            "min"
          ],
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "foxhunt_training_epoch_total_return{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
          "legendFormat": "{{model}}/{{fold}}",
          "refId": "A"
        }
      ],
      "title": "Total Return per Epoch",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Average return per trade",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisPlacement": "auto",
            "drawStyle": "line",
            "fillOpacity": 15,
            "gradientMode": "opacity",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "pointSize": 5,
            "showPoints": "auto",
            "spanNulls": true,
            "stacking": {
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "decimals": 4,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          },
          "unit": "none"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 69
      },
      "id": 114,
      "options": {
        "legend": {
          "calcs": [
            "lastNotNull",
            "max",
            "min"
          ],
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "foxhunt_training_epoch_avg_return{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
          "legendFormat": "{{model}}/{{fold}}",
          "refId": "A"
        }
      ],
      "title": "Avg Return per Trade",
      "type": "timeseries"
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 77
      },
      "id": 115,
      "panels": [],
      "title": "Supervised Model Eval",
      "type": "row"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Supervised eval accuracy and F1 score",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisPlacement": "auto",
            "drawStyle": "line",
            "fillOpacity": 15,
            "gradientMode": "opacity",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "pointSize": 5,
            "showPoints": "auto",
            "spanNulls": true,
            "stacking": {
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "decimals": 2,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          },
          "unit": "percentunit"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 78
      },
      "id": 116,
      "options": {
        "legend": {
          "calcs": [
            "lastNotNull",
            "max",
            "min"
          ],
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "foxhunt_training_eval_accuracy{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
          "legendFormat": "accuracy {{model}}/{{fold}}",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "foxhunt_training_eval_f1{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
          "legendFormat": "F1 {{model}}/{{fold}}",
          "refId": "B"
        }
      ],
      "title": "Accuracy & F1",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Supervised eval precision and recall",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisPlacement": "auto",
            "drawStyle": "line",
            "fillOpacity": 15,
            "gradientMode": "opacity",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "pointSize": 5,
            "showPoints": "auto",
            "spanNulls": true,
            "stacking": {
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "decimals": 2,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          },
          "unit": "percentunit"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 78
      },
      "id": 117,
      "options": {
        "legend": {
          "calcs": [
            "lastNotNull",
            "max",
            "min"
          ],
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "foxhunt_training_eval_precision{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
          "legendFormat": "precision {{model}}/{{fold}}",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "foxhunt_training_eval_recall{model=~\"$model\", fold=~\"$fold\", job=~\"$source\"}",
          "legendFormat": "recall {{model}}/{{fold}}",
          "refId": "B"
        }
      ],
      "title": "Precision & Recall",
      "type": "timeseries"
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 86
      },
      "id": 118,
      "panels": [],
      "title": "Hyperopt Details",
      "type": "row"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "1 = hyperopt active, 0 = training only",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "decimals": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "blue",
                "value": null
              },
              {
                "color": "green",
                "value": 1
              }
            ]
          },
          "unit": "none"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 5,
        "w": 4,
        "x": 0,
        "y": 87
      },
      "id": 119,
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "center",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "value"
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "foxhunt_hyperopt_mode{model=~\"$model\", job=~\"$source\"}",
          "legendFormat": "{{model}}",
          "refId": "A"
        }
      ],
      "title": "Hyperopt Mode",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Current epoch within the active trial",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisPlacement": "auto",
            "drawStyle": "line",
            "fillOpacity": 15,
            "gradientMode": "opacity",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "pointSize": 5,
            "showPoints": "auto",
            "spanNulls": true,
            "stacking": {
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "decimals": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          },
          "unit": "none"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 10,
        "x": 4,
        "y": 87
      },
      "id": 120,
      "options": {
        "legend": {
          "calcs": [
            "lastNotNull",
            "max",
            "min"
          ],
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "foxhunt_hyperopt_trial_epoch{model=~\"$model\", job=~\"$source\"}",
          "legendFormat": "{{model}}",
          "refId": "A"
        }
      ],
      "title": "Hyperopt Trial Epoch",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Cumulative failed trial count",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisPlacement": "auto",
            "drawStyle": "line",
            "fillOpacity": 15,
            "gradientMode": "opacity",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "pointSize": 5,
            "showPoints": "auto",
            "spanNulls": true,
            "stacking": {
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "decimals": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 3
              }
            ]
          },
          "unit": "none"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 10,
        "x": 14,
        "y": 87
      },
      "id": 121,
      "options": {
        "legend": {
          "calcs": [
            "lastNotNull",
            "max",
            "min"
          ],
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "foxhunt_hyperopt_trials_failed_total{model=~\"$model\", job=~\"$source\"}",
          "legendFormat": "{{model}}",
          "refId": "A"
        }
      ],
      "title": "Hyperopt Failed Trials",
      "type": "timeseries"
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 95
      },
      "id": 50,
      "panels": [],
      "title": "GPU Utilization",
      "type": "row"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "continuous-GrYlRd"
          },
          "custom": {
            "axisLabel": "Util %",
            "drawStyle": "line",
            "fillOpacity": 20,
            "gradientMode": "opacity",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "pointSize": 4,
            "showPoints": "never"
          },
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "red",
                "value": null
              },
              {
                "color": "yellow",
                "value": 50
              },
              {
                "color": "green",
                "value": 80
              }
            ]
          },
          "unit": "percent"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 6,
        "x": 0,
        "y": 96
      },
      "id": 51,
      "options": {
        "legend": {
          "calcs": [
            "mean",
            "max"
          ],
          "displayMode": "table",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "DCGM_FI_DEV_GPU_UTIL",
          "legendFormat": "GPU {{gpu}}",
          "refId": "A"
        }
      ],
      "title": "GPU Utilization %",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "MiB",
            "drawStyle": "line",
            "fillOpacity": 15,
            "gradientMode": "opacity",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "pointSize": 4,
            "showPoints": "never"
          },
          "unit": "decmbytes"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byRegexp",
              "options": "free.*"
            },
            "properties": [
              {
                "id": "custom.fillOpacity",
                "value": 5
              },
              {
                "id": "custom.lineStyle",
                "value": {
                  "dash": [
                    10,
                    5
                  ],
                  "fill": "dash"
                }
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 7,
        "w": 6,
        "x": 6,
        "y": 96
      },
      "id": 52,
      "options": {
        "legend": {
          "calcs": [
            "lastNotNull"
          ],
          "displayMode": "table",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "DCGM_FI_DEV_FB_USED",
          "legendFormat": "used GPU {{gpu}}",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "DCGM_FI_DEV_FB_FREE",
          "legendFormat": "free GPU {{gpu}}",
          "refId": "B"
        }
      ],
      "title": "VRAM Used / Free (MiB)",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "Temp",
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "opacity",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "pointSize": 4,
            "showPoints": "never"
          },
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "yellow",
                "value": 75
              },
              {
                "color": "red",
                "value": 85
              }
            ]
          },
          "unit": "celsius"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 4,
        "x": 12,
        "y": 96
      },
      "id": 53,
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "DCGM_FI_DEV_GPU_TEMP",
          "legendFormat": "GPU temp {{gpu}}",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "DCGM_FI_DEV_MEMORY_TEMP",
          "legendFormat": "mem temp {{gpu}}",
          "refId": "B"
        }
      ],
      "title": "GPU / Memory Temperature",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "Watts",
            "drawStyle": "line",
            "fillOpacity": 15,
            "gradientMode": "opacity",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "pointSize": 4,
            "showPoints": "never"
          },
          "unit": "watt"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 4,
        "x": 16,
        "y": 96
      },
      "id": 54,
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "DCGM_FI_DEV_POWER_USAGE",
          "legendFormat": "GPU {{gpu}}",
          "refId": "A"
        }
      ],
      "title": "Power Usage (W)",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "Activity",
            "drawStyle": "line",
            "fillOpacity": 15,
            "gradientMode": "opacity",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "pointSize": 4,
            "showPoints": "never"
          },
          "unit": "percentunit"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 4,
        "x": 20,
        "y": 96
      },
      "id": 55,
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "DCGM_FI_PROF_PIPE_TENSOR_ACTIVE",
          "legendFormat": "tensor {{gpu}}",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "DCGM_FI_PROF_DRAM_ACTIVE",
          "legendFormat": "DRAM {{gpu}}",
          "refId": "B"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "DCGM_FI_PROF_GR_ENGINE_ACTIVE",
          "legendFormat": "engine {{gpu}}",
          "refId": "C"
        }
      ],
      "title": "Tensor Core & DRAM Activity",
      "type": "timeseries"
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 103
      },
      "id": 60,
      "panels": [],
      "title": "Argo CI/CD",
      "type": "row"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "blue",
                "value": null
              }
            ]
          }
        },
        "overrides": [
          {
            "matcher": {
              "id": "byRegexp",
              "options": "Running"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "green",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byRegexp",
              "options": "Pending"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "yellow",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byRegexp",
              "options": "Failed|Error"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "red",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byRegexp",
              "options": "Succeeded"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "blue",
                  "mode": "fixed"
                }
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 5,
        "w": 6,
        "x": 0,
        "y": 104
      },
      "id": 61,
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "center",
        "orientation": "horizontal",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "auto"
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "argo_workflows_gauge{status=\"Running\"}",
          "legendFormat": "Running",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "argo_workflows_gauge{status=\"Pending\"}",
          "legendFormat": "Pending",
          "refId": "B"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "argo_workflows_gauge{status=\"Failed\"}",
          "legendFormat": "Failed",
          "refId": "C"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "argo_workflows_gauge{status=\"Succeeded\"}",
          "legendFormat": "Succeeded",
          "refId": "D"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "argo_workflows_gauge{status=\"Error\"}",
          "legendFormat": "Error",
          "refId": "E"
        }
      ],
      "title": "Workflows by Status",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "Pods",
            "drawStyle": "line",
            "fillOpacity": 20,
            "gradientMode": "opacity",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "pointSize": 4,
            "showPoints": "never"
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 5,
        "w": 4,
        "x": 6,
        "y": 104
      },
      "id": 62,
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "argo_workflows_pods_gauge",
          "legendFormat": "pods",
          "refId": "A"
        }
      ],
      "title": "Workflow Pods",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "fixedColor": "red",
            "mode": "fixed"
          },
          "custom": {
            "axisLabel": "Errors/s",
            "drawStyle": "bars",
            "fillOpacity": 50,
            "gradientMode": "opacity",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "pointSize": 4,
            "showPoints": "never"
          },
          "unit": "ops"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 5,
        "w": 4,
        "x": 10,
        "y": 104
      },
      "id": 63,
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "rate(argo_workflows_error_count[$__rate_interval])",
          "legendFormat": "errors/s",
          "refId": "A"
        }
      ],
      "title": "Error Rate",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "Duration",
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "opacity",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "pointSize": 4,
            "showPoints": "never"
          },
          "unit": "s"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 5,
        "w": 5,
        "x": 14,
        "y": 104
      },
      "id": 64,
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "histogram_quantile(0.50, rate(argo_workflows_operation_duration_seconds_bucket[$__rate_interval]))",
          "legendFormat": "p50",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "histogram_quantile(0.95, rate(argo_workflows_operation_duration_seconds_bucket[$__rate_interval]))",
          "legendFormat": "p95",
          "refId": "B"
        }
      ],
      "title": "Operation Duration (p50/p95)",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "Triggers/s",
            "drawStyle": "bars",
            "fillOpacity": 40,
            "gradientMode": "opacity",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "pointSize": 4,
            "showPoints": "never",
            "stacking": {
              "mode": "normal"
            }
          },
          "unit": "ops"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 5,
        "w": 5,
        "x": 19,
        "y": 104
      },
      "id": 65,
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "expr": "rate(argo_workflows_workflowtemplate_triggered_total[$__rate_interval])",
          "legendFormat": "triggers/s",
          "refId": "A"
        }
      ],
      "title": "Template Trigger Rate",
      "type": "timeseries"
    }
  ],
  "refresh": "15s",
  "schemaVersion": 39,
  "style": "dark",
  "tags": [
    "foxhunt",
    "training",
    "gpu",
    "ci"
  ],
  "templating": {
    "list": [
      {
        "auto": true,
        "auto_count": 30,
        "auto_min": "1m",
        "current": {
          "selected": true,
          "text": "auto",
          "value": "$__auto_interval_interval"
        },
        "label": "Interval",
        "name": "interval",
        "options": [
          {
            "selected": false,
            "text": "1m",
            "value": "1m"
          },
          {
            "selected": false,
            "text": "5m",
            "value": "5m"
          },
          {
            "selected": false,
            "text": "15m",
            "value": "15m"
          },
          {
            "selected": false,
            "text": "30m",
            "value": "30m"
          },
          {
            "selected": false,
            "text": "1h",
            "value": "1h"
          }
        ],
        "query": "1m,5m,15m,30m,1h",
        "type": "interval"
      },
      {
        "current": {
          "selected": true,
          "text": "Live",
          "value": "training-pods"
        },
        "description": "Live = running training pods, CI History = completed CI runs (pushgateway), All = both",
        "label": "Source",
        "name": "source",
        "options": [
          {
            "selected": true,
            "text": "Live",
            "value": "training-pods"
          },
          {
            "selected": false,
            "text": "CI History",
            "value": ".+_baseline.*"
          },
          {
            "selected": false,
            "text": "All",
            "value": "training-pods|.+_baseline.*"
          }
        ],
        "query": "Live : training-pods, CI History : .+_baseline.*, All : training-pods|.+_baseline.*",
        "type": "custom"
      },
      {
        "allValue": ".*",
        "current": {
          "selected": true,
          "text": "All",
          "value": "$__all"
        },
        "datasource": {
          "type": "prometheus",
          "uid": "${DS_PROMETHEUS}"
        },
        "includeAll": true,
        "label": "Model",
        "multi": true,
        "name": "model",
        "query": "label_values(foxhunt_training_step{job=~\"$source\"}, model)",
        "refresh": 2,
        "sort": 1,
        "type": "query"
      },
      {
        "allValue": ".*",
        "current": {
          "selected": true,
          "text": "All",
          "value": "$__all"
        },
        "datasource": {
          "type": "prometheus",
          "uid": "${DS_PROMETHEUS}"
        },
        "includeAll": true,
        "label": "Fold",
        "multi": true,
        "name": "fold",
        "query": "label_values(foxhunt_training_step{job=~\"$source\"}, fold)",
        "refresh": 2,
        "sort": 1,
        "type": "query"
      }
    ]
  },
  "time": {
    "from": "now-6h",
    "to": "now"
  },
  "timepicker": {},
  "timezone": "browser",
  "title": "Training & CI/CD",
  "uid": "foxhunt-training",
  "__inputs": [
    {
      "name": "DS_PROMETHEUS",
      "type": "datasource",
      "pluginId": "prometheus"
    },
    {
      "name": "DS_LOKI",
      "type": "datasource",
      "pluginId": "loki"
    },
    {
      "name": "DS_TEMPO",
      "type": "datasource",
      "pluginId": "${DS_TEMPO}"
    }
  ]
}
