{
  "annotations": {
    "list": [
      {
        "datasource": {
          "type": "loki",
          "uid": "${DS_LOKI}"
        },
        "enable": true,
        "expr": "{namespace=\"foxhunt\", pod=~\"$pod\", level=\"ERROR\"}",
        "iconColor": "red",
        "name": "Errors",
        "tagKeys": "level",
        "titleFormat": "Error"
      },
      {
        "datasource": {
          "type": "prometheus",
          "uid": "${DS_PROMETHEUS}"
        },
        "enable": true,
        "expr": "changes(foxhunt_training_checkpoint_saves_total{pod=~\"$pod\", namespace=\"foxhunt\"}[1m]) > 0",
        "iconColor": "blue",
        "name": "Checkpoints",
        "titleFormat": "Checkpoint Saved"
      }
    ]
  },
  "description": "Unified training pipeline monitor \u2014 target a running pod to see training curves, GPU health, eval metrics, hyperopt progress, container resources, and live logs in one place.",
  "editable": true,
  "fiscalYearStartMonth": 0,
  "links": [
    {
      "title": "CI Pipeline Logs",
      "type": "link",
      "url": "/d/foxhunt-ci-logs/foxhunt-ci-pipeline-logs"
    },
    {
      "title": "Training Logs",
      "type": "link",
      "url": "/d/foxhunt-training-logs/foxhunt-training-pipeline-logs"
    },
    {
      "title": "Training & CI/CD",
      "type": "link",
      "url": "/d/foxhunt-training/foxhunt-training-and-ci-cd"
    },
    {
      "title": "Operations Cockpit",
      "type": "link",
      "url": "/d/foxhunt-cockpit/foxhunt-operations-cockpit"
    }
  ],
  "liveNow": true,
  "panels": [
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 0
      },
      "id": 1,
      "panels": [],
      "title": "Job Overview",
      "type": "row"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "blue",
                "value": null
              }
            ]
          },
          "unit": "none"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 4,
        "w": 3,
        "x": 0,
        "y": 1
      },
      "id": 2,
      "options": {
        "colorMode": "background",
        "graphMode": "area",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ]
        },
        "textMode": "auto"
      },
      "targets": [
        {
          "expr": "foxhunt_training_current_epoch{pod=~\"$pod\", namespace=\"foxhunt\"}",
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "title": "Epoch",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "blue",
                "value": null
              }
            ]
          },
          "unit": "none"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 4,
        "w": 3,
        "x": 3,
        "y": 1
      },
      "id": 3,
      "options": {
        "colorMode": "background",
        "graphMode": "area",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ]
        },
        "textMode": "auto"
      },
      "targets": [
        {
          "expr": "foxhunt_training_step{pod=~\"$pod\", namespace=\"foxhunt\"}",
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "title": "Step",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "yellow",
                "value": 0.5
              },
              {
                "color": "red",
                "value": 1
              }
            ]
          },
          "unit": "none"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 4,
        "w": 3,
        "x": 6,
        "y": 1
      },
      "id": 4,
      "options": {
        "colorMode": "background",
        "graphMode": "area",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ]
        },
        "textMode": "auto"
      },
      "targets": [
        {
          "expr": "foxhunt_training_epoch_loss{pod=~\"$pod\", namespace=\"foxhunt\"}",
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "title": "Loss",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "yellow",
                "value": 0.5
              },
              {
                "color": "red",
                "value": 1
              }
            ]
          },
          "unit": "none"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 4,
        "w": 3,
        "x": 9,
        "y": 1
      },
      "id": 5,
      "options": {
        "colorMode": "background",
        "graphMode": "area",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ]
        },
        "textMode": "auto"
      },
      "targets": [
        {
          "expr": "foxhunt_training_validation_loss{pod=~\"$pod\", namespace=\"foxhunt\"}",
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "title": "Val Loss",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "red",
                "value": null
              },
              {
                "color": "yellow",
                "value": 0.5
              },
              {
                "color": "green",
                "value": 1
              }
            ]
          },
          "unit": "none"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 4,
        "w": 3,
        "x": 12,
        "y": 1
      },
      "id": 6,
      "options": {
        "colorMode": "background",
        "graphMode": "area",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ]
        },
        "textMode": "auto"
      },
      "targets": [
        {
          "expr": "foxhunt_training_epoch_sharpe{pod=~\"$pod\", namespace=\"foxhunt\"}",
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "title": "Sharpe",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "yellow",
                "value": 0.15
              },
              {
                "color": "red",
                "value": 0.3
              }
            ]
          },
          "unit": "percentunit"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 4,
        "w": 3,
        "x": 15,
        "y": 1
      },
      "id": 7,
      "options": {
        "colorMode": "background",
        "graphMode": "area",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ]
        },
        "textMode": "auto"
      },
      "targets": [
        {
          "expr": "foxhunt_training_epoch_max_drawdown{pod=~\"$pod\", namespace=\"foxhunt\"}",
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "title": "MaxDD",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "max": 100,
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "yellow",
                "value": 70
              },
              {
                "color": "red",
                "value": 90
              }
            ]
          },
          "unit": "percent"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 4,
        "w": 3,
        "x": 18,
        "y": 1
      },
      "id": 8,
      "options": {
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ]
        },
        "showThresholdLabels": false,
        "showThresholdMarkers": true
      },
      "targets": [
        {
          "expr": "DCGM_FI_DEV_GPU_UTIL{pod=~\"$pod\", namespace=\"foxhunt\"}",
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "title": "GPU Util",
      "type": "gauge"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "max": 100,
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "yellow",
                "value": 70
              },
              {
                "color": "red",
                "value": 90
              }
            ]
          },
          "unit": "percent"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 4,
        "w": 3,
        "x": 21,
        "y": 1
      },
      "id": 9,
      "options": {
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ]
        },
        "showThresholdLabels": false,
        "showThresholdMarkers": true
      },
      "targets": [
        {
          "expr": "DCGM_FI_DEV_FB_USED{pod=~\"$pod\", namespace=\"foxhunt\"} / (DCGM_FI_DEV_FB_USED{pod=~\"$pod\", namespace=\"foxhunt\"} + DCGM_FI_DEV_FB_FREE{pod=~\"$pod\", namespace=\"foxhunt\"}) * 100",
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "title": "VRAM Used",
      "type": "gauge"
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 5
      },
      "id": 10,
      "panels": [],
      "title": "CI Pipeline & Argo Workflows",
      "type": "row"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Currently running Argo workflows",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "blue",
                "value": null
              }
            ]
          },
          "unit": "none"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 4,
        "w": 3,
        "x": 0,
        "y": 6
      },
      "id": 11,
      "options": {
        "colorMode": "background",
        "graphMode": "area",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ]
        },
        "textMode": "auto"
      },
      "targets": [
        {
          "expr": "argo_workflows_gauge{status=\"Running\"}",
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "title": "Active Workflows",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "yellow",
                "value": 3
              },
              {
                "color": "red",
                "value": 10
              }
            ]
          },
          "unit": "none"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 4,
        "w": 3,
        "x": 3,
        "y": 6
      },
      "id": 12,
      "options": {
        "colorMode": "background",
        "graphMode": "area",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ]
        },
        "textMode": "auto"
      },
      "targets": [
        {
          "expr": "argo_workflows_pod_pending_count",
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "title": "Pending Pods",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          },
          "unit": "none"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 4,
        "w": 3,
        "x": 6,
        "y": 6
      },
      "id": 13,
      "options": {
        "colorMode": "background",
        "graphMode": "area",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ]
        },
        "textMode": "auto"
      },
      "targets": [
        {
          "expr": "argo_workflows_total_count{status=\"Succeeded\"}",
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "title": "Total Completed",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 1
              }
            ]
          },
          "unit": "none"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 4,
        "w": 3,
        "x": 9,
        "y": 6
      },
      "id": 14,
      "options": {
        "colorMode": "background",
        "graphMode": "area",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ]
        },
        "textMode": "auto"
      },
      "targets": [
        {
          "expr": "argo_workflows_error_count",
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "title": "Errors",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Average workflow operation duration",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "fillOpacity": 0,
            "lineWidth": 1,
            "spanNulls": true,
            "stacking": {
              "mode": "none"
            }
          },
          "unit": "s"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 6,
        "x": 12,
        "y": 6
      },
      "id": 15,
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "rate(argo_workflows_operation_duration_seconds_sum[5m]) / rate(argo_workflows_operation_duration_seconds_count[5m])",
          "legendFormat": "Avg Duration",
          "refId": "A"
        }
      ],
      "title": "Workflow Duration",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Argo controller K8s API calls \u2014 high rates may indicate thrashing",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "fillOpacity": 0,
            "lineWidth": 1,
            "spanNulls": true,
            "stacking": {
              "mode": "none"
            }
          },
          "unit": "none"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 6,
        "x": 18,
        "y": 6
      },
      "id": 16,
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "rate(argo_workflows_k8s_request_total[5m])",
          "legendFormat": "{{verb}} {{status_code}}",
          "refId": "A"
        }
      ],
      "title": "K8s API Request Rate",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "CPU usage of selected CI/training pods",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "fillOpacity": 0,
            "lineWidth": 1,
            "spanNulls": true,
            "stacking": {
              "mode": "none"
            }
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 6,
        "x": 0,
        "y": 10
      },
      "id": 17,
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "rate(container_cpu_usage_seconds_total{namespace=\"foxhunt\", pod=~\"$pod\"}[5m])",
          "legendFormat": "{{container}}",
          "refId": "A"
        }
      ],
      "title": "CI Pod CPU",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "RSS memory of selected CI/training pods",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "fillOpacity": 0,
            "lineWidth": 1,
            "spanNulls": true,
            "stacking": {
              "mode": "none"
            }
          },
          "unit": "bytes"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 6,
        "x": 6,
        "y": 10
      },
      "id": 18,
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "container_memory_rss{namespace=\"foxhunt\", pod=~\"$pod\"}",
          "legendFormat": "{{container}}",
          "refId": "A"
        }
      ],
      "title": "CI Pod Memory",
      "type": "timeseries"
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 17
      },
      "id": 19,
      "panels": [],
      "title": "CI Logs",
      "type": "row"
    },
    {
      "datasource": {
        "type": "loki",
        "uid": "${DS_LOKI}"
      },
      "description": "All logs from selected CI/training pods",
      "gridPos": {
        "h": 9,
        "w": 12,
        "x": 0,
        "y": 18
      },
      "id": 20,
      "options": {
        "dedupStrategy": "none",
        "enableLogDetails": true,
        "prettifyLogMessage": false,
        "showCommonLabels": false,
        "showLabels": false,
        "showTime": true,
        "sortOrder": "Descending",
        "wrapLogMessage": true
      },
      "targets": [
        {
          "expr": "{namespace=\"foxhunt\", pod=~\"$pod\"} |~ \"$search\" | pattern `<_> <_> <_> <content>` | regexp `msg=\"(?P<clean_msg>[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg",
          "refId": "A"
        }
      ],
      "title": "CI Full Log Stream",
      "type": "logs"
    },
    {
      "datasource": {
        "type": "loki",
        "uid": "${DS_LOKI}"
      },
      "description": "Rust compile output, test results, clippy warnings",
      "gridPos": {
        "h": 9,
        "w": 12,
        "x": 12,
        "y": 18
      },
      "id": 21,
      "options": {
        "dedupStrategy": "none",
        "enableLogDetails": true,
        "prettifyLogMessage": false,
        "showCommonLabels": false,
        "showLabels": false,
        "showTime": true,
        "sortOrder": "Descending",
        "wrapLogMessage": true
      },
      "targets": [
        {
          "expr": "{namespace=\"foxhunt\", pod=~\"$pod\"} |~ \"(?i)(cargo (build|test|check|clippy)|Compiling|Finished|running \\\\d+ test|test result|FAILED|passed|warning:)\" | pattern `<_> <_> <_> <content>` | regexp `msg=\"(?P<clean_msg>[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg",
          "refId": "A"
        }
      ],
      "title": "Compile & Test Results",
      "type": "logs"
    },
    {
      "datasource": {
        "type": "loki",
        "uid": "${DS_LOKI}"
      },
      "description": "S3 binary uploads, kubectl deploys, image builds",
      "gridPos": {
        "h": 9,
        "w": 12,
        "x": 0,
        "y": 27
      },
      "id": 22,
      "options": {
        "dedupStrategy": "none",
        "enableLogDetails": true,
        "prettifyLogMessage": false,
        "showCommonLabels": false,
        "showLabels": false,
        "showTime": true,
        "sortOrder": "Descending",
        "wrapLogMessage": true
      },
      "targets": [
        {
          "expr": "{namespace=\"foxhunt\", pod=~\"$pod\"} |~ \"(?i)(rclone|s3://|upload|download|kubectl|rollout|image|docker|kaniko|configmap)\" | pattern `<_> <_> <_> <content>` | regexp `msg=\"(?P<clean_msg>[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg",
          "refId": "A"
        }
      ],
      "title": "Build Artifacts & Deploy",
      "type": "logs"
    },
    {
      "datasource": {
        "type": "loki",
        "uid": "${DS_LOKI}"
      },
      "description": "Data fetching, redis cache, dbn file operations",
      "gridPos": {
        "h": 9,
        "w": 12,
        "x": 12,
        "y": 27
      },
      "id": 23,
      "options": {
        "dedupStrategy": "none",
        "enableLogDetails": true,
        "prettifyLogMessage": false,
        "showCommonLabels": false,
        "showLabels": false,
        "showTime": true,
        "sortOrder": "Descending",
        "wrapLogMessage": true
      },
      "targets": [
        {
          "expr": "{namespace=\"foxhunt\", pod=~\"$pod\"} |~ \"(?i)(redis|rclone|dbn|data|cache|download|decompress)\" | pattern `<_> <_> <_> <content>` | regexp `msg=\"(?P<clean_msg>[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg",
          "refId": "A"
        }
      ],
      "title": "Redis Sidecar & Data",
      "type": "logs"
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 36
      },
      "id": 24,
      "panels": [],
      "title": "Training Curves",
      "type": "row"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Training and validation loss over time",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "fillOpacity": 0,
            "lineWidth": 1,
            "spanNulls": true,
            "stacking": {
              "mode": "none"
            }
          },
          "unit": "none"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 8,
        "x": 0,
        "y": 37
      },
      "id": 25,
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "foxhunt_training_epoch_loss{pod=~\"$pod\", namespace=\"foxhunt\"}",
          "legendFormat": "Train Loss",
          "refId": "A"
        },
        {
          "expr": "foxhunt_training_validation_loss{pod=~\"$pod\", namespace=\"foxhunt\"}",
          "legendFormat": "Val Loss",
          "refId": "B"
        }
      ],
      "title": "Loss & Validation Loss",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Learning rate schedule",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "fillOpacity": 0,
            "lineWidth": 1,
            "spanNulls": true,
            "stacking": {
              "mode": "none"
            }
          },
          "unit": "none"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 8,
        "x": 8,
        "y": 37
      },
      "id": 26,
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "foxhunt_training_learning_rate{pod=~\"$pod\", namespace=\"foxhunt\"}",
          "legendFormat": "LR",
          "refId": "A"
        }
      ],
      "title": "Learning Rate",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Gradient L2 norm \u2014 spikes indicate instability",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "fillOpacity": 0,
            "lineWidth": 1,
            "spanNulls": true,
            "stacking": {
              "mode": "none"
            }
          },
          "unit": "none"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 8,
        "x": 16,
        "y": 37
      },
      "id": 27,
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "foxhunt_training_gradient_norm{pod=~\"$pod\", namespace=\"foxhunt\"}",
          "legendFormat": "Grad Norm",
          "refId": "A"
        }
      ],
      "title": "Gradient Norm",
      "type": "timeseries"
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 44
      },
      "id": 28,
      "panels": [],
      "title": "Trading Performance",
      "type": "row"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Risk-adjusted return metrics per epoch",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "fillOpacity": 0,
            "lineWidth": 1,
            "spanNulls": true,
            "stacking": {
              "mode": "none"
            }
          },
          "unit": "none"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 8,
        "x": 0,
        "y": 45
      },
      "id": 29,
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "foxhunt_training_epoch_sharpe{pod=~\"$pod\", namespace=\"foxhunt\"}",
          "legendFormat": "Sharpe",
          "refId": "A"
        },
        {
          "expr": "foxhunt_training_epoch_sortino{pod=~\"$pod\", namespace=\"foxhunt\"}",
          "legendFormat": "Sortino",
          "refId": "B"
        },
        {
          "expr": "foxhunt_training_epoch_max_drawdown{pod=~\"$pod\", namespace=\"foxhunt\"}",
          "legendFormat": "MaxDD",
          "refId": "C"
        }
      ],
      "title": "Sharpe / Sortino / MaxDD",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "fillOpacity": 0,
            "lineWidth": 1,
            "spanNulls": true,
            "stacking": {
              "mode": "none"
            }
          },
          "unit": "none"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 8,
        "x": 8,
        "y": 45
      },
      "id": 30,
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "foxhunt_training_epoch_win_rate{pod=~\"$pod\", namespace=\"foxhunt\"}",
          "legendFormat": "Win Rate",
          "refId": "A"
        },
        {
          "expr": "foxhunt_training_epoch_profit_factor{pod=~\"$pod\", namespace=\"foxhunt\"}",
          "legendFormat": "Profit Factor",
          "refId": "B"
        }
      ],
      "title": "Win Rate & Profit Factor",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "fillOpacity": 0,
            "lineWidth": 1,
            "spanNulls": true,
            "stacking": {
              "mode": "none"
            }
          },
          "unit": "none"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 8,
        "x": 16,
        "y": 45
      },
      "id": 31,
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "foxhunt_training_epoch_total_return{pod=~\"$pod\", namespace=\"foxhunt\"}",
          "legendFormat": "Total Return",
          "refId": "A"
        },
        {
          "expr": "foxhunt_training_epoch_avg_return{pod=~\"$pod\", namespace=\"foxhunt\"}",
          "legendFormat": "Avg Return",
          "refId": "B"
        }
      ],
      "title": "Total Return & Avg Return",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Action distribution per epoch \u2014 watch for collapse to single action",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "fillOpacity": 30,
            "lineWidth": 1,
            "spanNulls": true,
            "stacking": {
              "mode": "normal"
            }
          },
          "max": 1,
          "min": 0,
          "unit": "percentunit"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 8,
        "x": 0,
        "y": 52
      },
      "id": 32,
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "foxhunt_training_epoch_action_buy_pct{pod=~\"$pod\", namespace=\"foxhunt\"}",
          "legendFormat": "Buy %",
          "refId": "A"
        },
        {
          "expr": "foxhunt_training_epoch_action_hold_pct{pod=~\"$pod\", namespace=\"foxhunt\"}",
          "legendFormat": "Hold %",
          "refId": "B"
        },
        {
          "expr": "foxhunt_training_epoch_action_sell_pct{pod=~\"$pod\", namespace=\"foxhunt\"}",
          "legendFormat": "Sell %",
          "refId": "C"
        }
      ],
      "title": "Action Distribution",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Higher entropy = more exploration. Collapse \u2192 policy degeneration",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "fillOpacity": 0,
            "lineWidth": 1,
            "spanNulls": true,
            "stacking": {
              "mode": "none"
            }
          },
          "unit": "none"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 8,
        "x": 8,
        "y": 52
      },
      "id": 33,
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "foxhunt_training_action_entropy{pod=~\"$pod\", namespace=\"foxhunt\"}",
          "legendFormat": "Entropy",
          "refId": "A"
        },
        {
          "expr": "foxhunt_training_action_diversity{pod=~\"$pod\", namespace=\"foxhunt\"}",
          "legendFormat": "Diversity",
          "refId": "B"
        }
      ],
      "title": "Action Entropy & Diversity",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "fillOpacity": 0,
            "lineWidth": 1,
            "spanNulls": true,
            "stacking": {
              "mode": "none"
            }
          },
          "unit": "none"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 8,
        "x": 16,
        "y": 52
      },
      "id": 34,
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "foxhunt_training_epoch_total_trades{pod=~\"$pod\", namespace=\"foxhunt\"}",
          "legendFormat": "Trades/Epoch",
          "refId": "A"
        }
      ],
      "title": "Total Trades",
      "type": "timeseries"
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 59
      },
      "id": 35,
      "panels": [],
      "title": "Evaluation Metrics",
      "type": "row"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "fillOpacity": 0,
            "lineWidth": 1,
            "spanNulls": true,
            "stacking": {
              "mode": "none"
            }
          },
          "max": 1,
          "min": 0,
          "unit": "percentunit"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 6,
        "x": 0,
        "y": 60
      },
      "id": 36,
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "foxhunt_training_eval_accuracy{pod=~\"$pod\", namespace=\"foxhunt\"}",
          "legendFormat": "Accuracy",
          "refId": "A"
        }
      ],
      "title": "Eval Accuracy",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Classification quality of directional predictions",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "fillOpacity": 0,
            "lineWidth": 1,
            "spanNulls": true,
            "stacking": {
              "mode": "none"
            }
          },
          "max": 1,
          "min": 0,
          "unit": "percentunit"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 6,
        "x": 6,
        "y": 60
      },
      "id": 37,
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "foxhunt_training_eval_precision{pod=~\"$pod\", namespace=\"foxhunt\"}",
          "legendFormat": "Precision",
          "refId": "A"
        },
        {
          "expr": "foxhunt_training_eval_recall{pod=~\"$pod\", namespace=\"foxhunt\"}",
          "legendFormat": "Recall",
          "refId": "B"
        },
        {
          "expr": "foxhunt_training_eval_f1{pod=~\"$pod\", namespace=\"foxhunt\"}",
          "legendFormat": "F1",
          "refId": "C"
        }
      ],
      "title": "Precision / Recall / F1",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "DQN Q-value estimates \u2014 divergence from max indicates overestimation",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "fillOpacity": 0,
            "lineWidth": 1,
            "spanNulls": true,
            "stacking": {
              "mode": "none"
            }
          },
          "unit": "none"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 6,
        "x": 12,
        "y": 60
      },
      "id": 38,
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "foxhunt_training_q_value_mean{pod=~\"$pod\", namespace=\"foxhunt\"}",
          "legendFormat": "Q Mean",
          "refId": "A"
        },
        {
          "expr": "foxhunt_training_q_value_max{pod=~\"$pod\", namespace=\"foxhunt\"}",
          "legendFormat": "Q Max",
          "refId": "B"
        }
      ],
      "title": "Q-Values (DQN)",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Ratio > 1 = Q-values overestimate true returns",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "fillOpacity": 0,
            "lineWidth": 1,
            "spanNulls": true,
            "stacking": {
              "mode": "none"
            }
          },
          "unit": "none"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 6,
        "x": 18,
        "y": 60
      },
      "id": 39,
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "foxhunt_training_q_overestimation_ratio{pod=~\"$pod\", namespace=\"foxhunt\"}",
          "legendFormat": "Overestimation Ratio",
          "refId": "A"
        }
      ],
      "title": "Q-Overestimation & Replay Buffer",
      "type": "timeseries"
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 67
      },
      "id": 40,
      "panels": [],
      "title": "GPU & Hardware",
      "type": "row"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Overall GPU and graphics engine utilization",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "fillOpacity": 10,
            "lineWidth": 1,
            "spanNulls": true,
            "stacking": {
              "mode": "none"
            }
          },
          "max": 100,
          "min": 0,
          "unit": "percent"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 6,
        "x": 0,
        "y": 68
      },
      "id": 41,
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "DCGM_FI_DEV_GPU_UTIL{pod=~\"$pod\", namespace=\"foxhunt\"}",
          "legendFormat": "GPU %",
          "refId": "A"
        },
        {
          "expr": "DCGM_FI_PROF_GR_ENGINE_ACTIVE{pod=~\"$pod\", namespace=\"foxhunt\"} * 100",
          "legendFormat": "GR Engine %",
          "refId": "B"
        }
      ],
      "title": "GPU Utilization",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Tensor core and DRAM bandwidth utilization \u2014 key H100 saturation indicators",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "fillOpacity": 10,
            "lineWidth": 1,
            "spanNulls": true,
            "stacking": {
              "mode": "none"
            }
          },
          "max": 100,
          "min": 0,
          "unit": "percent"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 6,
        "x": 6,
        "y": 68
      },
      "id": 42,
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{pod=~\"$pod\", namespace=\"foxhunt\"} * 100",
          "legendFormat": "Tensor Cores %",
          "refId": "A"
        },
        {
          "expr": "DCGM_FI_PROF_DRAM_ACTIVE{pod=~\"$pod\", namespace=\"foxhunt\"} * 100",
          "legendFormat": "DRAM Active %",
          "refId": "B"
        }
      ],
      "title": "Tensor Core Activity",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "GPU framebuffer memory usage",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "fillOpacity": 20,
            "lineWidth": 1,
            "spanNulls": true,
            "stacking": {
              "mode": "normal"
            }
          },
          "unit": "decmbytes"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 6,
        "x": 12,
        "y": 68
      },
      "id": 43,
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "DCGM_FI_DEV_FB_USED{pod=~\"$pod\", namespace=\"foxhunt\"}",
          "legendFormat": "Used (MiB)",
          "refId": "A"
        },
        {
          "expr": "DCGM_FI_DEV_FB_FREE{pod=~\"$pod\", namespace=\"foxhunt\"}",
          "legendFormat": "Free (MiB)",
          "refId": "B"
        }
      ],
      "title": "VRAM (Framebuffer)",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Thermal throttling starts at ~83\u00b0C on H100",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "fillOpacity": 0,
            "lineWidth": 1,
            "spanNulls": true,
            "stacking": {
              "mode": "none"
            }
          },
          "unit": "celsius"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 6,
        "x": 18,
        "y": 68
      },
      "id": 44,
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "DCGM_FI_DEV_GPU_TEMP{pod=~\"$pod\", namespace=\"foxhunt\"}",
          "legendFormat": "GPU Temp",
          "refId": "A"
        },
        {
          "expr": "DCGM_FI_DEV_MEMORY_TEMP{pod=~\"$pod\", namespace=\"foxhunt\"}",
          "legendFormat": "Memory Temp",
          "refId": "B"
        }
      ],
      "title": "GPU Temperature",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "GPU power draw \u2014 H100 TDP is 700W",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "fillOpacity": 0,
            "lineWidth": 1,
            "spanNulls": true,
            "stacking": {
              "mode": "none"
            }
          },
          "unit": "watt"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 6,
        "x": 0,
        "y": 75
      },
      "id": 45,
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "DCGM_FI_DEV_POWER_USAGE{pod=~\"$pod\", namespace=\"foxhunt\"}",
          "legendFormat": "Power (W)",
          "refId": "A"
        }
      ],
      "title": "Power Usage",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "PCIe bandwidth \u2014 high RX during data loading, high TX during checkpoint saves",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "fillOpacity": 0,
            "lineWidth": 1,
            "spanNulls": true,
            "stacking": {
              "mode": "none"
            }
          },
          "unit": "Bps"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 6,
        "x": 6,
        "y": 75
      },
      "id": 46,
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "rate(DCGM_FI_PROF_PCIE_TX_BYTES{pod=~\"$pod\", namespace=\"foxhunt\"}[1m])",
          "legendFormat": "TX",
          "refId": "A"
        },
        {
          "expr": "rate(DCGM_FI_PROF_PCIE_RX_BYTES{pod=~\"$pod\", namespace=\"foxhunt\"}[1m])",
          "legendFormat": "RX",
          "refId": "B"
        }
      ],
      "title": "PCIe Throughput",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Clock throttling indicates thermal or power issues",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "fillOpacity": 0,
            "lineWidth": 1,
            "spanNulls": true,
            "stacking": {
              "mode": "none"
            }
          },
          "unit": "clockhertz"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 6,
        "x": 12,
        "y": 75
      },
      "id": 47,
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "DCGM_FI_DEV_SM_CLOCK{pod=~\"$pod\", namespace=\"foxhunt\"}",
          "legendFormat": "SM Clock",
          "refId": "A"
        },
        {
          "expr": "DCGM_FI_DEV_MEM_CLOCK{pod=~\"$pod\", namespace=\"foxhunt\"}",
          "legendFormat": "Mem Clock",
          "refId": "B"
        }
      ],
      "title": "SM & Memory Clock",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "fillOpacity": 10,
            "lineWidth": 1,
            "spanNulls": true,
            "stacking": {
              "mode": "none"
            }
          },
          "max": 100,
          "min": 0,
          "unit": "percent"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 6,
        "x": 18,
        "y": 75
      },
      "id": 48,
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "DCGM_FI_DEV_MEM_COPY_UTIL{pod=~\"$pod\", namespace=\"foxhunt\"}",
          "legendFormat": "Mem Copy %",
          "refId": "A"
        }
      ],
      "title": "Memory Copy Utilization",
      "type": "timeseries"
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 82
      },
      "id": 49,
      "panels": [],
      "title": "Throughput & Performance",
      "type": "row"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Training throughput \u2014 higher is better",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "fillOpacity": 0,
            "lineWidth": 1,
            "spanNulls": true,
            "stacking": {
              "mode": "none"
            }
          },
          "unit": "none"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 6,
        "x": 0,
        "y": 83
      },
      "id": 50,
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "foxhunt_training_batches_per_second{pod=~\"$pod\", namespace=\"foxhunt\"}",
          "legendFormat": "Batches/s",
          "refId": "A"
        }
      ],
      "title": "Batches / Second",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Wall-clock time per epoch",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "fillOpacity": 0,
            "lineWidth": 1,
            "spanNulls": true,
            "stacking": {
              "mode": "none"
            }
          },
          "unit": "s"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 6,
        "x": 6,
        "y": 83
      },
      "id": 51,
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "foxhunt_training_epoch_duration_seconds{pod=~\"$pod\", namespace=\"foxhunt\"}",
          "legendFormat": "Epoch Time",
          "refId": "A"
        }
      ],
      "title": "Epoch Duration",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Per-step training iteration time",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "fillOpacity": 0,
            "lineWidth": 1,
            "spanNulls": true,
            "stacking": {
              "mode": "none"
            }
          },
          "unit": "s"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 6,
        "x": 12,
        "y": 83
      },
      "id": 52,
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "foxhunt_training_iteration_seconds{pod=~\"$pod\", namespace=\"foxhunt\"}",
          "legendFormat": "Iter Time",
          "refId": "A"
        }
      ],
      "title": "Iteration Time",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {},
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 6,
        "x": 18,
        "y": 83
      },
      "id": 53,
      "options": {
        "calculate": false,
        "cellGap": 1,
        "color": {
          "mode": "scheme",
          "scheme": "Oranges"
        },
        "yAxis": {
          "unit": "s"
        }
      },
      "targets": [
        {
          "expr": "rate(foxhunt_training_data_load_seconds_bucket{pod=~\"$pod\", namespace=\"foxhunt\"}[$__rate_interval])",
          "format": "heatmap",
          "legendFormat": "{{le}}",
          "refId": "A"
        }
      ],
      "title": "Data Load Latency",
      "type": "heatmap"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Checkpoint save count over time",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "fillOpacity": 0,
            "lineWidth": 1,
            "spanNulls": true,
            "stacking": {
              "mode": "none"
            }
          },
          "unit": "none"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 6,
        "x": 0,
        "y": 90
      },
      "id": 54,
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "foxhunt_training_checkpoint_saves_total{pod=~\"$pod\", namespace=\"foxhunt\"}",
          "legendFormat": "Total Saves",
          "refId": "A"
        }
      ],
      "title": "Checkpoint Saves",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Checkpoint file size",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "fillOpacity": 0,
            "lineWidth": 1,
            "spanNulls": true,
            "stacking": {
              "mode": "none"
            }
          },
          "unit": "bytes"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 6,
        "x": 6,
        "y": 90
      },
      "id": 55,
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "foxhunt_training_checkpoint_size_bytes{pod=~\"$pod\", namespace=\"foxhunt\"}",
          "legendFormat": "Size (bytes)",
          "refId": "A"
        }
      ],
      "title": "Checkpoint Size & Duration",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "DQN/PER replay buffer population",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "fillOpacity": 0,
            "lineWidth": 1,
            "spanNulls": true,
            "stacking": {
              "mode": "none"
            }
          },
          "unit": "none"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 6,
        "x": 12,
        "y": 90
      },
      "id": 56,
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "foxhunt_training_replay_buffer_size{pod=~\"$pod\", namespace=\"foxhunt\"}",
          "legendFormat": "Buffer Size",
          "refId": "A"
        }
      ],
      "title": "Replay Buffer Size",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Parallel training workers",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "fillOpacity": 0,
            "lineWidth": 1,
            "spanNulls": true,
            "stacking": {
              "mode": "none"
            }
          },
          "unit": "none"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 6,
        "x": 18,
        "y": 90
      },
      "id": 57,
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "foxhunt_training_active_workers{pod=~\"$pod\", namespace=\"foxhunt\"}",
          "legendFormat": "Workers",
          "refId": "A"
        }
      ],
      "title": "Active Workers",
      "type": "timeseries"
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 97
      },
      "id": 58,
      "panels": [],
      "title": "Hyperopt Trials",
      "type": "row"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "purple",
                "value": null
              }
            ]
          },
          "unit": "none"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 4,
        "w": 3,
        "x": 0,
        "y": 98
      },
      "id": 59,
      "options": {
        "colorMode": "background",
        "graphMode": "area",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ]
        },
        "textMode": "auto"
      },
      "targets": [
        {
          "expr": "foxhunt_hyperopt_trial_current{pod=~\"$pod\", namespace=\"foxhunt\"}",
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "title": "Current Trial",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "purple",
                "value": null
              }
            ]
          },
          "unit": "none"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 4,
        "w": 3,
        "x": 3,
        "y": 98
      },
      "id": 60,
      "options": {
        "colorMode": "background",
        "graphMode": "area",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ]
        },
        "textMode": "auto"
      },
      "targets": [
        {
          "expr": "foxhunt_hyperopt_trial_total{pod=~\"$pod\", namespace=\"foxhunt\"}",
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "title": "Total Trials",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "yellow",
                "value": 0.5
              }
            ]
          },
          "unit": "none"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 4,
        "w": 3,
        "x": 6,
        "y": 98
      },
      "id": 61,
      "options": {
        "colorMode": "background",
        "graphMode": "area",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ]
        },
        "textMode": "auto"
      },
      "targets": [
        {
          "expr": "foxhunt_hyperopt_trial_best_loss{pod=~\"$pod\", namespace=\"foxhunt\"}",
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "title": "Best Loss",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "blue",
                "value": null
              }
            ]
          },
          "unit": "s"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 4,
        "w": 3,
        "x": 9,
        "y": 98
      },
      "id": 62,
      "options": {
        "colorMode": "background",
        "graphMode": "area",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ]
        },
        "textMode": "auto"
      },
      "targets": [
        {
          "expr": "foxhunt_hyperopt_elapsed_seconds{pod=~\"$pod\", namespace=\"foxhunt\"}",
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "title": "Elapsed",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Convergence curve \u2014 should decrease over trials",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "fillOpacity": 0,
            "lineWidth": 1,
            "spanNulls": true,
            "stacking": {
              "mode": "none"
            }
          },
          "unit": "none"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 12,
        "x": 12,
        "y": 98
      },
      "id": 63,
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "foxhunt_hyperopt_trial_best_loss{pod=~\"$pod\", namespace=\"foxhunt\"}",
          "legendFormat": "Best Loss",
          "refId": "A"
        },
        {
          "expr": "foxhunt_hyperopt_best_objective{pod=~\"$pod\", namespace=\"foxhunt\"}",
          "legendFormat": "Best Objective",
          "refId": "B"
        }
      ],
      "title": "Best Loss Over Trials",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {},
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 12,
        "x": 0,
        "y": 102
      },
      "id": 64,
      "options": {
        "calculate": false,
        "cellGap": 1,
        "color": {
          "mode": "scheme",
          "scheme": "Oranges"
        },
        "yAxis": {
          "unit": "s"
        }
      },
      "targets": [
        {
          "expr": "rate(foxhunt_hyperopt_trial_duration_seconds_bucket{pod=~\"$pod\", namespace=\"foxhunt\"}[$__rate_interval])",
          "format": "heatmap",
          "legendFormat": "{{le}}",
          "refId": "A"
        }
      ],
      "title": "Trial Duration Distribution",
      "type": "heatmap"
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 109
      },
      "id": 65,
      "panels": [],
      "title": "Container Resources",
      "type": "row"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "CPU cores consumed by each container in the pod",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "fillOpacity": 0,
            "lineWidth": 1,
            "spanNulls": true,
            "stacking": {
              "mode": "none"
            }
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 6,
        "x": 0,
        "y": 110
      },
      "id": 66,
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "rate(container_cpu_usage_seconds_total{pod=~\"$pod\", namespace=\"foxhunt\", container!=\"\"}[5m])",
          "legendFormat": "{{container}}",
          "refId": "A"
        }
      ],
      "title": "Container CPU Usage",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Resident set size \u2014 actual physical memory used",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "fillOpacity": 0,
            "lineWidth": 1,
            "spanNulls": true,
            "stacking": {
              "mode": "none"
            }
          },
          "unit": "bytes"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 6,
        "x": 6,
        "y": 110
      },
      "id": 67,
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "container_memory_rss{pod=~\"$pod\", namespace=\"foxhunt\", container!=\"\"}",
          "legendFormat": "{{container}}",
          "refId": "A"
        }
      ],
      "title": "Container Memory (RSS)",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "fillOpacity": 0,
            "lineWidth": 1,
            "spanNulls": true,
            "stacking": {
              "mode": "none"
            }
          },
          "unit": "Bps"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 6,
        "x": 12,
        "y": 110
      },
      "id": 68,
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "rate(container_network_receive_bytes_total{pod=~\"$pod\", namespace=\"foxhunt\"}[5m])",
          "legendFormat": "RX {{interface}}",
          "refId": "A"
        },
        {
          "expr": "rate(container_network_transmit_bytes_total{pod=~\"$pod\", namespace=\"foxhunt\"}[5m])",
          "legendFormat": "TX {{interface}}",
          "refId": "B"
        }
      ],
      "title": "Container Network I/O",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Non-zero restarts indicate OOM kills or crashes",
      "fieldConfig": {
        "defaults": {
          "custom": {
            "fillOpacity": 0,
            "lineWidth": 1,
            "spanNulls": true,
            "stacking": {
              "mode": "none"
            }
          },
          "unit": "none"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 6,
        "x": 18,
        "y": 110
      },
      "id": 69,
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "kube_pod_container_status_restarts_total{pod=~\"$pod\", namespace=\"foxhunt\"}",
          "legendFormat": "{{container}}",
          "refId": "A"
        }
      ],
      "title": "Pod Restarts",
      "type": "timeseries"
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 117
      },
      "id": 70,
      "panels": [],
      "title": "Live Logs",
      "type": "row"
    },
    {
      "datasource": {
        "type": "loki",
        "uid": "${DS_LOKI}"
      },
      "description": "All logs from the selected pod, filtered by level and search text",
      "gridPos": {
        "h": 9,
        "w": 12,
        "x": 0,
        "y": 118
      },
      "id": 71,
      "options": {
        "dedupStrategy": "none",
        "enableLogDetails": true,
        "prettifyLogMessage": false,
        "showCommonLabels": false,
        "showLabels": false,
        "showTime": true,
        "sortOrder": "Descending",
        "wrapLogMessage": true
      },
      "targets": [
        {
          "expr": "{namespace=\"foxhunt\", pod=~\"$pod\", level=~\"$level\"} |~ \"$search\" | pattern `<_> <_> <_> <content>` | regexp `msg=\"(?P<clean_msg>[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg",
          "refId": "A"
        }
      ],
      "title": "Full Log Stream",
      "type": "logs"
    },
    {
      "datasource": {
        "type": "loki",
        "uid": "${DS_LOKI}"
      },
      "description": "Only ERROR and WARN level logs",
      "gridPos": {
        "h": 9,
        "w": 12,
        "x": 12,
        "y": 118
      },
      "id": 72,
      "options": {
        "dedupStrategy": "none",
        "enableLogDetails": true,
        "prettifyLogMessage": false,
        "showCommonLabels": false,
        "showLabels": false,
        "showTime": true,
        "sortOrder": "Descending",
        "wrapLogMessage": true
      },
      "targets": [
        {
          "expr": "{namespace=\"foxhunt\", pod=~\"$pod\", level=~\"ERROR|WARN\"} | pattern `<_> <_> <_> <content>` | regexp `msg=\"(?P<clean_msg>[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg",
          "refId": "A"
        }
      ],
      "title": "Errors & Warnings",
      "type": "logs"
    },
    {
      "datasource": {
        "type": "loki",
        "uid": "${DS_LOKI}"
      },
      "description": "Log lines mentioning training metrics \u2014 epoch completions, loss values, Sharpe ratios",
      "gridPos": {
        "h": 9,
        "w": 12,
        "x": 0,
        "y": 127
      },
      "id": 73,
      "options": {
        "dedupStrategy": "none",
        "enableLogDetails": true,
        "prettifyLogMessage": false,
        "showCommonLabels": false,
        "showLabels": false,
        "showTime": true,
        "sortOrder": "Descending",
        "wrapLogMessage": true
      },
      "targets": [
        {
          "expr": "{namespace=\"foxhunt\", pod=~\"$pod\"} |~ \"(?i)(epoch|loss|sharpe|max_?dd|sortino|reward|return|accuracy|f1|precision|recall)\" | pattern `<_> <_> <_> <content>` | regexp `msg=\"(?P<clean_msg>[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg",
          "refId": "A"
        }
      ],
      "title": "Training Progress (Epoch/Loss/Sharpe)",
      "type": "logs"
    },
    {
      "datasource": {
        "type": "loki",
        "uid": "${DS_LOKI}"
      },
      "description": "GPU-related log lines \u2014 CUDA errors, OOM events, NCCL issues",
      "gridPos": {
        "h": 9,
        "w": 12,
        "x": 12,
        "y": 127
      },
      "id": 74,
      "options": {
        "dedupStrategy": "none",
        "enableLogDetails": true,
        "prettifyLogMessage": false,
        "showCommonLabels": false,
        "showLabels": false,
        "showTime": true,
        "sortOrder": "Descending",
        "wrapLogMessage": true
      },
      "targets": [
        {
          "expr": "{namespace=\"foxhunt\", pod=~\"$pod\"} |~ \"(?i)(cuda|gpu|vram|oom|out.of.memory|nccl|nvrtc|tensor.core|cublas)\" | pattern `<_> <_> <_> <content>` | regexp `msg=\"(?P<clean_msg>[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg",
          "refId": "A"
        }
      ],
      "title": "GPU / CUDA / OOM",
      "type": "logs"
    }
  ],
  "refresh": "10s",
  "schemaVersion": 39,
  "tags": [
    "foxhunt",
    "training",
    "gpu",
    "loki",
    "deep-dive"
  ],
  "templating": {
    "list": [
      {
        "current": {},
        "datasource": {
          "type": "loki",
          "uid": "${DS_LOKI}"
        },
        "includeAll": true,
        "label": "Pod",
        "multi": true,
        "name": "pod",
        "query": "{namespace=\"foxhunt\", pod\\!~\"(grafana|prometheus|loki|promtail|${DS_TEMPO}|redis|postgres|mattermost|tailscale|gitlab|cilium|csi|konnectivity|nvidia-gpu|argo-workflows|questdb|web-gateway|trading-service).*\"}",
        "refresh": 2,
        "regex": "",
        "sort": 2,
        "type": "query"
      },
      {
        "current": {},
        "datasource": {
          "type": "prometheus",
          "uid": "${DS_PROMETHEUS}"
        },
        "includeAll": true,
        "label": "Prometheus Job",
        "multi": false,
        "name": "prom_job",
        "query": "label_values(up{namespace=\"foxhunt\"}, job)",
        "refresh": 2,
        "regex": "/(train|hyperopt|evaluate|pushgateway|training-pods|dcgm)/",
        "sort": 1,
        "type": "query"
      },
      {
        "current": {
          "text": "All",
          "value": "$__all"
        },
        "includeAll": true,
        "label": "Log Level",
        "multi": true,
        "name": "level",
        "query": "ERROR,WARN,INFO,DEBUG,TRACE",
        "type": "custom"
      },
      {
        "current": {
          "text": "",
          "value": ""
        },
        "label": "Search",
        "name": "search",
        "type": "textbox"
      }
    ]
  },
  "time": {
    "from": "now-3h",
    "to": "now"
  },
  "timezone": "browser",
  "title": "Training Deep-Dive",
  "uid": "foxhunt-training-deep-dive",
  "__inputs": [
    {
      "name": "DS_PROMETHEUS",
      "type": "datasource",
      "pluginId": "prometheus"
    },
    {
      "name": "DS_LOKI",
      "type": "datasource",
      "pluginId": "loki"
    },
    {
      "name": "DS_TEMPO",
      "type": "datasource",
      "pluginId": "${DS_TEMPO}"
    }
  ]
}