{
  "description": "Loki-powered training log viewer \u2014 hyperopt trials, epoch progress, GPU status, errors",
  "editable": true,
  "graphTooltip": 2,
  "panels": [
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 0
      },
      "id": 1,
      "title": "Training Overview",
      "type": "row"
    },
    {
      "datasource": {
        "type": "loki",
        "uid": "${DS_LOKI}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "fixedColor": "red",
            "mode": "fixed"
          },
          "mappings": [],
          "noValue": "0",
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 1
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 4,
        "w": 4,
        "x": 0,
        "y": 1
      },
      "id": 2,
      "options": {
        "colorMode": "background",
        "graphMode": "none",
        "justifyMode": "center",
        "reduceOptions": {
          "calcs": [
            "sum"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "auto"
      },
      "targets": [
        {
          "datasource": {
            "type": "loki",
            "uid": "${DS_LOKI}"
          },
          "editorMode": "code",
          "expr": "sum(count_over_time({namespace=\"foxhunt\", pod=~\"$job-.*\", level=\"ERROR\"} [$__range]))",
          "queryType": "instant",
          "refId": "A"
        }
      ],
      "title": "Errors",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "loki",
        "uid": "${DS_LOKI}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "fixedColor": "yellow",
            "mode": "fixed"
          },
          "mappings": [],
          "noValue": "0",
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "yellow",
                "value": 1
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 4,
        "w": 4,
        "x": 4,
        "y": 1
      },
      "id": 3,
      "options": {
        "colorMode": "background",
        "graphMode": "none",
        "justifyMode": "center",
        "reduceOptions": {
          "calcs": [
            "sum"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "auto"
      },
      "targets": [
        {
          "datasource": {
            "type": "loki",
            "uid": "${DS_LOKI}"
          },
          "editorMode": "code",
          "expr": "sum(count_over_time({namespace=\"foxhunt\", pod=~\"$job-.*\", level=\"WARN\"} [$__range]))",
          "queryType": "instant",
          "refId": "A"
        }
      ],
      "title": "Warnings",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "loki",
        "uid": "${DS_LOKI}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "fixedColor": "blue",
            "mode": "fixed"
          },
          "mappings": [],
          "noValue": "0"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 4,
        "w": 4,
        "x": 8,
        "y": 1
      },
      "id": 4,
      "options": {
        "colorMode": "background",
        "graphMode": "none",
        "justifyMode": "center",
        "reduceOptions": {
          "calcs": [
            "sum"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "auto"
      },
      "targets": [
        {
          "datasource": {
            "type": "loki",
            "uid": "${DS_LOKI}"
          },
          "editorMode": "code",
          "expr": "sum(count_over_time({namespace=\"foxhunt\", pod=~\"$job-.*\"} [$__range]))",
          "queryType": "instant",
          "refId": "A"
        }
      ],
      "title": "Total Log Lines",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "loki",
        "uid": "${DS_LOKI}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisBorderShow": false,
            "axisLabel": "lines/interval",
            "drawStyle": "bars",
            "fillOpacity": 80,
            "lineWidth": 0,
            "showPoints": "never",
            "stacking": {
              "mode": "normal"
            }
          }
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "ERROR"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "red",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "WARN"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "yellow",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "INFO"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "green",
                  "mode": "fixed"
                }
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 6,
        "w": 12,
        "x": 12,
        "y": 1
      },
      "id": 5,
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "loki",
            "uid": "${DS_LOKI}"
          },
          "editorMode": "code",
          "expr": "sum by (level) (count_over_time({namespace=\"foxhunt\", pod=~\"$job-.*\"} [$__interval]))",
          "legendFormat": "{{level}}",
          "queryType": "range",
          "refId": "A"
        }
      ],
      "title": "Log Volume by Level",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "loki",
        "uid": "${DS_LOKI}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisBorderShow": false,
            "drawStyle": "bars",
            "fillOpacity": 60,
            "lineWidth": 0,
            "showPoints": "never",
            "stacking": {
              "mode": "normal"
            }
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 4,
        "w": 12,
        "x": 0,
        "y": 5
      },
      "id": 6,
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "right"
        },
        "tooltip": {
          "mode": "multi"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "loki",
            "uid": "${DS_LOKI}"
          },
          "editorMode": "code",
          "expr": "sum by (pod) (count_over_time({namespace=\"foxhunt\", pod=~\"$job-.*\"} [$__interval]))",
          "legendFormat": "{{pod}}",
          "queryType": "range",
          "refId": "A"
        }
      ],
      "title": "Log Volume by Pod",
      "type": "timeseries"
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 9
      },
      "id": 10,
      "title": "Full Training Log Stream",
      "type": "row"
    },
    {
      "datasource": {
        "type": "loki",
        "uid": "${DS_LOKI}"
      },
      "gridPos": {
        "h": 14,
        "w": 24,
        "x": 0,
        "y": 10
      },
      "id": 11,
      "options": {
        "dedupStrategy": "none",
        "enableLogDetails": true,
        "prettifyLogMessage": false,
        "showCommonLabels": false,
        "showLabels": true,
        "showTime": true,
        "sortOrder": "Descending",
        "wrapLogMessage": true
      },
      "targets": [
        {
          "datasource": {
            "type": "loki",
            "uid": "${DS_LOKI}"
          },
          "editorMode": "code",
          "expr": "{namespace=\"foxhunt\", pod=~\"$job-.*\", level=~\"$level\"} | pattern `<_> <_> <_> <content>` | regexp `msg=\"(?P<clean_msg>[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg ${search:pipe}",
          "queryType": "range",
          "refId": "A"
        }
      ],
      "title": "Training Logs \u2014 $job",
      "type": "logs"
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 24
      },
      "id": 20,
      "panels": [
        {
          "datasource": {
            "type": "loki",
            "uid": "${DS_LOKI}"
          },
          "gridPos": {
            "h": 12,
            "w": 24,
            "x": 0,
            "y": 25
          },
          "id": 21,
          "options": {
            "dedupStrategy": "none",
            "enableLogDetails": true,
            "showLabels": true,
            "showTime": true,
            "sortOrder": "Descending",
            "wrapLogMessage": true
          },
          "targets": [
            {
              "datasource": {
                "type": "loki",
                "uid": "${DS_LOKI}"
              },
              "editorMode": "code",
              "expr": "{namespace=\"foxhunt\", pod=~\"$job-.*\"} |~ \"(?i)(epoch|loss|reward|sharpe|accuracy|val_loss|train_loss|best_trial|trial|fold|window|learning_rate|lr=|MaxDD|win.?rate|drawdown)\" | pattern `<_> <_> <_> <content>` | regexp `msg=\"(?P<clean_msg>[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg",
              "queryType": "range",
              "refId": "A"
            }
          ],
          "title": "Epoch / Loss / Reward / Sharpe",
          "type": "logs"
        }
      ],
      "title": "Epoch Progress & Loss",
      "type": "row"
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 25
      },
      "id": 30,
      "panels": [
        {
          "datasource": {
            "type": "loki",
            "uid": "${DS_LOKI}"
          },
          "gridPos": {
            "h": 12,
            "w": 24,
            "x": 0,
            "y": 26
          },
          "id": 31,
          "options": {
            "dedupStrategy": "none",
            "enableLogDetails": true,
            "showLabels": true,
            "showTime": true,
            "sortOrder": "Descending",
            "wrapLogMessage": true
          },
          "targets": [
            {
              "datasource": {
                "type": "loki",
                "uid": "${DS_LOKI}"
              },
              "editorMode": "code",
              "expr": "{namespace=\"foxhunt\", pod=~\"(hyperopt|train)-.*\"} |~ \"(?i)(trial|best|cost|param|learning_rate|hidden_dim|batch_size|buffer_size|gamma|epsilon|clip|PSO|iteration|objective|optimal)\" | pattern `<_> <_> <_> <content>` | regexp `msg=\"(?P<clean_msg>[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg",
              "queryType": "range",
              "refId": "A"
            }
          ],
          "title": "Hyperopt Trial Results",
          "type": "logs"
        }
      ],
      "title": "Hyperopt Trials",
      "type": "row"
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 26
      },
      "id": 40,
      "panels": [
        {
          "datasource": {
            "type": "loki",
            "uid": "${DS_LOKI}"
          },
          "gridPos": {
            "h": 12,
            "w": 24,
            "x": 0,
            "y": 27
          },
          "id": 41,
          "options": {
            "dedupStrategy": "none",
            "enableLogDetails": true,
            "showLabels": true,
            "showTime": true,
            "sortOrder": "Descending",
            "wrapLogMessage": true
          },
          "targets": [
            {
              "datasource": {
                "type": "loki",
                "uid": "${DS_LOKI}"
              },
              "editorMode": "code",
              "expr": "{namespace=\"foxhunt\", pod=~\"$job-.*\"} |~ \"(?i)(GPU|CUDA|VRAM|cuda|gpu|vram|OOM|out.of.memory|nvrtc|tensor.core|BF16|bf16|SM|occupancy|kernel|device|nvidia)\" | pattern `<_> <_> <_> <content>` | regexp `msg=\"(?P<clean_msg>[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg",
              "queryType": "range",
              "refId": "A"
            }
          ],
          "title": "GPU / CUDA / VRAM Logs",
          "type": "logs"
        }
      ],
      "title": "GPU / CUDA / VRAM",
      "type": "row"
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 27
      },
      "id": 50,
      "panels": [
        {
          "datasource": {
            "type": "loki",
            "uid": "${DS_LOKI}"
          },
          "gridPos": {
            "h": 10,
            "w": 24,
            "x": 0,
            "y": 28
          },
          "id": 51,
          "options": {
            "dedupStrategy": "none",
            "enableLogDetails": true,
            "showLabels": true,
            "showTime": true,
            "sortOrder": "Descending",
            "wrapLogMessage": true
          },
          "targets": [
            {
              "datasource": {
                "type": "loki",
                "uid": "${DS_LOKI}"
              },
              "editorMode": "code",
              "expr": "{namespace=\"foxhunt\", pod=~\"$job-.*\"} |~ \"(?i)(walk.?forward|fold|backtest|window|train_start|train_end|val_start|val_end|test_start|test_end|embargo|purge|bars|features)\" | pattern `<_> <_> <_> <content>` | regexp `msg=\"(?P<clean_msg>[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg",
              "queryType": "range",
              "refId": "A"
            }
          ],
          "title": "Walk-Forward / Fold / Backtest",
          "type": "logs"
        }
      ],
      "title": "Walk-Forward Windows",
      "type": "row"
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 28
      },
      "id": 60,
      "panels": [
        {
          "datasource": {
            "type": "loki",
            "uid": "${DS_LOKI}"
          },
          "gridPos": {
            "h": 12,
            "w": 24,
            "x": 0,
            "y": 29
          },
          "id": 61,
          "options": {
            "dedupStrategy": "none",
            "enableLogDetails": true,
            "showLabels": true,
            "showTime": true,
            "sortOrder": "Descending",
            "wrapLogMessage": true
          },
          "targets": [
            {
              "datasource": {
                "type": "loki",
                "uid": "${DS_LOKI}"
              },
              "editorMode": "code",
              "expr": "{namespace=\"foxhunt\", pod=~\"$job-.*\"} |~ \"(?i)(error|failed|failure|panic|fatal|SIGABRT|SIGSEGV|thread .* panicked|OOM|NaN|inf|diverge)\" | pattern `<_> <_> <_> <content>` | regexp `msg=\"(?P<clean_msg>[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg",
              "queryType": "range",
              "refId": "A"
            }
          ],
          "title": "All Training Errors & Failures",
          "type": "logs"
        }
      ],
      "title": "Errors & Failures",
      "type": "row"
    },
    {
      "collapsed": true,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 29
      },
      "id": 70,
      "panels": [
        {
          "datasource": {
            "type": "loki",
            "uid": "${DS_LOKI}"
          },
          "gridPos": {
            "h": 10,
            "w": 24,
            "x": 0,
            "y": 30
          },
          "id": 71,
          "options": {
            "dedupStrategy": "none",
            "enableLogDetails": true,
            "showLabels": true,
            "showTime": true,
            "sortOrder": "Descending",
            "wrapLogMessage": true
          },
          "targets": [
            {
              "datasource": {
                "type": "loki",
                "uid": "${DS_LOKI}"
              },
              "editorMode": "code",
              "expr": "{namespace=\"foxhunt\", pod=~\"$job-.*\"} |~ \"(?i)(loading|loaded|dbn|ofi|features|bars|MBP.?10|OHLCV|symbol|normalize|norm_stats|state_dim|rclone|S3|download)\" | pattern `<_> <_> <_> <content>` | regexp `msg=\"(?P<clean_msg>[^\"]*)\"` | line_format `{{if .clean_msg}}{{.clean_msg}}{{else}}{{.content}}{{end}}` | drop content, clean_msg",
              "queryType": "range",
              "refId": "A"
            }
          ],
          "title": "Data Loading / OFI / DBN / Features",
          "type": "logs"
        }
      ],
      "title": "Data Loading & OFI",
      "type": "row"
    }
  ],
  "refresh": "30s",
  "schemaVersion": 39,
  "tags": [
    "foxhunt",
    "training",
    "loki",
    "logs",
    "gpu"
  ],
  "templating": {
    "list": [
      {
        "current": {
          "selected": true,
          "text": "All Training",
          "value": "(train|hyperopt|compile-and-train)"
        },
        "label": "Job Type",
        "name": "job",
        "options": [
          {
            "selected": true,
            "text": "All Training",
            "value": "(train|hyperopt|compile-and-train)"
          },
          {
            "selected": false,
            "text": "DQN",
            "value": "train-dqn"
          },
          {
            "selected": false,
            "text": "PPO",
            "value": "train-ppo"
          },
          {
            "selected": false,
            "text": "Supervised",
            "value": "train-supervised"
          },
          {
            "selected": false,
            "text": "Hyperopt",
            "value": "hyperopt"
          },
          {
            "selected": false,
            "text": "Compile+Train",
            "value": "compile-and-train"
          }
        ],
        "query": "All Training : (train|hyperopt|compile-and-train), DQN : train-dqn, PPO : train-ppo, Supervised : train-supervised, Hyperopt : hyperopt, Compile+Train : compile-and-train",
        "type": "custom"
      },
      {
        "allValue": ".*",
        "current": {
          "selected": true,
          "text": "All",
          "value": "$__all"
        },
        "includeAll": true,
        "label": "Level",
        "multi": true,
        "name": "level",
        "options": [
          {
            "selected": true,
            "text": "All",
            "value": "$__all"
          },
          {
            "selected": false,
            "text": "ERROR",
            "value": "ERROR"
          },
          {
            "selected": false,
            "text": "WARN",
            "value": "WARN"
          },
          {
            "selected": false,
            "text": "INFO",
            "value": "INFO"
          },
          {
            "selected": false,
            "text": "DEBUG",
            "value": "DEBUG"
          },
          {
            "selected": false,
            "text": "TRACE",
            "value": "TRACE"
          }
        ],
        "query": "ERROR,WARN,INFO,DEBUG,TRACE",
        "type": "custom"
      },
      {
        "current": {
          "selected": false,
          "text": "",
          "value": ""
        },
        "label": "Search",
        "name": "search",
        "type": "textbox"
      }
    ]
  },
  "time": {
    "from": "now-12h",
    "to": "now"
  },
  "timepicker": {},
  "timezone": "browser",
  "title": "Training Pipeline Logs",
  "uid": "foxhunt-training-logs",
  "__inputs": [
    {
      "name": "DS_PROMETHEUS",
      "type": "datasource",
      "pluginId": "prometheus"
    },
    {
      "name": "DS_LOKI",
      "type": "datasource",
      "pluginId": "loki"
    },
    {
      "name": "DS_TEMPO",
      "type": "datasource",
      "pluginId": "${DS_TEMPO}"
    }
  ]
}