From 2b77a7fbe1b60af37d58a547e3d17b7db441b5ef Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Fri, 6 Mar 2026 02:22:21 +0100 Subject: [PATCH] fix(dashboard): clamp degenerate outliers in training metrics panels Add clamp_max() to PromQL queries and hard Y-axis limits to prevent early-epoch degenerate values from blowing up panel scaling (Sharpe showing 12 instead of 1.3, Profit Factor at 30k). Run Summary: clamp_max on Best Val Loss (5), Sharpe (5), Profit Factor (20) Training Quality: clamp_max on Epoch Sharpe (5), Sortino (10), PF (20) Co-Authored-By: Claude Opus 4.6 --- .../dashboards/foxhunt-training.json | 105 ++++++++++++------ 1 file changed, 74 insertions(+), 31 deletions(-) diff --git a/infra/k8s/monitoring/dashboards/foxhunt-training.json b/infra/k8s/monitoring/dashboards/foxhunt-training.json index e827c0237..21516feb0 100644 --- a/infra/k8s/monitoring/dashboards/foxhunt-training.json +++ b/infra/k8s/monitoring/dashboards/foxhunt-training.json @@ -543,7 +543,8 @@ "axisPlacement": "auto", "thresholdsStyle": { "mode": "off" - } + }, + "axisSoftMax": 2 }, "thresholds": { "mode": "absolute", @@ -563,7 +564,9 @@ ] }, "unit": "none", - "decimals": 4 + "decimals": 4, + "min": 0, + "max": 5 }, "overrides": [] }, @@ -584,13 +587,14 @@ }, "targets": [ { - "expr": "foxhunt_training_validation_loss{model=~\"$model\", fold=~\"$fold\"}", + "expr": "clamp_max(min_over_time(foxhunt_training_validation_loss{model=~\"$model\", fold=~\"$fold\"}[$__range]), 5)", "legendFormat": "{{model}}/{{fold}}", "refId": "A", "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" - } + }, + "instant": false } ] }, @@ -628,7 +632,8 @@ "axisPlacement": "auto", "thresholdsStyle": { "mode": "off" - } + }, + "axisSoftMax": 3 }, "thresholds": { "mode": "absolute", @@ -648,7 +653,9 @@ ] }, "unit": "none", - "decimals": 2 + "decimals": 2, + "min": 0, + "max": 5 }, "overrides": [] }, @@ -669,13 +676,14 @@ }, "targets": [ { - "expr": "foxhunt_training_epoch_sharpe{model=~\"$model\", fold=~\"$fold\"}", + "expr": "clamp_max(max_over_time(foxhunt_training_epoch_sharpe{model=~\"$model\", fold=~\"$fold\"}[$__range]), 5)", "legendFormat": "{{model}}/{{fold}}", "refId": "A", "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" - } + }, + "instant": false } ] }, @@ -733,7 +741,9 @@ ] }, "unit": "percentunit", - "decimals": 1 + "decimals": 1, + "min": 0, + "max": 1 }, "overrides": [] }, @@ -754,13 +764,14 @@ }, "targets": [ { - "expr": "foxhunt_training_epoch_win_rate{model=~\"$model\", fold=~\"$fold\"}", + "expr": "max_over_time(foxhunt_training_epoch_win_rate{model=~\"$model\", fold=~\"$fold\"}[$__range])", "legendFormat": "{{model}}/{{fold}}", "refId": "A", "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" - } + }, + "instant": false } ] }, @@ -818,7 +829,9 @@ ] }, "unit": "percentunit", - "decimals": 1 + "decimals": 1, + "min": 0, + "max": 1 }, "overrides": [] }, @@ -839,13 +852,14 @@ }, "targets": [ { - "expr": "foxhunt_training_epoch_max_drawdown{model=~\"$model\", fold=~\"$fold\"}", + "expr": "min_over_time(foxhunt_training_epoch_max_drawdown{model=~\"$model\", fold=~\"$fold\"}[$__range])", "legendFormat": "{{model}}/{{fold}}", "refId": "A", "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" - } + }, + "instant": false } ] }, @@ -922,7 +936,8 @@ "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" - } + }, + "instant": false } ] }, @@ -960,7 +975,8 @@ "axisPlacement": "auto", "thresholdsStyle": { "mode": "off" - } + }, + "axisSoftMax": 10 }, "thresholds": { "mode": "absolute", @@ -980,7 +996,9 @@ ] }, "unit": "none", - "decimals": 2 + "decimals": 2, + "min": 0, + "max": 20 }, "overrides": [] }, @@ -1001,13 +1019,14 @@ }, "targets": [ { - "expr": "foxhunt_training_epoch_profit_factor{model=~\"$model\", fold=~\"$fold\"}", + "expr": "clamp_max(max_over_time(foxhunt_training_epoch_profit_factor{model=~\"$model\", fold=~\"$fold\"}[$__range]), 20)", "legendFormat": "{{model}}/{{fold}}", "refId": "A", "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" - } + }, + "instant": false } ] }, @@ -1319,7 +1338,9 @@ "gradientMode": "scheme", "showPoints": "auto", "pointSize": 4, - "axisLabel": "Sharpe" + "axisLabel": "Sharpe", + "axisSoftMin": -2, + "axisSoftMax": 3 }, "color": { "mode": "palette-classic" @@ -1340,7 +1361,9 @@ "value": 2 } ] - } + }, + "max": 5, + "min": -2 }, "overrides": [] }, @@ -1364,7 +1387,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "expr": "foxhunt_training_epoch_sharpe{model=~\"$model\", fold=~\"$fold\"}", + "expr": "clamp_max(foxhunt_training_epoch_sharpe{model=~\"$model\", fold=~\"$fold\"}, 5)", "legendFormat": "{{model}}/{{fold}}", "refId": "A" } @@ -1393,11 +1416,15 @@ "fillOpacity": 10, "showPoints": "auto", "pointSize": 4, - "axisLabel": "Sortino" + "axisLabel": "Sortino", + "axisSoftMin": -3, + "axisSoftMax": 5 }, "color": { "mode": "palette-classic" - } + }, + "max": 10, + "min": -3 }, "overrides": [] }, @@ -1421,7 +1448,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "expr": "foxhunt_training_epoch_sortino{model=~\"$model\", fold=~\"$fold\"}", + "expr": "clamp_max(foxhunt_training_epoch_sortino{model=~\"$model\", fold=~\"$fold\"}, 10)", "legendFormat": "{{model}}/{{fold}}", "refId": "A" } @@ -1506,12 +1533,14 @@ "fillOpacity": 10, "showPoints": "auto", "pointSize": 4, - "axisLabel": "Drawdown" + "axisLabel": "Drawdown", + "axisSoftMax": 1 }, "color": { "mode": "palette-classic" }, - "unit": "percentunit" + "unit": "percentunit", + "min": 0 }, "overrides": [] }, @@ -1559,11 +1588,13 @@ "lineWidth": 2, "fillOpacity": 5, "showPoints": "auto", - "pointSize": 4 + "pointSize": 4, + "axisSoftMax": 20 }, "color": { "mode": "palette-classic" - } + }, + "min": 0 }, "overrides": [ { @@ -1589,6 +1620,18 @@ "value": 30 } ] + }, + { + "matcher": { + "id": "byName", + "options": "foxhunt_training_epoch_profit_factor" + }, + "properties": [ + { + "id": "max", + "value": 20 + } + ] } ] }, @@ -1608,7 +1651,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "expr": "foxhunt_training_epoch_profit_factor{model=~\"$model\", fold=~\"$fold\"}", + "expr": "clamp_max(foxhunt_training_epoch_profit_factor{model=~\"$model\", fold=~\"$fold\"}, 20)", "legendFormat": "PF {{model}}", "refId": "A" }, @@ -3131,4 +3174,4 @@ ] } ] -} \ No newline at end of file +}