gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(dashboard): clamp degenerate outliers in training metrics panels

Browse Source 
Add clamp_max() to PromQL queries and hard Y-axis limits to prevent
early-epoch degenerate values from blowing up panel scaling (Sharpe
showing 12 instead of 1.3, Profit Factor at 30k).

Run Summary: clamp_max on Best Val Loss (5), Sharpe (5), Profit Factor (20)
Training Quality: clamp_max on Epoch Sharpe (5), Sortino (10), PF (20)

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-03-06 02:22:21 +01:00
parent 5323cf11e7
commit 2b77a7fbe1
1 changed files with 74 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

105
infra/k8s/monitoring/dashboards/foxhunt-training.json
View File

@@ -543,7 +543,8 @@
"axisPlacement": "auto",
"thresholdsStyle": {
"mode": "off"
}
},
"axisSoftMax": 2
},
"thresholds": {
"mode": "absolute",
@@ -563,7 +564,9 @@
]
},
"unit": "none",
"decimals": 4
"decimals": 4,
"min": 0,
"max": 5
},
"overrides": []
},
@@ -584,13 +587,14 @@
},
"targets": [
{
"expr": "foxhunt_training_validation_loss{model=~\"$model\", fold=~\"$fold\"}",
"expr": "clamp_max(min_over_time(foxhunt_training_validation_loss{model=~\"$model\", fold=~\"$fold\"}[$__range]), 5)",
"legendFormat": "{{model}}/{{fold}}",
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
}
},
"instant": false
}
]
},
@@ -628,7 +632,8 @@
"axisPlacement": "auto",
"thresholdsStyle": {
"mode": "off"
}
},
"axisSoftMax": 3
},
"thresholds": {
"mode": "absolute",
@@ -648,7 +653,9 @@
]
},
"unit": "none",
"decimals": 2
"decimals": 2,
"min": 0,
"max": 5
},
"overrides": []
},
@@ -669,13 +676,14 @@
},
"targets": [
{
"expr": "foxhunt_training_epoch_sharpe{model=~\"$model\", fold=~\"$fold\"}",
"expr": "clamp_max(max_over_time(foxhunt_training_epoch_sharpe{model=~\"$model\", fold=~\"$fold\"}[$__range]), 5)",
"legendFormat": "{{model}}/{{fold}}",
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
}
},
"instant": false
}
]
},
@@ -733,7 +741,9 @@
]
},
"unit": "percentunit",
"decimals": 1
"decimals": 1,
"min": 0,
"max": 1
},
"overrides": []
},
@@ -754,13 +764,14 @@
},
"targets": [
{
"expr": "foxhunt_training_epoch_win_rate{model=~\"$model\", fold=~\"$fold\"}",
"expr": "max_over_time(foxhunt_training_epoch_win_rate{model=~\"$model\", fold=~\"$fold\"}[$__range])",
"legendFormat": "{{model}}/{{fold}}",
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
}
},
"instant": false
}
]
},
@@ -818,7 +829,9 @@
]
},
"unit": "percentunit",
"decimals": 1
"decimals": 1,
"min": 0,
"max": 1
},
"overrides": []
},
@@ -839,13 +852,14 @@
},
"targets": [
{
"expr": "foxhunt_training_epoch_max_drawdown{model=~\"$model\", fold=~\"$fold\"}",
"expr": "min_over_time(foxhunt_training_epoch_max_drawdown{model=~\"$model\", fold=~\"$fold\"}[$__range])",
"legendFormat": "{{model}}/{{fold}}",
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
}
},
"instant": false
}
]
},
@@ -922,7 +936,8 @@
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
}
},
"instant": false
}
]
},
@@ -960,7 +975,8 @@
"axisPlacement": "auto",
"thresholdsStyle": {
"mode": "off"
}
},
"axisSoftMax": 10
},
"thresholds": {
"mode": "absolute",
@@ -980,7 +996,9 @@
]
},
"unit": "none",
"decimals": 2
"decimals": 2,
"min": 0,
"max": 20
},
"overrides": []
},
@@ -1001,13 +1019,14 @@
},
"targets": [
{
"expr": "foxhunt_training_epoch_profit_factor{model=~\"$model\", fold=~\"$fold\"}",
"expr": "clamp_max(max_over_time(foxhunt_training_epoch_profit_factor{model=~\"$model\", fold=~\"$fold\"}[$__range]), 20)",
"legendFormat": "{{model}}/{{fold}}",
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
}
},
"instant": false
}
]
},
@@ -1319,7 +1338,9 @@
"gradientMode": "scheme",
"showPoints": "auto",
"pointSize": 4,
"axisLabel": "Sharpe"
"axisLabel": "Sharpe",
"axisSoftMin": -2,
"axisSoftMax": 3
},
"color": {
"mode": "palette-classic"
@@ -1340,7 +1361,9 @@
"value": 2
}
]
}
},
"max": 5,
"min": -2
},
"overrides": []
},
@@ -1364,7 +1387,7 @@
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"expr": "foxhunt_training_epoch_sharpe{model=~\"$model\", fold=~\"$fold\"}",
"expr": "clamp_max(foxhunt_training_epoch_sharpe{model=~\"$model\", fold=~\"$fold\"}, 5)",
"legendFormat": "{{model}}/{{fold}}",
"refId": "A"
}
@@ -1393,11 +1416,15 @@
"fillOpacity": 10,
"showPoints": "auto",
"pointSize": 4,
"axisLabel": "Sortino"
"axisLabel": "Sortino",
"axisSoftMin": -3,
"axisSoftMax": 5
},
"color": {
"mode": "palette-classic"
}
},
"max": 10,
"min": -3
},
"overrides": []
},
@@ -1421,7 +1448,7 @@
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"expr": "foxhunt_training_epoch_sortino{model=~\"$model\", fold=~\"$fold\"}",
"expr": "clamp_max(foxhunt_training_epoch_sortino{model=~\"$model\", fold=~\"$fold\"}, 10)",
"legendFormat": "{{model}}/{{fold}}",
"refId": "A"
}
@@ -1506,12 +1533,14 @@
"fillOpacity": 10,
"showPoints": "auto",
"pointSize": 4,
"axisLabel": "Drawdown"
"axisLabel": "Drawdown",
"axisSoftMax": 1
},
"color": {
"mode": "palette-classic"
},
"unit": "percentunit"
"unit": "percentunit",
"min": 0
},
"overrides": []
},
@@ -1559,11 +1588,13 @@
"lineWidth": 2,
"fillOpacity": 5,
"showPoints": "auto",
"pointSize": 4
"pointSize": 4,
"axisSoftMax": 20
},
"color": {
"mode": "palette-classic"
}
},
"min": 0
},
"overrides": [
{
@@ -1589,6 +1620,18 @@
"value": 30
}
]
},
{
"matcher": {
"id": "byName",
"options": "foxhunt_training_epoch_profit_factor"
},
"properties": [
{
"id": "max",
"value": 20
}
]
}
]
},
@@ -1608,7 +1651,7 @@
"type": "prometheus",
"uid": "PBFA97CFB590B2093"
},
"expr": "foxhunt_training_epoch_profit_factor{model=~\"$model\", fold=~\"$fold\"}",
"expr": "clamp_max(foxhunt_training_epoch_profit_factor{model=~\"$model\", fold=~\"$fold\"}, 20)",
"legendFormat": "PF {{model}}",
"refId": "A"
},
@@ -3131,4 +3174,4 @@
]
}
]
}
}