gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(monitoring): add Training Deep-Dive dashboard with unified Prometheus + Loki

Browse Source 
74-panel dashboard for in-depth pipeline monitoring. Select a running pod
to see training curves, GPU health (DCGM), eval metrics, CI/Argo workflows,
container resources, and live logs in one place. Stored in Postgres via API
(no ConfigMap restarts needed).

Sections: Job Overview, CI Pipeline & Argo Workflows, CI Logs, Training
Curves, Trading Performance, Evaluation Metrics, GPU & Hardware, Throughput,
Hyperopt Trials, Container Resources, Live Logs.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-03-14 02:04:24 +01:00
parent e5daea44e0
commit 6349e384d2
2 changed files with 3349 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3345
infra/k8s/monitoring/dashboards/foxhunt-training-deep-dive.json Normal file
View File

File diff suppressed because it is too large Load Diff

5
infra/k8s/monitoring/dashboards/import.sh
View File

@@ -130,6 +130,7 @@ declare -A CONFIGMAP_GROUPS=(
[grafana-dashboards-observability]="foxhunt-observability.json"
[grafana-dashboards-ci-logs]="foxhunt-ci-logs.json"
[grafana-dashboards-training-logs]="foxhunt-training-logs.json"
[grafana-dashboards-deep-dive]="foxhunt-training-deep-dive.json"
)
# Folder mapping: ConfigMap name -> Grafana folder name (used by sidecar annotation)
@@ -141,6 +142,7 @@ declare -A CONFIGMAP_FOLDERS=(
[grafana-dashboards-observability]="Foxhunt"
[grafana-dashboards-ci-logs]="Foxhunt"
[grafana-dashboards-training-logs]="Foxhunt"
[grafana-dashboards-deep-dive]="Foxhunt"
)
deploy_configmaps() {
@@ -235,7 +237,8 @@ else
"${SCRIPT_DIR}/foxhunt-infrastructure.json" \
"${SCRIPT_DIR}/foxhunt-observability.json" \
"${SCRIPT_DIR}/foxhunt-ci-logs.json" \
"${SCRIPT_DIR}/foxhunt-training-logs.json"; do
"${SCRIPT_DIR}/foxhunt-training-logs.json" \
"${SCRIPT_DIR}/foxhunt-training-deep-dive.json"; do
if [[ -f "$dashboard" ]]; then
if ! import_dashboard "$dashboard"; then
failed=$((failed + 1))