diff --git a/crates/ml/src/trainers/dqn/trainer.rs b/crates/ml/src/trainers/dqn/trainer.rs index 57a82def0..05d51f26d 100644 --- a/crates/ml/src/trainers/dqn/trainer.rs +++ b/crates/ml/src/trainers/dqn/trainer.rs @@ -1595,6 +1595,11 @@ impl DQNTrainer { // WAVE 30: Log epoch start log_epoch_start(epoch + 1, self.hyperparams.epochs, self.hyperparams.learning_rate); + // Emit epoch gauge immediately so Prometheus/Grafana template variables resolve + // before the first epoch completes (foxhunt_training_step alone isn't enough + // because the dashboard model/fold dropdowns historically use current_epoch). + training_metrics::set_epoch("dqn", "current", (epoch + 1) as f64); + // Create monitor for this epoch let mut monitor = TrainingMonitor::new(epoch + 1); diff --git a/crates/ml/src/trainers/ppo.rs b/crates/ml/src/trainers/ppo.rs index 5892e5083..180bab3bc 100644 --- a/crates/ml/src/trainers/ppo.rs +++ b/crates/ml/src/trainers/ppo.rs @@ -539,6 +539,9 @@ impl PpoTrainer { let epoch_start = std::time::Instant::now(); debug!("Training epoch {}/{}", epoch + 1, self.hyperparams.epochs); + // Emit epoch gauge immediately so Grafana template variables resolve early + training_metrics::set_epoch("ppo", "current", (epoch + 1) as f64); + // Phase 3: GPU experience collection — bypasses CPU collect_rollouts + prepare_training_batch #[cfg(feature = "cuda")] let gpu_batch_result: Option = if let ( diff --git a/infra/k8s/monitoring/dashboards/foxhunt-training.json b/infra/k8s/monitoring/dashboards/foxhunt-training.json index f9dff2662..59ffa51b4 100644 --- a/infra/k8s/monitoring/dashboards/foxhunt-training.json +++ b/infra/k8s/monitoring/dashboards/foxhunt-training.json @@ -98,7 +98,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "query": "label_values(foxhunt_training_current_epoch{job=~\"$source\"}, model)", + "query": "label_values(foxhunt_training_step{job=~\"$source\"}, model)", "refresh": 2, "includeAll": true, "multi": true, @@ -118,7 +118,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "query": "label_values(foxhunt_training_current_epoch{job=~\"$source\"}, fold)", + "query": "label_values(foxhunt_training_step{job=~\"$source\"}, fold)", "refresh": 2, "includeAll": true, "multi": true, diff --git a/infra/scripts/train.sh b/infra/scripts/train.sh index 7f8141572..c43d46696 100755 --- a/infra/scripts/train.sh +++ b/infra/scripts/train.sh @@ -196,6 +196,10 @@ spec: foxhunt/job-type: training foxhunt/model: ${model} foxhunt/preset: ${PRESET} + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9094" + prometheus.io/path: "/metrics" spec: restartPolicy: Never nodeSelector: @@ -341,7 +345,12 @@ spec: metadata: labels: app.kubernetes.io/part-of: foxhunt + app.kubernetes.io/component: training-workflow foxhunt/job-type: evaluate + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9094" + prometheus.io/path: "/metrics" spec: restartPolicy: Never nodeSelector: