fix(metrics): wire training pod scraping to Grafana dashboard

Three root causes for "no metrics" on the training dashboard:

1. Dashboard template variables ($model, $fold) sourced from
   foxhunt_training_current_epoch which isn't emitted until the first
   epoch completes. Switch to foxhunt_training_step which fires from
   step 500 onward.

2. train.sh pod template missing Prometheus annotations
   (prometheus.io/scrape, port, path). Also add the
   app.kubernetes.io/component label to the eval manifest so
   evaluation pods are discoverable too.

3. DQN and PPO trainers only called set_epoch() at the END of each
   epoch. Move the call to the TOP of the epoch loop so the gauge
   exists from the first training iteration.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

crates/ml/src/trainers/dqn/trainer.rs

@@ -1595,6 +1595,11 @@ impl DQNTrainer {
             // WAVE 30: Log epoch start
             log_epoch_start(epoch + 1, self.hyperparams.epochs, self.hyperparams.learning_rate);
 
+            // Emit epoch gauge immediately so Prometheus/Grafana template variables resolve
+            // before the first epoch completes (foxhunt_training_step alone isn't enough
+            // because the dashboard model/fold dropdowns historically use current_epoch).
+            training_metrics::set_epoch("dqn", "current", (epoch + 1) as f64);
+
             // Create monitor for this epoch
             let mut monitor = TrainingMonitor::new(epoch + 1);
 

crates/ml/src/trainers/ppo.rs

@@ -539,6 +539,9 @@ impl PpoTrainer {
             let epoch_start = std::time::Instant::now();
             debug!("Training epoch {}/{}", epoch + 1, self.hyperparams.epochs);
 
+            // Emit epoch gauge immediately so Grafana template variables resolve early
+            training_metrics::set_epoch("ppo", "current", (epoch + 1) as f64);
+
             // Phase 3: GPU experience collection — bypasses CPU collect_rollouts + prepare_training_batch
             #[cfg(feature = "cuda")]
             let gpu_batch_result: Option<TrajectoryBatch> = if let (

infra/k8s/monitoring/dashboards/foxhunt-training.json

@@ -98,7 +98,7 @@
           "type": "prometheus",
           "uid": "PBFA97CFB590B2093"
         },
-        "query": "label_values(foxhunt_training_current_epoch{job=~\"$source\"}, model)",
+        "query": "label_values(foxhunt_training_step{job=~\"$source\"}, model)",
         "refresh": 2,
         "includeAll": true,
         "multi": true,
@@ -118,7 +118,7 @@
           "type": "prometheus",
           "uid": "PBFA97CFB590B2093"
         },
-        "query": "label_values(foxhunt_training_current_epoch{job=~\"$source\"}, fold)",
+        "query": "label_values(foxhunt_training_step{job=~\"$source\"}, fold)",
         "refresh": 2,
         "includeAll": true,
         "multi": true,

infra/scripts/train.sh

@@ -196,6 +196,10 @@ spec:
         foxhunt/job-type: training
         foxhunt/model: ${model}
         foxhunt/preset: ${PRESET}
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "9094"
+        prometheus.io/path: "/metrics"
     spec:
       restartPolicy: Never
       nodeSelector:
@@ -341,7 +345,12 @@ spec:
     metadata:
       labels:
         app.kubernetes.io/part-of: foxhunt
+        app.kubernetes.io/component: training-workflow
         foxhunt/job-type: evaluate
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "9094"
+        prometheus.io/path: "/metrics"
     spec:
       restartPolicy: Never
       nodeSelector: