From 894188d34f2fb6189a7e72cfae8240bdbbc95af3 Mon Sep 17 00:00:00 2001
From: jgrusewski <jgrusewski@localhost>
Date: Sun, 17 May 2026 11:46:06 +0200
Subject: [PATCH] feat(alpha_train): track best-by-mean-AUC alongside
 best-by-val_loss
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

cnjfl run showed val_loss and mean-AUC peak at different epochs:
  epoch 3: val_loss=0.5592 (best)  mean_auc=0.7608
  epoch 4: val_loss=0.5609 (worse) mean_auc=0.7670 (best — new h300 + h6000 peaks)

val_loss tracks probability calibration; AUC tracks ranking quality.
For downstream trading the ranking profile matters more — so we now
publish both bests in alpha_train_summary.json and log a "new best
mean_auc" line whenever a new mean-AUC peak lands.

Early stopping still gates on val_loss (the two policies stay
decoupled — mean-AUC is reported-only).

New summary fields:
  best_mean_auc_epoch
  best_mean_auc
  best_mean_auc_per_horizon

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
---
 crates/ml-alpha/examples/alpha_train.rs | 36 ++++++++++++++++++++++++-
 1 file changed, 35 insertions(+), 1 deletion(-)

diff --git a/crates/ml-alpha/examples/alpha_train.rs b/crates/ml-alpha/examples/alpha_train.rs
index 7c220decc..6f20d31b5 100644
--- a/crates/ml-alpha/examples/alpha_train.rs
+++ b/crates/ml-alpha/examples/alpha_train.rs
@@ -121,6 +121,15 @@ struct AlphaTrainSummary {
     best_val_loss: f32,
     /// Per-horizon AUCs at the best-val-loss epoch.
     best_val_auc: [f32; N_HORIZONS],
+    /// Epoch index that achieved the highest mean AUC across the 5 horizons.
+    /// Tracked independently of best_epoch because val_loss + AUC can
+    /// disagree — a calibration-flat / ranking-sharp epoch beats a
+    /// calibration-sharp / ranking-flat one for downstream trading.
+    best_mean_auc_epoch: usize,
+    /// Mean AUC at the best-mean-AUC epoch.
+    best_mean_auc: f32,
+    /// Per-horizon AUCs at the best-mean-AUC epoch.
+    best_mean_auc_per_horizon: [f32; N_HORIZONS],
     /// True if training stopped early (patience exceeded).
     early_stopped: bool,
 }
@@ -205,6 +214,15 @@ fn main() -> Result<()> {
     let mut early_stopped = false;
     let mut epochs_completed = 0usize;
 
+    // Independent tracker for best mean-AUC across horizons. val_loss and
+    // ranking quality (AUC) can disagree — e.g. when the model improves
+    // probability calibration on common cases (lower BCE) but loses
+    // ranking sharpness on the regime tails. For downstream trading the
+    // AUC profile usually matters more, so we publish both bests.
+    let mut best_mean_auc = f32::NEG_INFINITY;
+    let mut best_mean_auc_epoch = 0usize;
+    let mut best_mean_auc_per_horizon = [0.5_f32; N_HORIZONS];
+
     let lr_min_cfc = cli.lr_cfc * cli.lr_min_factor;
     let lr_min_m2 = cli.lr_mamba2 * cli.lr_min_factor;
     // Approximate the total step budget: epochs × n_train_seqs (each
@@ -352,7 +370,7 @@ fn main() -> Result<()> {
         n_val_seqs_consumed = val_loader.yielded();
         epochs_completed = epoch + 1;
 
-        // Best-checkpoint + early-stop bookkeeping.
+        // Best-checkpoint + early-stop bookkeeping (by val_loss).
         if val_avg < best_val_loss {
             best_val_loss = val_avg;
             best_epoch = epoch;
@@ -373,6 +391,19 @@ fn main() -> Result<()> {
                 break;
             }
         }
+
+        // Independent best-by-mean-AUC tracker — reported only, does NOT
+        // gate early stopping (val_loss remains the early-stop signal so
+        // the two policies stay decoupled).
+        let mean_auc = per_horizon_auc.iter().sum::<f32>() / per_horizon_auc.len() as f32;
+        if mean_auc > best_mean_auc {
+            best_mean_auc = mean_auc;
+            best_mean_auc_epoch = epoch;
+            best_mean_auc_per_horizon = per_horizon_auc;
+            tracing::info!(
+                epoch, mean_auc, "new best mean_auc"
+            );
+        }
     }
 
     let summary = AlphaTrainSummary {
@@ -387,6 +418,9 @@ fn main() -> Result<()> {
         best_epoch,
         best_val_loss,
         best_val_auc,
+        best_mean_auc_epoch,
+        best_mean_auc,
+        best_mean_auc_per_horizon,
         early_stopped,
     };
     let summary_path = cli.out.join("alpha_train_summary.json");