fix(ci): make CUDA non-default to unblock CPU service builds

- Remove cuda from default features in ml-core, ml-dqn, ml-ppo, ml
- Propagate cuda feature from ml → ml-core/ml-dqn/ml-ppo
- CI compile-training already uses --features ml/cuda explicitly
- Fix MaxDD log format: {:.1}% → {:.3}% (was rounding 0.033% to 0.0%)
- Suppress unused_labels/unused_variables warnings for cfg(cuda) code
- Add CALLBACK_ENDPOINT env to ml-training-service deployment
- Fix Grafana active_workers query to use sum() with fallback

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

crates/ml-core/Cargo.toml

@@ -14,7 +14,7 @@ categories.workspace = true
 description = "Shared ML types, traits, and infrastructure for Foxhunt"
 
 [features]
-default = ["cuda"]
+default = []
 cuda = ["candle-core/cuda", "candle-core/cudnn", "candle-nn/cuda", "candle-nn/cudnn"]
 s3-storage = ["aws-config", "aws-sdk-s3", "aws-types", "aws-credential-types", "urlencoding"]
 high-precision = ["rust_decimal/serde-float"]

crates/ml-dqn/Cargo.toml

@@ -14,7 +14,7 @@ categories.workspace = true
 description = "DQN reinforcement learning for Foxhunt trading"
 
 [features]
-default = ["cuda"]
+default = []
 cuda = ["candle-core/cuda", "candle-core/cudnn", "candle-nn/cuda", "candle-nn/cudnn"]
 
 [dependencies]

crates/ml-dqn/src/dqn.rs

@@ -2133,6 +2133,7 @@ impl DQN {
         // GPU FAST PATH: When GpuBatch is available, use pre-built GPU tensors directly.
         // Eliminates: CPU fold over experiences, 5× Tensor::from_vec CPU→GPU transfers,
         // and CPU action validation loop. Action bounds enforced via GPU clamp.
+        #[allow(unused_labels)] // label used only in #[cfg(feature = "cuda")] break
         let (batch_size, states_tensor, next_states_tensor, actions_tensor,
              rewards_tensor, dones_tensor, weights_tensor_cached) = 'tensor_prep: {
             #[cfg(feature = "cuda")]
@@ -2758,6 +2759,7 @@ impl DQN {
         #[cfg(not(feature = "cuda"))]
         let is_gpu_per = false;
 
+        #[allow(unused_variables)] // td_gpu/idx_gpu consumed in #[cfg(feature = "cuda")] struct fields
         let (td_errors_vec, td_gpu, idx_gpu) = if self.config.use_per {
             if is_gpu_per {
                 // GPU PER: keep TD errors on GPU, use GpuBatch indices directly

crates/ml-ppo/Cargo.toml

@@ -14,7 +14,7 @@ categories.workspace = true
 description = "PPO reinforcement learning for Foxhunt trading"
 
 [features]
-default = ["cuda"]
+default = []
 cuda = ["candle-core/cuda", "candle-core/cudnn", "candle-nn/cuda", "candle-nn/cudnn"]
 
 [dependencies]

crates/ml/Cargo.toml

@@ -14,8 +14,8 @@ categories.workspace = true
 
 [features]
 # MINIMAL features for HFT inference only - ALL HEAVY ML REMOVED
-# CUDA is now default for training - GPU acceleration mandatory
-default = ["minimal-inference", "cuda"]
+# CUDA opt-in: service crates build on CPU nodes without nvcc
+default = ["minimal-inference"]
 
 # PRODUCTION FEATURES - LIGHTWEIGHT ONLY
 minimal-inference = []  # Minimal inference with no optional deps
@@ -29,7 +29,7 @@ simd = []  # SIMD without heavy dependencies
 # Storage and memory management features
 gc = []  # Garbage collection features
 s3-storage = ["ml-checkpoint/s3-storage", "aws-config", "aws-sdk-s3", "aws-types", "aws-credential-types", "urlencoding"]  # S3 storage backend with AWS SDK
-cuda = ["candle-core/cuda", "candle-core/cudnn", "candle-nn/cuda", "candle-nn/cudnn"]  # CUDA support (includes LSTM sigmoid ops) - OPTIONAL for CI/Docker
+cuda = ["candle-core/cuda", "candle-core/cudnn", "candle-nn/cuda", "candle-nn/cudnn", "ml-core/cuda", "ml-dqn/cuda", "ml-ppo/cuda"]  # CUDA support — enabled by compile-training CI step via --features ml/cuda
 nccl = ["cuda"]  # NCCL multi-GPU data parallelism (requires NCCL library + cudarc nccl feature)
 
 # ALL HEAVY ML FEATURES REMOVED:

crates/ml/src/trainers/dqn/trainer.rs

@@ -3149,7 +3149,7 @@ impl DQNTrainer {
                     financials.hold_pct,
                 );
                 info!(
-                    "Epoch {}/{}: Sharpe={:.2} WinRate={:.1}% MaxDD={:.1}% PF={:.2} Return={:+.2}% Trades={}",
+                    "Epoch {}/{}: Sharpe={:.2} WinRate={:.1}% MaxDD={:.3}% PF={:.2} Return={:+.2}% Trades={}",
                     epoch + 1, self.hyperparams.epochs,
                     financials.sharpe, financials.win_rate * 100.0,
                     financials.max_drawdown * 100.0, financials.profit_factor,

infra/k8s/monitoring/dashboards/foxhunt-training.json

@@ -172,7 +172,7 @@
             "type": "prometheus",
             "uid": "PBFA97CFB590B2093"
           },
-          "expr": "foxhunt_training_active_workers",
+          "expr": "sum(foxhunt_training_active_workers{job=\"training-pods\"}) or vector(0)",
           "legendFormat": "Workers",
           "refId": "A"
         }

infra/k8s/services/ml-training-service.yaml

@@ -162,6 +162,8 @@ spec:
                   key: secret-key
             - name: TRAINING_RUNTIME_IMAGE
               value: "gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-training-runtime:latest"
+            - name: CALLBACK_ENDPOINT
+              value: "http://ml-training-service.foxhunt.svc.cluster.local:50053"
             - name: RUST_LOG
               value: "info,opentelemetry=warn,h2=warn,tonic=warn,hyper=warn"
             - name: OTEL_EXPORTER_OTLP_ENDPOINT