From 3d68e9feaf8b3fac9bf747b0f84fda3e4ec1f509 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sun, 8 Mar 2026 19:12:03 +0100 Subject: [PATCH] fix(ci): make CUDA non-default to unblock CPU service builds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove cuda from default features in ml-core, ml-dqn, ml-ppo, ml - Propagate cuda feature from ml → ml-core/ml-dqn/ml-ppo - CI compile-training already uses --features ml/cuda explicitly - Fix MaxDD log format: {:.1}% → {:.3}% (was rounding 0.033% to 0.0%) - Suppress unused_labels/unused_variables warnings for cfg(cuda) code - Add CALLBACK_ENDPOINT env to ml-training-service deployment - Fix Grafana active_workers query to use sum() with fallback Co-Authored-By: Claude Opus 4.6 --- crates/ml-core/Cargo.toml | 2 +- crates/ml-dqn/Cargo.toml | 2 +- crates/ml-dqn/src/dqn.rs | 2 ++ crates/ml-ppo/Cargo.toml | 2 +- crates/ml/Cargo.toml | 6 +++--- crates/ml/src/trainers/dqn/trainer.rs | 2 +- infra/k8s/monitoring/dashboards/foxhunt-training.json | 2 +- infra/k8s/services/ml-training-service.yaml | 2 ++ 8 files changed, 12 insertions(+), 8 deletions(-) diff --git a/crates/ml-core/Cargo.toml b/crates/ml-core/Cargo.toml index 659e69b48..184161e6b 100644 --- a/crates/ml-core/Cargo.toml +++ b/crates/ml-core/Cargo.toml @@ -14,7 +14,7 @@ categories.workspace = true description = "Shared ML types, traits, and infrastructure for Foxhunt" [features] -default = ["cuda"] +default = [] cuda = ["candle-core/cuda", "candle-core/cudnn", "candle-nn/cuda", "candle-nn/cudnn"] s3-storage = ["aws-config", "aws-sdk-s3", "aws-types", "aws-credential-types", "urlencoding"] high-precision = ["rust_decimal/serde-float"] diff --git a/crates/ml-dqn/Cargo.toml b/crates/ml-dqn/Cargo.toml index bcd249d76..88389411d 100644 --- a/crates/ml-dqn/Cargo.toml +++ b/crates/ml-dqn/Cargo.toml @@ -14,7 +14,7 @@ categories.workspace = true description = "DQN reinforcement learning for Foxhunt trading" [features] -default = ["cuda"] +default = [] cuda = ["candle-core/cuda", "candle-core/cudnn", "candle-nn/cuda", "candle-nn/cudnn"] [dependencies] diff --git a/crates/ml-dqn/src/dqn.rs b/crates/ml-dqn/src/dqn.rs index 3fbbcf0d4..a3649b916 100644 --- a/crates/ml-dqn/src/dqn.rs +++ b/crates/ml-dqn/src/dqn.rs @@ -2133,6 +2133,7 @@ impl DQN { // GPU FAST PATH: When GpuBatch is available, use pre-built GPU tensors directly. // Eliminates: CPU fold over experiences, 5× Tensor::from_vec CPU→GPU transfers, // and CPU action validation loop. Action bounds enforced via GPU clamp. + #[allow(unused_labels)] // label used only in #[cfg(feature = "cuda")] break let (batch_size, states_tensor, next_states_tensor, actions_tensor, rewards_tensor, dones_tensor, weights_tensor_cached) = 'tensor_prep: { #[cfg(feature = "cuda")] @@ -2758,6 +2759,7 @@ impl DQN { #[cfg(not(feature = "cuda"))] let is_gpu_per = false; + #[allow(unused_variables)] // td_gpu/idx_gpu consumed in #[cfg(feature = "cuda")] struct fields let (td_errors_vec, td_gpu, idx_gpu) = if self.config.use_per { if is_gpu_per { // GPU PER: keep TD errors on GPU, use GpuBatch indices directly diff --git a/crates/ml-ppo/Cargo.toml b/crates/ml-ppo/Cargo.toml index ee37f782f..dc6cf868a 100644 --- a/crates/ml-ppo/Cargo.toml +++ b/crates/ml-ppo/Cargo.toml @@ -14,7 +14,7 @@ categories.workspace = true description = "PPO reinforcement learning for Foxhunt trading" [features] -default = ["cuda"] +default = [] cuda = ["candle-core/cuda", "candle-core/cudnn", "candle-nn/cuda", "candle-nn/cudnn"] [dependencies] diff --git a/crates/ml/Cargo.toml b/crates/ml/Cargo.toml index 402045467..df3155321 100644 --- a/crates/ml/Cargo.toml +++ b/crates/ml/Cargo.toml @@ -14,8 +14,8 @@ categories.workspace = true [features] # MINIMAL features for HFT inference only - ALL HEAVY ML REMOVED -# CUDA is now default for training - GPU acceleration mandatory -default = ["minimal-inference", "cuda"] +# CUDA opt-in: service crates build on CPU nodes without nvcc +default = ["minimal-inference"] # PRODUCTION FEATURES - LIGHTWEIGHT ONLY minimal-inference = [] # Minimal inference with no optional deps @@ -29,7 +29,7 @@ simd = [] # SIMD without heavy dependencies # Storage and memory management features gc = [] # Garbage collection features s3-storage = ["ml-checkpoint/s3-storage", "aws-config", "aws-sdk-s3", "aws-types", "aws-credential-types", "urlencoding"] # S3 storage backend with AWS SDK -cuda = ["candle-core/cuda", "candle-core/cudnn", "candle-nn/cuda", "candle-nn/cudnn"] # CUDA support (includes LSTM sigmoid ops) - OPTIONAL for CI/Docker +cuda = ["candle-core/cuda", "candle-core/cudnn", "candle-nn/cuda", "candle-nn/cudnn", "ml-core/cuda", "ml-dqn/cuda", "ml-ppo/cuda"] # CUDA support — enabled by compile-training CI step via --features ml/cuda nccl = ["cuda"] # NCCL multi-GPU data parallelism (requires NCCL library + cudarc nccl feature) # ALL HEAVY ML FEATURES REMOVED: diff --git a/crates/ml/src/trainers/dqn/trainer.rs b/crates/ml/src/trainers/dqn/trainer.rs index c8d9599fa..d373f5025 100644 --- a/crates/ml/src/trainers/dqn/trainer.rs +++ b/crates/ml/src/trainers/dqn/trainer.rs @@ -3149,7 +3149,7 @@ impl DQNTrainer { financials.hold_pct, ); info!( - "Epoch {}/{}: Sharpe={:.2} WinRate={:.1}% MaxDD={:.1}% PF={:.2} Return={:+.2}% Trades={}", + "Epoch {}/{}: Sharpe={:.2} WinRate={:.1}% MaxDD={:.3}% PF={:.2} Return={:+.2}% Trades={}", epoch + 1, self.hyperparams.epochs, financials.sharpe, financials.win_rate * 100.0, financials.max_drawdown * 100.0, financials.profit_factor, diff --git a/infra/k8s/monitoring/dashboards/foxhunt-training.json b/infra/k8s/monitoring/dashboards/foxhunt-training.json index 69b54e306..1b9cc2a9d 100644 --- a/infra/k8s/monitoring/dashboards/foxhunt-training.json +++ b/infra/k8s/monitoring/dashboards/foxhunt-training.json @@ -172,7 +172,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "expr": "foxhunt_training_active_workers", + "expr": "sum(foxhunt_training_active_workers{job=\"training-pods\"}) or vector(0)", "legendFormat": "Workers", "refId": "A" } diff --git a/infra/k8s/services/ml-training-service.yaml b/infra/k8s/services/ml-training-service.yaml index 15903a83d..38bbef67f 100644 --- a/infra/k8s/services/ml-training-service.yaml +++ b/infra/k8s/services/ml-training-service.yaml @@ -162,6 +162,8 @@ spec: key: secret-key - name: TRAINING_RUNTIME_IMAGE value: "gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-training-runtime:latest" + - name: CALLBACK_ENDPOINT + value: "http://ml-training-service.foxhunt.svc.cluster.local:50053" - name: RUST_LOG value: "info,opentelemetry=warn,h2=warn,tonic=warn,hyper=warn" - name: OTEL_EXPORTER_OTLP_ENDPOINT