From ebe51f6109a6e85bc32fdbd402c2c584970ee5e2 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Wed, 25 Feb 2026 17:14:19 +0100 Subject: [PATCH] fix(ci): fix 6 GPU test failures, configure test-gpu runner, upgrade ci-build to GP1-XL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tests 15-16: from_checkpoint now correctly rejects invalid safetensors data instead of silently using random weights. Tests 17-20: construct untrained models directly via ::new() instead of broken from_checkpoint path — tests inference refusal, not checkpoint loading. test-gpu job: add kapsule+gpu tags for GPU runner targeting, remove TODO. CI build pool: GP1-M → GP1-XL (48 vCPU, 256GB) — request quota increase. Co-Authored-By: Claude Opus 4.6 --- .gitlab-ci.yml | 8 +- infra/live/production/kapsule/terragrunt.hcl | 2 +- .../src/services/enhanced_ml.rs | 137 +++++++++++------- 3 files changed, 93 insertions(+), 54 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c623191be..7f20a86bf 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,5 @@ # GitLab CI/CD — Foxhunt -# Runs on ci-build pool (GP1-XS, scale-to-zero) +# Runs on ci-build pool (GP1-XL, scale-to-zero) # CI builder image: CUDA 12.4 + Rust 1.89 + protoc + sccache (hosted on Scaleway CR) # Service images: pushed to internal GitLab registry (Kaniko) @@ -87,12 +87,14 @@ test: # Run all workspace tests EXCEPT ml crate (52 CUDA-dependent tests need real GPU) - cargo test --workspace --exclude ml --lib -- --skip model_loader::tests -# GPU test job — runs ml crate tests on real CUDA hardware -# TODO: Configure GPU runner with node_selector targeting gpu-inference pool +# GPU test job — runs ml crate tests on real CUDA hardware (gpu-inference pool, L4) test-gpu: extends: .rust-base stage: test needs: [check] + tags: + - kapsule + - gpu when: manual allow_failure: true script: diff --git a/infra/live/production/kapsule/terragrunt.hcl b/infra/live/production/kapsule/terragrunt.hcl index c389b2e74..eafd74271 100644 --- a/infra/live/production/kapsule/terragrunt.hcl +++ b/infra/live/production/kapsule/terragrunt.hcl @@ -27,6 +27,6 @@ inputs = { # CI build pool (ephemeral runner pods, scale-to-zero) enable_ci_build_pool = true - ci_build_type = "GP1-M" + ci_build_type = "GP1-XL" ci_build_max_size = 2 } diff --git a/services/trading_service/src/services/enhanced_ml.rs b/services/trading_service/src/services/enhanced_ml.rs index d0609e701..1ceac2ccd 100644 --- a/services/trading_service/src/services/enhanced_ml.rs +++ b/services/trading_service/src/services/enhanced_ml.rs @@ -2606,52 +2606,40 @@ mod enhanced_ml_tests { } // ----------------------------------------------------------------------- - // 15. TFT from_checkpoint does NOT set is_trained on random weights (C5 fix) + // 15. TFT from_checkpoint rejects invalid safetensors data (C5 fix) // ----------------------------------------------------------------------- #[test] fn test_tft_from_checkpoint_does_not_mark_trained_with_random_weights() { - // Create a real temp file so the existence check passes let tmp_dir = std::env::temp_dir(); let checkpoint_path = tmp_dir.join("test_tft_checkpoint_c5.safetensors"); std::fs::write(&checkpoint_path, b"fake checkpoint data").unwrap(); let result = RealTFTModel::from_checkpoint("tft-test".to_string(), &checkpoint_path); - // Clean up let _ = std::fs::remove_file(&checkpoint_path); - // The model should be created successfully... - let model = result.expect("TFT model creation should succeed when checkpoint exists"); - - // ...but is_trained must NOT be true (weights are random, not loaded from checkpoint) - let tft = model.model.try_read().unwrap(); + // from_checkpoint must reject invalid safetensors — never silently use random weights assert!( - !tft.is_trained, - "TFT model must NOT be marked as trained when using random weights" + result.is_err(), + "TFT from_checkpoint must fail on invalid safetensors data" ); } // ----------------------------------------------------------------------- - // 16. Mamba2 from_checkpoint does NOT set is_trained on random weights (C5 fix) + // 16. Mamba2 from_checkpoint rejects invalid safetensors data (C5 fix) // ----------------------------------------------------------------------- #[test] fn test_mamba2_from_checkpoint_does_not_mark_trained_with_random_weights() { - // Create a real temp file so the existence check passes let tmp_dir = std::env::temp_dir(); let checkpoint_path = tmp_dir.join("test_mamba2_checkpoint_c5.safetensors"); std::fs::write(&checkpoint_path, b"fake checkpoint data").unwrap(); let result = RealMamba2Model::from_checkpoint("mamba2-test".to_string(), &checkpoint_path); - // Clean up let _ = std::fs::remove_file(&checkpoint_path); - // The model should be created successfully... - let model = result.expect("Mamba2 model creation should succeed when checkpoint exists"); - - // ...but is_trained must NOT be true (weights are random, not loaded from checkpoint) - let mamba2 = model.model.try_read().unwrap(); + // from_checkpoint must reject invalid safetensors — never silently use random weights assert!( - !mamba2.is_trained, - "Mamba2 model must NOT be marked as trained when using random weights" + result.is_err(), + "Mamba2 from_checkpoint must fail on invalid safetensors data" ); } @@ -2660,16 +2648,19 @@ mod enhanced_ml_tests { // ----------------------------------------------------------------------- #[tokio::test] async fn test_mamba2_predict_refuses_untrained_model() { - // Create a real temp file so the existence check passes - let tmp_dir = std::env::temp_dir(); - let checkpoint_path = tmp_dir.join("test_mamba2_predict_m3.safetensors"); - std::fs::write(&checkpoint_path, b"fake checkpoint data").unwrap(); + use ml::mamba::{Mamba2Config, Mamba2SSM}; + use ml::prelude::Device; - let model = RealMamba2Model::from_checkpoint("mamba2-test".to_string(), &checkpoint_path) - .expect("Mamba2 model creation should succeed"); - let _ = std::fs::remove_file(&checkpoint_path); + // Construct untrained model directly (no checkpoint loading) + let config = Mamba2Config::default(); + let mamba2 = Mamba2SSM::new(config.clone(), &Device::Cpu) + .expect("Mamba2 creation should succeed"); + let model = RealMamba2Model { + model_id: "mamba2-test".to_string(), + model: Arc::new(RwLock::new(mamba2)), + config, + }; - // Model is untrained, predict should fail with InferenceError let features = ml::Features { values: vec![0.1, 0.2, 0.3], names: vec!["a".to_string(), "b".to_string(), "c".to_string()], @@ -2692,14 +2683,17 @@ mod enhanced_ml_tests { // ----------------------------------------------------------------------- #[test] fn test_mamba2_is_ready_false_when_untrained() { - // Create a real temp file so the existence check passes - let tmp_dir = std::env::temp_dir(); - let checkpoint_path = tmp_dir.join("test_mamba2_ready_m3.safetensors"); - std::fs::write(&checkpoint_path, b"fake checkpoint data").unwrap(); + use ml::mamba::{Mamba2Config, Mamba2SSM}; + use ml::prelude::Device; - let model = RealMamba2Model::from_checkpoint("mamba2-test".to_string(), &checkpoint_path) - .expect("Mamba2 model creation should succeed"); - let _ = std::fs::remove_file(&checkpoint_path); + let config = Mamba2Config::default(); + let mamba2 = Mamba2SSM::new(config.clone(), &Device::Cpu) + .expect("Mamba2 creation should succeed"); + let model = RealMamba2Model { + model_id: "mamba2-test".to_string(), + model: Arc::new(RwLock::new(mamba2)), + config, + }; assert!( !model.is_ready(), @@ -2712,16 +2706,37 @@ mod enhanced_ml_tests { // ----------------------------------------------------------------------- #[tokio::test] async fn test_tft_predict_refuses_untrained_model() { - // Create a real temp file so the existence check passes - let tmp_dir = std::env::temp_dir(); - let checkpoint_path = tmp_dir.join("test_tft_predict_c5.safetensors"); - std::fs::write(&checkpoint_path, b"fake checkpoint data").unwrap(); + use ml::tft::{TFTConfig, TemporalFusionTransformer}; - let model = RealTFTModel::from_checkpoint("tft-test".to_string(), &checkpoint_path) - .expect("TFT model creation should succeed"); - let _ = std::fs::remove_file(&checkpoint_path); + let config = TFTConfig { + input_dim: 31, + hidden_dim: 128, + num_heads: 8, + num_layers: 3, + prediction_horizon: 10, + sequence_length: 50, + num_quantiles: 9, + num_static_features: 5, + num_known_features: 10, + num_unknown_features: 16, + learning_rate: 1e-3, + batch_size: 64, + dropout_rate: 0.1, + l2_regularization: 1e-4, + use_flash_attention: true, + mixed_precision: false, + memory_efficient: true, + max_inference_latency_us: 50, + target_throughput_pps: 100_000, + }; + let tft = TemporalFusionTransformer::new(config.clone()) + .expect("TFT creation should succeed"); + let model = RealTFTModel { + model_id: "tft-test".to_string(), + model: Arc::new(RwLock::new(tft)), + config, + }; - // Model is untrained, predict should fail with InferenceError let features = ml::Features { values: vec![0.1, 0.2, 0.3], names: vec!["a".to_string(), "b".to_string(), "c".to_string()], @@ -2744,14 +2759,36 @@ mod enhanced_ml_tests { // ----------------------------------------------------------------------- #[test] fn test_tft_is_ready_false_when_untrained() { - // Create a real temp file so the existence check passes - let tmp_dir = std::env::temp_dir(); - let checkpoint_path = tmp_dir.join("test_tft_ready_c5.safetensors"); - std::fs::write(&checkpoint_path, b"fake checkpoint data").unwrap(); + use ml::tft::{TFTConfig, TemporalFusionTransformer}; - let model = RealTFTModel::from_checkpoint("tft-test".to_string(), &checkpoint_path) - .expect("TFT model creation should succeed"); - let _ = std::fs::remove_file(&checkpoint_path); + let config = TFTConfig { + input_dim: 31, + hidden_dim: 128, + num_heads: 8, + num_layers: 3, + prediction_horizon: 10, + sequence_length: 50, + num_quantiles: 9, + num_static_features: 5, + num_known_features: 10, + num_unknown_features: 16, + learning_rate: 1e-3, + batch_size: 64, + dropout_rate: 0.1, + l2_regularization: 1e-4, + use_flash_attention: true, + mixed_precision: false, + memory_efficient: true, + max_inference_latency_us: 50, + target_throughput_pps: 100_000, + }; + let tft = TemporalFusionTransformer::new(config.clone()) + .expect("TFT creation should succeed"); + let model = RealTFTModel { + model_id: "tft-test".to_string(), + model: Arc::new(RwLock::new(tft)), + config, + }; assert!( !model.is_ready(),