gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(ci): fix 6 GPU test failures, configure test-gpu runner, upgrade ci-build to GP1-XL

Browse Source 
Tests 15-16: from_checkpoint now correctly rejects invalid safetensors data
instead of silently using random weights.
Tests 17-20: construct untrained models directly via ::new() instead of
broken from_checkpoint path — tests inference refusal, not checkpoint loading.

test-gpu job: add kapsule+gpu tags for GPU runner targeting, remove TODO.
CI build pool: GP1-M → GP1-XL (48 vCPU, 256GB) — request quota increase.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-02-25 17:14:19 +01:00
parent e87b7ffb9e
commit ebe51f6109
3 changed files with 93 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
.gitlab-ci.yml
View File

@@ -1,5 +1,5 @@
# GitLab CI/CD — Foxhunt
# Runs on ci-build pool (GP1-XS, scale-to-zero)
# Runs on ci-build pool (GP1-XL, scale-to-zero)
# CI builder image: CUDA 12.4 + Rust 1.89 + protoc + sccache (hosted on Scaleway CR)
# Service images: pushed to internal GitLab registry (Kaniko)
@@ -87,12 +87,14 @@ test:
# Run all workspace tests EXCEPT ml crate (52 CUDA-dependent tests need real GPU)
- cargo test --workspace --exclude ml --lib -- --skip model_loader::tests
# GPU test job — runs ml crate tests on real CUDA hardware
# TODO: Configure GPU runner with node_selector targeting gpu-inference pool
# GPU test job — runs ml crate tests on real CUDA hardware (gpu-inference pool, L4)
test-gpu:
extends: .rust-base
stage: test
needs: [check]
tags:
- kapsule
- gpu
when: manual
allow_failure: true
script:

2
infra/live/production/kapsule/terragrunt.hcl
View File

@@ -27,6 +27,6 @@ inputs = {
# CI build pool (ephemeral runner pods, scale-to-zero)
enable_ci_build_pool = true
ci_build_type = "GP1-M"
ci_build_type = "GP1-XL"
ci_build_max_size = 2
}

137
services/trading_service/src/services/enhanced_ml.rs
View File

@@ -2606,52 +2606,40 @@ mod enhanced_ml_tests {
}
// -----------------------------------------------------------------------
// 15. TFT from_checkpoint does NOT set is_trained on random weights (C5 fix)
// 15. TFT from_checkpoint rejects invalid safetensors data (C5 fix)
// -----------------------------------------------------------------------
#[test]
fn test_tft_from_checkpoint_does_not_mark_trained_with_random_weights() {
// Create a real temp file so the existence check passes
let tmp_dir = std::env::temp_dir();
let checkpoint_path = tmp_dir.join("test_tft_checkpoint_c5.safetensors");
std::fs::write(&checkpoint_path, b"fake checkpoint data").unwrap();
let result = RealTFTModel::from_checkpoint("tft-test".to_string(), &checkpoint_path);
// Clean up
let _ = std::fs::remove_file(&checkpoint_path);
// The model should be created successfully...
let model = result.expect("TFT model creation should succeed when checkpoint exists");
// ...but is_trained must NOT be true (weights are random, not loaded from checkpoint)
let tft = model.model.try_read().unwrap();
// from_checkpoint must reject invalid safetensors — never silently use random weights
assert!(
!tft.is_trained,
"TFT model must NOT be marked as trained when using random weights"
result.is_err(),
"TFT from_checkpoint must fail on invalid safetensors data"
);
}
// -----------------------------------------------------------------------
// 16. Mamba2 from_checkpoint does NOT set is_trained on random weights (C5 fix)
// 16. Mamba2 from_checkpoint rejects invalid safetensors data (C5 fix)
// -----------------------------------------------------------------------
#[test]
fn test_mamba2_from_checkpoint_does_not_mark_trained_with_random_weights() {
// Create a real temp file so the existence check passes
let tmp_dir = std::env::temp_dir();
let checkpoint_path = tmp_dir.join("test_mamba2_checkpoint_c5.safetensors");
std::fs::write(&checkpoint_path, b"fake checkpoint data").unwrap();
let result = RealMamba2Model::from_checkpoint("mamba2-test".to_string(), &checkpoint_path);
// Clean up
let _ = std::fs::remove_file(&checkpoint_path);
// The model should be created successfully...
let model = result.expect("Mamba2 model creation should succeed when checkpoint exists");
// ...but is_trained must NOT be true (weights are random, not loaded from checkpoint)
let mamba2 = model.model.try_read().unwrap();
// from_checkpoint must reject invalid safetensors — never silently use random weights
assert!(
!mamba2.is_trained,
"Mamba2 model must NOT be marked as trained when using random weights"
result.is_err(),
"Mamba2 from_checkpoint must fail on invalid safetensors data"
);
}
@@ -2660,16 +2648,19 @@ mod enhanced_ml_tests {
// -----------------------------------------------------------------------
#[tokio::test]
async fn test_mamba2_predict_refuses_untrained_model() {
// Create a real temp file so the existence check passes
let tmp_dir = std::env::temp_dir();
let checkpoint_path = tmp_dir.join("test_mamba2_predict_m3.safetensors");
std::fs::write(&checkpoint_path, b"fake checkpoint data").unwrap();
use ml::mamba::{Mamba2Config, Mamba2SSM};
use ml::prelude::Device;
let model = RealMamba2Model::from_checkpoint("mamba2-test".to_string(), &checkpoint_path)
.expect("Mamba2 model creation should succeed");
let _ = std::fs::remove_file(&checkpoint_path);
// Construct untrained model directly (no checkpoint loading)
let config = Mamba2Config::default();
let mamba2 = Mamba2SSM::new(config.clone(), &Device::Cpu)
.expect("Mamba2 creation should succeed");
let model = RealMamba2Model {
model_id: "mamba2-test".to_string(),
model: Arc::new(RwLock::new(mamba2)),
config,
};
// Model is untrained, predict should fail with InferenceError
let features = ml::Features {
values: vec![0.1, 0.2, 0.3],
names: vec!["a".to_string(), "b".to_string(), "c".to_string()],
@@ -2692,14 +2683,17 @@ mod enhanced_ml_tests {
// -----------------------------------------------------------------------
#[test]
fn test_mamba2_is_ready_false_when_untrained() {
// Create a real temp file so the existence check passes
let tmp_dir = std::env::temp_dir();
let checkpoint_path = tmp_dir.join("test_mamba2_ready_m3.safetensors");
std::fs::write(&checkpoint_path, b"fake checkpoint data").unwrap();
use ml::mamba::{Mamba2Config, Mamba2SSM};
use ml::prelude::Device;
let model = RealMamba2Model::from_checkpoint("mamba2-test".to_string(), &checkpoint_path)
.expect("Mamba2 model creation should succeed");
let _ = std::fs::remove_file(&checkpoint_path);
let config = Mamba2Config::default();
let mamba2 = Mamba2SSM::new(config.clone(), &Device::Cpu)
.expect("Mamba2 creation should succeed");
let model = RealMamba2Model {
model_id: "mamba2-test".to_string(),
model: Arc::new(RwLock::new(mamba2)),
config,
};
assert!(
!model.is_ready(),
@@ -2712,16 +2706,37 @@ mod enhanced_ml_tests {
// -----------------------------------------------------------------------
#[tokio::test]
async fn test_tft_predict_refuses_untrained_model() {
// Create a real temp file so the existence check passes
let tmp_dir = std::env::temp_dir();
let checkpoint_path = tmp_dir.join("test_tft_predict_c5.safetensors");
std::fs::write(&checkpoint_path, b"fake checkpoint data").unwrap();
use ml::tft::{TFTConfig, TemporalFusionTransformer};
let model = RealTFTModel::from_checkpoint("tft-test".to_string(), &checkpoint_path)
.expect("TFT model creation should succeed");
let _ = std::fs::remove_file(&checkpoint_path);
let config = TFTConfig {
input_dim: 31,
hidden_dim: 128,
num_heads: 8,
num_layers: 3,
prediction_horizon: 10,
sequence_length: 50,
num_quantiles: 9,
num_static_features: 5,
num_known_features: 10,
num_unknown_features: 16,
learning_rate: 1e-3,
batch_size: 64,
dropout_rate: 0.1,
l2_regularization: 1e-4,
use_flash_attention: true,
mixed_precision: false,
memory_efficient: true,
max_inference_latency_us: 50,
target_throughput_pps: 100_000,
};
let tft = TemporalFusionTransformer::new(config.clone())
.expect("TFT creation should succeed");
let model = RealTFTModel {
model_id: "tft-test".to_string(),
model: Arc::new(RwLock::new(tft)),
config,
};
// Model is untrained, predict should fail with InferenceError
let features = ml::Features {
values: vec![0.1, 0.2, 0.3],
names: vec!["a".to_string(), "b".to_string(), "c".to_string()],
@@ -2744,14 +2759,36 @@ mod enhanced_ml_tests {
// -----------------------------------------------------------------------
#[test]
fn test_tft_is_ready_false_when_untrained() {
// Create a real temp file so the existence check passes
let tmp_dir = std::env::temp_dir();
let checkpoint_path = tmp_dir.join("test_tft_ready_c5.safetensors");
std::fs::write(&checkpoint_path, b"fake checkpoint data").unwrap();
use ml::tft::{TFTConfig, TemporalFusionTransformer};
let model = RealTFTModel::from_checkpoint("tft-test".to_string(), &checkpoint_path)
.expect("TFT model creation should succeed");
let _ = std::fs::remove_file(&checkpoint_path);
let config = TFTConfig {
input_dim: 31,
hidden_dim: 128,
num_heads: 8,
num_layers: 3,
prediction_horizon: 10,
sequence_length: 50,
num_quantiles: 9,
num_static_features: 5,
num_known_features: 10,
num_unknown_features: 16,
learning_rate: 1e-3,
batch_size: 64,
dropout_rate: 0.1,
l2_regularization: 1e-4,
use_flash_attention: true,
mixed_precision: false,
memory_efficient: true,
max_inference_latency_us: 50,
target_throughput_pps: 100_000,
};
let tft = TemporalFusionTransformer::new(config.clone())
.expect("TFT creation should succeed");
let model = RealTFTModel {
model_id: "tft-test".to_string(),
model: Arc::new(RwLock::new(tft)),
config,
};
assert!(
!model.is_ready(),