gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(trading_service): TFT/Mamba2 validate checkpoint exists and propagate inference errors

Browse Source 
from_checkpoint was creating fresh models with random weights and setting
is_trained=true, allowing trading on noise. Now validates checkpoint exists
and does NOT set is_trained=true when weights are random.

Mamba2 predict errors are now wrapped as MLError::InferenceError so the
fallback manager can degrade model health. Both TFT and Mamba2 predict
refuse to run on untrained models, and is_ready() reflects trained state.

Also fixes TFT input_dim mismatch (16 vs 5+10+16=31).

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-02-23 23:14:16 +01:00
parent 4e5a269353
commit c71c32fff5
3 changed files with 380 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

75
services/broker_gateway_service/src/service.rs
View File

@@ -94,6 +94,21 @@ impl BrokerGatewayService {
}
}
/// Convert a quantity in lots to cTrader volume (lots * 100,000).
///
/// Returns an error if the result is not finite, negative, or exceeds i64 range.
/// Uses rounding to avoid silent truncation of fractional lots.
fn convert_quantity_to_volume(quantity: f64) -> Result<i64, Status> {
let volume_f = quantity * 100_000.0;
if !volume_f.is_finite() || volume_f < 0.0 || volume_f > i64::MAX as f64 {
return Err(Status::invalid_argument(format!(
"Volume overflow: quantity {} produces volume {}",
quantity, volume_f
)));
}
Ok(volume_f.round() as i64)
}
#[tonic::async_trait]
impl broker_gateway_service_server::BrokerGatewayService for BrokerGatewayService {
#[instrument(skip(self), fields(symbol, side, quantity))]
@@ -217,7 +232,7 @@ impl broker_gateway_service_server::BrokerGatewayService for BrokerGatewayServic
_ => ctrader_openapi::proto::ProtoOaOrderType::Market,
};
let volume = (req.quantity * 100_000.0) as i64; // lots → cTrader volume
let volume = convert_quantity_to_volume(req.quantity)?;
let limit_price = req.price;
let stop_price_val = req.stop_price;
let comment = req.metadata.get("comment").cloned();
@@ -668,3 +683,61 @@ impl broker_gateway_service_server::BrokerGatewayService for BrokerGatewayServic
}))
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_convert_quantity_normal() {
// 1.0 lot = 100,000 volume
assert_eq!(convert_quantity_to_volume(1.0).ok(), Some(100_000));
// 0.01 lot (micro lot) = 1,000 volume
assert_eq!(convert_quantity_to_volume(0.01).ok(), Some(1_000));
// 10.5 lots = 1,050,000 volume
assert_eq!(convert_quantity_to_volume(10.5).ok(), Some(1_050_000));
}
#[test]
fn test_convert_quantity_fractional_rounding() {
// 0.011 lots should round to 1,100 (not truncate to 1,099)
assert_eq!(convert_quantity_to_volume(0.011).ok(), Some(1_100));
// 0.015 lots = 1,500.0 exactly
assert_eq!(convert_quantity_to_volume(0.015).ok(), Some(1_500));
}
#[test]
fn test_convert_quantity_overflow() {
// Huge value should overflow i64
let result = convert_quantity_to_volume(f64::MAX);
assert!(result.is_err());
let err = result.err().map(|s| format!("{}", s.message()));
assert!(err.as_deref().unwrap_or("").contains("Volume overflow"));
}
#[test]
fn test_convert_quantity_negative() {
let result = convert_quantity_to_volume(-1.0);
assert!(result.is_err());
}
#[test]
fn test_convert_quantity_nan() {
let result = convert_quantity_to_volume(f64::NAN);
assert!(result.is_err());
}
#[test]
fn test_convert_quantity_infinity() {
let result = convert_quantity_to_volume(f64::INFINITY);
assert!(result.is_err());
let result_neg = convert_quantity_to_volume(f64::NEG_INFINITY);
assert!(result_neg.is_err());
}
#[test]
fn test_convert_quantity_zero() {
// Zero is technically valid (non-negative, finite)
assert_eq!(convert_quantity_to_volume(0.0).ok(), Some(0));
}
}

304
services/trading_service/src/services/enhanced_ml.rs
View File

@@ -1689,23 +1689,32 @@ struct RealTFTModel {
impl RealTFTModel {
/// Create new TFT model from checkpoint
///
/// NOTE: Checkpoint loading deferred to ml crate to avoid dependency issues
/// Trading service creates TFT with default config, marked as "trained"
/// Actual checkpoint loading will be handled by ml crate's TFT implementation
/// Validates that the checkpoint file exists before creating the model.
/// Returns an error if the checkpoint is missing -- never sets is_trained=true
/// on random weights.
pub fn from_checkpoint(
model_id: String,
checkpoint_path: &std::path::Path,
) -> ml::MLResult<Self> {
use ml::tft::{TFTConfig, TemporalFusionTransformer};
// Validate checkpoint file exists before proceeding
if !checkpoint_path.exists() {
return Err(ml::MLError::CheckpointError(format!(
"TFT checkpoint not found: {}. Cannot load model with random weights.",
checkpoint_path.display()
)));
}
info!(
"Initializing TFT model (checkpoint: {})",
checkpoint_path.display()
);
// TFT configuration matching Wave 160 training
// input_dim must equal num_static + num_known + num_unknown = 5 + 10 + 16 = 31
let config = TFTConfig {
input_dim: 16, // From feature engineering
input_dim: 31,
hidden_dim: 128,
num_heads: 8,
num_layers: 3,
@@ -1726,16 +1735,20 @@ impl RealTFTModel {
target_throughput_pps: 100_000,
};
// Create TFT model (checkpoint loading handled by ml crate)
let mut tft = TemporalFusionTransformer::new(config.clone())
// Create TFT model -- checkpoint weight loading not yet implemented in ml crate,
// so we do NOT set is_trained=true. The model will refuse to produce predictions
// until real weight loading is added.
let tft = TemporalFusionTransformer::new(config.clone())
.map_err(|e| ml::MLError::ModelError(format!("Failed to create TFT: {}", e)))?;
// Mark as trained (production deployment assumes trained checkpoints)
tft.is_trained = true;
info!(
"✅ Initialized TFT model {} (checkpoint loading deferred to ml crate)",
model_id
// NOTE: is_trained is NOT set to true. The checkpoint file exists but the ml crate
// does not yet support loading TFT weights from safetensors. Callers must check
// is_ready() before using this model.
warn!(
"TFT model {} created with architecture only -- safetensors weight loading not yet implemented in ml crate. \
Checkpoint exists at {} but weights are random. is_trained=false.",
model_id,
checkpoint_path.display()
);
Ok(Self {
@@ -1757,12 +1770,20 @@ impl MLModel for RealTFTModel {
}
async fn predict(&self, features: &Features) -> ml::MLResult<ModelPrediction> {
let tft = self.model.read().await;
// Refuse to predict with untrained (random weight) model
if !tft.is_trained {
return Err(ml::MLError::InferenceError(format!(
"TFT model '{}' has not been trained -- refusing to predict with random weights",
self.model_id
)));
}
// Simplified TFT prediction using feature aggregation
// Full TFT multi-horizon prediction requires ndarray (in ml crate)
// This wrapper provides basic signal for ensemble voting
let _tft = self.model.read().await;
// Simple prediction based on features (TFT multi-horizon deferred to ml crate)
// In production, this would call tft.predict_fast() with proper tensor conversion
let feature_sum: f64 = features.values.iter().sum();
@@ -1795,14 +1816,18 @@ impl MLModel for RealTFTModel {
}
fn is_ready(&self) -> bool {
true
// Only report ready if the model has actually been trained with real weights
self.model
.try_read()
.map(|m| m.is_trained)
.unwrap_or(false)
}
fn get_metadata(&self) -> ModelMetadata {
ModelMetadata {
model_type: ModelType::TFT,
version: "1.0.0".to_string(),
features_used: 16, // num_unknown_features
features_used: self.config.input_dim,
memory_usage_mb: 180.0, // Estimated for transformer architecture
additional_metadata: std::collections::HashMap::new(),
}
@@ -1823,12 +1848,24 @@ struct RealMamba2Model {
impl RealMamba2Model {
/// Create new Mamba2 model from checkpoint
///
/// Validates that the checkpoint file exists before creating the model.
/// Returns an error if the checkpoint is missing -- never sets is_trained=true
/// on random weights.
pub fn from_checkpoint(
model_id: String,
checkpoint_path: &std::path::Path,
) -> ml::MLResult<Self> {
use ml::mamba::{Mamba2Config, Mamba2SSM};
// Validate checkpoint file exists before proceeding
if !checkpoint_path.exists() {
return Err(ml::MLError::CheckpointError(format!(
"Mamba2 checkpoint not found: {}. Cannot load model with random weights.",
checkpoint_path.display()
)));
}
info!(
"Initializing Mamba2 model (checkpoint: {})",
checkpoint_path.display()
@@ -1838,15 +1875,19 @@ impl RealMamba2Model {
use ml::prelude::Device;
let device = Device::Cpu;
let mut mamba2 = Mamba2SSM::new(config.clone(), &device)
// Create Mamba2 model -- checkpoint weight loading not yet implemented,
// so we do NOT set is_trained=true.
let mamba2 = Mamba2SSM::new(config.clone(), &device)
.map_err(|e| ml::MLError::ModelError(format!("Failed to create Mamba2: {}", e)))?;
// Mark as trained (production deployment assumes trained checkpoints)
mamba2.is_trained = true;
info!(
"Initialized Mamba2 model {} (d_model={}, layers={})",
model_id, config.d_model, config.num_layers
// NOTE: is_trained is NOT set to true. The checkpoint file exists but the ml crate
// does not yet support loading Mamba2 weights from safetensors. Callers must check
// is_ready() before using this model.
warn!(
"Mamba2 model {} created with architecture only -- safetensors weight loading not yet implemented. \
Checkpoint exists at {} but weights are random. is_trained=false.",
model_id,
checkpoint_path.display()
);
Ok(Self {
@@ -1870,20 +1911,40 @@ impl MLModel for RealMamba2Model {
async fn predict(&self, features: &Features) -> ml::MLResult<ModelPrediction> {
let mut mamba2 = self.model.write().await;
// Refuse to predict with untrained (random weight) model
if !mamba2.is_trained {
return Err(ml::MLError::InferenceError(format!(
"Mamba2 model '{}' has not been trained -- refusing to predict with random weights",
self.model_id
)));
}
// Pad or truncate feature vector to match d_model
let d_model = mamba2.config.d_model;
let mut input = vec![0.0f64; d_model];
let copy_len = features.values.len().min(d_model);
input[..copy_len].copy_from_slice(&features.values[..copy_len]);
// Use fast single-sample prediction
// Use fast single-sample prediction -- propagate errors as InferenceError
// so the fallback manager can degrade model health
let raw_prediction = mamba2
.predict_single_fast(&input)
.map_err(|e| {
warn!(model_id = %self.model_id, error = %e, "Mamba2 inference failed");
e
ml::MLError::InferenceError(format!(
"Mamba2 model '{}' inference failed: {}",
self.model_id, e
))
})?;
// Validate prediction is finite
if !raw_prediction.is_finite() {
return Err(ml::MLError::InferenceError(format!(
"Mamba2 model '{}' produced non-finite prediction: {}",
self.model_id, raw_prediction
)));
}
// Normalize to 0-1 range using sigmoid
let prediction_value = (1.0 / (1.0 + (-raw_prediction).exp())).clamp(0.0, 1.0);
@@ -1904,7 +1965,12 @@ impl MLModel for RealMamba2Model {
}
fn is_ready(&self) -> bool {
true
// Only report ready if the model has actually been trained with real weights
// This is checked synchronously so we use try_read
self.model
.try_read()
.map(|m| m.is_trained)
.unwrap_or(false)
}
fn get_metadata(&self) -> ModelMetadata {
@@ -2627,4 +2693,190 @@ mod enhanced_ml_tests {
"Error message should indicate the feature is not connected"
);
}
// -----------------------------------------------------------------------
// 13. TFT from_checkpoint fails on nonexistent path (C5 fix)
// -----------------------------------------------------------------------
#[test]
fn test_tft_from_checkpoint_fails_on_nonexistent_path() {
let fake_path = std::path::Path::new("/tmp/nonexistent_tft_checkpoint_12345.safetensors");
let result = RealTFTModel::from_checkpoint("tft-test".to_string(), fake_path);
assert!(result.is_err(), "from_checkpoint must fail when checkpoint file does not exist");
let err_msg = format!("{}", result.unwrap_err());
assert!(
err_msg.contains("not found"),
"Error message should mention 'not found', got: {}",
err_msg
);
}
// -----------------------------------------------------------------------
// 14. Mamba2 from_checkpoint fails on nonexistent path (C5 fix)
// -----------------------------------------------------------------------
#[test]
fn test_mamba2_from_checkpoint_fails_on_nonexistent_path() {
let fake_path = std::path::Path::new("/tmp/nonexistent_mamba2_checkpoint_12345.safetensors");
let result = RealMamba2Model::from_checkpoint("mamba2-test".to_string(), fake_path);
assert!(result.is_err(), "from_checkpoint must fail when checkpoint file does not exist");
let err_msg = format!("{}", result.unwrap_err());
assert!(
err_msg.contains("not found"),
"Error message should mention 'not found', got: {}",
err_msg
);
}
// -----------------------------------------------------------------------
// 15. TFT from_checkpoint does NOT set is_trained on random weights (C5 fix)
// -----------------------------------------------------------------------
#[test]
fn test_tft_from_checkpoint_does_not_mark_trained_with_random_weights() {
// Create a real temp file so the existence check passes
let tmp_dir = std::env::temp_dir();
let checkpoint_path = tmp_dir.join("test_tft_checkpoint_c5.safetensors");
std::fs::write(&checkpoint_path, b"fake checkpoint data").unwrap();
let result = RealTFTModel::from_checkpoint("tft-test".to_string(), &checkpoint_path);
// Clean up
let _ = std::fs::remove_file(&checkpoint_path);
// The model should be created successfully...
let model = result.expect("TFT model creation should succeed when checkpoint exists");
// ...but is_trained must NOT be true (weights are random, not loaded from checkpoint)
let tft = model.model.try_read().unwrap();
assert!(
!tft.is_trained,
"TFT model must NOT be marked as trained when using random weights"
);
}
// -----------------------------------------------------------------------
// 16. Mamba2 from_checkpoint does NOT set is_trained on random weights (C5 fix)
// -----------------------------------------------------------------------
#[test]
fn test_mamba2_from_checkpoint_does_not_mark_trained_with_random_weights() {
// Create a real temp file so the existence check passes
let tmp_dir = std::env::temp_dir();
let checkpoint_path = tmp_dir.join("test_mamba2_checkpoint_c5.safetensors");
std::fs::write(&checkpoint_path, b"fake checkpoint data").unwrap();
let result = RealMamba2Model::from_checkpoint("mamba2-test".to_string(), &checkpoint_path);
// Clean up
let _ = std::fs::remove_file(&checkpoint_path);
// The model should be created successfully...
let model = result.expect("Mamba2 model creation should succeed when checkpoint exists");
// ...but is_trained must NOT be true (weights are random, not loaded from checkpoint)
let mamba2 = model.model.try_read().unwrap();
assert!(
!mamba2.is_trained,
"Mamba2 model must NOT be marked as trained when using random weights"
);
}
// -----------------------------------------------------------------------
// 17. Mamba2 predict refuses untrained model (M3 fix)
// -----------------------------------------------------------------------
#[tokio::test]
async fn test_mamba2_predict_refuses_untrained_model() {
// Create a real temp file so the existence check passes
let tmp_dir = std::env::temp_dir();
let checkpoint_path = tmp_dir.join("test_mamba2_predict_m3.safetensors");
std::fs::write(&checkpoint_path, b"fake checkpoint data").unwrap();
let model = RealMamba2Model::from_checkpoint("mamba2-test".to_string(), &checkpoint_path)
.expect("Mamba2 model creation should succeed");
let _ = std::fs::remove_file(&checkpoint_path);
// Model is untrained, predict should fail with InferenceError
let features = ml::Features {
values: vec![0.1, 0.2, 0.3],
names: vec!["a".to_string(), "b".to_string(), "c".to_string()],
timestamp: 0,
symbol: None,
};
let result = model.predict(&features).await;
assert!(result.is_err(), "predict must fail on untrained Mamba2 model");
let err_msg = format!("{}", result.unwrap_err());
assert!(
err_msg.contains("not been trained"),
"Error should mention model is not trained, got: {}",
err_msg
);
}
// -----------------------------------------------------------------------
// 18. Mamba2 is_ready returns false for untrained model (M3 fix)
// -----------------------------------------------------------------------
#[test]
fn test_mamba2_is_ready_false_when_untrained() {
// Create a real temp file so the existence check passes
let tmp_dir = std::env::temp_dir();
let checkpoint_path = tmp_dir.join("test_mamba2_ready_m3.safetensors");
std::fs::write(&checkpoint_path, b"fake checkpoint data").unwrap();
let model = RealMamba2Model::from_checkpoint("mamba2-test".to_string(), &checkpoint_path)
.expect("Mamba2 model creation should succeed");
let _ = std::fs::remove_file(&checkpoint_path);
assert!(
!model.is_ready(),
"Mamba2 model must report is_ready=false when weights are random"
);
}
// -----------------------------------------------------------------------
// 19. TFT predict refuses untrained model (C5/M3 fix)
// -----------------------------------------------------------------------
#[tokio::test]
async fn test_tft_predict_refuses_untrained_model() {
// Create a real temp file so the existence check passes
let tmp_dir = std::env::temp_dir();
let checkpoint_path = tmp_dir.join("test_tft_predict_c5.safetensors");
std::fs::write(&checkpoint_path, b"fake checkpoint data").unwrap();
let model = RealTFTModel::from_checkpoint("tft-test".to_string(), &checkpoint_path)
.expect("TFT model creation should succeed");
let _ = std::fs::remove_file(&checkpoint_path);
// Model is untrained, predict should fail with InferenceError
let features = ml::Features {
values: vec![0.1, 0.2, 0.3],
names: vec!["a".to_string(), "b".to_string(), "c".to_string()],
timestamp: 0,
symbol: None,
};
let result = model.predict(&features).await;
assert!(result.is_err(), "predict must fail on untrained TFT model");
let err_msg = format!("{}", result.unwrap_err());
assert!(
err_msg.contains("not been trained"),
"Error should mention model is not trained, got: {}",
err_msg
);
}
// -----------------------------------------------------------------------
// 20. TFT is_ready returns false for untrained model
// -----------------------------------------------------------------------
#[test]
fn test_tft_is_ready_false_when_untrained() {
// Create a real temp file so the existence check passes
let tmp_dir = std::env::temp_dir();
let checkpoint_path = tmp_dir.join("test_tft_ready_c5.safetensors");
std::fs::write(&checkpoint_path, b"fake checkpoint data").unwrap();
let model = RealTFTModel::from_checkpoint("tft-test".to_string(), &checkpoint_path)
.expect("TFT model creation should succeed");
let _ = std::fs::remove_file(&checkpoint_path);
assert!(
!model.is_ready(),
"TFT model must report is_ready=false when weights are random"
);
}
}

29
web-gateway/src/routes/auth.rs
View File

@@ -1,6 +1,7 @@
use axum::{extract::State, routing, Json, Router};
use jsonwebtoken::{encode, EncodingKey, Header};
use serde::{Deserialize, Serialize};
use tracing::warn;
use crate::auth::claims::Claims;
use crate::error::AppError;
@@ -23,11 +24,24 @@ pub fn router() -> Router<AppState> {
// SECURITY: This is a development-only login stub. Before production deployment,
// replace with real authentication (LDAP, OAuth2, bcrypt-verified credentials, etc.).
// Currently accepts any non-empty username/password combination.
// Gated behind FOXHUNT_DEV_AUTH=true — disabled by default.
async fn login(
State(state): State<AppState>,
Json(body): Json<LoginRequest>,
) -> Result<Json<LoginResponse>, AppError> {
// Gate: dev auth stub must be explicitly enabled
let dev_auth = std::env::var("FOXHUNT_DEV_AUTH").unwrap_or_default();
if dev_auth != "true" {
return Err(AppError::Internal(anyhow::anyhow!(
"Production auth provider not configured. Set FOXHUNT_DEV_AUTH=true for development stub."
)));
}
warn!(
"DEV AUTH STUB: Accepting login for '{}' without password verification.",
body.username
);
if body.username.is_empty() || body.password.is_empty() {
return Err(AppError::BadRequest("Username and password required".into()));
}
@@ -87,8 +101,14 @@ mod tests {
.with_state(state)
}
/// Enable the dev auth stub for tests that need it
fn enable_dev_auth() {
unsafe { std::env::set_var("FOXHUNT_DEV_AUTH", "true") };
}
#[tokio::test]
async fn test_login_returns_token() {
enable_dev_auth();
let app = test_app("test-secret");
let body = serde_json::json!({"username": "admin", "password": "pass123"});
@@ -109,6 +129,7 @@ mod tests {
#[tokio::test]
async fn test_login_empty_username_returns_400() {
enable_dev_auth();
let app = test_app("test-secret");
let body = serde_json::json!({"username": "", "password": "pass123"});
@@ -125,6 +146,7 @@ mod tests {
#[tokio::test]
async fn test_login_no_jwt_secret_returns_500() {
enable_dev_auth();
let app = test_app("");
let body = serde_json::json!({"username": "admin", "password": "pass123"});
@@ -138,4 +160,9 @@ mod tests {
let resp = app.oneshot(req).await.unwrap();
assert_eq!(resp.status(), StatusCode::INTERNAL_SERVER_ERROR);
}
// Note: negative test for FOXHUNT_DEV_AUTH=false is not included because
// env vars are process-global and tests run in parallel, causing races.
// The safety guarantee is enforced by the handler checking the env var at
// runtime — without FOXHUNT_DEV_AUTH=true, login returns 500.
}