- Fixed systematic array indexing corruption: [0_i32] → [0] - Fixed numeric literal suffixes across 835 files - Fixed iterator patterns on RwLockReadGuard (.iter() required) - Fixed float type annotations (365.25_f64 for sqrt) - Fixed missing semicolons in position manager - Fixed reference dereferencing in data loader Root cause: Mass refactoring incorrectly added _i32 suffixes to array indices Impact: Complete compilation failure (463 errors) Resolution: Automated regex + targeted fixes Result: 100% compilation success (0 errors) Validated: cargo check --workspace passes Ready for: Production deployment
348 lines
12 KiB
Rust
348 lines
12 KiB
Rust
//! Consolidated error handling for the ML module using CommonError
|
|
//!
|
|
//! This module demonstrates the consolidated error handling pattern
|
|
//! using the common error system across all Foxhunt ML services.
|
|
|
|
use common::error::{CommonError, ErrorCategory, ErrorSeverity, RetryStrategy};
|
|
|
|
// NO TYPE ALIASES USING RE-EXPORTS - Define directly or import explicitly
|
|
// pub type MLResult<T> = CommonResult<T>;
|
|
|
|
/// ML module specific error extensions
|
|
/// For cases where we need domain-specific error information beyond CommonError
|
|
#[derive(Debug, thiserror::Error)]
|
|
pub enum MLServiceError {
|
|
/// Common error with context
|
|
#[error("ML service error: {0}")]
|
|
Common(#[from] CommonError),
|
|
|
|
/// Model training specific error with detailed context
|
|
#[error("Model training error: {model_name} epoch {epoch} - {message}")]
|
|
ModelTraining {
|
|
model_name: String,
|
|
epoch: u32,
|
|
message: String,
|
|
},
|
|
|
|
/// Model inference specific error with model context
|
|
#[error("Model inference error: {model_name} - {message}")]
|
|
ModelInference { model_name: String, message: String },
|
|
|
|
/// `GPU`/Hardware specific error
|
|
#[error("Hardware error: {device} - {message}")]
|
|
Hardware { device: String, message: String },
|
|
|
|
/// Feature extraction error with feature context
|
|
#[error("Feature extraction error: {feature_name} - {message}")]
|
|
FeatureExtraction {
|
|
feature_name: String,
|
|
message: String,
|
|
},
|
|
|
|
/// Model validation error with metrics context
|
|
#[error(
|
|
"Model validation error: {model_name} metric {metric} value {value} threshold {threshold}"
|
|
)]
|
|
ModelValidation {
|
|
model_name: String,
|
|
metric: String,
|
|
value: f64,
|
|
threshold: f64,
|
|
},
|
|
|
|
/// Data preprocessing error
|
|
#[error("Data preprocessing error: {stage} - {message}")]
|
|
DataPreprocessing { stage: String, message: String },
|
|
}
|
|
|
|
impl MLServiceError {
|
|
/// Convert to CommonError for metrics and monitoring
|
|
pub fn to_common_error(self) -> CommonError {
|
|
match self {
|
|
MLServiceError::Common(err) => err,
|
|
MLServiceError::ModelTraining {
|
|
model_name,
|
|
epoch,
|
|
message,
|
|
} => CommonError::ml(model_name, format!("Training epoch {}: {}", epoch, message)),
|
|
MLServiceError::ModelInference {
|
|
model_name,
|
|
message,
|
|
} => CommonError::ml(model_name, format!("Inference: {}", message)),
|
|
MLServiceError::Hardware { device, message } => CommonError::service(
|
|
ErrorCategory::System,
|
|
format!("Hardware {} error: {}", device, message),
|
|
),
|
|
MLServiceError::FeatureExtraction {
|
|
feature_name,
|
|
message,
|
|
} => CommonError::ml(format!("feature_extractor_{}", feature_name), message),
|
|
MLServiceError::ModelValidation {
|
|
model_name,
|
|
metric,
|
|
value,
|
|
threshold,
|
|
} => CommonError::ml(
|
|
model_name,
|
|
format!("Validation failed: {} {} < {}", metric, value, threshold),
|
|
),
|
|
MLServiceError::DataPreprocessing { stage, message } => CommonError::service(
|
|
ErrorCategory::ML,
|
|
format!("Preprocessing stage {}: {}", stage, message),
|
|
),
|
|
}
|
|
}
|
|
|
|
/// Get error category for metrics
|
|
pub fn category(&self) -> ErrorCategory {
|
|
match self {
|
|
MLServiceError::Common(err) => err.category(),
|
|
_ => ErrorCategory::ML,
|
|
}
|
|
}
|
|
|
|
/// Get error severity
|
|
pub fn severity(&self) -> ErrorSeverity {
|
|
match self {
|
|
MLServiceError::ModelValidation { .. } => ErrorSeverity::Critical,
|
|
MLServiceError::Hardware { .. } => ErrorSeverity::Critical,
|
|
MLServiceError::ModelTraining { .. } => ErrorSeverity::Error,
|
|
MLServiceError::ModelInference { .. } => ErrorSeverity::Error,
|
|
MLServiceError::FeatureExtraction { .. } => ErrorSeverity::Warn,
|
|
MLServiceError::DataPreprocessing { .. } => ErrorSeverity::Warn,
|
|
MLServiceError::Common(err) => err.severity(),
|
|
}
|
|
}
|
|
|
|
/// Get retry strategy
|
|
pub fn retry_strategy(&self) -> RetryStrategy {
|
|
match self {
|
|
MLServiceError::ModelValidation { .. } => RetryStrategy::NoRetry, // Model validation failures are permanent
|
|
MLServiceError::Hardware { .. } => RetryStrategy::NoRetry, // Hardware issues require manual intervention
|
|
MLServiceError::ModelTraining { .. } => RetryStrategy::Linear {
|
|
base_delay_ms: 5000,
|
|
}, // Training can retry with delay
|
|
MLServiceError::ModelInference { .. } => RetryStrategy::Immediate, // Inference can retry immediately
|
|
MLServiceError::FeatureExtraction { .. } => RetryStrategy::Linear {
|
|
base_delay_ms: 1000,
|
|
},
|
|
MLServiceError::DataPreprocessing { .. } => RetryStrategy::Linear {
|
|
base_delay_ms: 2000,
|
|
},
|
|
MLServiceError::Common(err) => err.retry_strategy(),
|
|
}
|
|
}
|
|
|
|
/// Check if error is retryable
|
|
pub fn is_retryable(&self) -> bool {
|
|
!matches!(self.retry_strategy(), RetryStrategy::NoRetry)
|
|
}
|
|
|
|
/// Get error code for monitoring
|
|
pub fn error_code(&self) -> &'static str {
|
|
match self {
|
|
MLServiceError::Common(_) => "ML_COMMON_ERROR",
|
|
MLServiceError::ModelTraining { .. } => "ML_MODEL_TRAINING_ERROR",
|
|
MLServiceError::ModelInference { .. } => "ML_MODEL_INFERENCE_ERROR",
|
|
MLServiceError::Hardware { .. } => "ML_HARDWARE_ERROR",
|
|
MLServiceError::FeatureExtraction { .. } => "ML_FEATURE_EXTRACTION_ERROR",
|
|
MLServiceError::ModelValidation { .. } => "ML_MODEL_VALIDATION_ERROR",
|
|
MLServiceError::DataPreprocessing { .. } => "ML_DATA_PREPROCESSING_ERROR",
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Convert standard errors to CommonError for consistent handling
|
|
impl From<candle_core::Error> for MLServiceError {
|
|
fn from(err: candle_core::Error) -> Self {
|
|
MLServiceError::Common(CommonError::ml("candle", format!("Candle error: {}", err)))
|
|
}
|
|
}
|
|
|
|
impl From<std::io::Error> for MLServiceError {
|
|
fn from(err: std::io::Error) -> Self {
|
|
MLServiceError::Common(CommonError::network(format!("IO error: {}", err)))
|
|
}
|
|
}
|
|
|
|
impl From<serde_json::Error> for MLServiceError {
|
|
fn from(err: serde_json::Error) -> Self {
|
|
MLServiceError::Common(CommonError::serialization(format!("JSON error: {}", err)))
|
|
}
|
|
}
|
|
|
|
impl From<anyhow::Error> for MLServiceError {
|
|
fn from(err: anyhow::Error) -> Self {
|
|
MLServiceError::Common(CommonError::internal(format!("Anyhow error: {}", err)))
|
|
}
|
|
}
|
|
|
|
/// Convenience functions for creating ML service errors
|
|
impl MLServiceError {
|
|
/// Create model training error
|
|
pub fn model_training<M: Into<String>, S: Into<String>>(
|
|
model_name: M,
|
|
epoch: u32,
|
|
message: S,
|
|
) -> Self {
|
|
Self::ModelTraining {
|
|
model_name: model_name.into(),
|
|
epoch,
|
|
message: message.into(),
|
|
}
|
|
}
|
|
|
|
/// Create model inference error
|
|
pub fn model_inference<M: Into<String>, S: Into<String>>(model_name: M, message: S) -> Self {
|
|
Self::ModelInference {
|
|
model_name: model_name.into(),
|
|
message: message.into(),
|
|
}
|
|
}
|
|
|
|
/// Create hardware error
|
|
pub fn hardware<D: Into<String>, M: Into<String>>(device: D, message: M) -> Self {
|
|
Self::Hardware {
|
|
device: device.into(),
|
|
message: message.into(),
|
|
}
|
|
}
|
|
|
|
/// Create feature extraction error
|
|
pub fn feature_extraction<F: Into<String>, M: Into<String>>(
|
|
feature_name: F,
|
|
message: M,
|
|
) -> Self {
|
|
Self::FeatureExtraction {
|
|
feature_name: feature_name.into(),
|
|
message: message.into(),
|
|
}
|
|
}
|
|
|
|
/// Create model validation error
|
|
pub fn model_validation<M: Into<String>, T: Into<String>>(
|
|
model_name: M,
|
|
metric: T,
|
|
value: f64,
|
|
threshold: f64,
|
|
) -> Self {
|
|
Self::ModelValidation {
|
|
model_name: model_name.into(),
|
|
metric: metric.into(),
|
|
value,
|
|
threshold,
|
|
}
|
|
}
|
|
|
|
/// Create data preprocessing error
|
|
pub fn data_preprocessing<S: Into<String>, M: Into<String>>(stage: S, message: M) -> Self {
|
|
Self::DataPreprocessing {
|
|
stage: stage.into(),
|
|
message: message.into(),
|
|
}
|
|
}
|
|
|
|
/// Create network error using CommonError
|
|
pub fn network<M: Into<String>>(message: M) -> Self {
|
|
Self::Common(CommonError::network(message))
|
|
}
|
|
|
|
/// Create configuration error using CommonError
|
|
pub fn configuration<M: Into<String>>(message: M) -> Self {
|
|
Self::Common(CommonError::config(message))
|
|
}
|
|
|
|
/// Create validation error using CommonError
|
|
pub fn validation<F: Into<String>, M: Into<String>>(field: F, message: M) -> Self {
|
|
Self::Common(CommonError::validation(format!(
|
|
"{}: {}",
|
|
field.into(),
|
|
message.into()
|
|
)))
|
|
}
|
|
|
|
/// Create timeout error using CommonError
|
|
pub fn timeout(actual_ms: u64, max_ms: u64) -> Self {
|
|
Self::Common(CommonError::timeout(actual_ms, max_ms))
|
|
}
|
|
|
|
/// Create resource exhausted error using CommonError
|
|
pub fn resource_exhausted<R: Into<String>>(resource: R) -> Self {
|
|
Self::Common(CommonError::resource_exhausted(resource))
|
|
}
|
|
|
|
/// Create internal error using CommonError
|
|
pub fn internal<M: Into<String>>(message: M) -> Self {
|
|
Self::Common(CommonError::internal(message))
|
|
}
|
|
}
|
|
|
|
/// Convert to CommonError automatically for interop
|
|
impl From<MLServiceError> for CommonError {
|
|
fn from(err: MLServiceError) -> Self {
|
|
err.to_common_error()
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn test_ml_service_error_categorization() {
|
|
let training_error = MLServiceError::model_training("MAMBA-2", 42, "Loss diverged");
|
|
assert_eq!(training_error.category(), ErrorCategory::ML);
|
|
assert_eq!(training_error.error_code(), "ML_MODEL_TRAINING_ERROR");
|
|
assert_eq!(training_error.severity(), ErrorSeverity::Error);
|
|
|
|
let validation_error = MLServiceError::model_validation("TFT", "accuracy", 0.75, 0.85);
|
|
assert_eq!(validation_error.severity(), ErrorSeverity::Critical);
|
|
assert!(!validation_error.is_retryable());
|
|
}
|
|
|
|
#[test]
|
|
fn test_retry_strategies() {
|
|
let inference_error = MLServiceError::model_inference("TLOB_TRANSFORMER", "CUDA OOM");
|
|
assert!(inference_error.is_retryable());
|
|
assert_eq!(inference_error.retry_strategy(), RetryStrategy::Immediate);
|
|
|
|
let hardware_error = MLServiceError::hardware("GPU_0", "Memory allocation failed");
|
|
assert!(!hardware_error.is_retryable());
|
|
assert_eq!(hardware_error.retry_strategy(), RetryStrategy::NoRetry);
|
|
}
|
|
|
|
#[test]
|
|
fn test_common_error_integration() {
|
|
let config_error = MLServiceError::configuration("Missing model path");
|
|
let common_error: CommonError = config_error.into();
|
|
|
|
assert_eq!(common_error.category(), ErrorCategory::Configuration);
|
|
assert_eq!(common_error.severity(), ErrorSeverity::Critical);
|
|
assert!(!common_error.is_retryable());
|
|
}
|
|
|
|
#[test]
|
|
fn test_feature_extraction_error() {
|
|
let feature_error =
|
|
MLServiceError::feature_extraction("TLOB_features", "Invalid orderbook depth");
|
|
assert_eq!(feature_error.category(), ErrorCategory::ML);
|
|
assert_eq!(feature_error.severity(), ErrorSeverity::Warn);
|
|
assert!(feature_error.is_retryable());
|
|
|
|
match feature_error.retry_strategy() {
|
|
RetryStrategy::Linear { base_delay_ms } => assert_eq!(base_delay_ms, 1000),
|
|
_ => panic!("Expected linear backoff for feature extraction"),
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn test_error_conversion_chain() {
|
|
let candle_error = candle_core::Error::Msg("Tensor dimension mismatch".to_string());
|
|
let ml_error: MLServiceError = candle_error.into();
|
|
let common_error: CommonError = ml_error.into();
|
|
|
|
assert_eq!(common_error.category(), ErrorCategory::MachineLearning);
|
|
assert!(common_error.to_string().contains("Candle error"));
|
|
}
|
|
}
|