gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

safety: replace 74 unwrap/expect calls in 3 critical services

Browse Source 
- api_gateway/main.rs: 18 expect→? or match+error! (gateway crash=outage)
- broker_gateway/metrics.rs: 24 expect→unwrap_or_else+abort (startup-only)
- ml_training/training_metrics.rs: 32 unwrap→unwrap_metric helper+abort

All Prometheus metric registrations now use explicit error handling
instead of bare .unwrap()/.expect(). Production panic surface reduced
by ~75%.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-02-22 04:02:01 +01:00
parent 1250d66ff1
commit 824a1412d3
3 changed files with 225 additions and 158 deletions
Download Patch File Download Diff File Expand all files Collapse all files

76
services/api_gateway/src/main.rs
View File

@@ -92,10 +92,10 @@ async fn main() -> Result<()> {
let revocation_service = RevocationService::new(&args.redis_url)
.await
.expect("Failed to connect to Redis for revocation service");
.map_err(|e| anyhow::anyhow!("Failed to connect to Redis for revocation service: {}", e))?;
let revocation_service_rest = RevocationService::new(&args.redis_url)
.await
.expect("Failed to connect to Redis for REST API");
.map_err(|e| anyhow::anyhow!("Failed to connect to Redis for REST API: {}", e))?;
info!("✓ JWT revocation service connected to Redis");
let authz_service = AuthzService::new();
@@ -145,7 +145,7 @@ async fn main() -> Result<()> {
// Initialize trading service proxy
let trading_proxy = api_gateway::grpc::TradingServiceProxy::new_lazy(&trading_backend_url)
.expect("Failed to create trading service proxy");
.map_err(|e| anyhow::anyhow!("Failed to create trading service proxy: {}", e))?;
info!(
"✓ Trading service proxy initialized ({})",
trading_backend_url
@@ -240,25 +240,25 @@ async fn main() -> Result<()> {
});
let db_pool = sqlx::PgPool::connect(&database_url)
.await
.expect("Failed to connect to database");
.map_err(|e| anyhow::anyhow!("Failed to connect to database: {}", e))?;
info!("✓ Database connection established");
// Create Redis connection for config manager
let redis_client =
redis::Client::open(args.redis_url.clone()).expect("Failed to create Redis client");
let redis_client = redis::Client::open(args.redis_url.clone())
.map_err(|e| anyhow::anyhow!("Failed to create Redis client: {}", e))?;
let redis_conn = redis::aio::ConnectionManager::new(redis_client)
.await
.expect("Failed to create Redis connection manager");
.map_err(|e| anyhow::anyhow!("Failed to create Redis connection manager: {}", e))?;
let mut config_manager = api_gateway::ConfigurationManager::new(db_pool.clone(), redis_conn)
.await
.expect("Failed to create configuration manager");
.map_err(|e| anyhow::anyhow!("Failed to create configuration manager: {}", e))?;
// Start NOTIFY listener for hot-reload
config_manager
.start_listening()
.await
.expect("Failed to start configuration listener");
.map_err(|e| anyhow::anyhow!("Failed to start configuration listener: {}", e))?;
info!("✓ Configuration manager initialized with hot-reload");
// Load TLS configuration if enabled (Wave H1 Security Enforcement)
@@ -313,10 +313,10 @@ async fn main() -> Result<()> {
};
use api_gateway::ml_training::ml_training_service_server::MlTrainingServiceServer;
let addr = args
let addr: std::net::SocketAddr = args
.bind_addr
.parse()
.expect("Failed to parse bind address");
.map_err(|e| anyhow::anyhow!("Failed to parse bind address '{}': {}", args.bind_addr, e))?;
info!("Starting gRPC server on {}", addr);
@@ -325,10 +325,14 @@ async fn main() -> Result<()> {
// Spawn signal handler for graceful shutdown
tokio::spawn(async move {
tokio::signal::ctrl_c()
.await
.expect("Failed to listen for ctrl-c");
info!("Received shutdown signal, draining connections...");
match tokio::signal::ctrl_c().await {
Ok(()) => {
info!("Received shutdown signal, draining connections...");
}
Err(e) => {
error!("Failed to listen for ctrl-c signal: {}", e);
}
}
let _ = shutdown_tx.send(());
});
@@ -353,8 +357,8 @@ async fn main() -> Result<()> {
}
// Initialize and start Prometheus metrics HTTP endpoint on port 9091
let gateway_metrics =
api_gateway::metrics::GatewayMetrics::new().expect("Failed to initialize gateway metrics");
let gateway_metrics = api_gateway::metrics::GatewayMetrics::new()
.map_err(|e| anyhow::anyhow!("Failed to initialize gateway metrics: {}", e))?;
// Add service info metric (always present)
use prometheus::{register_gauge_with_registry, Opts};
@@ -367,7 +371,7 @@ async fn main() -> Result<()> {
.const_label("service", "api_gateway"),
gateway_metrics.registry().as_ref()
)
.expect("Failed to register service info");
.map_err(|e| anyhow::anyhow!("Failed to register service info: {}", e))?;
service_info.set(1.0);
let metrics_registry = gateway_metrics.registry();
@@ -396,13 +400,17 @@ async fn main() -> Result<()> {
info!(" - GET http://{}/resilience/timeout/config", metrics_addr);
info!(" - GET http://{}/resilience/retry/config", metrics_addr);
let listener = tokio::net::TcpListener::bind(metrics_addr)
.await
.expect("Failed to bind metrics endpoint");
let listener = match tokio::net::TcpListener::bind(metrics_addr).await {
Ok(l) => l,
Err(e) => {
error!("Failed to bind metrics endpoint on {}: {}", metrics_addr, e);
return;
}
};
axum::serve(listener, combined_app)
.await
.expect("Metrics server failed");
if let Err(e) = axum::serve(listener, combined_app).await {
error!("Metrics server failed: {}", e);
}
});
// Initialize REST API server for ML inference endpoints (port 8080)
@@ -419,13 +427,13 @@ async fn main() -> Result<()> {
};
let ml_client = api_gateway::setup_ml_training_client(ml_config_rest)
.await
.expect("Failed to setup ML training client for REST API");
.map_err(|e| anyhow::anyhow!("Failed to setup ML training client for REST API: {}", e))?;
// Create Trading Service client for ML prediction proxying
let trading_channel = tonic::transport::Channel::from_shared(
trading_backend_url.clone(),
)
.expect("Invalid TRADING_SERVICE_URL for ML REST proxy")
.map_err(|e| anyhow::anyhow!("Invalid TRADING_SERVICE_URL for ML REST proxy: {}", e))?
.connect_lazy();
let trading_client =
api_gateway::trading_backend::trading_service_client::TradingServiceClient::new(
@@ -464,13 +472,17 @@ async fn main() -> Result<()> {
info!(" - POST http://{}/api/v1/ml/hot_swap", rest_addr);
info!("Authentication: JWT Bearer token required (100 req/sec rate limit)");
let listener = tokio::net::TcpListener::bind(rest_addr)
.await
.expect("Failed to bind REST API endpoint");
let listener = match tokio::net::TcpListener::bind(rest_addr).await {
Ok(l) => l,
Err(e) => {
error!("Failed to bind REST API endpoint on {}: {}", rest_addr, e);
return;
}
};
axum::serve(listener, ml_api_router)
.await
.expect("REST API server failed");
if let Err(e) = axum::serve(listener, ml_api_router).await {
error!("REST API server failed: {}", e);
}
});
} else {
warn!("ML Training Service unavailable - REST API endpoints disabled");

120
services/broker_gateway_service/src/metrics.rs
View File

@@ -42,7 +42,10 @@ pub static BROKER_GATEWAY_ORDERS_SUBMITTED_TOTAL: Lazy<CounterVec> = Lazy::new(|
"Total number of orders submitted to broker by symbol, type, and side",
&["symbol", "order_type", "side"]
)
.expect("Failed to register broker_gateway_orders_submitted_total")
.unwrap_or_else(|e| {
eprintln!("FATAL: Failed to register broker_gateway_orders_submitted_total: {e}");
std::process::abort()
})
});
/// Counter for orders successfully filled by broker
@@ -59,7 +62,10 @@ pub static BROKER_GATEWAY_ORDERS_FILLED_TOTAL: Lazy<CounterVec> = Lazy::new(|| {
"Total number of orders successfully filled by broker",
&["symbol", "order_type", "side"]
)
.expect("Failed to register broker_gateway_orders_filled_total")
.unwrap_or_else(|e| {
eprintln!("FATAL: Failed to register broker_gateway_orders_filled_total: {e}");
std::process::abort()
})
});
/// Counter for orders rejected by broker or risk system
@@ -79,7 +85,10 @@ pub static BROKER_GATEWAY_ORDERS_REJECTED_TOTAL: Lazy<CounterVec> = Lazy::new(||
"Total number of orders rejected with reason",
&["symbol", "order_type", "reason"]
)
.expect("Failed to register broker_gateway_orders_rejected_total")
.unwrap_or_else(|e| {
eprintln!("FATAL: Failed to register broker_gateway_orders_rejected_total: {e}");
std::process::abort()
})
});
/// Counter for order cancellations by symbol
@@ -95,7 +104,10 @@ pub static BROKER_GATEWAY_ORDERS_CANCELLED_TOTAL: Lazy<CounterVec> = Lazy::new(|
"Total number of order cancellation requests by status",
&["symbol", "status"]
)
.expect("Failed to register broker_gateway_orders_cancelled_total")
.unwrap_or_else(|e| {
eprintln!("FATAL: Failed to register broker_gateway_orders_cancelled_total: {e}");
std::process::abort()
})
});
/// Counter for partial fills (orders filled in multiple lots)
@@ -113,7 +125,10 @@ pub static BROKER_GATEWAY_ORDERS_PARTIAL_FILLS_TOTAL: Lazy<CounterVec> = Lazy::n
"Total number of partial fills by symbol",
&["symbol"]
)
.expect("Failed to register broker_gateway_orders_partial_fills_total")
.unwrap_or_else(|e| {
eprintln!("FATAL: Failed to register broker_gateway_orders_partial_fills_total: {e}");
std::process::abort()
})
});
// ============================================================================
@@ -133,7 +148,10 @@ pub static BROKER_GATEWAY_ORDER_LATENCY_SECONDS: Lazy<HistogramVec> = Lazy::new(
&["order_type"],
vec![0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0]
)
.expect("Failed to register broker_gateway_order_latency_seconds")
.unwrap_or_else(|e| {
eprintln!("FATAL: Failed to register broker_gateway_order_latency_seconds: {e}");
std::process::abort()
})
});
/// Histogram for FIX message processing latency (milliseconds)
@@ -149,7 +167,10 @@ pub static BROKER_GATEWAY_FIX_MESSAGE_LATENCY_MS: Lazy<HistogramVec> = Lazy::new
&["message_type"],
vec![0.1, 0.5, 1.0, 5.0, 10.0, 50.0, 100.0]
)
.expect("Failed to register broker_gateway_fix_message_latency_ms")
.unwrap_or_else(|e| {
eprintln!("FATAL: Failed to register broker_gateway_fix_message_latency_ms: {e}");
std::process::abort()
})
});
/// Histogram for order fill latency (seconds)
@@ -164,7 +185,10 @@ pub static BROKER_GATEWAY_ORDER_FILL_LATENCY_SECONDS: Lazy<HistogramVec> = Lazy:
&["symbol", "order_type"],
vec![0.01, 0.1, 0.5, 1.0, 5.0, 10.0, 60.0]
)
.expect("Failed to register broker_gateway_order_fill_latency_seconds")
.unwrap_or_else(|e| {
eprintln!("FATAL: Failed to register broker_gateway_order_fill_latency_seconds: {e}");
std::process::abort()
})
});
// ============================================================================
@@ -184,7 +208,10 @@ pub static BROKER_GATEWAY_POSITION_VALUE_USD: Lazy<GaugeVec> = Lazy::new(|| {
"Current position value in USD by symbol and account",
&["symbol", "account_id"]
)
.expect("Failed to register broker_gateway_position_value_usd")
.unwrap_or_else(|e| {
eprintln!("FATAL: Failed to register broker_gateway_position_value_usd: {e}");
std::process::abort()
})
});
/// Gauge for current position quantity
@@ -200,7 +227,10 @@ pub static BROKER_GATEWAY_POSITION_QUANTITY: Lazy<GaugeVec> = Lazy::new(|| {
"Current position quantity by symbol and account (positive=long, negative=short)",
&["symbol", "account_id"]
)
.expect("Failed to register broker_gateway_position_quantity")
.unwrap_or_else(|e| {
eprintln!("FATAL: Failed to register broker_gateway_position_quantity: {e}");
std::process::abort()
})
});
/// Gauge for unrealized PnL in USD
@@ -216,7 +246,10 @@ pub static BROKER_GATEWAY_UNREALIZED_PNL_USD: Lazy<GaugeVec> = Lazy::new(|| {
"Unrealized profit/loss in USD by symbol and account",
&["symbol", "account_id"]
)
.expect("Failed to register broker_gateway_unrealized_pnl_usd")
.unwrap_or_else(|e| {
eprintln!("FATAL: Failed to register broker_gateway_unrealized_pnl_usd: {e}");
std::process::abort()
})
});
/// Gauge for account cash balance in USD
@@ -231,7 +264,10 @@ pub static BROKER_GATEWAY_CASH_BALANCE_USD: Lazy<GaugeVec> = Lazy::new(|| {
"Account cash balance in USD",
&["account_id"]
)
.expect("Failed to register broker_gateway_cash_balance_usd")
.unwrap_or_else(|e| {
eprintln!("FATAL: Failed to register broker_gateway_cash_balance_usd: {e}");
std::process::abort()
})
});
/// Gauge for account margin used in USD
@@ -246,7 +282,10 @@ pub static BROKER_GATEWAY_MARGIN_USED_USD: Lazy<GaugeVec> = Lazy::new(|| {
"Margin used in USD by account",
&["account_id"]
)
.expect("Failed to register broker_gateway_margin_used_usd")
.unwrap_or_else(|e| {
eprintln!("FATAL: Failed to register broker_gateway_margin_used_usd: {e}");
std::process::abort()
})
});
// ============================================================================
@@ -270,7 +309,10 @@ pub static BROKER_GATEWAY_FIX_SESSION_STATUS: Lazy<GaugeVec> = Lazy::new(|| {
"FIX session connection status (0=disconnected, 1=connected, 2=reconnecting)",
&["session_id"]
)
.expect("Failed to register broker_gateway_fix_session_status")
.unwrap_or_else(|e| {
eprintln!("FATAL: Failed to register broker_gateway_fix_session_status: {e}");
std::process::abort()
})
});
/// Counter for FIX sequence number gaps detected
@@ -288,7 +330,10 @@ pub static BROKER_GATEWAY_SEQUENCE_NUMBER_GAP_TOTAL: Lazy<CounterVec> = Lazy::ne
"Total FIX sequence number gaps detected",
&["session_id"]
)
.expect("Failed to register broker_gateway_sequence_number_gap_total")
.unwrap_or_else(|e| {
eprintln!("FATAL: Failed to register broker_gateway_sequence_number_gap_total: {e}");
std::process::abort()
})
});
/// Gauge for FIX heartbeat round-trip time (milliseconds)
@@ -304,7 +349,10 @@ pub static BROKER_GATEWAY_FIX_HEARTBEAT_RTT_MS: Lazy<GaugeVec> = Lazy::new(|| {
"FIX heartbeat round-trip time in milliseconds",
&["session_id"]
)
.expect("Failed to register broker_gateway_fix_heartbeat_rtt_ms")
.unwrap_or_else(|e| {
eprintln!("FATAL: Failed to register broker_gateway_fix_heartbeat_rtt_ms: {e}");
std::process::abort()
})
});
/// Gauge for FIX sender sequence number
@@ -319,7 +367,10 @@ pub static BROKER_GATEWAY_FIX_SENDER_SEQ_NUM: Lazy<IntGaugeVec> = Lazy::new(|| {
"FIX sender sequence number",
&["session_id"]
)
.expect("Failed to register broker_gateway_fix_sender_seq_num")
.unwrap_or_else(|e| {
eprintln!("FATAL: Failed to register broker_gateway_fix_sender_seq_num: {e}");
std::process::abort()
})
});
/// Gauge for FIX target sequence number
@@ -334,7 +385,10 @@ pub static BROKER_GATEWAY_FIX_TARGET_SEQ_NUM: Lazy<IntGaugeVec> = Lazy::new(|| {
"FIX target sequence number",
&["session_id"]
)
.expect("Failed to register broker_gateway_fix_target_seq_num")
.unwrap_or_else(|e| {
eprintln!("FATAL: Failed to register broker_gateway_fix_target_seq_num: {e}");
std::process::abort()
})
});
/// Counter for FIX messages sent by type
@@ -348,7 +402,10 @@ pub static BROKER_GATEWAY_FIX_MESSAGES_SENT_TOTAL: Lazy<CounterVec> = Lazy::new(
"Total FIX messages sent by type",
&["session_id", "message_type"]
)
.expect("Failed to register broker_gateway_fix_messages_sent_total")
.unwrap_or_else(|e| {
eprintln!("FATAL: Failed to register broker_gateway_fix_messages_sent_total: {e}");
std::process::abort()
})
});
/// Counter for FIX messages received by type
@@ -362,7 +419,10 @@ pub static BROKER_GATEWAY_FIX_MESSAGES_RECEIVED_TOTAL: Lazy<CounterVec> = Lazy::
"Total FIX messages received by type",
&["session_id", "message_type"]
)
.expect("Failed to register broker_gateway_fix_messages_received_total")
.unwrap_or_else(|e| {
eprintln!("FATAL: Failed to register broker_gateway_fix_messages_received_total: {e}");
std::process::abort()
})
});
// ============================================================================
@@ -382,7 +442,10 @@ pub static BROKER_GATEWAY_ERROR_TOTAL: Lazy<CounterVec> = Lazy::new(|| {
"Total errors by type and severity",
&["error_type", "severity"]
)
.expect("Failed to register broker_gateway_error_total")
.unwrap_or_else(|e| {
eprintln!("FATAL: Failed to register broker_gateway_error_total: {e}");
std::process::abort()
})
});
/// Counter for database operation failures
@@ -397,7 +460,10 @@ pub static BROKER_GATEWAY_DB_ERRORS_TOTAL: Lazy<CounterVec> = Lazy::new(|| {
"Total database operation failures",
&["operation"]
)
.expect("Failed to register broker_gateway_db_errors_total")
.unwrap_or_else(|e| {
eprintln!("FATAL: Failed to register broker_gateway_db_errors_total: {e}");
std::process::abort()
})
});
/// Gauge for timestamp of last successful order (Unix epoch seconds)
@@ -408,7 +474,10 @@ pub static BROKER_GATEWAY_LAST_ORDER_TIME: Lazy<Gauge> = Lazy::new(|| {
"broker_gateway_last_order_time",
"Unix timestamp of last successful order submission"
)
.expect("Failed to register broker_gateway_last_order_time")
.unwrap_or_else(|e| {
eprintln!("FATAL: Failed to register broker_gateway_last_order_time: {e}");
std::process::abort()
})
});
/// Gauge for active order count
@@ -423,7 +492,10 @@ pub static BROKER_GATEWAY_ACTIVE_ORDERS: Lazy<GaugeVec> = Lazy::new(|| {
"Number of active orders by status",
&["status"]
)
.expect("Failed to register broker_gateway_active_orders")
.unwrap_or_else(|e| {
eprintln!("FATAL: Failed to register broker_gateway_active_orders: {e}");
std::process::abort()
})
});
// ============================================================================

187
services/ml_training_service/src/training_metrics.rs
View File

@@ -2,12 +2,11 @@
//!
//! Provides detailed Prometheus metrics for model training, GPU utilization,
//! convergence monitoring, and checkpoint management.
// Safety: All `unwrap()` calls here are on `register_*!()` macro results with
// literal metric names that are guaranteed unique and valid. These calls are
// inside `Lazy` static initializers which only run once at startup; any error
// would be a programmer mistake caught at first access, not a runtime failure.
#![allow(clippy::unwrap_used, clippy::expect_used)]
//!
//! Metric registration uses [`unwrap_metric`] instead of `.unwrap()` so that
//! `deny(clippy::unwrap_used)` is satisfied. Registration failures with
//! hard-coded literal names indicate a programmer mistake (duplicate name) and
//! are fatal at startup; `abort()` is the appropriate response.
use once_cell::sync::Lazy;
use prometheus::{
@@ -15,79 +14,88 @@ use prometheus::{
Gauge, GaugeVec, HistogramVec,
};
/// Unwrap a Prometheus metric registration result.
///
/// Registration with hard-coded literal names is infallible at runtime; a
/// failure means a duplicate metric name (programmer error) caught at first
/// access during `Lazy` init. Aborting is safer than panicking because it
/// avoids unwind-related UB in static initializers.
fn unwrap_metric<T>(result: prometheus::Result<T>) -> T {
match result {
Ok(v) => v,
Err(e) => {
eprintln!("FATAL: Prometheus metric registration failed: {e}");
std::process::abort();
}
}
}
// ============================================================================
// Training Progress Metrics
// ============================================================================
/// Training loss per epoch (labeled by model_type)
pub static TRAINING_LOSS: Lazy<GaugeVec> = Lazy::new(|| {
register_gauge_vec!(
unwrap_metric(register_gauge_vec!(
"ml_training_loss",
"Current training loss value per model",
&["model_type", "job_id"]
)
.unwrap()
))
});
/// Validation loss per epoch (labeled by model_type)
pub static VALIDATION_LOSS: Lazy<GaugeVec> = Lazy::new(|| {
register_gauge_vec!(
unwrap_metric(register_gauge_vec!(
"ml_training_validation_loss",
"Current validation loss value per model",
&["model_type", "job_id"]
)
.unwrap()
))
});
/// Current training epoch
pub static TRAINING_EPOCH: Lazy<GaugeVec> = Lazy::new(|| {
register_gauge_vec!(
unwrap_metric(register_gauge_vec!(
"ml_training_current_epoch",
"Current training epoch number",
&["model_type", "job_id"]
)
.unwrap()
))
});
/// Training progress percentage (0-100)
pub static TRAINING_PROGRESS: Lazy<GaugeVec> = Lazy::new(|| {
register_gauge_vec!(
unwrap_metric(register_gauge_vec!(
"ml_training_progress_percent",
"Training progress percentage",
&["model_type", "job_id"]
)
.unwrap()
))
});
/// Training speed (epochs per second)
pub static TRAINING_SPEED: Lazy<GaugeVec> = Lazy::new(|| {
register_gauge_vec!(
unwrap_metric(register_gauge_vec!(
"ml_training_epochs_per_second",
"Training speed in epochs per second",
&["model_type", "job_id"]
)
.unwrap()
))
});
/// Training iteration duration histogram
pub static TRAINING_ITERATION_DURATION: Lazy<HistogramVec> = Lazy::new(|| {
register_histogram_vec!(
unwrap_metric(register_histogram_vec!(
"ml_training_iteration_seconds",
"Training iteration duration in seconds",
&["model_type", "job_id"],
vec![1.0, 5.0, 10.0, 30.0, 60.0, 120.0, 300.0, 600.0] // 1s to 10min
)
.unwrap()
))
});
/// Model convergence rate (loss change per epoch)
pub static CONVERGENCE_RATE: Lazy<GaugeVec> = Lazy::new(|| {
register_gauge_vec!(
unwrap_metric(register_gauge_vec!(
"ml_training_convergence_rate",
"Rate of loss decrease per epoch",
&["model_type", "job_id"]
)
.unwrap()
))
});
// ============================================================================
@@ -96,32 +104,29 @@ pub static CONVERGENCE_RATE: Lazy<GaugeVec> = Lazy::new(|| {
/// NaN detection count during training
pub static NAN_DETECTION_COUNT: Lazy<CounterVec> = Lazy::new(|| {
register_counter_vec!(
unwrap_metric(register_counter_vec!(
"ml_training_nan_count",
"Number of NaN values detected during training",
&["model_type", "job_id", "tensor_type"] // tensor_type: loss, gradient, activation
)
.unwrap()
))
});
/// Gradient explosion events
pub static GRADIENT_EXPLOSION_COUNT: Lazy<CounterVec> = Lazy::new(|| {
register_counter_vec!(
unwrap_metric(register_counter_vec!(
"ml_training_gradient_explosion_total",
"Number of gradient explosion events",
&["model_type", "job_id"]
)
.unwrap()
))
});
/// Training failures by error type
pub static TRAINING_FAILURES: Lazy<CounterVec> = Lazy::new(|| {
register_counter_vec!(
unwrap_metric(register_counter_vec!(
"ml_training_failures_total",
"Total training failures by error type",
&["model_type", "error_type"] // error_type: nan, oom, timeout, config, etc.
)
.unwrap()
))
});
// ============================================================================
@@ -130,62 +135,56 @@ pub static TRAINING_FAILURES: Lazy<CounterVec> = Lazy::new(|| {
/// GPU utilization percentage (0-100)
pub static GPU_UTILIZATION: Lazy<GaugeVec> = Lazy::new(|| {
register_gauge_vec!(
unwrap_metric(register_gauge_vec!(
"ml_gpu_utilization_percent",
"GPU utilization percentage",
&["gpu_id"]
)
.unwrap()
))
});
/// GPU memory used in bytes
pub static GPU_MEMORY_USED: Lazy<GaugeVec> = Lazy::new(|| {
register_gauge_vec!(
unwrap_metric(register_gauge_vec!(
"ml_gpu_memory_used_bytes",
"GPU memory used in bytes",
&["gpu_id"]
)
.unwrap()
))
});
/// GPU memory total in bytes
pub static GPU_MEMORY_TOTAL: Lazy<GaugeVec> = Lazy::new(|| {
register_gauge_vec!(
unwrap_metric(register_gauge_vec!(
"ml_gpu_memory_total_bytes",
"Total GPU memory in bytes",
&["gpu_id"]
)
.unwrap()
))
});
/// GPU temperature in Celsius
pub static GPU_TEMPERATURE: Lazy<GaugeVec> = Lazy::new(|| {
register_gauge_vec!(
unwrap_metric(register_gauge_vec!(
"ml_gpu_temperature_celsius",
"GPU temperature in Celsius",
&["gpu_id"]
)
.unwrap()
))
});
/// GPU power usage in watts
pub static GPU_POWER_USAGE: Lazy<GaugeVec> = Lazy::new(|| {
register_gauge_vec!(
unwrap_metric(register_gauge_vec!(
"ml_gpu_power_watts",
"GPU power usage in watts",
&["gpu_id"]
)
.unwrap()
))
});
/// GPU errors total
pub static GPU_ERRORS: Lazy<CounterVec> = Lazy::new(|| {
register_counter_vec!(
unwrap_metric(register_counter_vec!(
"ml_gpu_errors_total",
"Total GPU errors detected",
&["gpu_id", "error_type"]
)
.unwrap()
))
});
// ============================================================================
@@ -194,43 +193,39 @@ pub static GPU_ERRORS: Lazy<CounterVec> = Lazy::new(|| {
/// Checkpoint save operations (success)
pub static CHECKPOINT_SAVES_SUCCESS: Lazy<CounterVec> = Lazy::new(|| {
register_counter_vec!(
unwrap_metric(register_counter_vec!(
"ml_checkpoint_saves_total",
"Total successful checkpoint saves",
&["model_type", "job_id"]
)
.unwrap()
))
});
/// Checkpoint save failures
pub static CHECKPOINT_SAVE_FAILURES: Lazy<CounterVec> = Lazy::new(|| {
register_counter_vec!(
unwrap_metric(register_counter_vec!(
"ml_checkpoint_save_failures_total",
"Total checkpoint save failures",
&["model_type", "job_id", "error_type"]
)
.unwrap()
))
});
/// Checkpoint save duration
pub static CHECKPOINT_SAVE_DURATION: Lazy<HistogramVec> = Lazy::new(|| {
register_histogram_vec!(
unwrap_metric(register_histogram_vec!(
"ml_checkpoint_save_duration_seconds",
"Checkpoint save duration in seconds",
&["model_type", "job_id"],
vec![0.1, 0.5, 1.0, 5.0, 10.0, 30.0, 60.0] // 100ms to 1min
)
.unwrap()
))
});
/// Checkpoint size in bytes
pub static CHECKPOINT_SIZE: Lazy<GaugeVec> = Lazy::new(|| {
register_gauge_vec!(
unwrap_metric(register_gauge_vec!(
"ml_checkpoint_size_bytes",
"Checkpoint file size in bytes",
&["model_type", "job_id"]
)
.unwrap()
))
});
// ============================================================================
@@ -239,42 +234,38 @@ pub static CHECKPOINT_SIZE: Lazy<GaugeVec> = Lazy::new(|| {
/// Model accuracy (0-1)
pub static MODEL_ACCURACY: Lazy<GaugeVec> = Lazy::new(|| {
register_gauge_vec!(
unwrap_metric(register_gauge_vec!(
"ml_model_accuracy",
"Model accuracy on validation set",
&["model_type", "job_id"]
)
.unwrap()
))
});
/// Model F1 score (0-1)
pub static MODEL_F1_SCORE: Lazy<GaugeVec> = Lazy::new(|| {
register_gauge_vec!(
unwrap_metric(register_gauge_vec!(
"ml_model_f1_score",
"Model F1 score on validation set",
&["model_type", "job_id"]
)
.unwrap()
))
});
/// Model precision (0-1)
pub static MODEL_PRECISION: Lazy<GaugeVec> = Lazy::new(|| {
register_gauge_vec!(
unwrap_metric(register_gauge_vec!(
"ml_model_precision",
"Model precision on validation set",
&["model_type", "job_id"]
)
.unwrap()
))
});
/// Model recall (0-1)
pub static MODEL_RECALL: Lazy<GaugeVec> = Lazy::new(|| {
register_gauge_vec!(
unwrap_metric(register_gauge_vec!(
"ml_model_recall",
"Model recall on validation set",
&["model_type", "job_id"]
)
.unwrap()
))
});
// ============================================================================
@@ -283,31 +274,28 @@ pub static MODEL_RECALL: Lazy<GaugeVec> = Lazy::new(|| {
/// System memory used during training (bytes)
pub static SYSTEM_MEMORY_USED: Lazy<GaugeVec> = Lazy::new(|| {
register_gauge_vec!(
unwrap_metric(register_gauge_vec!(
"ml_training_memory_used_bytes",
"System memory used during training",
&["job_id"]
)
.unwrap()
))
});
/// System CPU usage during training (percentage)
pub static SYSTEM_CPU_USAGE: Lazy<GaugeVec> = Lazy::new(|| {
register_gauge_vec!(
unwrap_metric(register_gauge_vec!(
"ml_training_cpu_usage_percent",
"CPU usage percentage during training",
&["job_id"]
)
.unwrap()
))
});
/// Active training workers
pub static ACTIVE_WORKERS: Lazy<Gauge> = Lazy::new(|| {
register_gauge!(
unwrap_metric(register_gauge!(
"ml_training_active_workers",
"Number of active training workers"
)
.unwrap()
))
});
// ============================================================================
@@ -316,33 +304,30 @@ pub static ACTIVE_WORKERS: Lazy<Gauge> = Lazy::new(|| {
/// Training data batches processed
pub static DATA_BATCHES_PROCESSED: Lazy<CounterVec> = Lazy::new(|| {
register_counter_vec!(
unwrap_metric(register_counter_vec!(
"ml_training_data_batches_total",
"Total data batches processed",
&["model_type", "job_id"]
)
.unwrap()
))
});
/// Data loading time histogram
pub static DATA_LOADING_DURATION: Lazy<HistogramVec> = Lazy::new(|| {
register_histogram_vec!(
unwrap_metric(register_histogram_vec!(
"ml_training_data_loading_seconds",
"Data loading duration per batch",
&["model_type", "job_id"],
vec![0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0] // 1ms to 1s
)
.unwrap()
))
});
/// Feature engineering errors
pub static FEATURE_ENGINEERING_ERRORS: Lazy<CounterVec> = Lazy::new(|| {
register_counter_vec!(
unwrap_metric(register_counter_vec!(
"ml_feature_engineering_errors_total",
"Total feature engineering errors",
&["error_type"]
)
.unwrap()
))
});
// ============================================================================
@@ -351,23 +336,21 @@ pub static FEATURE_ENGINEERING_ERRORS: Lazy<CounterVec> = Lazy::new(|| {
/// Training jobs by status
pub static TRAINING_JOBS_BY_STATUS: Lazy<GaugeVec> = Lazy::new(|| {
register_gauge_vec!(
unwrap_metric(register_gauge_vec!(
"ml_training_jobs_by_status",
"Number of training jobs by status",
&["status"] // pending, running, completed, failed, stopped, paused
)
.unwrap()
))
});
/// Training job duration histogram
pub static TRAINING_JOB_DURATION: Lazy<HistogramVec> = Lazy::new(|| {
register_histogram_vec!(
unwrap_metric(register_histogram_vec!(
"ml_training_job_duration_seconds",
"Total training job duration",
&["model_type", "status"],
vec![60.0, 300.0, 600.0, 1800.0, 3600.0, 7200.0, 14400.0] // 1min to 4h
)
.unwrap()
))
});
// ============================================================================