feat(observability): add #[instrument] tracing to all gRPC service handlers

Add tracing::instrument(skip_all) to gRPC handlers across all services for distributed trace spans via OTLP. Pairs with Prometheus scrape annotations from previous commit. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-01 01:20:15 +01:00
parent 995b46a680
commit c457e5c4d9
11 changed files with 130 additions and 14 deletions
--- a/services/api_gateway/src/config/endpoints.rs
+++ b/services/api_gateway/src/config/endpoints.rs
@@ -3,7 +3,7 @@
 use crate::config::ConfigurationManager;
 use crate::error::GatewayConfigError;
 use tonic::{Request, Response, Status};
-use tracing::{debug, info};
+use tracing::{debug, info, instrument};

 // Import generated proto code from crate root
 use crate::foxhunt::config_proto::{
@@ -34,6 +34,7 @@ impl ConfigurationServiceImpl {

 #[tonic::async_trait]
 impl ConfigurationService for ConfigurationServiceImpl {
+    #[instrument(skip_all)]
    async fn get_config(
        &self,
        request: Request<GetConfigRequest>,
@@ -60,6 +61,7 @@ impl ConfigurationService for ConfigurationServiceImpl {
        }))
    }

+    #[instrument(skip_all)]
    async fn update_config(
        &self,
        request: Request<UpdateConfigRequest>,
@@ -98,6 +100,7 @@ impl ConfigurationService for ConfigurationServiceImpl {
        }))
    }

+    #[instrument(skip_all)]
    async fn list_configs(
        &self,
        request: Request<ListConfigsRequest>,
@@ -133,6 +136,7 @@ impl ConfigurationService for ConfigurationServiceImpl {
        }))
    }

+    #[instrument(skip_all)]
    async fn reload_config(
        &self,
        request: Request<ReloadConfigRequest>,
--- a/services/api_gateway/src/grpc/backtesting_proxy.rs
+++ b/services/api_gateway/src/grpc/backtesting_proxy.rs
@@ -14,7 +14,7 @@ use tokio::sync::RwLock;
 use tonic::transport::{Certificate, ClientTlsConfig, Identity};
 use tonic::{Request, Response, Status};
 use tonic_health::pb::health_client::HealthClient;
-use tracing::{debug, error, info, warn};
+use tracing::{debug, error, info, warn, instrument};

 // Import generated protobuf types from build.rs
 use crate::foxhunt::tli::{
@@ -386,6 +386,7 @@ impl BacktestingService for BacktestingServiceProxy {
    type SubscribeBacktestProgressStream = tonic::codec::Streaming<BacktestProgressEvent>;

    /// Forward start_backtest request to backend
+    #[instrument(skip_all)]
    async fn start_backtest(
        &self,
        request: Request<StartBacktestRequest>,
@@ -404,6 +405,7 @@ impl BacktestingService for BacktestingServiceProxy {
    }

    /// Forward get_backtest_status request to backend
+    #[instrument(skip_all)]
    async fn get_backtest_status(
        &self,
        request: Request<GetBacktestStatusRequest>,
@@ -417,6 +419,7 @@ impl BacktestingService for BacktestingServiceProxy {
    }

    /// Forward get_backtest_results request to backend
+    #[instrument(skip_all)]
    async fn get_backtest_results(
        &self,
        request: Request<GetBacktestResultsRequest>,
@@ -430,6 +433,7 @@ impl BacktestingService for BacktestingServiceProxy {
    }

    /// Forward list_backtests request to backend
+    #[instrument(skip_all)]
    async fn list_backtests(
        &self,
        request: Request<ListBacktestsRequest>,
@@ -443,6 +447,7 @@ impl BacktestingService for BacktestingServiceProxy {
    }

    /// Forward subscribe_backtest_progress streaming request to backend
+    #[instrument(skip_all)]
    async fn subscribe_backtest_progress(
        &self,
        request: Request<SubscribeBacktestProgressRequest>,
@@ -461,6 +466,7 @@ impl BacktestingService for BacktestingServiceProxy {
    }

    /// Forward stop_backtest request to backend
+    #[instrument(skip_all)]
    async fn stop_backtest(
        &self,
        request: Request<StopBacktestRequest>,
@@ -479,6 +485,7 @@ impl BacktestingService for BacktestingServiceProxy {
 impl BacktestingService for Arc<BacktestingServiceProxy> {
    type SubscribeBacktestProgressStream = tonic::codec::Streaming<BacktestProgressEvent>;

+    #[instrument(skip_all)]
    async fn start_backtest(
        &self,
        request: Request<StartBacktestRequest>,
@@ -486,6 +493,7 @@ impl BacktestingService for Arc<BacktestingServiceProxy> {
        self.as_ref().start_backtest(request).await
    }

+    #[instrument(skip_all)]
    async fn get_backtest_status(
        &self,
        request: Request<GetBacktestStatusRequest>,
@@ -493,6 +501,7 @@ impl BacktestingService for Arc<BacktestingServiceProxy> {
        self.as_ref().get_backtest_status(request).await
    }

+    #[instrument(skip_all)]
    async fn get_backtest_results(
        &self,
        request: Request<GetBacktestResultsRequest>,
@@ -500,6 +509,7 @@ impl BacktestingService for Arc<BacktestingServiceProxy> {
        self.as_ref().get_backtest_results(request).await
    }

+    #[instrument(skip_all)]
    async fn list_backtests(
        &self,
        request: Request<ListBacktestsRequest>,
@@ -507,6 +517,7 @@ impl BacktestingService for Arc<BacktestingServiceProxy> {
        self.as_ref().list_backtests(request).await
    }

+    #[instrument(skip_all)]
    async fn subscribe_backtest_progress(
        &self,
        request: Request<SubscribeBacktestProgressRequest>,
@@ -514,6 +525,7 @@ impl BacktestingService for Arc<BacktestingServiceProxy> {
        self.as_ref().subscribe_backtest_progress(request).await
    }

+    #[instrument(skip_all)]
    async fn stop_backtest(
        &self,
        request: Request<StopBacktestRequest>,
--- a/services/api_gateway/src/grpc/trading_proxy.rs
+++ b/services/api_gateway/src/grpc/trading_proxy.rs
@@ -23,7 +23,7 @@ use std::sync::Arc;
 use std::time::Instant;
 use tonic::transport::Channel;
 use tonic::{Request, Response, Status};
-use tracing::{debug, error, warn};
+use tracing::{debug, error, warn, instrument};

 // Import TLI proto (client-facing interface)
 use crate::foxhunt::tli::trading_service_server::TradingService as TliTradingService;
@@ -409,6 +409,7 @@ impl TliTradingService for TradingServiceProxy {
        Pin<Box<dyn Stream<Item = Result<crate::foxhunt::tli::SystemStatusEvent, Status>> + Send>>;

    /// Submit order with protocol translation
+    #[instrument(skip_all)]
    async fn submit_order(
        &self,
        request: Request<crate::foxhunt::tli::SubmitOrderRequest>,
@@ -496,6 +497,7 @@ impl TliTradingService for TradingServiceProxy {
    }

    /// Cancel order with protocol translation
+    #[instrument(skip_all)]
    async fn cancel_order(
        &self,
        request: Request<crate::foxhunt::tli::CancelOrderRequest>,
@@ -553,6 +555,7 @@ impl TliTradingService for TradingServiceProxy {
    }

    /// Get order status with protocol translation
+    #[instrument(skip_all)]
    async fn get_order_status(
        &self,
        request: Request<crate::foxhunt::tli::GetOrderStatusRequest>,
@@ -619,6 +622,7 @@ impl TliTradingService for TradingServiceProxy {
    }

    /// Get account info with protocol translation
+    #[instrument(skip_all)]
    async fn get_account_info(
        &self,
        request: Request<crate::foxhunt::tli::GetAccountInfoRequest>,
@@ -677,6 +681,7 @@ impl TliTradingService for TradingServiceProxy {
    }

    /// Get positions with protocol translation
+    #[instrument(skip_all)]
    async fn get_positions(
        &self,
        request: Request<crate::foxhunt::tli::GetPositionsRequest>,
@@ -742,6 +747,7 @@ impl TliTradingService for TradingServiceProxy {
    // ========================================================================

    /// Subscribe to market data stream with protocol translation
+    #[instrument(skip_all)]
    async fn subscribe_market_data(
        &self,
        request: Request<crate::foxhunt::tli::SubscribeMarketDataRequest>,
@@ -811,6 +817,7 @@ impl TliTradingService for TradingServiceProxy {
    }

    /// Subscribe to order updates stream with protocol translation
+    #[instrument(skip_all)]
    async fn subscribe_order_updates(
        &self,
        request: Request<crate::foxhunt::tli::SubscribeOrderUpdatesRequest>,
@@ -883,6 +890,7 @@ impl TliTradingService for TradingServiceProxy {
    // Risk Management Methods (connect to Risk Service backend)
    // ========================================================================

+    #[instrument(skip_all)]
    async fn get_va_r(
        &self,
        request: Request<crate::foxhunt::tli::GetVaRRequest>,
@@ -956,6 +964,7 @@ impl TliTradingService for TradingServiceProxy {
        Ok(Response::new(tli_resp))
    }

+    #[instrument(skip_all)]
    async fn get_position_risk(
        &self,
        request: Request<crate::foxhunt::tli::GetPositionRiskRequest>,
@@ -1046,6 +1055,7 @@ impl TliTradingService for TradingServiceProxy {
        Ok(Response::new(tli_resp))
    }

+    #[instrument(skip_all)]
    async fn validate_order(
        &self,
        request: Request<crate::foxhunt::tli::ValidateOrderRequest>,
@@ -1128,6 +1138,7 @@ impl TliTradingService for TradingServiceProxy {
        Ok(Response::new(tli_resp))
    }

+    #[instrument(skip_all)]
    async fn get_risk_metrics(
        &self,
        request: Request<crate::foxhunt::tli::GetRiskMetricsRequest>,
@@ -1218,6 +1229,7 @@ impl TliTradingService for TradingServiceProxy {
        Ok(Response::new(tli_resp))
    }

+    #[instrument(skip_all)]
    async fn subscribe_risk_alerts(
        &self,
        request: Request<crate::foxhunt::tli::SubscribeRiskAlertsRequest>,
@@ -1291,6 +1303,7 @@ impl TliTradingService for TradingServiceProxy {
        Ok(Response::new(Box::pin(tli_stream)))
    }

+    #[instrument(skip_all)]
    async fn emergency_stop(
        &self,
        request: Request<crate::foxhunt::tli::EmergencyStopRequest>,
@@ -1363,6 +1376,7 @@ impl TliTradingService for TradingServiceProxy {
    // Monitoring Methods (connect to Monitoring Service backend)
    // ========================================================================

+    #[instrument(skip_all)]
    async fn get_metrics(
        &self,
        request: Request<crate::foxhunt::tli::GetMetricsRequest>,
@@ -1427,6 +1441,7 @@ impl TliTradingService for TradingServiceProxy {
        Ok(Response::new(tli_resp))
    }

+    #[instrument(skip_all)]
    async fn get_latency(
        &self,
        request: Request<crate::foxhunt::tli::GetLatencyRequest>,
@@ -1514,6 +1529,7 @@ impl TliTradingService for TradingServiceProxy {
        Ok(Response::new(tli_resp))
    }

+    #[instrument(skip_all)]
    async fn get_throughput(
        &self,
        request: Request<crate::foxhunt::tli::GetThroughputRequest>,
@@ -1593,6 +1609,7 @@ impl TliTradingService for TradingServiceProxy {
        Ok(Response::new(tli_resp))
    }

+    #[instrument(skip_all)]
    async fn subscribe_metrics(
        &self,
        request: Request<crate::foxhunt::tli::SubscribeMetricsRequest>,
@@ -1678,6 +1695,7 @@ impl TliTradingService for TradingServiceProxy {
    // Configuration Methods (connect to Config Service backend)
    // ========================================================================

+    #[instrument(skip_all)]
    async fn update_parameters(
        &self,
        request: Request<crate::foxhunt::tli::UpdateParametersRequest>,
@@ -1759,6 +1777,7 @@ impl TliTradingService for TradingServiceProxy {
        Ok(Response::new(tli_resp))
    }

+    #[instrument(skip_all)]
    async fn get_config(
        &self,
        request: Request<crate::foxhunt::tli::GetConfigRequest>,
@@ -1841,6 +1860,7 @@ impl TliTradingService for TradingServiceProxy {
        Ok(Response::new(tli_resp))
    }

+    #[instrument(skip_all)]
    async fn subscribe_config(
        &self,
        request: Request<crate::foxhunt::tli::SubscribeConfigRequest>,
@@ -1912,6 +1932,7 @@ impl TliTradingService for TradingServiceProxy {
    // System Status Methods (from Monitoring Service backend)
    // ========================================================================

+    #[instrument(skip_all)]
    async fn get_system_status(
        &self,
        request: Request<crate::foxhunt::tli::GetSystemStatusRequest>,
@@ -1998,6 +2019,7 @@ impl TliTradingService for TradingServiceProxy {
        Ok(Response::new(tli_resp))
    }

+    #[instrument(skip_all)]
    async fn subscribe_system_status(
        &self,
        request: Request<crate::foxhunt::tli::SubscribeSystemStatusRequest>,
@@ -2083,6 +2105,7 @@ impl TliTradingService for TradingServiceProxy {
    // ========================================================================

    /// Submit ML-powered trading order with ensemble predictions
+    #[instrument(skip_all)]
    async fn submit_ml_order(
        &self,
        request: Request<crate::foxhunt::tli::SubmitMlOrderRequest>,
@@ -2167,6 +2190,7 @@ impl TliTradingService for TradingServiceProxy {
    }

    /// Get ML prediction history with outcomes
+    #[instrument(skip_all)]
    async fn get_ml_predictions(
        &self,
        request: Request<crate::foxhunt::tli::GetMlPredictionsRequest>,
@@ -2235,6 +2259,7 @@ impl TliTradingService for TradingServiceProxy {
    }

    /// Get ML model performance metrics
+    #[instrument(skip_all)]
    async fn get_ml_performance(
        &self,
        request: Request<crate::foxhunt::tli::GetMlPerformanceRequest>,
@@ -2307,6 +2332,7 @@ impl TliTradingService for TradingServiceProxy {
    }

    /// Get current regime state for a symbol (Wave D)
+    #[instrument(skip_all)]
    async fn get_regime_state(
        &self,
        request: Request<crate::foxhunt::tli::GetRegimeStateRequest>,
@@ -2368,6 +2394,7 @@ impl TliTradingService for TradingServiceProxy {
    }

    /// Get regime transition history for a symbol (Wave D)
+    #[instrument(skip_all)]
    async fn get_regime_transitions(
        &self,
        request: Request<crate::foxhunt::tli::GetRegimeTransitionsRequest>,
--- a/services/backtesting_service/src/service.rs
+++ b/services/backtesting_service/src/service.rs
@@ -6,7 +6,7 @@ use std::sync::Arc;
 use tokio::sync::{broadcast, RwLock};
 use tokio_util::sync::CancellationToken;
 use tonic::{Request, Response, Status};
-use tracing::{debug, error, info};
+use tracing::{debug, error, info, instrument};
 use uuid::Uuid;

 use crate::foxhunt::tli::{backtesting_service_server::BacktestingService, *};
@@ -362,6 +362,7 @@ impl BacktestingService for BacktestingServiceImpl {
    >;

    /// Start a new backtest
+    #[instrument(skip_all)]
    async fn start_backtest(
        &self,
        request: Request<StartBacktestRequest>,
@@ -444,6 +445,7 @@ impl BacktestingService for BacktestingServiceImpl {
    }

    /// Get backtest status
+    #[instrument(skip_all)]
    async fn get_backtest_status(
        &self,
        request: Request<GetBacktestStatusRequest>,
@@ -469,6 +471,7 @@ impl BacktestingService for BacktestingServiceImpl {
    }

    /// Get backtest results
+    #[instrument(skip_all)]
    async fn get_backtest_results(
        &self,
        request: Request<GetBacktestResultsRequest>,
@@ -547,6 +550,7 @@ impl BacktestingService for BacktestingServiceImpl {
    }

    /// List historical backtests
+    #[instrument(skip_all)]
    async fn list_backtests(
        &self,
        request: Request<ListBacktestsRequest>,
@@ -573,6 +577,7 @@ impl BacktestingService for BacktestingServiceImpl {
    }

    /// Subscribe to backtest progress
+    #[instrument(skip_all)]
    async fn subscribe_backtest_progress(
        &self,
        request: Request<SubscribeBacktestProgressRequest>,
@@ -610,6 +615,7 @@ impl BacktestingService for BacktestingServiceImpl {
    }

    /// Stop a running backtest
+    #[instrument(skip_all)]
    async fn stop_backtest(
        &self,
        request: Request<StopBacktestRequest>,
--- a/services/data_acquisition_service/src/service.rs
+++ b/services/data_acquisition_service/src/service.rs
@@ -16,6 +16,7 @@ use std::sync::Arc;
 use tokio::sync::RwLock;
 use tonic::{Request, Response, Status};
 use uuid::Uuid;
+use tracing::instrument;

 /// In-memory job store backed by a `HashMap<String, DownloadJobDetails>`.
 ///
@@ -72,6 +73,7 @@ impl Default for DataAcquisitionServiceImpl {

 #[tonic::async_trait]
 impl DataAcquisitionService for DataAcquisitionServiceImpl {
+    #[instrument(skip_all)]
    async fn schedule_download(
        &self,
        request: Request<ScheduleDownloadRequest>,
@@ -178,6 +180,7 @@ impl DataAcquisitionService for DataAcquisitionServiceImpl {
        }))
    }

+    #[instrument(skip_all)]
    async fn get_download_status(
        &self,
        request: Request<GetDownloadStatusRequest>,
@@ -195,6 +198,7 @@ impl DataAcquisitionService for DataAcquisitionServiceImpl {
        }))
    }

+    #[instrument(skip_all)]
    async fn cancel_download(
        &self,
        request: Request<CancelDownloadRequest>,
@@ -224,6 +228,7 @@ impl DataAcquisitionService for DataAcquisitionServiceImpl {
        }))
    }

+    #[instrument(skip_all)]
    async fn list_download_jobs(
        &self,
        request: Request<ListDownloadJobsRequest>,
@@ -278,6 +283,7 @@ impl DataAcquisitionService for DataAcquisitionServiceImpl {
        }))
    }

+    #[instrument(skip_all)]
    async fn health_check(
        &self,
        _request: Request<HealthCheckRequest>,
--- a/services/ml_training_service/src/service.rs
+++ b/services/ml_training_service/src/service.rs
@@ -11,7 +11,7 @@ use anyhow::Result;
 use tokio::time::{timeout, Duration};
 use tokio_stream::Stream;
 use tonic::{Request, Response, Status};
-use tracing::{debug, info, warn};
+use tracing::{debug, info, warn, instrument};
 use uuid::Uuid;

 // Import the generated protobuf types
@@ -252,6 +252,7 @@ impl MLTrainingServiceImpl {
 #[tonic::async_trait]
 impl MlTrainingService for MLTrainingServiceImpl {
    /// Start a new training job
+    #[instrument(skip_all)]
    async fn start_training(
        &self,
        request: Request<StartTrainingRequest>,
@@ -420,6 +421,7 @@ impl MlTrainingService for MLTrainingServiceImpl {
    type SubscribeToTrainingStatusStream =
        Pin<Box<dyn Stream<Item = Result<ProtoStatusUpdate, Status>> + Send>>;

+    #[instrument(skip_all)]
    async fn subscribe_to_training_status(
        &self,
        request: Request<SubscribeToTrainingStatusRequest>,
@@ -455,6 +457,7 @@ impl MlTrainingService for MLTrainingServiceImpl {
    }

    /// Stop a running training job
+    #[instrument(skip_all)]
    async fn stop_training(
        &self,
        request: Request<StopTrainingRequest>,
@@ -484,6 +487,7 @@ impl MlTrainingService for MLTrainingServiceImpl {
    }

    /// List available models for training
+    #[instrument(skip_all)]
    async fn list_available_models(
        &self,
        _request: Request<ListAvailableModelsRequest>,
@@ -657,6 +661,7 @@ impl MlTrainingService for MLTrainingServiceImpl {
    }

    /// List training jobs with filtering
+    #[instrument(skip_all)]
    async fn list_training_jobs(
        &self,
        request: Request<ListTrainingJobsRequest>,
@@ -733,6 +738,7 @@ impl MlTrainingService for MLTrainingServiceImpl {
    }

    /// Get detailed information about a training job
+    #[instrument(skip_all)]
    async fn get_training_job_details(
        &self,
        request: Request<GetTrainingJobDetailsRequest>,
@@ -777,6 +783,7 @@ impl MlTrainingService for MLTrainingServiceImpl {
    }

    /// Health check
+    #[instrument(skip_all)]
    async fn health_check(
        &self,
        _request: Request<HealthCheckRequest>,
@@ -798,6 +805,7 @@ impl MlTrainingService for MLTrainingServiceImpl {
    }

    /// Start a new hyperparameter tuning job
+    #[instrument(skip_all)]
    async fn start_tuning_job(
        &self,
        request: Request<proto::StartTuningJobRequest>,
@@ -806,6 +814,7 @@ impl MlTrainingService for MLTrainingServiceImpl {
    }

    /// Get status of a tuning job
+    #[instrument(skip_all)]
    async fn get_tuning_job_status(
        &self,
        request: Request<proto::GetTuningJobStatusRequest>,
@@ -814,6 +823,7 @@ impl MlTrainingService for MLTrainingServiceImpl {
    }

    /// Stop a running tuning job
+    #[instrument(skip_all)]
    async fn stop_tuning_job(
        &self,
        request: Request<proto::StopTuningJobRequest>,
@@ -825,6 +835,7 @@ impl MlTrainingService for MLTrainingServiceImpl {
    type StreamTuningProgressStream =
        Pin<Box<dyn Stream<Item = Result<proto::ProgressUpdate, Status>> + Send>>;

+    #[instrument(skip_all)]
    async fn stream_tuning_progress(
        &self,
        request: Request<proto::StreamProgressRequest>,
@@ -833,6 +844,7 @@ impl MlTrainingService for MLTrainingServiceImpl {
    }

    /// INTERNAL: Train a single model with specific hyperparameters (called by Optuna subprocess)
+    #[instrument(skip_all)]
    async fn train_model(
        &self,
        request: Request<proto::TrainModelRequest>,
@@ -956,6 +968,7 @@ impl MlTrainingService for MLTrainingServiceImpl {
    }

    /// Start batch tuning job for multiple models
+    #[instrument(skip_all)]
    async fn batch_start_tuning_jobs(
        &self,
        request: Request<proto::BatchStartTuningJobsRequest>,
@@ -1041,6 +1054,7 @@ impl MlTrainingService for MLTrainingServiceImpl {
    }

    /// Get batch tuning job status
+    #[instrument(skip_all)]
    async fn get_batch_tuning_status(
        &self,
        request: Request<proto::GetBatchTuningStatusRequest>,
@@ -1099,6 +1113,7 @@ impl MlTrainingService for MLTrainingServiceImpl {
    }

    /// Stop a running batch tuning job
+    #[instrument(skip_all)]
    async fn stop_batch_tuning_job(
        &self,
        request: Request<proto::StopBatchTuningJobRequest>,
@@ -1165,6 +1180,7 @@ impl MlTrainingService for MLTrainingServiceImpl {

    // -- Job completion & model promotion (on-demand training pipeline) --------

+    #[instrument(skip_all)]
    async fn report_job_completion(
        &self,
        request: Request<JobCompletionReport>,
@@ -1266,6 +1282,7 @@ impl MlTrainingService for MLTrainingServiceImpl {
        }))
    }

+    #[instrument(skip_all)]
    async fn list_pending_promotions(
        &self,
        _request: Request<ListPendingPromotionsRequest>,
@@ -1290,6 +1307,7 @@ impl MlTrainingService for MLTrainingServiceImpl {
        Ok(Response::new(ListPendingPromotionsResponse { promotions }))
    }

+    #[instrument(skip_all)]
    async fn approve_promotion(
        &self,
        request: Request<ApprovePromotionRequest>,
@@ -1318,6 +1336,7 @@ impl MlTrainingService for MLTrainingServiceImpl {
        }
    }

+    #[instrument(skip_all)]
    async fn reject_promotion(
        &self,
        request: Request<RejectPromotionRequest>,
--- a/services/trading_service/src/services/enhanced_ml.rs
+++ b/services/trading_service/src/services/enhanced_ml.rs
@@ -26,7 +26,7 @@ use std::time::{Instant, SystemTime, UNIX_EPOCH};
 use tokio::sync::{broadcast, RwLock};
 use tokio_stream::{wrappers::BroadcastStream, wrappers::ReceiverStream, Stream, StreamExt};
 use tonic::{Request, Response, Status};
-use tracing::{debug, info, warn};
+use tracing::{debug, info, warn, instrument};

 // Production ML imports
 use ml::{Features, MLModel, ModelMetadata, ModelPrediction, ModelType};
@@ -721,6 +721,7 @@ impl EnhancedMLServiceImpl {

 #[tonic::async_trait]
 impl MlService for EnhancedMLServiceImpl {
+    #[instrument(skip_all)]
    async fn get_prediction(
        &self,
        request: Request<GetPredictionRequest>,
@@ -786,6 +787,7 @@ impl MlService for EnhancedMLServiceImpl {
        }))
    }

+    #[instrument(skip_all)]
    async fn get_model_status(
        &self,
        _request: Request<GetModelStatusRequest>,
@@ -817,6 +819,7 @@ impl MlService for EnhancedMLServiceImpl {
        Ok(Response::new(GetModelStatusResponse { model_statuses }))
    }

+    #[instrument(skip_all)]
    async fn get_available_models(
        &self,
        _request: Request<GetAvailableModelsRequest>,
@@ -870,6 +873,7 @@ impl MlService for EnhancedMLServiceImpl {
        }))
    }

+    #[instrument(skip_all)]
    async fn get_ensemble_vote(
        &self,
        request: Request<GetEnsembleVoteRequest>,
@@ -1003,6 +1007,7 @@ impl MlService for EnhancedMLServiceImpl {
    type StreamPredictionsStream =
        std::pin::Pin<Box<dyn Stream<Item = Result<PredictionEvent, Status>> + Send>>;

+    #[instrument(skip_all)]
    async fn stream_predictions(
        &self,
        request: Request<StreamPredictionsRequest>,
@@ -1017,6 +1022,7 @@ impl MlService for EnhancedMLServiceImpl {
    }
    // Additional MLService methods implementation

+    #[instrument(skip_all)]
    async fn retrain_model(
        &self,
        request: Request<RetrainModelRequest>,
@@ -1034,6 +1040,7 @@ impl MlService for EnhancedMLServiceImpl {
        )))
    }

+    #[instrument(skip_all)]
    async fn get_model_performance(
        &self,
        request: Request<GetModelPerformanceRequest>,
@@ -1103,6 +1110,7 @@ impl MlService for EnhancedMLServiceImpl {
        }))
    }

+    #[instrument(skip_all)]
    async fn get_feature_importance(
        &self,
        request: Request<GetFeatureImportanceRequest>,
@@ -1119,6 +1127,7 @@ impl MlService for EnhancedMLServiceImpl {
    type StreamModelMetricsStream =
        std::pin::Pin<Box<dyn Stream<Item = Result<ModelMetricsEvent, Status>> + Send>>;

+    #[instrument(skip_all)]
    async fn stream_model_metrics(
        &self,
        request: Request<StreamModelMetricsRequest>,
@@ -1221,6 +1230,7 @@ impl MlService for EnhancedMLServiceImpl {
    type StreamSignalStrengthStream =
        std::pin::Pin<Box<dyn Stream<Item = Result<SignalStrengthEvent, Status>> + Send>>;

+    #[instrument(skip_all)]
    async fn stream_signal_strength(
        &self,
        request: Request<StreamSignalStrengthRequest>,
--- a/services/trading_service/src/services/ml.rs
+++ b/services/trading_service/src/services/ml.rs
@@ -10,6 +10,7 @@ use ml::ensemble::TradingAction;
 use std::collections::HashMap;
 use std::time::{SystemTime, UNIX_EPOCH};
 use tonic::{Request, Response, Status};
+use tracing::instrument;

 /// ML service implementation
 #[derive(Debug, Clone)]
@@ -35,6 +36,7 @@ fn action_to_prediction_type(action: TradingActionType) -> i32 {

 #[tonic::async_trait]
 impl MlService for MLServiceImpl {
+    #[instrument(skip_all)]
    async fn get_prediction(
        &self,
        request: Request<GetPredictionRequest>,
@@ -85,6 +87,7 @@ impl MlService for MLServiceImpl {
        }))
    }

+    #[instrument(skip_all)]
    async fn get_model_status(
        &self,
        _request: Request<GetModelStatusRequest>,
--- a/services/trading_service/src/services/monitoring.rs
+++ b/services/trading_service/src/services/monitoring.rs
@@ -17,7 +17,7 @@ use std::sync::Arc;
 use std::time::Instant;
 use tokio::sync::RwLock;
 use tonic::{Request, Response, Status};
-use tracing::info;
+use tracing::{info, instrument};

 /// Monitoring service implementation
 #[derive(Debug, Clone)]
@@ -116,6 +116,7 @@ async fn collect_system_metrics(

 #[tonic::async_trait]
 impl MonitoringService for MonitoringServiceImpl {
+    #[instrument(skip_all)]
    async fn get_health_check(
        &self,
        _request: Request<GetHealthCheckRequest>,
@@ -147,6 +148,7 @@ impl MonitoringService for MonitoringServiceImpl {
        }))
    }

+    #[instrument(skip_all)]
    async fn get_metrics(
        &self,
        _request: Request<GetMetricsRequest>,
@@ -195,6 +197,7 @@ impl MonitoringService for MonitoringServiceImpl {
        }))
    }

+    #[instrument(skip_all)]
    async fn get_system_status(
        &self,
        _request: Request<GetSystemStatusRequest>,
@@ -273,6 +276,7 @@ impl MonitoringService for MonitoringServiceImpl {
        }))
    }

+    #[instrument(skip_all)]
    async fn get_latency_metrics(
        &self,
        _request: Request<GetLatencyMetricsRequest>,
@@ -293,6 +297,7 @@ impl MonitoringService for MonitoringServiceImpl {
        Ok(Response::new(GetLatencyMetricsResponse { latency_metrics }))
    }

+    #[instrument(skip_all)]
    async fn get_throughput_metrics(
        &self,
        _request: Request<GetThroughputMetricsRequest>,
@@ -313,6 +318,7 @@ impl MonitoringService for MonitoringServiceImpl {
        }))
    }

+    #[instrument(skip_all)]
    async fn acknowledge_alert(
        &self,
        request: Request<AcknowledgeAlertRequest>,
@@ -352,6 +358,7 @@ impl MonitoringService for MonitoringServiceImpl {
        }
    }

+    #[instrument(skip_all)]
    async fn get_active_alerts(
        &self,
        _request: Request<GetActiveAlertsRequest>,
@@ -374,6 +381,7 @@ impl MonitoringService for MonitoringServiceImpl {
        Box<dyn tokio_stream::Stream<Item = Result<SystemStatusEvent, Status>> + Send>,
    >;

+    #[instrument(skip_all)]
    async fn stream_system_status(
        &self,
        request: Request<StreamSystemStatusRequest>,
@@ -476,6 +484,7 @@ impl MonitoringService for MonitoringServiceImpl {
    type StreamMetricsStream =
        std::pin::Pin<Box<dyn tokio_stream::Stream<Item = Result<MetricsEvent, Status>> + Send>>;

+    #[instrument(skip_all)]
    async fn stream_metrics(
        &self,
        request: Request<StreamMetricsRequest>,
@@ -595,6 +604,7 @@ impl MonitoringService for MonitoringServiceImpl {
    type StreamAlertsStream =
        std::pin::Pin<Box<dyn tokio_stream::Stream<Item = Result<AlertEvent, Status>> + Send>>;

+    #[instrument(skip_all)]
    async fn stream_alerts(
        &self,
        request: Request<StreamAlertsRequest>,
--- a/services/trading_service/src/services/risk.rs
+++ b/services/trading_service/src/services/risk.rs
@@ -13,7 +13,7 @@ use crate::proto::risk::{
 use crate::repositories::ConfigRepository;
 use crate::state::TradingServiceState;
 use tonic::{Request, Response, Status};
-use tracing::{debug, error, info, warn};
+use tracing::{debug, error, info, warn, instrument};

 /// Risk service implementation
 #[derive(Debug, Clone)]
@@ -361,6 +361,7 @@ impl RiskServiceImpl {

 #[tonic::async_trait]
 impl RiskService for RiskServiceImpl {
+    #[instrument(skip_all)]
    async fn get_va_r(
        &self,
        request: Request<GetVaRRequest>,
@@ -476,6 +477,7 @@ impl RiskService for RiskServiceImpl {
        }))
    }

+    #[instrument(skip_all)]
    async fn get_risk_metrics(
        &self,
        _request: Request<GetRiskMetricsRequest>,
@@ -579,6 +581,7 @@ impl RiskService for RiskServiceImpl {
        }))
    }

+    #[instrument(skip_all)]
    async fn get_position_risk(
        &self,
        request: Request<GetPositionRiskRequest>,
@@ -634,6 +637,7 @@ impl RiskService for RiskServiceImpl {
        }))
    }

+    #[instrument(skip_all)]
    async fn validate_order(
        &self,
        request: Request<ValidateOrderRequest>,
@@ -842,6 +846,7 @@ impl RiskService for RiskServiceImpl {
        }))
    }

+    #[instrument(skip_all)]
    async fn emergency_stop(
        &self,
        request: Request<EmergencyStopRequest>,
@@ -903,6 +908,7 @@ impl RiskService for RiskServiceImpl {
        }))
    }

+    #[instrument(skip_all)]
    async fn get_circuit_breaker_status(
        &self,
        _request: Request<GetCircuitBreakerStatusRequest>,
@@ -954,6 +960,7 @@ impl RiskService for RiskServiceImpl {
        Box<dyn tokio_stream::Stream<Item = Result<crate::proto::risk::VaREvent, Status>> + Send>,
    >;

+    #[instrument(skip_all)]
    async fn stream_va_r_updates(
        &self,
        request: Request<StreamVaRRequest>,
@@ -1044,6 +1051,7 @@ impl RiskService for RiskServiceImpl {
        >,
    >;

+    #[instrument(skip_all)]
    async fn stream_risk_alerts(
        &self,
        request: Request<StreamRiskAlertsRequest>,
--- a/services/trading_service/src/services/trading.rs
+++ b/services/trading_service/src/services/trading.rs
@@ -8,7 +8,7 @@ use std::sync::Arc;
 use tokio::sync::mpsc;
 use tokio_stream::{wrappers::ReceiverStream, Stream};
 use tonic::{Request, Response, Result as TonicResult, Status};
-use tracing::{debug, error, info, warn};
+use tracing::{debug, error, info, instrument, warn};

 use crate::error::TradingServiceResult;
 use crate::latency_recorder::{time_async, LatencyCategory};
@@ -40,7 +40,7 @@ impl TradingServiceImpl {

 #[tonic::async_trait]
 impl trading_service_server::TradingService for TradingServiceImpl {
-    // Order Management Implementation
+    #[instrument(skip_all)]
    async fn submit_order(
        &self,
        request: Request<SubmitOrderRequest>,
@@ -177,6 +177,7 @@ impl trading_service_server::TradingService for TradingServiceImpl {
        }
    }

+    #[instrument(skip_all)]
    async fn cancel_order(
        &self,
        request: Request<CancelOrderRequest>,
@@ -236,6 +237,7 @@ impl trading_service_server::TradingService for TradingServiceImpl {
        }
    }

+    #[instrument(skip_all)]
    async fn get_order_status(
        &self,
        request: Request<GetOrderStatusRequest>,
@@ -284,6 +286,7 @@ impl trading_service_server::TradingService for TradingServiceImpl {
    // Order Streaming Implementation
    type StreamOrdersStream = Pin<Box<dyn Stream<Item = Result<OrderEvent, Status>> + Send>>;

+    #[instrument(skip_all)]
    async fn stream_orders(
        &self,
        request: Request<StreamOrdersRequest>,
@@ -335,7 +338,7 @@ impl trading_service_server::TradingService for TradingServiceImpl {
        Ok(Response::new(Box::pin(stream)))
    }

-    // Position Management Implementation
+    #[instrument(skip_all)]
    async fn get_positions(
        &self,
        request: Request<GetPositionsRequest>,
@@ -383,6 +386,7 @@ impl trading_service_server::TradingService for TradingServiceImpl {
    // Position Streaming Implementation
    type StreamPositionsStream = Pin<Box<dyn Stream<Item = Result<PositionEvent, Status>> + Send>>;

+    #[instrument(skip_all)]
    async fn stream_positions(
        &self,
        request: Request<StreamPositionsRequest>,
@@ -421,6 +425,7 @@ impl trading_service_server::TradingService for TradingServiceImpl {
        Ok(Response::new(Box::pin(stream)))
    }

+    #[instrument(skip_all)]
    async fn get_portfolio_summary(
        &self,
        request: Request<GetPortfolioSummaryRequest>,
@@ -490,6 +495,7 @@ impl trading_service_server::TradingService for TradingServiceImpl {
    type StreamMarketDataStream =
        Pin<Box<dyn Stream<Item = Result<MarketDataEvent, Status>> + Send>>;

+    #[instrument(skip_all)]
    async fn stream_market_data(
        &self,
        request: Request<StreamMarketDataRequest>,
@@ -529,6 +535,7 @@ impl trading_service_server::TradingService for TradingServiceImpl {
        Ok(Response::new(Box::pin(stream)))
    }

+    #[instrument(skip_all)]
    async fn get_order_book(
        &self,
        request: Request<GetOrderBookRequest>,
@@ -599,6 +606,7 @@ impl trading_service_server::TradingService for TradingServiceImpl {
    type StreamExecutionsStream =
        Pin<Box<dyn Stream<Item = Result<ExecutionEvent, Status>> + Send>>;

+    #[instrument(skip_all)]
    async fn stream_executions(
        &self,
        request: Request<StreamExecutionsRequest>,
@@ -637,6 +645,7 @@ impl trading_service_server::TradingService for TradingServiceImpl {
        Ok(Response::new(Box::pin(stream)))
    }

+    #[instrument(skip_all)]
    async fn get_execution_history(
        &self,
        request: Request<GetExecutionHistoryRequest>,
@@ -678,7 +687,7 @@ impl trading_service_server::TradingService for TradingServiceImpl {
        }
    }

-    // ML Trading Operations Implementation
+    #[instrument(skip_all)]
    async fn submit_ml_order(
        &self,
        request: Request<crate::proto::trading::MlOrderRequest>,
@@ -797,6 +806,7 @@ impl trading_service_server::TradingService for TradingServiceImpl {
        }
    }

+    #[instrument(skip_all)]
    async fn get_ml_predictions(
        &self,
        request: Request<crate::proto::trading::MlPredictionsRequest>,
@@ -962,6 +972,7 @@ impl trading_service_server::TradingService for TradingServiceImpl {
        ))
    }

+    #[instrument(skip_all)]
    async fn get_ml_performance(
        &self,
        request: Request<crate::proto::trading::MlPerformanceRequest>,
@@ -999,7 +1010,7 @@ impl trading_service_server::TradingService for TradingServiceImpl {
        ))
    }

-    /// Get current regime state for a symbol
+    #[instrument(skip_all)]
    async fn get_regime_state(
        &self,
        request: Request<GetRegimeStateRequest>,
@@ -1047,7 +1058,7 @@ impl trading_service_server::TradingService for TradingServiceImpl {
        Ok(Response::new(response))
    }

-    /// Get regime transition history for a symbol
+    #[instrument(skip_all)]
    async fn get_regime_transitions(
        &self,
        request: Request<GetRegimeTransitionsRequest>,