From 0c7f6362d98da2e18a753ef792abfeb5c0d788f0 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sun, 1 Mar 2026 00:32:35 +0100 Subject: [PATCH] fix(observability): combine fmt + OTLP layers in single subscriber Previously init_observability() called init_json_logger() (which set the global subscriber with a fmt layer) and then init_tracing() (which only set the global tracer provider). Because no tracing_opentelemetry::layer() was ever added to the subscriber, #[instrument] spans never reached Tempo. Now init_observability() builds a single Registry subscriber that composes an EnvFilter, a JSON fmt layer, and an optional OTLP layer (via tracing_opentelemetry). The OTLP layer gracefully degrades to None when Tempo is unavailable, so logging always works. Also adds build_otel_tracer() to tracing_config.rs which returns Option for the caller to wrap in an OpenTelemetryLayer inside their subscriber stack, letting the S type parameter be inferred correctly by the compiler. Co-Authored-By: Claude Opus 4.6 --- crates/common/src/observability/mod.rs | 47 +++++++++------ .../src/observability/tracing_config.rs | 59 ++++++++++++++++++- 2 files changed, 86 insertions(+), 20 deletions(-) diff --git a/crates/common/src/observability/mod.rs b/crates/common/src/observability/mod.rs index a945805eb..f112cd455 100644 --- a/crates/common/src/observability/mod.rs +++ b/crates/common/src/observability/mod.rs @@ -74,7 +74,7 @@ mod tests; // Re-export commonly used types pub use correlation::{CorrelationId, CorrelationIdExt, CORRELATION_ID_HEADER}; pub use logger::{init_json_logger, JsonLoggerConfig, LogLevel}; -pub use tracing_config::{init_tracing, TracingConfig}; +pub use tracing_config::{build_otel_tracer, init_tracing, TracingConfig}; use crate::error::CommonResult; @@ -110,32 +110,41 @@ pub async fn init_observability( service_name: &str, otlp_endpoint: &str, ) -> CommonResult<()> { - // Console-only logging: stdout is captured by containerd and shipped - // to Loki via Promtail. No file logging needed in K8s. - let logger_config = JsonLoggerConfig { - service_name: service_name.to_string(), - log_level: LogLevel::Info, - enable_console: true, - enable_file: false, - log_directory: String::new(), - rotation: logger::LogRotation::Daily, - max_files: 10, - max_file_size_mb: 100, - }; + use tracing_subscriber::prelude::*; - init_json_logger(logger_config)?; + // Build env filter: honour RUST_LOG, default to info. + let env_filter = tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")); - // Initialize OpenTelemetry tracing with OTLP export. - // Gracefully degrade if Tempo is unavailable — logging still works. + // JSON fmt layer — stdout captured by containerd / Promtail → Loki. + let fmt_layer = tracing_subscriber::fmt::layer() + .json() + .with_target(true) + .with_current_span(true) + .with_span_list(false); + + // OTLP layer — gracefully degrade if Tempo is unavailable. + // build_otel_tracer returns Option; wrapping with + // tracing_opentelemetry::layer() inside the subscriber chain lets + // the `S` type parameter be inferred correctly by the compiler. let tracing_config = TracingConfig { service_name: service_name.to_string(), otlp_endpoint: otlp_endpoint.to_string(), enable_export: true, }; - - if let Err(e) = init_tracing(tracing_config).await { - eprintln!("OTLP tracing unavailable (non-fatal): {}", e); + let otel_tracer = tracing_config::build_otel_tracer(&tracing_config); + let otel_layer = otel_tracer + .map(|tracer| tracing_opentelemetry::layer().with_tracer(tracer)); + if otel_layer.is_none() { + eprintln!("OTLP tracing layer unavailable — logs-only mode"); } + // Single subscriber with all layers. `.with(Option)` is a no-op when None. + tracing_subscriber::registry() + .with(env_filter) + .with(fmt_layer) + .with(otel_layer) + .init(); + Ok(()) } diff --git a/crates/common/src/observability/tracing_config.rs b/crates/common/src/observability/tracing_config.rs index 5a1d4cbd3..50d5c58f8 100644 --- a/crates/common/src/observability/tracing_config.rs +++ b/crates/common/src/observability/tracing_config.rs @@ -21,7 +21,7 @@ //! using the W3C Trace Context specification. use crate::error::{CommonError, CommonResult}; -use opentelemetry::{global, KeyValue}; +use opentelemetry::{global, trace::TracerProvider as _, KeyValue}; use opentelemetry_otlp::{SpanExporter, WithExportConfig}; use opentelemetry_sdk::{ runtime::Tokio as OtelTokio, @@ -57,12 +57,69 @@ impl Default for TracingConfig { } } +/// Build an OpenTelemetry tracer backed by an OTLP exporter. +/// +/// Returns `Some(tracer)` when OTLP export is enabled and the exporter can be +/// constructed, or `None` on failure (graceful degradation). The caller wraps +/// the tracer in a `tracing_opentelemetry::layer().with_tracer(tracer)` layer +/// inside their subscriber stack so the `S` type parameter is inferred +/// correctly. +/// +/// This function also registers the tracer provider as the global provider +/// (required for propagation helpers like `inject_trace_context`). +/// +/// # Arguments +/// +/// * `config` - Tracing configuration +/// +/// # Returns +/// +/// Returns `Some(Tracer)` on success, or `None` if export is disabled or the +/// OTLP exporter fails to build. +pub fn build_otel_tracer( + config: &TracingConfig, +) -> Option { + if !config.enable_export { + return None; + } + + let exporter = match SpanExporter::builder() + .with_tonic() + .with_endpoint(&config.otlp_endpoint) + .build() + { + Ok(e) => e, + Err(e) => { + eprintln!("OTLP exporter build failed (non-fatal): {e}"); + return None; + } + }; + + let provider = TracerProvider::builder() + .with_sampler(Sampler::AlwaysOn) + .with_batch_exporter(exporter, OtelTokio) + .with_resource(Resource::new(vec![ + KeyValue::new("service.name", config.service_name.clone()), + KeyValue::new("service.version", env!("CARGO_PKG_VERSION")), + ])) + .build(); + + let tracer = provider.tracer(config.service_name.clone()); + global::set_tracer_provider(provider); + + Some(tracer) +} + /// Initialize OpenTelemetry tracing with OTLP export. /// /// This function sets up the OpenTelemetry tracing pipeline with OTLP /// (OpenTelemetry Protocol) as the trace exporter. OTLP is the vendor-neutral /// standard supported by Jaeger (v1.35+), Grafana Tempo, Datadog, and others. /// +/// **Note:** Prefer [`build_otel_tracer`] for new code. This function is kept +/// for backward compatibility but only sets the global tracer provider without +/// adding a layer to the subscriber. +/// /// # Arguments /// /// * `config` - Tracing configuration