fix(observability): combine fmt + OTLP layers in single subscriber

Previously init_observability() called init_json_logger() (which set the
global subscriber with a fmt layer) and then init_tracing() (which only
set the global tracer provider). Because no tracing_opentelemetry::layer()
was ever added to the subscriber, #[instrument] spans never reached Tempo.

Now init_observability() builds a single Registry subscriber that
composes an EnvFilter, a JSON fmt layer, and an optional OTLP layer
(via tracing_opentelemetry). The OTLP layer gracefully degrades to None
when Tempo is unavailable, so logging always works.

Also adds build_otel_tracer() to tracing_config.rs which returns
Option<Tracer> for the caller to wrap in an OpenTelemetryLayer inside
their subscriber stack, letting the S type parameter be inferred
correctly by the compiler.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

crates/common/src/observability/mod.rs

@@ -74,7 +74,7 @@ mod tests;
 // Re-export commonly used types
 pub use correlation::{CorrelationId, CorrelationIdExt, CORRELATION_ID_HEADER};
 pub use logger::{init_json_logger, JsonLoggerConfig, LogLevel};
-pub use tracing_config::{init_tracing, TracingConfig};
+pub use tracing_config::{build_otel_tracer, init_tracing, TracingConfig};
 
 use crate::error::CommonResult;
 
@@ -110,32 +110,41 @@ pub async fn init_observability(
     service_name: &str,
     otlp_endpoint: &str,
 ) -> CommonResult<()> {
-    // Console-only logging: stdout is captured by containerd and shipped
-    // to Loki via Promtail. No file logging needed in K8s.
-    let logger_config = JsonLoggerConfig {
-        service_name: service_name.to_string(),
-        log_level: LogLevel::Info,
-        enable_console: true,
-        enable_file: false,
-        log_directory: String::new(),
-        rotation: logger::LogRotation::Daily,
-        max_files: 10,
-        max_file_size_mb: 100,
-    };
+    use tracing_subscriber::prelude::*;
 
-    init_json_logger(logger_config)?;
+    // Build env filter: honour RUST_LOG, default to info.
+    let env_filter = tracing_subscriber::EnvFilter::try_from_default_env()
+        .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info"));
 
-    // Initialize OpenTelemetry tracing with OTLP export.
-    // Gracefully degrade if Tempo is unavailable — logging still works.
+    // JSON fmt layer — stdout captured by containerd / Promtail → Loki.
+    let fmt_layer = tracing_subscriber::fmt::layer()
+        .json()
+        .with_target(true)
+        .with_current_span(true)
+        .with_span_list(false);
+
+    // OTLP layer — gracefully degrade if Tempo is unavailable.
+    // build_otel_tracer returns Option<Tracer>; wrapping with
+    // tracing_opentelemetry::layer() inside the subscriber chain lets
+    // the `S` type parameter be inferred correctly by the compiler.
     let tracing_config = TracingConfig {
         service_name: service_name.to_string(),
         otlp_endpoint: otlp_endpoint.to_string(),
         enable_export: true,
     };
-
-    if let Err(e) = init_tracing(tracing_config).await {
-        eprintln!("OTLP tracing unavailable (non-fatal): {}", e);
+    let otel_tracer = tracing_config::build_otel_tracer(&tracing_config);
+    let otel_layer = otel_tracer
+        .map(|tracer| tracing_opentelemetry::layer().with_tracer(tracer));
+    if otel_layer.is_none() {
+        eprintln!("OTLP tracing layer unavailable — logs-only mode");
     }
 
+    // Single subscriber with all layers. `.with(Option<L>)` is a no-op when None.
+    tracing_subscriber::registry()
+        .with(env_filter)
+        .with(fmt_layer)
+        .with(otel_layer)
+        .init();
+
     Ok(())
 }

crates/common/src/observability/tracing_config.rs

@@ -21,7 +21,7 @@
 //! using the W3C Trace Context specification.
 
 use crate::error::{CommonError, CommonResult};
-use opentelemetry::{global, KeyValue};
+use opentelemetry::{global, trace::TracerProvider as _, KeyValue};
 use opentelemetry_otlp::{SpanExporter, WithExportConfig};
 use opentelemetry_sdk::{
     runtime::Tokio as OtelTokio,
@@ -57,12 +57,69 @@ impl Default for TracingConfig {
     }
 }
 
+/// Build an OpenTelemetry tracer backed by an OTLP exporter.
+///
+/// Returns `Some(tracer)` when OTLP export is enabled and the exporter can be
+/// constructed, or `None` on failure (graceful degradation). The caller wraps
+/// the tracer in a `tracing_opentelemetry::layer().with_tracer(tracer)` layer
+/// inside their subscriber stack so the `S` type parameter is inferred
+/// correctly.
+///
+/// This function also registers the tracer provider as the global provider
+/// (required for propagation helpers like `inject_trace_context`).
+///
+/// # Arguments
+///
+/// * `config` - Tracing configuration
+///
+/// # Returns
+///
+/// Returns `Some(Tracer)` on success, or `None` if export is disabled or the
+/// OTLP exporter fails to build.
+pub fn build_otel_tracer(
+    config: &TracingConfig,
+) -> Option<opentelemetry_sdk::trace::Tracer> {
+    if !config.enable_export {
+        return None;
+    }
+
+    let exporter = match SpanExporter::builder()
+        .with_tonic()
+        .with_endpoint(&config.otlp_endpoint)
+        .build()
+    {
+        Ok(e) => e,
+        Err(e) => {
+            eprintln!("OTLP exporter build failed (non-fatal): {e}");
+            return None;
+        }
+    };
+
+    let provider = TracerProvider::builder()
+        .with_sampler(Sampler::AlwaysOn)
+        .with_batch_exporter(exporter, OtelTokio)
+        .with_resource(Resource::new(vec![
+            KeyValue::new("service.name", config.service_name.clone()),
+            KeyValue::new("service.version", env!("CARGO_PKG_VERSION")),
+        ]))
+        .build();
+
+    let tracer = provider.tracer(config.service_name.clone());
+    global::set_tracer_provider(provider);
+
+    Some(tracer)
+}
+
 /// Initialize OpenTelemetry tracing with OTLP export.
 ///
 /// This function sets up the OpenTelemetry tracing pipeline with OTLP
 /// (OpenTelemetry Protocol) as the trace exporter. OTLP is the vendor-neutral
 /// standard supported by Jaeger (v1.35+), Grafana Tempo, Datadog, and others.
 ///
+/// **Note:** Prefer [`build_otel_tracer`] for new code. This function is kept
+/// for backward compatibility but only sets the global tracer provider without
+/// adding a layer to the subscriber.
+///
 /// # Arguments
 ///
 /// * `config` - Tracing configuration