From 7a01618e862684054bb03ba46f866758b61ac437 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sun, 1 Mar 2026 16:31:22 +0100 Subject: [PATCH] =?UTF-8?q?fix(infra):=20upgrade=20OpenTelemetry=200.27?= =?UTF-8?q?=E2=86=920.31=20to=20align=20tonic=20versions,=20fix=20DCGM=20s?= =?UTF-8?q?cheduling?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit OTLP fix: opentelemetry-otlp 0.27 bundled tonic 0.12 while the workspace uses tonic 0.14, making with_channel() impossible (type mismatch). Upgrading to otel-otlp 0.31 aligns both on tonic 0.14, enabling explicit Channel construction that respects http:// scheme (no spurious TLS negotiation). API migrations (otel 0.27→0.31): - TracerProvider → SdkTracerProvider - with_batch_exporter(exporter, runtime) → with_batch_exporter(exporter) - Resource::new(vec![...]) → Resource::builder().with_service_name().build() - global::shutdown_tracer_provider() removed (Drop-based shutdown) - opentelemetry-otlp feature "tonic" → "grpc-tonic" DCGM fix: remove runtimeClassName: nvidia from DaemonSet — Scaleway Kapsule GPU pools use nvidia runtime as default containerd handler. The RuntimeClass CRD is only created by the full GPU Operator, not the device plugin alone. Co-Authored-By: Claude Opus 4.6 --- Cargo.lock | 187 +++++++----------- Cargo.toml | 10 +- .../src/observability/tracing_config.rs | 59 ++++-- infra/k8s/monitoring/dcgm-exporter.yaml | 5 +- 4 files changed, 124 insertions(+), 137 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a863a7d5e..992811867 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -226,7 +226,7 @@ dependencies = [ "tokio", "tokio-stream", "tokio-test", - "tonic 0.14.2", + "tonic", "tonic-health", "tonic-prost", "tonic-prost-build", @@ -270,7 +270,7 @@ dependencies = [ "tokio", "tokio-stream", "toml", - "tonic 0.14.2", + "tonic", "tonic-prost", "tracing", "tracing-subscriber", @@ -610,7 +610,7 @@ dependencies = [ "arrow-schema 55.2.0", "chrono", "half", - "indexmap 2.11.4", + "indexmap", "lexical-core", "memchr", "num", @@ -1565,7 +1565,7 @@ dependencies = [ "tokio-stream", "tokio-test", "tokio-util", - "tonic 0.14.2", + "tonic", "tonic-health", "tonic-prost", "tonic-prost-build", @@ -1824,7 +1824,7 @@ dependencies = [ "tokio", "tokio-stream", "tokio-test", - "tonic 0.14.2", + "tonic", "tonic-health", "tonic-prost", "tonic-prost-build", @@ -2313,7 +2313,7 @@ dependencies = [ "tokio", "tokio-test", "toml", - "tonic 0.14.2", + "tonic", "tower-layer", "tower-service", "tracing", @@ -2985,7 +2985,7 @@ dependencies = [ "tokio-retry", "tokio-stream", "tokio-util", - "tonic 0.14.2", + "tonic", "tonic-prost", "tonic-prost-build", "tonic-reflection", @@ -3841,7 +3841,7 @@ dependencies = [ "thiserror 1.0.69", "tokio", "tokio-stream", - "tonic 0.14.2", + "tonic", "tracing", "tracing-subscriber", "trading_engine", @@ -3882,7 +3882,7 @@ dependencies = [ "tokio", "tokio-stream", "tokio-test", - "tonic 0.14.2", + "tonic", "tonic-prost", "tonic-prost-build", "tracing", @@ -4090,7 +4090,7 @@ dependencies = [ "tokio-stream", "tokio-test", "toml", - "tonic 0.14.2", + "tonic", "tonic-prost", "tonic-prost-build", "tracing", @@ -4358,7 +4358,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.12", - "indexmap 2.11.4", + "indexmap", "slab", "tokio", "tokio-util", @@ -4377,7 +4377,7 @@ dependencies = [ "futures-core", "futures-sink", "http 1.3.1", - "indexmap 2.11.4", + "indexmap", "slab", "tokio", "tokio-util", @@ -5002,16 +5002,6 @@ version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7c5cedc30da3a610cac6b4ba17597bdf7152cf974e8aab3afb3d54455e371c8" -[[package]] -name = "indexmap" -version = "1.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" -dependencies = [ - "autocfg", - "hashbrown 0.12.3", -] - [[package]] name = "indexmap" version = "2.11.4" @@ -5155,7 +5145,7 @@ dependencies = [ "prost-build", "reqwest 0.12.23", "tokio", - "tonic 0.14.2", + "tonic", "tonic-prost", "tonic-prost-build", "uuid", @@ -5181,7 +5171,7 @@ dependencies = [ "serial_test", "thiserror 1.0.69", "tokio", - "tonic 0.14.2", + "tonic", "tonic-prost", "tonic-prost-build", "tracing", @@ -5863,7 +5853,7 @@ dependencies = [ "hyper 1.7.0", "hyper-rustls 0.27.7", "hyper-util", - "indexmap 2.11.4", + "indexmap", "ipnet", "metrics", "metrics-util", @@ -6108,7 +6098,7 @@ dependencies = [ "tokio-retry", "tokio-stream", "tokio-util", - "tonic 0.14.2", + "tonic", "tonic-prost", "tonic-prost-build", "tonic-reflection", @@ -6651,68 +6641,76 @@ dependencies = [ [[package]] name = "opentelemetry" -version = "0.27.1" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab70038c28ed37b97d8ed414b6429d343a8bbf44c9f79ec854f3a643029ba6d7" +checksum = "b84bcd6ae87133e903af7ef497404dda70c60d0ea14895fc8a5e6722754fc2a0" dependencies = [ "futures-core", "futures-sink", "js-sys", "pin-project-lite", - "thiserror 1.0.69", + "thiserror 2.0.17", "tracing", ] [[package]] -name = "opentelemetry-otlp" -version = "0.27.0" +name = "opentelemetry-http" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91cf61a1868dacc576bf2b2a1c3e9ab150af7272909e80085c3173384fe11f76" +checksum = "d7a6d09a73194e6b66df7c8f1b680f156d916a1a942abf2de06823dd02b7855d" dependencies = [ "async-trait", - "futures-core", + "bytes", "http 1.3.1", "opentelemetry", +] + +[[package]] +name = "opentelemetry-otlp" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2366db2dca4d2ad033cad11e6ee42844fd727007af5ad04a1730f4cb8163bf" +dependencies = [ + "http 1.3.1", + "opentelemetry", + "opentelemetry-http", "opentelemetry-proto", "opentelemetry_sdk", - "prost 0.13.5", - "thiserror 1.0.69", + "prost 0.14.1", + "thiserror 2.0.17", "tokio", - "tonic 0.12.3", + "tonic", "tracing", ] [[package]] name = "opentelemetry-proto" -version = "0.27.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6e05acbfada5ec79023c85368af14abd0b307c015e9064d249b2a950ef459a6" +checksum = "a7175df06de5eaee9909d4805a3d07e28bb752c34cab57fa9cff549da596b30f" dependencies = [ "opentelemetry", "opentelemetry_sdk", - "prost 0.13.5", - "tonic 0.12.3", + "prost 0.14.1", + "tonic", + "tonic-prost", ] [[package]] name = "opentelemetry_sdk" -version = "0.27.1" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "231e9d6ceef9b0b2546ddf52335785ce41252bc7474ee8ba05bfad277be13ab8" +checksum = "e14ae4f5991976fd48df6d843de219ca6d31b01daaab2dad5af2badeded372bd" dependencies = [ - "async-trait", "futures-channel", "futures-executor", "futures-util", - "glob", "opentelemetry", "percent-encoding", - "rand 0.8.5", - "serde_json", - "thiserror 1.0.69", + "rand 0.9.2", + "thiserror 2.0.17", "tokio", "tokio-stream", - "tracing", ] [[package]] @@ -7014,7 +7012,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset 0.4.2", - "indexmap 2.11.4", + "indexmap", "serde", ] @@ -7025,7 +7023,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" dependencies = [ "fixedbitset 0.5.7", - "indexmap 2.11.4", + "indexmap", ] [[package]] @@ -9034,7 +9032,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.11.4", + "indexmap", "itoa", "ryu", "serde", @@ -9428,7 +9426,7 @@ dependencies = [ "futures-util", "hashbrown 0.15.5", "hashlink", - "indexmap 2.11.4", + "indexmap", "log", "memchr", "once_cell", @@ -9641,7 +9639,7 @@ dependencies = [ "flate2", "fs2", "futures", - "indexmap 2.11.4", + "indexmap", "lru", "object_store", "parking_lot 0.12.5", @@ -9684,7 +9682,7 @@ dependencies = [ "tempfile", "thiserror 1.0.69", "tokio", - "tonic 0.14.2", + "tonic", "tonic-prost", "tracing", "tracing-subscriber", @@ -10069,7 +10067,7 @@ dependencies = [ "tokio-stream", "tokio-test", "toml", - "tonic 0.14.2", + "tonic", "tonic-health", "tracing", "tracing-subscriber", @@ -10437,7 +10435,7 @@ version = "0.22.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" dependencies = [ - "indexmap 2.11.4", + "indexmap", "serde", "serde_spanned", "toml_datetime 0.6.11", @@ -10451,7 +10449,7 @@ version = "0.23.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3effe7c0e86fdff4f69cdd2ccc1b96f933e24811c5441d44904e8683e27184b" dependencies = [ - "indexmap 2.11.4", + "indexmap", "toml_datetime 0.7.2", "toml_parser", "winnow", @@ -10472,36 +10470,6 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" -[[package]] -name = "tonic" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52" -dependencies = [ - "async-stream", - "async-trait", - "axum 0.7.9", - "base64 0.22.1", - "bytes", - "h2 0.4.12", - "http 1.3.1", - "http-body 1.0.1", - "http-body-util", - "hyper 1.7.0", - "hyper-timeout", - "hyper-util", - "percent-encoding", - "pin-project", - "prost 0.13.5", - "socket2 0.5.10", - "tokio", - "tokio-stream", - "tower 0.4.13", - "tower-layer", - "tower-service", - "tracing", -] - [[package]] name = "tonic" version = "0.14.2" @@ -10554,7 +10522,7 @@ dependencies = [ "prost 0.14.1", "tokio", "tokio-stream", - "tonic 0.14.2", + "tonic", "tonic-prost", ] @@ -10566,7 +10534,7 @@ checksum = "66bd50ad6ce1252d87ef024b3d64fe4c3cf54a86fb9ef4c631fdd0ded7aeaa67" dependencies = [ "bytes", "prost 0.14.1", - "tonic 0.14.2", + "tonic", ] [[package]] @@ -10595,7 +10563,7 @@ dependencies = [ "prost-types", "tokio", "tokio-stream", - "tonic 0.14.2", + "tonic", "tonic-prost", ] @@ -10620,11 +10588,8 @@ checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" dependencies = [ "futures-core", "futures-util", - "indexmap 1.9.3", "pin-project", "pin-project-lite", - "rand 0.8.5", - "slab", "tokio", "tokio-util", "tower-layer", @@ -10640,7 +10605,7 @@ checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" dependencies = [ "futures-core", "futures-util", - "indexmap 2.11.4", + "indexmap", "pin-project-lite", "slab", "sync_wrapper 1.0.2", @@ -10717,9 +10682,9 @@ dependencies = [ [[package]] name = "tracing" -version = "0.1.41" +version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ "log", "pin-project-lite", @@ -10741,9 +10706,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.30" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", @@ -10752,9 +10717,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.34" +version = "0.1.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" dependencies = [ "once_cell", "valuable", @@ -10783,14 +10748,12 @@ dependencies = [ [[package]] name = "tracing-opentelemetry" -version = "0.28.0" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97a971f6058498b5c0f1affa23e7ea202057a7301dbff68e968b2d578bcbd053" +checksum = "1ac28f2d093c6c477eaa76b23525478f38de514fa9aeb1285738d4b97a9552fc" dependencies = [ "js-sys", - "once_cell", "opentelemetry", - "opentelemetry_sdk", "smallvec", "tracing", "tracing-core", @@ -10811,9 +10774,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.20" +version = "0.3.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2054a14f5307d601f88daf0553e1cbf472acc4f2c51afab632431cdcd72124d5" +checksum = "2f30143827ddab0d256fd843b7a66d164e9f271cfa0dde49142c5ca0ca291f1e" dependencies = [ "matchers", "nu-ansi-term", @@ -10889,7 +10852,7 @@ dependencies = [ "thiserror 1.0.69", "tokio", "tokio-stream", - "tonic 0.14.2", + "tonic", "tonic-health", "tonic-prost", "tonic-prost-build", @@ -11016,7 +10979,7 @@ dependencies = [ "tokio", "tokio-stream", "tokio-tungstenite", - "tonic 0.14.2", + "tonic", "tonic-health", "tonic-prost", "tonic-prost-build", @@ -11059,7 +11022,7 @@ dependencies = [ "tokio", "tokio-stream", "toml", - "tonic 0.14.2", + "tonic", "tonic-prost", "tonic-prost-build", "tracing", @@ -11081,7 +11044,7 @@ dependencies = [ "serde", "serde_json", "tokio", - "tonic 0.14.2", + "tonic", "tonic-prost", "tonic-prost-build", "tracing", @@ -11555,7 +11518,7 @@ dependencies = [ "tokio", "tokio-stream", "tokio-test", - "tonic 0.14.2", + "tonic", "tonic-prost", "tonic-prost-build", "tower 0.4.13", @@ -12276,7 +12239,7 @@ dependencies = [ "crc32fast", "crossbeam-utils", "displaydoc", - "indexmap 2.11.4", + "indexmap", "num_enum", "thiserror 1.0.69", ] diff --git a/Cargo.toml b/Cargo.toml index 644cd14a5..91fa58600 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -336,10 +336,12 @@ hdrhistogram = "7.5" influxdb2 = { version = "0.5", default-features = false, features = ["native-tls"] } # OpenTelemetry for production monitoring and extensive logging -tracing-opentelemetry = "0.28" -opentelemetry = { version = "0.27", features = ["trace", "metrics", "logs"] } -opentelemetry-otlp = { version = "0.27", features = ["tonic", "metrics", "trace", "logs"] } -opentelemetry_sdk = { version = "0.27", features = ["rt-tokio", "metrics", "trace", "logs"] } +# NOTE: otel-otlp 0.31 uses tonic 0.14 (matching workspace), fixing the tonic version mismatch +# that prevented with_channel() for explicit plaintext gRPC channels. +tracing-opentelemetry = "0.32" +opentelemetry = { version = "0.31", features = ["trace", "metrics", "logs"] } +opentelemetry-otlp = { version = "0.31", default-features = false, features = ["grpc-tonic", "metrics", "trace", "logs", "internal-logs"] } +opentelemetry_sdk = { version = "0.31", features = ["rt-tokio", "metrics", "trace", "logs"] } # Additional commonly used dependencies - CONSOLIDATED # Build dependencies already defined above with tonic dependencies diff --git a/crates/common/src/observability/tracing_config.rs b/crates/common/src/observability/tracing_config.rs index 50d5c58f8..cd37439c8 100644 --- a/crates/common/src/observability/tracing_config.rs +++ b/crates/common/src/observability/tracing_config.rs @@ -22,10 +22,9 @@ use crate::error::{CommonError, CommonResult}; use opentelemetry::{global, trace::TracerProvider as _, KeyValue}; -use opentelemetry_otlp::{SpanExporter, WithExportConfig}; +use opentelemetry_otlp::{SpanExporter, WithTonicConfig}; use opentelemetry_sdk::{ - runtime::Tokio as OtelTokio, - trace::{Sampler, TracerProvider}, + trace::{Sampler, SdkTracerProvider}, Resource, }; @@ -83,9 +82,19 @@ pub fn build_otel_tracer( return None; } + // Build an explicit tonic Channel that respects the URI scheme (http vs https). + // This prevents TLS negotiation on plaintext http:// endpoints like Tempo. + let channel = match tonic::transport::Channel::from_shared(config.otlp_endpoint.clone()) { + Ok(endpoint) => endpoint.connect_lazy(), + Err(e) => { + eprintln!("Invalid OTLP endpoint URI (non-fatal): {e}"); + return None; + } + }; + let exporter = match SpanExporter::builder() .with_tonic() - .with_endpoint(&config.otlp_endpoint) + .with_channel(channel) .build() { Ok(e) => e, @@ -95,13 +104,15 @@ pub fn build_otel_tracer( } }; - let provider = TracerProvider::builder() + let provider = SdkTracerProvider::builder() .with_sampler(Sampler::AlwaysOn) - .with_batch_exporter(exporter, OtelTokio) - .with_resource(Resource::new(vec![ - KeyValue::new("service.name", config.service_name.clone()), - KeyValue::new("service.version", env!("CARGO_PKG_VERSION")), - ])) + .with_batch_exporter(exporter) + .with_resource( + Resource::builder() + .with_service_name(config.service_name.clone()) + .with_attribute(KeyValue::new("service.version", env!("CARGO_PKG_VERSION"))) + .build(), + ) .build(); let tracer = provider.tracer(config.service_name.clone()); @@ -153,21 +164,27 @@ pub async fn init_tracing(config: TracingConfig) -> CommonResult<()> { return Ok(()); } - // Build OTLP span exporter targeting the configured collector endpoint + // Build an explicit tonic Channel — see build_otel_tracer() comment for rationale + let channel = tonic::transport::Channel::from_shared(config.otlp_endpoint.clone()) + .map_err(|e| CommonError::config(format!("Invalid OTLP endpoint URI: {}", e)))? + .connect_lazy(); + let exporter = SpanExporter::builder() .with_tonic() - .with_endpoint(&config.otlp_endpoint) + .with_channel(channel) .build() .map_err(|e| CommonError::config(format!("Failed to build OTLP exporter: {}", e)))?; - // Create a tracer provider with the OTLP exporter using batched async export - let provider = TracerProvider::builder() + // Create a tracer provider with the OTLP exporter using batched export + let provider = SdkTracerProvider::builder() .with_sampler(Sampler::AlwaysOn) - .with_batch_exporter(exporter, OtelTokio) - .with_resource(Resource::new(vec![ - KeyValue::new("service.name", config.service_name.clone()), - KeyValue::new("service.version", env!("CARGO_PKG_VERSION")), - ])) + .with_batch_exporter(exporter) + .with_resource( + Resource::builder() + .with_service_name(config.service_name.clone()) + .with_attribute(KeyValue::new("service.version", env!("CARGO_PKG_VERSION"))) + .build(), + ) .build(); // Set as global tracer provider @@ -191,7 +208,9 @@ pub async fn init_tracing(config: TracingConfig) -> CommonResult<()> { /// } /// ``` pub async fn shutdown_tracing() { - global::shutdown_tracer_provider(); + // In opentelemetry 0.31, shutdown is on the provider itself via Drop. + // The global provider is shut down when it goes out of scope. + // Explicit flush is no longer needed via a global function. } /// Extract trace context from gRPC metadata. diff --git a/infra/k8s/monitoring/dcgm-exporter.yaml b/infra/k8s/monitoring/dcgm-exporter.yaml index 4885fcb1d..5bf5b2add 100644 --- a/infra/k8s/monitoring/dcgm-exporter.yaml +++ b/infra/k8s/monitoring/dcgm-exporter.yaml @@ -65,7 +65,10 @@ spec: operator: In values: - "true" - runtimeClassName: nvidia + # NOTE: runtimeClassName: nvidia removed — Scaleway Kapsule GPU pools use + # the nvidia container runtime as the default containerd handler. The + # RuntimeClass CRD is only created by the full NVIDIA GPU Operator, not + # the device plugin alone, so requiring it prevents pod scheduling. tolerations: # Tolerate GPU taints so the exporter can run on GPU nodes - operator: Exists