From 2cc68af7d603b2eccd226ffc861803cfac0a860d Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sun, 1 Mar 2026 22:53:08 +0100 Subject: [PATCH] feat(ml): add OTLP tracing to all 6 training/eval binaries Replace tracing_subscriber::fmt() with init_observability() which adds JSON structured logging + optional OTLP export to Tempo. When OTEL_EXPORTER_OTLP_ENDPOINT env var is unset, falls back to fmt-only. Co-Authored-By: Claude Opus 4.6 --- crates/ml/examples/evaluate_baseline.rs | 15 ++++++++------- crates/ml/examples/evaluate_supervised.rs | 14 ++++++++------ crates/ml/examples/hyperopt_baseline_rl.rs | 15 +++++++++------ .../ml/examples/hyperopt_baseline_supervised.rs | 14 +++++++++----- crates/ml/examples/train_baseline_rl.rs | 15 ++++++++------- crates/ml/examples/train_baseline_supervised.rs | 14 ++++++++------ 6 files changed, 50 insertions(+), 37 deletions(-) diff --git a/crates/ml/examples/evaluate_baseline.rs b/crates/ml/examples/evaluate_baseline.rs index f474447bc..9d88bcec5 100644 --- a/crates/ml/examples/evaluate_baseline.rs +++ b/crates/ml/examples/evaluate_baseline.rs @@ -636,13 +636,14 @@ fn run_sanity_checks( #[allow(clippy::cognitive_complexity, clippy::too_many_lines)] fn main() -> Result<()> { - // Initialize tracing - tracing_subscriber::fmt() - .with_env_filter( - tracing_subscriber::EnvFilter::try_from_default_env() - .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")), - ) - .init(); + // Initialize tracing with optional OTLP export to Tempo + let otlp_endpoint = std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT").ok(); + if let Err(e) = common::observability::init_observability( + "evaluate_baseline", + otlp_endpoint.as_deref(), + ) { + eprintln!("Observability init failed (non-fatal): {e}"); + } let args = Args::parse(); diff --git a/crates/ml/examples/evaluate_supervised.rs b/crates/ml/examples/evaluate_supervised.rs index 8ce0c4d18..cafda3ba0 100644 --- a/crates/ml/examples/evaluate_supervised.rs +++ b/crates/ml/examples/evaluate_supervised.rs @@ -532,12 +532,14 @@ fn evaluate_fold( #[allow(clippy::cognitive_complexity, clippy::too_many_lines)] fn main() -> Result<()> { - tracing_subscriber::fmt() - .with_env_filter( - tracing_subscriber::EnvFilter::try_from_default_env() - .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")), - ) - .init(); + // Initialize tracing with optional OTLP export to Tempo + let otlp_endpoint = std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT").ok(); + if let Err(e) = common::observability::init_observability( + "evaluate_supervised", + otlp_endpoint.as_deref(), + ) { + eprintln!("Observability init failed (non-fatal): {e}"); + } let args = Args::parse(); diff --git a/crates/ml/examples/hyperopt_baseline_rl.rs b/crates/ml/examples/hyperopt_baseline_rl.rs index 868a871f9..01f7301a5 100644 --- a/crates/ml/examples/hyperopt_baseline_rl.rs +++ b/crates/ml/examples/hyperopt_baseline_rl.rs @@ -40,7 +40,7 @@ use clap::Parser; use serde_json::Value; use std::path::PathBuf; use std::time::Instant; -use tracing::{error, info, warn, Level}; +use tracing::{error, info, warn}; use ml::hyperopt::adapters::dqn::DQNTrainer; use ml::hyperopt::adapters::ppo::PPOTrainer; @@ -262,11 +262,14 @@ fn run_ppo_hyperopt(args: &Args, parallel: usize, device: &candle_core::Device) #[allow(clippy::cognitive_complexity)] fn main() -> Result<()> { - // Initialize tracing - tracing_subscriber::fmt() - .with_max_level(Level::INFO) - .with_target(false) - .init(); + // Initialize tracing with optional OTLP export to Tempo + let otlp_endpoint = std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT").ok(); + if let Err(e) = common::observability::init_observability( + "hyperopt_baseline_rl", + otlp_endpoint.as_deref(), + ) { + eprintln!("Observability init failed (non-fatal): {e}"); + } training_metrics::init(); metrics_server::start_metrics_server(9094); diff --git a/crates/ml/examples/hyperopt_baseline_supervised.rs b/crates/ml/examples/hyperopt_baseline_supervised.rs index 2dd96c105..742338b02 100644 --- a/crates/ml/examples/hyperopt_baseline_supervised.rs +++ b/crates/ml/examples/hyperopt_baseline_supervised.rs @@ -35,7 +35,7 @@ use clap::Parser; use serde_json::Value; use std::path::PathBuf; use std::time::Instant; -use tracing::{error, info, warn, Level}; +use tracing::{error, info, warn}; use common::metrics::{server as metrics_server, training_metrics}; @@ -544,10 +544,14 @@ const VALID_MODELS: &[&str] = &[ #[allow(clippy::cognitive_complexity)] fn main() -> Result<()> { - tracing_subscriber::fmt() - .with_max_level(Level::INFO) - .with_target(false) - .init(); + // Initialize tracing with optional OTLP export to Tempo + let otlp_endpoint = std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT").ok(); + if let Err(e) = common::observability::init_observability( + "hyperopt_baseline_supervised", + otlp_endpoint.as_deref(), + ) { + eprintln!("Observability init failed (non-fatal): {e}"); + } training_metrics::init(); metrics_server::start_metrics_server(9094); diff --git a/crates/ml/examples/train_baseline_rl.rs b/crates/ml/examples/train_baseline_rl.rs index aaf164846..3b5841910 100644 --- a/crates/ml/examples/train_baseline_rl.rs +++ b/crates/ml/examples/train_baseline_rl.rs @@ -759,13 +759,14 @@ fn run_training(args: &Args) -> Result> { // --------------------------------------------------------------------------- fn main() -> Result<()> { - // Initialize tracing - tracing_subscriber::fmt() - .with_env_filter( - tracing_subscriber::EnvFilter::try_from_default_env() - .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")), - ) - .init(); + // Initialize tracing with optional OTLP export to Tempo + let otlp_endpoint = std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT").ok(); + if let Err(e) = common::observability::init_observability( + "train_baseline_rl", + otlp_endpoint.as_deref(), + ) { + eprintln!("Observability init failed (non-fatal): {e}"); + } metrics::init(); metrics_server::start_metrics_server(9094); diff --git a/crates/ml/examples/train_baseline_supervised.rs b/crates/ml/examples/train_baseline_supervised.rs index 411eef143..ee8c54c16 100644 --- a/crates/ml/examples/train_baseline_supervised.rs +++ b/crates/ml/examples/train_baseline_supervised.rs @@ -797,12 +797,14 @@ fn run_training(args: &Args) -> Result> { // --------------------------------------------------------------------------- fn main() -> Result<()> { - tracing_subscriber::fmt() - .with_env_filter( - tracing_subscriber::EnvFilter::try_from_default_env() - .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")), - ) - .init(); + // Initialize tracing with optional OTLP export to Tempo + let otlp_endpoint = std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT").ok(); + if let Err(e) = common::observability::init_observability( + "train_baseline_supervised", + otlp_endpoint.as_deref(), + ) { + eprintln!("Observability init failed (non-fatal): {e}"); + } metrics::init(); metrics_server::start_metrics_server(9094);