gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(ml): add OTLP tracing to all 6 training/eval binaries

Browse Source 
Replace tracing_subscriber::fmt() with init_observability() which adds
JSON structured logging + optional OTLP export to Tempo. When
OTEL_EXPORTER_OTLP_ENDPOINT env var is unset, falls back to fmt-only.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-03-01 22:53:08 +01:00
parent e741b50932
commit 2cc68af7d6
6 changed files with 50 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
crates/ml/examples/evaluate_baseline.rs
View File

@@ -636,13 +636,14 @@ fn run_sanity_checks(
#[allow(clippy::cognitive_complexity, clippy::too_many_lines)]
fn main() -> Result<()> {
// Initialize tracing
tracing_subscriber::fmt()
.with_env_filter(
tracing_subscriber::EnvFilter::try_from_default_env()
.unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")),
)
.init();
// Initialize tracing with optional OTLP export to Tempo
let otlp_endpoint = std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT").ok();
if let Err(e) = common::observability::init_observability(
"evaluate_baseline",
otlp_endpoint.as_deref(),
) {
eprintln!("Observability init failed (non-fatal): {e}");
}
let args = Args::parse();

14
crates/ml/examples/evaluate_supervised.rs
View File

@@ -532,12 +532,14 @@ fn evaluate_fold(
#[allow(clippy::cognitive_complexity, clippy::too_many_lines)]
fn main() -> Result<()> {
tracing_subscriber::fmt()
.with_env_filter(
tracing_subscriber::EnvFilter::try_from_default_env()
.unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")),
)
.init();
// Initialize tracing with optional OTLP export to Tempo
let otlp_endpoint = std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT").ok();
if let Err(e) = common::observability::init_observability(
"evaluate_supervised",
otlp_endpoint.as_deref(),
) {
eprintln!("Observability init failed (non-fatal): {e}");
}
let args = Args::parse();

15
crates/ml/examples/hyperopt_baseline_rl.rs
View File

@@ -40,7 +40,7 @@ use clap::Parser;
use serde_json::Value;
use std::path::PathBuf;
use std::time::Instant;
use tracing::{error, info, warn, Level};
use tracing::{error, info, warn};
use ml::hyperopt::adapters::dqn::DQNTrainer;
use ml::hyperopt::adapters::ppo::PPOTrainer;
@@ -262,11 +262,14 @@ fn run_ppo_hyperopt(args: &Args, parallel: usize, device: &candle_core::Device)
#[allow(clippy::cognitive_complexity)]
fn main() -> Result<()> {
// Initialize tracing
tracing_subscriber::fmt()
.with_max_level(Level::INFO)
.with_target(false)
.init();
// Initialize tracing with optional OTLP export to Tempo
let otlp_endpoint = std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT").ok();
if let Err(e) = common::observability::init_observability(
"hyperopt_baseline_rl",
otlp_endpoint.as_deref(),
) {
eprintln!("Observability init failed (non-fatal): {e}");
}
training_metrics::init();
metrics_server::start_metrics_server(9094);

14
crates/ml/examples/hyperopt_baseline_supervised.rs
View File

@@ -35,7 +35,7 @@ use clap::Parser;
use serde_json::Value;
use std::path::PathBuf;
use std::time::Instant;
use tracing::{error, info, warn, Level};
use tracing::{error, info, warn};
use common::metrics::{server as metrics_server, training_metrics};
@@ -544,10 +544,14 @@ const VALID_MODELS: &[&str] = &[
#[allow(clippy::cognitive_complexity)]
fn main() -> Result<()> {
tracing_subscriber::fmt()
.with_max_level(Level::INFO)
.with_target(false)
.init();
// Initialize tracing with optional OTLP export to Tempo
let otlp_endpoint = std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT").ok();
if let Err(e) = common::observability::init_observability(
"hyperopt_baseline_supervised",
otlp_endpoint.as_deref(),
) {
eprintln!("Observability init failed (non-fatal): {e}");
}
training_metrics::init();
metrics_server::start_metrics_server(9094);

15
crates/ml/examples/train_baseline_rl.rs
View File

@@ -759,13 +759,14 @@ fn run_training(args: &Args) -> Result<Vec<RlTrainingResult>> {
// ---------------------------------------------------------------------------
fn main() -> Result<()> {
// Initialize tracing
tracing_subscriber::fmt()
.with_env_filter(
tracing_subscriber::EnvFilter::try_from_default_env()
.unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")),
)
.init();
// Initialize tracing with optional OTLP export to Tempo
let otlp_endpoint = std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT").ok();
if let Err(e) = common::observability::init_observability(
"train_baseline_rl",
otlp_endpoint.as_deref(),
) {
eprintln!("Observability init failed (non-fatal): {e}");
}
metrics::init();
metrics_server::start_metrics_server(9094);

14
crates/ml/examples/train_baseline_supervised.rs
View File

@@ -797,12 +797,14 @@ fn run_training(args: &Args) -> Result<Vec<TrainingResult>> {
// ---------------------------------------------------------------------------
fn main() -> Result<()> {
tracing_subscriber::fmt()
.with_env_filter(
tracing_subscriber::EnvFilter::try_from_default_env()
.unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")),
)
.init();
// Initialize tracing with optional OTLP export to Tempo
let otlp_endpoint = std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT").ok();
if let Err(e) = common::observability::init_observability(
"train_baseline_supervised",
otlp_endpoint.as_deref(),
) {
eprintln!("Observability init failed (non-fatal): {e}");
}
metrics::init();
metrics_server::start_metrics_server(9094);