diff --git a/crates/common/src/observability/mod.rs b/crates/common/src/observability/mod.rs index 7237a2104..a945805eb 100644 --- a/crates/common/src/observability/mod.rs +++ b/crates/common/src/observability/mod.rs @@ -110,22 +110,14 @@ pub async fn init_observability( service_name: &str, otlp_endpoint: &str, ) -> CommonResult<()> { - // Determine log directory from environment. - // In K8s pods, LOG_DIR is typically set to /app/logs (backed by an emptyDir volume). - // Locally, falls back to logs/. - let log_dir = std::env::var("LOG_DIR") - .map(|d| format!("{}/{}", d, service_name)) - .unwrap_or_else(|_| format!("logs/{}", service_name)); - - // Disable file logging if the directory cannot be created (e.g. read-only fs). - let enable_file = std::fs::create_dir_all(&log_dir).is_ok(); - + // Console-only logging: stdout is captured by containerd and shipped + // to Loki via Promtail. No file logging needed in K8s. let logger_config = JsonLoggerConfig { service_name: service_name.to_string(), log_level: LogLevel::Info, enable_console: true, - enable_file, - log_directory: log_dir, + enable_file: false, + log_directory: String::new(), rotation: logger::LogRotation::Daily, max_files: 10, max_file_size_mb: 100, @@ -133,14 +125,17 @@ pub async fn init_observability( init_json_logger(logger_config)?; - // Initialize OpenTelemetry tracing with OTLP export + // Initialize OpenTelemetry tracing with OTLP export. + // Gracefully degrade if Tempo is unavailable — logging still works. let tracing_config = TracingConfig { service_name: service_name.to_string(), otlp_endpoint: otlp_endpoint.to_string(), enable_export: true, }; - init_tracing(tracing_config).await?; + if let Err(e) = init_tracing(tracing_config).await { + eprintln!("OTLP tracing unavailable (non-fatal): {}", e); + } Ok(()) } diff --git a/crates/web-gateway/src/grpc/streams.rs b/crates/web-gateway/src/grpc/streams.rs index 64af53acd..f3b3705ca 100644 --- a/crates/web-gateway/src/grpc/streams.rs +++ b/crates/web-gateway/src/grpc/streams.rs @@ -184,14 +184,14 @@ async fn heartbeat_loop(tx: broadcast::Sender) { } /// Reconnect wrapper with exponential backoff for a gRPC stream bridge task. -#[allow(clippy::infinite_loop)] +/// Stops retrying if the server returns Unimplemented (endpoint doesn't exist). async fn reconnect_loop(stream_name: &str, connect_fn: F) where F: Fn() -> Fut, Fut: std::future::Future>, { let mut backoff = Duration::from_secs(1); - let max_backoff = Duration::from_secs(60); + let max_backoff = Duration::from_secs(300); loop { info!("Connecting gRPC stream: {}", stream_name); @@ -202,6 +202,13 @@ where backoff = Duration::from_secs(1); info!("gRPC stream {} ended, reconnecting", stream_name); } + Err(e) if e.code() == tonic::Code::Unimplemented => { + info!( + "gRPC stream {} not implemented on server, disabling", + stream_name + ); + return; + } Err(e) => { warn!( "gRPC stream {} error: {}, reconnecting in {:?}", diff --git a/infra/k8s/services/api-gateway.yaml b/infra/k8s/services/api-gateway.yaml index c98110a89..21af499c1 100644 --- a/infra/k8s/services/api-gateway.yaml +++ b/infra/k8s/services/api-gateway.yaml @@ -114,6 +114,8 @@ spec: value: "http://trading-agent-service:50055" - name: RUST_LOG value: info + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: "http://tempo.foxhunt.svc.cluster.local:4317" volumeMounts: - name: binaries mountPath: /binaries diff --git a/infra/k8s/services/backtesting-service.yaml b/infra/k8s/services/backtesting-service.yaml index 7ba4a7500..73815da70 100644 --- a/infra/k8s/services/backtesting-service.yaml +++ b/infra/k8s/services/backtesting-service.yaml @@ -108,16 +108,12 @@ spec: value: "placeholder" - name: RUST_LOG value: info - - name: LOG_DIR - value: /app/logs - name: OTEL_EXPORTER_OTLP_ENDPOINT value: "http://tempo.foxhunt.svc.cluster.local:4317" volumeMounts: - name: binaries mountPath: /binaries readOnly: true - - name: logs - mountPath: /app/logs readinessProbe: httpGet: path: /health @@ -145,8 +141,6 @@ spec: - name: binary-cache persistentVolumeClaim: claimName: binary-cache-backtesting-service - - name: logs - emptyDir: {} --- apiVersion: v1 kind: Service diff --git a/infra/k8s/services/broker-gateway.yaml b/infra/k8s/services/broker-gateway.yaml index 7f1067d71..bba74d6f7 100644 --- a/infra/k8s/services/broker-gateway.yaml +++ b/infra/k8s/services/broker-gateway.yaml @@ -117,6 +117,8 @@ spec: key: account-id - name: RUST_LOG value: info + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: "http://tempo.foxhunt.svc.cluster.local:4317" volumeMounts: - name: binaries mountPath: /binaries diff --git a/infra/k8s/services/data-acquisition-service.yaml b/infra/k8s/services/data-acquisition-service.yaml index fb303a5a2..5dbbd37fe 100644 --- a/infra/k8s/services/data-acquisition-service.yaml +++ b/infra/k8s/services/data-acquisition-service.yaml @@ -101,6 +101,8 @@ spec: value: "50057" - name: RUST_LOG value: info + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: "http://tempo.foxhunt.svc.cluster.local:4317" volumeMounts: - name: binaries mountPath: /binaries diff --git a/infra/k8s/services/ml-training-service-gpu.yaml b/infra/k8s/services/ml-training-service-gpu.yaml index 4f8b220db..848b0971f 100644 --- a/infra/k8s/services/ml-training-service-gpu.yaml +++ b/infra/k8s/services/ml-training-service-gpu.yaml @@ -120,8 +120,6 @@ spec: value: "true" - name: RUST_LOG value: info - - name: LOG_DIR - value: /app/logs - name: OTEL_EXPORTER_OTLP_ENDPOINT value: "http://tempo.foxhunt.svc.cluster.local:4317" volumeMounts: @@ -131,8 +129,6 @@ spec: - name: tls-certs mountPath: /app/certs/ml_training_service readOnly: true - - name: logs - mountPath: /app/logs readinessProbe: tcpSocket: port: 50053 @@ -163,5 +159,4 @@ spec: - name: tls-certs secret: secretName: ml-training-tls - - name: logs - emptyDir: {} + diff --git a/infra/k8s/services/ml-training-service.yaml b/infra/k8s/services/ml-training-service.yaml index ed501a1b1..600a91d3d 100644 --- a/infra/k8s/services/ml-training-service.yaml +++ b/infra/k8s/services/ml-training-service.yaml @@ -165,8 +165,6 @@ spec: optional: true - name: RUST_LOG value: info - - name: LOG_DIR - value: /app/logs - name: OTEL_EXPORTER_OTLP_ENDPOINT value: "http://tempo.foxhunt.svc.cluster.local:4317" volumeMounts: @@ -176,8 +174,6 @@ spec: - name: tls-certs mountPath: /app/certs/ml_training_service readOnly: true - - name: logs - mountPath: /app/logs readinessProbe: tcpSocket: port: 50053 @@ -206,8 +202,6 @@ spec: - name: tls-certs secret: secretName: ml-training-tls - - name: logs - emptyDir: {} --- apiVersion: v1 kind: Service diff --git a/infra/k8s/services/trading-agent-service.yaml b/infra/k8s/services/trading-agent-service.yaml index 869c5bb22..556ffbf98 100644 --- a/infra/k8s/services/trading-agent-service.yaml +++ b/infra/k8s/services/trading-agent-service.yaml @@ -106,6 +106,8 @@ spec: value: foxhunt-services - name: RUST_LOG value: info + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: "http://tempo.foxhunt.svc.cluster.local:4317" volumeMounts: - name: binaries mountPath: /binaries diff --git a/infra/k8s/services/trading-service-gpu.yaml b/infra/k8s/services/trading-service-gpu.yaml index 69d5aa58a..092f62b7b 100644 --- a/infra/k8s/services/trading-service-gpu.yaml +++ b/infra/k8s/services/trading-service-gpu.yaml @@ -123,8 +123,6 @@ spec: - name: binaries mountPath: /binaries readOnly: true - - name: logs - mountPath: /app/logs readinessProbe: exec: command: @@ -148,5 +146,4 @@ spec: - name: binary-cache persistentVolumeClaim: claimName: binary-cache-trading-service - - name: logs - emptyDir: {} + diff --git a/infra/k8s/services/trading-service.yaml b/infra/k8s/services/trading-service.yaml index 277964bb3..a18cf3126 100644 --- a/infra/k8s/services/trading-service.yaml +++ b/infra/k8s/services/trading-service.yaml @@ -121,16 +121,12 @@ spec: value: "50051" - name: RUST_LOG value: info - - name: LOG_DIR - value: /app/logs - name: OTEL_EXPORTER_OTLP_ENDPOINT value: "http://tempo.foxhunt.svc.cluster.local:4317" volumeMounts: - name: binaries mountPath: /binaries readOnly: true - - name: logs - mountPath: /app/logs readinessProbe: exec: command: @@ -160,8 +156,6 @@ spec: - name: binary-cache persistentVolumeClaim: claimName: binary-cache-trading-service - - name: logs - emptyDir: {} --- apiVersion: v1 kind: Service diff --git a/infra/k8s/services/web-gateway.yaml b/infra/k8s/services/web-gateway.yaml index 7fd9cf82e..7661b1433 100644 --- a/infra/k8s/services/web-gateway.yaml +++ b/infra/k8s/services/web-gateway.yaml @@ -106,6 +106,8 @@ spec: value: "http://api-gateway:50051" - name: RUST_LOG value: info + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: "http://tempo.foxhunt.svc.cluster.local:4317" volumeMounts: - name: binaries mountPath: /binaries