gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(observability): switch to containerd stdout logging, fix stream retry

Browse Source 
- Disable file logging in init_observability — use console-only (stdout).
  Containerd captures stdout, Promtail ships to Loki. Eliminates
  "Failed to create log directory: Permission denied" errors on 3 services.
- Remove LOG_DIR env vars and /app/logs emptyDir volumes from 5 deployments.
- Add OTEL_EXPORTER_OTLP_ENDPOINT=tempo to 5 services that were missing it
  (api-gateway, web-gateway, data-acquisition, trading-agent, broker-gateway).
- Make OTLP tracing init non-fatal — services degrade gracefully if Tempo is
  unavailable instead of failing entire observability stack.
- Stop web-gateway gRPC stream retry on Unimplemented status. When trading-service
  doesn't implement streaming endpoints, log once and stop instead of retrying
  every 60s indefinitely.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-02-28 22:49:57 +01:00
parent 48316d6874
commit 6b94045bcd
12 changed files with 30 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
crates/common/src/observability/mod.rs
View File

@@ -110,22 +110,14 @@ pub async fn init_observability(
service_name: &str,
otlp_endpoint: &str,
) -> CommonResult<()> {
// Determine log directory from environment.
// In K8s pods, LOG_DIR is typically set to /app/logs (backed by an emptyDir volume).
// Locally, falls back to logs/<service_name>.
let log_dir = std::env::var("LOG_DIR")
.map(|d| format!("{}/{}", d, service_name))
.unwrap_or_else(|_| format!("logs/{}", service_name));
// Disable file logging if the directory cannot be created (e.g. read-only fs).
let enable_file = std::fs::create_dir_all(&log_dir).is_ok();
// Console-only logging: stdout is captured by containerd and shipped
// to Loki via Promtail. No file logging needed in K8s.
let logger_config = JsonLoggerConfig {
service_name: service_name.to_string(),
log_level: LogLevel::Info,
enable_console: true,
enable_file,
log_directory: log_dir,
enable_file: false,
log_directory: String::new(),
rotation: logger::LogRotation::Daily,
max_files: 10,
max_file_size_mb: 100,
@@ -133,14 +125,17 @@ pub async fn init_observability(
init_json_logger(logger_config)?;
// Initialize OpenTelemetry tracing with OTLP export
// Initialize OpenTelemetry tracing with OTLP export.
// Gracefully degrade if Tempo is unavailable — logging still works.
let tracing_config = TracingConfig {
service_name: service_name.to_string(),
otlp_endpoint: otlp_endpoint.to_string(),
enable_export: true,
};
init_tracing(tracing_config).await?;
if let Err(e) = init_tracing(tracing_config).await {
eprintln!("OTLP tracing unavailable (non-fatal): {}", e);
}
Ok(())
}

11
crates/web-gateway/src/grpc/streams.rs
View File

@@ -184,14 +184,14 @@ async fn heartbeat_loop(tx: broadcast::Sender<String>) {
}
/// Reconnect wrapper with exponential backoff for a gRPC stream bridge task.
#[allow(clippy::infinite_loop)]
/// Stops retrying if the server returns Unimplemented (endpoint doesn't exist).
async fn reconnect_loop<F, Fut>(stream_name: &str, connect_fn: F)
where
F: Fn() -> Fut,
Fut: std::future::Future<Output = Result<(), tonic::Status>>,
{
let mut backoff = Duration::from_secs(1);
let max_backoff = Duration::from_secs(60);
let max_backoff = Duration::from_secs(300);
loop {
info!("Connecting gRPC stream: {}", stream_name);
@@ -202,6 +202,13 @@ where
backoff = Duration::from_secs(1);
info!("gRPC stream {} ended, reconnecting", stream_name);
}
Err(e) if e.code() == tonic::Code::Unimplemented => {
info!(
"gRPC stream {} not implemented on server, disabling",
stream_name
);
return;
}
Err(e) => {
warn!(
"gRPC stream {} error: {}, reconnecting in {:?}",

2
infra/k8s/services/api-gateway.yaml
View File

@@ -114,6 +114,8 @@ spec:
value: "http://trading-agent-service:50055"
- name: RUST_LOG
value: info
- name: OTEL_EXPORTER_OTLP_ENDPOINT
value: "http://tempo.foxhunt.svc.cluster.local:4317"
volumeMounts:
- name: binaries
mountPath: /binaries

6
infra/k8s/services/backtesting-service.yaml
View File

@@ -108,16 +108,12 @@ spec:
value: "placeholder"
- name: RUST_LOG
value: info
- name: LOG_DIR
value: /app/logs
- name: OTEL_EXPORTER_OTLP_ENDPOINT
value: "http://tempo.foxhunt.svc.cluster.local:4317"
volumeMounts:
- name: binaries
mountPath: /binaries
readOnly: true
- name: logs
mountPath: /app/logs
readinessProbe:
httpGet:
path: /health
@@ -145,8 +141,6 @@ spec:
- name: binary-cache
persistentVolumeClaim:
claimName: binary-cache-backtesting-service
- name: logs
emptyDir: {}
---
apiVersion: v1
kind: Service

2
infra/k8s/services/broker-gateway.yaml
View File

@@ -117,6 +117,8 @@ spec:
key: account-id
- name: RUST_LOG
value: info
- name: OTEL_EXPORTER_OTLP_ENDPOINT
value: "http://tempo.foxhunt.svc.cluster.local:4317"
volumeMounts:
- name: binaries
mountPath: /binaries

2
infra/k8s/services/data-acquisition-service.yaml
View File

@@ -101,6 +101,8 @@ spec:
value: "50057"
- name: RUST_LOG
value: info
- name: OTEL_EXPORTER_OTLP_ENDPOINT
value: "http://tempo.foxhunt.svc.cluster.local:4317"
volumeMounts:
- name: binaries
mountPath: /binaries

7
infra/k8s/services/ml-training-service-gpu.yaml
View File

@@ -120,8 +120,6 @@ spec:
value: "true"
- name: RUST_LOG
value: info
- name: LOG_DIR
value: /app/logs
- name: OTEL_EXPORTER_OTLP_ENDPOINT
value: "http://tempo.foxhunt.svc.cluster.local:4317"
volumeMounts:
@@ -131,8 +129,6 @@ spec:
- name: tls-certs
mountPath: /app/certs/ml_training_service
readOnly: true
- name: logs
mountPath: /app/logs
readinessProbe:
tcpSocket:
port: 50053
@@ -163,5 +159,4 @@ spec:
- name: tls-certs
secret:
secretName: ml-training-tls
- name: logs
emptyDir: {}

6
infra/k8s/services/ml-training-service.yaml
View File

@@ -165,8 +165,6 @@ spec:
optional: true
- name: RUST_LOG
value: info
- name: LOG_DIR
value: /app/logs
- name: OTEL_EXPORTER_OTLP_ENDPOINT
value: "http://tempo.foxhunt.svc.cluster.local:4317"
volumeMounts:
@@ -176,8 +174,6 @@ spec:
- name: tls-certs
mountPath: /app/certs/ml_training_service
readOnly: true
- name: logs
mountPath: /app/logs
readinessProbe:
tcpSocket:
port: 50053
@@ -206,8 +202,6 @@ spec:
- name: tls-certs
secret:
secretName: ml-training-tls
- name: logs
emptyDir: {}
---
apiVersion: v1
kind: Service

2
infra/k8s/services/trading-agent-service.yaml
View File

@@ -106,6 +106,8 @@ spec:
value: foxhunt-services
- name: RUST_LOG
value: info
- name: OTEL_EXPORTER_OTLP_ENDPOINT
value: "http://tempo.foxhunt.svc.cluster.local:4317"
volumeMounts:
- name: binaries
mountPath: /binaries

5
infra/k8s/services/trading-service-gpu.yaml
View File

@@ -123,8 +123,6 @@ spec:
- name: binaries
mountPath: /binaries
readOnly: true
- name: logs
mountPath: /app/logs
readinessProbe:
exec:
command:
@@ -148,5 +146,4 @@ spec:
- name: binary-cache
persistentVolumeClaim:
claimName: binary-cache-trading-service
- name: logs
emptyDir: {}

6
infra/k8s/services/trading-service.yaml
View File

@@ -121,16 +121,12 @@ spec:
value: "50051"
- name: RUST_LOG
value: info
- name: LOG_DIR
value: /app/logs
- name: OTEL_EXPORTER_OTLP_ENDPOINT
value: "http://tempo.foxhunt.svc.cluster.local:4317"
volumeMounts:
- name: binaries
mountPath: /binaries
readOnly: true
- name: logs
mountPath: /app/logs
readinessProbe:
exec:
command:
@@ -160,8 +156,6 @@ spec:
- name: binary-cache
persistentVolumeClaim:
claimName: binary-cache-trading-service
- name: logs
emptyDir: {}
---
apiVersion: v1
kind: Service

2
infra/k8s/services/web-gateway.yaml
View File

@@ -106,6 +106,8 @@ spec:
value: "http://api-gateway:50051"
- name: RUST_LOG
value: info
- name: OTEL_EXPORTER_OTLP_ENDPOINT
value: "http://tempo.foxhunt.svc.cluster.local:4317"
volumeMounts:
- name: binaries
mountPath: /binaries