diff --git a/crates/common/src/observability/mod.rs b/crates/common/src/observability/mod.rs index 5c9ecddde..7237a2104 100644 --- a/crates/common/src/observability/mod.rs +++ b/crates/common/src/observability/mod.rs @@ -110,13 +110,22 @@ pub async fn init_observability( service_name: &str, otlp_endpoint: &str, ) -> CommonResult<()> { - // Initialize JSON logger with default configuration + // Determine log directory from environment. + // In K8s pods, LOG_DIR is typically set to /app/logs (backed by an emptyDir volume). + // Locally, falls back to logs/. + let log_dir = std::env::var("LOG_DIR") + .map(|d| format!("{}/{}", d, service_name)) + .unwrap_or_else(|_| format!("logs/{}", service_name)); + + // Disable file logging if the directory cannot be created (e.g. read-only fs). + let enable_file = std::fs::create_dir_all(&log_dir).is_ok(); + let logger_config = JsonLoggerConfig { service_name: service_name.to_string(), log_level: LogLevel::Info, enable_console: true, - enable_file: true, - log_directory: format!("logs/{}", service_name), + enable_file, + log_directory: log_dir, rotation: logger::LogRotation::Daily, max_files: 10, max_file_size_mb: 100, diff --git a/infra/k8s/services/backtesting-service.yaml b/infra/k8s/services/backtesting-service.yaml index 9a34b957c..4100a928f 100644 --- a/infra/k8s/services/backtesting-service.yaml +++ b/infra/k8s/services/backtesting-service.yaml @@ -108,6 +108,8 @@ spec: value: "placeholder" - name: RUST_LOG value: info + - name: LOG_DIR + value: /app/logs volumeMounts: - name: binaries mountPath: /binaries diff --git a/infra/k8s/services/ml-training-service-gpu.yaml b/infra/k8s/services/ml-training-service-gpu.yaml index 449b29cdc..ae09244d0 100644 --- a/infra/k8s/services/ml-training-service-gpu.yaml +++ b/infra/k8s/services/ml-training-service-gpu.yaml @@ -24,6 +24,7 @@ spec: labels: app.kubernetes.io/name: ml-training-service spec: + serviceAccountName: ml-training-service nodeSelector: k8s.scaleway.com/pool-name: gpu-inference tolerations: @@ -119,6 +120,8 @@ spec: value: "true" - name: RUST_LOG value: info + - name: LOG_DIR + value: /app/logs volumeMounts: - name: binaries mountPath: /binaries diff --git a/infra/k8s/services/ml-training-service.yaml b/infra/k8s/services/ml-training-service.yaml index 06c4a1a68..230f65189 100644 --- a/infra/k8s/services/ml-training-service.yaml +++ b/infra/k8s/services/ml-training-service.yaml @@ -1,3 +1,45 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ml-training-service + namespace: foxhunt + labels: + app.kubernetes.io/name: ml-training-service + app.kubernetes.io/part-of: foxhunt +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: ml-training-job-manager + namespace: foxhunt + labels: + app.kubernetes.io/name: ml-training-service + app.kubernetes.io/part-of: foxhunt +rules: + - apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["create", "get", "list", "watch", "delete"] + - apiGroups: [""] + resources: ["pods", "pods/log"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: ml-training-job-manager + namespace: foxhunt + labels: + app.kubernetes.io/name: ml-training-service + app.kubernetes.io/part-of: foxhunt +subjects: + - kind: ServiceAccount + name: ml-training-service + namespace: foxhunt +roleRef: + kind: Role + name: ml-training-job-manager + apiGroup: rbac.authorization.k8s.io +--- apiVersion: apps/v1 kind: Deployment metadata: @@ -21,6 +63,7 @@ spec: labels: app.kubernetes.io/name: ml-training-service spec: + serviceAccountName: ml-training-service imagePullSecrets: - name: scw-registry nodeSelector: @@ -122,6 +165,8 @@ spec: optional: true - name: RUST_LOG value: info + - name: LOG_DIR + value: /app/logs volumeMounts: - name: binaries mountPath: /binaries diff --git a/infra/k8s/services/trading-service.yaml b/infra/k8s/services/trading-service.yaml index 27c597eb0..3fc2ab291 100644 --- a/infra/k8s/services/trading-service.yaml +++ b/infra/k8s/services/trading-service.yaml @@ -121,6 +121,8 @@ spec: value: "50051" - name: RUST_LOG value: info + - name: LOG_DIR + value: /app/logs volumeMounts: - name: binaries mountPath: /binaries