diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1371e6267..7a775e0a0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1429,6 +1429,6 @@ deploy: for svc in trading-service api-gateway broker-gateway \ ml-training-service backtesting-service \ trading-agent-service data-acquisition-service web-gateway; do - kubectl -n foxhunt rollout status deployment/$svc --timeout=120s + kubectl -n foxhunt rollout status deployment/$svc --timeout=300s done - echo "All services deployed with S3 binaries" diff --git a/infra/k8s/services/api-gateway.yaml b/infra/k8s/services/api-gateway.yaml index 4201ee8b9..c98110a89 100644 --- a/infra/k8s/services/api-gateway.yaml +++ b/infra/k8s/services/api-gateway.yaml @@ -11,8 +11,8 @@ spec: strategy: type: RollingUpdate rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 + maxSurge: 0 + maxUnavailable: 1 selector: matchLabels: app.kubernetes.io/name: api-gateway diff --git a/infra/k8s/services/backtesting-service.yaml b/infra/k8s/services/backtesting-service.yaml index a435adbbc..9a34b957c 100644 --- a/infra/k8s/services/backtesting-service.yaml +++ b/infra/k8s/services/backtesting-service.yaml @@ -11,8 +11,8 @@ spec: strategy: type: RollingUpdate rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 + maxSurge: 0 + maxUnavailable: 1 selector: matchLabels: app.kubernetes.io/name: backtesting-service diff --git a/infra/k8s/services/broker-gateway.yaml b/infra/k8s/services/broker-gateway.yaml index e607b6e5a..7f1067d71 100644 --- a/infra/k8s/services/broker-gateway.yaml +++ b/infra/k8s/services/broker-gateway.yaml @@ -11,8 +11,8 @@ spec: strategy: type: RollingUpdate rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 + maxSurge: 0 + maxUnavailable: 1 selector: matchLabels: app.kubernetes.io/name: broker-gateway diff --git a/infra/k8s/services/data-acquisition-service.yaml b/infra/k8s/services/data-acquisition-service.yaml index e54d53f36..fb303a5a2 100644 --- a/infra/k8s/services/data-acquisition-service.yaml +++ b/infra/k8s/services/data-acquisition-service.yaml @@ -11,8 +11,8 @@ spec: strategy: type: RollingUpdate rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 + maxSurge: 0 + maxUnavailable: 1 selector: matchLabels: app.kubernetes.io/name: data-acquisition-service diff --git a/infra/k8s/services/ml-training-service-gpu.yaml b/infra/k8s/services/ml-training-service-gpu.yaml index a3f8db92a..449b29cdc 100644 --- a/infra/k8s/services/ml-training-service-gpu.yaml +++ b/infra/k8s/services/ml-training-service-gpu.yaml @@ -14,8 +14,8 @@ spec: strategy: type: RollingUpdate rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 + maxSurge: 0 + maxUnavailable: 1 selector: matchLabels: app.kubernetes.io/name: ml-training-service @@ -129,11 +129,16 @@ spec: - name: logs mountPath: /app/logs readinessProbe: - httpGet: - path: /health - port: 8080 + tcpSocket: + port: 50053 initialDelaySeconds: 15 periodSeconds: 10 + livenessProbe: + tcpSocket: + port: 50053 + initialDelaySeconds: 30 + periodSeconds: 15 + failureThreshold: 5 resources: requests: nvidia.com/gpu: "1" diff --git a/infra/k8s/services/ml-training-service.yaml b/infra/k8s/services/ml-training-service.yaml index dcc4a169b..06c4a1a68 100644 --- a/infra/k8s/services/ml-training-service.yaml +++ b/infra/k8s/services/ml-training-service.yaml @@ -11,8 +11,8 @@ spec: strategy: type: RollingUpdate rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 + maxSurge: 0 + maxUnavailable: 1 selector: matchLabels: app.kubernetes.io/name: ml-training-service @@ -132,15 +132,13 @@ spec: - name: logs mountPath: /app/logs readinessProbe: - httpGet: - path: /health - port: 8080 + tcpSocket: + port: 50053 initialDelaySeconds: 15 periodSeconds: 10 livenessProbe: - httpGet: - path: /health - port: 8080 + tcpSocket: + port: 50053 initialDelaySeconds: 30 periodSeconds: 15 failureThreshold: 5 diff --git a/infra/k8s/services/trading-agent-service.yaml b/infra/k8s/services/trading-agent-service.yaml index 5296bf66f..869c5bb22 100644 --- a/infra/k8s/services/trading-agent-service.yaml +++ b/infra/k8s/services/trading-agent-service.yaml @@ -11,8 +11,8 @@ spec: strategy: type: RollingUpdate rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 + maxSurge: 0 + maxUnavailable: 1 selector: matchLabels: app.kubernetes.io/name: trading-agent-service diff --git a/infra/k8s/services/trading-service-gpu.yaml b/infra/k8s/services/trading-service-gpu.yaml index 599832522..69d5aa58a 100644 --- a/infra/k8s/services/trading-service-gpu.yaml +++ b/infra/k8s/services/trading-service-gpu.yaml @@ -14,8 +14,8 @@ spec: strategy: type: RollingUpdate rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 + maxSurge: 0 + maxUnavailable: 1 selector: matchLabels: app.kubernetes.io/name: trading-service diff --git a/infra/k8s/services/trading-service.yaml b/infra/k8s/services/trading-service.yaml index ac5a86b0b..27c597eb0 100644 --- a/infra/k8s/services/trading-service.yaml +++ b/infra/k8s/services/trading-service.yaml @@ -11,8 +11,8 @@ spec: strategy: type: RollingUpdate rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 + maxSurge: 0 + maxUnavailable: 1 selector: matchLabels: app.kubernetes.io/name: trading-service diff --git a/infra/k8s/services/web-gateway.yaml b/infra/k8s/services/web-gateway.yaml index 9f5c3dd30..7fd9cf82e 100644 --- a/infra/k8s/services/web-gateway.yaml +++ b/infra/k8s/services/web-gateway.yaml @@ -11,8 +11,8 @@ spec: strategy: type: RollingUpdate rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 + maxSurge: 0 + maxUnavailable: 1 selector: matchLabels: app.kubernetes.io/name: web-gateway diff --git a/infra/k8s/training/job-template.yaml b/infra/k8s/training/job-template.yaml index d7841a754..663b0d3cc 100644 --- a/infra/k8s/training/job-template.yaml +++ b/infra/k8s/training/job-template.yaml @@ -1,7 +1,7 @@ apiVersion: batch/v1 kind: Job metadata: - name: training-MODEL-TIMESTAMP + generateName: training- namespace: foxhunt labels: app.kubernetes.io/name: training