fix(infra): RWO PVC strategy, probe fixes, deploy hardening

- All services: maxSurge=0, maxUnavailable=1 (ReadWriteOnce PVCs
  deadlock with maxSurge=1 — new pod can't attach volume while old
  pod still holds it)
- ml-training-service: switch httpGet /health:8080 to tcpSocket:50053
  (service crashes on log dir permission, never binds health port)
- GPU overlays: same strategy and probe fixes
- job-template.yaml: use generateName instead of placeholder name
  (training-MODEL-TIMESTAMP fails k8s RFC 1123 validation)
- deploy job: increase rollout timeout 120s → 300s

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-02-28 19:50:53 +01:00
parent e6d0abccf0
commit 64d57e7454
12 changed files with 34 additions and 31 deletions

View File

@@ -1429,6 +1429,6 @@ deploy:
for svc in trading-service api-gateway broker-gateway \
ml-training-service backtesting-service \
trading-agent-service data-acquisition-service web-gateway; do
kubectl -n foxhunt rollout status deployment/$svc --timeout=120s
kubectl -n foxhunt rollout status deployment/$svc --timeout=300s
done
- echo "All services deployed with S3 binaries"

View File

@@ -11,8 +11,8 @@ spec:
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
maxUnavailable: 0
maxSurge: 0
maxUnavailable: 1
selector:
matchLabels:
app.kubernetes.io/name: api-gateway

View File

@@ -11,8 +11,8 @@ spec:
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
maxUnavailable: 0
maxSurge: 0
maxUnavailable: 1
selector:
matchLabels:
app.kubernetes.io/name: backtesting-service

View File

@@ -11,8 +11,8 @@ spec:
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
maxUnavailable: 0
maxSurge: 0
maxUnavailable: 1
selector:
matchLabels:
app.kubernetes.io/name: broker-gateway

View File

@@ -11,8 +11,8 @@ spec:
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
maxUnavailable: 0
maxSurge: 0
maxUnavailable: 1
selector:
matchLabels:
app.kubernetes.io/name: data-acquisition-service

View File

@@ -14,8 +14,8 @@ spec:
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
maxUnavailable: 0
maxSurge: 0
maxUnavailable: 1
selector:
matchLabels:
app.kubernetes.io/name: ml-training-service
@@ -129,11 +129,16 @@ spec:
- name: logs
mountPath: /app/logs
readinessProbe:
httpGet:
path: /health
port: 8080
tcpSocket:
port: 50053
initialDelaySeconds: 15
periodSeconds: 10
livenessProbe:
tcpSocket:
port: 50053
initialDelaySeconds: 30
periodSeconds: 15
failureThreshold: 5
resources:
requests:
nvidia.com/gpu: "1"

View File

@@ -11,8 +11,8 @@ spec:
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
maxUnavailable: 0
maxSurge: 0
maxUnavailable: 1
selector:
matchLabels:
app.kubernetes.io/name: ml-training-service
@@ -132,15 +132,13 @@ spec:
- name: logs
mountPath: /app/logs
readinessProbe:
httpGet:
path: /health
port: 8080
tcpSocket:
port: 50053
initialDelaySeconds: 15
periodSeconds: 10
livenessProbe:
httpGet:
path: /health
port: 8080
tcpSocket:
port: 50053
initialDelaySeconds: 30
periodSeconds: 15
failureThreshold: 5

View File

@@ -11,8 +11,8 @@ spec:
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
maxUnavailable: 0
maxSurge: 0
maxUnavailable: 1
selector:
matchLabels:
app.kubernetes.io/name: trading-agent-service

View File

@@ -14,8 +14,8 @@ spec:
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
maxUnavailable: 0
maxSurge: 0
maxUnavailable: 1
selector:
matchLabels:
app.kubernetes.io/name: trading-service

View File

@@ -11,8 +11,8 @@ spec:
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
maxUnavailable: 0
maxSurge: 0
maxUnavailable: 1
selector:
matchLabels:
app.kubernetes.io/name: trading-service

View File

@@ -11,8 +11,8 @@ spec:
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
maxUnavailable: 0
maxSurge: 0
maxUnavailable: 1
selector:
matchLabels:
app.kubernetes.io/name: web-gateway

View File

@@ -1,7 +1,7 @@
apiVersion: batch/v1
kind: Job
metadata:
name: training-MODEL-TIMESTAMP
generateName: training-
namespace: foxhunt
labels:
app.kubernetes.io/name: training