fix(infra): RWO PVC strategy, probe fixes, deploy hardening
- All services: maxSurge=0, maxUnavailable=1 (ReadWriteOnce PVCs deadlock with maxSurge=1 — new pod can't attach volume while old pod still holds it) - ml-training-service: switch httpGet /health:8080 to tcpSocket:50053 (service crashes on log dir permission, never binds health port) - GPU overlays: same strategy and probe fixes - job-template.yaml: use generateName instead of placeholder name (training-MODEL-TIMESTAMP fails k8s RFC 1123 validation) - deploy job: increase rollout timeout 120s → 300s Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -1429,6 +1429,6 @@ deploy:
|
||||
for svc in trading-service api-gateway broker-gateway \
|
||||
ml-training-service backtesting-service \
|
||||
trading-agent-service data-acquisition-service web-gateway; do
|
||||
kubectl -n foxhunt rollout status deployment/$svc --timeout=120s
|
||||
kubectl -n foxhunt rollout status deployment/$svc --timeout=300s
|
||||
done
|
||||
- echo "All services deployed with S3 binaries"
|
||||
|
||||
@@ -11,8 +11,8 @@ spec:
|
||||
strategy:
|
||||
type: RollingUpdate
|
||||
rollingUpdate:
|
||||
maxSurge: 1
|
||||
maxUnavailable: 0
|
||||
maxSurge: 0
|
||||
maxUnavailable: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: api-gateway
|
||||
|
||||
@@ -11,8 +11,8 @@ spec:
|
||||
strategy:
|
||||
type: RollingUpdate
|
||||
rollingUpdate:
|
||||
maxSurge: 1
|
||||
maxUnavailable: 0
|
||||
maxSurge: 0
|
||||
maxUnavailable: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: backtesting-service
|
||||
|
||||
@@ -11,8 +11,8 @@ spec:
|
||||
strategy:
|
||||
type: RollingUpdate
|
||||
rollingUpdate:
|
||||
maxSurge: 1
|
||||
maxUnavailable: 0
|
||||
maxSurge: 0
|
||||
maxUnavailable: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: broker-gateway
|
||||
|
||||
@@ -11,8 +11,8 @@ spec:
|
||||
strategy:
|
||||
type: RollingUpdate
|
||||
rollingUpdate:
|
||||
maxSurge: 1
|
||||
maxUnavailable: 0
|
||||
maxSurge: 0
|
||||
maxUnavailable: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: data-acquisition-service
|
||||
|
||||
@@ -14,8 +14,8 @@ spec:
|
||||
strategy:
|
||||
type: RollingUpdate
|
||||
rollingUpdate:
|
||||
maxSurge: 1
|
||||
maxUnavailable: 0
|
||||
maxSurge: 0
|
||||
maxUnavailable: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: ml-training-service
|
||||
@@ -129,11 +129,16 @@ spec:
|
||||
- name: logs
|
||||
mountPath: /app/logs
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: 8080
|
||||
tcpSocket:
|
||||
port: 50053
|
||||
initialDelaySeconds: 15
|
||||
periodSeconds: 10
|
||||
livenessProbe:
|
||||
tcpSocket:
|
||||
port: 50053
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 15
|
||||
failureThreshold: 5
|
||||
resources:
|
||||
requests:
|
||||
nvidia.com/gpu: "1"
|
||||
|
||||
@@ -11,8 +11,8 @@ spec:
|
||||
strategy:
|
||||
type: RollingUpdate
|
||||
rollingUpdate:
|
||||
maxSurge: 1
|
||||
maxUnavailable: 0
|
||||
maxSurge: 0
|
||||
maxUnavailable: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: ml-training-service
|
||||
@@ -132,15 +132,13 @@ spec:
|
||||
- name: logs
|
||||
mountPath: /app/logs
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: 8080
|
||||
tcpSocket:
|
||||
port: 50053
|
||||
initialDelaySeconds: 15
|
||||
periodSeconds: 10
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: 8080
|
||||
tcpSocket:
|
||||
port: 50053
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 15
|
||||
failureThreshold: 5
|
||||
|
||||
@@ -11,8 +11,8 @@ spec:
|
||||
strategy:
|
||||
type: RollingUpdate
|
||||
rollingUpdate:
|
||||
maxSurge: 1
|
||||
maxUnavailable: 0
|
||||
maxSurge: 0
|
||||
maxUnavailable: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: trading-agent-service
|
||||
|
||||
@@ -14,8 +14,8 @@ spec:
|
||||
strategy:
|
||||
type: RollingUpdate
|
||||
rollingUpdate:
|
||||
maxSurge: 1
|
||||
maxUnavailable: 0
|
||||
maxSurge: 0
|
||||
maxUnavailable: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: trading-service
|
||||
|
||||
@@ -11,8 +11,8 @@ spec:
|
||||
strategy:
|
||||
type: RollingUpdate
|
||||
rollingUpdate:
|
||||
maxSurge: 1
|
||||
maxUnavailable: 0
|
||||
maxSurge: 0
|
||||
maxUnavailable: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: trading-service
|
||||
|
||||
@@ -11,8 +11,8 @@ spec:
|
||||
strategy:
|
||||
type: RollingUpdate
|
||||
rollingUpdate:
|
||||
maxSurge: 1
|
||||
maxUnavailable: 0
|
||||
maxSurge: 0
|
||||
maxUnavailable: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: web-gateway
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: training-MODEL-TIMESTAMP
|
||||
generateName: training-
|
||||
namespace: foxhunt
|
||||
labels:
|
||||
app.kubernetes.io/name: training
|
||||
|
||||
Reference in New Issue
Block a user