foxhunt/k8s/deployments/ml-training-service-deployment.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: ml-training-service
  namespace: foxhunt
  labels:
    app: ml-training-service
    component: ml-training
    tier: backend
    version: v1
spec:
  replicas: 1
  revisionHistoryLimit: 10
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxUnavailable: 1
      maxSurge: 1
  selector:
    matchLabels:
      app: ml-training-service
  template:
    metadata:
      labels:
        app: ml-training-service
        component: ml-training
        tier: backend
        version: v1
      annotations:
        prometheus.io/scrape: "true"
        prometheus.io/port: "9094"
        prometheus.io/path: "/metrics"
    spec:
      serviceAccountName: foxhunt-ml-training-service
      securityContext:
        runAsNonRoot: true
        runAsUser: 1000
        runAsGroup: 1000
        fsGroup: 1000
      containers:
      - name: ml-training-service
        image: foxhunt/ml-training-service:latest
        imagePullPolicy: Always
        ports:
        - name: grpc
          containerPort: 50054
          protocol: TCP
        - name: metrics
          containerPort: 9094
          protocol: TCP
        - name: health
          containerPort: 8095
          protocol: TCP
        env:
        - name: ML_TRAINING_SERVICE_HOST
          value: "0.0.0.0"
        - name: ML_TRAINING_SERVICE_PORT
          value: "50054"
        - name: METRICS_PORT
          value: "9094"
        - name: DATABASE_URL
          valueFrom:
            secretKeyRef:
              name: foxhunt-secrets
              key: database-url
        - name: REDIS_URL
          valueFrom:
            configMapKeyRef:
              name: foxhunt-config
              key: redis-url
        - name: VAULT_ADDR
          valueFrom:
            configMapKeyRef:
              name: foxhunt-config
              key: vault-addr
        - name: VAULT_TOKEN
          valueFrom:
            secretKeyRef:
              name: foxhunt-secrets
              key: vault-token
        - name: AWS_ACCESS_KEY_ID
          valueFrom:
            secretKeyRef:
              name: foxhunt-secrets
              key: aws-access-key-id
        - name: AWS_SECRET_ACCESS_KEY
          valueFrom:
            secretKeyRef:
              name: foxhunt-secrets
              key: aws-secret-access-key
        - name: AWS_REGION
          value: "us-east-1"
        - name: S3_MODEL_BUCKET
          value: "foxhunt-models"
        - name: RUST_LOG
          value: "info,ml_training_service=debug"
        - name: RUST_BACKTRACE
          value: "1"
        resources:
          requests:
            cpu: "2000m"
            memory: "4Gi"
            nvidia.com/gpu: 1
          limits:
            cpu: "4000m"
            memory: "8Gi"
            nvidia.com/gpu: 1
        livenessProbe:
          httpGet:
            path: /health
            port: 8095
          initialDelaySeconds: 30
          periodSeconds: 10
          timeoutSeconds: 5
          successThreshold: 1
          failureThreshold: 3
        readinessProbe:
          httpGet:
            path: /ready
            port: 8095
          initialDelaySeconds: 10
          periodSeconds: 5
          timeoutSeconds: 3
          successThreshold: 1
          failureThreshold: 3
        volumeMounts:
        - name: config
          mountPath: /app/config
          readOnly: true
        - name: logs
          mountPath: /app/logs
        - name: models
          mountPath: /app/models
        - name: checkpoints
          mountPath: /app/checkpoints
      volumes:
      - name: config
        configMap:
          name: ml-training-service-config
      - name: logs
        emptyDir: {}
      - name: models
        persistentVolumeClaim:
          claimName: ml-models-pvc
      - name: checkpoints
        persistentVolumeClaim:
          claimName: ml-checkpoints-pvc
      nodeSelector:
        nvidia.com/gpu: "true"
      tolerations:
      - key: nvidia.com/gpu
        operator: Exists
        effect: NoSchedule