foxhunt/infra/k8s/services/ml-training-service.yaml

apiVersion: v1
kind: ServiceAccount
metadata:
  name: ml-training-service
  namespace: foxhunt
  labels:
    app.kubernetes.io/name: ml-training-service
    app.kubernetes.io/part-of: foxhunt
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: ml-training-job-manager
  namespace: foxhunt
  labels:
    app.kubernetes.io/name: ml-training-service
    app.kubernetes.io/part-of: foxhunt
rules:
  - apiGroups: ["batch"]
    resources: ["jobs"]
    verbs: ["create", "get", "list", "watch", "delete"]
  - apiGroups: [""]
    resources: ["pods", "pods/log"]
    verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: ml-training-job-manager
  namespace: foxhunt
  labels:
    app.kubernetes.io/name: ml-training-service
    app.kubernetes.io/part-of: foxhunt
subjects:
  - kind: ServiceAccount
    name: ml-training-service
    namespace: foxhunt
roleRef:
  kind: Role
  name: ml-training-job-manager
  apiGroup: rbac.authorization.k8s.io
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: ml-training-service
  namespace: foxhunt
  labels:
    app.kubernetes.io/name: ml-training-service
    app.kubernetes.io/part-of: foxhunt
spec:
  replicas: 1
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxSurge: 0
      maxUnavailable: 1
  selector:
    matchLabels:
      app.kubernetes.io/name: ml-training-service
  template:
    metadata:
      annotations:
        gitlab.com/prometheus_scrape: "true"
        gitlab.com/prometheus_port: "9094"
        gitlab.com/prometheus_path: "/metrics"
      labels:
        app.kubernetes.io/name: ml-training-service
        app.kubernetes.io/part-of: foxhunt
    spec:
      serviceAccountName: ml-training-service
      securityContext:
        runAsNonRoot: true
        runAsUser: 1000
        runAsGroup: 1000
        fsGroup: 1000
        seccompProfile:
          type: RuntimeDefault
      imagePullSecrets:
        - name: gitlab-registry
      nodeSelector:
        k8s.scaleway.com/pool-name: foxhunt
      initContainers:
        - name: fetch-binary
          image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest
          command: ["/bin/sh", "-c"]
          args:
            - |
              set -e
              rclone copyto \
                ":s3:foxhunt-binaries/services/ml-training-service" \
                "/binaries/ml-training-service" \
                --s3-provider=Minio \
                --s3-endpoint=https://minio.foxhunt.svc.cluster.local:9000 \
                --s3-access-key-id="${MINIO_ACCESS_KEY}" \
                --s3-secret-access-key="${MINIO_SECRET_KEY}" \
                --s3-no-check-bucket \
                --no-check-certificate
              chmod +x /binaries/ml-training-service
              echo "Fetched ml-training-service ($(stat -c%s /binaries/ml-training-service) bytes)"
          env:
            - name: MINIO_ACCESS_KEY
              valueFrom:
                secretKeyRef:
                  name: minio-credentials
                  key: access-key
            - name: MINIO_SECRET_KEY
              valueFrom:
                secretKeyRef:
                  name: minio-credentials
                  key: secret-key
          volumeMounts:
            - name: binaries
              mountPath: /binaries
          resources:
            requests:
              cpu: 100m
              memory: 64Mi
            limits:
              cpu: 500m
              memory: 128Mi
      containers:
        - name: ml-training-service
          image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest
          securityContext:
            allowPrivilegeEscalation: false
            readOnlyRootFilesystem: true
            capabilities:
              drop: ["ALL"]
          command: ["/binaries/ml-training-service", "serve"]
          ports:
            - containerPort: 50053
              name: grpc
            - containerPort: 9094
              name: metrics
          env:
            - name: DATABASE_PASSWORD
              valueFrom:
                secretKeyRef:
                  name: db-credentials
                  key: password
            - name: DATABASE_URL
              value: "postgresql://foxhunt:$(DATABASE_PASSWORD)@postgres:5432/foxhunt"
            - name: REDIS_URL
              value: "redis://redis:6379"
            - name: JWT_SECRET
              valueFrom:
                secretKeyRef:
                  name: jwt-secret
                  key: secret
            - name: JWT_ISSUER
              value: foxhunt-api
            - name: JWT_AUDIENCE
              value: foxhunt-services
            - name: S3_ENDPOINT
              value: "https://minio.foxhunt.svc.cluster.local:9000"
            - name: S3_BUCKET
              value: foxhunt-models
            - name: MINIO_CA_CERT_PATH
              value: /etc/ssl/minio-ca/ca.crt
            - name: AWS_ACCESS_KEY_ID
              valueFrom:
                secretKeyRef:
                  name: minio-credentials
                  key: access-key
            - name: AWS_SECRET_ACCESS_KEY
              valueFrom:
                secretKeyRef:
                  name: minio-credentials
                  key: secret-key
            - name: RUST_LOG
              value: "info,opentelemetry=warn,h2=warn,tonic=warn,hyper=warn"
            - name: OTEL_EXPORTER_OTLP_ENDPOINT
              value: "http://tempo.foxhunt.svc.cluster.local:4317"
          volumeMounts:
            - name: binaries
              mountPath: /binaries
              readOnly: true
            - name: tls-certs
              mountPath: /app/certs/ml_training_service
              readOnly: true
            - name: minio-ca
              mountPath: /etc/ssl/minio-ca
              readOnly: true
            - name: tmp
              mountPath: /tmp
          readinessProbe:
            tcpSocket:
              port: 50053
            initialDelaySeconds: 15
            periodSeconds: 10
          livenessProbe:
            tcpSocket:
              port: 50053
            initialDelaySeconds: 30
            periodSeconds: 15
            failureThreshold: 5
          resources:
            requests:
              cpu: 200m
              memory: 256Mi
            limits:
              cpu: 500m
              memory: 512Mi
      volumes:
        - name: binaries
          emptyDir:
            sizeLimit: 200Mi
        - name: tls-certs
          secret:
            secretName: ml-training-tls
        - name: minio-ca
          configMap:
            name: minio-ca-cert
        - name: tmp
          emptyDir:
            sizeLimit: 50Mi
---
apiVersion: v1
kind: Service
metadata:
  name: ml-training-service
  namespace: foxhunt
  labels:
    app.kubernetes.io/name: ml-training-service
    app.kubernetes.io/part-of: foxhunt
spec:
  selector:
    app.kubernetes.io/name: ml-training-service
  ports:
    - port: 50053
      targetPort: 50053
      name: grpc
    - port: 9094
      targetPort: 9094
      name: metrics