foxhunt/infra/k8s/services/ml-training-service.yaml

apiVersion: v1
kind: ServiceAccount
metadata:
  name: ml-training-service
  namespace: foxhunt
  labels:
    app.kubernetes.io/name: ml-training-service
    app.kubernetes.io/part-of: foxhunt
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: ml-training-job-manager
  namespace: foxhunt
  labels:
    app.kubernetes.io/name: ml-training-service
    app.kubernetes.io/part-of: foxhunt
rules:
  - apiGroups: ["batch"]
    resources: ["jobs"]
    verbs: ["create", "get", "list", "watch", "delete"]
  - apiGroups: [""]
    resources: ["pods", "pods/log"]
    verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: ml-training-job-manager
  namespace: foxhunt
  labels:
    app.kubernetes.io/name: ml-training-service
    app.kubernetes.io/part-of: foxhunt
subjects:
  - kind: ServiceAccount
    name: ml-training-service
    namespace: foxhunt
roleRef:
  kind: Role
  name: ml-training-job-manager
  apiGroup: rbac.authorization.k8s.io
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: ml-training-service
  namespace: foxhunt
  labels:
    app.kubernetes.io/name: ml-training-service
    app.kubernetes.io/part-of: foxhunt
spec:
  replicas: 1
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxSurge: 0
      maxUnavailable: 1
  selector:
    matchLabels:
      app.kubernetes.io/name: ml-training-service
  template:
    metadata:
      annotations:
        prometheus.io/scrape: "true"
        prometheus.io/port: "9094"
        prometheus.io/path: "/metrics"
      labels:
        app.kubernetes.io/name: ml-training-service
        app.kubernetes.io/part-of: foxhunt
    spec:
      serviceAccountName: ml-training-service
      securityContext:
        runAsNonRoot: true
        runAsUser: 1000
        runAsGroup: 1000
        fsGroup: 1000
        seccompProfile:
          type: RuntimeDefault
      imagePullSecrets:
        - name: gitlab-registry
      nodeSelector:
        k8s.scaleway.com/pool-name: platform
      initContainers:
        - name: fetch-binary
          image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest
          command: ["/bin/sh", "-c"]
          args:
            - |
              set -e
              BINARY="ml-training-service"
              curl -fSL -o "/binaries/${BINARY}" \
                --header "DEPLOY-TOKEN: ${GITLAB_DEPLOY_TOKEN}" \
                "${GITLAB_API}/projects/1/packages/generic/foxhunt-services/${FOXHUNT_RELEASE}/${BINARY}"
              chmod +x "/binaries/${BINARY}"
              echo "Fetched ${BINARY} ${FOXHUNT_RELEASE} ($(stat -c%s /binaries/${BINARY}) bytes)"
          env:
            - name: GITLAB_DEPLOY_TOKEN
              valueFrom:
                secretKeyRef:
                  name: gitlab-deploy-token
                  key: token
            - name: GITLAB_API
              value: "http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181/api/v4"
            - name: FOXHUNT_RELEASE
              value: "latest"
          volumeMounts:
            - name: binaries
              mountPath: /binaries
          resources:
            requests:
              cpu: 100m
              memory: 64Mi
            limits:
              cpu: 500m
              memory: 128Mi
      containers:
        - name: ml-training-service
          image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest
          securityContext:
            allowPrivilegeEscalation: false
            readOnlyRootFilesystem: true
            capabilities:
              drop: ["ALL"]
          command: ["/binaries/ml-training-service", "serve"]
          ports:
            - containerPort: 50053
              name: grpc
            - containerPort: 9094
              name: metrics
          env:
            - name: DATABASE_PASSWORD
              valueFrom:
                secretKeyRef:
                  name: db-credentials
                  key: password
            - name: DATABASE_URL
              value: "postgresql://foxhunt:$(DATABASE_PASSWORD)@postgres:5432/foxhunt"
            - name: REDIS_URL
              value: "redis://redis:6379"
            - name: JWT_SECRET
              valueFrom:
                secretKeyRef:
                  name: jwt-secret
                  key: secret
            - name: JWT_ISSUER
              value: foxhunt-api
            - name: JWT_AUDIENCE
              value: foxhunt-services
            - name: S3_ENDPOINT
              value: "http://minio.foxhunt.svc.cluster.local:9000"
            - name: S3_BUCKET
              value: foxhunt-models
            - name: AWS_ACCESS_KEY_ID
              valueFrom:
                secretKeyRef:
                  name: minio-credentials
                  key: access-key
            - name: AWS_SECRET_ACCESS_KEY
              valueFrom:
                secretKeyRef:
                  name: minio-credentials
                  key: secret-key
            - name: TRAINING_RUNTIME_IMAGE
              value: "gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-training-runtime:latest"
            - name: CALLBACK_ENDPOINT
              value: "http://ml-training-service.foxhunt.svc.cluster.local:50053"
            - name: RUST_LOG
              value: "info,opentelemetry=warn,h2=warn,tonic=warn,hyper=warn"
            - name: OTEL_EXPORTER_OTLP_ENDPOINT
              value: "http://tempo.foxhunt.svc.cluster.local:4317"
          volumeMounts:
            - name: binaries
              mountPath: /binaries
              readOnly: true
            - name: tls-certs
              mountPath: /app/certs/ml_training_service
              readOnly: true
            - name: tmp
              mountPath: /tmp
          readinessProbe:
            tcpSocket:
              port: 50053
            initialDelaySeconds: 15
            periodSeconds: 10
          livenessProbe:
            tcpSocket:
              port: 50053
            initialDelaySeconds: 30
            periodSeconds: 15
            failureThreshold: 5
          resources:
            requests:
              cpu: 200m
              memory: 256Mi
            limits:
              cpu: 500m
              memory: 512Mi
      volumes:
        - name: binaries
          emptyDir:
            sizeLimit: 200Mi
        - name: tls-certs
          secret:
            secretName: ml-training-tls
        - name: tmp
          emptyDir:
            sizeLimit: 50Mi
---
apiVersion: v1
kind: Service
metadata:
  name: ml-training-service
  namespace: foxhunt
  labels:
    app.kubernetes.io/name: ml-training-service
    app.kubernetes.io/part-of: foxhunt
spec:
  selector:
    app.kubernetes.io/name: ml-training-service
  ports:
    - port: 50053
      targetPort: 50053
      name: grpc
    - port: 9094
      targetPort: 9094
      name: metrics