foxhunt/infra/k8s/argo/compile-training-template.yaml

apiVersion: argoproj.io/v1alpha1
kind: WorkflowTemplate
metadata:
  name: compile-training
  namespace: foxhunt
  labels:
    app.kubernetes.io/name: compile-training
    app.kubernetes.io/part-of: foxhunt
spec:
  activeDeadlineSeconds: 3600
  serviceAccountName: argo-workflow
  entrypoint: compile
  podMetadata:
    labels:
      app.kubernetes.io/component: compile
      app.kubernetes.io/part-of: foxhunt
  ttlStrategy:
    secondsAfterCompletion: 3600
  arguments:
    parameters:
      - name: commit-sha
        value: HEAD
      - name: cuda-compute-cap
        value: "89"
  volumeClaimTemplates:
    - metadata:
        name: workspace
      spec:
        accessModes: [ReadWriteOnce]
        resources:
          requests:
            storage: 20Gi
        storageClassName: scw-bssd
  volumes:
    - name: sccache
      persistentVolumeClaim:
        claimName: sccache-pvc
    - name: git-ssh-key
      secret:
        secretName: argo-git-ssh-key
        defaultMode: 256
  templates:
    - name: compile
      inputs:
        parameters:
          - name: commit-sha
            value: "{{workflow.parameters.commit-sha}}"
          - name: cuda-compute-cap
            value: "{{workflow.parameters.cuda-compute-cap}}"
      dag:
        tasks:
          - name: checkout
            template: git-checkout
            arguments:
              parameters:
                - name: commit-sha
                  value: "{{inputs.parameters.commit-sha}}"
          - name: build
            dependencies: [checkout]
            template: build-training
            arguments:
              parameters:
                - name: cuda-compute-cap
                  value: "{{inputs.parameters.cuda-compute-cap}}"
          - name: upload
            dependencies: [build]
            template: upload-binaries

    - name: git-checkout
      nodeSelector:
        k8s.scaleway.com/pool-name: ci-compile-cpu
      tolerations:
        - key: node.cilium.io/agent-not-ready
          operator: Exists
          effect: NoSchedule
      inputs:
        parameters:
          - name: commit-sha
      container:
        image: rg.fr-par.scw.cloud/foxhunt-ci/ci-builder:latest
        command: ["/bin/sh", "-c"]
        resources:
          requests:
            cpu: 500m
            memory: 512Mi
          limits:
            cpu: "2"
            memory: 2Gi
        volumeMounts:
          - name: workspace
            mountPath: /workspace
          - name: git-ssh-key
            mountPath: /etc/git-ssh
            readOnly: true
        args:
          - |
            set -e
            mkdir -p ~/.ssh
            cp /etc/git-ssh/ssh-privatekey ~/.ssh/id_ed25519
            chmod 600 ~/.ssh/id_ed25519
            printf 'Host *\n  StrictHostKeyChecking no\n  UserKnownHostsFile /dev/null\n' > ~/.ssh/config
            chmod 600 ~/.ssh/config

            REPO="ssh://git@gitlab-gitlab-shell.foxhunt.svc.cluster.local:2222/root/foxhunt.git"
            SHA="{{inputs.parameters.commit-sha}}"
            echo "Cloning foxhunt at ${SHA}..."
            if [ "$SHA" = "HEAD" ]; then
              git clone --depth=1 "$REPO" /workspace/src
            else
              git clone "$REPO" /workspace/src
              cd /workspace/src
              git checkout "$SHA"
            fi
            cd /workspace/src
            echo "Checked out $(git rev-parse --short HEAD)"

    - name: build-training
      nodeSelector:
        k8s.scaleway.com/pool-name: ci-compile-cpu
      tolerations:
        - key: node.cilium.io/agent-not-ready
          operator: Exists
          effect: NoSchedule
      inputs:
        parameters:
          - name: cuda-compute-cap
      container:
        image: rg.fr-par.scw.cloud/foxhunt-ci/ci-builder:latest
        command: ["/bin/sh", "-c"]
        env:
          - name: SQLX_OFFLINE
            value: "true"
          - name: CARGO_TERM_COLOR
            value: always
        resources:
          requests:
            cpu: "14"
            memory: 32Gi
          limits:
            cpu: "30"
            memory: 64Gi
        volumeMounts:
          - name: workspace
            mountPath: /workspace
          - name: sccache
            mountPath: /mnt/sccache
        args:
          - |
            set -e
            cd /workspace/src

            export CUDA_COMPUTE_CAP={{inputs.parameters.cuda-compute-cap}}
            export SCCACHE_DIR="/mnt/sccache/sm_${CUDA_COMPUTE_CAP}"
            export RUSTC_WRAPPER=sccache
            mkdir -p "$SCCACHE_DIR"
            sccache --zero-stats || true

            YEAR=$(date +%Y)
            MONTH=$(date +%m)
            export FOXHUNT_BUILD_VERSION="${YEAR}.${MONTH}.argo"

            echo "=== Compiling training binaries (CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP) ==="
            cargo build --release -p ml --features ml/cuda \
              --example train_baseline_rl \
              --example train_baseline_supervised \
              --example evaluate_baseline \
              --example evaluate_supervised \
              --example hyperopt_baseline_rl \
              --example hyperopt_baseline_supervised

            cargo build --release -p training_uploader

            sccache --show-stats || true

            echo "=== Copying and stripping binaries ==="
            mkdir -p /workspace/bin
            for bin in train_baseline_rl train_baseline_supervised evaluate_baseline evaluate_supervised hyperopt_baseline_rl hyperopt_baseline_supervised; do
              cp target/release/examples/$bin /workspace/bin/
              strip /workspace/bin/$bin
            done
            cp target/release/training_uploader /workspace/bin/
            strip /workspace/bin/training_uploader

            echo "=== Build complete ==="
            ls -lh /workspace/bin/

    - name: upload-binaries
      nodeSelector:
        k8s.scaleway.com/pool-name: platform
      tolerations:
        - key: gitlab
          operator: Equal
          value: "true"
          effect: NoSchedule
      container:
        image: rg.fr-par.scw.cloud/foxhunt-ci/foxhunt-runtime:latest
        command: ["/bin/sh", "-c"]
        env:
          - name: MINIO_ACCESS_KEY
            valueFrom:
              secretKeyRef:
                name: minio-credentials
                key: access-key
          - name: MINIO_SECRET_KEY
            valueFrom:
              secretKeyRef:
                name: minio-credentials
                key: secret-key
        resources:
          requests:
            cpu: 100m
            memory: 128Mi
          limits:
            cpu: 500m
            memory: 256Mi
        volumeMounts:
          - name: workspace
            mountPath: /workspace
        args:
          - |
            set -e
            echo "Uploading training binaries to MinIO..."
            rclone copy /workspace/bin/ :s3:foxhunt-binaries/training/ \
              --s3-provider=Minio \
              --s3-endpoint=https://minio.foxhunt.svc.cluster.local:9000 \
              --s3-access-key-id="${MINIO_ACCESS_KEY}" \
              --s3-secret-access-key="${MINIO_SECRET_KEY}" \
              --s3-no-check-bucket \
              --no-check-certificate

            echo "=== Uploaded binaries ==="
            rclone ls :s3:foxhunt-binaries/training/ \
              --s3-provider=Minio \
              --s3-endpoint=https://minio.foxhunt.svc.cluster.local:9000 \
              --s3-access-key-id="${MINIO_ACCESS_KEY}" \
              --s3-secret-access-key="${MINIO_SECRET_KEY}" \
              --s3-no-check-bucket \
              --no-check-certificate