# Download MBP-10 order book data for ES.FUT from Databento → MinIO.
#
# Prerequisites:
#   1. Compile + upload the binary:
#        cargo build -p ml --example download_baseline --release
#        rclone copyto target/release/examples/download_baseline \
#          :s3:foxhunt-binaries/tools/download_baseline \
#          --s3-provider=Minio --s3-endpoint=<MINIO_ENDPOINT> \
#          --s3-access-key-id=<KEY> --s3-secret-access-key=<SECRET> \
#          --s3-no-check-bucket
#
#   2. Create Databento API key secret:
#        kubectl -n foxhunt create secret generic databento-credentials \
#          --from-literal=api-key=$DATABENTO_API_KEY
#
#   3. Apply:
#        kubectl apply -f infra/k8s/training/download-mbp10-job.yaml
#
# The job downloads ES.FUT MBP-10 data (730 days, quarterly chunks) and
# uploads each file to s3:foxhunt-training-data/futures-baseline-mbp10/.
apiVersion: batch/v1
kind: Job
metadata:
  name: download-mbp10
  namespace: foxhunt
  labels:
    app.kubernetes.io/name: download-mbp10
    app.kubernetes.io/part-of: foxhunt
    foxhunt/job-type: data-download
spec:
  backoffLimit: 2
  activeDeadlineSeconds: 14400  # 4 hours
  ttlSecondsAfterFinished: 3600
  template:
    metadata:
      labels:
        app.kubernetes.io/name: download-mbp10
        app.kubernetes.io/part-of: foxhunt
        foxhunt/job-type: data-download
    spec:
      nodeSelector:
        k8s.scaleway.com/pool-name: platform
      imagePullSecrets:
        - name: gitlab-registry
      restartPolicy: Never
      initContainers:
        - name: fetch-binary
          image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest
          command: ["/bin/sh", "-c"]
          args:
            - |
              set -e
              rclone copyto \
                :s3:foxhunt-binaries/tools/download_baseline \
                /binaries/download_baseline \
                --s3-provider=Minio \
                --s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 \
                --s3-access-key-id="${MINIO_ACCESS_KEY}" \
                --s3-secret-access-key="${MINIO_SECRET_KEY}" \
                --s3-no-check-bucket
              chmod +x /binaries/download_baseline
              echo "Fetched download_baseline ($(stat -c%s /binaries/download_baseline) bytes)"
          env:
            - name: MINIO_ACCESS_KEY
              valueFrom:
                secretKeyRef:
                  name: minio-credentials
                  key: access-key
            - name: MINIO_SECRET_KEY
              valueFrom:
                secretKeyRef:
                  name: minio-credentials
                  key: secret-key
          volumeMounts:
            - name: binaries
              mountPath: /binaries
          resources:
            requests:
              cpu: 100m
              memory: 64Mi
            limits:
              cpu: 500m
              memory: 128Mi
      containers:
        - name: downloader
          image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest
          command: ["/bin/sh", "-c"]
          args:
            - |
              set -e

              # Write universe config inline (avoids ConfigMap for a one-off job)
              mkdir -p /tmp/config
              cat > /tmp/config/universe-es-mbp10.toml <<'TOML'
              [universe]
              name = "es-mbp10"
              description = "ES.FUT MBP-10 order book data for OFI features"
              date_range_start = "2024-03-01"
              date_range_end = "2026-02-22"
              bar_size = "tick"
              databento_dataset = "GLBX.MDP3"
              databento_schema = "mbp-10"

              [[symbols]]
              symbol = "ES.FUT"
              exchange = "GLBX.MDP3"
              asset_class = "Future"
              TOML

              echo "=== ES.FUT MBP-10 Download Job ==="
              echo "Downloading from Databento and uploading to MinIO..."

              /binaries/download_baseline \
                --universe-config /tmp/config/universe-es-mbp10.toml \
                --output-dir /tmp/mbp10-download \
                --rclone-dest "s3:foxhunt-training-data/futures-baseline-mbp10" \
                --parallel 3 \
                --yes

              echo "=== Download complete ==="
              echo "Files in MinIO: foxhunt-training-data/futures-baseline-mbp10/ES.FUT/"
          env:
            - name: DATABENTO_API_KEY
              valueFrom:
                secretKeyRef:
                  name: databento-credentials
                  key: api-key
            # rclone env-var config (used by download_baseline --rclone-dest)
            - name: RCLONE_S3_PROVIDER
              value: Minio
            - name: RCLONE_S3_ENDPOINT
              value: http://minio.foxhunt.svc.cluster.local:9000
            - name: RCLONE_S3_ACCESS_KEY_ID
              valueFrom:
                secretKeyRef:
                  name: minio-credentials
                  key: access-key
            - name: RCLONE_S3_SECRET_ACCESS_KEY
              valueFrom:
                secretKeyRef:
                  name: minio-credentials
                  key: secret-key
            - name: RCLONE_S3_NO_CHECK_BUCKET
              value: "true"
          volumeMounts:
            - name: binaries
              mountPath: /binaries
              readOnly: true
            - name: scratch
              mountPath: /tmp
          resources:
            requests:
              cpu: "1"
              memory: 1Gi
            limits:
              cpu: "2"
              memory: 4Gi
      volumes:
        - name: binaries
          emptyDir:
            sizeLimit: 100Mi
        - name: scratch
          emptyDir:
            sizeLimit: 100Gi  # MBP-10: ~5GB/quarter × 9 quarters + headroom
