# Download ES.FUT trade data (Schema::Trades) from Databento → PVC.
#
# Writes directly to the training-data PVC, then backs up to MinIO.
# Trade data is used by VPIN and Kyle's Lambda OFI features.
#
# Prerequisites:
#   1. Compile + upload the binary (same as download-mbp10-job):
#        cargo build -p ml --example download_baseline --release
#        rclone copyto target/release/examples/download_baseline \
#          :s3:foxhunt-binaries/tools/download_baseline \
#          --s3-provider=Minio --s3-endpoint=<MINIO_ENDPOINT> \
#          --s3-access-key-id=<KEY> --s3-secret-access-key=<SECRET> \
#          --s3-no-check-bucket
#
#   2. Create Databento API key secret (if not already done):
#        kubectl -n foxhunt create secret generic databento-credentials \
#          --from-literal=api-key=$DATABENTO_API_KEY
#
#   3. Apply:
#        kubectl apply -f infra/k8s/training/download-trades-job.yaml
#
# NOTE: training-data-pvc is RWO — ensure no training job is running.
apiVersion: batch/v1
kind: Job
metadata:
  name: download-trades
  namespace: foxhunt
  labels:
    app.kubernetes.io/name: download-trades
    app.kubernetes.io/part-of: foxhunt
    foxhunt/job-type: data-download
spec:
  backoffLimit: 2
  activeDeadlineSeconds: 14400  # 4 hours
  ttlSecondsAfterFinished: 3600
  template:
    metadata:
      labels:
        app.kubernetes.io/name: download-trades
        app.kubernetes.io/part-of: foxhunt
        foxhunt/job-type: data-download
    spec:
      nodeSelector:
        k8s.scaleway.com/pool-name: ci-compile-cpu
      tolerations:
        - key: node.cilium.io/agent-not-ready
          operator: Exists
          effect: NoSchedule
      imagePullSecrets:
        - name: gitlab-registry
      restartPolicy: Never
      initContainers:
        - name: fetch-binary
          image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest
          command: ["/bin/sh", "-c"]
          args:
            - |
              set -e
              rclone copyto \
                :s3:foxhunt-binaries/tools/download_baseline \
                /binaries/download_baseline \
                --s3-provider=Minio \
                --s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 \
                --s3-access-key-id="${MINIO_ACCESS_KEY}" \
                --s3-secret-access-key="${MINIO_SECRET_KEY}" \
                --s3-no-check-bucket
              chmod +x /binaries/download_baseline
              echo "Fetched download_baseline ($(stat -c%s /binaries/download_baseline) bytes)"
          env:
            - name: MINIO_ACCESS_KEY
              valueFrom:
                secretKeyRef:
                  name: minio-credentials
                  key: access-key
            - name: MINIO_SECRET_KEY
              valueFrom:
                secretKeyRef:
                  name: minio-credentials
                  key: secret-key
          volumeMounts:
            - name: binaries
              mountPath: /binaries
          resources:
            requests:
              cpu: 100m
              memory: 64Mi
            limits:
              cpu: 500m
              memory: 128Mi
      containers:
        - name: downloader
          image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest
          command: ["/bin/sh", "-c"]
          args:
            - |
              set -e

              # Write universe config for trades schema
              mkdir -p /tmp/config
              cat > /tmp/config/universe-es-trades.toml <<'TOML'
              [universe]
              name = "es-trades"
              date_range_start = "2024-03-01"
              date_range_end = "2026-02-22"
              databento_dataset = "GLBX.MDP3"
              databento_schema = "trades"

              [[symbols]]
              symbol = "ES.FUT"
              exchange = "GLBX.MDP3"
              asset_class = "Future"
              TOML

              echo "=== ES.FUT Trades Download Job ==="
              echo "Downloading from Databento directly to PVC..."

              /binaries/download_baseline \
                --universe-config /tmp/config/universe-es-trades.toml \
                --output-dir /data/futures-baseline-trades \
                --parallel 6 \
                --yes

              echo ""
              echo "=== Download complete ==="
              du -sh /data/futures-baseline-trades/
              ls -lhS /data/futures-baseline-trades/ES.FUT/

              # Backup to MinIO
              RCLONE_FLAGS="--s3-provider=Minio --s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 --s3-access-key-id=${MINIO_ACCESS_KEY} --s3-secret-access-key=${MINIO_SECRET_KEY} --s3-no-check-bucket"
              echo ""
              echo "=== Backing up to MinIO ==="
              rclone sync \
                /data/futures-baseline-trades \
                ":s3:foxhunt-training-data/futures-baseline-trades" \
                $RCLONE_FLAGS --stats-one-line -v
              echo "MinIO backup complete."
          env:
            - name: DATABENTO_API_KEY
              valueFrom:
                secretKeyRef:
                  name: databento-credentials
                  key: api-key
            - name: MINIO_ACCESS_KEY
              valueFrom:
                secretKeyRef:
                  name: minio-credentials
                  key: access-key
            - name: MINIO_SECRET_KEY
              valueFrom:
                secretKeyRef:
                  name: minio-credentials
                  key: secret-key
          volumeMounts:
            - name: binaries
              mountPath: /binaries
              readOnly: true
            - name: training-data
              mountPath: /data
            - name: scratch
              mountPath: /tmp
          resources:
            requests:
              cpu: "2"
              memory: 4Gi
            limits:
              cpu: "4"
              memory: 16Gi
      volumes:
        - name: binaries
          emptyDir:
            sizeLimit: 100Mi
        - name: training-data
          persistentVolumeClaim:
            claimName: training-data-pvc
        - name: scratch
          emptyDir:
            sizeLimit: 1Gi
