# Databento data download via streaming API (get_range).
#
# Parameterized Argo WorkflowTemplate — replaces the hardcoded K8s Job.
# Binary pre-uploaded to /data/bin/download_baseline on training-data-pvc.
# Universe configs loaded from configmap databento-universe-all.
#
# Usage:
#   argo submit -n foxhunt --from=wftmpl/databento-download
#   argo submit -n foxhunt --from=wftmpl/databento-download -p schema=trades -p output-dir=/data/futures-baseline-trades
#   ./scripts/argo-download.sh mbp-10 --parallel 9 --pool ci-compile-cpu
#
# Databento rate limits: 100 concurrent connections per IP, 100 req/s.
---
apiVersion: argoproj.io/v1alpha1
kind: WorkflowTemplate
metadata:
  name: databento-download
  namespace: foxhunt
  labels:
    app.kubernetes.io/name: databento-download
    app.kubernetes.io/part-of: foxhunt
spec:
  entrypoint: download
  serviceAccountName: argo-workflow
  podMetadata:
    labels:
      app.kubernetes.io/part-of: foxhunt
      app.kubernetes.io/component: databento-download
  securityContext:
    fsGroup: 1000
  ttlStrategy:
    secondsAfterCompletion: 3600
  activeDeadlineSeconds: 86400  # 24 hours

  arguments:
    parameters:
      - name: schema
        value: mbp-10
        description: "Databento schema: ohlcv-1m, ohlcv-1s, trades, mbp-10"
      - name: output-dir
        value: /data/futures-baseline-mbp10
        description: "Output directory on training-data PVC"
      - name: parallel
        value: "9"
        description: "Number of parallel quarterly downloads (max ~100 per Databento rate limit)"
      - name: node-pool
        value: ci-compile-cpu
        description: "K8s node pool: ci-compile-cpu (32c/64G) or platform (4c/8G)"

  volumes:
    - name: training-data
      persistentVolumeClaim:
        claimName: training-data-pvc
    - name: universe-configs
      configMap:
        name: databento-universe-all

  templates:
    - name: download
      inputs:
        parameters:
          - name: schema
          - name: output-dir
          - name: parallel
          - name: node-pool
      nodeSelector:
        k8s.scaleway.com/pool-name: "{{inputs.parameters.node-pool}}"
        topology.kubernetes.io/zone: fr-par-2
      container:
        image: ubuntu:24.04
        command: ["/bin/bash", "-c"]
        env:
          - name: RUST_LOG
            value: info
          - name: DATABENTO_API_KEY
            valueFrom:
              secretKeyRef:
                name: databento-credentials
                key: api-key
        resources:
          requests:
            cpu: 500m
            memory: 512Mi
          limits:
            cpu: "2"
            memory: 4Gi
        volumeMounts:
          - name: training-data
            mountPath: /data
          - name: universe-configs
            mountPath: /configs
        args:
          - |
            set -e
            apt-get update -qq > /dev/null 2>&1 && apt-get install -y -qq ca-certificates > /dev/null 2>&1

            BINARY=/data/bin/download_baseline
            chmod +x "$BINARY"

            SCHEMA="{{inputs.parameters.schema}}"
            OUTPUT_DIR="{{inputs.parameters.output-dir}}"
            PARALLEL="{{inputs.parameters.parallel}}"

            # Map schema name to configmap key
            CONFIG="/configs/${SCHEMA}.toml"
            if [ ! -f "$CONFIG" ]; then
              echo "ERROR: Config not found: $CONFIG"
              echo "Available configs:"
              ls /configs/
              exit 1
            fi

            mkdir -p "$OUTPUT_DIR"

            echo "=== Databento Download ==="
            echo "Schema:    $SCHEMA"
            echo "Config:    $CONFIG"
            echo "Output:    $OUTPUT_DIR"
            echo "Parallel:  $PARALLEL"
            echo "Node:      $(hostname), CPUs: $(nproc), RAM: $(free -g | awk '/Mem:/{print $2}')G"
            echo ""

            $BINARY \
              --universe-config "$CONFIG" \
              --output-dir "$OUTPUT_DIR" \
              --parallel "$PARALLEL" \
              --yes

            echo ""
            echo "=== Download complete ==="
            echo "Files:"
            find "$OUTPUT_DIR" -name '*.dbn.zst' -exec ls -lh {} \;
            echo ""
            du -sh "$OUTPUT_DIR"
            df -h /data
