diff --git a/infra/k8s/jobs/fxhnt-opra-backfill.yaml b/infra/k8s/jobs/fxhnt-opra-backfill.yaml deleted file mode 100644 index 110a134..0000000 --- a/infra/k8s/jobs/fxhnt-opra-backfill.yaml +++ /dev/null @@ -1,99 +0,0 @@ -# ============================================================================================ -# ONE-TIME MANUAL APPLY ONLY. Do NOT add this file to any automated/bulk `kubectl apply -f infra/k8s/jobs/` -# path or CI pipeline. It is a `kind: Job` (not a CronJob), so there is no `suspend` flag to gate it — -# submitting this manifest starts the backfill immediately. Apply it by hand when ready: -# kubectl apply -f infra/k8s/jobs/fxhnt-opra-backfill.yaml -# and delete it (`kubectl delete job/fxhnt-opra-backfill -n foxhunt`) before ever re-applying. -# ============================================================================================ -# -# One-time historical freeze: `fxhnt backfill-xsp-opra --start 2013-04-01 --max-cost 25.0`, from 2013-04-01 -# (XSP's mini-SPX option listing era) through today, freezing the WIDE DTE[1,45]/moneyness[0.70,1.20] put -# slice + near-ATM calls into `xsp_option_bars`. BATCHED per MONTH (`_backfill_month`): one range `definition` -# + one range `ohlcv-1d` fetch per month, then every trading day processed LOCALLY — ~2 OPRA calls/month vs -# the old ~6-8/day (~1h vs ~45h over 2013->). `--max-cost 25.0` is the per-MONTH cost cap (the guard stays -# ACTIVE, never disabled; a month over budget is logged + skipped, never a silent partial-freeze). -# -# Idempotent/resumable: `upsert_bars` merges on (osi_symbol, date), so RE-APPLYING this exact manifest after -# an interruption is safe (already-frozen days are silently re-written, not duplicated) — it just re-spends -# the OPRA cost on the days already done. To resume WITHOUT re-paying for completed months, read the last -# "reached YYYY-MM" line from `kubectl logs`, delete the Job, and re-apply with `--start` bumped to that -# month before resubmitting. -apiVersion: batch/v1 -kind: Job -metadata: - name: fxhnt-opra-backfill - namespace: foxhunt - labels: { app.kubernetes.io/name: fxhnt-opra-backfill, app.kubernetes.io/part-of: foxhunt } -spec: - backoffLimit: 2 - ttlSecondsAfterFinished: 604800 # 7d — keep the finished Job around for log inspection before GC - template: - metadata: - labels: - app.kubernetes.io/name: fxhnt-opra-backfill # egress NetworkPolicy selects this - app.kubernetes.io/part-of: foxhunt # gitea ingress (git-sync) + base egress - spec: - restartPolicy: Never - imagePullSecrets: - - name: scw-registry - initContainers: - - name: git-sync - image: alpine/git:latest - command: [/bin/sh, -c] - args: - - | - mkdir -p /root/.ssh && cp /etc/git-ssh/ssh-privatekey /root/.ssh/id_ed25519 && chmod 600 /root/.ssh/id_ed25519 - printf 'Host *\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n' > /root/.ssh/config - rm -rf /code/src - git clone --depth 1 --branch master --filter=blob:none --sparse \ - "ssh://git@gitea-ssh.gitea.svc.cluster.local:22/gitadmin/fxhnt.git" /tmp/repo - cd /tmp/repo && git sparse-checkout set src && [ -d /tmp/repo/src ] - cp -a /tmp/repo/src /code/src && echo "git-sync OK: $(git rev-parse --short HEAD)" - resources: { requests: { cpu: 50m, memory: 64Mi }, limits: { cpu: 500m, memory: 256Mi } } - volumeMounts: - - { name: code, mountPath: /code } - - { name: git-ssh-key, mountPath: /etc/git-ssh, readOnly: true } - containers: - - name: backfill - image: rg.fr-par.scw.cloud/bizworx/fxhnt-cockpit:latest - command: ["/bin/sh", "-c"] - args: - - | - pip install --quiet --no-cache-dir databento==0.79.0 - fxhnt backfill-xsp-opra --start 2013-04-01 --max-cost 25.0 - env: - - { name: PYTHONPATH, value: /code/src } - - { name: FXHNT_OPERATIONAL_DSN, value: "postgresql+psycopg://foxhunt@postgres.foxhunt.svc.cluster.local:5432/fxhnt" } - - name: PGPASSWORD - valueFrom: { secretKeyRef: { name: db-credentials, key: password } } - - name: DATABENTO_API_KEY - valueFrom: { secretKeyRef: { name: databento-credentials, key: api-key } } - - { name: PYTHONUNBUFFERED, value: "1" } # stream monthly progress logs live - resources: - requests: { memory: "512Mi", cpu: "250m" } - limits: { memory: "1Gi", cpu: "1" } - volumeMounts: - - { name: code, mountPath: /code, readOnly: true } - volumes: - - { name: code, emptyDir: {} } - - { name: git-ssh-key, secret: { secretName: argo-git-ssh-key, defaultMode: 256 } } ---- -# Egress: DNS + Postgres (5432, write xsp_option_bars) + gitea-sshd (22, git-sync) + external 443 (OPRA via -# Databento historical API). No IBKR/ib-gateway needed — this Job never places an order. -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: fxhnt-opra-backfill - namespace: foxhunt - labels: { app.kubernetes.io/part-of: foxhunt } -spec: - podSelector: { matchLabels: { app.kubernetes.io/name: fxhnt-opra-backfill } } - policyTypes: [Egress] - egress: - - ports: [{ port: 53, protocol: UDP }, { port: 53, protocol: TCP }] - - to: [{ podSelector: { matchLabels: { app.kubernetes.io/name: postgres } } }] - ports: [{ port: 5432, protocol: TCP }] - - to: [{ namespaceSelector: { matchLabels: { kubernetes.io/metadata.name: gitea } }, podSelector: { matchLabels: { app.kubernetes.io/name: gitea } } }] - ports: [{ port: 2222, protocol: TCP }, { port: 22, protocol: TCP }] - - to: [{ ipBlock: { cidr: 0.0.0.0/0, except: [10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16] } }] # OPRA 443 - ports: [{ port: 443, protocol: TCP }]