100 lines
5.5 KiB
YAML
100 lines
5.5 KiB
YAML
# ============================================================================================
|
|
# ONE-TIME MANUAL APPLY ONLY. Do NOT add this file to any automated/bulk `kubectl apply -f infra/k8s/jobs/`
|
|
# path or CI pipeline. It is a `kind: Job` (not a CronJob), so there is no `suspend` flag to gate it —
|
|
# submitting this manifest starts the backfill immediately. Apply it by hand when ready:
|
|
# kubectl apply -f infra/k8s/jobs/fxhnt-opra-backfill.yaml
|
|
# and delete it (`kubectl delete job/fxhnt-opra-backfill -n foxhunt`) before ever re-applying.
|
|
# ============================================================================================
|
|
#
|
|
# One-time historical freeze: `fxhnt backfill-xsp-opra --start 2013-04-01 --max-cost 25.0`, from 2013-04-01
|
|
# (XSP's mini-SPX option listing era) through today, freezing the WIDE DTE[1,45]/moneyness[0.70,1.20] put
|
|
# slice + near-ATM calls into `xsp_option_bars`. BATCHED per MONTH (`_backfill_month`): one range `definition`
|
|
# + one range `ohlcv-1d` fetch per month, then every trading day processed LOCALLY — ~2 OPRA calls/month vs
|
|
# the old ~6-8/day (~1h vs ~45h over 2013->). `--max-cost 25.0` is the per-MONTH cost cap (the guard stays
|
|
# ACTIVE, never disabled; a month over budget is logged + skipped, never a silent partial-freeze).
|
|
#
|
|
# Idempotent/resumable: `upsert_bars` merges on (osi_symbol, date), so RE-APPLYING this exact manifest after
|
|
# an interruption is safe (already-frozen days are silently re-written, not duplicated) — it just re-spends
|
|
# the OPRA cost on the days already done. To resume WITHOUT re-paying for completed months, read the last
|
|
# "reached YYYY-MM" line from `kubectl logs`, delete the Job, and re-apply with `--start` bumped to that
|
|
# month before resubmitting.
|
|
apiVersion: batch/v1
|
|
kind: Job
|
|
metadata:
|
|
name: fxhnt-opra-backfill
|
|
namespace: foxhunt
|
|
labels: { app.kubernetes.io/name: fxhnt-opra-backfill, app.kubernetes.io/part-of: foxhunt }
|
|
spec:
|
|
backoffLimit: 2
|
|
ttlSecondsAfterFinished: 604800 # 7d — keep the finished Job around for log inspection before GC
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app.kubernetes.io/name: fxhnt-opra-backfill # egress NetworkPolicy selects this
|
|
app.kubernetes.io/part-of: foxhunt # gitea ingress (git-sync) + base egress
|
|
spec:
|
|
restartPolicy: Never
|
|
imagePullSecrets:
|
|
- name: scw-registry
|
|
initContainers:
|
|
- name: git-sync
|
|
image: alpine/git:latest
|
|
command: [/bin/sh, -c]
|
|
args:
|
|
- |
|
|
mkdir -p /root/.ssh && cp /etc/git-ssh/ssh-privatekey /root/.ssh/id_ed25519 && chmod 600 /root/.ssh/id_ed25519
|
|
printf 'Host *\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n' > /root/.ssh/config
|
|
rm -rf /code/src
|
|
git clone --depth 1 --branch master --filter=blob:none --sparse \
|
|
"ssh://git@gitea-ssh.gitea.svc.cluster.local:22/gitadmin/fxhnt.git" /tmp/repo
|
|
cd /tmp/repo && git sparse-checkout set src && [ -d /tmp/repo/src ]
|
|
cp -a /tmp/repo/src /code/src && echo "git-sync OK: $(git rev-parse --short HEAD)"
|
|
resources: { requests: { cpu: 50m, memory: 64Mi }, limits: { cpu: 500m, memory: 256Mi } }
|
|
volumeMounts:
|
|
- { name: code, mountPath: /code }
|
|
- { name: git-ssh-key, mountPath: /etc/git-ssh, readOnly: true }
|
|
containers:
|
|
- name: backfill
|
|
image: rg.fr-par.scw.cloud/bizworx/fxhnt-cockpit:latest
|
|
command: ["/bin/sh", "-c"]
|
|
args:
|
|
- |
|
|
pip install --quiet --no-cache-dir databento==0.79.0
|
|
fxhnt backfill-xsp-opra --start 2013-04-01 --max-cost 25.0
|
|
env:
|
|
- { name: PYTHONPATH, value: /code/src }
|
|
- { name: FXHNT_OPERATIONAL_DSN, value: "postgresql+psycopg://foxhunt@postgres.foxhunt.svc.cluster.local:5432/fxhnt" }
|
|
- name: PGPASSWORD
|
|
valueFrom: { secretKeyRef: { name: db-credentials, key: password } }
|
|
- name: DATABENTO_API_KEY
|
|
valueFrom: { secretKeyRef: { name: databento-credentials, key: api-key } }
|
|
- { name: PYTHONUNBUFFERED, value: "1" } # stream monthly progress logs live
|
|
resources:
|
|
requests: { memory: "512Mi", cpu: "250m" }
|
|
limits: { memory: "1Gi", cpu: "1" }
|
|
volumeMounts:
|
|
- { name: code, mountPath: /code, readOnly: true }
|
|
volumes:
|
|
- { name: code, emptyDir: {} }
|
|
- { name: git-ssh-key, secret: { secretName: argo-git-ssh-key, defaultMode: 256 } }
|
|
---
|
|
# Egress: DNS + Postgres (5432, write xsp_option_bars) + gitea-sshd (22, git-sync) + external 443 (OPRA via
|
|
# Databento historical API). No IBKR/ib-gateway needed — this Job never places an order.
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: NetworkPolicy
|
|
metadata:
|
|
name: fxhnt-opra-backfill
|
|
namespace: foxhunt
|
|
labels: { app.kubernetes.io/part-of: foxhunt }
|
|
spec:
|
|
podSelector: { matchLabels: { app.kubernetes.io/name: fxhnt-opra-backfill } }
|
|
policyTypes: [Egress]
|
|
egress:
|
|
- ports: [{ port: 53, protocol: UDP }, { port: 53, protocol: TCP }]
|
|
- to: [{ podSelector: { matchLabels: { app.kubernetes.io/name: postgres } } }]
|
|
ports: [{ port: 5432, protocol: TCP }]
|
|
- to: [{ namespaceSelector: { matchLabels: { kubernetes.io/metadata.name: gitea } }, podSelector: { matchLabels: { app.kubernetes.io/name: gitea } } }]
|
|
ports: [{ port: 2222, protocol: TCP }, { port: 22, protocol: TCP }]
|
|
- to: [{ ipBlock: { cidr: 0.0.0.0/0, except: [10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16] } }] # OPRA 443
|
|
ports: [{ port: 443, protocol: TCP }]
|