gitadmin/fxhnt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

deploy(orchestration): Dagster manifest (single-pod webserver+daemon+tailscale, pg storage) + Argo deploy + Dockerfile extra

Browse Source 
Task 10 repo prep: one-pod Dagster (RWO surfer PVC shared by webserver+daemon), tailnet-private UI,
storage in cockpit Postgres `dagster` DB, FXHNT_COMBINED_BOOK_DATA_SOURCE=warehouse. Argo build-deploy
extended to apply+roll dagster. Cron suspend = deliberate post-verify cutover step (RWO-forced).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-06-15 12:47:38 +02:00
parent 8efb78ab15
commit bf2029868b
3 changed files with 168 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
infra/argo/cockpit-build-deploy.yaml
View File

@@ -111,8 +111,12 @@ spec:
cd /workspace/src
kubectl apply -f infra/k8s/services/fxhnt-dashboard.yaml
kubectl apply -f infra/k8s/jobs/fxhnt-forward-cronjob.yaml
kubectl apply -f infra/k8s/orchestration/dagster.yaml
kubectl apply -f infra/k8s/network-policies/fxhnt-cockpit.yaml
kubectl -n foxhunt rollout restart deploy/fxhnt-dashboard
kubectl -n foxhunt rollout restart deploy/dagster
kubectl -n foxhunt rollout status deploy/fxhnt-dashboard --timeout=180s
kubectl -n foxhunt rollout status deploy/dagster --timeout=300s
volumeMounts:
- { name: workspace, mountPath: /workspace }
volumes:

2
infra/docker/fxhnt.Dockerfile
View File

@@ -5,7 +5,7 @@ ENV PYTHONUNBUFFERED=1 PIP_NO_CACHE_DIR=1
WORKDIR /app
COPY pyproject.toml README.md ./
COPY src/ ./src/
RUN pip install --upgrade pip && pip install '.[web,pg,data,factory]'
RUN pip install --upgrade pip && pip install '.[web,pg,data,factory,orchestration]'
# DSN is injected at runtime via secretKeyRef in the K8s manifests; empty here so the image contains no credentials.
ENV FXHNT_OPERATIONAL_DSN=""
EXPOSE 8080

163
infra/k8s/orchestration/dagster.yaml Normal file
View File

@@ -0,0 +1,163 @@
# Dagster B0 — the lifecycle orchestrator (combined-book asset graph) on the cluster.
#
# ONE pod, three containers (webserver + daemon + tailscale userspace sidecar) sharing the RWO
# `fxhnt-surfer-data` PVC (warehouse.duckdb + forward state live there). RWO = single-node, so webserver
# and daemon MUST co-locate; the daemon runs the daily schedule, runs execute in-process (DefaultRunLauncher).
# Storage (runs/events/schedules) is the existing cockpit Postgres, separate `dagster` DB. UI is tailnet-
# private via `tailscale serve` (CGNAT-safe userspace mode, mirrors fxhnt-dashboard) → fxhnt-dagster.<tailnet>.ts.net
#
# Cutover (RWO-forced): bringing this up REPLACES the fxhnt-forward CronJob — suspend that cron at go-live.
# Revert = `kubectl -n foxhunt scale deploy/dagster --replicas=0` (releases the PVC) + un-suspend the cron.
# Data-path revert (independent) = set FXHNT_COMBINED_BOOK_DATA_SOURCE=raw below.
---
apiVersion: v1
kind: ConfigMap
metadata:
name: dagster-instance
namespace: foxhunt
data:
dagster.yaml: |
storage:
postgres:
postgres_db:
username: foxhunt
password:
env: PGPASSWORD
hostname: postgres.foxhunt.svc.cluster.local
db_name: dagster
port: 5432
telemetry:
enabled: false
workspace.yaml: |
load_from:
- python_module:
module_name: fxhnt.adapters.orchestration.definitions
working_directory: /app
---
apiVersion: v1
kind: ConfigMap
metadata:
name: fxhnt-dagster-ts-serve
namespace: foxhunt
data:
serve.json: |
{
"TCP": { "443": { "HTTPS": true } },
"Web": {
"${TS_CERT_DOMAIN}:443": {
"Handlers": { "/": { "Proxy": "http://127.0.0.1:3000" } }
}
}
}
---
apiVersion: v1
kind: ServiceAccount
metadata: { name: tailscale-dagster, namespace: foxhunt }
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata: { name: tailscale-dagster, namespace: foxhunt }
rules:
- apiGroups: [""]
resources: ["secrets"]
verbs: ["create", "get", "update", "patch", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata: { name: tailscale-dagster, namespace: foxhunt }
subjects:
- { kind: ServiceAccount, name: tailscale-dagster, namespace: foxhunt }
roleRef: { kind: Role, name: tailscale-dagster, apiGroup: rbac.authorization.k8s.io }
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: dagster
namespace: foxhunt
labels: { app.kubernetes.io/name: dagster, app.kubernetes.io/part-of: foxhunt }
spec:
replicas: 1
strategy: { type: Recreate } # RWO PVC: never run two pods holding it at once
selector: { matchLabels: { app.kubernetes.io/name: dagster } }
template:
metadata:
labels: { app.kubernetes.io/name: dagster, app.kubernetes.io/part-of: foxhunt } # part-of -> postgres ingress + base egress
spec:
serviceAccountName: tailscale-dagster
nodeSelector: { k8s.scaleway.com/pool-name: platform }
imagePullSecrets: [{ name: gitlab-registry }]
initContainers:
- name: dagster-home-init # DAGSTER_HOME must be writable; seed it from the configmap
image: busybox:1.36
command: ["sh", "-c", "cp /cfg/dagster.yaml /cfg/workspace.yaml /dagster-home/"]
volumeMounts:
- { name: dagster-home, mountPath: /dagster-home }
- { name: dagster-cfg, mountPath: /cfg, readOnly: true }
containers:
- name: webserver
image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/fxhnt-cockpit:latest
command: ["dagster-webserver", "-h", "0.0.0.0", "-p", "3000",
"-w", "/dagster-home/workspace.yaml"]
env: &dagster_env
- { name: DAGSTER_HOME, value: /dagster-home }
- { name: FXHNT_COMBINED_BOOK_DATA_SOURCE, value: "warehouse" }
- { name: FXHNT_WAREHOUSE_PATH, value: /data/surfer/warehouse.duckdb }
- { name: FXHNT_SURFER_DATA_DIR, value: /data/surfer }
- { name: FXHNT_OPERATIONAL_DSN, value: "postgresql+psycopg://foxhunt@postgres.foxhunt.svc.cluster.local:5432/fxhnt" }
- { name: FXHNT_DATABENTO_MAX_COST_USD, value: "40.0" }
- name: PGPASSWORD
valueFrom: { secretKeyRef: { name: db-credentials, key: password } }
- name: DATABENTO_API_KEY
valueFrom: { secretKeyRef: { name: databento-credentials, key: api-key } }
readinessProbe: { tcpSocket: { port: 3000 }, initialDelaySeconds: 10, periodSeconds: 15 }
volumeMounts:
- { name: dagster-home, mountPath: /dagster-home }
- { name: surfer-data, mountPath: /data }
resources: { requests: { cpu: 100m, memory: 512Mi }, limits: { cpu: "1", memory: 1Gi } }
- name: daemon
image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/fxhnt-cockpit:latest
command: ["dagster-daemon", "run", "-w", "/dagster-home/workspace.yaml"]
env: *dagster_env
volumeMounts:
- { name: dagster-home, mountPath: /dagster-home }
- { name: surfer-data, mountPath: /data }
resources: { requests: { cpu: 100m, memory: 512Mi }, limits: { cpu: "2", memory: 2Gi } }
- name: tailscale
image: ghcr.io/tailscale/tailscale:latest
env:
- name: TS_AUTHKEY
valueFrom: { secretKeyRef: { name: tailscale-auth, key: TS_AUTHKEY } }
- { name: TS_KUBE_SECRET, value: fxhnt-dagster-ts-state }
- { name: TS_USERSPACE, value: "true" }
- { name: TS_HOSTNAME, value: "fxhnt-dagster" }
- { name: TS_SERVE_CONFIG, value: /etc/ts/serve.json }
volumeMounts:
- { name: ts-serve, mountPath: /etc/ts, readOnly: true }
resources: { requests: { cpu: 25m, memory: 64Mi }, limits: { cpu: 200m, memory: 128Mi } }
volumes:
- { name: dagster-home, emptyDir: {} }
- { name: dagster-cfg, configMap: { name: dagster-instance } }
- { name: ts-serve, configMap: { name: fxhnt-dagster-ts-serve } }
- { name: surfer-data, persistentVolumeClaim: { claimName: fxhnt-surfer-data } }
---
apiVersion: v1
kind: Service
metadata: { name: dagster, namespace: foxhunt }
spec:
selector: { app.kubernetes.io/name: dagster }
ports: [{ port: 80, targetPort: 3000, name: http }]
---
# Egress: postgres (storage + cockpit ingest) + DNS + public HTTPS (Binance/Databento fetch in the assets)
# + tailscale coordination/DERP. part-of=foxhunt already grants base egress + satisfies postgres ingress.
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata: { name: dagster, namespace: foxhunt, labels: { app.kubernetes.io/part-of: foxhunt } }
spec:
podSelector: { matchLabels: { app.kubernetes.io/name: dagster } }
policyTypes: [Egress]
egress:
- to: [{ podSelector: { matchLabels: { app.kubernetes.io/name: postgres } } }]
ports: [{ port: 5432, protocol: TCP }]
- ports: [{ port: 53, protocol: UDP }, { port: 53, protocol: TCP }]
- to: [{ ipBlock: { cidr: 0.0.0.0/0, except: [10.32.0.0/16, 172.16.0.0/16] } }] # Binance/Databento + tailscale
ports: [{ port: 443, protocol: TCP }, { port: 3478, protocol: UDP }, { port: 41641, protocol: UDP }]