From bf2029868bafbbdc4426b19e4fc937715a66095e Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Mon, 15 Jun 2026 12:47:38 +0200 Subject: [PATCH] deploy(orchestration): Dagster manifest (single-pod webserver+daemon+tailscale, pg storage) + Argo deploy + Dockerfile extra Task 10 repo prep: one-pod Dagster (RWO surfer PVC shared by webserver+daemon), tailnet-private UI, storage in cockpit Postgres `dagster` DB, FXHNT_COMBINED_BOOK_DATA_SOURCE=warehouse. Argo build-deploy extended to apply+roll dagster. Cron suspend = deliberate post-verify cutover step (RWO-forced). Co-Authored-By: Claude Opus 4.8 (1M context) --- infra/argo/cockpit-build-deploy.yaml | 4 + infra/docker/fxhnt.Dockerfile | 2 +- infra/k8s/orchestration/dagster.yaml | 163 +++++++++++++++++++++++++++ 3 files changed, 168 insertions(+), 1 deletion(-) create mode 100644 infra/k8s/orchestration/dagster.yaml diff --git a/infra/argo/cockpit-build-deploy.yaml b/infra/argo/cockpit-build-deploy.yaml index 8516e16..6556914 100644 --- a/infra/argo/cockpit-build-deploy.yaml +++ b/infra/argo/cockpit-build-deploy.yaml @@ -111,8 +111,12 @@ spec: cd /workspace/src kubectl apply -f infra/k8s/services/fxhnt-dashboard.yaml kubectl apply -f infra/k8s/jobs/fxhnt-forward-cronjob.yaml + kubectl apply -f infra/k8s/orchestration/dagster.yaml + kubectl apply -f infra/k8s/network-policies/fxhnt-cockpit.yaml kubectl -n foxhunt rollout restart deploy/fxhnt-dashboard + kubectl -n foxhunt rollout restart deploy/dagster kubectl -n foxhunt rollout status deploy/fxhnt-dashboard --timeout=180s + kubectl -n foxhunt rollout status deploy/dagster --timeout=300s volumeMounts: - { name: workspace, mountPath: /workspace } volumes: diff --git a/infra/docker/fxhnt.Dockerfile b/infra/docker/fxhnt.Dockerfile index cccaf76..7f3cc9d 100644 --- a/infra/docker/fxhnt.Dockerfile +++ b/infra/docker/fxhnt.Dockerfile @@ -5,7 +5,7 @@ ENV PYTHONUNBUFFERED=1 PIP_NO_CACHE_DIR=1 WORKDIR /app COPY pyproject.toml README.md ./ COPY src/ ./src/ -RUN pip install --upgrade pip && pip install '.[web,pg,data,factory]' +RUN pip install --upgrade pip && pip install '.[web,pg,data,factory,orchestration]' # DSN is injected at runtime via secretKeyRef in the K8s manifests; empty here so the image contains no credentials. ENV FXHNT_OPERATIONAL_DSN="" EXPOSE 8080 diff --git a/infra/k8s/orchestration/dagster.yaml b/infra/k8s/orchestration/dagster.yaml new file mode 100644 index 0000000..ccbeecc --- /dev/null +++ b/infra/k8s/orchestration/dagster.yaml @@ -0,0 +1,163 @@ +# Dagster B0 — the lifecycle orchestrator (combined-book asset graph) on the cluster. +# +# ONE pod, three containers (webserver + daemon + tailscale userspace sidecar) sharing the RWO +# `fxhnt-surfer-data` PVC (warehouse.duckdb + forward state live there). RWO = single-node, so webserver +# and daemon MUST co-locate; the daemon runs the daily schedule, runs execute in-process (DefaultRunLauncher). +# Storage (runs/events/schedules) is the existing cockpit Postgres, separate `dagster` DB. UI is tailnet- +# private via `tailscale serve` (CGNAT-safe userspace mode, mirrors fxhnt-dashboard) → fxhnt-dagster..ts.net +# +# Cutover (RWO-forced): bringing this up REPLACES the fxhnt-forward CronJob — suspend that cron at go-live. +# Revert = `kubectl -n foxhunt scale deploy/dagster --replicas=0` (releases the PVC) + un-suspend the cron. +# Data-path revert (independent) = set FXHNT_COMBINED_BOOK_DATA_SOURCE=raw below. +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: dagster-instance + namespace: foxhunt +data: + dagster.yaml: | + storage: + postgres: + postgres_db: + username: foxhunt + password: + env: PGPASSWORD + hostname: postgres.foxhunt.svc.cluster.local + db_name: dagster + port: 5432 + telemetry: + enabled: false + workspace.yaml: | + load_from: + - python_module: + module_name: fxhnt.adapters.orchestration.definitions + working_directory: /app +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: fxhnt-dagster-ts-serve + namespace: foxhunt +data: + serve.json: | + { + "TCP": { "443": { "HTTPS": true } }, + "Web": { + "${TS_CERT_DOMAIN}:443": { + "Handlers": { "/": { "Proxy": "http://127.0.0.1:3000" } } + } + } + } +--- +apiVersion: v1 +kind: ServiceAccount +metadata: { name: tailscale-dagster, namespace: foxhunt } +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: { name: tailscale-dagster, namespace: foxhunt } +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["create", "get", "update", "patch", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: { name: tailscale-dagster, namespace: foxhunt } +subjects: + - { kind: ServiceAccount, name: tailscale-dagster, namespace: foxhunt } +roleRef: { kind: Role, name: tailscale-dagster, apiGroup: rbac.authorization.k8s.io } +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: dagster + namespace: foxhunt + labels: { app.kubernetes.io/name: dagster, app.kubernetes.io/part-of: foxhunt } +spec: + replicas: 1 + strategy: { type: Recreate } # RWO PVC: never run two pods holding it at once + selector: { matchLabels: { app.kubernetes.io/name: dagster } } + template: + metadata: + labels: { app.kubernetes.io/name: dagster, app.kubernetes.io/part-of: foxhunt } # part-of -> postgres ingress + base egress + spec: + serviceAccountName: tailscale-dagster + nodeSelector: { k8s.scaleway.com/pool-name: platform } + imagePullSecrets: [{ name: gitlab-registry }] + initContainers: + - name: dagster-home-init # DAGSTER_HOME must be writable; seed it from the configmap + image: busybox:1.36 + command: ["sh", "-c", "cp /cfg/dagster.yaml /cfg/workspace.yaml /dagster-home/"] + volumeMounts: + - { name: dagster-home, mountPath: /dagster-home } + - { name: dagster-cfg, mountPath: /cfg, readOnly: true } + containers: + - name: webserver + image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/fxhnt-cockpit:latest + command: ["dagster-webserver", "-h", "0.0.0.0", "-p", "3000", + "-w", "/dagster-home/workspace.yaml"] + env: &dagster_env + - { name: DAGSTER_HOME, value: /dagster-home } + - { name: FXHNT_COMBINED_BOOK_DATA_SOURCE, value: "warehouse" } + - { name: FXHNT_WAREHOUSE_PATH, value: /data/surfer/warehouse.duckdb } + - { name: FXHNT_SURFER_DATA_DIR, value: /data/surfer } + - { name: FXHNT_OPERATIONAL_DSN, value: "postgresql+psycopg://foxhunt@postgres.foxhunt.svc.cluster.local:5432/fxhnt" } + - { name: FXHNT_DATABENTO_MAX_COST_USD, value: "40.0" } + - name: PGPASSWORD + valueFrom: { secretKeyRef: { name: db-credentials, key: password } } + - name: DATABENTO_API_KEY + valueFrom: { secretKeyRef: { name: databento-credentials, key: api-key } } + readinessProbe: { tcpSocket: { port: 3000 }, initialDelaySeconds: 10, periodSeconds: 15 } + volumeMounts: + - { name: dagster-home, mountPath: /dagster-home } + - { name: surfer-data, mountPath: /data } + resources: { requests: { cpu: 100m, memory: 512Mi }, limits: { cpu: "1", memory: 1Gi } } + - name: daemon + image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/fxhnt-cockpit:latest + command: ["dagster-daemon", "run", "-w", "/dagster-home/workspace.yaml"] + env: *dagster_env + volumeMounts: + - { name: dagster-home, mountPath: /dagster-home } + - { name: surfer-data, mountPath: /data } + resources: { requests: { cpu: 100m, memory: 512Mi }, limits: { cpu: "2", memory: 2Gi } } + - name: tailscale + image: ghcr.io/tailscale/tailscale:latest + env: + - name: TS_AUTHKEY + valueFrom: { secretKeyRef: { name: tailscale-auth, key: TS_AUTHKEY } } + - { name: TS_KUBE_SECRET, value: fxhnt-dagster-ts-state } + - { name: TS_USERSPACE, value: "true" } + - { name: TS_HOSTNAME, value: "fxhnt-dagster" } + - { name: TS_SERVE_CONFIG, value: /etc/ts/serve.json } + volumeMounts: + - { name: ts-serve, mountPath: /etc/ts, readOnly: true } + resources: { requests: { cpu: 25m, memory: 64Mi }, limits: { cpu: 200m, memory: 128Mi } } + volumes: + - { name: dagster-home, emptyDir: {} } + - { name: dagster-cfg, configMap: { name: dagster-instance } } + - { name: ts-serve, configMap: { name: fxhnt-dagster-ts-serve } } + - { name: surfer-data, persistentVolumeClaim: { claimName: fxhnt-surfer-data } } +--- +apiVersion: v1 +kind: Service +metadata: { name: dagster, namespace: foxhunt } +spec: + selector: { app.kubernetes.io/name: dagster } + ports: [{ port: 80, targetPort: 3000, name: http }] +--- +# Egress: postgres (storage + cockpit ingest) + DNS + public HTTPS (Binance/Databento fetch in the assets) +# + tailscale coordination/DERP. part-of=foxhunt already grants base egress + satisfies postgres ingress. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: { name: dagster, namespace: foxhunt, labels: { app.kubernetes.io/part-of: foxhunt } } +spec: + podSelector: { matchLabels: { app.kubernetes.io/name: dagster } } + policyTypes: [Egress] + egress: + - to: [{ podSelector: { matchLabels: { app.kubernetes.io/name: postgres } } }] + ports: [{ port: 5432, protocol: TCP }] + - ports: [{ port: 53, protocol: UDP }, { port: 53, protocol: TCP }] + - to: [{ ipBlock: { cidr: 0.0.0.0/0, except: [10.32.0.0/16, 172.16.0.0/16] } }] # Binance/Databento + tailscale + ports: [{ port: 443, protocol: TCP }, { port: 3478, protocol: UDP }, { port: 41641, protocol: UDP }]