diff --git a/docs/superpowers/plans/CONSOLIDATION-PROGRESS.md b/docs/superpowers/plans/CONSOLIDATION-PROGRESS.md index 259fcc0..8508d1d 100644 --- a/docs/superpowers/plans/CONSOLIDATION-PROGRESS.md +++ b/docs/superpowers/plans/CONSOLIDATION-PROGRESS.md @@ -72,6 +72,26 @@ Rollback until T11 = re-point DNS + un-suspend foxhunt. - ALSO for cutover: the tailscale bizworx pods joined as extra nodes (fresh key); at T8 suspend foxhunt so the real hostnames + IBKR session move cleanly. ROTATE the pasted tailscale key post-cutover. +## P1 EXECUTION (2026-07-18) — wildcard cert + the cross-project DNS-01 IAM fix +- Applied `infra/k8s/services/fxhnt-wildcard-cert.yaml` (Certificate fxhnt.ai + *.fxhnt.ai, ClusterIssuer + letsencrypt-prod, secret fxhnt-wildcard-tls, ns foxhunt) on BIZ. +- **DNS-01 FAILED: `403 Forbidden: domain not found`.** Root cause: `fxhnt.ai` DNS zone lives in the **foxhunt + project c293eb98**; the bizworx cert-manager key (`SCWW9261…`, app `bizworx-prod-runtime` 48418a71) has + `DomainsDNSFullAccess` but SCOPED to the **bizworx project 96d0717d** only → can't touch fxhnt.ai's zone. +- **DEAD END (operator's first pick "move the domain to bizworx project"): Scaleway BLOCKS it.** fxhnt.ai is an + EXTERNAL domain (`scw domain domain list` → IS EXTERNAL=true, REGISTRAR=EXTERNAL). `scw dns zone update + … project-id=…` on the apex → **`Root zone can't be updated`**; on a scratch sub-zone → **`Subdomain not + allowed`**; `scw domain domain` has NO move-project subcommand. Verified via a throwaway `_cutovertest.fxhnt.ai` + sub-zone (created+move-attempt+deleted). So the domain CANNOT be reassigned to the bizworx project via API. +- **ACTUAL FIX (operator's 2nd pick, applied): add an IAM rule.** `scw iam rule create + policy-id=8573ce25-… permission-set-names.0=DomainsDNSFullAccess project-ids.0=c293eb98-…` → policy + `bizworx-prod-runtime-scoped` now has 2 rules (orig 6-perms@96d0717d untouched + new DNS@c293eb98). This lets + the existing bizworx cert-manager key write the DNS-01 TXT on fxhnt.ai. Domain STAYS external in c293eb98; + only IAM changed. NOTE for teardown: this rule references the foxhunt project — harmless to keep, or tighten + later; if the foxhunt PROJECT (not just cluster) is ever deleted, the fxhnt.ai domain/zone must first move + registrar-side or certs break. (Cluster teardown ≠ project deletion; the DNS zone + domain survive T12.) +- Forced cert-manager retry (deleted order+CR) after the IAM change. Monitoring for READY. + ## T8 PREP (2026-07-18) — DNS/ingress map + fixes (foxhunt still fully live) ### Gotchas hit this session diff --git a/infra/k8s/services/fxhnt-ingress-proxy.yaml b/infra/k8s/services/fxhnt-ingress-proxy.yaml new file mode 100644 index 0000000..8a011d9 --- /dev/null +++ b/infra/k8s/services/fxhnt-ingress-proxy.yaml @@ -0,0 +1,179 @@ +# fxhnt-ingress — the bizworx front for *.fxhnt.ai (fund-only; replaces the foxhunt tailscale-gitlab-proxy). +# nginx terminates the *.fxhnt.ai wildcard (cert-manager secret fxhnt-wildcard-tls) and reverse-proxies the 3 +# fund vhosts by ClusterIP + a socat sidecar forwards git SSH. A tailscale sidecar joins the tailnet as +# `fxhnt-ingress` so the Scaleway fxhnt.ai A-records can point at its stable tailnet IP (C4). +# +# WHY ClusterIP (not pod tailnet IPs like the foxhunt proxy did): on bizworx a tailscale-sidecar pod CAN reach +# fund ClusterIPs (10.32.x.x) — verified: cockpit ClusterIP returned HTTP 200 from a TS sidecar — but CANNOT +# reach pod IPs in 100.64.0.0/15 (Tailscale CGNAT overlap; verified: pod-IP request timed out). So proxy to +# Services by name, never to pod IPs. +apiVersion: v1 +kind: ConfigMap +metadata: + name: fxhnt-ingress-nginx + namespace: foxhunt + labels: { app.kubernetes.io/name: fxhnt-ingress, app.kubernetes.io/part-of: foxhunt } +data: + default.conf: | + # HTTP -> HTTPS + server { + listen 80; + server_name *.fxhnt.ai; + return 301 https://$host$request_uri; + } + # git.fxhnt.ai -> bizworx gitea (ns gitea) + server { + listen 443 ssl; + server_name git.fxhnt.ai; + ssl_certificate /etc/nginx/certs/tls.crt; + ssl_certificate_key /etc/nginx/certs/tls.key; + ssl_protocols TLSv1.2 TLSv1.3; + client_max_body_size 0; + location / { + proxy_pass http://gitea-http.gitea.svc.cluster.local:3000; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto https; + proxy_buffering off; + proxy_request_buffering off; + proxy_http_version 1.1; + proxy_read_timeout 3600s; + proxy_send_timeout 3600s; + proxy_connect_timeout 300s; + } + } + # dashboard.fxhnt.ai + cockpit.fxhnt.ai -> bizworx cockpit (ns foxhunt) + server { + listen 443 ssl; + server_name dashboard.fxhnt.ai cockpit.fxhnt.ai; + ssl_certificate /etc/nginx/certs/tls.crt; + ssl_certificate_key /etc/nginx/certs/tls.key; + ssl_protocols TLSv1.2 TLSv1.3; + location / { + proxy_pass http://fxhnt-dashboard.foxhunt.svc.cluster.local:80; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto https; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } + } + # dagster.fxhnt.ai -> bizworx dagster (ns foxhunt) + server { + listen 443 ssl; + server_name dagster.fxhnt.ai; + ssl_certificate /etc/nginx/certs/tls.crt; + ssl_certificate_key /etc/nginx/certs/tls.key; + ssl_protocols TLSv1.2 TLSv1.3; + location / { + proxy_pass http://dagster.foxhunt.svc.cluster.local:80; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto https; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_read_timeout 3600s; + } + } +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: fxhnt-ingress + namespace: foxhunt + labels: { app.kubernetes.io/name: fxhnt-ingress, app.kubernetes.io/part-of: foxhunt } +spec: + replicas: 1 + selector: + matchLabels: { app.kubernetes.io/name: fxhnt-ingress } + template: + metadata: + labels: { app.kubernetes.io/name: fxhnt-ingress, app.kubernetes.io/part-of: foxhunt } + spec: + nodeSelector: + k8s.scaleway.com/pool-name: large + containers: + - name: nginx + image: nginx:alpine + ports: + - { containerPort: 80, name: http } + - { containerPort: 443, name: https } + volumeMounts: + - { name: nginx-conf, mountPath: /etc/nginx/conf.d, readOnly: true } + - { name: wildcard-cert, mountPath: /etc/nginx/certs, readOnly: true } + resources: + requests: { cpu: 25m, memory: 32Mi } + limits: { cpu: 200m, memory: 128Mi } + # git SSH: git.fxhnt.ai:22 -> bizworx gitea-ssh (ns gitea). socat TCP forward on the same tailnet node. + - name: ssh-proxy + image: alpine/socat:latest + args: + - TCP-LISTEN:22,fork,reuseaddr + - TCP:gitea-ssh.gitea.svc.cluster.local:22 + resources: + requests: { cpu: 10m, memory: 16Mi } + limits: { cpu: 100m, memory: 64Mi } + - name: tailscale + image: ghcr.io/tailscale/tailscale:stable + env: + - { name: TS_HOSTNAME, value: "fxhnt-ingress" } + - { name: TS_KUBE_SECRET, value: "fxhnt-ingress-ts-state" } + - { name: TS_USERSPACE, value: "false" } + - name: TS_AUTHKEY + valueFrom: { secretKeyRef: { name: tailscale-auth, key: TS_AUTHKEY } } + - name: POD_NAME + valueFrom: { fieldRef: { fieldPath: metadata.name } } + - name: POD_UID + valueFrom: { fieldRef: { fieldPath: metadata.uid } } + securityContext: + capabilities: { add: ["NET_ADMIN"] } + resources: + requests: { cpu: 25m, memory: 64Mi } + limits: { cpu: 200m, memory: 128Mi } + volumes: + - name: nginx-conf + configMap: { name: fxhnt-ingress-nginx } + - name: wildcard-cert + secret: { secretName: fxhnt-wildcard-tls } +--- +# ClusterIP Service (in-cluster reachability; the public entrypoint is the tailscale node, not this Service) +apiVersion: v1 +kind: Service +metadata: + name: fxhnt-ingress + namespace: foxhunt + labels: { app.kubernetes.io/name: fxhnt-ingress, app.kubernetes.io/part-of: foxhunt } +spec: + selector: { app.kubernetes.io/name: fxhnt-ingress } + ports: + - { port: 443, targetPort: 443, name: https } + - { port: 80, targetPort: 80, name: http } + - { port: 22, targetPort: 22, name: ssh } +--- +# NetworkPolicy: allow this proxy the egress it needs (postgres NOT needed; it only fronts HTTP/SSH to services). +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: fxhnt-ingress + namespace: foxhunt + labels: { app.kubernetes.io/name: fxhnt-ingress, app.kubernetes.io/part-of: foxhunt } +spec: + podSelector: { matchLabels: { app.kubernetes.io/name: fxhnt-ingress } } + policyTypes: [Egress] + egress: + - ports: [{ port: 53, protocol: UDP }, { port: 53, protocol: TCP }] # DNS + - to: [{ podSelector: { matchLabels: { app.kubernetes.io/name: fxhnt-dashboard } } }] + ports: [{ port: 8080, protocol: TCP }, { port: 80, protocol: TCP }] # cockpit + - to: [{ podSelector: { matchLabels: { app.kubernetes.io/name: dagster } } }] + ports: [{ port: 3000, protocol: TCP }] # dagster webserver + - to: [{ namespaceSelector: { matchLabels: { kubernetes.io/metadata.name: gitea } } }] + ports: [{ port: 3000, protocol: TCP }, { port: 22, protocol: TCP }] # gitea web + ssh (cross-ns) + - to: [{ ipBlock: { cidr: 172.16.0.11/32 } }] # kube-apiserver (tailscale sidecar ts-state Secret; post-DNAT dest) + ports: [{ port: 6443, protocol: TCP }] + - to: [{ ipBlock: { cidr: 0.0.0.0/0, except: [10.32.0.0/16, 172.16.0.0/16] } }] # tailscale coordination/DERP + ports: [{ port: 443, protocol: TCP }, { port: 3478, protocol: UDP }, { port: 41641, protocol: UDP }] diff --git a/infra/k8s/services/fxhnt-wildcard-cert.yaml b/infra/k8s/services/fxhnt-wildcard-cert.yaml new file mode 100644 index 0000000..0ddfba2 --- /dev/null +++ b/infra/k8s/services/fxhnt-wildcard-cert.yaml @@ -0,0 +1,22 @@ +# Wildcard TLS cert for the fxhnt.ai fund ingress on bizworx. +# Issued by the existing bizworx letsencrypt-prod ClusterIssuer (Scaleway DNS-01 webhook), which already +# issues real LE certs across ~10 namespaces. The fxhnt-ingress proxy (P2) terminates *.fxhnt.ai with this. +# DNS-01 provisions a TXT on the Scaleway fxhnt.ai zone we control -> no HTTP reachability needed to issue. +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: fxhnt-wildcard + namespace: foxhunt + labels: + app.kubernetes.io/part-of: foxhunt +spec: + commonName: fxhnt.ai + dnsNames: + - fxhnt.ai + - "*.fxhnt.ai" + issuerRef: + kind: ClusterIssuer + name: letsencrypt-prod + secretName: fxhnt-wildcard-tls + privateKey: + rotationPolicy: Always