fix(ci): C1 dagster to :latest+Always-pull; I1 trap restores dashboard replicas
C1 (Critical, CI review): dagster webserver/daemon + K8sRunLauncher job_image were pinned to a hardcoded tag (dagster-k8s-25e3d7c), so a mode=build dep bump updated the dashboard (:latest) but left dagster + its run Jobs on stale deps -> version skew in the nightly ETL. Now :latest with image_pull_policy Always. I1 (Important): a trap scales the dashboard back to replicas=1 on ANY exit path, so a failure between scale-0 and scale-1 can't strand the cockpit at 0.
This commit is contained in:
@@ -188,9 +188,16 @@ spec:
|
||||
# the external proxy link goes stale and the cockpit reads "down" post-deploy. Scaling to 0 cleanly
|
||||
# terminates the session before a fresh, non-overlapping one comes up; the git-sync initContainer
|
||||
# re-pulls src the same way, so code still updates. See reference_fxhnt_dashboard_proxy.
|
||||
#
|
||||
# SAFETY (CI review I1): once we scale to 0, a trap guarantees the deployment is scaled BACK to 1 on
|
||||
# ANY exit path — so a failure between scale-0 and scale-1 (transient apiserver error, throttle) can
|
||||
# never strand the production dashboard at 0 replicas. The trap is best-effort (its own failure is
|
||||
# swallowed) and cleared on success.
|
||||
trap 'kubectl -n foxhunt scale deploy/fxhnt-dashboard --replicas=1 || true' EXIT
|
||||
kubectl -n foxhunt scale deploy/fxhnt-dashboard --replicas=0
|
||||
kubectl -n foxhunt wait --for=delete pod -l app.kubernetes.io/name=fxhnt-dashboard --timeout=90s || true
|
||||
kubectl -n foxhunt scale deploy/fxhnt-dashboard --replicas=1
|
||||
trap - EXIT # scaled back up cleanly; no longer need the safety net
|
||||
kubectl -n foxhunt rollout restart deploy/dagster # dagster has no external-proxy race
|
||||
kubectl -n foxhunt rollout status deploy/fxhnt-dashboard --timeout=180s
|
||||
kubectl -n foxhunt rollout status deploy/dagster --timeout=300s
|
||||
|
||||
@@ -40,8 +40,8 @@ data:
|
||||
config:
|
||||
service_account_name: tailscale-dagster
|
||||
job_namespace: foxhunt
|
||||
job_image: rg.fr-par.scw.cloud/bizworx/fxhnt-cockpit:dagster-k8s-25e3d7c
|
||||
image_pull_policy: IfNotPresent
|
||||
job_image: rg.fr-par.scw.cloud/bizworx/fxhnt-cockpit:latest
|
||||
image_pull_policy: Always # :latest -> Always-pull so a dep-bump build reaches launched run Jobs (CI review C1)
|
||||
image_pull_secrets:
|
||||
- name: scw-registry
|
||||
load_incluster_config: true
|
||||
@@ -201,7 +201,7 @@ spec:
|
||||
- { name: git-ssh-key, mountPath: /etc/git-ssh, readOnly: true }
|
||||
containers:
|
||||
- name: webserver
|
||||
image: rg.fr-par.scw.cloud/bizworx/fxhnt-cockpit:dagster-k8s-25e3d7c
|
||||
image: rg.fr-par.scw.cloud/bizworx/fxhnt-cockpit:latest
|
||||
command: ["dagster-webserver", "-h", "0.0.0.0", "-p", "3000",
|
||||
"-w", "/dagster-home/workspace.yaml"]
|
||||
env: &dagster_env
|
||||
@@ -229,7 +229,7 @@ spec:
|
||||
- { name: code, mountPath: /code, readOnly: true } # git-sync'd src (PYTHONPATH=/code/src)
|
||||
resources: { requests: { cpu: 100m, memory: 512Mi }, limits: { cpu: "1", memory: 1Gi } }
|
||||
- name: daemon
|
||||
image: rg.fr-par.scw.cloud/bizworx/fxhnt-cockpit:dagster-k8s-25e3d7c
|
||||
image: rg.fr-par.scw.cloud/bizworx/fxhnt-cockpit:latest
|
||||
command: ["dagster-daemon", "run", "-w", "/dagster-home/workspace.yaml"]
|
||||
env: *dagster_env
|
||||
volumeMounts:
|
||||
|
||||
Reference in New Issue
Block a user