From 5cb9be71e49cc38d9df77f5d81d2d6aa8c1acc83 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Fri, 13 Mar 2026 12:20:26 +0100 Subject: [PATCH] fix(infra): add gpu-test egress NetworkPolicy, revert archiveLogs hack MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause: default-deny-all blocks egress from gpu-test pods to MinIO, causing the Argo wait sidecar to hang on log upload. Fix: add egress policy for gpu-test component (DNS, git, MinIO, Mattermost, registry). Revert archiveLogs: false — logs should be stored. Co-Authored-By: Claude Opus 4.6 --- infra/k8s/argo/gpu-test-network-policy.yaml | 85 +++++++++++++++++++ .../k8s/argo/gpu-test-pipeline-template.yaml | 7 -- 2 files changed, 85 insertions(+), 7 deletions(-) create mode 100644 infra/k8s/argo/gpu-test-network-policy.yaml diff --git a/infra/k8s/argo/gpu-test-network-policy.yaml b/infra/k8s/argo/gpu-test-network-policy.yaml new file mode 100644 index 000000000..58ffceaca --- /dev/null +++ b/infra/k8s/argo/gpu-test-network-policy.yaml @@ -0,0 +1,85 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: argo-gpu-test-workflow + namespace: foxhunt + labels: + app.kubernetes.io/part-of: foxhunt +spec: + podSelector: + matchLabels: + app.kubernetes.io/component: gpu-test + policyTypes: + - Egress + egress: + # DNS + - ports: + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP + # Kubernetes API + internal services (service CIDR) + - ports: + - port: 443 + protocol: TCP + - port: 6443 + protocol: TCP + to: + - ipBlock: + cidr: 10.32.0.0/16 + - ipBlock: + cidr: 172.16.0.4/32 + # HTTPS egress (crates.io, etc.) + - ports: + - port: 443 + protocol: TCP + # Git SSH (GitLab) + - ports: + - port: 2222 + protocol: TCP + to: + - ipBlock: + cidr: 100.90.76.85/32 + - podSelector: + matchLabels: + app: gitlab-shell + # GitLab webservice (git-http) + - ports: + - port: 8181 + protocol: TCP + to: + - podSelector: + matchLabels: + app: webservice + # MinIO (artifact/log storage) + - ports: + - port: 9000 + protocol: TCP + to: + - podSelector: + matchLabels: + app.kubernetes.io/name: minio + # OTLP (Tempo) + - ports: + - port: 4317 + protocol: TCP + to: + - podSelector: + matchLabels: + app.kubernetes.io/name: tempo + # Mattermost (notifications) + - ports: + - port: 8065 + protocol: TCP + to: + - podSelector: + matchLabels: + app.kubernetes.io/name: mattermost + # GitLab container registry + - ports: + - port: 5000 + protocol: TCP + to: + - podSelector: + matchLabels: + app: registry diff --git a/infra/k8s/argo/gpu-test-pipeline-template.yaml b/infra/k8s/argo/gpu-test-pipeline-template.yaml index cc761fef0..f29d06e39 100644 --- a/infra/k8s/argo/gpu-test-pipeline-template.yaml +++ b/infra/k8s/argo/gpu-test-pipeline-template.yaml @@ -62,10 +62,7 @@ spec: # ── gpu-warmup: trigger H100 autoscale ── # Requests GPU to force autoscaler to provision node, then releases it. - # archiveLogs: false — no MinIO dependency for this throwaway step. - name: gpu-warmup - archiveLocation: - archiveLogs: false nodeSelector: k8s.scaleway.com/pool-name: "{{workflow.parameters.gpu-pool}}" tolerations: @@ -95,8 +92,6 @@ spec: # ── compile-and-test: compile + run GPU tests in single H100 pod ── - name: compile-and-test - archiveLocation: - archiveLogs: false nodeSelector: k8s.scaleway.com/pool-name: "{{workflow.parameters.gpu-pool}}" tolerations: @@ -307,8 +302,6 @@ spec: # ── notify-result: post test outcome to Mattermost (onExit) ── - name: notify-result - archiveLocation: - archiveLogs: false nodeSelector: k8s.scaleway.com/pool-name: platform container: