From a77c873da4f8c49f5bca5db00f2e927a8a8b969a Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Thu, 5 Mar 2026 22:34:44 +0100 Subject: [PATCH] fix(ci): add deploy step, podGC, move compile-services to ci-compile-cpu MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three issues prevented services from being deployed after push: 1. No deploy step: pipeline compiled + uploaded binaries to MinIO but never restarted service deployments. Added deploy-services step (bitnami/kubectl) that runs rollout restart after compile-services. 2. compile-services couldn't schedule: targeted platform node (4 CPU, 6Gi allocatable) but requested 6Gi memory. Moved to ci-compile-cpu (POP2-32C-128G) alongside compile-training — both fit simultaneously (28/32 CPU, 48/128Gi). 3. No podGC: completed Argo pods held resources indefinitely, blocking subsequent pipeline steps. Added podGC:OnPodCompletion. Also: detect-changes now triggers service rebuild when CI pipeline template or k8s service manifests change. Co-Authored-By: Claude Opus 4.6 --- infra/k8s/argo/ci-deploy-rbac.yaml | 28 ++++++++++++ infra/k8s/argo/ci-pipeline-template.yaml | 57 +++++++++++++++++++++--- infra/k8s/argo/kustomization.yaml | 1 + 3 files changed, 79 insertions(+), 7 deletions(-) create mode 100644 infra/k8s/argo/ci-deploy-rbac.yaml diff --git a/infra/k8s/argo/ci-deploy-rbac.yaml b/infra/k8s/argo/ci-deploy-rbac.yaml new file mode 100644 index 000000000..e9fa70415 --- /dev/null +++ b/infra/k8s/argo/ci-deploy-rbac.yaml @@ -0,0 +1,28 @@ +# RBAC for CI pipeline deploy step — allows argo-workflow SA to restart deployments +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: ci-deploy + namespace: foxhunt + labels: + app.kubernetes.io/part-of: foxhunt +rules: + - apiGroups: ["apps"] + resources: [deployments] + verbs: [get, list, patch] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: ci-deploy + namespace: foxhunt + labels: + app.kubernetes.io/part-of: foxhunt +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ci-deploy +subjects: + - kind: ServiceAccount + name: argo-workflow + namespace: foxhunt diff --git a/infra/k8s/argo/ci-pipeline-template.yaml b/infra/k8s/argo/ci-pipeline-template.yaml index b5cbf3cdd..023b15209 100644 --- a/infra/k8s/argo/ci-pipeline-template.yaml +++ b/infra/k8s/argo/ci-pipeline-template.yaml @@ -14,6 +14,8 @@ spec: labels: app.kubernetes.io/component: ci-pipeline app.kubernetes.io/part-of: foxhunt + podGC: + strategy: OnPodCompletion ttlStrategy: secondsAfterCompletion: 3600 volumes: @@ -64,6 +66,11 @@ spec: template: build-web-dashboard when: "{{tasks.detect-changes.outputs.parameters.needs-dashboard}} == true" + - name: deploy-services + dependencies: [compile-services] + template: deploy-services + when: "{{tasks.detect-changes.outputs.parameters.needs-services}} == true" + - name: rebuild-ci-builder dependencies: [detect-changes] templateRef: @@ -178,7 +185,7 @@ spec: } SHARED=$(check_paths "crates/common/ crates/config/ Cargo.toml Cargo.lock") - SVC_ONLY=$(check_paths "services/ bin/fxt/ crates/trading_engine/ crates/risk/ crates/data/ crates/storage/ crates/database/ crates/broker crates/ctrader") + SVC_ONLY=$(check_paths "services/ bin/fxt/ crates/trading_engine/ crates/risk/ crates/data/ crates/storage/ crates/database/ crates/broker crates/ctrader infra/k8s/argo/ci-pipeline infra/k8s/services/") ML_CHANGED=$(check_paths "crates/ml/") DOCKER_IMAGES=$(check_paths "infra/docker/") NEEDS_DASHBOARD=$(check_paths "web-dashboard/") @@ -299,11 +306,15 @@ spec: echo "=== Web dashboard build + upload done ===" - # ── compile-services: CPU-only, runs on platform (no CUDA needed) ── + # ── compile-services: CPU-only, runs on ci-compile-cpu alongside compile-training ── # sccache backed by MinIO (s3://foxhunt-sccache/cpu/) — shared across nodes - name: compile-services nodeSelector: - k8s.scaleway.com/pool-name: platform + k8s.scaleway.com/pool-name: ci-compile-cpu + tolerations: + - key: node.cilium.io/agent-not-ready + operator: Exists + effect: NoSchedule container: image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/ci-builder-cpu:latest imagePullPolicy: Always @@ -347,11 +358,11 @@ spec: key: secret-key resources: requests: - cpu: "3" - memory: 6Gi + cpu: "14" + memory: 16Gi limits: - cpu: "3800m" - memory: 10Gi + cpu: "30" + memory: 32Gi volumeMounts: - name: git-ssh-key mountPath: /etc/git-ssh @@ -550,3 +561,35 @@ spec: --no-check-certificate echo "=== Training compile + upload done ===" + + # ── deploy-services: rolling restart after new binaries are uploaded ── + - name: deploy-services + serviceAccountName: argo-workflow + nodeSelector: + k8s.scaleway.com/pool-name: platform + container: + image: bitnami/kubectl:1.31 + command: ["/bin/sh", "-c"] + resources: + requests: + cpu: 100m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi + args: + - | + set -e + echo "=== Rolling restart of service deployments ===" + SERVICES="api trading-service ml-training-service backtesting-service trading-agent-service broker-gateway data-acquisition-service" + for svc in $SERVICES; do + echo "Restarting $svc..." + kubectl -n foxhunt rollout restart deployment "$svc" || echo "WARN: $svc restart failed (may not exist)" + done + + echo "=== Waiting for rollouts ===" + for svc in $SERVICES; do + kubectl -n foxhunt rollout status deployment "$svc" --timeout=120s || echo "WARN: $svc rollout timeout" + done + + echo "=== Deploy complete ===" diff --git a/infra/k8s/argo/kustomization.yaml b/infra/k8s/argo/kustomization.yaml index 92a26f48c..9f2d6a84e 100644 --- a/infra/k8s/argo/kustomization.yaml +++ b/infra/k8s/argo/kustomization.yaml @@ -6,3 +6,4 @@ resources: - build-ci-image-template.yaml - training-workflow-template.yaml - argo-workflow-netpol.yaml + - ci-deploy-rbac.yaml