diff --git a/.cargo/config.toml b/.cargo/config.toml index cd418dbec..c3336ad2a 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -26,7 +26,7 @@ progress.when = "auto" [target.x86_64-unknown-linux-gnu] linker = "clang" rustflags = [ - "-C", "link-arg=-fuse-ld=lld", + "-C", "link-arg=-fuse-ld=mold", "-C", "link-arg=-Wl,-z,relro,-z,now", "-C", "link-arg=-Wl,--as-needed", # AVX2/FMA baseline — matches both Scaleway L4/H100 and most dev machines diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8068c10ac..33ec3fa5d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,5 @@ # GitLab CI/CD — Foxhunt -# Runs on ci-build pool (L4, CUDA 8.9, scale-to-zero), linker: clang+lld +# Runs on ci-build pool (L40S, CUDA 8.9, autoscales 0-2), linker: clang+mold # CI builder image: CUDA 12.4 + Rust 1.89 + protoc + sccache (hosted on Scaleway CR) # Service images: pushed to Scaleway Container Registry (Kaniko) # Note: .gitlab-ci.yml itself is NOT in changes: filters — CI-only edits skip build diff --git a/infra/docker/Dockerfile.ci-builder b/infra/docker/Dockerfile.ci-builder index d2f14df65..12b2a8dcd 100644 --- a/infra/docker/Dockerfile.ci-builder +++ b/infra/docker/Dockerfile.ci-builder @@ -23,6 +23,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ unzip \ && rm -rf /var/lib/apt/lists/* +# mold linker — 2-5x faster than lld for final link step on large Rust binaries +RUN curl -fsSL https://github.com/rui314/mold/releases/download/v2.35.1/mold-2.35.1-x86_64-linux.tar.gz \ + | tar xz -C /usr/local --strip-components=1 \ + && chmod +x /usr/local/bin/mold + # protoc 28.3 (proto3 optional support, matches local dev) RUN curl -fsSL https://github.com/protocolbuffers/protobuf/releases/download/v28.3/protoc-28.3-linux-x86_64.zip -o /tmp/protoc.zip \ && unzip -o /tmp/protoc.zip -d /usr/local bin/protoc 'include/*' \ diff --git a/infra/k8s/gitlab/runner-values.yaml b/infra/k8s/gitlab/runner-values.yaml index 91c1f03e9..4c2555a20 100644 --- a/infra/k8s/gitlab/runner-values.yaml +++ b/infra/k8s/gitlab/runner-values.yaml @@ -1,6 +1,6 @@ -# GitLab Runner — Kubernetes executor targeting ci-build pool (L4) +# GitLab Runner — Kubernetes executor targeting ci-build pool (L40S) # Runner manager pod lives on gitlab node pool -# Build pods spawn on ci-build pool (L4-1-24G, scale-to-zero) +# Build pods spawn on ci-build pool (L40S-1-48G, autoscales 0-2) gitlabUrl: http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181 # runnerToken set via --set at install time @@ -26,13 +26,13 @@ runners: namespace = "foxhunt" image = "rust:1.89-slim" privileged = false - # L4 scale-to-zero needs ~3-5 min to provision; default 180s times out + # L40S scale-to-zero needs ~3-5 min to provision; default 180s times out poll_timeout = 600 - # Resource limits for build pods (L4 node: 12 vCPU, 45.5GB) - cpu_request = "4000m" - cpu_limit = "10000m" - memory_request = "16Gi" - memory_limit = "38Gi" + # Resource limits for build pods (L40S node: ~24 vCPU, ~96GB) + cpu_request = "6000m" + cpu_limit = "20000m" + memory_request = "24Gi" + memory_limit = "80Gi" helper_cpu_request = "100m" helper_cpu_limit = "500m" helper_memory_request = "128Mi" diff --git a/infra/live/production/kapsule/terragrunt.hcl b/infra/live/production/kapsule/terragrunt.hcl index 9be51f067..f6f2a0e96 100644 --- a/infra/live/production/kapsule/terragrunt.hcl +++ b/infra/live/production/kapsule/terragrunt.hcl @@ -25,10 +25,11 @@ inputs = { enable_gitlab_pool = true gitlab_type = "GP1-XS" - # CI build pool (L4 for cost-effective CI: 12 vCPU, 45.5GB, CUDA CC 89) + # CI build pool (L40S for faster CI: 48GB VRAM, ~2x L4 throughput, CUDA CC 89) + # max_size=2 allows autoscaler to add a second node for concurrent jobs enable_ci_build_pool = true - ci_build_type = "L4-1-24G" - ci_build_max_size = 1 + ci_build_type = "L40S-1-48G" + ci_build_max_size = 2 # Dev pool (GP1-L for DevPod remote development, autoscales to zero) enable_dev_pool = true diff --git a/infra/modules/kapsule/main.tf b/infra/modules/kapsule/main.tf index 91f059328..30217c35d 100644 --- a/infra/modules/kapsule/main.tf +++ b/infra/modules/kapsule/main.tf @@ -89,7 +89,7 @@ resource "scaleway_k8s_pool" "gitlab" { region = var.region } -# CI build pool (L4 — cheaper than H100, has CUDA for compilation + GPU tests) +# CI build pool (L40S — 48GB VRAM, ~2x L4 throughput, same Ada arch / CUDA CC 89) resource "scaleway_k8s_pool" "ci_build" { count = var.enable_ci_build_pool ? 1 : 0 cluster_id = scaleway_k8s_cluster.foxhunt.id diff --git a/infra/modules/kapsule/variables.tf b/infra/modules/kapsule/variables.tf index a6352cd7c..407a514f6 100644 --- a/infra/modules/kapsule/variables.tf +++ b/infra/modules/kapsule/variables.tf @@ -76,15 +76,15 @@ variable "enable_ci_build_pool" { } variable "ci_build_type" { - description = "Instance type for the CI build node pool" + description = "Instance type for the CI build node pool (L40S for fast compilation + training)" type = string - default = "L4-1-24G" + default = "L40S-1-48G" } variable "ci_build_max_size" { - description = "Maximum number of nodes in the CI build pool" + description = "Maximum number of nodes in the CI build pool (2 allows concurrent jobs)" type = number - default = 1 + default = 2 } variable "enable_dev_pool" {