68b6aa8313
Full migration off Scaleway Container Registry to internal GitLab registry backed by MinIO S3. All 4 images (ci-builder, ci-builder-cpu, foxhunt-runtime, foxhunt-training-runtime) rebuilt in internal registry. Registry & images: - All image refs → gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/ - imagePullSecrets: scw-registry → gitlab-registry - Kaniko build template: two-step DAG (git-clone → kaniko-build) with shared PVC - Kaniko layer cache enabled at root/foxhunt/cache - AWS_ACCESS_KEY_ID: $SCW_ACCESS_KEY → $MINIO_ACCESS_KEY in .gitlab-ci.yml Network policies: - ci-pipeline: add HTTP/80, registry/5000, webservice/8181 egress rules DNS & Tailscale proxy cleanup: - Remove ci, prometheus, monitor DNS records (no longer exposed) - Rename s3 → minio DNS record - Remove Argo UI, Prometheus, monitor nginx server blocks - Remove argo-htpasswd volume mount - Tailscale proxy nodeSelector: infra → platform Terraform cleanup: - Delete infra/modules/registry/ (SCW CR namespace) - Delete infra/modules/object-storage/ (SCW S3 buckets) - Delete infra/modules/secrets/ (SCW secrets) - Delete corresponding live configs - TF state backend: S3 → GitLab HTTP Argo workflows: - Add events/ (GitLab push eventsource + ci-pipeline sensor) - ci-pipeline + training templates: SCW → internal registry - Delete obsolete compile-training-template.yaml Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
70 lines
2.4 KiB
YAML
70 lines
2.4 KiB
YAML
# GitLab Runner — 2×H100 GPU training workloads (multi-GPU hyperopt)
|
||
# Runs on ci-training-h100x2 pool (2× H100 PCIe: 160GB total VRAM)
|
||
# Multi-GPU auto-detected by MultiGpuConfig::detect() in ml crate.
|
||
|
||
gitlabUrl: http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181
|
||
|
||
replicas: 1
|
||
|
||
rbac:
|
||
create: false
|
||
serviceAccount:
|
||
create: false
|
||
name: gitlab-runner
|
||
|
||
nodeSelector:
|
||
k8s.scaleway.com/pool-name: infra
|
||
|
||
runners:
|
||
cloneUrl: http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181
|
||
config: |
|
||
[[runners]]
|
||
clone_url = "http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181"
|
||
tag_list = ["kapsule", "h100x2"]
|
||
[runners.kubernetes]
|
||
namespace = "foxhunt"
|
||
service_account = "gitlab-runner"
|
||
image = "rust:1.89-slim"
|
||
privileged = false
|
||
node_selector_overwrite_allowed = ".*"
|
||
cpu_request_overwrite_max_allowed = "16000m"
|
||
cpu_limit_overwrite_max_allowed = "16000m"
|
||
memory_request_overwrite_max_allowed = "192Gi"
|
||
memory_limit_overwrite_max_allowed = "192Gi"
|
||
poll_timeout = 600
|
||
runtime_class_name = "nvidia"
|
||
pod_annotations_overwrite_allowed = ".*"
|
||
cpu_request = "4000m"
|
||
cpu_limit = "8000m"
|
||
memory_request = "8Gi"
|
||
memory_limit = "16Gi"
|
||
helper_cpu_request = "100m"
|
||
helper_cpu_limit = "500m"
|
||
helper_memory_request = "128Mi"
|
||
helper_memory_limit = "512Mi"
|
||
image_pull_secrets = ["gitlab-registry"]
|
||
[runners.kubernetes.node_selector]
|
||
"k8s.scaleway.com/pool-name" = "ci-training-h100x2"
|
||
[runners.kubernetes.node_tolerations]
|
||
"nvidia.com/gpu" = "NoSchedule"
|
||
"node.cilium.io/agent-not-ready" = "NoSchedule"
|
||
[runners.kubernetes.pod_labels]
|
||
"app.kubernetes.io/part-of" = "foxhunt-ci"
|
||
# Request both GPUs via K8s scheduler for exclusive access
|
||
[[runners.kubernetes.pod_spec]]
|
||
name = "build"
|
||
patch_type = "strategic"
|
||
patch = '{"containers":[{"name":"build","resources":{"requests":{"nvidia.com/gpu":"2"},"limits":{"nvidia.com/gpu":"2"}}}]}'
|
||
[[runners.kubernetes.volumes.pvc]]
|
||
name = "training-data-h100x2-pvc"
|
||
mount_path = "/mnt/training-data"
|
||
read_only = true
|
||
[[runners.kubernetes.volumes.pvc]]
|
||
name = "sccache-h100x2-pvc"
|
||
mount_path = "/mnt/sccache"
|
||
read_only = false
|
||
|
||
tags: "kapsule,h100x2"
|
||
|
||
concurrent: 2
|