b31329931f
- Remove all HTTPS/TLS from MinIO (plain HTTP for internal cluster traffic) - Fix sccache 0% cache hit rate (rustls rejected self-signed MinIO cert) - Remove hardcoded URLs from k8s_dispatcher.rs (S3_ENDPOINT, TRAINING_RUNTIME_IMAGE, CALLBACK_ENDPOINT now required env vars) - Update GitLab registry S3 credentials to HTTP endpoint - Fix PVC manifest (20Gi → 100Gi to match cluster) - Fix nodeSelector: infra/foxhunt → platform (match actual node pool) - Fix rclone trailing backslash causing chmod to be parsed as rclone args - Remove minio-ca-cert ConfigMap references from all manifests - Update trading-service GPU overlay to l40s pool 20 files changed, -118 lines net Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
87 lines
3.5 KiB
YAML
87 lines
3.5 KiB
YAML
# GitLab Runner — Kubernetes executor
|
|
# Runner manager pod lives on gitlab node pool
|
|
# Default: build pods spawn on ci-compile-cpu pool (POP2-32C-128G)
|
|
# CI jobs override node_selector for their target pool
|
|
|
|
gitlabUrl: http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181
|
|
# runnerToken set via --set at install time
|
|
|
|
replicas: 1
|
|
|
|
nodeSelector:
|
|
k8s.scaleway.com/pool-name: platform
|
|
|
|
runners:
|
|
# Override clone URL to internal service (pods can't reach Tailscale IPs)
|
|
cloneUrl: http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181
|
|
config: |
|
|
[[runners]]
|
|
clone_url = "http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181"
|
|
tag_list = ["kapsule", "rust", "docker", "gpu"]
|
|
[runners.kubernetes]
|
|
namespace = "foxhunt"
|
|
image = "rust:1.89-slim"
|
|
privileged = false
|
|
# Allow CI jobs to override node selector via KUBERNETES_NODE_SELECTOR_* vars
|
|
# Format: KUBERNETES_NODE_SELECTOR_<LABEL>: "key=value"
|
|
node_selector_overwrite_allowed = ".*"
|
|
# Allow CI jobs to override resources via KUBERNETES_*_{REQUEST,LIMIT} vars
|
|
# Values are maximums (not regexes) — training can request up to 80Gi
|
|
cpu_request_overwrite_max_allowed = "24000m"
|
|
cpu_limit_overwrite_max_allowed = "24000m"
|
|
memory_request_overwrite_max_allowed = "80Gi"
|
|
memory_limit_overwrite_max_allowed = "80Gi"
|
|
# Scale-to-zero pools need ~3-5 min to provision; default 180s times out
|
|
poll_timeout = 600
|
|
# Allow CI jobs to override runtimeClassName via KUBERNETES_RUNTIME_CLASS_NAME
|
|
# GPU jobs set "nvidia"; compile/kaniko jobs leave unset (default runc)
|
|
runtime_class_name_overwrite_allowed = ".*"
|
|
# Allow CI jobs to set pod annotations (training jobs expose Prometheus metrics)
|
|
pod_annotations_overwrite_allowed = ".*"
|
|
# Default resource limits for ci-compile-cpu (POP2-32C-128G)
|
|
cpu_request = "3500m"
|
|
cpu_limit = "7800m"
|
|
memory_request = "12Gi"
|
|
memory_limit = "28Gi"
|
|
helper_cpu_request = "100m"
|
|
helper_cpu_limit = "500m"
|
|
helper_memory_request = "128Mi"
|
|
helper_memory_limit = "512Mi"
|
|
image_pull_secrets = ["gitlab-registry"]
|
|
# Sub-tables must come AFTER all scalar values (TOML rule)
|
|
[runners.kubernetes.node_selector]
|
|
"k8s.scaleway.com/pool-name" = "ci-compile-cpu"
|
|
[runners.kubernetes.node_tolerations]
|
|
"nvidia.com/gpu" = "NoSchedule"
|
|
# Cilium CNI agent takes ~30s to initialize on fresh scale-from-zero nodes
|
|
"node.cilium.io/agent-not-ready" = "NoSchedule"
|
|
[runners.kubernetes.pod_labels]
|
|
"app.kubernetes.io/part-of" = "foxhunt-ci"
|
|
# Mount training data PVC (Databento futures .dbn.zst files)
|
|
# Path must NOT be under /data — Redis image's WORKDIR is /data and entrypoint chowns it
|
|
[[runners.kubernetes.volumes.pvc]]
|
|
name = "training-data-pvc"
|
|
mount_path = "/mnt/training-data"
|
|
read_only = true
|
|
|
|
# Runner tags for job matching
|
|
tags: "kapsule,rust,docker,gpu"
|
|
|
|
# Concurrency — 10 allows all 9 Kaniko builds + test to run in parallel
|
|
concurrent: 10
|
|
|
|
# RBAC for runner to spawn pods
|
|
rbac:
|
|
create: true
|
|
rules:
|
|
- apiGroups: [""]
|
|
resources: ["pods", "pods/exec", "pods/log", "secrets", "configmaps"]
|
|
verbs: ["get", "list", "watch", "create", "delete", "update", "patch"]
|
|
- apiGroups: [""]
|
|
resources: ["pods/attach"]
|
|
verbs: ["create", "get"]
|
|
|
|
serviceAccount:
|
|
create: true
|
|
name: gitlab-runner
|