From 1bc4e7e10a9ef5809aa4b38286943ae1b95d4397 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Tue, 24 Feb 2026 23:16:50 +0100 Subject: [PATCH] docs: GitLab CE migration implementation plan (12 tasks, 5 phases) Covers: Terragrunt node pools, object storage buckets, Postgres init, GitLab CE Helm deploy, Runner with K8s executor, .gitlab-ci.yml translation (Kaniko builds), manifest cutover, decommission steps. Co-Authored-By: Claude Opus 4.6 --- ...6-02-24-gitlab-migration-implementation.md | 1288 +++++++++++++++++ 1 file changed, 1288 insertions(+) create mode 100644 docs/plans/2026-02-24-gitlab-migration-implementation.md diff --git a/docs/plans/2026-02-24-gitlab-migration-implementation.md b/docs/plans/2026-02-24-gitlab-migration-implementation.md new file mode 100644 index 000000000..ed14dbbab --- /dev/null +++ b/docs/plans/2026-02-24-gitlab-migration-implementation.md @@ -0,0 +1,1288 @@ +# GitLab CE Migration Implementation Plan + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Migrate from Gitea to GitLab CE running in Kapsule with K8s-native CI (scale-to-zero build pods), GitLab built-in registry, and external Postgres/Redis. + +**Architecture:** GitLab CE deployed via Helm on a dedicated `gitlab` node pool (DEV1-L). GitLab Runner with Kubernetes executor spawns ephemeral build pods on a `ci-build` pool (GP1-XS, 0-2). External Postgres and Redis on existing always-on pool. Kaniko for unprivileged Docker builds. Tailscale-only access. + +**Tech Stack:** Terraform/Terragrunt (Scaleway provider), Helm 3, GitLab CE Helm chart, GitLab Runner Helm chart, Kaniko, Scaleway Kapsule, Scaleway Object Storage (S3-compatible) + +**Design doc:** `docs/plans/2026-02-24-gitlab-migration-design.md` + +--- + +## Phase 1: Infrastructure (Terragrunt) + +### Task 1: Add `gitlab` and `ci-build` node pools to Kapsule module + +**Files:** +- Modify: `infra/modules/kapsule/variables.tf` +- Modify: `infra/modules/kapsule/main.tf` +- Modify: `infra/modules/kapsule/outputs.tf` +- Modify: `infra/live/production/kapsule/terragrunt.hcl` + +**Step 1: Add variables for new pools** + +Append to `infra/modules/kapsule/variables.tf`: + +```hcl +variable "enable_gitlab_pool" { + description = "Create dedicated node pool for GitLab CE" + type = bool + default = false +} + +variable "gitlab_type" { + description = "Instance type for the GitLab node pool" + type = string + default = "DEV1-L" +} + +variable "enable_ci_build_pool" { + description = "Create dedicated node pool for CI build pods" + type = bool + default = false +} + +variable "ci_build_type" { + description = "Instance type for the CI build node pool" + type = string + default = "GP1-XS" +} + +variable "ci_build_max_size" { + description = "Maximum number of nodes in the CI build pool" + type = number + default = 2 +} +``` + +**Step 2: Add node pool resources** + +Append to `infra/modules/kapsule/main.tf` (after the `gpu_inference` resource): + +```hcl +# GitLab CE node pool (dedicated DEV1-L for GitLab + Runner manager) +resource "scaleway_k8s_pool" "gitlab" { + count = var.enable_gitlab_pool ? 1 : 0 + cluster_id = scaleway_k8s_cluster.foxhunt.id + name = "gitlab" + node_type = var.gitlab_type + size = 1 + min_size = 1 + max_size = 1 + autoscaling = false + autohealing = true + region = var.region +} + +# CI build pod pool (scale-to-zero for ephemeral GitLab Runner build pods) +resource "scaleway_k8s_pool" "ci_build" { + count = var.enable_ci_build_pool ? 1 : 0 + cluster_id = scaleway_k8s_cluster.foxhunt.id + name = "ci-build" + node_type = var.ci_build_type + size = 0 + min_size = 0 + max_size = var.ci_build_max_size + autoscaling = true + autohealing = true + region = var.region + + lifecycle { + ignore_changes = [size] + } +} +``` + +**Step 3: Add outputs** + +Append to `infra/modules/kapsule/outputs.tf`: + +```hcl +output "gitlab_pool_id" { + description = "ID of the GitLab node pool" + value = var.enable_gitlab_pool ? scaleway_k8s_pool.gitlab[0].id : "" +} + +output "ci_build_pool_id" { + description = "ID of the CI build node pool" + value = var.enable_ci_build_pool ? scaleway_k8s_pool.ci_build[0].id : "" +} +``` + +**Step 4: Enable pools in terragrunt inputs** + +Add to `infra/live/production/kapsule/terragrunt.hcl` inputs block: + +```hcl + # GitLab CE pool (dedicated for GitLab + Runner manager) + enable_gitlab_pool = true + gitlab_type = "DEV1-L" + + # CI build pool (ephemeral runner pods, scale-to-zero) + enable_ci_build_pool = true + ci_build_type = "GP1-XS" + ci_build_max_size = 2 +``` + +**Step 5: Validate terraform plan** + +Run: `cd infra/live/production/kapsule && terragrunt plan` + +Expected: Plan shows 2 new resources (`scaleway_k8s_pool.gitlab[0]`, `scaleway_k8s_pool.ci_build[0]`), 0 destroyed. + +**Step 6: Apply** + +Run: `cd infra/live/production/kapsule && terragrunt apply` + +Expected: 2 added. Verify with `kubectl get nodes` — new `gitlab` node should appear within ~3 minutes. + +**Step 7: Commit** + +```bash +git add infra/modules/kapsule/ infra/live/production/kapsule/terragrunt.hcl +git commit -m "infra(kapsule): add gitlab and ci-build node pools" +``` + +--- + +### Task 2: Add GitLab object storage buckets + +**Files:** +- Modify: `infra/modules/object-storage/main.tf` +- Modify: `infra/modules/object-storage/variables.tf` +- Modify: `infra/modules/object-storage/outputs.tf` (create if missing) + +**Step 1: Add bucket resources** + +Append to `infra/modules/object-storage/main.tf`: + +```hcl +resource "scaleway_object_bucket" "gitlab_registry" { + name = "${var.bucket_name_prefix}-gitlab-registry" + region = var.region + + versioning { + enabled = false + } +} + +resource "scaleway_object_bucket" "gitlab_artifacts" { + name = "${var.bucket_name_prefix}-gitlab-artifacts" + region = var.region + + lifecycle_rule { + enabled = true + prefix = "tmp/" + + expiration { + days = 7 + } + } + + versioning { + enabled = false + } +} +``` + +**Step 2: Add variable** + +Append to `infra/modules/object-storage/variables.tf`: + +```hcl +variable "bucket_name_prefix" { + description = "Prefix for additional buckets (e.g., foxhunt)" + type = string + default = "foxhunt" +} +``` + +**Step 3: Create outputs file** + +Create `infra/modules/object-storage/outputs.tf`: + +```hcl +output "artifacts_bucket_name" { + description = "Name of the main artifacts bucket" + value = scaleway_object_bucket.artifacts.name +} + +output "gitlab_registry_bucket_name" { + description = "Name of the GitLab registry bucket" + value = scaleway_object_bucket.gitlab_registry.name +} + +output "gitlab_artifacts_bucket_name" { + description = "Name of the GitLab artifacts bucket" + value = scaleway_object_bucket.gitlab_artifacts.name +} +``` + +**Step 4: Validate and apply** + +Run: `cd infra/live/production/object-storage && terragrunt plan` + +Expected: 2 new buckets, 0 changes to existing `artifacts` bucket. + +Run: `cd infra/live/production/object-storage && terragrunt apply` + +**Step 5: Commit** + +```bash +git add infra/modules/object-storage/ +git commit -m "infra(storage): add GitLab registry and artifacts buckets" +``` + +--- + +### Task 3: Create GitLab database and user in Postgres + +**Files:** +- Create: `infra/k8s/gitlab/postgres-init.yaml` (one-time Job) + +**Step 1: Create the init job manifest** + +Create `infra/k8s/gitlab/postgres-init.yaml`: + +```yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: gitlab-postgres-init + namespace: foxhunt + labels: + app.kubernetes.io/name: gitlab-postgres-init + app.kubernetes.io/part-of: foxhunt +spec: + ttlSecondsAfterFinished: 300 + template: + spec: + nodeSelector: + k8s.scaleway.com/pool-name: always-on + restartPolicy: Never + containers: + - name: init + image: timescale/timescaledb:latest-pg16 + command: + - bash + - -c + - | + set -euo pipefail + export PGPASSWORD="$POSTGRES_PASSWORD" + + # Wait for postgres to be ready + until pg_isready -h postgres -U foxhunt; do + echo "Waiting for postgres..." + sleep 2 + done + + # Create gitlab database and user + psql -h postgres -U foxhunt -d foxhunt -c " + DO \$\$ + BEGIN + IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = 'gitlab') THEN + CREATE ROLE gitlab WITH LOGIN PASSWORD '$GITLAB_DB_PASSWORD'; + END IF; + END + \$\$; + " + psql -h postgres -U foxhunt -c " + SELECT 'CREATE DATABASE gitlab OWNER gitlab' + WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'gitlab') + \gexec" + psql -h postgres -U foxhunt -d gitlab -c "CREATE EXTENSION IF NOT EXISTS pg_trgm;" + psql -h postgres -U foxhunt -d gitlab -c "CREATE EXTENSION IF NOT EXISTS btree_gist;" + psql -h postgres -U foxhunt -d gitlab -c "GRANT ALL PRIVILEGES ON DATABASE gitlab TO gitlab;" + psql -h postgres -U foxhunt -d gitlab -c "GRANT ALL ON SCHEMA public TO gitlab;" + + echo "GitLab database initialized successfully." + env: + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: foxhunt-secrets + key: db-password + - name: GITLAB_DB_PASSWORD + valueFrom: + secretKeyRef: + name: gitlab-secrets + key: db-password +``` + +**Step 2: Create GitLab secrets** + +Generate a password and create the secret (run locally, NOT in a file): + +```bash +GITLAB_DB_PASS=$(openssl rand -base64 32) +kubectl -n foxhunt create secret generic gitlab-secrets \ + --from-literal=db-password="$GITLAB_DB_PASS" \ + --from-literal=root-password="$(openssl rand -base64 24)" \ + --from-literal=runner-token="" \ + --dry-run=client -o yaml | kubectl apply -f - +echo "GitLab DB password: $GITLAB_DB_PASS" +``` + +Save the `GITLAB_DB_PASS` value — needed for Helm values in Task 4. + +**Step 3: Run the init job** + +```bash +kubectl apply -f infra/k8s/gitlab/postgres-init.yaml +kubectl -n foxhunt wait --for=condition=complete job/gitlab-postgres-init --timeout=60s +kubectl -n foxhunt logs job/gitlab-postgres-init +``` + +Expected: "GitLab database initialized successfully." + +**Step 4: Verify** + +```bash +kubectl -n foxhunt exec deployment/postgres -- psql -U foxhunt -c "\l" | grep gitlab +``` + +Expected: Row showing `gitlab` database owned by `gitlab`. + +**Step 5: Commit** + +```bash +git add infra/k8s/gitlab/ +git commit -m "infra(gitlab): postgres init job for GitLab database" +``` + +--- + +## Phase 2: GitLab CE Deployment + +### Task 4: Create GitLab CE Helm values and deploy + +**Files:** +- Create: `infra/k8s/gitlab/values.yaml` + +**Step 1: Add GitLab Helm repo** + +```bash +helm repo add gitlab https://charts.gitlab.io/ +helm repo update +``` + +**Step 2: Create Helm values file** + +Create `infra/k8s/gitlab/values.yaml`: + +```yaml +# GitLab CE - Foxhunt deployment +# Runs on dedicated 'gitlab' node pool (DEV1-L) +# External Postgres + Redis on always-on pool + +global: + edition: ce + hosts: + domain: fxhnt.ai + gitlab: + name: gitlab.fxhnt.ai + registry: + name: gitlab.fxhnt.ai + ingress: + enabled: false + configureCertmanager: false + shell: + port: 2222 + registry: + bucket: foxhunt-gitlab-registry + appConfig: + lfs: + bucket: foxhunt-gitlab-artifacts + connection: + secret: gitlab-s3-credentials + key: connection + artifacts: + bucket: foxhunt-gitlab-artifacts + connection: + secret: gitlab-s3-credentials + key: connection + uploads: + bucket: foxhunt-gitlab-artifacts + connection: + secret: gitlab-s3-credentials + key: connection + packages: + bucket: foxhunt-gitlab-artifacts + connection: + secret: gitlab-s3-credentials + key: connection + +# Disable bundled databases — use external +postgresql: + install: false +redis: + install: false + +# External Postgres (existing pod on always-on pool) +global.psql: + host: postgres.foxhunt.svc.cluster.local + port: 5432 + database: gitlab + username: gitlab + password: + secret: gitlab-secrets + key: db-password + +# External Redis (existing pod on always-on pool, DB index 8) +global.redis: + host: redis.foxhunt.svc.cluster.local + port: 6379 + password: + enabled: false + # Use DB index 8 to avoid collision with application data + # Set via extraEnv on individual components + +# Disable components we don't need +certmanager: + install: false +nginx-ingress: + enabled: false +prometheus: + install: false +grafana: + install: false +gitlab-runner: + install: false # We deploy runner separately with K8s executor config +minio: + install: false + +# Registry — S3-backed +registry: + storage: + secret: gitlab-s3-credentials + key: registry + nodeSelector: + k8s.scaleway.com/pool-name: gitlab + tolerations: + - key: gitlab + operator: Equal + value: "true" + effect: NoSchedule + +# GitLab core components — all on gitlab node pool +gitlab: + webservice: + replicaCount: 1 + workerProcesses: 2 + nodeSelector: + k8s.scaleway.com/pool-name: gitlab + tolerations: + - key: gitlab + operator: Equal + value: "true" + effect: NoSchedule + resources: + requests: + cpu: 500m + memory: 2Gi + limits: + cpu: 2000m + memory: 4Gi + extraEnv: + REDIS_URL: "redis://redis.foxhunt.svc.cluster.local:6379/8" + + sidekiq: + replicas: 1 + concurrency: 10 + nodeSelector: + k8s.scaleway.com/pool-name: gitlab + tolerations: + - key: gitlab + operator: Equal + value: "true" + effect: NoSchedule + resources: + requests: + cpu: 250m + memory: 1Gi + limits: + cpu: 1000m + memory: 2Gi + extraEnv: + REDIS_URL: "redis://redis.foxhunt.svc.cluster.local:6379/8" + + gitaly: + persistence: + enabled: true + size: 50Gi + nodeSelector: + k8s.scaleway.com/pool-name: gitlab + tolerations: + - key: gitlab + operator: Equal + value: "true" + effect: NoSchedule + resources: + requests: + cpu: 200m + memory: 512Mi + limits: + cpu: 1000m + memory: 2Gi + + gitlab-shell: + nodeSelector: + k8s.scaleway.com/pool-name: gitlab + tolerations: + - key: gitlab + operator: Equal + value: "true" + effect: NoSchedule + service: + type: NodePort + nodePort: 32222 + + toolbox: + nodeSelector: + k8s.scaleway.com/pool-name: gitlab + tolerations: + - key: gitlab + operator: Equal + value: "true" + effect: NoSchedule + + kas: + enabled: false + + gitlab-exporter: + enabled: false +``` + +**Step 3: Create S3 credentials secret** + +The GitLab Helm chart expects S3 credentials in a specific format. Create the secret (run locally): + +```bash +# S3 connection config for GitLab (Scaleway Object Storage) +cat <<'CONN_EOF' > /tmp/gitlab-s3-connection.yaml +provider: AWS +aws_access_key_id: "" +aws_secret_access_key: "" +region: fr-par +endpoint: "https://s3.fr-par.scw.cloud" +path_style: true +CONN_EOF + +# Registry-specific S3 config +cat <<'REG_EOF' > /tmp/gitlab-s3-registry.yaml +s3: + accesskey: "" + secretkey: "" + region: fr-par + regionendpoint: "https://s3.fr-par.scw.cloud" + bucket: foxhunt-gitlab-registry +REG_EOF + +# Replace placeholders with real values from ~/.config/scw/config.yaml +# Then create the secret: +kubectl -n foxhunt create secret generic gitlab-s3-credentials \ + --from-file=connection=/tmp/gitlab-s3-connection.yaml \ + --from-file=registry=/tmp/gitlab-s3-registry.yaml \ + --dry-run=client -o yaml | kubectl apply -f - + +rm /tmp/gitlab-s3-connection.yaml /tmp/gitlab-s3-registry.yaml +``` + +**Step 4: Taint the gitlab node** + +Once the gitlab node pool is running (from Task 1): + +```bash +# Find the gitlab node name +GITLAB_NODE=$(kubectl get nodes -l k8s.scaleway.com/pool-name=gitlab -o jsonpath='{.items[0].metadata.name}') +kubectl taint nodes "$GITLAB_NODE" gitlab=true:NoSchedule --overwrite +``` + +**Step 5: Deploy GitLab CE** + +```bash +helm upgrade --install gitlab gitlab/gitlab \ + --namespace foxhunt \ + --values infra/k8s/gitlab/values.yaml \ + --timeout 10m \ + --wait +``` + +Expected: All GitLab pods running on the gitlab node. This may take 5-8 minutes. + +**Step 6: Verify** + +```bash +kubectl -n foxhunt get pods -l app.kubernetes.io/part-of=gitlab -o wide +``` + +Expected: webservice, sidekiq, gitaly, gitlab-shell, toolbox pods all `Running` on the gitlab node. + +**Step 7: Expose GitLab via Tailscale** + +GitLab webservice is a ClusterIP service. Access it via the Tailscale subnet router (already advertising `10.42.0.0/16` pod CIDR): + +```bash +# Get the webservice ClusterIP +kubectl -n foxhunt get svc -l app.kubernetes.io/name=webservice +``` + +Alternatively, create a NodePort service or use `kubectl port-forward` for initial testing: + +```bash +kubectl -n foxhunt port-forward svc/gitlab-webservice-default 8443:8443 +# Then access https://localhost:8443 +``` + +**Step 8: Add DNS record** + +```bash +# Get the gitlab node's Tailscale-routable IP (via subnet router) +# Or use the webservice ClusterIP if the subnet router is advertising pod CIDR +scw dns record add fxhnt.ai name=gitlab type=A data= ttl=300 +``` + +**Step 9: Set root password and create project** + +Access `https://gitlab.fxhnt.ai` and: +1. Log in with `root` / the password from `gitlab-secrets` `root-password` +2. Create group `foxhunt` +3. Create project `foxhunt/foxhunt` +4. Go to Settings → CI/CD → Runners and note the registration token (for Task 6) + +**Step 10: Commit** + +```bash +git add infra/k8s/gitlab/values.yaml +git commit -m "infra(gitlab): Helm values for GitLab CE deployment" +``` + +--- + +### Task 5: Push-mirror repository from Gitea to GitLab + +**No files to modify — operational step.** + +**Step 1: Add GitLab as a remote** + +```bash +git remote add gitlab ssh://git@gitlab.fxhnt.ai:2222/foxhunt/foxhunt.git +``` + +**Step 2: Push mirror** + +```bash +git push gitlab --mirror +``` + +Expected: All branches, tags, and refs pushed. Output shows `main` and all tags. + +**Step 3: Verify on GitLab** + +Navigate to `https://gitlab.fxhnt.ai/foxhunt/foxhunt` and verify: +- Full commit history present +- All branches visible +- File tree matches Gitea + +--- + +### Task 6: Deploy GitLab Runner with Kubernetes executor + +**Files:** +- Create: `infra/k8s/gitlab/runner-values.yaml` + +**Step 1: Create runner Helm values** + +Create `infra/k8s/gitlab/runner-values.yaml`: + +```yaml +# GitLab Runner — Kubernetes executor targeting ci-build pool +# Runner manager pod lives on gitlab node pool +# Build pods spawn on ci-build pool (scale-to-zero) + +gitlabUrl: https://gitlab.fxhnt.ai +# runnerToken set via --set at install time + +replicas: 1 + +nodeSelector: + k8s.scaleway.com/pool-name: gitlab +tolerations: + - key: gitlab + operator: Equal + value: "true" + effect: NoSchedule + +runners: + config: | + [[runners]] + [runners.kubernetes] + namespace = "foxhunt" + image = "rust:1.89-slim" + privileged = false + + # Build pods on ci-build pool + [runners.kubernetes.node_selector] + "k8s.scaleway.com/pool-name" = "ci-build" + [[runners.kubernetes.node_tolerations]] + key = "ci-build" + operator = "Equal" + value = "true" + effect = "NoSchedule" + + # Resource limits for build pods + cpu_request = "2000m" + cpu_limit = "6000m" + memory_request = "4Gi" + memory_limit = "12Gi" + + # Helper container + helper_cpu_request = "100m" + helper_cpu_limit = "500m" + helper_memory_request = "128Mi" + helper_memory_limit = "512Mi" + + # Service containers (for kaniko sidecar) + [runners.kubernetes.pod_labels] + "app.kubernetes.io/part-of" = "foxhunt-ci" + + # Runner tags for job matching + tags: "kapsule,rust,docker" + + # Concurrency +concurrent: 4 + +# RBAC for runner to spawn pods +rbac: + create: true + rules: + - apiGroups: [""] + resources: ["pods", "pods/exec", "pods/log", "secrets", "configmaps"] + verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] + - apiGroups: [""] + resources: ["pods/attach"] + verbs: ["create", "get"] + +serviceAccount: + create: true + name: gitlab-runner +``` + +**Step 2: Register and deploy runner** + +Get the runner registration token from GitLab UI (Settings → CI/CD → Runners), then: + +```bash +helm upgrade --install gitlab-runner gitlab/gitlab-runner \ + --namespace foxhunt \ + --values infra/k8s/gitlab/runner-values.yaml \ + --set runnerToken="" \ + --timeout 5m \ + --wait +``` + +**Step 3: Verify runner is registered** + +```bash +kubectl -n foxhunt get pods -l app=gitlab-runner +``` + +Expected: 1 runner manager pod `Running` on the gitlab node. + +Also verify in GitLab UI: Settings → CI/CD → Runners → should show 1 runner online. + +**Step 4: Commit** + +```bash +git add infra/k8s/gitlab/runner-values.yaml +git commit -m "infra(gitlab): Runner Helm values with K8s executor for ci-build pool" +``` + +--- + +## Phase 3: CI Pipeline + +### Task 7: Translate Gitea Actions workflow to `.gitlab-ci.yml` + +**Files:** +- Create: `.gitlab-ci.yml` +- Reference: `.gitea/workflows/ci.yaml` (existing, for translation) + +**Step 1: Create `.gitlab-ci.yml`** + +Create `.gitlab-ci.yml` at the repo root: + +```yaml +# GitLab CI/CD — Foxhunt +# Runs on ci-build pool (GP1-XS, scale-to-zero) +# Uses sccache for Rust build caching, Kaniko for Docker builds + +stages: + - check + - test + - build + - deploy + +variables: + SQLX_OFFLINE: "true" + CARGO_TERM_COLOR: always + SCCACHE_BUCKET: $SCCACHE_BUCKET + SCCACHE_ENDPOINT: $SCCACHE_ENDPOINT + SCCACHE_S3_USE_SSL: "true" + AWS_ACCESS_KEY_ID: $SCW_ACCESS_KEY + AWS_SECRET_ACCESS_KEY: $SCW_SECRET_KEY + RUSTC_WRAPPER: /usr/local/bin/sccache + REGISTRY: ${CI_REGISTRY_IMAGE} + +# Base template for Rust jobs +.rust-base: + image: rust:1.89-slim + tags: + - kapsule + - rust + before_script: + - apt-get update && apt-get install -y protobuf-compiler curl pkg-config libssl-dev + - curl -fsSL https://github.com/mozilla/sccache/releases/download/v0.8.1/sccache-v0.8.1-x86_64-unknown-linux-musl.tar.gz + | tar xz --strip-components=1 -C /usr/local/bin sccache-v0.8.1-x86_64-unknown-linux-musl/sccache + - chmod +x /usr/local/bin/sccache + - rustup component add clippy + +# -------------------------------------------------------------------------- +# Stage 1: cargo check + clippy +# -------------------------------------------------------------------------- +check: + extends: .rust-base + stage: check + script: + - cargo check --workspace + - cargo clippy --workspace -- -D warnings + +# -------------------------------------------------------------------------- +# Stage 2: tests +# -------------------------------------------------------------------------- +test: + extends: .rust-base + stage: test + needs: [check] + script: + - cargo test --workspace --lib + +# -------------------------------------------------------------------------- +# Stage 3: Build + push images (main only, Kaniko) +# -------------------------------------------------------------------------- +.kaniko-base: + stage: build + needs: [test] + image: + name: gcr.io/kaniko-project/executor:debug + entrypoint: [""] + tags: + - kapsule + rules: + - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push" + before_script: + # Authenticate to GitLab container registry + - mkdir -p /kaniko/.docker + - echo "{\"auths\":{\"${CI_REGISTRY}\":{\"auth\":\"$(printf '%s:%s' "${CI_REGISTRY_USER}" "${CI_REGISTRY_PASSWORD}" | base64 | tr -d '\n')\"}}}" > /kaniko/.docker/config.json + +build-trading-service: + extends: .kaniko-base + script: + - /kaniko/executor + --context "${CI_PROJECT_DIR}" + --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.service" + --build-arg SERVICE=trading_service + --build-arg SCCACHE_BUCKET=${SCCACHE_BUCKET} + --build-arg AWS_ACCESS_KEY_ID=${SCW_ACCESS_KEY} + --build-arg AWS_SECRET_ACCESS_KEY=${SCW_SECRET_KEY} + --build-arg SCCACHE_ENDPOINT=${SCCACHE_ENDPOINT} + --destination "${REGISTRY}/trading_service:${CI_COMMIT_SHA}" + --destination "${REGISTRY}/trading_service:latest" + +build-api-gateway: + extends: .kaniko-base + script: + - /kaniko/executor + --context "${CI_PROJECT_DIR}" + --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.service" + --build-arg SERVICE=api_gateway + --build-arg SCCACHE_BUCKET=${SCCACHE_BUCKET} + --build-arg AWS_ACCESS_KEY_ID=${SCW_ACCESS_KEY} + --build-arg AWS_SECRET_ACCESS_KEY=${SCW_SECRET_KEY} + --build-arg SCCACHE_ENDPOINT=${SCCACHE_ENDPOINT} + --destination "${REGISTRY}/api_gateway:${CI_COMMIT_SHA}" + --destination "${REGISTRY}/api_gateway:latest" + +build-broker-gateway: + extends: .kaniko-base + script: + - /kaniko/executor + --context "${CI_PROJECT_DIR}" + --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.service" + --build-arg SERVICE=broker_gateway_service + --build-arg SCCACHE_BUCKET=${SCCACHE_BUCKET} + --build-arg AWS_ACCESS_KEY_ID=${SCW_ACCESS_KEY} + --build-arg AWS_SECRET_ACCESS_KEY=${SCW_SECRET_KEY} + --build-arg SCCACHE_ENDPOINT=${SCCACHE_ENDPOINT} + --destination "${REGISTRY}/broker_gateway_service:${CI_COMMIT_SHA}" + --destination "${REGISTRY}/broker_gateway_service:latest" + +build-ml-training: + extends: .kaniko-base + script: + - /kaniko/executor + --context "${CI_PROJECT_DIR}" + --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.service" + --build-arg SERVICE=ml_training_service + --build-arg SCCACHE_BUCKET=${SCCACHE_BUCKET} + --build-arg AWS_ACCESS_KEY_ID=${SCW_ACCESS_KEY} + --build-arg AWS_SECRET_ACCESS_KEY=${SCW_SECRET_KEY} + --build-arg SCCACHE_ENDPOINT=${SCCACHE_ENDPOINT} + --destination "${REGISTRY}/ml_training_service:${CI_COMMIT_SHA}" + --destination "${REGISTRY}/ml_training_service:latest" + +build-backtesting: + extends: .kaniko-base + script: + - /kaniko/executor + --context "${CI_PROJECT_DIR}" + --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.service" + --build-arg SERVICE=backtesting_service + --build-arg SCCACHE_BUCKET=${SCCACHE_BUCKET} + --build-arg AWS_ACCESS_KEY_ID=${SCW_ACCESS_KEY} + --build-arg AWS_SECRET_ACCESS_KEY=${SCW_SECRET_KEY} + --build-arg SCCACHE_ENDPOINT=${SCCACHE_ENDPOINT} + --destination "${REGISTRY}/backtesting_service:${CI_COMMIT_SHA}" + --destination "${REGISTRY}/backtesting_service:latest" + +build-trading-agent: + extends: .kaniko-base + script: + - /kaniko/executor + --context "${CI_PROJECT_DIR}" + --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.service" + --build-arg SERVICE=trading_agent_service + --build-arg SCCACHE_BUCKET=${SCCACHE_BUCKET} + --build-arg AWS_ACCESS_KEY_ID=${SCW_ACCESS_KEY} + --build-arg AWS_SECRET_ACCESS_KEY=${SCW_SECRET_KEY} + --build-arg SCCACHE_ENDPOINT=${SCCACHE_ENDPOINT} + --destination "${REGISTRY}/trading_agent_service:${CI_COMMIT_SHA}" + --destination "${REGISTRY}/trading_agent_service:latest" + +build-web-gateway: + extends: .kaniko-base + script: + - /kaniko/executor + --context "${CI_PROJECT_DIR}" + --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.web-gateway" + --destination "${REGISTRY}/web-gateway:${CI_COMMIT_SHA}" + --destination "${REGISTRY}/web-gateway:latest" + +build-training: + extends: .kaniko-base + script: + - /kaniko/executor + --context "${CI_PROJECT_DIR}" + --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.training" + --build-arg SCCACHE_BUCKET=${SCCACHE_BUCKET} + --build-arg AWS_ACCESS_KEY_ID=${SCW_ACCESS_KEY} + --build-arg AWS_SECRET_ACCESS_KEY=${SCW_SECRET_KEY} + --build-arg SCCACHE_ENDPOINT=${SCCACHE_ENDPOINT} + --destination "${REGISTRY}/training:${CI_COMMIT_SHA}" + --destination "${REGISTRY}/training:latest" + +# -------------------------------------------------------------------------- +# Stage 4: Deploy to Kapsule (main only) +# -------------------------------------------------------------------------- +deploy: + stage: deploy + image: bitnami/kubectl:latest + tags: + - kapsule + needs: + - build-trading-service + - build-api-gateway + - build-broker-gateway + - build-ml-training + - build-backtesting + - build-trading-agent + - build-web-gateway + - build-training + rules: + - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push" + before_script: + - mkdir -p ~/.kube + - echo "$KUBECONFIG_DATA" | base64 -d > ~/.kube/config + - chmod 600 ~/.kube/config + script: + - | + for svc in trading-service api-gateway broker-gateway-service ml-training-service backtesting-service trading-agent-service; do + kubectl -n foxhunt set image deployment/${svc} ${svc}=${REGISTRY}/${svc}:${CI_COMMIT_SHA} || true + done + kubectl -n foxhunt set image deployment/web-gateway web-gateway=${REGISTRY}/web-gateway:${CI_COMMIT_SHA} || true + - | + for svc in trading-service api-gateway broker-gateway-service ml-training-service backtesting-service trading-agent-service web-gateway; do + kubectl -n foxhunt rollout status deployment/${svc} --timeout=300s || true + done +``` + +**Step 2: Configure CI/CD variables in GitLab UI** + +Go to GitLab → foxhunt/foxhunt → Settings → CI/CD → Variables. Add: + +| Key | Value | Type | Protected | Masked | +|-----|-------|------|-----------|--------| +| `SCW_ACCESS_KEY` | (from `~/.config/scw/config.yaml`) | Variable | Yes | Yes | +| `SCW_SECRET_KEY` | (from `~/.config/scw/config.yaml`) | Variable | Yes | Yes | +| `SCCACHE_BUCKET` | `foxhunt-artifacts` | Variable | No | No | +| `SCCACHE_ENDPOINT` | `https://s3.fr-par.scw.cloud` | Variable | No | No | +| `KUBECONFIG_DATA` | (base64 of `~/.kube/foxhunt-fr-par.yaml`) | Variable | Yes | Yes | + +**Step 3: Push and test pipeline** + +```bash +git add .gitlab-ci.yml +git commit -m "ci: add GitLab CI/CD pipeline (check, test, build, deploy)" +git push gitlab main +``` + +**Step 4: Monitor pipeline** + +Go to GitLab → foxhunt/foxhunt → CI/CD → Pipelines. Verify: +- `check` job runs, passes (cargo check + clippy) +- `test` job runs, passes (cargo test --workspace --lib) +- `build-*` jobs run in parallel (8 Kaniko builds) +- `deploy` job updates K8s deployments + +Watch for ci-build node autoscaling: +```bash +kubectl get nodes -w # Should see ci-build node appear, then disappear after ~10min idle +``` + +**Step 5: Commit is already done in Step 3.** + +--- + +## Phase 4: Cutover + +### Task 8: Update K8s manifests for GitLab registry + +**Files:** +- Modify: `infra/k8s/services/api-gateway.yaml` +- Modify: `infra/k8s/services/trading-service.yaml` +- Modify: `infra/k8s/services/trading-service-gpu.yaml` +- Modify: `infra/k8s/services/ml-training-service.yaml` +- Modify: `infra/k8s/services/ml-training-service-gpu.yaml` +- Modify: `infra/k8s/services/broker-gateway.yaml` +- Modify: `infra/k8s/services/backtesting-service.yaml` +- Modify: `infra/k8s/services/trading-agent-service.yaml` +- Modify: `infra/k8s/services/web-gateway.yaml` + +**Step 1: Create GitLab registry pull secret** + +Create a GitLab deploy token first (GitLab UI → Settings → Repository → Deploy Tokens): +- Name: `k8s-deploy` +- Scopes: `read_registry` + +Then create the K8s secret: + +```bash +kubectl -n foxhunt create secret docker-registry gitlab-registry \ + --docker-server=gitlab.fxhnt.ai:5050 \ + --docker-username= \ + --docker-password= \ + --dry-run=client -o yaml | kubectl apply -f - +``` + +**Step 2: Update all service manifests** + +In every service YAML file, make two changes: + +1. Replace `imagePullSecrets`: +```yaml +# OLD +imagePullSecrets: + - name: scw-registry + +# NEW +imagePullSecrets: + - name: gitlab-registry +``` + +2. Replace image references: +```yaml +# OLD +image: rg.fr-par.scw.cloud/foxhunt/:latest + +# NEW +image: gitlab.fxhnt.ai:5050/foxhunt/foxhunt/:latest +``` + +Service name mappings (container image name → manifest): +| Manifest | Old image | New image | +|----------|-----------|-----------| +| api-gateway.yaml | `rg.fr-par.scw.cloud/foxhunt/api_gateway` | `gitlab.fxhnt.ai:5050/foxhunt/foxhunt/api_gateway` | +| trading-service.yaml | `rg.fr-par.scw.cloud/foxhunt/trading_service` | `gitlab.fxhnt.ai:5050/foxhunt/foxhunt/trading_service` | +| trading-service-gpu.yaml | `rg.fr-par.scw.cloud/foxhunt/trading_service` | `gitlab.fxhnt.ai:5050/foxhunt/foxhunt/trading_service` | +| ml-training-service.yaml | `rg.fr-par.scw.cloud/foxhunt/ml_training_service` | `gitlab.fxhnt.ai:5050/foxhunt/foxhunt/ml_training_service` | +| ml-training-service-gpu.yaml | `rg.fr-par.scw.cloud/foxhunt/ml_training_service` | `gitlab.fxhnt.ai:5050/foxhunt/foxhunt/ml_training_service` | +| broker-gateway.yaml | `rg.fr-par.scw.cloud/foxhunt/broker_gateway_service` | `gitlab.fxhnt.ai:5050/foxhunt/foxhunt/broker_gateway_service` | +| backtesting-service.yaml | `rg.fr-par.scw.cloud/foxhunt/backtesting_service` | `gitlab.fxhnt.ai:5050/foxhunt/foxhunt/backtesting_service` | +| trading-agent-service.yaml | `rg.fr-par.scw.cloud/foxhunt/trading_agent_service` | `gitlab.fxhnt.ai:5050/foxhunt/foxhunt/trading_agent_service` | +| web-gateway.yaml | `rg.fr-par.scw.cloud/foxhunt/web-gateway` | `gitlab.fxhnt.ai:5050/foxhunt/foxhunt/web-gateway` | + +**Step 3: Apply updated manifests** + +```bash +kubectl apply -f infra/k8s/services/ +``` + +**Step 4: Verify all pods can pull from GitLab registry** + +```bash +kubectl -n foxhunt get pods +# All pods should be Running, no ImagePullBackOff errors +``` + +**Step 5: Commit** + +```bash +git add infra/k8s/services/ +git commit -m "infra(k8s): switch image refs from Scaleway registry to GitLab registry" +``` + +--- + +### Task 9: Switch git remote to GitLab + +**No files to create — operational step.** + +**Step 1: Update origin remote** + +```bash +git remote set-url origin ssh://git@gitlab.fxhnt.ai:2222/foxhunt/foxhunt.git +git remote remove gitlab # Remove the temporary mirror remote +``` + +**Step 2: Verify push works** + +```bash +git push origin main +``` + +Expected: Push succeeds, GitLab shows new commit. + +**Step 3: Verify full CI cycle** + +Push triggers pipeline → check → test → build → deploy. Verify services update with new images. + +--- + +## Phase 5: Decommission + +### Task 10: Destroy CI runner infrastructure + +**Step 1: Destroy the CI runner VM** + +```bash +cd infra/live/production/ci-runner && terragrunt destroy +``` + +Expected: GP1-S instance and associated resources destroyed. + +**Step 2: Remove ci-runner module from repo** + +```bash +rm -rf infra/modules/ci-runner/ +rm -rf infra/live/production/ci-runner/ +``` + +**Step 3: Remove Gitea workflow** + +```bash +rm -rf .gitea/ +``` + +**Step 4: Commit** + +```bash +git add -A +git commit -m "infra: decommission Gitea CI runner and remove Gitea Actions workflow" +git push origin main +``` + +--- + +### Task 11: Decommission Gitea VM + +**Operational step — verify everything works on GitLab first.** + +**Step 1: Final verification checklist** + +- [ ] GitLab UI accessible at `https://gitlab.fxhnt.ai` +- [ ] Git push/pull works via SSH +- [ ] CI pipeline runs: check → test → build → deploy +- [ ] Build pods scale up on ci-build pool, then scale to zero +- [ ] Images pushed to GitLab registry +- [ ] K8s pods pull images from GitLab registry +- [ ] Deploy stage updates K8s deployments successfully + +**Step 2: Decommission Gitea (once all checks pass)** + +```bash +# Stop Gitea service +ssh root@vm-fxhnt-git "systemctl stop gitea" + +# Remove auto-start watcher (was deployed by ci-runner terraform) +ssh root@vm-fxhnt-git "systemctl stop ci-watcher.timer && systemctl disable ci-watcher.timer" + +# Delete Scaleway instance (via CLI, since it wasn't Terragrunt-managed) +scw instance server terminate 8e8f0ac7-3745-412e-9db2-f288e2938035 with-ip=true with-block=local +``` + +**Step 3: Remove DNS record** + +```bash +scw dns record delete fxhnt.ai name=git type=A data=100.92.231.92 +``` + +**Step 4: Optionally delete Scaleway container registry** + +```bash +# Only after confirming no running pods reference old images +scw registry namespace delete e8b31a6b-0dba-442f-8b34-23be2b3a3e52 +``` + +--- + +### Task 12: Update documentation and memory + +**Files:** +- Modify: `CLAUDE.md` (update Git remote, CI references) +- Modify: Memory files + +**Step 1: Update CLAUDE.md** + +Find and replace: +- `git.fxhnt.ai` → `gitlab.fxhnt.ai` where referencing git remote +- Remove references to Gitea, act_runner, ci-runner VM +- Update CI pipeline description from Gitea Actions to GitLab CI + +**Step 2: Update memory files** + +Update `/home/jgrusewski/.claude/projects/-home-jgrusewski-Work-foxhunt/memory/infrastructure.md`: +- Replace Gitea section with GitLab section +- Update CI runner section (K8s executor instead of VM) +- Update registry info (GitLab instead of Scaleway) +- Add gitlab node pool to Kapsule section + +**Step 3: Commit** + +```bash +git add CLAUDE.md +git commit -m "docs: update references for GitLab CE migration" +git push origin main +```