266 lines
9.7 KiB
YAML
266 lines
9.7 KiB
YAML
# GitLab CI/CD — Foxhunt
|
|
# Runs on ci-build pool (GP1-XS, scale-to-zero)
|
|
# CI builder image: CUDA 12.4 + Rust 1.89 + protoc + sccache (hosted on Scaleway CR)
|
|
# Service images: pushed to internal GitLab registry (Kaniko)
|
|
|
|
stages:
|
|
- prepare
|
|
- check
|
|
- test
|
|
- build
|
|
- deploy
|
|
|
|
variables:
|
|
SQLX_OFFLINE: "true"
|
|
CARGO_TERM_COLOR: always
|
|
SCCACHE_BUCKET: $SCCACHE_BUCKET
|
|
SCCACHE_ENDPOINT: $SCCACHE_ENDPOINT
|
|
SCCACHE_S3_USE_SSL: "true"
|
|
AWS_ACCESS_KEY_ID: $SCW_ACCESS_KEY
|
|
AWS_SECRET_ACCESS_KEY: $SCW_SECRET_KEY
|
|
RUSTC_WRAPPER: /usr/local/bin/sccache
|
|
# CI builder image on Scaleway Container Registry (reachable by Kapsule nodes)
|
|
CI_BUILDER_IMAGE: rg.fr-par.scw.cloud/foxhunt-ci/ci-builder:latest
|
|
# Internal GitLab registry (HTTP, reachable from pods only — for service images)
|
|
INTERNAL_REGISTRY: gitlab-registry.foxhunt.svc.cluster.local:5000
|
|
REGISTRY: ${INTERNAL_REGISTRY}/root/foxhunt
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Stage 0: Build CI builder image → push to Scaleway CR
|
|
# --------------------------------------------------------------------------
|
|
build-ci-builder:
|
|
stage: prepare
|
|
image:
|
|
name: gcr.io/kaniko-project/executor:debug
|
|
entrypoint: [""]
|
|
tags:
|
|
- kapsule
|
|
- docker
|
|
rules:
|
|
- changes:
|
|
- infra/docker/Dockerfile.ci-builder
|
|
when: on_success
|
|
- when: manual
|
|
allow_failure: true
|
|
before_script:
|
|
- mkdir -p /kaniko/.docker
|
|
- echo "{\"auths\":{\"rg.fr-par.scw.cloud\":{\"username\":\"nologin\",\"password\":\"${SCW_SECRET_KEY}\"}}}" > /kaniko/.docker/config.json
|
|
script:
|
|
- /kaniko/executor
|
|
--context "${CI_PROJECT_DIR}"
|
|
--dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.ci-builder"
|
|
--destination "${CI_BUILDER_IMAGE}"
|
|
|
|
# Base template for Rust jobs — pre-baked CI builder from Scaleway CR
|
|
# Image pre-exists in SCR; rebuild via build-ci-builder when Dockerfile changes
|
|
.rust-base:
|
|
image: ${CI_BUILDER_IMAGE}
|
|
tags:
|
|
- kapsule
|
|
- rust
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Stage 1: cargo check + clippy
|
|
# --------------------------------------------------------------------------
|
|
check:
|
|
extends: .rust-base
|
|
stage: check
|
|
script:
|
|
- sccache --zero-stats || true
|
|
- cargo check --workspace
|
|
- cargo clippy --workspace -- -D warnings
|
|
- sccache --show-stats || true
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Stage 2: tests
|
|
# --------------------------------------------------------------------------
|
|
test:
|
|
extends: .rust-base
|
|
stage: test
|
|
needs: [check]
|
|
script:
|
|
- cargo test --workspace --lib
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Stage 3: Build + push service images (main only, Kaniko → GitLab registry)
|
|
# --------------------------------------------------------------------------
|
|
.kaniko-base:
|
|
stage: build
|
|
needs: [test]
|
|
image:
|
|
name: gcr.io/kaniko-project/executor:debug
|
|
entrypoint: [""]
|
|
tags:
|
|
- kapsule
|
|
- docker
|
|
rules:
|
|
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
|
|
before_script:
|
|
- mkdir -p /kaniko/.docker
|
|
- echo "{\"auths\":{\"${INTERNAL_REGISTRY}\":{\"username\":\"${CI_REGISTRY_USER}\",\"password\":\"${CI_REGISTRY_PASSWORD}\"}}}" > /kaniko/.docker/config.json
|
|
|
|
build-trading-service:
|
|
extends: .kaniko-base
|
|
script:
|
|
- /kaniko/executor
|
|
--context "${CI_PROJECT_DIR}"
|
|
--dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.service"
|
|
--insecure-registry "${INTERNAL_REGISTRY}"
|
|
--build-arg SERVICE=trading_service
|
|
--build-arg SCCACHE_BUCKET=${SCCACHE_BUCKET}
|
|
--build-arg AWS_ACCESS_KEY_ID=${SCW_ACCESS_KEY}
|
|
--build-arg AWS_SECRET_ACCESS_KEY=${SCW_SECRET_KEY}
|
|
--build-arg SCCACHE_ENDPOINT=${SCCACHE_ENDPOINT}
|
|
--destination "${REGISTRY}/trading_service:${CI_COMMIT_SHA}"
|
|
--destination "${REGISTRY}/trading_service:latest"
|
|
|
|
build-api-gateway:
|
|
extends: .kaniko-base
|
|
script:
|
|
- /kaniko/executor
|
|
--context "${CI_PROJECT_DIR}"
|
|
--dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.service"
|
|
--insecure-registry "${INTERNAL_REGISTRY}"
|
|
--build-arg SERVICE=api_gateway
|
|
--build-arg SCCACHE_BUCKET=${SCCACHE_BUCKET}
|
|
--build-arg AWS_ACCESS_KEY_ID=${SCW_ACCESS_KEY}
|
|
--build-arg AWS_SECRET_ACCESS_KEY=${SCW_SECRET_KEY}
|
|
--build-arg SCCACHE_ENDPOINT=${SCCACHE_ENDPOINT}
|
|
--destination "${REGISTRY}/api_gateway:${CI_COMMIT_SHA}"
|
|
--destination "${REGISTRY}/api_gateway:latest"
|
|
|
|
build-broker-gateway:
|
|
extends: .kaniko-base
|
|
script:
|
|
- /kaniko/executor
|
|
--context "${CI_PROJECT_DIR}"
|
|
--dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.service"
|
|
--insecure-registry "${INTERNAL_REGISTRY}"
|
|
--build-arg SERVICE=broker_gateway_service
|
|
--build-arg SCCACHE_BUCKET=${SCCACHE_BUCKET}
|
|
--build-arg AWS_ACCESS_KEY_ID=${SCW_ACCESS_KEY}
|
|
--build-arg AWS_SECRET_ACCESS_KEY=${SCW_SECRET_KEY}
|
|
--build-arg SCCACHE_ENDPOINT=${SCCACHE_ENDPOINT}
|
|
--destination "${REGISTRY}/broker_gateway_service:${CI_COMMIT_SHA}"
|
|
--destination "${REGISTRY}/broker_gateway_service:latest"
|
|
|
|
build-ml-training:
|
|
extends: .kaniko-base
|
|
script:
|
|
- /kaniko/executor
|
|
--context "${CI_PROJECT_DIR}"
|
|
--dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.service"
|
|
--insecure-registry "${INTERNAL_REGISTRY}"
|
|
--build-arg SERVICE=ml_training_service
|
|
--build-arg SCCACHE_BUCKET=${SCCACHE_BUCKET}
|
|
--build-arg AWS_ACCESS_KEY_ID=${SCW_ACCESS_KEY}
|
|
--build-arg AWS_SECRET_ACCESS_KEY=${SCW_SECRET_KEY}
|
|
--build-arg SCCACHE_ENDPOINT=${SCCACHE_ENDPOINT}
|
|
--destination "${REGISTRY}/ml_training_service:${CI_COMMIT_SHA}"
|
|
--destination "${REGISTRY}/ml_training_service:latest"
|
|
|
|
build-backtesting:
|
|
extends: .kaniko-base
|
|
script:
|
|
- /kaniko/executor
|
|
--context "${CI_PROJECT_DIR}"
|
|
--dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.service"
|
|
--insecure-registry "${INTERNAL_REGISTRY}"
|
|
--build-arg SERVICE=backtesting_service
|
|
--build-arg SCCACHE_BUCKET=${SCCACHE_BUCKET}
|
|
--build-arg AWS_ACCESS_KEY_ID=${SCW_ACCESS_KEY}
|
|
--build-arg AWS_SECRET_ACCESS_KEY=${SCW_SECRET_KEY}
|
|
--build-arg SCCACHE_ENDPOINT=${SCCACHE_ENDPOINT}
|
|
--destination "${REGISTRY}/backtesting_service:${CI_COMMIT_SHA}"
|
|
--destination "${REGISTRY}/backtesting_service:latest"
|
|
|
|
build-trading-agent:
|
|
extends: .kaniko-base
|
|
script:
|
|
- /kaniko/executor
|
|
--context "${CI_PROJECT_DIR}"
|
|
--dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.service"
|
|
--insecure-registry "${INTERNAL_REGISTRY}"
|
|
--build-arg SERVICE=trading_agent_service
|
|
--build-arg SCCACHE_BUCKET=${SCCACHE_BUCKET}
|
|
--build-arg AWS_ACCESS_KEY_ID=${SCW_ACCESS_KEY}
|
|
--build-arg AWS_SECRET_ACCESS_KEY=${SCW_SECRET_KEY}
|
|
--build-arg SCCACHE_ENDPOINT=${SCCACHE_ENDPOINT}
|
|
--destination "${REGISTRY}/trading_agent_service:${CI_COMMIT_SHA}"
|
|
--destination "${REGISTRY}/trading_agent_service:latest"
|
|
|
|
build-data-acquisition:
|
|
extends: .kaniko-base
|
|
script:
|
|
- /kaniko/executor
|
|
--context "${CI_PROJECT_DIR}"
|
|
--dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.service"
|
|
--insecure-registry "${INTERNAL_REGISTRY}"
|
|
--build-arg SERVICE=data_acquisition_service
|
|
--build-arg SCCACHE_BUCKET=${SCCACHE_BUCKET}
|
|
--build-arg AWS_ACCESS_KEY_ID=${SCW_ACCESS_KEY}
|
|
--build-arg AWS_SECRET_ACCESS_KEY=${SCW_SECRET_KEY}
|
|
--build-arg SCCACHE_ENDPOINT=${SCCACHE_ENDPOINT}
|
|
--destination "${REGISTRY}/data_acquisition_service:${CI_COMMIT_SHA}"
|
|
--destination "${REGISTRY}/data_acquisition_service:latest"
|
|
|
|
build-web-gateway:
|
|
extends: .kaniko-base
|
|
script:
|
|
- /kaniko/executor
|
|
--context "${CI_PROJECT_DIR}"
|
|
--dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.web-gateway"
|
|
--insecure-registry "${INTERNAL_REGISTRY}"
|
|
--destination "${REGISTRY}/web-gateway:${CI_COMMIT_SHA}"
|
|
--destination "${REGISTRY}/web-gateway:latest"
|
|
|
|
build-training:
|
|
extends: .kaniko-base
|
|
script:
|
|
- /kaniko/executor
|
|
--context "${CI_PROJECT_DIR}"
|
|
--dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.training"
|
|
--insecure-registry "${INTERNAL_REGISTRY}"
|
|
--build-arg SCCACHE_BUCKET=${SCCACHE_BUCKET}
|
|
--build-arg AWS_ACCESS_KEY_ID=${SCW_ACCESS_KEY}
|
|
--build-arg AWS_SECRET_ACCESS_KEY=${SCW_SECRET_KEY}
|
|
--build-arg SCCACHE_ENDPOINT=${SCCACHE_ENDPOINT}
|
|
--destination "${REGISTRY}/training:${CI_COMMIT_SHA}"
|
|
--destination "${REGISTRY}/training:latest"
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Stage 4: Deploy to Kapsule (main only)
|
|
# --------------------------------------------------------------------------
|
|
deploy:
|
|
stage: deploy
|
|
image: bitnami/kubectl:latest
|
|
tags:
|
|
- kapsule
|
|
needs:
|
|
- build-trading-service
|
|
- build-api-gateway
|
|
- build-broker-gateway
|
|
- build-ml-training
|
|
- build-backtesting
|
|
- build-trading-agent
|
|
- build-data-acquisition
|
|
- build-web-gateway
|
|
- build-training
|
|
rules:
|
|
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
|
|
before_script:
|
|
- mkdir -p ~/.kube
|
|
- echo "$KUBECONFIG_B64" | base64 -d > ~/.kube/config
|
|
- chmod 600 ~/.kube/config
|
|
environment:
|
|
name: production
|
|
kubernetes:
|
|
namespace: foxhunt
|
|
script:
|
|
# Update images to this commit's build
|
|
- |
|
|
for svc in trading_service api_gateway broker_gateway_service ml_training_service backtesting_service trading_agent_service data_acquisition_service; do
|
|
kubectl -n foxhunt set image deployment/${svc} ${svc}=${REGISTRY}/${svc}:${CI_COMMIT_SHA} || true
|
|
done
|
|
kubectl -n foxhunt set image deployment/web-gateway web-gateway=${REGISTRY}/web-gateway:${CI_COMMIT_SHA} || true
|