gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
f565a039b607069da8bf11ab64afd3c3b8166efb
foxhunt/.gitlab-ci.yml
jgrusewski f565a039b6 ci: trigger rebuild for TFT zero-dim fix 
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-26 16:02:14 +01:00

586 lines
21 KiB
YAML
Raw Blame History

# GitLab CI/CD — Foxhunt
# Runs on ci-build pool (L4 or H100: scale-to-zero)
# CI builder image: CUDA 12.4 + Rust 1.89 + protoc + sccache (hosted on Scaleway CR)
# Service images: pushed to Scaleway Container Registry (Kaniko)
stages:
- prepare
- check
- test
- compile
- build
- deploy
variables:
SQLX_OFFLINE: "true"
CARGO_TERM_COLOR: always
# CUDA_COMPUTE_CAP is set at runtime by detect_gpu() — L4=89, H100=90
# Do NOT hardcode here; sccache is partitioned by compute cap below
# Stable JWT secret for api_gateway tests (prevents env-var race between parallel tests)
JWT_SECRET: "CiTestSecret_Kx7mP9nR2sW5vY8bC3fG6jH1kL4pQ7tZ0uN9dM5eV8xS2wT6yA4zB_64chars_min"
# Training data on block storage PVC (mounted at /mnt/training-data by runner)
FOXHUNT_DATA_DIR: "/mnt/training-data/futures-baseline"
# sccache base dir on PVC — GPU stages override to /mnt/sccache/sm_<cap>
SCCACHE_DIR: "/mnt/sccache"
RUSTC_WRAPPER: /usr/local/bin/sccache
# S3 credentials for Kaniko registry auth and Dockerfile builds
AWS_ACCESS_KEY_ID: $SCW_ACCESS_KEY
AWS_SECRET_ACCESS_KEY: $SCW_SECRET_KEY
# CI builder image on Scaleway Container Registry (reachable by Kapsule nodes)
CI_BUILDER_IMAGE: rg.fr-par.scw.cloud/foxhunt-ci/ci-builder:latest
# Scaleway Container Registry for service images
REGISTRY: rg.fr-par.scw.cloud/foxhunt-ci
# IaC runner image on Scaleway Container Registry
INFRA_RUNNER_IMAGE: rg.fr-par.scw.cloud/foxhunt-ci/infra-runner:latest
# --------------------------------------------------------------------------
# Stage 0: Build CI builder image → push to Scaleway CR
# --------------------------------------------------------------------------
build-ci-builder:
stage: prepare
image:
name: gcr.io/kaniko-project/executor:debug
entrypoint: [""]
tags:
- kapsule
- docker
rules:
- if: $CI_PIPELINE_SOURCE == "push"
changes:
- infra/docker/Dockerfile.ci-builder
when: on_success
- when: manual
allow_failure: true
before_script:
- mkdir -p /kaniko/.docker
- >-
echo "{\"auths\":{
\"rg.fr-par.scw.cloud\":{\"username\":\"nologin\",\"password\":\"${SCW_SECRET_KEY}\"},
\"https://index.docker.io/v1/\":{\"username\":\"${DOCKERHUB_USERNAME}\",\"password\":\"${DOCKERHUB_TOKEN}\"}
}}" > /kaniko/.docker/config.json
script:
- /kaniko/executor
--context "${CI_PROJECT_DIR}"
--dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.ci-builder"
--cache=true --cache-repo="${REGISTRY}/cache"
--destination "${CI_BUILDER_IMAGE}"
# --------------------------------------------------------------------------
# Stage 0b: Build devcontainer image → push to internal GitLab registry
# --------------------------------------------------------------------------
build-devcontainer:
stage: prepare
image:
name: gcr.io/kaniko-project/executor:debug
entrypoint: [""]
tags:
- kapsule
- docker
rules:
- if: $CI_PIPELINE_SOURCE == "push"
changes:
- .devcontainer/**
when: on_success
- when: manual
allow_failure: true
before_script:
- mkdir -p /kaniko/.docker
- >-
echo "{\"auths\":{
\"rg.fr-par.scw.cloud\":{\"username\":\"nologin\",\"password\":\"${SCW_SECRET_KEY}\"},
\"https://index.docker.io/v1/\":{\"username\":\"${DOCKERHUB_USERNAME}\",\"password\":\"${DOCKERHUB_TOKEN}\"}
}}" > /kaniko/.docker/config.json
script:
- /kaniko/executor
--context "${CI_PROJECT_DIR}"
--dockerfile "${CI_PROJECT_DIR}/.devcontainer/Dockerfile"
--cache=true --cache-repo="${REGISTRY}/cache"
--destination "rg.fr-par.scw.cloud/foxhunt-ci/devcontainer:${CI_COMMIT_SHA}"
--destination "rg.fr-par.scw.cloud/foxhunt-ci/devcontainer:latest"
# --------------------------------------------------------------------------
# Stage 0c: Build infra-runner image → push to Scaleway CR
# --------------------------------------------------------------------------
build-infra-runner:
stage: prepare
image:
name: gcr.io/kaniko-project/executor:debug
entrypoint: [""]
tags:
- kapsule
- docker
rules:
- if: $CI_PIPELINE_SOURCE == "push"
changes:
- infra/docker/Dockerfile.infra-runner
when: on_success
- when: manual
allow_failure: true
before_script:
- mkdir -p /kaniko/.docker
- >-
echo "{\"auths\":{
\"rg.fr-par.scw.cloud\":{\"username\":\"nologin\",\"password\":\"${SCW_SECRET_KEY}\"},
\"https://index.docker.io/v1/\":{\"username\":\"${DOCKERHUB_USERNAME}\",\"password\":\"${DOCKERHUB_TOKEN}\"}
}}" > /kaniko/.docker/config.json
script:
- /kaniko/executor
--context "${CI_PROJECT_DIR}"
--dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.infra-runner"
--cache=true --cache-repo="${REGISTRY}/cache"
--destination "${INFRA_RUNNER_IMAGE}"
# Base template for Rust jobs — pre-baked CI builder from Scaleway CR
# Image pre-exists in SCR; rebuild via build-ci-builder when Dockerfile changes
.rust-base:
image: ${CI_BUILDER_IMAGE}
tags:
- kapsule
- rust
# --------------------------------------------------------------------------
# Stage 1: cargo check + clippy (SKIPPED — test stage already compiles)
# Uncomment the script lines below to re-enable check+clippy before tests.
# --------------------------------------------------------------------------
check:
extends: .rust-base
stage: check
needs: []
rules:
- if: $CI_PIPELINE_SOURCE == "push"
changes:
- Cargo.toml
- Cargo.lock
- crates/**
- bin/**
- services/**
- testing/**
- infra/docker/Dockerfile.service
- infra/docker/Dockerfile.web-gateway
- infra/docker/Dockerfile.training
- infra/docker/Dockerfile.training-runtime
- .gitlab-ci.yml
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
changes:
- Cargo.toml
- Cargo.lock
- crates/**
- bin/**
- services/**
- testing/**
script:
- echo "check+clippy skipped — test stage compiles the workspace"
# - sccache --zero-stats || true
# - cargo check --workspace
# - cargo clippy --workspace -- -D warnings
# - sccache --show-stats || true
# --------------------------------------------------------------------------
# Stage 2: tests
# --------------------------------------------------------------------------
test:
extends: .rust-base
stage: test
needs: [check] # check is a fast no-op; keeps stage ordering
rules:
- if: $CI_PIPELINE_SOURCE == "push"
changes:
- Cargo.toml
- Cargo.lock
- crates/**
- bin/**
- services/**
- testing/**
- infra/docker/Dockerfile.service
- infra/docker/Dockerfile.web-gateway
- infra/docker/Dockerfile.training
- infra/docker/Dockerfile.training-runtime
- .gitlab-ci.yml
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
changes:
- Cargo.toml
- Cargo.lock
- crates/**
- bin/**
- services/**
- testing/**
services:
- name: redis:7-alpine
alias: redis
script:
# Remove CUDA stubs from LD_LIBRARY_PATH — nvidia RuntimeClass provides real libcuda.so
- export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//')
- nvidia-smi # verify GPU access
# Detect GPU compute capability and partition sccache per architecture
- export CUDA_COMPUTE_CAP=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader | head -1 | tr -d '.')
- echo "Detected CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP"
- export SCCACHE_DIR="/mnt/sccache/sm_${CUDA_COMPUTE_CAP}"
- mkdir -p "$SCCACHE_DIR"
# Wait for Redis sidecar (K8s executor doesn't wait for service readiness)
- |
for i in $(seq 1 30); do
if bash -c 'echo PING > /dev/tcp/localhost/6379' 2>/dev/null; then
echo "Redis ready"; break
fi
echo "Waiting for Redis... ($i/30)"; sleep 1
done
- sccache --zero-stats || true
- cargo test --workspace --lib -- --skip model_loader::tests
- sccache --show-stats || true
# --------------------------------------------------------------------------
# Stage 3a: Compile all service binaries (single job, PVC sccache)
# --------------------------------------------------------------------------
compile-services:
extends: .rust-base
stage: compile
needs: [test]
rules:
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
changes:
- Cargo.toml
- Cargo.lock
- crates/**
- bin/**
- services/**
- testing/**
- infra/docker/Dockerfile.service
- infra/docker/Dockerfile.web-gateway
- infra/docker/Dockerfile.training
- infra/docker/Dockerfile.training-runtime
- .gitlab-ci.yml
script:
# Detect GPU compute capability and partition sccache per architecture
- export CUDA_COMPUTE_CAP=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader | head -1 | tr -d '.')
- echo "Detected CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP"
- export SCCACHE_DIR="/mnt/sccache/sm_${CUDA_COMPUTE_CAP}"
- mkdir -p "$SCCACHE_DIR"
- sccache --zero-stats || true
# 1) Build service binaries (no CUDA feature)
- cargo build --release
-p trading_service
-p api_gateway
-p broker_gateway_service
-p ml_training_service
-p backtesting_service
-p trading_agent_service
-p data_acquisition_service
-p web-gateway
# 2) Build training binaries with CUDA (sccache already warm from step 1)
- cargo build --release -p ml --features ml/cuda
--example train_baseline_rl
--example train_baseline_supervised
--example evaluate_baseline
--example hyperopt_baseline_rl
--example hyperopt_baseline_supervised
# 3) Build training uploader sidecar (no CUDA needed)
- cargo build --release -p training_uploader
- sccache --show-stats || true
- mkdir -p build-out
- |
for bin in trading_service api_gateway broker_gateway_service ml_training_service backtesting_service trading_agent_service data_acquisition_service web-gateway; do
cp target/release/$bin build-out/
strip build-out/$bin
done
for bin in train_baseline_rl train_baseline_supervised evaluate_baseline hyperopt_baseline_rl hyperopt_baseline_supervised; do
cp target/release/examples/$bin build-out/
strip build-out/$bin
done
cp target/release/training_uploader build-out/
strip build-out/training_uploader
- ls -lh build-out/
artifacts:
paths:
- build-out/
expire_in: 1 hour
# --------------------------------------------------------------------------
# Stage 3b: Package service images (Kaniko → Scaleway CR, uses pre-built binaries)
# --------------------------------------------------------------------------
.kaniko-base:
stage: build
needs: [compile-services]
image:
name: gcr.io/kaniko-project/executor:debug
entrypoint: [""]
tags:
- kapsule
- docker
rules:
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
changes:
- Cargo.toml
- Cargo.lock
- crates/**
- bin/**
- services/**
- testing/**
- infra/docker/Dockerfile.service
- infra/docker/Dockerfile.runtime
- infra/docker/Dockerfile.web-gateway
- infra/docker/Dockerfile.web-gateway-runtime
- infra/docker/Dockerfile.training
- infra/docker/Dockerfile.training-runtime
- .gitlab-ci.yml
before_script:
- mkdir -p /kaniko/.docker
- >-
echo "{\"auths\":{
\"rg.fr-par.scw.cloud\":{\"username\":\"nologin\",\"password\":\"${SCW_SECRET_KEY}\"},
\"https://index.docker.io/v1/\":{\"username\":\"${DOCKERHUB_USERNAME}\",\"password\":\"${DOCKERHUB_TOKEN}\"}
}}" > /kaniko/.docker/config.json
# 7 services: pre-built binary → Dockerfile.runtime (no Rust toolchain, ~30s each)
build-trading-service:
extends: .kaniko-base
script:
- /kaniko/executor
--context dir://${CI_PROJECT_DIR}/build-out
--dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.runtime"
--build-arg SERVICE=trading_service
--destination "${REGISTRY}/trading_service:${CI_COMMIT_SHA}"
--destination "${REGISTRY}/trading_service:latest"
build-api-gateway:
extends: .kaniko-base
script:
- /kaniko/executor
--context dir://${CI_PROJECT_DIR}/build-out
--dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.runtime"
--build-arg SERVICE=api_gateway
--destination "${REGISTRY}/api_gateway:${CI_COMMIT_SHA}"
--destination "${REGISTRY}/api_gateway:latest"
build-broker-gateway:
extends: .kaniko-base
script:
- /kaniko/executor
--context dir://${CI_PROJECT_DIR}/build-out
--dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.runtime"
--build-arg SERVICE=broker_gateway_service
--destination "${REGISTRY}/broker_gateway_service:${CI_COMMIT_SHA}"
--destination "${REGISTRY}/broker_gateway_service:latest"
build-ml-training:
extends: .kaniko-base
script:
- /kaniko/executor
--context dir://${CI_PROJECT_DIR}/build-out
--dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.runtime"
--build-arg SERVICE=ml_training_service
--destination "${REGISTRY}/ml_training_service:${CI_COMMIT_SHA}"
--destination "${REGISTRY}/ml_training_service:latest"
build-backtesting:
extends: .kaniko-base
script:
- /kaniko/executor
--context dir://${CI_PROJECT_DIR}/build-out
--dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.runtime"
--build-arg SERVICE=backtesting_service
--destination "${REGISTRY}/backtesting_service:${CI_COMMIT_SHA}"
--destination "${REGISTRY}/backtesting_service:latest"
build-trading-agent:
extends: .kaniko-base
script:
- /kaniko/executor
--context dir://${CI_PROJECT_DIR}/build-out
--dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.runtime"
--build-arg SERVICE=trading_agent_service
--destination "${REGISTRY}/trading_agent_service:${CI_COMMIT_SHA}"
--destination "${REGISTRY}/trading_agent_service:latest"
build-data-acquisition:
extends: .kaniko-base
script:
- /kaniko/executor
--context dir://${CI_PROJECT_DIR}/build-out
--dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.runtime"
--build-arg SERVICE=data_acquisition_service
--destination "${REGISTRY}/data_acquisition_service:${CI_COMMIT_SHA}"
--destination "${REGISTRY}/data_acquisition_service:latest"
# web-gateway: Node.js dashboard build + pre-built Rust binary
build-web-gateway:
extends: .kaniko-base
script:
- /kaniko/executor
--context "${CI_PROJECT_DIR}"
--dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.web-gateway-runtime"
--cache=true --cache-repo="${REGISTRY}/cache"
--destination "${REGISTRY}/web-gateway:${CI_COMMIT_SHA}"
--destination "${REGISTRY}/web-gateway:latest"
# training: pre-built CUDA binaries → CUDA runtime image (no recompilation)
build-training:
extends: .kaniko-base
script:
- /kaniko/executor
--context dir://${CI_PROJECT_DIR}/build-out
--dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.training-runtime"
--destination "${REGISTRY}/training:${CI_COMMIT_SHA}"
--destination "${REGISTRY}/training:latest"
# --------------------------------------------------------------------------
# IaC: Terragrunt plan on MR (runs on gitlab pool)
# --------------------------------------------------------------------------
infra-plan:
stage: check
image: ${INFRA_RUNNER_IMAGE}
tags:
- kapsule
needs: []
rules:
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
changes:
- infra/**
before_script:
- export SCW_ACCESS_KEY=${SCW_ACCESS_KEY}
- export SCW_SECRET_KEY=${SCW_SECRET_KEY}
- export SCW_DEFAULT_PROJECT_ID=${SCW_DEFAULT_PROJECT_ID}
- export AWS_ACCESS_KEY_ID=${SCW_ACCESS_KEY}
- export AWS_SECRET_ACCESS_KEY=${SCW_SECRET_KEY}
script:
- cd infra/live/production
- terragrunt run-all plan --terragrunt-non-interactive 2>&1 | tee ${CI_PROJECT_DIR}/plan-output.txt
- echo "Plan completed successfully"
artifacts:
paths:
- plan-output.txt
when: always
expire_in: 7 days
# --------------------------------------------------------------------------
# IaC: Terragrunt apply on merge to main (runs on gitlab pool)
# --------------------------------------------------------------------------
infra-apply:
stage: deploy
image: ${INFRA_RUNNER_IMAGE}
tags:
- kapsule
needs: []
rules:
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
changes:
- infra/**
before_script:
- export SCW_ACCESS_KEY=${SCW_ACCESS_KEY}
- export SCW_SECRET_KEY=${SCW_SECRET_KEY}
- export SCW_DEFAULT_PROJECT_ID=${SCW_DEFAULT_PROJECT_ID}
- export AWS_ACCESS_KEY_ID=${SCW_ACCESS_KEY}
- export AWS_SECRET_ACCESS_KEY=${SCW_SECRET_KEY}
script:
- cd infra/live/production
- terragrunt run-all apply --terragrunt-non-interactive -auto-approve
environment:
name: production/infrastructure
# --------------------------------------------------------------------------
# IaC: Weekly drift detection (scheduled pipeline, gitlab pool)
# --------------------------------------------------------------------------
infra-drift-check:
stage: check
image: ${INFRA_RUNNER_IMAGE}
tags:
- kapsule
needs: []
rules:
- if: $CI_PIPELINE_SOURCE == "schedule" && $DRIFT_CHECK == "true"
before_script:
- export SCW_ACCESS_KEY=${SCW_ACCESS_KEY}
- export SCW_SECRET_KEY=${SCW_SECRET_KEY}
- export SCW_DEFAULT_PROJECT_ID=${SCW_DEFAULT_PROJECT_ID}
- export AWS_ACCESS_KEY_ID=${SCW_ACCESS_KEY}
- export AWS_SECRET_ACCESS_KEY=${SCW_SECRET_KEY}
script:
- cd infra/live/production
- |
terragrunt run-all plan -detailed-exitcode --terragrunt-non-interactive \
2>&1 | tee ${CI_PROJECT_DIR}/drift-output.txt; EXIT_CODE=$?
if [ "$EXIT_CODE" -eq 2 ]; then
echo "DRIFT DETECTED — creating GitLab issue"
glab issue create \
--title "IaC Drift Detected ($(date +%Y-%m-%d))" \
--description "$(tail -100 ${CI_PROJECT_DIR}/drift-output.txt)" \
--label "infrastructure,drift"
exit 1
elif [ "$EXIT_CODE" -ne 0 ]; then
exit $EXIT_CODE
fi
echo "No drift detected"
artifacts:
paths:
- drift-output.txt
when: always
expire_in: 7 days
# --------------------------------------------------------------------------
# Stage 4: Deploy to Kapsule (main only)
# --------------------------------------------------------------------------
deploy:
stage: deploy
image: ${INFRA_RUNNER_IMAGE}
tags:
- kapsule
needs:
- build-trading-service
- build-api-gateway
- build-broker-gateway
- build-ml-training
- build-backtesting
- build-trading-agent
- build-data-acquisition
- build-web-gateway
- build-training
rules:
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
before_script:
# Install kubectl if not in image yet (removed once infra-runner is rebuilt)
- which kubectl || (curl -fsSL "https://dl.k8s.io/release/v1.31.4/bin/linux/amd64/kubectl" -o /usr/local/bin/kubectl && chmod +x /usr/local/bin/kubectl)
# scw CLI needs these for kubeconfig exec plugin (get-token)
- export SCW_ACCESS_KEY=${SCW_ACCESS_KEY}
- export SCW_SECRET_KEY=${SCW_SECRET_KEY}
- export SCW_DEFAULT_PROJECT_ID=${SCW_DEFAULT_PROJECT_ID}
- export SCW_DEFAULT_ORGANIZATION_ID=${SCW_DEFAULT_ORGANIZATION_ID:-${SCW_DEFAULT_PROJECT_ID}}
- mkdir -p ~/.kube
- echo "$KUBECONFIG_B64" | base64 -d > ~/.kube/config
- chmod 600 ~/.kube/config
# Rewrite external API URL to in-cluster endpoint (pod can't reach external API due to CIDR overlap)
- sed -i 's|https://.*\.api\.k8s\.fr-par\.scw\.cloud:6443|https://kubernetes.default.svc|g' ~/.kube/config
# Verify cluster connectivity before deploying
- kubectl cluster-info
environment:
name: production
kubernetes:
namespace: foxhunt
script:
# Update images to this commit's build
# Format: image_repo:deployment_name:container_name
- |
SERVICES="
trading_service:trading-service:trading-service
api_gateway:api-gateway:api-gateway
broker_gateway_service:broker-gateway:broker-gateway
ml_training_service:ml-training-service:ml-training-service
backtesting_service:backtesting-service:backtesting-service
trading_agent_service:trading-agent-service:trading-agent-service
web-gateway:web-gateway:web-gateway
"
TOTAL=0
FAILED=0
for entry in $SERVICES; do
IMAGE_REPO=$(echo "$entry" | cut -d: -f1)
DEPLOY_NAME=$(echo "$entry" | cut -d: -f2)
CONTAINER_NAME=$(echo "$entry" | cut -d: -f3)
TOTAL=$((TOTAL + 1))
echo "Deploying ${DEPLOY_NAME} (image: ${IMAGE_REPO}:${CI_COMMIT_SHA})..."
if ! kubectl -n foxhunt set image "deployment/${DEPLOY_NAME}" "${CONTAINER_NAME}=${REGISTRY}/${IMAGE_REPO}:${CI_COMMIT_SHA}"; then
echo "WARNING: failed to update ${DEPLOY_NAME} (deployment may not exist yet)"
FAILED=$((FAILED + 1))
fi
done
echo "Deploy complete: $((TOTAL - FAILED))/${TOTAL} services updated"
# Fail if ALL deployments failed (cluster probably unreachable)
[ "$FAILED" -lt "$TOTAL" ]
Reference in New Issue View Git Blame Copy Permalink