b31329931f
- Remove all HTTPS/TLS from MinIO (plain HTTP for internal cluster traffic) - Fix sccache 0% cache hit rate (rustls rejected self-signed MinIO cert) - Remove hardcoded URLs from k8s_dispatcher.rs (S3_ENDPOINT, TRAINING_RUNTIME_IMAGE, CALLBACK_ENDPOINT now required env vars) - Update GitLab registry S3 credentials to HTTP endpoint - Fix PVC manifest (20Gi → 100Gi to match cluster) - Fix nodeSelector: infra/foxhunt → platform (match actual node pool) - Fix rclone trailing backslash causing chmod to be parsed as rclone args - Remove minio-ca-cert ConfigMap references from all manifests - Update trading-service GPU overlay to l40s pool 20 files changed, -118 lines net Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1816 lines
65 KiB
YAML
1816 lines
65 KiB
YAML
# GitLab CI/CD — Foxhunt
|
||
# Two runners: CPU (compile/build/deploy) and GPU (training)
|
||
# CI builder images:
|
||
# ci-builder: CUDA 12.4 + Rust 1.89 + protoc + sccache (CUDA stubs for compile)
|
||
# ci-builder-cpu: Rust 1.89 + protoc + sccache, no CUDA (for service binaries)
|
||
# Service images: pushed to Scaleway Container Registry (Kaniko)
|
||
#
|
||
# Runner routing (by tags):
|
||
# kapsule,rust → CPU runner → ci-compile-cpu pool (POP2-32C-128G, 32 vCPU)
|
||
# kapsule,l40s → L40S runner → ci-training (L40S 48GB)
|
||
# kapsule,h100 → H100 runner → ci-training-h100 (H100 80GB)
|
||
# Override nodeSelector: KUBERNETES_NODE_SELECTOR_k8s.scaleway.com/pool-name: "<name>" (k=v format)
|
||
# Pools: ci-compile-cpu (POP2-32C-128G), ci-training (L40S), ci-training-h100 (H100), services, gitlab
|
||
# All compilation (services + training) runs on ci-compile-cpu. CUDA stubs in ci-builder image.
|
||
|
||
# DISABLED 2026-03-04: Migrated to Argo Workflows (infra/k8s/argo/).
|
||
# GitLab still serves as git host; Argo Events webhook triggers pipelines.
|
||
# To re-enable: remove the "when: never" rule below and restore the original rules.
|
||
workflow:
|
||
rules:
|
||
- when: never
|
||
# Original rules (disabled):
|
||
# - if: $CI_PIPELINE_SOURCE == "push"
|
||
# - if: $CI_PIPELINE_SOURCE == "merge_request_event"
|
||
# - if: $CI_PIPELINE_SOURCE == "schedule"
|
||
# - if: $CI_PIPELINE_SOURCE == "api"
|
||
# - if: $CI_PIPELINE_SOURCE == "web"
|
||
|
||
stages:
|
||
- prepare
|
||
- scan
|
||
- test
|
||
- compile
|
||
- train
|
||
- deploy
|
||
|
||
variables:
|
||
SQLX_OFFLINE: "true"
|
||
CARGO_TERM_COLOR: always
|
||
# CUDA_COMPUTE_CAP is set at runtime by detect_gpu() — L4=89, H100=90
|
||
# Do NOT hardcode here; sccache is partitioned by compute cap below
|
||
# Stable JWT secret for api tests (prevents env-var race between parallel tests)
|
||
JWT_SECRET: "CiTestSecret_Kx7mP9nR2sW5vY8bC3fG6jH1kL4pQ7tZ0uN9dM5eV8xS2wT6yA4zB_64chars_min"
|
||
# Training data on block storage PVC (mounted at /mnt/training-data by runner)
|
||
FOXHUNT_DATA_DIR: "/mnt/training-data/futures-baseline"
|
||
# sccache backed by MinIO S3 (HTTP) — shared across all nodes (no PVC needed)
|
||
SCCACHE_BUCKET: foxhunt-sccache
|
||
SCCACHE_ENDPOINT: http://minio.foxhunt.svc.cluster.local:9000
|
||
SCCACHE_S3_KEY_PREFIX: cpu
|
||
SCCACHE_S3_USE_SSL: "false"
|
||
SCCACHE_S3_NO_CREDENTIALS: "false"
|
||
RUSTC_WRAPPER: /usr/local/bin/sccache
|
||
# Build profile: "dev-release" (thin LTO, fast) is default for quick iteration
|
||
# Set DEV_RELEASE=false in pipeline vars to use full "release" (fat LTO) profile
|
||
DEV_RELEASE: "true"
|
||
CARGO_BUILD_PROFILE: dev-release
|
||
# S3 credentials for sccache MinIO backend (also used by rclone binary uploads)
|
||
AWS_ACCESS_KEY_ID: $MINIO_ACCESS_KEY
|
||
AWS_SECRET_ACCESS_KEY: $MINIO_SECRET_KEY
|
||
# CI builder images on internal GitLab registry (backed by MinIO)
|
||
CI_BUILDER_IMAGE: localhost:30500/root/foxhunt/ci-builder:latest
|
||
CI_BUILDER_CPU_IMAGE: localhost:30500/root/foxhunt/ci-builder-cpu:latest
|
||
# Container registry for all images
|
||
REGISTRY: localhost:30500/root/foxhunt
|
||
# Infrastructure runner image
|
||
INFRA_RUNNER_IMAGE: localhost:30500/root/foxhunt/infra-runner:latest
|
||
# Generic runtime images for S3 binary share (service pods + training pods + node builder)
|
||
RUNTIME_IMAGE: localhost:30500/root/foxhunt/foxhunt-runtime:latest
|
||
TRAINING_RUNTIME_IMAGE: localhost:30500/root/foxhunt/foxhunt-training-runtime:latest
|
||
# Node builder image eliminated — web dashboard uses node:22-slim + inline rclone
|
||
|
||
# --------------------------------------------------------------------------
|
||
# Stage 0: Build CI builder image → push to Scaleway CR
|
||
# --------------------------------------------------------------------------
|
||
build-ci-builder:
|
||
stage: prepare
|
||
image:
|
||
name: gcr.io/kaniko-project/executor:debug
|
||
entrypoint: [""]
|
||
tags:
|
||
- kapsule
|
||
- docker
|
||
rules:
|
||
- if: $CI_PIPELINE_SOURCE == "push"
|
||
changes:
|
||
- infra/docker/Dockerfile.ci-builder
|
||
when: on_success
|
||
- when: manual
|
||
allow_failure: true
|
||
before_script:
|
||
- mkdir -p /kaniko/.docker
|
||
- >-
|
||
echo "{\"auths\":{
|
||
\"rg.fr-par.scw.cloud\":{\"username\":\"nologin\",\"password\":\"${SCW_SECRET_KEY}\"},
|
||
\"https://index.docker.io/v1/\":{\"username\":\"${DOCKERHUB_USERNAME}\",\"password\":\"${DOCKERHUB_TOKEN}\"}
|
||
}}" > /kaniko/.docker/config.json
|
||
script:
|
||
- /kaniko/executor
|
||
--context "${CI_PROJECT_DIR}"
|
||
--dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.ci-builder"
|
||
--cache=true --cache-repo="${REGISTRY}/cache"
|
||
--destination "${CI_BUILDER_IMAGE}"
|
||
|
||
# --------------------------------------------------------------------------
|
||
# Stage 0a: Build CPU-only CI builder image → push to Scaleway CR
|
||
# --------------------------------------------------------------------------
|
||
build-ci-builder-cpu:
|
||
stage: prepare
|
||
image:
|
||
name: gcr.io/kaniko-project/executor:debug
|
||
entrypoint: [""]
|
||
tags:
|
||
- kapsule
|
||
- docker
|
||
rules:
|
||
- if: $CI_PIPELINE_SOURCE == "push"
|
||
changes:
|
||
- infra/docker/Dockerfile.ci-builder-cpu
|
||
when: on_success
|
||
- when: manual
|
||
allow_failure: true
|
||
before_script:
|
||
- mkdir -p /kaniko/.docker
|
||
- >-
|
||
echo "{\"auths\":{
|
||
\"rg.fr-par.scw.cloud\":{\"username\":\"nologin\",\"password\":\"${SCW_SECRET_KEY}\"},
|
||
\"https://index.docker.io/v1/\":{\"username\":\"${DOCKERHUB_USERNAME}\",\"password\":\"${DOCKERHUB_TOKEN}\"}
|
||
}}" > /kaniko/.docker/config.json
|
||
script:
|
||
- /kaniko/executor
|
||
--context "${CI_PROJECT_DIR}"
|
||
--dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.ci-builder-cpu"
|
||
--cache=true --cache-repo="${REGISTRY}/cache"
|
||
--destination "${CI_BUILDER_CPU_IMAGE}"
|
||
|
||
# --------------------------------------------------------------------------
|
||
# Stage 0b: Build devcontainer image → push to internal GitLab registry
|
||
# --------------------------------------------------------------------------
|
||
build-devcontainer:
|
||
stage: prepare
|
||
image:
|
||
name: gcr.io/kaniko-project/executor:debug
|
||
entrypoint: [""]
|
||
tags:
|
||
- kapsule
|
||
- docker
|
||
rules:
|
||
- if: $CI_PIPELINE_SOURCE == "push"
|
||
changes:
|
||
- .devcontainer/**
|
||
when: on_success
|
||
- when: manual
|
||
allow_failure: true
|
||
before_script:
|
||
- mkdir -p /kaniko/.docker
|
||
- >-
|
||
echo "{\"auths\":{
|
||
\"rg.fr-par.scw.cloud\":{\"username\":\"nologin\",\"password\":\"${SCW_SECRET_KEY}\"},
|
||
\"https://index.docker.io/v1/\":{\"username\":\"${DOCKERHUB_USERNAME}\",\"password\":\"${DOCKERHUB_TOKEN}\"}
|
||
}}" > /kaniko/.docker/config.json
|
||
script:
|
||
- /kaniko/executor
|
||
--context "${CI_PROJECT_DIR}"
|
||
--dockerfile "${CI_PROJECT_DIR}/.devcontainer/Dockerfile"
|
||
--cache=true --cache-repo="${REGISTRY}/cache"
|
||
--destination "localhost:30500/root/foxhunt/devcontainer:${CI_COMMIT_SHA}"
|
||
--destination "localhost:30500/root/foxhunt/devcontainer:latest"
|
||
|
||
# --------------------------------------------------------------------------
|
||
# Stage 0c: Build infra-runner image → push to Scaleway CR
|
||
# --------------------------------------------------------------------------
|
||
build-infra-runner:
|
||
stage: prepare
|
||
image:
|
||
name: gcr.io/kaniko-project/executor:debug
|
||
entrypoint: [""]
|
||
tags:
|
||
- kapsule
|
||
- docker
|
||
rules:
|
||
- if: $CI_PIPELINE_SOURCE == "push"
|
||
changes:
|
||
- infra/docker/Dockerfile.infra-runner
|
||
when: on_success
|
||
- when: manual
|
||
allow_failure: true
|
||
before_script:
|
||
- mkdir -p /kaniko/.docker
|
||
- >-
|
||
echo "{\"auths\":{
|
||
\"rg.fr-par.scw.cloud\":{\"username\":\"nologin\",\"password\":\"${SCW_SECRET_KEY}\"},
|
||
\"https://index.docker.io/v1/\":{\"username\":\"${DOCKERHUB_USERNAME}\",\"password\":\"${DOCKERHUB_TOKEN}\"}
|
||
}}" > /kaniko/.docker/config.json
|
||
script:
|
||
- /kaniko/executor
|
||
--context "${CI_PROJECT_DIR}"
|
||
--dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.infra-runner"
|
||
--cache=true --cache-repo="${REGISTRY}/cache"
|
||
--destination "${INFRA_RUNNER_IMAGE}"
|
||
|
||
# --------------------------------------------------------------------------
|
||
# Stage 0d: Build generic runtime base images → push to Scaleway CR
|
||
# --------------------------------------------------------------------------
|
||
build-foxhunt-runtime:
|
||
stage: prepare
|
||
image:
|
||
name: gcr.io/kaniko-project/executor:debug
|
||
entrypoint: [""]
|
||
tags:
|
||
- kapsule
|
||
- docker
|
||
rules:
|
||
- if: $CI_PIPELINE_SOURCE == "push"
|
||
changes:
|
||
- infra/docker/Dockerfile.foxhunt-runtime
|
||
when: on_success
|
||
- when: manual
|
||
allow_failure: true
|
||
before_script:
|
||
- mkdir -p /kaniko/.docker
|
||
- >-
|
||
echo "{\"auths\":{
|
||
\"rg.fr-par.scw.cloud\":{\"username\":\"nologin\",\"password\":\"${SCW_SECRET_KEY}\"},
|
||
\"https://index.docker.io/v1/\":{\"username\":\"${DOCKERHUB_USERNAME}\",\"password\":\"${DOCKERHUB_TOKEN}\"}
|
||
}}" > /kaniko/.docker/config.json
|
||
script:
|
||
- /kaniko/executor
|
||
--context "${CI_PROJECT_DIR}"
|
||
--dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.foxhunt-runtime"
|
||
--cache=true --cache-repo="${REGISTRY}/cache"
|
||
--destination "${REGISTRY}/foxhunt-runtime:latest"
|
||
|
||
build-foxhunt-training-runtime:
|
||
stage: prepare
|
||
image:
|
||
name: gcr.io/kaniko-project/executor:debug
|
||
entrypoint: [""]
|
||
tags:
|
||
- kapsule
|
||
- docker
|
||
rules:
|
||
- if: $CI_PIPELINE_SOURCE == "push"
|
||
changes:
|
||
- infra/docker/Dockerfile.foxhunt-training-runtime
|
||
when: on_success
|
||
- when: manual
|
||
allow_failure: true
|
||
before_script:
|
||
- mkdir -p /kaniko/.docker
|
||
- >-
|
||
echo "{\"auths\":{
|
||
\"rg.fr-par.scw.cloud\":{\"username\":\"nologin\",\"password\":\"${SCW_SECRET_KEY}\"},
|
||
\"https://index.docker.io/v1/\":{\"username\":\"${DOCKERHUB_USERNAME}\",\"password\":\"${DOCKERHUB_TOKEN}\"}
|
||
}}" > /kaniko/.docker/config.json
|
||
script:
|
||
- /kaniko/executor
|
||
--context "${CI_PROJECT_DIR}"
|
||
--dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.foxhunt-training-runtime"
|
||
--cache=true --cache-repo="${REGISTRY}/cache"
|
||
--destination "${REGISTRY}/foxhunt-training-runtime:latest"
|
||
|
||
# --------------------------------------------------------------------------
|
||
# Stage 0.5: Scan runtime images for vulnerabilities (Trivy)
|
||
# --------------------------------------------------------------------------
|
||
scan-runtime-images:
|
||
stage: scan
|
||
image:
|
||
name: aquasec/trivy:latest
|
||
entrypoint: [""]
|
||
tags:
|
||
- kapsule
|
||
- docker
|
||
variables:
|
||
KUBERNETES_CPU_REQUEST: "200m"
|
||
KUBERNETES_CPU_LIMIT: "500m"
|
||
KUBERNETES_MEMORY_REQUEST: "256Mi"
|
||
KUBERNETES_MEMORY_LIMIT: "512Mi"
|
||
needs:
|
||
- job: build-foxhunt-runtime
|
||
optional: true
|
||
- job: build-foxhunt-training-runtime
|
||
optional: true
|
||
rules:
|
||
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
|
||
- if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
|
||
cache:
|
||
key: trivy-db
|
||
paths:
|
||
- .trivy-cache/
|
||
before_script:
|
||
- export TRIVY_USERNAME=nologin
|
||
- export TRIVY_PASSWORD=${SCW_SECRET_KEY}
|
||
script:
|
||
- echo "Scanning runtime images for CRITICAL/HIGH vulnerabilities..."
|
||
- trivy image
|
||
--cache-dir .trivy-cache
|
||
--exit-code 1
|
||
--severity CRITICAL,HIGH
|
||
--ignore-unfixed
|
||
--no-progress
|
||
"${REGISTRY}/foxhunt-runtime:latest"
|
||
- trivy image
|
||
--cache-dir .trivy-cache
|
||
--exit-code 1
|
||
--severity CRITICAL,HIGH
|
||
--ignore-unfixed
|
||
--no-progress
|
||
"${REGISTRY}/foxhunt-training-runtime:latest"
|
||
- echo "Image scan passed — no fixable CRITICAL/HIGH vulnerabilities."
|
||
allow_failure: false
|
||
|
||
|
||
# Base template for Rust jobs — pre-baked CI builder from Scaleway CR
|
||
# Image pre-exists in SCR; rebuild via build-ci-builder when Dockerfile changes
|
||
# Runs on ci-compile-cpu pool (POP2-32C-128G: 32 vCPU, 128GB RAM) via CPU runner
|
||
# CPU budget: 14 CPUs per job for tests; per-service compile jobs use 4 CPUs each
|
||
# (Scaleway quota: 1× POP2-32C-128G, so both jobs must share)
|
||
# Base template for jobs needing CUDA builder (test stage only — compile uses .rust-base-cpu)
|
||
.rust-base:
|
||
image: ${CI_BUILDER_IMAGE}
|
||
tags:
|
||
- kapsule
|
||
- rust
|
||
variables:
|
||
CUDA_COMPUTE_CAP: "89"
|
||
CARGO_BUILD_JOBS: "14"
|
||
KUBERNETES_CPU_REQUEST: "14000m"
|
||
KUBERNETES_CPU_LIMIT: "15500m"
|
||
KUBERNETES_MEMORY_REQUEST: "24Gi"
|
||
KUBERNETES_MEMORY_LIMIT: "56Gi"
|
||
before_script:
|
||
# Mold linker fallback — .cargo/config.toml uses -fuse-ld=mold via clang.
|
||
# clang's -fuse-ld=mold searches for "ld.mold" (not "mold") in its linker paths.
|
||
# If mold isn't installed, create ld.mold wrapper that delegates to ld.lld.
|
||
- |
|
||
if ! command -v mold &>/dev/null; then
|
||
printf '#!/bin/sh\nexec ld.lld "$@"\n' > /usr/local/bin/ld.mold
|
||
chmod +x /usr/local/bin/ld.mold
|
||
echo "mold not found, falling back to lld via ld.mold wrapper"
|
||
fi
|
||
|
||
# Base template for CPU-only Rust jobs — lightweight builder without CUDA
|
||
# Uses ci-builder-cpu image (rust:1.89-slim, ~2-3GB, no CUDA toolkit)
|
||
# Per-service compile jobs override CPU/memory to 4 CPUs each (see .compile-service-base)
|
||
.rust-base-cpu:
|
||
image: ${CI_BUILDER_CPU_IMAGE}
|
||
tags:
|
||
- kapsule
|
||
- rust
|
||
variables:
|
||
# No CUDA_COMPUTE_CAP — CPU-only builds don't compile CUDA kernels
|
||
# Request 10 CPUs (burstable to 15.5) — two compile jobs + web-dashboard fit on 1 POP2-32C-128G node
|
||
CARGO_BUILD_JOBS: "10"
|
||
KUBERNETES_NODE_SELECTOR_POOL: "k8s.scaleway.com/pool-name=ci-compile-cpu"
|
||
KUBERNETES_CPU_REQUEST: "10000m"
|
||
KUBERNETES_CPU_LIMIT: "15500m"
|
||
KUBERNETES_MEMORY_REQUEST: "18Gi"
|
||
KUBERNETES_MEMORY_LIMIT: "56Gi"
|
||
before_script:
|
||
# Mold linker fallback — same as .rust-base
|
||
- |
|
||
if ! command -v mold &>/dev/null; then
|
||
printf '#!/bin/sh\nexec ld.lld "$@"\n' > /usr/local/bin/ld.mold
|
||
chmod +x /usr/local/bin/ld.mold
|
||
echo "mold not found, falling back to lld via ld.mold wrapper"
|
||
fi
|
||
|
||
# Base template for per-service compile jobs (CPU-only, no CUDA)
|
||
# Each service gets its own job with dependency-aware changes: filters.
|
||
# CARGO_BUILD_JOBS=4 (down from 10) since up to 8 jobs may run on the same node.
|
||
.compile-service-base:
|
||
extends: .rust-base-cpu
|
||
stage: compile
|
||
needs: []
|
||
interruptible: true
|
||
variables:
|
||
CARGO_BUILD_JOBS: "4"
|
||
KUBERNETES_CPU_REQUEST: "4000m"
|
||
KUBERNETES_CPU_LIMIT: "6000m"
|
||
KUBERNETES_MEMORY_REQUEST: "8Gi"
|
||
KUBERNETES_MEMORY_LIMIT: "16Gi"
|
||
script:
|
||
- sccache --zero-stats || true
|
||
- PROFILE=${DEV_RELEASE:+dev-release}; PROFILE=${PROFILE:-release}
|
||
- |
|
||
# CalVer: YYYY.MM.<pipeline_iid>
|
||
YEAR=$(date -d "$CI_COMMIT_TIMESTAMP" +%Y 2>/dev/null || date +%Y)
|
||
MONTH=$(date -d "$CI_COMMIT_TIMESTAMP" +%m 2>/dev/null || date +%m)
|
||
export FOXHUNT_BUILD_VERSION="${YEAR}.${MONTH}.${CI_PIPELINE_IID:-0}"
|
||
echo "Build version: $FOXHUNT_BUILD_VERSION"
|
||
- echo "Building ${SERVICE_PACKAGE} with --profile $PROFILE (CPU-only, no CUDA)"
|
||
- TARGET_DIR="target/$PROFILE"
|
||
- cargo build --profile $PROFILE -p ${SERVICE_PACKAGE}
|
||
- sccache --show-stats || true
|
||
- mkdir -p build-out/services
|
||
- |
|
||
cp $TARGET_DIR/${SERVICE_PACKAGE} build-out/services/
|
||
strip build-out/services/${SERVICE_PACKAGE}
|
||
- ls -lh build-out/services/
|
||
- echo "${SERVICE_PACKAGE} binary ready as CI artifact"
|
||
artifacts:
|
||
paths:
|
||
- build-out/services/
|
||
expire_in: 1 day
|
||
|
||
# --------------------------------------------------------------------------
|
||
# Stage 2: tests
|
||
# --------------------------------------------------------------------------
|
||
test:
|
||
extends: .rust-base
|
||
stage: test
|
||
needs: []
|
||
rules:
|
||
# Tests disabled by default — compile+deploy is the fast path for now.
|
||
# Trigger manually from the pipeline UI when needed.
|
||
- if: $CI_PIPELINE_SOURCE == "push"
|
||
when: manual
|
||
allow_failure: true
|
||
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
|
||
when: manual
|
||
allow_failure: true
|
||
- if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
|
||
when: manual
|
||
allow_failure: true
|
||
services:
|
||
- name: redis:7-alpine
|
||
alias: redis
|
||
script:
|
||
# Detect GPU if available (nvidia runtime), otherwise use CUDA_COMPUTE_CAP from variables
|
||
- |
|
||
if command -v nvidia-smi &>/dev/null; then
|
||
export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//')
|
||
export CUDA_COMPUTE_CAP=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader | head -1 | tr -d '.')
|
||
echo "GPU=$(nvidia-smi --query-gpu=name --format=csv,noheader) CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP"
|
||
else
|
||
echo "No nvidia-smi — using CUDA stubs, CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP"
|
||
fi
|
||
- export SCCACHE_S3_KEY_PREFIX="cuda/sm_${CUDA_COMPUTE_CAP}"
|
||
# Wait for Redis sidecar (K8s executor doesn't wait for service readiness)
|
||
- |
|
||
for i in $(seq 1 30); do
|
||
if bash -c 'echo PING > /dev/tcp/localhost/6379' 2>/dev/null; then
|
||
echo "Redis ready"; break
|
||
fi
|
||
echo "Waiting for Redis... ($i/30)"; sleep 1
|
||
done
|
||
- sccache --zero-stats || true
|
||
- cargo test --workspace --lib -- --skip model_loader::tests
|
||
- sccache --show-stats || true
|
||
|
||
# --------------------------------------------------------------------------
|
||
# Stage 3a: Per-service compile jobs (CPU-only, no CUDA)
|
||
# Each service compiled independently with dependency-aware changes: filters.
|
||
# sccache shared across jobs — incremental crate compilation is cached.
|
||
# --------------------------------------------------------------------------
|
||
compile-api:
|
||
extends: .compile-service-base
|
||
variables:
|
||
SERVICE_PACKAGE: api
|
||
rules: &rules-api
|
||
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
|
||
changes:
|
||
- services/api/**
|
||
- crates/common/**
|
||
- crates/config/**
|
||
- crates/trading_engine/**
|
||
- bin/fxt/**
|
||
- Cargo.toml
|
||
- Cargo.lock
|
||
- if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
|
||
|
||
compile-trading-service:
|
||
extends: .compile-service-base
|
||
variables:
|
||
SERVICE_PACKAGE: trading-service
|
||
rules: &rules-trading-service
|
||
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
|
||
changes:
|
||
- services/trading_service/**
|
||
- services/api/**
|
||
- crates/common/**
|
||
- crates/config/**
|
||
- crates/data/**
|
||
- crates/database/**
|
||
- crates/ml/**
|
||
- crates/risk/**
|
||
- crates/storage/**
|
||
- crates/trading_engine/**
|
||
- Cargo.toml
|
||
- Cargo.lock
|
||
- if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
|
||
|
||
compile-ml-training-service:
|
||
extends: .compile-service-base
|
||
variables:
|
||
SERVICE_PACKAGE: ml-training-service
|
||
rules: &rules-ml-training-service
|
||
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
|
||
changes:
|
||
- services/ml_training_service/**
|
||
- crates/common/**
|
||
- crates/config/**
|
||
- crates/data/**
|
||
- crates/ml/**
|
||
- crates/risk/**
|
||
- crates/storage/**
|
||
- crates/trading_engine/**
|
||
- Cargo.toml
|
||
- Cargo.lock
|
||
- if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
|
||
|
||
compile-backtesting-service:
|
||
extends: .compile-service-base
|
||
variables:
|
||
SERVICE_PACKAGE: backtesting-service
|
||
rules: &rules-backtesting-service
|
||
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
|
||
changes:
|
||
- services/backtesting_service/**
|
||
- crates/common/**
|
||
- crates/config/**
|
||
- crates/data/**
|
||
- crates/ml/**
|
||
- crates/model_loader/**
|
||
- crates/risk/**
|
||
- crates/storage/**
|
||
- crates/trading_engine/**
|
||
- bin/fxt/**
|
||
- Cargo.toml
|
||
- Cargo.lock
|
||
- if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
|
||
|
||
compile-trading-agent-service:
|
||
extends: .compile-service-base
|
||
variables:
|
||
SERVICE_PACKAGE: trading-agent-service
|
||
rules: &rules-trading-agent-service
|
||
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
|
||
changes:
|
||
- services/trading_agent_service/**
|
||
- crates/common/**
|
||
- crates/config/**
|
||
- crates/ml/**
|
||
- crates/risk/**
|
||
- Cargo.toml
|
||
- Cargo.lock
|
||
- if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
|
||
|
||
compile-broker-gateway:
|
||
extends: .compile-service-base
|
||
variables:
|
||
SERVICE_PACKAGE: broker-gateway
|
||
rules: &rules-broker-gateway
|
||
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
|
||
changes:
|
||
- services/broker_gateway_service/**
|
||
- crates/common/**
|
||
- crates/config/**
|
||
- crates/trading_engine/**
|
||
- vendor/ctrader-openapi/**
|
||
- Cargo.toml
|
||
- Cargo.lock
|
||
- if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
|
||
|
||
compile-data-acquisition-service:
|
||
extends: .compile-service-base
|
||
variables:
|
||
SERVICE_PACKAGE: data-acquisition-service
|
||
rules: &rules-data-acquisition-service
|
||
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
|
||
changes:
|
||
- services/data_acquisition_service/**
|
||
- crates/common/**
|
||
- crates/config/**
|
||
- crates/storage/**
|
||
- Cargo.toml
|
||
- Cargo.lock
|
||
- if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
|
||
|
||
# --------------------------------------------------------------------------
|
||
# Stage 3a: Compile training binaries (CPU node, CUDA stubs from ci-builder image)
|
||
# --------------------------------------------------------------------------
|
||
compile-training:
|
||
extends: .rust-base-cpu
|
||
stage: compile
|
||
image: ${CI_BUILDER_IMAGE}
|
||
needs: [] # parallel with per-service compile jobs — shares 1 POP2-32C-128G node
|
||
resource_group: compile-training
|
||
interruptible: true
|
||
variables:
|
||
# CUDA stubs from ci-builder image — compiles CUDA kernels without GPU hardware.
|
||
# L4 GPU = compute capability 8.9 → candle-kernels generates sm_89 PTX
|
||
CUDA_COMPUTE_CAP: "89"
|
||
# When all 8 service jobs fire (Cargo.toml change), the node is fully loaded.
|
||
# Increase pod scheduling timeout so training waits for service jobs to finish.
|
||
KUBERNETES_POLL_TIMEOUT: "1200"
|
||
rules:
|
||
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
|
||
changes:
|
||
- Cargo.toml
|
||
- Cargo.lock
|
||
- crates/ml/**
|
||
- crates/trading_engine/**
|
||
- crates/common/**
|
||
- crates/risk/**
|
||
- crates/data/**
|
||
- crates/config/**
|
||
- crates/storage/**
|
||
- services/training_uploader/**
|
||
- if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
|
||
script:
|
||
- echo "CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP"
|
||
- export SCCACHE_S3_KEY_PREFIX="cuda/sm_${CUDA_COMPUTE_CAP}"
|
||
- sccache --zero-stats || true
|
||
- PROFILE=${DEV_RELEASE:+dev-release}; PROFILE=${PROFILE:-release}
|
||
- |
|
||
# CalVer: YYYY.MM.<pipeline_iid>
|
||
YEAR=$(date -d "$CI_COMMIT_TIMESTAMP" +%Y 2>/dev/null || date +%Y)
|
||
MONTH=$(date -d "$CI_COMMIT_TIMESTAMP" +%m 2>/dev/null || date +%m)
|
||
export FOXHUNT_BUILD_VERSION="${YEAR}.${MONTH}.${CI_PIPELINE_IID:-0}"
|
||
echo "Build version: $FOXHUNT_BUILD_VERSION"
|
||
- echo "Building training binaries with --profile $PROFILE (CUDA stubs)"
|
||
- TARGET_DIR="target/$PROFILE"
|
||
- cargo build --profile $PROFILE -p ml --features ml/cuda
|
||
--example train_baseline_rl
|
||
--example train_baseline_supervised
|
||
--example evaluate_baseline
|
||
--example evaluate_supervised
|
||
--example hyperopt_baseline_rl
|
||
--example hyperopt_baseline_supervised
|
||
- cargo build --profile $PROFILE -p training_uploader
|
||
- sccache --show-stats || true
|
||
- mkdir -p build-out/training
|
||
- |
|
||
for bin in train_baseline_rl train_baseline_supervised evaluate_baseline evaluate_supervised hyperopt_baseline_rl hyperopt_baseline_supervised; do
|
||
cp $TARGET_DIR/examples/$bin build-out/training/
|
||
strip build-out/training/$bin
|
||
done
|
||
cp $TARGET_DIR/training_uploader build-out/training/
|
||
strip build-out/training/training_uploader
|
||
- ls -lh build-out/training/
|
||
- echo "Training binaries ready as CI artifacts"
|
||
artifacts:
|
||
paths:
|
||
- build-out/training/
|
||
expire_in: 3 days
|
||
|
||
# --------------------------------------------------------------------------
|
||
# Stage 3c: Build web-dashboard static assets (Vite)
|
||
# --------------------------------------------------------------------------
|
||
build-web-dashboard:
|
||
stage: compile
|
||
image: node:22-slim
|
||
needs: []
|
||
resource_group: compile-web-dashboard
|
||
interruptible: true
|
||
tags:
|
||
- kapsule
|
||
- docker
|
||
variables:
|
||
KUBERNETES_NODE_SELECTOR_POOL: "k8s.scaleway.com/pool-name=ci-compile-cpu"
|
||
KUBERNETES_CPU_REQUEST: "2000m"
|
||
KUBERNETES_CPU_LIMIT: "4000m"
|
||
KUBERNETES_MEMORY_REQUEST: "2Gi"
|
||
KUBERNETES_MEMORY_LIMIT: "4Gi"
|
||
rules:
|
||
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
|
||
changes:
|
||
- web-dashboard/**
|
||
- if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
|
||
script:
|
||
- cd web-dashboard
|
||
- npm ci
|
||
- npm run build
|
||
- echo "Web dashboard assets ready as CI artifacts"
|
||
artifacts:
|
||
paths:
|
||
- web-dashboard/dist/
|
||
expire_in: 1 day
|
||
|
||
# --------------------------------------------------------------------------
|
||
# Stage 3c: Training — RL on L4 (CPU-heavy), Supervised on L40S (GPU-heavy)
|
||
# --------------------------------------------------------------------------
|
||
|
||
# --------------------------------------------------------------------------
|
||
# RL training base: DQN + PPO → ci-training pool (L40S)
|
||
# RL hyperopt runs parallel PSO trials — CPU+GPU bound, low VRAM.
|
||
# --------------------------------------------------------------------------
|
||
.train-rl-base:
|
||
stage: train
|
||
image: ${TRAINING_RUNTIME_IMAGE}
|
||
tags:
|
||
- kapsule
|
||
- l40s
|
||
variables:
|
||
# Route to ci-training pool (L40S GPU)
|
||
KUBERNETES_NODE_SELECTOR_POOL: "k8s.scaleway.com/pool-name=ci-training"
|
||
KUBERNETES_RUNTIME_CLASS_NAME: "nvidia"
|
||
KUBERNETES_CPU_REQUEST: "7000m"
|
||
KUBERNETES_CPU_LIMIT: "8000m"
|
||
KUBERNETES_MEMORY_REQUEST: "8Gi"
|
||
KUBERNETES_MEMORY_LIMIT: "20Gi"
|
||
CUBLAS_WORKSPACE_CONFIG: ":4096:8"
|
||
OTEL_EXPORTER_OTLP_ENDPOINT: "http://tempo.foxhunt.svc.cluster.local:4317"
|
||
# Expose training metrics to Prometheus (binaries serve /metrics on :9094)
|
||
KUBERNETES_POD_ANNOTATIONS_1: "gitlab.com/prometheus_scrape=true"
|
||
KUBERNETES_POD_ANNOTATIONS_2: "gitlab.com/prometheus_port=9094"
|
||
KUBERNETES_POD_ANNOTATIONS_3: "gitlab.com/prometheus_path=/metrics"
|
||
rules:
|
||
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
|
||
when: manual
|
||
allow_failure: true
|
||
- if: $CI_PIPELINE_SOURCE == "schedule" && $TRAIN_VALIDATE == "true"
|
||
- if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
|
||
when: manual
|
||
allow_failure: true
|
||
before_script:
|
||
- export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//')
|
||
- nvidia-smi
|
||
- mkdir -p ${CI_PROJECT_DIR}/output ${CI_PROJECT_DIR}/training-bin
|
||
# Fetch training binaries from MinIO (CI artifacts as fallback)
|
||
- |
|
||
if cp build-out/training/* ${CI_PROJECT_DIR}/training-bin/ 2>/dev/null; then
|
||
echo "Using CI artifact binaries"
|
||
else
|
||
echo "Fetching training binaries from MinIO..."
|
||
rclone copy :s3:foxhunt-binaries/training/ ${CI_PROJECT_DIR}/training-bin/ \
|
||
--s3-provider=Minio \
|
||
--s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 \
|
||
--s3-access-key-id="${MINIO_ACCESS_KEY}" \
|
||
--s3-secret-access-key="${MINIO_SECRET_KEY}" \
|
||
--s3-no-check-bucket \
|
||
echo "Fetched $(ls ${CI_PROJECT_DIR}/training-bin/ | wc -l) training binaries from MinIO"
|
||
fi
|
||
- chmod +x ${CI_PROJECT_DIR}/training-bin/* 2>/dev/null || true
|
||
- export PATH="${CI_PROJECT_DIR}/training-bin:$PATH"
|
||
|
||
# --------------------------------------------------------------------------
|
||
# Supervised training base: TFT, Mamba2, TGGN, TLOB, Liquid, KAN, xLSTM, Diffusion → L40S
|
||
# Supervised models are GPU-bound — low CPU, high VRAM.
|
||
# --------------------------------------------------------------------------
|
||
.train-validate-base:
|
||
stage: train
|
||
image: ${TRAINING_RUNTIME_IMAGE}
|
||
tags:
|
||
- kapsule
|
||
- l40s
|
||
variables:
|
||
# nvidia runtime for GPU nodes (ci-training has L40S 48GB)
|
||
KUBERNETES_RUNTIME_CLASS_NAME: "nvidia"
|
||
# Route to L40S pool (48GB VRAM for large supervised models)
|
||
# Overrides runner default to L40S pool
|
||
KUBERNETES_NODE_SELECTOR_POOL: "k8s.scaleway.com/pool-name=ci-training"
|
||
# Supervised is GPU-bound — minimal CPU. Low request allows 3-4 concurrent jobs.
|
||
KUBERNETES_CPU_REQUEST: "1000m"
|
||
KUBERNETES_CPU_LIMIT: "2000m"
|
||
KUBERNETES_MEMORY_REQUEST: "16Gi"
|
||
KUBERNETES_MEMORY_LIMIT: "40Gi"
|
||
OTEL_EXPORTER_OTLP_ENDPOINT: "http://tempo.foxhunt.svc.cluster.local:4317"
|
||
# Expose training metrics to Prometheus (binaries serve /metrics on :9094)
|
||
KUBERNETES_POD_ANNOTATIONS_1: "gitlab.com/prometheus_scrape=true"
|
||
KUBERNETES_POD_ANNOTATIONS_2: "gitlab.com/prometheus_port=9094"
|
||
KUBERNETES_POD_ANNOTATIONS_3: "gitlab.com/prometheus_path=/metrics"
|
||
rules:
|
||
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
|
||
when: manual
|
||
allow_failure: true
|
||
- if: $CI_PIPELINE_SOURCE == "schedule" && $TRAIN_VALIDATE == "true"
|
||
- if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
|
||
when: manual
|
||
allow_failure: true
|
||
before_script:
|
||
- export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//')
|
||
- nvidia-smi
|
||
- mkdir -p ${CI_PROJECT_DIR}/output ${CI_PROJECT_DIR}/training-bin
|
||
# Fetch training binaries from MinIO (CI artifacts as fallback)
|
||
- |
|
||
if cp build-out/training/* ${CI_PROJECT_DIR}/training-bin/ 2>/dev/null; then
|
||
echo "Using CI artifact binaries"
|
||
else
|
||
echo "Fetching training binaries from MinIO..."
|
||
rclone copy :s3:foxhunt-binaries/training/ ${CI_PROJECT_DIR}/training-bin/ \
|
||
--s3-provider=Minio \
|
||
--s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 \
|
||
--s3-access-key-id="${MINIO_ACCESS_KEY}" \
|
||
--s3-secret-access-key="${MINIO_SECRET_KEY}" \
|
||
--s3-no-check-bucket \
|
||
echo "Fetched $(ls ${CI_PROJECT_DIR}/training-bin/ | wc -l) training binaries from MinIO"
|
||
fi
|
||
- chmod +x ${CI_PROJECT_DIR}/training-bin/* 2>/dev/null || true
|
||
- export PATH="${CI_PROJECT_DIR}/training-bin:$PATH"
|
||
|
||
train-validate-rl:
|
||
extends: .train-rl-base
|
||
needs:
|
||
- job: compile-training
|
||
optional: true
|
||
script:
|
||
- |
|
||
train_baseline_rl \
|
||
--model both \
|
||
--symbol ES.FUT \
|
||
--data-dir /mnt/training-data/futures-baseline \
|
||
--output-dir ${CI_PROJECT_DIR}/output \
|
||
--max-steps-per-epoch 50 \
|
||
&& TRAIN_OK=1 || TRAIN_OK=0
|
||
echo "=== RL training exit: $TRAIN_OK ==="
|
||
echo "=== Running evaluation ==="
|
||
evaluate_baseline \
|
||
--model both \
|
||
--models-dir ${CI_PROJECT_DIR}/output \
|
||
--data-dir /mnt/training-data/futures-baseline \
|
||
--output ${CI_PROJECT_DIR}/output/rl_eval_report.json \
|
||
--symbol ES.FUT \
|
||
|| true
|
||
echo "=== Eval report ==="
|
||
cat ${CI_PROJECT_DIR}/output/rl_eval_report.json 2>/dev/null || echo "No eval report"
|
||
[ "$TRAIN_OK" = "1" ] || exit 1
|
||
artifacts:
|
||
paths:
|
||
- output/rl_eval_report.json
|
||
when: always
|
||
expire_in: 30 days
|
||
|
||
train-validate-tft:
|
||
extends: .train-validate-base
|
||
needs:
|
||
- job: compile-training
|
||
optional: true
|
||
script:
|
||
- train_baseline_supervised
|
||
--model tft
|
||
--symbol ES.FUT
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output-dir ${CI_PROJECT_DIR}/output
|
||
--max-steps-per-epoch 50
|
||
- echo "=== TFT training complete, running evaluation ==="
|
||
- evaluate_supervised
|
||
--model tft
|
||
--models-dir ${CI_PROJECT_DIR}/output
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output ${CI_PROJECT_DIR}/output/tft_eval_report.json
|
||
--symbol ES.FUT
|
||
- echo "TFT validation + evaluation passed"
|
||
- cat ${CI_PROJECT_DIR}/output/tft_eval_report.json
|
||
artifacts:
|
||
paths:
|
||
- output/tft_eval_report.json
|
||
when: always
|
||
expire_in: 30 days
|
||
|
||
hyperopt-ppo:
|
||
extends: .train-rl-base
|
||
tags:
|
||
- kapsule
|
||
- h100
|
||
variables:
|
||
KUBERNETES_NODE_SELECTOR_POOL: "k8s.scaleway.com/pool-name=ci-training-h100"
|
||
KUBERNETES_MEMORY_REQUEST: "16Gi"
|
||
KUBERNETES_MEMORY_LIMIT: "48Gi"
|
||
needs:
|
||
- job: compile-training
|
||
optional: true
|
||
timeout: 4h
|
||
script:
|
||
- hyperopt_baseline_rl
|
||
--model ppo
|
||
--trials 20
|
||
--n-initial 5
|
||
--epochs 8
|
||
--parallel 0
|
||
--symbol ES.FUT
|
||
--tx-cost-bps 0.1
|
||
--tick-size 0.25
|
||
--spread-ticks 1.0
|
||
--initial-capital 35000
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--base-dir ${CI_PROJECT_DIR}/output/hyperopt
|
||
--output ${CI_PROJECT_DIR}/output/ppo_hyperopt_results.json
|
||
- echo "=== PPO Hyperopt complete ==="
|
||
- cat ${CI_PROJECT_DIR}/output/ppo_hyperopt_results.json
|
||
- echo "=== Training PPO with best params ==="
|
||
- train_baseline_rl
|
||
--model ppo
|
||
--symbol ES.FUT
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output-dir ${CI_PROJECT_DIR}/output
|
||
--hyperopt-params ${CI_PROJECT_DIR}/output/ppo_hyperopt_results.json
|
||
- echo "=== Evaluating PPO with best params ==="
|
||
- evaluate_baseline
|
||
--model ppo
|
||
--models-dir ${CI_PROJECT_DIR}/output
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output ${CI_PROJECT_DIR}/output/ppo_eval_report.json
|
||
--symbol ES.FUT
|
||
--hyperopt-params ${CI_PROJECT_DIR}/output/ppo_hyperopt_results.json
|
||
- cat ${CI_PROJECT_DIR}/output/ppo_eval_report.json
|
||
artifacts:
|
||
paths:
|
||
- output/ppo_hyperopt_results.json
|
||
- output/ppo_eval_report.json
|
||
when: always
|
||
expire_in: 90 days
|
||
|
||
# --- Train-validate: remaining supervised models ---
|
||
|
||
train-validate-mamba2:
|
||
extends: .train-validate-base
|
||
needs:
|
||
- job: compile-training
|
||
optional: true
|
||
script:
|
||
- train_baseline_supervised
|
||
--model mamba2
|
||
--symbol ES.FUT
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output-dir ${CI_PROJECT_DIR}/output
|
||
--max-steps-per-epoch 50
|
||
- echo "=== Mamba2 training complete, running evaluation ==="
|
||
- evaluate_supervised
|
||
--model mamba2
|
||
--models-dir ${CI_PROJECT_DIR}/output
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output ${CI_PROJECT_DIR}/output/mamba2_eval_report.json
|
||
--symbol ES.FUT
|
||
- echo "Mamba2 validation + evaluation passed"
|
||
- cat ${CI_PROJECT_DIR}/output/mamba2_eval_report.json
|
||
artifacts:
|
||
paths:
|
||
- output/mamba2_eval_report.json
|
||
when: always
|
||
expire_in: 30 days
|
||
|
||
train-validate-liquid:
|
||
extends: .train-validate-base
|
||
needs:
|
||
- job: compile-training
|
||
optional: true
|
||
script:
|
||
- train_baseline_supervised
|
||
--model liquid
|
||
--symbol ES.FUT
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output-dir ${CI_PROJECT_DIR}/output
|
||
--max-steps-per-epoch 50
|
||
- echo "=== Liquid training complete, running evaluation ==="
|
||
- evaluate_supervised
|
||
--model liquid
|
||
--models-dir ${CI_PROJECT_DIR}/output
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output ${CI_PROJECT_DIR}/output/liquid_eval_report.json
|
||
--symbol ES.FUT
|
||
- echo "Liquid validation + evaluation passed"
|
||
- cat ${CI_PROJECT_DIR}/output/liquid_eval_report.json
|
||
artifacts:
|
||
paths:
|
||
- output/liquid_eval_report.json
|
||
when: always
|
||
expire_in: 30 days
|
||
|
||
train-validate-tggn:
|
||
extends: .train-validate-base
|
||
needs:
|
||
- job: compile-training
|
||
optional: true
|
||
script:
|
||
- train_baseline_supervised
|
||
--model tggn
|
||
--symbol ES.FUT
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output-dir ${CI_PROJECT_DIR}/output
|
||
--max-steps-per-epoch 50
|
||
- echo "=== TGGN training complete, running evaluation ==="
|
||
- evaluate_supervised
|
||
--model tggn
|
||
--models-dir ${CI_PROJECT_DIR}/output
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output ${CI_PROJECT_DIR}/output/tggn_eval_report.json
|
||
--symbol ES.FUT
|
||
- echo "TGGN validation + evaluation passed"
|
||
- cat ${CI_PROJECT_DIR}/output/tggn_eval_report.json
|
||
artifacts:
|
||
paths:
|
||
- output/tggn_eval_report.json
|
||
when: always
|
||
expire_in: 30 days
|
||
|
||
train-validate-tlob:
|
||
extends: .train-validate-base
|
||
needs:
|
||
- job: compile-training
|
||
optional: true
|
||
script:
|
||
- train_baseline_supervised
|
||
--model tlob
|
||
--symbol ES.FUT
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output-dir ${CI_PROJECT_DIR}/output
|
||
--max-steps-per-epoch 50
|
||
- echo "=== TLOB training complete, running evaluation ==="
|
||
- evaluate_supervised
|
||
--model tlob
|
||
--models-dir ${CI_PROJECT_DIR}/output
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output ${CI_PROJECT_DIR}/output/tlob_eval_report.json
|
||
--symbol ES.FUT
|
||
- echo "TLOB validation + evaluation passed"
|
||
- cat ${CI_PROJECT_DIR}/output/tlob_eval_report.json
|
||
artifacts:
|
||
paths:
|
||
- output/tlob_eval_report.json
|
||
when: always
|
||
expire_in: 30 days
|
||
|
||
train-validate-kan:
|
||
extends: .train-validate-base
|
||
needs:
|
||
- job: compile-training
|
||
optional: true
|
||
script:
|
||
- train_baseline_supervised
|
||
--model kan
|
||
--symbol ES.FUT
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output-dir ${CI_PROJECT_DIR}/output
|
||
--max-steps-per-epoch 50
|
||
- echo "=== KAN training complete, running evaluation ==="
|
||
- evaluate_supervised
|
||
--model kan
|
||
--models-dir ${CI_PROJECT_DIR}/output
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output ${CI_PROJECT_DIR}/output/kan_eval_report.json
|
||
--symbol ES.FUT
|
||
- echo "KAN validation + evaluation passed"
|
||
- cat ${CI_PROJECT_DIR}/output/kan_eval_report.json
|
||
artifacts:
|
||
paths:
|
||
- output/kan_eval_report.json
|
||
when: always
|
||
expire_in: 30 days
|
||
|
||
train-validate-xlstm:
|
||
extends: .train-validate-base
|
||
needs:
|
||
- job: compile-training
|
||
optional: true
|
||
script:
|
||
- train_baseline_supervised
|
||
--model xlstm
|
||
--symbol ES.FUT
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output-dir ${CI_PROJECT_DIR}/output
|
||
--max-steps-per-epoch 50
|
||
- echo "=== xLSTM training complete, running evaluation ==="
|
||
- evaluate_supervised
|
||
--model xlstm
|
||
--models-dir ${CI_PROJECT_DIR}/output
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output ${CI_PROJECT_DIR}/output/xlstm_eval_report.json
|
||
--symbol ES.FUT
|
||
- echo "xLSTM validation + evaluation passed"
|
||
- cat ${CI_PROJECT_DIR}/output/xlstm_eval_report.json
|
||
artifacts:
|
||
paths:
|
||
- output/xlstm_eval_report.json
|
||
when: always
|
||
expire_in: 30 days
|
||
|
||
train-validate-diffusion:
|
||
extends: .train-validate-base
|
||
needs:
|
||
- job: compile-training
|
||
optional: true
|
||
script:
|
||
- train_baseline_supervised
|
||
--model diffusion
|
||
--symbol ES.FUT
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output-dir ${CI_PROJECT_DIR}/output
|
||
--max-steps-per-epoch 50
|
||
- echo "=== Diffusion training complete, running evaluation ==="
|
||
- evaluate_supervised
|
||
--model diffusion
|
||
--models-dir ${CI_PROJECT_DIR}/output
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output ${CI_PROJECT_DIR}/output/diffusion_eval_report.json
|
||
--symbol ES.FUT
|
||
- echo "Diffusion validation + evaluation passed"
|
||
- cat ${CI_PROJECT_DIR}/output/diffusion_eval_report.json
|
||
artifacts:
|
||
paths:
|
||
- output/diffusion_eval_report.json
|
||
when: always
|
||
expire_in: 30 days
|
||
|
||
# --- Hyperopt: all models ---
|
||
|
||
hyperopt-dqn:
|
||
extends: .train-rl-base
|
||
tags:
|
||
- kapsule
|
||
- h100
|
||
variables:
|
||
KUBERNETES_NODE_SELECTOR_POOL: "k8s.scaleway.com/pool-name=ci-training-h100"
|
||
KUBERNETES_MEMORY_REQUEST: "16Gi"
|
||
KUBERNETES_MEMORY_LIMIT: "48Gi"
|
||
needs:
|
||
- job: compile-training
|
||
optional: true
|
||
timeout: 4h
|
||
script:
|
||
- |
|
||
hyperopt_baseline_rl \
|
||
--model dqn \
|
||
--trials 20 \
|
||
--n-initial 5 \
|
||
--epochs 8 \
|
||
--parallel 0 \
|
||
--symbol ES.FUT \
|
||
--tx-cost-bps 0.1 \
|
||
--tick-size 0.25 \
|
||
--spread-ticks 1.0 \
|
||
--initial-capital 35000 \
|
||
--data-dir /mnt/training-data/futures-baseline \
|
||
--base-dir ${CI_PROJECT_DIR}/output/hyperopt \
|
||
--output ${CI_PROJECT_DIR}/output/dqn_hyperopt_results.json \
|
||
&& HYPEROPT_OK=1 || HYPEROPT_OK=0
|
||
echo "=== DQN Hyperopt exit: $HYPEROPT_OK ==="
|
||
cat ${CI_PROJECT_DIR}/output/dqn_hyperopt_results.json 2>/dev/null || echo "No hyperopt results"
|
||
echo "=== Training DQN with best params ==="
|
||
train_baseline_rl \
|
||
--model dqn \
|
||
--symbol ES.FUT \
|
||
--tx-cost-bps 0.1 \
|
||
--tick-size 0.25 \
|
||
--spread-ticks 1.0 \
|
||
--data-dir /mnt/training-data/futures-baseline \
|
||
--output-dir ${CI_PROJECT_DIR}/output \
|
||
--hyperopt-params ${CI_PROJECT_DIR}/output/dqn_hyperopt_results.json \
|
||
&& TRAIN_OK=1 || TRAIN_OK=0
|
||
echo "=== Training exit: $TRAIN_OK ==="
|
||
echo "=== Evaluating DQN ==="
|
||
evaluate_baseline \
|
||
--model dqn \
|
||
--models-dir ${CI_PROJECT_DIR}/output \
|
||
--data-dir /mnt/training-data/futures-baseline \
|
||
--output ${CI_PROJECT_DIR}/output/dqn_eval_report.json \
|
||
--symbol ES.FUT \
|
||
--tx-cost-bps 0.1 \
|
||
--tick-size 0.25 \
|
||
--spread-ticks 1.0 \
|
||
--hyperopt-params ${CI_PROJECT_DIR}/output/dqn_hyperopt_results.json \
|
||
|| true
|
||
echo "=== Eval report ==="
|
||
cat ${CI_PROJECT_DIR}/output/dqn_eval_report.json 2>/dev/null || echo "No eval report"
|
||
[ "$HYPEROPT_OK" = "1" ] && [ "$TRAIN_OK" = "1" ] || exit 1
|
||
artifacts:
|
||
paths:
|
||
- output/dqn_hyperopt_results.json
|
||
- output/dqn_eval_report.json
|
||
when: always
|
||
expire_in: 90 days
|
||
|
||
hyperopt-tft:
|
||
extends: .train-validate-base
|
||
needs:
|
||
- job: compile-training
|
||
optional: true
|
||
timeout: 4h
|
||
script:
|
||
- hyperopt_baseline_supervised
|
||
--model tft
|
||
--trials 20
|
||
--n-initial 5
|
||
--epochs 15
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--base-dir ${CI_PROJECT_DIR}/output/hyperopt
|
||
--output ${CI_PROJECT_DIR}/output/tft_hyperopt_results.json
|
||
- echo "=== TFT Hyperopt complete ==="
|
||
- cat ${CI_PROJECT_DIR}/output/tft_hyperopt_results.json
|
||
- echo "=== Training TFT with best params ==="
|
||
- train_baseline_supervised
|
||
--model tft
|
||
--symbol ES.FUT
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output-dir ${CI_PROJECT_DIR}/output
|
||
--hyperopt-params ${CI_PROJECT_DIR}/output/tft_hyperopt_results.json
|
||
- echo "=== Evaluating TFT with best params ==="
|
||
- evaluate_supervised
|
||
--model tft
|
||
--models-dir ${CI_PROJECT_DIR}/output
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output ${CI_PROJECT_DIR}/output/tft_eval_report.json
|
||
--symbol ES.FUT
|
||
- cat ${CI_PROJECT_DIR}/output/tft_eval_report.json
|
||
artifacts:
|
||
paths:
|
||
- output/tft_hyperopt_results.json
|
||
- output/tft_eval_report.json
|
||
when: always
|
||
expire_in: 90 days
|
||
|
||
hyperopt-mamba2:
|
||
extends: .train-validate-base
|
||
needs:
|
||
- job: compile-training
|
||
optional: true
|
||
timeout: 4h
|
||
script:
|
||
- hyperopt_baseline_supervised
|
||
--model mamba2
|
||
--trials 20
|
||
--n-initial 5
|
||
--epochs 15
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--base-dir ${CI_PROJECT_DIR}/output/hyperopt
|
||
--output ${CI_PROJECT_DIR}/output/mamba2_hyperopt_results.json
|
||
- echo "=== Mamba2 Hyperopt complete ==="
|
||
- cat ${CI_PROJECT_DIR}/output/mamba2_hyperopt_results.json
|
||
- echo "=== Training Mamba2 with best params ==="
|
||
- train_baseline_supervised
|
||
--model mamba2
|
||
--symbol ES.FUT
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output-dir ${CI_PROJECT_DIR}/output
|
||
--hyperopt-params ${CI_PROJECT_DIR}/output/mamba2_hyperopt_results.json
|
||
- echo "=== Evaluating Mamba2 with best params ==="
|
||
- evaluate_supervised
|
||
--model mamba2
|
||
--models-dir ${CI_PROJECT_DIR}/output
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output ${CI_PROJECT_DIR}/output/mamba2_eval_report.json
|
||
--symbol ES.FUT
|
||
- cat ${CI_PROJECT_DIR}/output/mamba2_eval_report.json
|
||
artifacts:
|
||
paths:
|
||
- output/mamba2_hyperopt_results.json
|
||
- output/mamba2_eval_report.json
|
||
when: always
|
||
expire_in: 90 days
|
||
|
||
hyperopt-liquid:
|
||
extends: .train-validate-base
|
||
needs:
|
||
- job: compile-training
|
||
optional: true
|
||
timeout: 4h
|
||
script:
|
||
- hyperopt_baseline_supervised
|
||
--model liquid
|
||
--trials 20
|
||
--n-initial 5
|
||
--epochs 15
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--base-dir ${CI_PROJECT_DIR}/output/hyperopt
|
||
--output ${CI_PROJECT_DIR}/output/liquid_hyperopt_results.json
|
||
- echo "=== Liquid Hyperopt complete ==="
|
||
- cat ${CI_PROJECT_DIR}/output/liquid_hyperopt_results.json
|
||
- echo "=== Training Liquid with best params ==="
|
||
- train_baseline_supervised
|
||
--model liquid
|
||
--symbol ES.FUT
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output-dir ${CI_PROJECT_DIR}/output
|
||
--hyperopt-params ${CI_PROJECT_DIR}/output/liquid_hyperopt_results.json
|
||
- echo "=== Evaluating Liquid with best params ==="
|
||
- evaluate_supervised
|
||
--model liquid
|
||
--models-dir ${CI_PROJECT_DIR}/output
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output ${CI_PROJECT_DIR}/output/liquid_eval_report.json
|
||
--symbol ES.FUT
|
||
- cat ${CI_PROJECT_DIR}/output/liquid_eval_report.json
|
||
artifacts:
|
||
paths:
|
||
- output/liquid_hyperopt_results.json
|
||
- output/liquid_eval_report.json
|
||
when: always
|
||
expire_in: 90 days
|
||
|
||
hyperopt-tggn:
|
||
extends: .train-validate-base
|
||
needs:
|
||
- job: compile-training
|
||
optional: true
|
||
timeout: 4h
|
||
script:
|
||
- hyperopt_baseline_supervised
|
||
--model tggn
|
||
--trials 20
|
||
--n-initial 5
|
||
--epochs 15
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--base-dir ${CI_PROJECT_DIR}/output/hyperopt
|
||
--output ${CI_PROJECT_DIR}/output/tggn_hyperopt_results.json
|
||
- echo "=== TGGN Hyperopt complete ==="
|
||
- cat ${CI_PROJECT_DIR}/output/tggn_hyperopt_results.json
|
||
- echo "=== Training TGGN with best params ==="
|
||
- train_baseline_supervised
|
||
--model tggn
|
||
--symbol ES.FUT
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output-dir ${CI_PROJECT_DIR}/output
|
||
--hyperopt-params ${CI_PROJECT_DIR}/output/tggn_hyperopt_results.json
|
||
- echo "=== Evaluating TGGN with best params ==="
|
||
- evaluate_supervised
|
||
--model tggn
|
||
--models-dir ${CI_PROJECT_DIR}/output
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output ${CI_PROJECT_DIR}/output/tggn_eval_report.json
|
||
--symbol ES.FUT
|
||
- cat ${CI_PROJECT_DIR}/output/tggn_eval_report.json
|
||
artifacts:
|
||
paths:
|
||
- output/tggn_hyperopt_results.json
|
||
- output/tggn_eval_report.json
|
||
when: always
|
||
expire_in: 90 days
|
||
|
||
hyperopt-tlob:
|
||
extends: .train-validate-base
|
||
needs:
|
||
- job: compile-training
|
||
optional: true
|
||
timeout: 4h
|
||
script:
|
||
- hyperopt_baseline_supervised
|
||
--model tlob
|
||
--trials 20
|
||
--n-initial 5
|
||
--epochs 15
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--base-dir ${CI_PROJECT_DIR}/output/hyperopt
|
||
--output ${CI_PROJECT_DIR}/output/tlob_hyperopt_results.json
|
||
- echo "=== TLOB Hyperopt complete ==="
|
||
- cat ${CI_PROJECT_DIR}/output/tlob_hyperopt_results.json
|
||
- echo "=== Training TLOB with best params ==="
|
||
- train_baseline_supervised
|
||
--model tlob
|
||
--symbol ES.FUT
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output-dir ${CI_PROJECT_DIR}/output
|
||
--hyperopt-params ${CI_PROJECT_DIR}/output/tlob_hyperopt_results.json
|
||
- echo "=== Evaluating TLOB with best params ==="
|
||
- evaluate_supervised
|
||
--model tlob
|
||
--models-dir ${CI_PROJECT_DIR}/output
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output ${CI_PROJECT_DIR}/output/tlob_eval_report.json
|
||
--symbol ES.FUT
|
||
- cat ${CI_PROJECT_DIR}/output/tlob_eval_report.json
|
||
artifacts:
|
||
paths:
|
||
- output/tlob_hyperopt_results.json
|
||
- output/tlob_eval_report.json
|
||
when: always
|
||
expire_in: 90 days
|
||
|
||
hyperopt-kan:
|
||
extends: .train-validate-base
|
||
needs:
|
||
- job: compile-training
|
||
optional: true
|
||
timeout: 4h
|
||
script:
|
||
- hyperopt_baseline_supervised
|
||
--model kan
|
||
--trials 20
|
||
--n-initial 5
|
||
--epochs 15
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--base-dir ${CI_PROJECT_DIR}/output/hyperopt
|
||
--output ${CI_PROJECT_DIR}/output/kan_hyperopt_results.json
|
||
- echo "=== KAN Hyperopt complete ==="
|
||
- cat ${CI_PROJECT_DIR}/output/kan_hyperopt_results.json
|
||
- echo "=== Training KAN with best params ==="
|
||
- train_baseline_supervised
|
||
--model kan
|
||
--symbol ES.FUT
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output-dir ${CI_PROJECT_DIR}/output
|
||
--hyperopt-params ${CI_PROJECT_DIR}/output/kan_hyperopt_results.json
|
||
- echo "=== Evaluating KAN with best params ==="
|
||
- evaluate_supervised
|
||
--model kan
|
||
--models-dir ${CI_PROJECT_DIR}/output
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output ${CI_PROJECT_DIR}/output/kan_eval_report.json
|
||
--symbol ES.FUT
|
||
- cat ${CI_PROJECT_DIR}/output/kan_eval_report.json
|
||
artifacts:
|
||
paths:
|
||
- output/kan_hyperopt_results.json
|
||
- output/kan_eval_report.json
|
||
when: always
|
||
expire_in: 90 days
|
||
|
||
hyperopt-xlstm:
|
||
extends: .train-validate-base
|
||
needs:
|
||
- job: compile-training
|
||
optional: true
|
||
timeout: 4h
|
||
script:
|
||
- hyperopt_baseline_supervised
|
||
--model xlstm
|
||
--trials 20
|
||
--n-initial 5
|
||
--epochs 15
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--base-dir ${CI_PROJECT_DIR}/output/hyperopt
|
||
--output ${CI_PROJECT_DIR}/output/xlstm_hyperopt_results.json
|
||
- echo "=== xLSTM Hyperopt complete ==="
|
||
- cat ${CI_PROJECT_DIR}/output/xlstm_hyperopt_results.json
|
||
- echo "=== Training xLSTM with best params ==="
|
||
- train_baseline_supervised
|
||
--model xlstm
|
||
--symbol ES.FUT
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output-dir ${CI_PROJECT_DIR}/output
|
||
--hyperopt-params ${CI_PROJECT_DIR}/output/xlstm_hyperopt_results.json
|
||
- echo "=== Evaluating xLSTM with best params ==="
|
||
- evaluate_supervised
|
||
--model xlstm
|
||
--models-dir ${CI_PROJECT_DIR}/output
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output ${CI_PROJECT_DIR}/output/xlstm_eval_report.json
|
||
--symbol ES.FUT
|
||
- cat ${CI_PROJECT_DIR}/output/xlstm_eval_report.json
|
||
artifacts:
|
||
paths:
|
||
- output/xlstm_hyperopt_results.json
|
||
- output/xlstm_eval_report.json
|
||
when: always
|
||
expire_in: 90 days
|
||
|
||
hyperopt-diffusion:
|
||
extends: .train-validate-base
|
||
needs:
|
||
- job: compile-training
|
||
optional: true
|
||
timeout: 4h
|
||
script:
|
||
- hyperopt_baseline_supervised
|
||
--model diffusion
|
||
--trials 20
|
||
--n-initial 5
|
||
--epochs 15
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--base-dir ${CI_PROJECT_DIR}/output/hyperopt
|
||
--output ${CI_PROJECT_DIR}/output/diffusion_hyperopt_results.json
|
||
- echo "=== Diffusion Hyperopt complete ==="
|
||
- cat ${CI_PROJECT_DIR}/output/diffusion_hyperopt_results.json
|
||
- echo "=== Training Diffusion with best params ==="
|
||
- train_baseline_supervised
|
||
--model diffusion
|
||
--symbol ES.FUT
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output-dir ${CI_PROJECT_DIR}/output
|
||
--hyperopt-params ${CI_PROJECT_DIR}/output/diffusion_hyperopt_results.json
|
||
- echo "=== Evaluating Diffusion with best params ==="
|
||
- evaluate_supervised
|
||
--model diffusion
|
||
--models-dir ${CI_PROJECT_DIR}/output
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output ${CI_PROJECT_DIR}/output/diffusion_eval_report.json
|
||
--symbol ES.FUT
|
||
- cat ${CI_PROJECT_DIR}/output/diffusion_eval_report.json
|
||
artifacts:
|
||
paths:
|
||
- output/diffusion_hyperopt_results.json
|
||
- output/diffusion_eval_report.json
|
||
when: always
|
||
expire_in: 90 days
|
||
|
||
# --------------------------------------------------------------------------
|
||
# IaC: Terragrunt plan on MR (runs on gitlab pool)
|
||
# --------------------------------------------------------------------------
|
||
infra-plan:
|
||
stage: test
|
||
image: ${INFRA_RUNNER_IMAGE}
|
||
tags:
|
||
- kapsule
|
||
- docker
|
||
needs: []
|
||
rules:
|
||
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
|
||
changes:
|
||
- infra/**
|
||
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
|
||
changes:
|
||
- infra/**
|
||
before_script:
|
||
- export SCW_ACCESS_KEY=${SCW_ACCESS_KEY}
|
||
- export SCW_SECRET_KEY=${SCW_SECRET_KEY}
|
||
- export SCW_DEFAULT_PROJECT_ID=${SCW_DEFAULT_PROJECT_ID}
|
||
- export AWS_ACCESS_KEY_ID=${SCW_ACCESS_KEY}
|
||
- export AWS_SECRET_ACCESS_KEY=${SCW_SECRET_KEY}
|
||
script:
|
||
- cd infra/live/production
|
||
- terragrunt run-all plan --terragrunt-non-interactive 2>&1 | tee ${CI_PROJECT_DIR}/plan-output.txt
|
||
- echo "Plan completed — review output, then trigger infra-apply manually"
|
||
artifacts:
|
||
paths:
|
||
- plan-output.txt
|
||
when: always
|
||
expire_in: 7 days
|
||
|
||
# --------------------------------------------------------------------------
|
||
# IaC: Terragrunt apply (auto on main push, manual from web/api)
|
||
# --------------------------------------------------------------------------
|
||
infra-apply:
|
||
stage: deploy
|
||
image: ${INFRA_RUNNER_IMAGE}
|
||
tags:
|
||
- kapsule
|
||
- docker
|
||
needs:
|
||
- job: infra-plan
|
||
optional: true
|
||
rules:
|
||
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
|
||
changes:
|
||
- infra/**
|
||
- if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
|
||
when: manual
|
||
before_script:
|
||
- export SCW_ACCESS_KEY=${SCW_ACCESS_KEY}
|
||
- export SCW_SECRET_KEY=${SCW_SECRET_KEY}
|
||
- export SCW_DEFAULT_PROJECT_ID=${SCW_DEFAULT_PROJECT_ID}
|
||
- export AWS_ACCESS_KEY_ID=${SCW_ACCESS_KEY}
|
||
- export AWS_SECRET_ACCESS_KEY=${SCW_SECRET_KEY}
|
||
script:
|
||
- cd infra/live/production
|
||
- terragrunt run-all apply --terragrunt-non-interactive -auto-approve
|
||
environment:
|
||
name: production/infrastructure
|
||
|
||
# --------------------------------------------------------------------------
|
||
# IaC: Weekly drift detection (scheduled pipeline, gitlab pool)
|
||
# --------------------------------------------------------------------------
|
||
infra-drift-check:
|
||
stage: test
|
||
image: ${INFRA_RUNNER_IMAGE}
|
||
tags:
|
||
- kapsule
|
||
- docker
|
||
needs: []
|
||
rules:
|
||
- if: $CI_PIPELINE_SOURCE == "schedule" && $DRIFT_CHECK == "true"
|
||
before_script:
|
||
- export SCW_ACCESS_KEY=${SCW_ACCESS_KEY}
|
||
- export SCW_SECRET_KEY=${SCW_SECRET_KEY}
|
||
- export SCW_DEFAULT_PROJECT_ID=${SCW_DEFAULT_PROJECT_ID}
|
||
- export AWS_ACCESS_KEY_ID=${SCW_ACCESS_KEY}
|
||
- export AWS_SECRET_ACCESS_KEY=${SCW_SECRET_KEY}
|
||
script:
|
||
- cd infra/live/production
|
||
- |
|
||
terragrunt run-all plan -detailed-exitcode --terragrunt-non-interactive \
|
||
2>&1 | tee ${CI_PROJECT_DIR}/drift-output.txt; EXIT_CODE=$?
|
||
if [ "$EXIT_CODE" -eq 2 ]; then
|
||
echo "DRIFT DETECTED — creating GitLab issue"
|
||
glab issue create \
|
||
--title "IaC Drift Detected ($(date +%Y-%m-%d))" \
|
||
--description "$(tail -100 ${CI_PROJECT_DIR}/drift-output.txt)" \
|
||
--label "infrastructure,drift"
|
||
exit 1
|
||
elif [ "$EXIT_CODE" -ne 0 ]; then
|
||
exit $EXIT_CODE
|
||
fi
|
||
echo "No drift detected"
|
||
artifacts:
|
||
paths:
|
||
- drift-output.txt
|
||
when: always
|
||
expire_in: 7 days
|
||
|
||
# --------------------------------------------------------------------------
|
||
# Stage 4: Deploy base templates
|
||
# --------------------------------------------------------------------------
|
||
# Shared kubeconfig setup for all deploy jobs
|
||
.deploy-base:
|
||
stage: deploy
|
||
image: ${INFRA_RUNNER_IMAGE}
|
||
tags:
|
||
- kapsule
|
||
- docker
|
||
variables:
|
||
KUBERNETES_CPU_REQUEST: "200m"
|
||
KUBERNETES_CPU_LIMIT: "500m"
|
||
KUBERNETES_MEMORY_REQUEST: "256Mi"
|
||
KUBERNETES_MEMORY_LIMIT: "512Mi"
|
||
before_script:
|
||
- which kubectl || (curl -fsSL "https://dl.k8s.io/release/v1.31.4/bin/linux/amd64/kubectl" -o /usr/local/bin/kubectl && chmod +x /usr/local/bin/kubectl)
|
||
- export SCW_ACCESS_KEY=${SCW_ACCESS_KEY}
|
||
- export SCW_SECRET_KEY=${SCW_SECRET_KEY}
|
||
- export SCW_DEFAULT_PROJECT_ID=${SCW_DEFAULT_PROJECT_ID}
|
||
- export SCW_DEFAULT_ORGANIZATION_ID=${SCW_DEFAULT_ORGANIZATION_ID:-${SCW_DEFAULT_PROJECT_ID}}
|
||
- mkdir -p ~/.kube
|
||
- echo "$KUBECONFIG_B64" | base64 -d > ~/.kube/config
|
||
- chmod 600 ~/.kube/config
|
||
- sed -i 's|https://.*\.api\.k8s\.fr-par\.scw\.cloud:6443|https://kubernetes.default.svc|g' ~/.kube/config
|
||
- kubectl cluster-info
|
||
environment:
|
||
name: production
|
||
kubernetes:
|
||
namespace: foxhunt
|
||
|
||
# Per-service deploy: upload binary → apply manifest → rollout restart
|
||
# Each service deploys IMMEDIATELY after its compile job finishes.
|
||
.deploy-service-base:
|
||
extends: .deploy-base
|
||
script:
|
||
- export AWS_ACCESS_KEY_ID="${MINIO_ACCESS_KEY}"
|
||
- export AWS_SECRET_ACCESS_KEY="${MINIO_SECRET_KEY}"
|
||
- S3="aws s3 --endpoint-url http://minio.foxhunt.svc.cluster.local:9000"
|
||
- echo "Deploying ${SERVICE_PACKAGE}..."
|
||
- $S3 cp build-out/services/${SERVICE_PACKAGE} s3://foxhunt-binaries/services/${SERVICE_PACKAGE}
|
||
- kubectl apply -f infra/k8s/services/${SERVICE_PACKAGE}.yaml
|
||
- kubectl -n foxhunt rollout restart deployment/${SERVICE_PACKAGE}
|
||
- kubectl -n foxhunt rollout status deployment/${SERVICE_PACKAGE} --timeout=300s
|
||
- echo "Deployed ${SERVICE_PACKAGE}"
|
||
|
||
# --------------------------------------------------------------------------
|
||
# Stage 4a: Per-service deploy (fires as each compile finishes)
|
||
# Rules use YAML anchors shared with compile jobs — ensures compile+deploy
|
||
# are created/skipped together.
|
||
# --------------------------------------------------------------------------
|
||
deploy-api:
|
||
extends: .deploy-service-base
|
||
needs: [{job: compile-api, artifacts: true}]
|
||
variables:
|
||
SERVICE_PACKAGE: api
|
||
rules: *rules-api
|
||
|
||
deploy-trading-service:
|
||
extends: .deploy-service-base
|
||
needs: [{job: compile-trading-service, artifacts: true}]
|
||
variables:
|
||
SERVICE_PACKAGE: trading-service
|
||
rules: *rules-trading-service
|
||
|
||
deploy-ml-training-service:
|
||
extends: .deploy-service-base
|
||
needs: [{job: compile-ml-training-service, artifacts: true}]
|
||
variables:
|
||
SERVICE_PACKAGE: ml-training-service
|
||
rules: *rules-ml-training-service
|
||
|
||
deploy-backtesting-service:
|
||
extends: .deploy-service-base
|
||
needs: [{job: compile-backtesting-service, artifacts: true}]
|
||
variables:
|
||
SERVICE_PACKAGE: backtesting-service
|
||
rules: *rules-backtesting-service
|
||
|
||
deploy-trading-agent-service:
|
||
extends: .deploy-service-base
|
||
needs: [{job: compile-trading-agent-service, artifacts: true}]
|
||
variables:
|
||
SERVICE_PACKAGE: trading-agent-service
|
||
rules: *rules-trading-agent-service
|
||
|
||
deploy-broker-gateway:
|
||
extends: .deploy-service-base
|
||
needs: [{job: compile-broker-gateway, artifacts: true}]
|
||
variables:
|
||
SERVICE_PACKAGE: broker-gateway
|
||
rules: *rules-broker-gateway
|
||
|
||
deploy-data-acquisition-service:
|
||
extends: .deploy-service-base
|
||
needs: [{job: compile-data-acquisition-service, artifacts: true}]
|
||
variables:
|
||
SERVICE_PACKAGE: data-acquisition-service
|
||
rules: *rules-data-acquisition-service
|
||
|
||
# --------------------------------------------------------------------------
|
||
# Stage 4b: Deploy training binaries to MinIO
|
||
# --------------------------------------------------------------------------
|
||
deploy-training:
|
||
extends: .deploy-base
|
||
needs:
|
||
- job: compile-training
|
||
artifacts: true
|
||
rules:
|
||
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
|
||
changes:
|
||
- Cargo.toml
|
||
- Cargo.lock
|
||
- crates/ml/**
|
||
- crates/trading_engine/**
|
||
- crates/common/**
|
||
- crates/risk/**
|
||
- crates/data/**
|
||
- crates/config/**
|
||
- crates/storage/**
|
||
- services/training_uploader/**
|
||
- if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
|
||
script:
|
||
- export AWS_ACCESS_KEY_ID="${MINIO_ACCESS_KEY}"
|
||
- export AWS_SECRET_ACCESS_KEY="${MINIO_SECRET_KEY}"
|
||
- S3="aws s3 --endpoint-url http://minio.foxhunt.svc.cluster.local:9000"
|
||
- $S3 cp build-out/training/ s3://foxhunt-binaries/training/ --recursive
|
||
- echo "Training binaries uploaded to MinIO"
|
||
|
||
# --------------------------------------------------------------------------
|
||
# Stage 4c: Deploy web dashboard assets to MinIO
|
||
# --------------------------------------------------------------------------
|
||
deploy-web-dashboard:
|
||
extends: .deploy-base
|
||
needs:
|
||
- job: build-web-dashboard
|
||
artifacts: true
|
||
rules:
|
||
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
|
||
changes:
|
||
- web-dashboard/**
|
||
- if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
|
||
script:
|
||
- export AWS_ACCESS_KEY_ID="${MINIO_ACCESS_KEY}"
|
||
- export AWS_SECRET_ACCESS_KEY="${MINIO_SECRET_KEY}"
|
||
- S3="aws s3 --endpoint-url http://minio.foxhunt.svc.cluster.local:9000"
|
||
- $S3 cp web-dashboard/dist/ s3://foxhunt-binaries/static/ --recursive
|
||
- echo "Web dashboard assets uploaded to MinIO"
|
||
|
||
# --------------------------------------------------------------------------
|
||
# Stage 4d: Deploy infrastructure (migrations, monitoring, dashboards, proxy)
|
||
# Runs on every main push — no compile dependency.
|
||
# --------------------------------------------------------------------------
|
||
deploy-infra:
|
||
extends: .deploy-base
|
||
needs: []
|
||
rules:
|
||
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
|
||
- if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
|
||
script:
|
||
- echo "Deploying infrastructure ${CI_COMMIT_SHA}..."
|
||
# Apply pending database migrations
|
||
- |
|
||
echo "Applying database migrations..."
|
||
DB_POD=$(kubectl -n foxhunt get pod -l app.kubernetes.io/name=postgres -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
|
||
if [ -n "$DB_POD" ]; then
|
||
kubectl -n foxhunt exec "$DB_POD" -- psql -U foxhunt -d foxhunt -c \
|
||
"CREATE TABLE IF NOT EXISTS _applied_migrations (name TEXT PRIMARY KEY, applied_at TIMESTAMPTZ DEFAULT now())" 2>/dev/null || true
|
||
for f in $(ls migrations/*.sql | grep -v '\.down\.' | sort); do
|
||
MIGRATION_NAME=$(basename "$f")
|
||
ALREADY=$(kubectl -n foxhunt exec "$DB_POD" -- psql -U foxhunt -d foxhunt -tAc \
|
||
"SELECT 1 FROM _applied_migrations WHERE name = '${MIGRATION_NAME}'" 2>/dev/null || true)
|
||
if [ "$ALREADY" = "1" ]; then
|
||
echo " skip: $MIGRATION_NAME (already applied)"
|
||
else
|
||
echo " apply: $MIGRATION_NAME"
|
||
kubectl -n foxhunt exec -i "$DB_POD" -- psql -U foxhunt -d foxhunt < "$f"
|
||
kubectl -n foxhunt exec "$DB_POD" -- psql -U foxhunt -d foxhunt -c \
|
||
"INSERT INTO _applied_migrations (name) VALUES ('${MIGRATION_NAME}')"
|
||
fi
|
||
done
|
||
echo "Migrations complete"
|
||
else
|
||
echo "WARNING: postgres pod not found, skipping migrations"
|
||
fi
|
||
# Deploy monitoring stack
|
||
- kubectl apply -f infra/k8s/monitoring/loki.yaml -f infra/k8s/monitoring/tempo.yaml -f infra/k8s/monitoring/promtail.yaml -f infra/k8s/monitoring/node-exporter.yaml -f infra/k8s/monitoring/kube-state-metrics.yaml
|
||
# Deploy Grafana dashboards
|
||
- |
|
||
PROM_UID="PBFA97CFB590B2093"
|
||
LOKI_UID="P8E80F9AEF21F6940"
|
||
TEMPO_UID="tempo"
|
||
DASH_DIR="infra/k8s/monitoring/dashboards"
|
||
TMP_DIR=$(mktemp -d)
|
||
for f in "$DASH_DIR"/*.json; do
|
||
python3 -c "
|
||
import json, sys
|
||
with open('$f') as fh:
|
||
d = json.load(fh)
|
||
d.pop('__inputs', None)
|
||
d.pop('__requires', None)
|
||
raw = json.dumps(d)
|
||
raw = raw.replace('\${DS_PROMETHEUS}', '${PROM_UID}')
|
||
raw = raw.replace('\${DS_LOKI}', '${LOKI_UID}')
|
||
raw = raw.replace('\${DS_TEMPO}', '${TEMPO_UID}')
|
||
print(raw)
|
||
" > "$TMP_DIR/$(basename "$f")"
|
||
done
|
||
apply_dashboard_cm() {
|
||
cm_name="$1"; folder="$2"; shift 2
|
||
args=""
|
||
for df in "$@"; do
|
||
[ -f "$TMP_DIR/$df" ] && args="$args --from-file=$df=$TMP_DIR/$df"
|
||
done
|
||
if [ -n "$args" ]; then
|
||
kubectl create configmap "$cm_name" -n foxhunt $args \
|
||
--dry-run=client -o yaml | \
|
||
kubectl label -f - --dry-run=client -o yaml --local grafana_dashboard=1 | \
|
||
kubectl annotate -f - --dry-run=client -o yaml --local "grafana_folder=$folder" | \
|
||
kubectl apply -f -
|
||
echo " Applied ConfigMap: $cm_name (folder: $folder)"
|
||
fi
|
||
}
|
||
apply_dashboard_cm grafana-dashboards-foxhunt Foxhunt \
|
||
foxhunt-logs.json foxhunt-overview.json foxhunt-services.json \
|
||
foxhunt-cockpit.json foxhunt-trading-cockpit.json foxhunt-training-cockpit.json foxhunt-traces.json
|
||
apply_dashboard_cm grafana-dashboards-infra Infrastructure \
|
||
cluster-overview.json gpu-overview.json foxhunt-gpu-training.json foxhunt-infrastructure.json
|
||
apply_dashboard_cm grafana-dashboards-cicd "CI/CD" \
|
||
foxhunt-ci-pipelines.json gitlab-services.json
|
||
rm -rf "$TMP_DIR"
|
||
echo "Dashboard ConfigMaps deployed"
|
||
# Apply proxy + service manifests
|
||
- kubectl apply -f infra/k8s/gitlab/tailscale-proxy.yaml
|
||
- kubectl -n foxhunt rollout restart deployment/tailscale-gitlab-proxy
|
||
- kubectl apply -f infra/k8s/services/
|
||
- echo "Infrastructure deploy complete"
|