fcf9ec3ba5
The Grafana sidecar supports grafana_folder annotations to place dashboards in specific folders. Without this, all provisioned dashboards land in the General folder. Add Foxhunt/Infrastructure/CI-CD folder mapping to both CI pipeline and import.sh. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1528 lines
57 KiB
YAML
1528 lines
57 KiB
YAML
# GitLab CI/CD — Foxhunt
|
||
# Two runners: CPU (compile/build/deploy) and GPU (training)
|
||
# CI builder images:
|
||
# ci-builder: CUDA 12.4 + Rust 1.89 + protoc + sccache (CUDA stubs for compile)
|
||
# ci-builder-cpu: Rust 1.89 + protoc + sccache, no CUDA (for service binaries)
|
||
# Service images: pushed to Scaleway Container Registry (Kaniko)
|
||
#
|
||
# Runner routing (by tags):
|
||
# kapsule,rust → CPU runner → ci-compile-cpu pool (POP2-32C-128G, 32 vCPU)
|
||
# kapsule,gpu → GPU runner → ci-training (L40S)
|
||
# Override nodeSelector: KUBERNETES_NODE_SELECTOR_k8s.scaleway.com/pool-name: "<name>" (k=v format)
|
||
# Pools: ci-compile-cpu (POP2-32C-128G), ci-rl (L4), ci-training (L40S), services, gitlab
|
||
# All compilation (services + training) runs on ci-compile-cpu. CUDA stubs in ci-builder image.
|
||
|
||
# Ensure pipeline is always created — individual job rules handle filtering.
|
||
# Without this, GitLab CE may silently drop push pipelines when changes:
|
||
# evaluation fails to match (known issue with path glob evaluation).
|
||
workflow:
|
||
rules:
|
||
- if: $CI_PIPELINE_SOURCE == "push"
|
||
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
|
||
- if: $CI_PIPELINE_SOURCE == "schedule"
|
||
- if: $CI_PIPELINE_SOURCE == "api"
|
||
- if: $CI_PIPELINE_SOURCE == "web"
|
||
|
||
stages:
|
||
- prepare
|
||
- test
|
||
- compile
|
||
- train
|
||
- deploy
|
||
|
||
variables:
|
||
SQLX_OFFLINE: "true"
|
||
CARGO_TERM_COLOR: always
|
||
# CUDA_COMPUTE_CAP is set at runtime by detect_gpu() — L4=89, H100=90
|
||
# Do NOT hardcode here; sccache is partitioned by compute cap below
|
||
# Stable JWT secret for api_gateway tests (prevents env-var race between parallel tests)
|
||
JWT_SECRET: "CiTestSecret_Kx7mP9nR2sW5vY8bC3fG6jH1kL4pQ7tZ0uN9dM5eV8xS2wT6yA4zB_64chars_min"
|
||
# Training data on block storage PVC (mounted at /mnt/training-data by runner)
|
||
FOXHUNT_DATA_DIR: "/mnt/training-data/futures-baseline"
|
||
# sccache base dir on PVC — GPU stages override to /mnt/sccache/sm_<cap>
|
||
SCCACHE_DIR: "/mnt/sccache"
|
||
RUSTC_WRAPPER: /usr/local/bin/sccache
|
||
# Build profile: "dev-release" (thin LTO, fast) is default for quick iteration
|
||
# Set DEV_RELEASE=false in pipeline vars to use full "release" (fat LTO) profile
|
||
DEV_RELEASE: "true"
|
||
CARGO_BUILD_PROFILE: dev-release
|
||
# S3 credentials for Kaniko registry auth and Dockerfile builds
|
||
AWS_ACCESS_KEY_ID: $SCW_ACCESS_KEY
|
||
AWS_SECRET_ACCESS_KEY: $SCW_SECRET_KEY
|
||
# CI builder image on Scaleway Container Registry (reachable by Kapsule nodes)
|
||
CI_BUILDER_IMAGE: rg.fr-par.scw.cloud/foxhunt-ci/ci-builder:latest
|
||
CI_BUILDER_CPU_IMAGE: rg.fr-par.scw.cloud/foxhunt-ci/ci-builder-cpu:latest
|
||
# Scaleway Container Registry for service images
|
||
REGISTRY: rg.fr-par.scw.cloud/foxhunt-ci
|
||
# IaC runner image on Scaleway Container Registry
|
||
INFRA_RUNNER_IMAGE: rg.fr-par.scw.cloud/foxhunt-ci/infra-runner:latest
|
||
# Generic runtime images for S3 binary share (service pods + training pods + node builder)
|
||
RUNTIME_IMAGE: rg.fr-par.scw.cloud/foxhunt-ci/foxhunt-runtime:latest
|
||
TRAINING_RUNTIME_IMAGE: rg.fr-par.scw.cloud/foxhunt-ci/foxhunt-training-runtime:latest
|
||
# Node builder image eliminated — web dashboard uses node:22-slim + inline rclone
|
||
|
||
# --------------------------------------------------------------------------
|
||
# Stage 0: Build CI builder image → push to Scaleway CR
|
||
# --------------------------------------------------------------------------
|
||
build-ci-builder:
|
||
stage: prepare
|
||
image:
|
||
name: gcr.io/kaniko-project/executor:debug
|
||
entrypoint: [""]
|
||
tags:
|
||
- kapsule
|
||
- docker
|
||
rules:
|
||
- if: $CI_PIPELINE_SOURCE == "push"
|
||
changes:
|
||
- infra/docker/Dockerfile.ci-builder
|
||
when: on_success
|
||
- when: manual
|
||
allow_failure: true
|
||
before_script:
|
||
- mkdir -p /kaniko/.docker
|
||
- >-
|
||
echo "{\"auths\":{
|
||
\"rg.fr-par.scw.cloud\":{\"username\":\"nologin\",\"password\":\"${SCW_SECRET_KEY}\"},
|
||
\"https://index.docker.io/v1/\":{\"username\":\"${DOCKERHUB_USERNAME}\",\"password\":\"${DOCKERHUB_TOKEN}\"}
|
||
}}" > /kaniko/.docker/config.json
|
||
script:
|
||
- /kaniko/executor
|
||
--context "${CI_PROJECT_DIR}"
|
||
--dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.ci-builder"
|
||
--cache=true --cache-repo="${REGISTRY}/cache"
|
||
--destination "${CI_BUILDER_IMAGE}"
|
||
|
||
# --------------------------------------------------------------------------
|
||
# Stage 0a: Build CPU-only CI builder image → push to Scaleway CR
|
||
# --------------------------------------------------------------------------
|
||
build-ci-builder-cpu:
|
||
stage: prepare
|
||
image:
|
||
name: gcr.io/kaniko-project/executor:debug
|
||
entrypoint: [""]
|
||
tags:
|
||
- kapsule
|
||
- docker
|
||
rules:
|
||
- if: $CI_PIPELINE_SOURCE == "push"
|
||
changes:
|
||
- infra/docker/Dockerfile.ci-builder-cpu
|
||
when: on_success
|
||
- when: manual
|
||
allow_failure: true
|
||
before_script:
|
||
- mkdir -p /kaniko/.docker
|
||
- >-
|
||
echo "{\"auths\":{
|
||
\"rg.fr-par.scw.cloud\":{\"username\":\"nologin\",\"password\":\"${SCW_SECRET_KEY}\"},
|
||
\"https://index.docker.io/v1/\":{\"username\":\"${DOCKERHUB_USERNAME}\",\"password\":\"${DOCKERHUB_TOKEN}\"}
|
||
}}" > /kaniko/.docker/config.json
|
||
script:
|
||
- /kaniko/executor
|
||
--context "${CI_PROJECT_DIR}"
|
||
--dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.ci-builder-cpu"
|
||
--cache=true --cache-repo="${REGISTRY}/cache"
|
||
--destination "${CI_BUILDER_CPU_IMAGE}"
|
||
|
||
# --------------------------------------------------------------------------
|
||
# Stage 0b: Build devcontainer image → push to internal GitLab registry
|
||
# --------------------------------------------------------------------------
|
||
build-devcontainer:
|
||
stage: prepare
|
||
image:
|
||
name: gcr.io/kaniko-project/executor:debug
|
||
entrypoint: [""]
|
||
tags:
|
||
- kapsule
|
||
- docker
|
||
rules:
|
||
- if: $CI_PIPELINE_SOURCE == "push"
|
||
changes:
|
||
- .devcontainer/**
|
||
when: on_success
|
||
- when: manual
|
||
allow_failure: true
|
||
before_script:
|
||
- mkdir -p /kaniko/.docker
|
||
- >-
|
||
echo "{\"auths\":{
|
||
\"rg.fr-par.scw.cloud\":{\"username\":\"nologin\",\"password\":\"${SCW_SECRET_KEY}\"},
|
||
\"https://index.docker.io/v1/\":{\"username\":\"${DOCKERHUB_USERNAME}\",\"password\":\"${DOCKERHUB_TOKEN}\"}
|
||
}}" > /kaniko/.docker/config.json
|
||
script:
|
||
- /kaniko/executor
|
||
--context "${CI_PROJECT_DIR}"
|
||
--dockerfile "${CI_PROJECT_DIR}/.devcontainer/Dockerfile"
|
||
--cache=true --cache-repo="${REGISTRY}/cache"
|
||
--destination "rg.fr-par.scw.cloud/foxhunt-ci/devcontainer:${CI_COMMIT_SHA}"
|
||
--destination "rg.fr-par.scw.cloud/foxhunt-ci/devcontainer:latest"
|
||
|
||
# --------------------------------------------------------------------------
|
||
# Stage 0c: Build infra-runner image → push to Scaleway CR
|
||
# --------------------------------------------------------------------------
|
||
build-infra-runner:
|
||
stage: prepare
|
||
image:
|
||
name: gcr.io/kaniko-project/executor:debug
|
||
entrypoint: [""]
|
||
tags:
|
||
- kapsule
|
||
- docker
|
||
rules:
|
||
- if: $CI_PIPELINE_SOURCE == "push"
|
||
changes:
|
||
- infra/docker/Dockerfile.infra-runner
|
||
when: on_success
|
||
- when: manual
|
||
allow_failure: true
|
||
before_script:
|
||
- mkdir -p /kaniko/.docker
|
||
- >-
|
||
echo "{\"auths\":{
|
||
\"rg.fr-par.scw.cloud\":{\"username\":\"nologin\",\"password\":\"${SCW_SECRET_KEY}\"},
|
||
\"https://index.docker.io/v1/\":{\"username\":\"${DOCKERHUB_USERNAME}\",\"password\":\"${DOCKERHUB_TOKEN}\"}
|
||
}}" > /kaniko/.docker/config.json
|
||
script:
|
||
- /kaniko/executor
|
||
--context "${CI_PROJECT_DIR}"
|
||
--dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.infra-runner"
|
||
--cache=true --cache-repo="${REGISTRY}/cache"
|
||
--destination "${INFRA_RUNNER_IMAGE}"
|
||
|
||
# --------------------------------------------------------------------------
|
||
# Stage 0d: Build generic runtime base images → push to Scaleway CR
|
||
# --------------------------------------------------------------------------
|
||
build-foxhunt-runtime:
|
||
stage: prepare
|
||
image:
|
||
name: gcr.io/kaniko-project/executor:debug
|
||
entrypoint: [""]
|
||
tags:
|
||
- kapsule
|
||
- docker
|
||
rules:
|
||
- if: $CI_PIPELINE_SOURCE == "push"
|
||
changes:
|
||
- infra/docker/Dockerfile.foxhunt-runtime
|
||
when: on_success
|
||
- when: manual
|
||
allow_failure: true
|
||
before_script:
|
||
- mkdir -p /kaniko/.docker
|
||
- >-
|
||
echo "{\"auths\":{
|
||
\"rg.fr-par.scw.cloud\":{\"username\":\"nologin\",\"password\":\"${SCW_SECRET_KEY}\"},
|
||
\"https://index.docker.io/v1/\":{\"username\":\"${DOCKERHUB_USERNAME}\",\"password\":\"${DOCKERHUB_TOKEN}\"}
|
||
}}" > /kaniko/.docker/config.json
|
||
script:
|
||
- /kaniko/executor
|
||
--context "${CI_PROJECT_DIR}"
|
||
--dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.foxhunt-runtime"
|
||
--cache=true --cache-repo="${REGISTRY}/cache"
|
||
--destination "${REGISTRY}/foxhunt-runtime:latest"
|
||
|
||
build-foxhunt-training-runtime:
|
||
stage: prepare
|
||
image:
|
||
name: gcr.io/kaniko-project/executor:debug
|
||
entrypoint: [""]
|
||
tags:
|
||
- kapsule
|
||
- docker
|
||
rules:
|
||
- if: $CI_PIPELINE_SOURCE == "push"
|
||
changes:
|
||
- infra/docker/Dockerfile.foxhunt-training-runtime
|
||
when: on_success
|
||
- when: manual
|
||
allow_failure: true
|
||
before_script:
|
||
- mkdir -p /kaniko/.docker
|
||
- >-
|
||
echo "{\"auths\":{
|
||
\"rg.fr-par.scw.cloud\":{\"username\":\"nologin\",\"password\":\"${SCW_SECRET_KEY}\"},
|
||
\"https://index.docker.io/v1/\":{\"username\":\"${DOCKERHUB_USERNAME}\",\"password\":\"${DOCKERHUB_TOKEN}\"}
|
||
}}" > /kaniko/.docker/config.json
|
||
script:
|
||
- /kaniko/executor
|
||
--context "${CI_PROJECT_DIR}"
|
||
--dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.foxhunt-training-runtime"
|
||
--cache=true --cache-repo="${REGISTRY}/cache"
|
||
--destination "${REGISTRY}/foxhunt-training-runtime:latest"
|
||
|
||
|
||
# Base template for Rust jobs — pre-baked CI builder from Scaleway CR
|
||
# Image pre-exists in SCR; rebuild via build-ci-builder when Dockerfile changes
|
||
# Runs on ci-compile-cpu pool (POP2-32C-128G: 32 vCPU, 128GB RAM) via CPU runner
|
||
# CPU budget: 14 CPUs per job → compile-services + compile-training run in parallel on 1 node
|
||
# (Scaleway quota: 1× POP2-32C-128G, so both jobs must share)
|
||
# Base template for jobs needing CUDA builder (test stage only — compile uses .rust-base-cpu)
|
||
.rust-base:
|
||
image: ${CI_BUILDER_IMAGE}
|
||
tags:
|
||
- kapsule
|
||
- rust
|
||
variables:
|
||
CUDA_COMPUTE_CAP: "89"
|
||
CARGO_BUILD_JOBS: "14"
|
||
KUBERNETES_CPU_REQUEST: "14000m"
|
||
KUBERNETES_CPU_LIMIT: "15500m"
|
||
KUBERNETES_MEMORY_REQUEST: "24Gi"
|
||
KUBERNETES_MEMORY_LIMIT: "56Gi"
|
||
before_script:
|
||
# Mold linker fallback — .cargo/config.toml uses -fuse-ld=mold via clang.
|
||
# clang's -fuse-ld=mold searches for "ld.mold" (not "mold") in its linker paths.
|
||
# If mold isn't installed, create ld.mold wrapper that delegates to ld.lld.
|
||
- |
|
||
if ! command -v mold &>/dev/null; then
|
||
printf '#!/bin/sh\nexec ld.lld "$@"\n' > /usr/local/bin/ld.mold
|
||
chmod +x /usr/local/bin/ld.mold
|
||
echo "mold not found, falling back to lld via ld.mold wrapper"
|
||
fi
|
||
|
||
# Base template for CPU-only Rust jobs — lightweight builder without CUDA
|
||
# Uses ci-builder-cpu image (rust:1.89-slim, ~2-3GB, no CUDA toolkit)
|
||
# CPU budget: 14 CPUs per job → runs in parallel with compile-training on same node
|
||
.rust-base-cpu:
|
||
image: ${CI_BUILDER_CPU_IMAGE}
|
||
tags:
|
||
- kapsule
|
||
- rust
|
||
variables:
|
||
# No CUDA_COMPUTE_CAP — CPU-only builds don't compile CUDA kernels
|
||
CARGO_BUILD_JOBS: "14"
|
||
KUBERNETES_CPU_REQUEST: "14000m"
|
||
KUBERNETES_CPU_LIMIT: "15500m"
|
||
KUBERNETES_MEMORY_REQUEST: "24Gi"
|
||
KUBERNETES_MEMORY_LIMIT: "56Gi"
|
||
before_script:
|
||
# Mold linker fallback — same as .rust-base
|
||
- |
|
||
if ! command -v mold &>/dev/null; then
|
||
printf '#!/bin/sh\nexec ld.lld "$@"\n' > /usr/local/bin/ld.mold
|
||
chmod +x /usr/local/bin/ld.mold
|
||
echo "mold not found, falling back to lld via ld.mold wrapper"
|
||
fi
|
||
|
||
# --------------------------------------------------------------------------
|
||
# Stage 2: tests
|
||
# --------------------------------------------------------------------------
|
||
test:
|
||
extends: .rust-base
|
||
stage: test
|
||
needs: []
|
||
rules:
|
||
# Tests disabled by default — compile+deploy is the fast path for now.
|
||
# Trigger manually from the pipeline UI when needed.
|
||
- if: $CI_PIPELINE_SOURCE == "push"
|
||
when: manual
|
||
allow_failure: true
|
||
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
|
||
when: manual
|
||
allow_failure: true
|
||
- if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
|
||
when: manual
|
||
allow_failure: true
|
||
services:
|
||
- name: redis:7-alpine
|
||
alias: redis
|
||
script:
|
||
# Detect GPU if available (nvidia runtime), otherwise use CUDA_COMPUTE_CAP from variables
|
||
- |
|
||
if command -v nvidia-smi &>/dev/null; then
|
||
export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//')
|
||
export CUDA_COMPUTE_CAP=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader | head -1 | tr -d '.')
|
||
echo "GPU=$(nvidia-smi --query-gpu=name --format=csv,noheader) CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP"
|
||
else
|
||
echo "No nvidia-smi — using CUDA stubs, CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP"
|
||
fi
|
||
- export SCCACHE_DIR="/mnt/sccache/sm_${CUDA_COMPUTE_CAP}"
|
||
- mkdir -p "$SCCACHE_DIR"
|
||
# Wait for Redis sidecar (K8s executor doesn't wait for service readiness)
|
||
- |
|
||
for i in $(seq 1 30); do
|
||
if bash -c 'echo PING > /dev/tcp/localhost/6379' 2>/dev/null; then
|
||
echo "Redis ready"; break
|
||
fi
|
||
echo "Waiting for Redis... ($i/30)"; sleep 1
|
||
done
|
||
- sccache --zero-stats || true
|
||
- cargo test --workspace --lib -- --skip model_loader::tests
|
||
- sccache --show-stats || true
|
||
|
||
# --------------------------------------------------------------------------
|
||
# Stage 3a: Compile service binaries (CPU-only, no CUDA)
|
||
# --------------------------------------------------------------------------
|
||
compile-services:
|
||
extends: .rust-base-cpu
|
||
stage: compile
|
||
needs: []
|
||
rules:
|
||
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
|
||
changes:
|
||
- Cargo.toml
|
||
- Cargo.lock
|
||
- crates/**
|
||
- bin/**
|
||
- services/**
|
||
- infra/docker/Dockerfile.*
|
||
- if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
|
||
script:
|
||
- mkdir -p "$SCCACHE_DIR" && sccache --zero-stats || true
|
||
- PROFILE=${DEV_RELEASE:+dev-release}; PROFILE=${PROFILE:-release}
|
||
- echo "Building services with --profile $PROFILE (CPU-only, no CUDA)"
|
||
- TARGET_DIR="target/$PROFILE"
|
||
- cargo build --profile $PROFILE
|
||
-p trading_service
|
||
-p api_gateway
|
||
-p broker_gateway_service
|
||
-p ml_training_service
|
||
-p backtesting_service
|
||
-p trading_agent_service
|
||
-p data_acquisition_service
|
||
-p web-gateway
|
||
- sccache --show-stats || true
|
||
- mkdir -p build-out/services
|
||
- |
|
||
for bin in trading_service api_gateway broker_gateway_service ml_training_service backtesting_service trading_agent_service data_acquisition_service web-gateway; do
|
||
cp $TARGET_DIR/$bin build-out/services/
|
||
strip build-out/services/$bin
|
||
done
|
||
- ls -lh build-out/services/
|
||
# Upload stripped binaries to MinIO (in-cluster S3)
|
||
- export RCLONE_S3_PROVIDER=Minio RCLONE_S3_ENDPOINT=http://minio.foxhunt.svc.cluster.local:9000 RCLONE_S3_REGION=us-east-1
|
||
- export RCLONE_S3_ACCESS_KEY_ID=$MINIO_ACCESS_KEY RCLONE_S3_SECRET_ACCESS_KEY=$MINIO_SECRET_KEY
|
||
- |
|
||
for bin in trading_service api_gateway broker_gateway_service ml_training_service backtesting_service trading_agent_service data_acquisition_service web-gateway; do
|
||
rclone copyto build-out/services/$bin :s3:foxhunt-binaries/latest/services/$bin
|
||
done
|
||
- rclone copy build-out/services/ :s3:foxhunt-binaries/archive/${CI_COMMIT_SHA}/services/
|
||
- echo "Service binaries uploaded to MinIO"
|
||
|
||
# --------------------------------------------------------------------------
|
||
# Stage 3a: Compile training binaries (CPU node, CUDA stubs from ci-builder image)
|
||
# --------------------------------------------------------------------------
|
||
compile-training:
|
||
extends: .rust-base-cpu
|
||
stage: compile
|
||
image: ${CI_BUILDER_IMAGE}
|
||
needs: [] # parallel with compile-services — both share 1 POP2-32C-128G node (14 CPUs each)
|
||
variables:
|
||
# CUDA stubs from ci-builder image — compiles CUDA kernels without GPU hardware.
|
||
# L4 GPU = compute capability 8.9 → candle-kernels generates sm_89 PTX
|
||
CUDA_COMPUTE_CAP: "89"
|
||
rules:
|
||
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
|
||
changes:
|
||
- Cargo.toml
|
||
- Cargo.lock
|
||
- crates/ml/**
|
||
- crates/trading_engine/**
|
||
- crates/common/**
|
||
- crates/risk/**
|
||
- crates/data/**
|
||
- crates/config/**
|
||
- crates/storage/**
|
||
- services/training_uploader/**
|
||
- if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
|
||
script:
|
||
- echo "CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP"
|
||
- export SCCACHE_DIR="/mnt/sccache/sm_${CUDA_COMPUTE_CAP}"
|
||
- mkdir -p "$SCCACHE_DIR" && sccache --zero-stats || true
|
||
- PROFILE=${DEV_RELEASE:+dev-release}; PROFILE=${PROFILE:-release}
|
||
- echo "Building training binaries with --profile $PROFILE (CUDA stubs)"
|
||
- TARGET_DIR="target/$PROFILE"
|
||
- cargo build --profile $PROFILE -p ml --features ml/cuda
|
||
--example train_baseline_rl
|
||
--example train_baseline_supervised
|
||
--example evaluate_baseline
|
||
--example evaluate_supervised
|
||
--example hyperopt_baseline_rl
|
||
--example hyperopt_baseline_supervised
|
||
- cargo build --profile $PROFILE -p training_uploader
|
||
- sccache --show-stats || true
|
||
- mkdir -p build-out/training
|
||
- |
|
||
for bin in train_baseline_rl train_baseline_supervised evaluate_baseline evaluate_supervised hyperopt_baseline_rl hyperopt_baseline_supervised; do
|
||
cp $TARGET_DIR/examples/$bin build-out/training/
|
||
strip build-out/training/$bin
|
||
done
|
||
cp $TARGET_DIR/training_uploader build-out/training/
|
||
strip build-out/training/training_uploader
|
||
- ls -lh build-out/training/
|
||
# Upload stripped binaries to MinIO (in-cluster S3)
|
||
- export RCLONE_S3_PROVIDER=Minio RCLONE_S3_ENDPOINT=http://minio.foxhunt.svc.cluster.local:9000 RCLONE_S3_REGION=us-east-1
|
||
- export RCLONE_S3_ACCESS_KEY_ID=$MINIO_ACCESS_KEY RCLONE_S3_SECRET_ACCESS_KEY=$MINIO_SECRET_KEY
|
||
- |
|
||
for bin in train_baseline_rl train_baseline_supervised evaluate_baseline evaluate_supervised hyperopt_baseline_rl hyperopt_baseline_supervised training_uploader; do
|
||
rclone copyto build-out/training/$bin :s3:foxhunt-binaries/latest/training/$bin
|
||
done
|
||
- rclone copy build-out/training/ :s3:foxhunt-binaries/archive/${CI_COMMIT_SHA}/training/
|
||
- echo "Training binaries uploaded to MinIO"
|
||
|
||
# --------------------------------------------------------------------------
|
||
# Stage 3c: Build web-dashboard static assets (Vite)
|
||
# --------------------------------------------------------------------------
|
||
build-web-dashboard:
|
||
stage: compile
|
||
image: node:22-slim
|
||
needs: []
|
||
tags:
|
||
- kapsule
|
||
- docker
|
||
variables:
|
||
KUBERNETES_CPU_REQUEST: "2000m"
|
||
KUBERNETES_CPU_LIMIT: "4000m"
|
||
KUBERNETES_MEMORY_REQUEST: "2Gi"
|
||
KUBERNETES_MEMORY_LIMIT: "4Gi"
|
||
rules:
|
||
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
|
||
changes:
|
||
- web-dashboard/**
|
||
- if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
|
||
before_script:
|
||
- apt-get update && apt-get install -y --no-install-recommends curl unzip ca-certificates
|
||
- curl -fsSL https://downloads.rclone.org/v1.69.1/rclone-v1.69.1-linux-amd64.zip -o /tmp/rclone.zip
|
||
- unzip -j /tmp/rclone.zip '*/rclone' -d /usr/local/bin/ && rm /tmp/rclone.zip
|
||
script:
|
||
- cd web-dashboard
|
||
- npm ci
|
||
- npm run build
|
||
- export RCLONE_S3_PROVIDER=Minio RCLONE_S3_ENDPOINT=http://minio.foxhunt.svc.cluster.local:9000 RCLONE_S3_REGION=us-east-1
|
||
- export RCLONE_S3_ACCESS_KEY_ID=$MINIO_ACCESS_KEY RCLONE_S3_SECRET_ACCESS_KEY=$MINIO_SECRET_KEY
|
||
- rclone sync dist/ :s3:foxhunt-binaries/latest/web-dashboard/dist/
|
||
- echo "Web dashboard assets uploaded to MinIO"
|
||
|
||
# --------------------------------------------------------------------------
|
||
# Stage 3d: Write manifest after all compiles complete
|
||
# --------------------------------------------------------------------------
|
||
write-manifest:
|
||
stage: compile
|
||
image: ${RUNTIME_IMAGE}
|
||
needs:
|
||
- job: compile-services
|
||
optional: true
|
||
- job: compile-training
|
||
optional: true
|
||
- job: build-web-dashboard
|
||
optional: true
|
||
tags:
|
||
- kapsule
|
||
- docker
|
||
variables:
|
||
KUBERNETES_CPU_REQUEST: "100m"
|
||
KUBERNETES_MEMORY_REQUEST: "64Mi"
|
||
rules:
|
||
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
|
||
- if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
|
||
script:
|
||
- export RCLONE_S3_PROVIDER=Minio RCLONE_S3_ENDPOINT=http://minio.foxhunt.svc.cluster.local:9000 RCLONE_S3_REGION=us-east-1
|
||
- export RCLONE_S3_ACCESS_KEY_ID=$MINIO_ACCESS_KEY RCLONE_S3_SECRET_ACCESS_KEY=$MINIO_SECRET_KEY
|
||
- echo "{\"sha\":\"${CI_COMMIT_SHA}\",\"ts\":\"$(date -Iseconds)\"}" | rclone rcat :s3:foxhunt-binaries/latest/MANIFEST.json
|
||
- echo "Manifest written for ${CI_COMMIT_SHA}"
|
||
|
||
# --------------------------------------------------------------------------
|
||
# Stage 3c: Training — RL on L4 (CPU-heavy), Supervised on L40S (GPU-heavy)
|
||
# --------------------------------------------------------------------------
|
||
|
||
# --------------------------------------------------------------------------
|
||
# RL training base: DQN + PPO → ci-rl pool (L4-1-24G)
|
||
# RL hyperopt runs parallel PSO trials — CPU+GPU bound, low VRAM.
|
||
# --------------------------------------------------------------------------
|
||
.train-rl-base:
|
||
stage: train
|
||
image: ${TRAINING_RUNTIME_IMAGE}
|
||
tags:
|
||
- kapsule
|
||
- gpu
|
||
# Runner default node_selector: k8s.scaleway.com/pool-name=ci-rl
|
||
variables:
|
||
# nvidia runtime provided by runner default (runtime_class_name = "nvidia")
|
||
# RL hyperopt is CPU-bound (parallel PSO trials). Use full L4 (8 vCPU).
|
||
KUBERNETES_CPU_REQUEST: "6000m"
|
||
KUBERNETES_CPU_LIMIT: "7800m"
|
||
KUBERNETES_MEMORY_REQUEST: "8Gi"
|
||
KUBERNETES_MEMORY_LIMIT: "20Gi"
|
||
rules:
|
||
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
|
||
when: manual
|
||
allow_failure: true
|
||
- if: $CI_PIPELINE_SOURCE == "schedule" && $TRAIN_VALIDATE == "true"
|
||
- if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
|
||
when: manual
|
||
allow_failure: true
|
||
before_script:
|
||
- export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//')
|
||
- nvidia-smi
|
||
- mkdir -p ${CI_PROJECT_DIR}/output ${CI_PROJECT_DIR}/bin
|
||
# Fetch training binaries from MinIO (in-cluster S3)
|
||
- export RCLONE_S3_PROVIDER=Minio RCLONE_S3_ENDPOINT=http://minio.foxhunt.svc.cluster.local:9000 RCLONE_S3_REGION=us-east-1
|
||
- export RCLONE_S3_ACCESS_KEY_ID=$MINIO_ACCESS_KEY RCLONE_S3_SECRET_ACCESS_KEY=$MINIO_SECRET_KEY
|
||
- rclone sync :s3:foxhunt-binaries/latest/training/ ${CI_PROJECT_DIR}/bin/
|
||
- chmod +x ${CI_PROJECT_DIR}/bin/*
|
||
- export PATH="${CI_PROJECT_DIR}/bin:$PATH"
|
||
|
||
# --------------------------------------------------------------------------
|
||
# Supervised training base: TFT, Mamba2, TGGN, TLOB, Liquid, KAN, xLSTM, Diffusion → L40S
|
||
# Supervised models are GPU-bound — low CPU, high VRAM.
|
||
# --------------------------------------------------------------------------
|
||
.train-validate-base:
|
||
stage: train
|
||
image: ${TRAINING_RUNTIME_IMAGE}
|
||
tags:
|
||
- kapsule
|
||
- gpu
|
||
variables:
|
||
# nvidia runtime provided by runner default (runtime_class_name = "nvidia")
|
||
# Route to L40S pool (48GB VRAM for large supervised models)
|
||
# Overrides runner default to L40S pool
|
||
KUBERNETES_NODE_SELECTOR_k8s.scaleway.com/pool-name: "ci-training"
|
||
# Supervised is GPU-bound — minimal CPU. Low request allows 3-4 concurrent jobs.
|
||
KUBERNETES_CPU_REQUEST: "1000m"
|
||
KUBERNETES_CPU_LIMIT: "2000m"
|
||
KUBERNETES_MEMORY_REQUEST: "16Gi"
|
||
KUBERNETES_MEMORY_LIMIT: "40Gi"
|
||
rules:
|
||
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
|
||
when: manual
|
||
allow_failure: true
|
||
- if: $CI_PIPELINE_SOURCE == "schedule" && $TRAIN_VALIDATE == "true"
|
||
- if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
|
||
when: manual
|
||
allow_failure: true
|
||
before_script:
|
||
- export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//')
|
||
- nvidia-smi
|
||
- mkdir -p ${CI_PROJECT_DIR}/output ${CI_PROJECT_DIR}/bin
|
||
# Fetch training binaries from MinIO (in-cluster S3)
|
||
- export RCLONE_S3_PROVIDER=Minio RCLONE_S3_ENDPOINT=http://minio.foxhunt.svc.cluster.local:9000 RCLONE_S3_REGION=us-east-1
|
||
- export RCLONE_S3_ACCESS_KEY_ID=$MINIO_ACCESS_KEY RCLONE_S3_SECRET_ACCESS_KEY=$MINIO_SECRET_KEY
|
||
- rclone sync :s3:foxhunt-binaries/latest/training/ ${CI_PROJECT_DIR}/bin/
|
||
- chmod +x ${CI_PROJECT_DIR}/bin/*
|
||
- export PATH="${CI_PROJECT_DIR}/bin:$PATH"
|
||
|
||
train-validate-rl:
|
||
extends: .train-rl-base
|
||
needs:
|
||
- job: compile-training
|
||
optional: true
|
||
script:
|
||
- |
|
||
train_baseline_rl \
|
||
--model both \
|
||
--symbol ES.FUT \
|
||
--data-dir /mnt/training-data/futures-baseline \
|
||
--output-dir ${CI_PROJECT_DIR}/output \
|
||
--max-steps-per-epoch 50 \
|
||
&& TRAIN_OK=1 || TRAIN_OK=0
|
||
echo "=== RL training exit: $TRAIN_OK ==="
|
||
echo "=== Running evaluation ==="
|
||
evaluate_baseline \
|
||
--model both \
|
||
--models-dir ${CI_PROJECT_DIR}/output \
|
||
--data-dir /mnt/training-data/futures-baseline \
|
||
--output ${CI_PROJECT_DIR}/output/rl_eval_report.json \
|
||
--symbol ES.FUT \
|
||
|| true
|
||
echo "=== Eval report ==="
|
||
cat ${CI_PROJECT_DIR}/output/rl_eval_report.json 2>/dev/null || echo "No eval report"
|
||
[ "$TRAIN_OK" = "1" ] || exit 1
|
||
artifacts:
|
||
paths:
|
||
- output/rl_eval_report.json
|
||
when: always
|
||
expire_in: 30 days
|
||
|
||
train-validate-tft:
|
||
extends: .train-validate-base
|
||
needs:
|
||
- job: compile-training
|
||
optional: true
|
||
script:
|
||
- train_baseline_supervised
|
||
--model tft
|
||
--symbol ES.FUT
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output-dir ${CI_PROJECT_DIR}/output
|
||
--max-steps-per-epoch 50
|
||
- echo "=== TFT training complete, running evaluation ==="
|
||
- evaluate_supervised
|
||
--model tft
|
||
--models-dir ${CI_PROJECT_DIR}/output
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output ${CI_PROJECT_DIR}/output/tft_eval_report.json
|
||
--symbol ES.FUT
|
||
- echo "TFT validation + evaluation passed"
|
||
- cat ${CI_PROJECT_DIR}/output/tft_eval_report.json
|
||
artifacts:
|
||
paths:
|
||
- output/tft_eval_report.json
|
||
when: always
|
||
expire_in: 30 days
|
||
|
||
hyperopt-ppo:
|
||
extends: .train-rl-base
|
||
needs:
|
||
- job: compile-training
|
||
optional: true
|
||
timeout: 4h
|
||
script:
|
||
- hyperopt_baseline_rl
|
||
--model ppo
|
||
--trials 20
|
||
--n-initial 5
|
||
--epochs 8
|
||
--parallel 0
|
||
--symbol ES.FUT
|
||
--tx-cost-bps 0.1
|
||
--tick-size 0.25
|
||
--spread-ticks 1.0
|
||
--initial-capital 35000
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--base-dir ${CI_PROJECT_DIR}/output/hyperopt
|
||
--output ${CI_PROJECT_DIR}/output/ppo_hyperopt_results.json
|
||
- echo "=== PPO Hyperopt complete ==="
|
||
- cat ${CI_PROJECT_DIR}/output/ppo_hyperopt_results.json
|
||
- echo "=== Training PPO with best params ==="
|
||
- train_baseline_rl
|
||
--model ppo
|
||
--symbol ES.FUT
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output-dir ${CI_PROJECT_DIR}/output
|
||
--hyperopt-params ${CI_PROJECT_DIR}/output/ppo_hyperopt_results.json
|
||
- echo "=== Evaluating PPO with best params ==="
|
||
- evaluate_baseline
|
||
--model ppo
|
||
--models-dir ${CI_PROJECT_DIR}/output
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output ${CI_PROJECT_DIR}/output/ppo_eval_report.json
|
||
--symbol ES.FUT
|
||
- cat ${CI_PROJECT_DIR}/output/ppo_eval_report.json
|
||
artifacts:
|
||
paths:
|
||
- output/ppo_hyperopt_results.json
|
||
- output/ppo_eval_report.json
|
||
when: always
|
||
expire_in: 90 days
|
||
|
||
# --- Train-validate: remaining supervised models ---
|
||
|
||
train-validate-mamba2:
|
||
extends: .train-validate-base
|
||
needs:
|
||
- job: compile-training
|
||
optional: true
|
||
script:
|
||
- train_baseline_supervised
|
||
--model mamba2
|
||
--symbol ES.FUT
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output-dir ${CI_PROJECT_DIR}/output
|
||
--max-steps-per-epoch 50
|
||
- echo "=== Mamba2 training complete, running evaluation ==="
|
||
- evaluate_supervised
|
||
--model mamba2
|
||
--models-dir ${CI_PROJECT_DIR}/output
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output ${CI_PROJECT_DIR}/output/mamba2_eval_report.json
|
||
--symbol ES.FUT
|
||
- echo "Mamba2 validation + evaluation passed"
|
||
- cat ${CI_PROJECT_DIR}/output/mamba2_eval_report.json
|
||
artifacts:
|
||
paths:
|
||
- output/mamba2_eval_report.json
|
||
when: always
|
||
expire_in: 30 days
|
||
|
||
train-validate-liquid:
|
||
extends: .train-validate-base
|
||
needs:
|
||
- job: compile-training
|
||
optional: true
|
||
script:
|
||
- train_baseline_supervised
|
||
--model liquid
|
||
--symbol ES.FUT
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output-dir ${CI_PROJECT_DIR}/output
|
||
--max-steps-per-epoch 50
|
||
- echo "=== Liquid training complete, running evaluation ==="
|
||
- evaluate_supervised
|
||
--model liquid
|
||
--models-dir ${CI_PROJECT_DIR}/output
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output ${CI_PROJECT_DIR}/output/liquid_eval_report.json
|
||
--symbol ES.FUT
|
||
- echo "Liquid validation + evaluation passed"
|
||
- cat ${CI_PROJECT_DIR}/output/liquid_eval_report.json
|
||
artifacts:
|
||
paths:
|
||
- output/liquid_eval_report.json
|
||
when: always
|
||
expire_in: 30 days
|
||
|
||
train-validate-tggn:
|
||
extends: .train-validate-base
|
||
needs:
|
||
- job: compile-training
|
||
optional: true
|
||
script:
|
||
- train_baseline_supervised
|
||
--model tggn
|
||
--symbol ES.FUT
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output-dir ${CI_PROJECT_DIR}/output
|
||
--max-steps-per-epoch 50
|
||
- echo "=== TGGN training complete, running evaluation ==="
|
||
- evaluate_supervised
|
||
--model tggn
|
||
--models-dir ${CI_PROJECT_DIR}/output
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output ${CI_PROJECT_DIR}/output/tggn_eval_report.json
|
||
--symbol ES.FUT
|
||
- echo "TGGN validation + evaluation passed"
|
||
- cat ${CI_PROJECT_DIR}/output/tggn_eval_report.json
|
||
artifacts:
|
||
paths:
|
||
- output/tggn_eval_report.json
|
||
when: always
|
||
expire_in: 30 days
|
||
|
||
train-validate-tlob:
|
||
extends: .train-validate-base
|
||
needs:
|
||
- job: compile-training
|
||
optional: true
|
||
script:
|
||
- train_baseline_supervised
|
||
--model tlob
|
||
--symbol ES.FUT
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output-dir ${CI_PROJECT_DIR}/output
|
||
--max-steps-per-epoch 50
|
||
- echo "=== TLOB training complete, running evaluation ==="
|
||
- evaluate_supervised
|
||
--model tlob
|
||
--models-dir ${CI_PROJECT_DIR}/output
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output ${CI_PROJECT_DIR}/output/tlob_eval_report.json
|
||
--symbol ES.FUT
|
||
- echo "TLOB validation + evaluation passed"
|
||
- cat ${CI_PROJECT_DIR}/output/tlob_eval_report.json
|
||
artifacts:
|
||
paths:
|
||
- output/tlob_eval_report.json
|
||
when: always
|
||
expire_in: 30 days
|
||
|
||
train-validate-kan:
|
||
extends: .train-validate-base
|
||
needs:
|
||
- job: compile-training
|
||
optional: true
|
||
script:
|
||
- train_baseline_supervised
|
||
--model kan
|
||
--symbol ES.FUT
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output-dir ${CI_PROJECT_DIR}/output
|
||
--max-steps-per-epoch 50
|
||
- echo "=== KAN training complete, running evaluation ==="
|
||
- evaluate_supervised
|
||
--model kan
|
||
--models-dir ${CI_PROJECT_DIR}/output
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output ${CI_PROJECT_DIR}/output/kan_eval_report.json
|
||
--symbol ES.FUT
|
||
- echo "KAN validation + evaluation passed"
|
||
- cat ${CI_PROJECT_DIR}/output/kan_eval_report.json
|
||
artifacts:
|
||
paths:
|
||
- output/kan_eval_report.json
|
||
when: always
|
||
expire_in: 30 days
|
||
|
||
train-validate-xlstm:
|
||
extends: .train-validate-base
|
||
needs:
|
||
- job: compile-training
|
||
optional: true
|
||
script:
|
||
- train_baseline_supervised
|
||
--model xlstm
|
||
--symbol ES.FUT
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output-dir ${CI_PROJECT_DIR}/output
|
||
--max-steps-per-epoch 50
|
||
- echo "=== xLSTM training complete, running evaluation ==="
|
||
- evaluate_supervised
|
||
--model xlstm
|
||
--models-dir ${CI_PROJECT_DIR}/output
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output ${CI_PROJECT_DIR}/output/xlstm_eval_report.json
|
||
--symbol ES.FUT
|
||
- echo "xLSTM validation + evaluation passed"
|
||
- cat ${CI_PROJECT_DIR}/output/xlstm_eval_report.json
|
||
artifacts:
|
||
paths:
|
||
- output/xlstm_eval_report.json
|
||
when: always
|
||
expire_in: 30 days
|
||
|
||
train-validate-diffusion:
|
||
extends: .train-validate-base
|
||
needs:
|
||
- job: compile-training
|
||
optional: true
|
||
script:
|
||
- train_baseline_supervised
|
||
--model diffusion
|
||
--symbol ES.FUT
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output-dir ${CI_PROJECT_DIR}/output
|
||
--max-steps-per-epoch 50
|
||
- echo "=== Diffusion training complete, running evaluation ==="
|
||
- evaluate_supervised
|
||
--model diffusion
|
||
--models-dir ${CI_PROJECT_DIR}/output
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output ${CI_PROJECT_DIR}/output/diffusion_eval_report.json
|
||
--symbol ES.FUT
|
||
- echo "Diffusion validation + evaluation passed"
|
||
- cat ${CI_PROJECT_DIR}/output/diffusion_eval_report.json
|
||
artifacts:
|
||
paths:
|
||
- output/diffusion_eval_report.json
|
||
when: always
|
||
expire_in: 30 days
|
||
|
||
# --- Hyperopt: all models ---
|
||
|
||
hyperopt-dqn:
|
||
extends: .train-rl-base
|
||
needs:
|
||
- job: compile-training
|
||
optional: true
|
||
timeout: 4h
|
||
script:
|
||
- |
|
||
hyperopt_baseline_rl \
|
||
--model dqn \
|
||
--trials 20 \
|
||
--n-initial 5 \
|
||
--epochs 8 \
|
||
--parallel 0 \
|
||
--symbol ES.FUT \
|
||
--tx-cost-bps 0.1 \
|
||
--tick-size 0.25 \
|
||
--spread-ticks 1.0 \
|
||
--initial-capital 35000 \
|
||
--data-dir /mnt/training-data/futures-baseline \
|
||
--base-dir ${CI_PROJECT_DIR}/output/hyperopt \
|
||
--output ${CI_PROJECT_DIR}/output/dqn_hyperopt_results.json \
|
||
&& HYPEROPT_OK=1 || HYPEROPT_OK=0
|
||
echo "=== DQN Hyperopt exit: $HYPEROPT_OK ==="
|
||
cat ${CI_PROJECT_DIR}/output/dqn_hyperopt_results.json 2>/dev/null || echo "No hyperopt results"
|
||
echo "=== Training DQN with best params ==="
|
||
train_baseline_rl \
|
||
--model dqn \
|
||
--symbol ES.FUT \
|
||
--data-dir /mnt/training-data/futures-baseline \
|
||
--output-dir ${CI_PROJECT_DIR}/output \
|
||
--hyperopt-params ${CI_PROJECT_DIR}/output/dqn_hyperopt_results.json \
|
||
&& TRAIN_OK=1 || TRAIN_OK=0
|
||
echo "=== Training exit: $TRAIN_OK ==="
|
||
echo "=== Evaluating DQN ==="
|
||
evaluate_baseline \
|
||
--model dqn \
|
||
--models-dir ${CI_PROJECT_DIR}/output \
|
||
--data-dir /mnt/training-data/futures-baseline \
|
||
--output ${CI_PROJECT_DIR}/output/dqn_eval_report.json \
|
||
--symbol ES.FUT \
|
||
|| true
|
||
echo "=== Eval report ==="
|
||
cat ${CI_PROJECT_DIR}/output/dqn_eval_report.json 2>/dev/null || echo "No eval report"
|
||
[ "$HYPEROPT_OK" = "1" ] && [ "$TRAIN_OK" = "1" ] || exit 1
|
||
artifacts:
|
||
paths:
|
||
- output/dqn_hyperopt_results.json
|
||
- output/dqn_eval_report.json
|
||
when: always
|
||
expire_in: 90 days
|
||
|
||
hyperopt-tft:
|
||
extends: .train-validate-base
|
||
needs:
|
||
- job: compile-training
|
||
optional: true
|
||
timeout: 4h
|
||
script:
|
||
- hyperopt_baseline_supervised
|
||
--model tft
|
||
--trials 20
|
||
--n-initial 5
|
||
--epochs 15
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--base-dir ${CI_PROJECT_DIR}/output/hyperopt
|
||
--output ${CI_PROJECT_DIR}/output/tft_hyperopt_results.json
|
||
- echo "=== TFT Hyperopt complete ==="
|
||
- cat ${CI_PROJECT_DIR}/output/tft_hyperopt_results.json
|
||
- echo "=== Training TFT with best params ==="
|
||
- train_baseline_supervised
|
||
--model tft
|
||
--symbol ES.FUT
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output-dir ${CI_PROJECT_DIR}/output
|
||
--hyperopt-params ${CI_PROJECT_DIR}/output/tft_hyperopt_results.json
|
||
- echo "=== Evaluating TFT with best params ==="
|
||
- evaluate_supervised
|
||
--model tft
|
||
--models-dir ${CI_PROJECT_DIR}/output
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output ${CI_PROJECT_DIR}/output/tft_eval_report.json
|
||
--symbol ES.FUT
|
||
- cat ${CI_PROJECT_DIR}/output/tft_eval_report.json
|
||
artifacts:
|
||
paths:
|
||
- output/tft_hyperopt_results.json
|
||
- output/tft_eval_report.json
|
||
when: always
|
||
expire_in: 90 days
|
||
|
||
hyperopt-mamba2:
|
||
extends: .train-validate-base
|
||
needs:
|
||
- job: compile-training
|
||
optional: true
|
||
timeout: 4h
|
||
script:
|
||
- hyperopt_baseline_supervised
|
||
--model mamba2
|
||
--trials 20
|
||
--n-initial 5
|
||
--epochs 15
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--base-dir ${CI_PROJECT_DIR}/output/hyperopt
|
||
--output ${CI_PROJECT_DIR}/output/mamba2_hyperopt_results.json
|
||
- echo "=== Mamba2 Hyperopt complete ==="
|
||
- cat ${CI_PROJECT_DIR}/output/mamba2_hyperopt_results.json
|
||
- echo "=== Training Mamba2 with best params ==="
|
||
- train_baseline_supervised
|
||
--model mamba2
|
||
--symbol ES.FUT
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output-dir ${CI_PROJECT_DIR}/output
|
||
--hyperopt-params ${CI_PROJECT_DIR}/output/mamba2_hyperopt_results.json
|
||
- echo "=== Evaluating Mamba2 with best params ==="
|
||
- evaluate_supervised
|
||
--model mamba2
|
||
--models-dir ${CI_PROJECT_DIR}/output
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output ${CI_PROJECT_DIR}/output/mamba2_eval_report.json
|
||
--symbol ES.FUT
|
||
- cat ${CI_PROJECT_DIR}/output/mamba2_eval_report.json
|
||
artifacts:
|
||
paths:
|
||
- output/mamba2_hyperopt_results.json
|
||
- output/mamba2_eval_report.json
|
||
when: always
|
||
expire_in: 90 days
|
||
|
||
hyperopt-liquid:
|
||
extends: .train-validate-base
|
||
needs:
|
||
- job: compile-training
|
||
optional: true
|
||
timeout: 4h
|
||
script:
|
||
- hyperopt_baseline_supervised
|
||
--model liquid
|
||
--trials 20
|
||
--n-initial 5
|
||
--epochs 15
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--base-dir ${CI_PROJECT_DIR}/output/hyperopt
|
||
--output ${CI_PROJECT_DIR}/output/liquid_hyperopt_results.json
|
||
- echo "=== Liquid Hyperopt complete ==="
|
||
- cat ${CI_PROJECT_DIR}/output/liquid_hyperopt_results.json
|
||
- echo "=== Training Liquid with best params ==="
|
||
- train_baseline_supervised
|
||
--model liquid
|
||
--symbol ES.FUT
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output-dir ${CI_PROJECT_DIR}/output
|
||
--hyperopt-params ${CI_PROJECT_DIR}/output/liquid_hyperopt_results.json
|
||
- echo "=== Evaluating Liquid with best params ==="
|
||
- evaluate_supervised
|
||
--model liquid
|
||
--models-dir ${CI_PROJECT_DIR}/output
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output ${CI_PROJECT_DIR}/output/liquid_eval_report.json
|
||
--symbol ES.FUT
|
||
- cat ${CI_PROJECT_DIR}/output/liquid_eval_report.json
|
||
artifacts:
|
||
paths:
|
||
- output/liquid_hyperopt_results.json
|
||
- output/liquid_eval_report.json
|
||
when: always
|
||
expire_in: 90 days
|
||
|
||
hyperopt-tggn:
|
||
extends: .train-validate-base
|
||
needs:
|
||
- job: compile-training
|
||
optional: true
|
||
timeout: 4h
|
||
script:
|
||
- hyperopt_baseline_supervised
|
||
--model tggn
|
||
--trials 20
|
||
--n-initial 5
|
||
--epochs 15
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--base-dir ${CI_PROJECT_DIR}/output/hyperopt
|
||
--output ${CI_PROJECT_DIR}/output/tggn_hyperopt_results.json
|
||
- echo "=== TGGN Hyperopt complete ==="
|
||
- cat ${CI_PROJECT_DIR}/output/tggn_hyperopt_results.json
|
||
- echo "=== Training TGGN with best params ==="
|
||
- train_baseline_supervised
|
||
--model tggn
|
||
--symbol ES.FUT
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output-dir ${CI_PROJECT_DIR}/output
|
||
--hyperopt-params ${CI_PROJECT_DIR}/output/tggn_hyperopt_results.json
|
||
- echo "=== Evaluating TGGN with best params ==="
|
||
- evaluate_supervised
|
||
--model tggn
|
||
--models-dir ${CI_PROJECT_DIR}/output
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output ${CI_PROJECT_DIR}/output/tggn_eval_report.json
|
||
--symbol ES.FUT
|
||
- cat ${CI_PROJECT_DIR}/output/tggn_eval_report.json
|
||
artifacts:
|
||
paths:
|
||
- output/tggn_hyperopt_results.json
|
||
- output/tggn_eval_report.json
|
||
when: always
|
||
expire_in: 90 days
|
||
|
||
hyperopt-tlob:
|
||
extends: .train-validate-base
|
||
needs:
|
||
- job: compile-training
|
||
optional: true
|
||
timeout: 4h
|
||
script:
|
||
- hyperopt_baseline_supervised
|
||
--model tlob
|
||
--trials 20
|
||
--n-initial 5
|
||
--epochs 15
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--base-dir ${CI_PROJECT_DIR}/output/hyperopt
|
||
--output ${CI_PROJECT_DIR}/output/tlob_hyperopt_results.json
|
||
- echo "=== TLOB Hyperopt complete ==="
|
||
- cat ${CI_PROJECT_DIR}/output/tlob_hyperopt_results.json
|
||
- echo "=== Training TLOB with best params ==="
|
||
- train_baseline_supervised
|
||
--model tlob
|
||
--symbol ES.FUT
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output-dir ${CI_PROJECT_DIR}/output
|
||
--hyperopt-params ${CI_PROJECT_DIR}/output/tlob_hyperopt_results.json
|
||
- echo "=== Evaluating TLOB with best params ==="
|
||
- evaluate_supervised
|
||
--model tlob
|
||
--models-dir ${CI_PROJECT_DIR}/output
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output ${CI_PROJECT_DIR}/output/tlob_eval_report.json
|
||
--symbol ES.FUT
|
||
- cat ${CI_PROJECT_DIR}/output/tlob_eval_report.json
|
||
artifacts:
|
||
paths:
|
||
- output/tlob_hyperopt_results.json
|
||
- output/tlob_eval_report.json
|
||
when: always
|
||
expire_in: 90 days
|
||
|
||
hyperopt-kan:
|
||
extends: .train-validate-base
|
||
needs:
|
||
- job: compile-training
|
||
optional: true
|
||
timeout: 4h
|
||
script:
|
||
- hyperopt_baseline_supervised
|
||
--model kan
|
||
--trials 20
|
||
--n-initial 5
|
||
--epochs 15
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--base-dir ${CI_PROJECT_DIR}/output/hyperopt
|
||
--output ${CI_PROJECT_DIR}/output/kan_hyperopt_results.json
|
||
- echo "=== KAN Hyperopt complete ==="
|
||
- cat ${CI_PROJECT_DIR}/output/kan_hyperopt_results.json
|
||
- echo "=== Training KAN with best params ==="
|
||
- train_baseline_supervised
|
||
--model kan
|
||
--symbol ES.FUT
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output-dir ${CI_PROJECT_DIR}/output
|
||
--hyperopt-params ${CI_PROJECT_DIR}/output/kan_hyperopt_results.json
|
||
- echo "=== Evaluating KAN with best params ==="
|
||
- evaluate_supervised
|
||
--model kan
|
||
--models-dir ${CI_PROJECT_DIR}/output
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output ${CI_PROJECT_DIR}/output/kan_eval_report.json
|
||
--symbol ES.FUT
|
||
- cat ${CI_PROJECT_DIR}/output/kan_eval_report.json
|
||
artifacts:
|
||
paths:
|
||
- output/kan_hyperopt_results.json
|
||
- output/kan_eval_report.json
|
||
when: always
|
||
expire_in: 90 days
|
||
|
||
hyperopt-xlstm:
|
||
extends: .train-validate-base
|
||
needs:
|
||
- job: compile-training
|
||
optional: true
|
||
timeout: 4h
|
||
script:
|
||
- hyperopt_baseline_supervised
|
||
--model xlstm
|
||
--trials 20
|
||
--n-initial 5
|
||
--epochs 15
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--base-dir ${CI_PROJECT_DIR}/output/hyperopt
|
||
--output ${CI_PROJECT_DIR}/output/xlstm_hyperopt_results.json
|
||
- echo "=== xLSTM Hyperopt complete ==="
|
||
- cat ${CI_PROJECT_DIR}/output/xlstm_hyperopt_results.json
|
||
- echo "=== Training xLSTM with best params ==="
|
||
- train_baseline_supervised
|
||
--model xlstm
|
||
--symbol ES.FUT
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output-dir ${CI_PROJECT_DIR}/output
|
||
--hyperopt-params ${CI_PROJECT_DIR}/output/xlstm_hyperopt_results.json
|
||
- echo "=== Evaluating xLSTM with best params ==="
|
||
- evaluate_supervised
|
||
--model xlstm
|
||
--models-dir ${CI_PROJECT_DIR}/output
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output ${CI_PROJECT_DIR}/output/xlstm_eval_report.json
|
||
--symbol ES.FUT
|
||
- cat ${CI_PROJECT_DIR}/output/xlstm_eval_report.json
|
||
artifacts:
|
||
paths:
|
||
- output/xlstm_hyperopt_results.json
|
||
- output/xlstm_eval_report.json
|
||
when: always
|
||
expire_in: 90 days
|
||
|
||
hyperopt-diffusion:
|
||
extends: .train-validate-base
|
||
needs:
|
||
- job: compile-training
|
||
optional: true
|
||
timeout: 4h
|
||
script:
|
||
- hyperopt_baseline_supervised
|
||
--model diffusion
|
||
--trials 20
|
||
--n-initial 5
|
||
--epochs 15
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--base-dir ${CI_PROJECT_DIR}/output/hyperopt
|
||
--output ${CI_PROJECT_DIR}/output/diffusion_hyperopt_results.json
|
||
- echo "=== Diffusion Hyperopt complete ==="
|
||
- cat ${CI_PROJECT_DIR}/output/diffusion_hyperopt_results.json
|
||
- echo "=== Training Diffusion with best params ==="
|
||
- train_baseline_supervised
|
||
--model diffusion
|
||
--symbol ES.FUT
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output-dir ${CI_PROJECT_DIR}/output
|
||
--hyperopt-params ${CI_PROJECT_DIR}/output/diffusion_hyperopt_results.json
|
||
- echo "=== Evaluating Diffusion with best params ==="
|
||
- evaluate_supervised
|
||
--model diffusion
|
||
--models-dir ${CI_PROJECT_DIR}/output
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output ${CI_PROJECT_DIR}/output/diffusion_eval_report.json
|
||
--symbol ES.FUT
|
||
- cat ${CI_PROJECT_DIR}/output/diffusion_eval_report.json
|
||
artifacts:
|
||
paths:
|
||
- output/diffusion_hyperopt_results.json
|
||
- output/diffusion_eval_report.json
|
||
when: always
|
||
expire_in: 90 days
|
||
|
||
# --------------------------------------------------------------------------
|
||
# IaC: Terragrunt plan on MR (runs on gitlab pool)
|
||
# --------------------------------------------------------------------------
|
||
infra-plan:
|
||
stage: test
|
||
image: ${INFRA_RUNNER_IMAGE}
|
||
tags:
|
||
- kapsule
|
||
- docker
|
||
needs: []
|
||
rules:
|
||
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
|
||
changes:
|
||
- infra/**
|
||
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
|
||
changes:
|
||
- infra/**
|
||
before_script:
|
||
- export SCW_ACCESS_KEY=${SCW_ACCESS_KEY}
|
||
- export SCW_SECRET_KEY=${SCW_SECRET_KEY}
|
||
- export SCW_DEFAULT_PROJECT_ID=${SCW_DEFAULT_PROJECT_ID}
|
||
- export AWS_ACCESS_KEY_ID=${SCW_ACCESS_KEY}
|
||
- export AWS_SECRET_ACCESS_KEY=${SCW_SECRET_KEY}
|
||
script:
|
||
- cd infra/live/production
|
||
- terragrunt run-all plan --terragrunt-non-interactive 2>&1 | tee ${CI_PROJECT_DIR}/plan-output.txt
|
||
- echo "Plan completed — review output, then trigger infra-apply manually"
|
||
artifacts:
|
||
paths:
|
||
- plan-output.txt
|
||
when: always
|
||
expire_in: 7 days
|
||
|
||
# --------------------------------------------------------------------------
|
||
# IaC: Terragrunt apply (auto on main push, manual from web/api)
|
||
# --------------------------------------------------------------------------
|
||
infra-apply:
|
||
stage: deploy
|
||
image: ${INFRA_RUNNER_IMAGE}
|
||
tags:
|
||
- kapsule
|
||
- docker
|
||
needs:
|
||
- job: infra-plan
|
||
optional: true
|
||
rules:
|
||
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
|
||
changes:
|
||
- infra/**
|
||
- if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
|
||
when: manual
|
||
before_script:
|
||
- export SCW_ACCESS_KEY=${SCW_ACCESS_KEY}
|
||
- export SCW_SECRET_KEY=${SCW_SECRET_KEY}
|
||
- export SCW_DEFAULT_PROJECT_ID=${SCW_DEFAULT_PROJECT_ID}
|
||
- export AWS_ACCESS_KEY_ID=${SCW_ACCESS_KEY}
|
||
- export AWS_SECRET_ACCESS_KEY=${SCW_SECRET_KEY}
|
||
script:
|
||
- cd infra/live/production
|
||
- terragrunt run-all apply --terragrunt-non-interactive -auto-approve
|
||
environment:
|
||
name: production/infrastructure
|
||
|
||
# --------------------------------------------------------------------------
|
||
# IaC: Weekly drift detection (scheduled pipeline, gitlab pool)
|
||
# --------------------------------------------------------------------------
|
||
infra-drift-check:
|
||
stage: test
|
||
image: ${INFRA_RUNNER_IMAGE}
|
||
tags:
|
||
- kapsule
|
||
- docker
|
||
needs: []
|
||
rules:
|
||
- if: $CI_PIPELINE_SOURCE == "schedule" && $DRIFT_CHECK == "true"
|
||
before_script:
|
||
- export SCW_ACCESS_KEY=${SCW_ACCESS_KEY}
|
||
- export SCW_SECRET_KEY=${SCW_SECRET_KEY}
|
||
- export SCW_DEFAULT_PROJECT_ID=${SCW_DEFAULT_PROJECT_ID}
|
||
- export AWS_ACCESS_KEY_ID=${SCW_ACCESS_KEY}
|
||
- export AWS_SECRET_ACCESS_KEY=${SCW_SECRET_KEY}
|
||
script:
|
||
- cd infra/live/production
|
||
- |
|
||
terragrunt run-all plan -detailed-exitcode --terragrunt-non-interactive \
|
||
2>&1 | tee ${CI_PROJECT_DIR}/drift-output.txt; EXIT_CODE=$?
|
||
if [ "$EXIT_CODE" -eq 2 ]; then
|
||
echo "DRIFT DETECTED — creating GitLab issue"
|
||
glab issue create \
|
||
--title "IaC Drift Detected ($(date +%Y-%m-%d))" \
|
||
--description "$(tail -100 ${CI_PROJECT_DIR}/drift-output.txt)" \
|
||
--label "infrastructure,drift"
|
||
exit 1
|
||
elif [ "$EXIT_CODE" -ne 0 ]; then
|
||
exit $EXIT_CODE
|
||
fi
|
||
echo "No drift detected"
|
||
artifacts:
|
||
paths:
|
||
- drift-output.txt
|
||
when: always
|
||
expire_in: 7 days
|
||
|
||
# --------------------------------------------------------------------------
|
||
# Stage 4: Deploy to Kapsule (main only)
|
||
# --------------------------------------------------------------------------
|
||
deploy:
|
||
stage: deploy
|
||
image: ${INFRA_RUNNER_IMAGE}
|
||
tags:
|
||
- kapsule
|
||
- docker
|
||
variables:
|
||
# Deploy runs on CPU compile pool (lightweight kubectl apply)
|
||
KUBERNETES_CPU_REQUEST: "200m"
|
||
KUBERNETES_CPU_LIMIT: "500m"
|
||
KUBERNETES_MEMORY_REQUEST: "256Mi"
|
||
KUBERNETES_MEMORY_LIMIT: "512Mi"
|
||
needs:
|
||
- job: write-manifest
|
||
optional: true
|
||
rules:
|
||
# Deploy when source code or infra manifests changed
|
||
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
|
||
changes:
|
||
- crates/**
|
||
- bin/**
|
||
- services/**
|
||
- Cargo.toml
|
||
- Cargo.lock
|
||
- infra/k8s/**
|
||
- .gitlab-ci.yml
|
||
- if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
|
||
before_script:
|
||
# Install kubectl if not in image yet (removed once infra-runner is rebuilt)
|
||
- which kubectl || (curl -fsSL "https://dl.k8s.io/release/v1.31.4/bin/linux/amd64/kubectl" -o /usr/local/bin/kubectl && chmod +x /usr/local/bin/kubectl)
|
||
# scw CLI needs these for kubeconfig exec plugin (get-token)
|
||
- export SCW_ACCESS_KEY=${SCW_ACCESS_KEY}
|
||
- export SCW_SECRET_KEY=${SCW_SECRET_KEY}
|
||
- export SCW_DEFAULT_PROJECT_ID=${SCW_DEFAULT_PROJECT_ID}
|
||
- export SCW_DEFAULT_ORGANIZATION_ID=${SCW_DEFAULT_ORGANIZATION_ID:-${SCW_DEFAULT_PROJECT_ID}}
|
||
- mkdir -p ~/.kube
|
||
- echo "$KUBECONFIG_B64" | base64 -d > ~/.kube/config
|
||
- chmod 600 ~/.kube/config
|
||
# Rewrite external API URL to in-cluster endpoint (pod can't reach external API due to CIDR overlap)
|
||
- sed -i 's|https://.*\.api\.k8s\.fr-par\.scw\.cloud:6443|https://kubernetes.default.svc|g' ~/.kube/config
|
||
# Verify cluster connectivity before deploying
|
||
- kubectl cluster-info
|
||
environment:
|
||
name: production
|
||
kubernetes:
|
||
namespace: foxhunt
|
||
script:
|
||
# S3 binary share: services use generic runtime image + initContainer to fetch binaries from S3.
|
||
# Deploy = apply K8s manifests (picks up YAML changes) + rollout restart (fetch new binaries).
|
||
- echo "Deploying ${CI_COMMIT_SHA}..."
|
||
# Apply pending database migrations before deploying services
|
||
- |
|
||
echo "Applying database migrations..."
|
||
DB_POD=$(kubectl -n foxhunt get pod -l app.kubernetes.io/name=postgres -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
|
||
if [ -n "$DB_POD" ]; then
|
||
# Create tracking table if it doesn't exist
|
||
kubectl -n foxhunt exec "$DB_POD" -- psql -U foxhunt -d foxhunt -c \
|
||
"CREATE TABLE IF NOT EXISTS _applied_migrations (name TEXT PRIMARY KEY, applied_at TIMESTAMPTZ DEFAULT now())" 2>/dev/null || true
|
||
# Apply each migration in order, skipping already-applied ones
|
||
for f in $(ls migrations/*.sql | grep -v '\.down\.' | sort); do
|
||
MIGRATION_NAME=$(basename "$f")
|
||
ALREADY=$(kubectl -n foxhunt exec "$DB_POD" -- psql -U foxhunt -d foxhunt -tAc \
|
||
"SELECT 1 FROM _applied_migrations WHERE name = '${MIGRATION_NAME}'" 2>/dev/null || true)
|
||
if [ "$ALREADY" = "1" ]; then
|
||
echo " skip: $MIGRATION_NAME (already applied)"
|
||
else
|
||
echo " apply: $MIGRATION_NAME"
|
||
kubectl -n foxhunt exec -i "$DB_POD" -- psql -U foxhunt -d foxhunt < "$f"
|
||
kubectl -n foxhunt exec "$DB_POD" -- psql -U foxhunt -d foxhunt -c \
|
||
"INSERT INTO _applied_migrations (name) VALUES ('${MIGRATION_NAME}')"
|
||
fi
|
||
done
|
||
echo "Migrations complete"
|
||
else
|
||
echo "WARNING: postgres pod not found, skipping migrations"
|
||
fi
|
||
# Deploy MinIO (in-cluster S3 for compiled binaries and trained models)
|
||
# Must be ready BEFORE service rollout — initContainers fetch binaries from MinIO.
|
||
- kubectl apply -k infra/k8s/minio/
|
||
- |
|
||
echo "Waiting for MinIO to be ready..."
|
||
kubectl -n foxhunt rollout status deployment/minio --timeout=120s
|
||
echo "Waiting for MinIO init-buckets job..."
|
||
kubectl -n foxhunt wait --for=condition=complete job/minio-init-buckets --timeout=60s 2>/dev/null || true
|
||
# Deploy monitoring stack (Loki, Tempo, Promtail, node-exporter, kube-state-metrics)
|
||
- kubectl apply -f infra/k8s/monitoring/loki.yaml -f infra/k8s/monitoring/tempo.yaml -f infra/k8s/monitoring/promtail.yaml -f infra/k8s/monitoring/node-exporter.yaml -f infra/k8s/monitoring/kube-state-metrics.yaml
|
||
# Deploy Grafana dashboards as ConfigMaps (sidecar auto-reloads)
|
||
# Resolve datasource variable placeholders to actual UIDs before applying
|
||
- |
|
||
PROM_UID="PBFA97CFB590B2093"
|
||
LOKI_UID="P8E80F9AEF21F6940"
|
||
TEMPO_UID="tempo"
|
||
DASH_DIR="infra/k8s/monitoring/dashboards"
|
||
TMP_DIR=$(mktemp -d)
|
||
for f in "$DASH_DIR"/*.json; do
|
||
python3 -c "
|
||
import json, sys
|
||
with open('$f') as fh:
|
||
d = json.load(fh)
|
||
d.pop('__inputs', None)
|
||
d.pop('__requires', None)
|
||
raw = json.dumps(d)
|
||
raw = raw.replace('\${DS_PROMETHEUS}', '${PROM_UID}')
|
||
raw = raw.replace('\${DS_LOKI}', '${LOKI_UID}')
|
||
raw = raw.replace('\${DS_TEMPO}', '${TEMPO_UID}')
|
||
print(raw)
|
||
" > "$TMP_DIR/$(basename "$f")"
|
||
done
|
||
# Build ConfigMaps grouped by category
|
||
declare -A GROUPS=(
|
||
[grafana-dashboards-foxhunt]="foxhunt-logs.json foxhunt-overview.json foxhunt-services.json foxhunt-cockpit.json foxhunt-trading-cockpit.json foxhunt-training-cockpit.json foxhunt-traces.json"
|
||
[grafana-dashboards-infra]="cluster-overview.json gpu-overview.json foxhunt-gpu-training.json foxhunt-infrastructure.json"
|
||
[grafana-dashboards-cicd]="foxhunt-ci-pipelines.json gitlab-services.json"
|
||
)
|
||
declare -A FOLDERS=(
|
||
[grafana-dashboards-foxhunt]="Foxhunt"
|
||
[grafana-dashboards-infra]="Infrastructure"
|
||
[grafana-dashboards-cicd]="CI/CD"
|
||
)
|
||
for cm_name in "${!GROUPS[@]}"; do
|
||
args=""
|
||
for df in ${GROUPS[$cm_name]}; do
|
||
[ -f "$TMP_DIR/$df" ] && args="$args --from-file=$df=$TMP_DIR/$df"
|
||
done
|
||
if [ -n "$args" ]; then
|
||
kubectl create configmap "$cm_name" -n foxhunt $args \
|
||
--dry-run=client -o yaml | \
|
||
kubectl label -f - --dry-run=client -o yaml --local grafana_dashboard=1 | \
|
||
kubectl annotate -f - --dry-run=client -o yaml --local grafana_folder="${FOLDERS[$cm_name]}" | \
|
||
kubectl apply -f -
|
||
echo " Applied ConfigMap: $cm_name (folder: ${FOLDERS[$cm_name]})"
|
||
fi
|
||
done
|
||
rm -rf "$TMP_DIR"
|
||
echo "Dashboard ConfigMaps deployed"
|
||
# Apply proxy (picks up nginx config changes for dashboard.fxhnt.ai routing)
|
||
- kubectl apply -f infra/k8s/gitlab/tailscale-proxy.yaml
|
||
# Apply service deployments and binary cache PVCs (idempotent — picks up any YAML changes)
|
||
# Skip training/job-template.yaml (placeholder names, not directly applicable)
|
||
# Skip storage/training-data-pv.yaml (PVC spec immutable after creation)
|
||
- kubectl apply -f infra/k8s/services/ -f infra/k8s/storage/binary-cache-pvcs.yaml
|
||
# Rolling restart triggers initContainers to fetch latest binaries from MinIO
|
||
- |
|
||
for svc in trading-service api-gateway broker-gateway \
|
||
ml-training-service backtesting-service \
|
||
trading-agent-service data-acquisition-service web-gateway; do
|
||
kubectl -n foxhunt rollout restart deployment/$svc
|
||
echo "Restarted $svc"
|
||
done
|
||
- |
|
||
for svc in trading-service api-gateway broker-gateway \
|
||
ml-training-service backtesting-service \
|
||
trading-agent-service data-acquisition-service web-gateway; do
|
||
kubectl -n foxhunt rollout status deployment/$svc --timeout=300s
|
||
done
|
||
- echo "All services deployed with MinIO binaries"
|