85725abd8f
- DQN/PPO trainers now record epoch, loss, val_loss, batch/s every epoch - CI training jobs get Prometheus scrape annotations via runner overrides - Allow Prometheus (foxhunt namespace) to scrape foxhunt-ci pods - Skip no-model metrics in monitoring service session grouping Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1661 lines
63 KiB
YAML
1661 lines
63 KiB
YAML
# GitLab CI/CD — Foxhunt
|
||
# Two runners: CPU (compile/build/deploy) and GPU (training)
|
||
# CI builder images:
|
||
# ci-builder: CUDA 12.4 + Rust 1.89 + protoc + sccache (CUDA stubs for compile)
|
||
# ci-builder-cpu: Rust 1.89 + protoc + sccache, no CUDA (for service binaries)
|
||
# Service images: pushed to Scaleway Container Registry (Kaniko)
|
||
#
|
||
# Runner routing (by tags):
|
||
# kapsule,rust → CPU runner → ci-compile-cpu pool (POP2-32C-128G, 32 vCPU)
|
||
# kapsule,gpu → GPU runner → ci-training (L40S)
|
||
# Override nodeSelector: KUBERNETES_NODE_SELECTOR_k8s.scaleway.com/pool-name: "<name>" (k=v format)
|
||
# Pools: ci-compile-cpu (POP2-32C-128G), ci-training (L40S training), services, gitlab
|
||
# All compilation (services + training) runs on ci-compile-cpu. CUDA stubs in ci-builder image.
|
||
|
||
# Ensure pipeline is always created — individual job rules handle filtering.
|
||
# Without this, GitLab CE may silently drop push pipelines when changes:
|
||
# evaluation fails to match (known issue with path glob evaluation).
|
||
workflow:
|
||
rules:
|
||
- if: $CI_PIPELINE_SOURCE == "push"
|
||
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
|
||
- if: $CI_PIPELINE_SOURCE == "schedule"
|
||
- if: $CI_PIPELINE_SOURCE == "api"
|
||
- if: $CI_PIPELINE_SOURCE == "web"
|
||
|
||
stages:
|
||
- prepare
|
||
- scan
|
||
- test
|
||
- compile
|
||
- train
|
||
- deploy
|
||
|
||
variables:
|
||
SQLX_OFFLINE: "true"
|
||
CARGO_TERM_COLOR: always
|
||
# CUDA_COMPUTE_CAP is set at runtime by detect_gpu() — L4=89, H100=90
|
||
# Do NOT hardcode here; sccache is partitioned by compute cap below
|
||
# Stable JWT secret for api_gateway tests (prevents env-var race between parallel tests)
|
||
JWT_SECRET: "CiTestSecret_Kx7mP9nR2sW5vY8bC3fG6jH1kL4pQ7tZ0uN9dM5eV8xS2wT6yA4zB_64chars_min"
|
||
# Training data on block storage PVC (mounted at /mnt/training-data by runner)
|
||
FOXHUNT_DATA_DIR: "/mnt/training-data/futures-baseline"
|
||
# sccache base dir on PVC — GPU stages override to /mnt/sccache/sm_<cap>
|
||
SCCACHE_DIR: "/mnt/sccache"
|
||
RUSTC_WRAPPER: /usr/local/bin/sccache
|
||
# Build profile: "dev-release" (thin LTO, fast) is default for quick iteration
|
||
# Set DEV_RELEASE=false in pipeline vars to use full "release" (fat LTO) profile
|
||
DEV_RELEASE: "true"
|
||
CARGO_BUILD_PROFILE: dev-release
|
||
# S3 credentials for Kaniko registry auth and Dockerfile builds
|
||
AWS_ACCESS_KEY_ID: $SCW_ACCESS_KEY
|
||
AWS_SECRET_ACCESS_KEY: $SCW_SECRET_KEY
|
||
# CI builder image on Scaleway Container Registry (reachable by Kapsule nodes)
|
||
CI_BUILDER_IMAGE: rg.fr-par.scw.cloud/foxhunt-ci/ci-builder:latest
|
||
CI_BUILDER_CPU_IMAGE: rg.fr-par.scw.cloud/foxhunt-ci/ci-builder-cpu:latest
|
||
# Scaleway Container Registry for service images
|
||
REGISTRY: rg.fr-par.scw.cloud/foxhunt-ci
|
||
# IaC runner image on Scaleway Container Registry
|
||
INFRA_RUNNER_IMAGE: rg.fr-par.scw.cloud/foxhunt-ci/infra-runner:latest
|
||
# Generic runtime images for S3 binary share (service pods + training pods + node builder)
|
||
RUNTIME_IMAGE: rg.fr-par.scw.cloud/foxhunt-ci/foxhunt-runtime:latest
|
||
TRAINING_RUNTIME_IMAGE: rg.fr-par.scw.cloud/foxhunt-ci/foxhunt-training-runtime:latest
|
||
# Node builder image eliminated — web dashboard uses node:22-slim + inline rclone
|
||
|
||
# --------------------------------------------------------------------------
|
||
# Stage 0: Build CI builder image → push to Scaleway CR
|
||
# --------------------------------------------------------------------------
|
||
build-ci-builder:
|
||
stage: prepare
|
||
image:
|
||
name: gcr.io/kaniko-project/executor:debug
|
||
entrypoint: [""]
|
||
tags:
|
||
- kapsule
|
||
- docker
|
||
rules:
|
||
- if: $CI_PIPELINE_SOURCE == "push"
|
||
changes:
|
||
- infra/docker/Dockerfile.ci-builder
|
||
when: on_success
|
||
- when: manual
|
||
allow_failure: true
|
||
before_script:
|
||
- mkdir -p /kaniko/.docker
|
||
- >-
|
||
echo "{\"auths\":{
|
||
\"rg.fr-par.scw.cloud\":{\"username\":\"nologin\",\"password\":\"${SCW_SECRET_KEY}\"},
|
||
\"https://index.docker.io/v1/\":{\"username\":\"${DOCKERHUB_USERNAME}\",\"password\":\"${DOCKERHUB_TOKEN}\"}
|
||
}}" > /kaniko/.docker/config.json
|
||
script:
|
||
- /kaniko/executor
|
||
--context "${CI_PROJECT_DIR}"
|
||
--dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.ci-builder"
|
||
--cache=true --cache-repo="${REGISTRY}/cache"
|
||
--destination "${CI_BUILDER_IMAGE}"
|
||
|
||
# --------------------------------------------------------------------------
|
||
# Stage 0a: Build CPU-only CI builder image → push to Scaleway CR
|
||
# --------------------------------------------------------------------------
|
||
build-ci-builder-cpu:
|
||
stage: prepare
|
||
image:
|
||
name: gcr.io/kaniko-project/executor:debug
|
||
entrypoint: [""]
|
||
tags:
|
||
- kapsule
|
||
- docker
|
||
rules:
|
||
- if: $CI_PIPELINE_SOURCE == "push"
|
||
changes:
|
||
- infra/docker/Dockerfile.ci-builder-cpu
|
||
when: on_success
|
||
- when: manual
|
||
allow_failure: true
|
||
before_script:
|
||
- mkdir -p /kaniko/.docker
|
||
- >-
|
||
echo "{\"auths\":{
|
||
\"rg.fr-par.scw.cloud\":{\"username\":\"nologin\",\"password\":\"${SCW_SECRET_KEY}\"},
|
||
\"https://index.docker.io/v1/\":{\"username\":\"${DOCKERHUB_USERNAME}\",\"password\":\"${DOCKERHUB_TOKEN}\"}
|
||
}}" > /kaniko/.docker/config.json
|
||
script:
|
||
- /kaniko/executor
|
||
--context "${CI_PROJECT_DIR}"
|
||
--dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.ci-builder-cpu"
|
||
--cache=true --cache-repo="${REGISTRY}/cache"
|
||
--destination "${CI_BUILDER_CPU_IMAGE}"
|
||
|
||
# --------------------------------------------------------------------------
|
||
# Stage 0b: Build devcontainer image → push to internal GitLab registry
|
||
# --------------------------------------------------------------------------
|
||
build-devcontainer:
|
||
stage: prepare
|
||
image:
|
||
name: gcr.io/kaniko-project/executor:debug
|
||
entrypoint: [""]
|
||
tags:
|
||
- kapsule
|
||
- docker
|
||
rules:
|
||
- if: $CI_PIPELINE_SOURCE == "push"
|
||
changes:
|
||
- .devcontainer/**
|
||
when: on_success
|
||
- when: manual
|
||
allow_failure: true
|
||
before_script:
|
||
- mkdir -p /kaniko/.docker
|
||
- >-
|
||
echo "{\"auths\":{
|
||
\"rg.fr-par.scw.cloud\":{\"username\":\"nologin\",\"password\":\"${SCW_SECRET_KEY}\"},
|
||
\"https://index.docker.io/v1/\":{\"username\":\"${DOCKERHUB_USERNAME}\",\"password\":\"${DOCKERHUB_TOKEN}\"}
|
||
}}" > /kaniko/.docker/config.json
|
||
script:
|
||
- /kaniko/executor
|
||
--context "${CI_PROJECT_DIR}"
|
||
--dockerfile "${CI_PROJECT_DIR}/.devcontainer/Dockerfile"
|
||
--cache=true --cache-repo="${REGISTRY}/cache"
|
||
--destination "rg.fr-par.scw.cloud/foxhunt-ci/devcontainer:${CI_COMMIT_SHA}"
|
||
--destination "rg.fr-par.scw.cloud/foxhunt-ci/devcontainer:latest"
|
||
|
||
# --------------------------------------------------------------------------
|
||
# Stage 0c: Build infra-runner image → push to Scaleway CR
|
||
# --------------------------------------------------------------------------
|
||
build-infra-runner:
|
||
stage: prepare
|
||
image:
|
||
name: gcr.io/kaniko-project/executor:debug
|
||
entrypoint: [""]
|
||
tags:
|
||
- kapsule
|
||
- docker
|
||
rules:
|
||
- if: $CI_PIPELINE_SOURCE == "push"
|
||
changes:
|
||
- infra/docker/Dockerfile.infra-runner
|
||
when: on_success
|
||
- when: manual
|
||
allow_failure: true
|
||
before_script:
|
||
- mkdir -p /kaniko/.docker
|
||
- >-
|
||
echo "{\"auths\":{
|
||
\"rg.fr-par.scw.cloud\":{\"username\":\"nologin\",\"password\":\"${SCW_SECRET_KEY}\"},
|
||
\"https://index.docker.io/v1/\":{\"username\":\"${DOCKERHUB_USERNAME}\",\"password\":\"${DOCKERHUB_TOKEN}\"}
|
||
}}" > /kaniko/.docker/config.json
|
||
script:
|
||
- /kaniko/executor
|
||
--context "${CI_PROJECT_DIR}"
|
||
--dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.infra-runner"
|
||
--cache=true --cache-repo="${REGISTRY}/cache"
|
||
--destination "${INFRA_RUNNER_IMAGE}"
|
||
|
||
# --------------------------------------------------------------------------
|
||
# Stage 0d: Build generic runtime base images → push to Scaleway CR
|
||
# --------------------------------------------------------------------------
|
||
build-foxhunt-runtime:
|
||
stage: prepare
|
||
image:
|
||
name: gcr.io/kaniko-project/executor:debug
|
||
entrypoint: [""]
|
||
tags:
|
||
- kapsule
|
||
- docker
|
||
rules:
|
||
- if: $CI_PIPELINE_SOURCE == "push"
|
||
changes:
|
||
- infra/docker/Dockerfile.foxhunt-runtime
|
||
when: on_success
|
||
- when: manual
|
||
allow_failure: true
|
||
before_script:
|
||
- mkdir -p /kaniko/.docker
|
||
- >-
|
||
echo "{\"auths\":{
|
||
\"rg.fr-par.scw.cloud\":{\"username\":\"nologin\",\"password\":\"${SCW_SECRET_KEY}\"},
|
||
\"https://index.docker.io/v1/\":{\"username\":\"${DOCKERHUB_USERNAME}\",\"password\":\"${DOCKERHUB_TOKEN}\"}
|
||
}}" > /kaniko/.docker/config.json
|
||
script:
|
||
- /kaniko/executor
|
||
--context "${CI_PROJECT_DIR}"
|
||
--dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.foxhunt-runtime"
|
||
--cache=true --cache-repo="${REGISTRY}/cache"
|
||
--destination "${REGISTRY}/foxhunt-runtime:latest"
|
||
|
||
build-foxhunt-training-runtime:
|
||
stage: prepare
|
||
image:
|
||
name: gcr.io/kaniko-project/executor:debug
|
||
entrypoint: [""]
|
||
tags:
|
||
- kapsule
|
||
- docker
|
||
rules:
|
||
- if: $CI_PIPELINE_SOURCE == "push"
|
||
changes:
|
||
- infra/docker/Dockerfile.foxhunt-training-runtime
|
||
when: on_success
|
||
- when: manual
|
||
allow_failure: true
|
||
before_script:
|
||
- mkdir -p /kaniko/.docker
|
||
- >-
|
||
echo "{\"auths\":{
|
||
\"rg.fr-par.scw.cloud\":{\"username\":\"nologin\",\"password\":\"${SCW_SECRET_KEY}\"},
|
||
\"https://index.docker.io/v1/\":{\"username\":\"${DOCKERHUB_USERNAME}\",\"password\":\"${DOCKERHUB_TOKEN}\"}
|
||
}}" > /kaniko/.docker/config.json
|
||
script:
|
||
- /kaniko/executor
|
||
--context "${CI_PROJECT_DIR}"
|
||
--dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.foxhunt-training-runtime"
|
||
--cache=true --cache-repo="${REGISTRY}/cache"
|
||
--destination "${REGISTRY}/foxhunt-training-runtime:latest"
|
||
|
||
# --------------------------------------------------------------------------
|
||
# Stage 0.5: Scan runtime images for vulnerabilities (Trivy)
|
||
# --------------------------------------------------------------------------
|
||
scan-runtime-images:
|
||
stage: scan
|
||
image:
|
||
name: aquasec/trivy:latest
|
||
entrypoint: [""]
|
||
tags:
|
||
- kapsule
|
||
- docker
|
||
variables:
|
||
KUBERNETES_CPU_REQUEST: "200m"
|
||
KUBERNETES_CPU_LIMIT: "500m"
|
||
KUBERNETES_MEMORY_REQUEST: "256Mi"
|
||
KUBERNETES_MEMORY_LIMIT: "512Mi"
|
||
needs:
|
||
- job: build-foxhunt-runtime
|
||
optional: true
|
||
- job: build-foxhunt-training-runtime
|
||
optional: true
|
||
rules:
|
||
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
|
||
- if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
|
||
cache:
|
||
key: trivy-db
|
||
paths:
|
||
- .trivy-cache/
|
||
before_script:
|
||
- export TRIVY_USERNAME=nologin
|
||
- export TRIVY_PASSWORD=${SCW_SECRET_KEY}
|
||
script:
|
||
- echo "Scanning runtime images for CRITICAL/HIGH vulnerabilities..."
|
||
- trivy image
|
||
--cache-dir .trivy-cache
|
||
--exit-code 1
|
||
--severity CRITICAL,HIGH
|
||
--ignore-unfixed
|
||
--no-progress
|
||
"${REGISTRY}/foxhunt-runtime:latest"
|
||
- trivy image
|
||
--cache-dir .trivy-cache
|
||
--exit-code 1
|
||
--severity CRITICAL,HIGH
|
||
--ignore-unfixed
|
||
--no-progress
|
||
"${REGISTRY}/foxhunt-training-runtime:latest"
|
||
- echo "Image scan passed — no fixable CRITICAL/HIGH vulnerabilities."
|
||
allow_failure: false
|
||
|
||
|
||
# Base template for Rust jobs — pre-baked CI builder from Scaleway CR
|
||
# Image pre-exists in SCR; rebuild via build-ci-builder when Dockerfile changes
|
||
# Runs on ci-compile-cpu pool (POP2-32C-128G: 32 vCPU, 128GB RAM) via CPU runner
|
||
# CPU budget: 14 CPUs per job → compile-services + compile-training run in parallel on 1 node
|
||
# (Scaleway quota: 1× POP2-32C-128G, so both jobs must share)
|
||
# Base template for jobs needing CUDA builder (test stage only — compile uses .rust-base-cpu)
|
||
.rust-base:
|
||
image: ${CI_BUILDER_IMAGE}
|
||
tags:
|
||
- kapsule
|
||
- rust
|
||
variables:
|
||
CUDA_COMPUTE_CAP: "89"
|
||
CARGO_BUILD_JOBS: "14"
|
||
KUBERNETES_CPU_REQUEST: "14000m"
|
||
KUBERNETES_CPU_LIMIT: "15500m"
|
||
KUBERNETES_MEMORY_REQUEST: "24Gi"
|
||
KUBERNETES_MEMORY_LIMIT: "56Gi"
|
||
before_script:
|
||
# Mold linker fallback — .cargo/config.toml uses -fuse-ld=mold via clang.
|
||
# clang's -fuse-ld=mold searches for "ld.mold" (not "mold") in its linker paths.
|
||
# If mold isn't installed, create ld.mold wrapper that delegates to ld.lld.
|
||
- |
|
||
if ! command -v mold &>/dev/null; then
|
||
printf '#!/bin/sh\nexec ld.lld "$@"\n' > /usr/local/bin/ld.mold
|
||
chmod +x /usr/local/bin/ld.mold
|
||
echo "mold not found, falling back to lld via ld.mold wrapper"
|
||
fi
|
||
|
||
# Base template for CPU-only Rust jobs — lightweight builder without CUDA
|
||
# Uses ci-builder-cpu image (rust:1.89-slim, ~2-3GB, no CUDA toolkit)
|
||
# CPU budget: 14 CPUs per job → runs in parallel with compile-training on same node
|
||
.rust-base-cpu:
|
||
image: ${CI_BUILDER_CPU_IMAGE}
|
||
tags:
|
||
- kapsule
|
||
- rust
|
||
variables:
|
||
# No CUDA_COMPUTE_CAP — CPU-only builds don't compile CUDA kernels
|
||
CARGO_BUILD_JOBS: "14"
|
||
KUBERNETES_NODE_SELECTOR_POOL: "k8s.scaleway.com/pool-name=ci-compile-cpu"
|
||
KUBERNETES_CPU_REQUEST: "14000m"
|
||
KUBERNETES_CPU_LIMIT: "15500m"
|
||
KUBERNETES_MEMORY_REQUEST: "24Gi"
|
||
KUBERNETES_MEMORY_LIMIT: "56Gi"
|
||
before_script:
|
||
# Mold linker fallback — same as .rust-base
|
||
- |
|
||
if ! command -v mold &>/dev/null; then
|
||
printf '#!/bin/sh\nexec ld.lld "$@"\n' > /usr/local/bin/ld.mold
|
||
chmod +x /usr/local/bin/ld.mold
|
||
echo "mold not found, falling back to lld via ld.mold wrapper"
|
||
fi
|
||
|
||
# --------------------------------------------------------------------------
|
||
# Stage 2: tests
|
||
# --------------------------------------------------------------------------
|
||
test:
|
||
extends: .rust-base
|
||
stage: test
|
||
needs: []
|
||
rules:
|
||
# Tests disabled by default — compile+deploy is the fast path for now.
|
||
# Trigger manually from the pipeline UI when needed.
|
||
- if: $CI_PIPELINE_SOURCE == "push"
|
||
when: manual
|
||
allow_failure: true
|
||
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
|
||
when: manual
|
||
allow_failure: true
|
||
- if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
|
||
when: manual
|
||
allow_failure: true
|
||
services:
|
||
- name: redis:7-alpine
|
||
alias: redis
|
||
script:
|
||
# Detect GPU if available (nvidia runtime), otherwise use CUDA_COMPUTE_CAP from variables
|
||
- |
|
||
if command -v nvidia-smi &>/dev/null; then
|
||
export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//')
|
||
export CUDA_COMPUTE_CAP=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader | head -1 | tr -d '.')
|
||
echo "GPU=$(nvidia-smi --query-gpu=name --format=csv,noheader) CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP"
|
||
else
|
||
echo "No nvidia-smi — using CUDA stubs, CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP"
|
||
fi
|
||
- export SCCACHE_DIR="/mnt/sccache/sm_${CUDA_COMPUTE_CAP}"
|
||
- mkdir -p "$SCCACHE_DIR"
|
||
# Wait for Redis sidecar (K8s executor doesn't wait for service readiness)
|
||
- |
|
||
for i in $(seq 1 30); do
|
||
if bash -c 'echo PING > /dev/tcp/localhost/6379' 2>/dev/null; then
|
||
echo "Redis ready"; break
|
||
fi
|
||
echo "Waiting for Redis... ($i/30)"; sleep 1
|
||
done
|
||
- sccache --zero-stats || true
|
||
- cargo test --workspace --lib -- --skip model_loader::tests
|
||
- sccache --show-stats || true
|
||
|
||
# --------------------------------------------------------------------------
|
||
# Stage 3a: Compile service binaries (CPU-only, no CUDA)
|
||
# --------------------------------------------------------------------------
|
||
compile-services:
|
||
extends: .rust-base-cpu
|
||
stage: compile
|
||
needs: []
|
||
rules:
|
||
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
|
||
changes:
|
||
- Cargo.toml
|
||
- Cargo.lock
|
||
- crates/**
|
||
- bin/**
|
||
- services/**
|
||
- infra/docker/Dockerfile.*
|
||
- if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
|
||
script:
|
||
- mkdir -p "$SCCACHE_DIR" && sccache --zero-stats || true
|
||
- PROFILE=${DEV_RELEASE:+dev-release}; PROFILE=${PROFILE:-release}
|
||
- echo "Building services with --profile $PROFILE (CPU-only, no CUDA)"
|
||
- TARGET_DIR="target/$PROFILE"
|
||
- cargo build --profile $PROFILE
|
||
-p trading_service
|
||
-p api_gateway
|
||
-p broker_gateway_service
|
||
-p ml_training_service
|
||
-p backtesting_service
|
||
-p trading_agent_service
|
||
-p data_acquisition_service
|
||
-p web-gateway
|
||
-p monitoring_service
|
||
- sccache --show-stats || true
|
||
- mkdir -p build-out/services
|
||
- |
|
||
for bin in trading_service api_gateway broker_gateway_service ml_training_service backtesting_service trading_agent_service data_acquisition_service web-gateway monitoring_service; do
|
||
cp $TARGET_DIR/$bin build-out/services/
|
||
strip build-out/services/$bin
|
||
done
|
||
- ls -lh build-out/services/
|
||
- echo "Service binaries ready as CI artifacts"
|
||
artifacts:
|
||
paths:
|
||
- build-out/services/
|
||
expire_in: 1 day
|
||
|
||
# --------------------------------------------------------------------------
|
||
# Stage 3a: Compile training binaries (CPU node, CUDA stubs from ci-builder image)
|
||
# --------------------------------------------------------------------------
|
||
compile-training:
|
||
extends: .rust-base-cpu
|
||
stage: compile
|
||
image: ${CI_BUILDER_IMAGE}
|
||
needs: [] # parallel with compile-services — both share 1 POP2-32C-128G node (14 CPUs each)
|
||
variables:
|
||
# CUDA stubs from ci-builder image — compiles CUDA kernels without GPU hardware.
|
||
# L4 GPU = compute capability 8.9 → candle-kernels generates sm_89 PTX
|
||
CUDA_COMPUTE_CAP: "89"
|
||
rules:
|
||
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
|
||
changes:
|
||
- Cargo.toml
|
||
- Cargo.lock
|
||
- crates/ml/**
|
||
- crates/trading_engine/**
|
||
- crates/common/**
|
||
- crates/risk/**
|
||
- crates/data/**
|
||
- crates/config/**
|
||
- crates/storage/**
|
||
- services/training_uploader/**
|
||
- if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
|
||
script:
|
||
- echo "CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP"
|
||
- export SCCACHE_DIR="/mnt/sccache/sm_${CUDA_COMPUTE_CAP}"
|
||
- mkdir -p "$SCCACHE_DIR" && sccache --zero-stats || true
|
||
- PROFILE=${DEV_RELEASE:+dev-release}; PROFILE=${PROFILE:-release}
|
||
- echo "Building training binaries with --profile $PROFILE (CUDA stubs)"
|
||
- TARGET_DIR="target/$PROFILE"
|
||
- cargo build --profile $PROFILE -p ml --features ml/cuda
|
||
--example train_baseline_rl
|
||
--example train_baseline_supervised
|
||
--example evaluate_baseline
|
||
--example evaluate_supervised
|
||
--example hyperopt_baseline_rl
|
||
--example hyperopt_baseline_supervised
|
||
- cargo build --profile $PROFILE -p training_uploader
|
||
- sccache --show-stats || true
|
||
- mkdir -p build-out/training
|
||
- |
|
||
for bin in train_baseline_rl train_baseline_supervised evaluate_baseline evaluate_supervised hyperopt_baseline_rl hyperopt_baseline_supervised; do
|
||
cp $TARGET_DIR/examples/$bin build-out/training/
|
||
strip build-out/training/$bin
|
||
done
|
||
cp $TARGET_DIR/training_uploader build-out/training/
|
||
strip build-out/training/training_uploader
|
||
- ls -lh build-out/training/
|
||
- echo "Training binaries ready as CI artifacts"
|
||
artifacts:
|
||
paths:
|
||
- build-out/training/
|
||
expire_in: 3 days
|
||
|
||
# --------------------------------------------------------------------------
|
||
# Stage 3c: Build web-dashboard static assets (Vite)
|
||
# --------------------------------------------------------------------------
|
||
build-web-dashboard:
|
||
stage: compile
|
||
image: node:22-slim
|
||
needs: []
|
||
tags:
|
||
- kapsule
|
||
- docker
|
||
variables:
|
||
KUBERNETES_NODE_SELECTOR_POOL: "k8s.scaleway.com/pool-name=ci-compile-cpu"
|
||
KUBERNETES_CPU_REQUEST: "2000m"
|
||
KUBERNETES_CPU_LIMIT: "4000m"
|
||
KUBERNETES_MEMORY_REQUEST: "2Gi"
|
||
KUBERNETES_MEMORY_LIMIT: "4Gi"
|
||
rules:
|
||
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
|
||
changes:
|
||
- web-dashboard/**
|
||
- if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
|
||
script:
|
||
- cd web-dashboard
|
||
- npm ci
|
||
- npm run build
|
||
- echo "Web dashboard assets ready as CI artifacts"
|
||
artifacts:
|
||
paths:
|
||
- web-dashboard/dist/
|
||
expire_in: 1 day
|
||
|
||
# --------------------------------------------------------------------------
|
||
# Stage 3c: Training — RL on L4 (CPU-heavy), Supervised on L40S (GPU-heavy)
|
||
# --------------------------------------------------------------------------
|
||
|
||
# --------------------------------------------------------------------------
|
||
# RL training base: DQN + PPO → ci-training pool (L40S)
|
||
# RL hyperopt runs parallel PSO trials — CPU+GPU bound, low VRAM.
|
||
# --------------------------------------------------------------------------
|
||
.train-rl-base:
|
||
stage: train
|
||
image: ${TRAINING_RUNTIME_IMAGE}
|
||
tags:
|
||
- kapsule
|
||
- gpu
|
||
variables:
|
||
# Route to ci-training pool (L40S GPU)
|
||
KUBERNETES_NODE_SELECTOR_POOL: "k8s.scaleway.com/pool-name=ci-training"
|
||
KUBERNETES_RUNTIME_CLASS_NAME: "nvidia"
|
||
KUBERNETES_CPU_REQUEST: "7000m"
|
||
KUBERNETES_CPU_LIMIT: "8000m"
|
||
KUBERNETES_MEMORY_REQUEST: "8Gi"
|
||
KUBERNETES_MEMORY_LIMIT: "20Gi"
|
||
CUBLAS_WORKSPACE_CONFIG: ":4096:8"
|
||
OTEL_EXPORTER_OTLP_ENDPOINT: "http://tempo.foxhunt.svc.cluster.local:4317"
|
||
# Expose training metrics to Prometheus (binaries serve /metrics on :9094)
|
||
KUBERNETES_POD_ANNOTATIONS_1: "gitlab.com/prometheus_scrape=true"
|
||
KUBERNETES_POD_ANNOTATIONS_2: "gitlab.com/prometheus_port=9094"
|
||
KUBERNETES_POD_ANNOTATIONS_3: "gitlab.com/prometheus_path=/metrics"
|
||
rules:
|
||
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
|
||
when: manual
|
||
allow_failure: true
|
||
- if: $CI_PIPELINE_SOURCE == "schedule" && $TRAIN_VALIDATE == "true"
|
||
- if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
|
||
when: manual
|
||
allow_failure: true
|
||
before_script:
|
||
- export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//')
|
||
- nvidia-smi
|
||
- mkdir -p ${CI_PROJECT_DIR}/output ${CI_PROJECT_DIR}/training-bin
|
||
# Training binaries come from compile-training CI artifacts
|
||
- cp build-out/training/* ${CI_PROJECT_DIR}/training-bin/ 2>/dev/null || true
|
||
- chmod +x ${CI_PROJECT_DIR}/training-bin/* 2>/dev/null || true
|
||
- export PATH="${CI_PROJECT_DIR}/training-bin:$PATH"
|
||
|
||
# --------------------------------------------------------------------------
|
||
# Supervised training base: TFT, Mamba2, TGGN, TLOB, Liquid, KAN, xLSTM, Diffusion → L40S
|
||
# Supervised models are GPU-bound — low CPU, high VRAM.
|
||
# --------------------------------------------------------------------------
|
||
.train-validate-base:
|
||
stage: train
|
||
image: ${TRAINING_RUNTIME_IMAGE}
|
||
tags:
|
||
- kapsule
|
||
- gpu
|
||
variables:
|
||
# nvidia runtime for GPU nodes (ci-training has L40S 48GB)
|
||
KUBERNETES_RUNTIME_CLASS_NAME: "nvidia"
|
||
# Route to L40S pool (48GB VRAM for large supervised models)
|
||
# Overrides runner default to L40S pool
|
||
KUBERNETES_NODE_SELECTOR_POOL: "k8s.scaleway.com/pool-name=ci-training"
|
||
# Supervised is GPU-bound — minimal CPU. Low request allows 3-4 concurrent jobs.
|
||
KUBERNETES_CPU_REQUEST: "1000m"
|
||
KUBERNETES_CPU_LIMIT: "2000m"
|
||
KUBERNETES_MEMORY_REQUEST: "16Gi"
|
||
KUBERNETES_MEMORY_LIMIT: "40Gi"
|
||
OTEL_EXPORTER_OTLP_ENDPOINT: "http://tempo.foxhunt.svc.cluster.local:4317"
|
||
# Expose training metrics to Prometheus (binaries serve /metrics on :9094)
|
||
KUBERNETES_POD_ANNOTATIONS_1: "gitlab.com/prometheus_scrape=true"
|
||
KUBERNETES_POD_ANNOTATIONS_2: "gitlab.com/prometheus_port=9094"
|
||
KUBERNETES_POD_ANNOTATIONS_3: "gitlab.com/prometheus_path=/metrics"
|
||
rules:
|
||
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
|
||
when: manual
|
||
allow_failure: true
|
||
- if: $CI_PIPELINE_SOURCE == "schedule" && $TRAIN_VALIDATE == "true"
|
||
- if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
|
||
when: manual
|
||
allow_failure: true
|
||
before_script:
|
||
- export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//')
|
||
- nvidia-smi
|
||
- mkdir -p ${CI_PROJECT_DIR}/output ${CI_PROJECT_DIR}/training-bin
|
||
# Training binaries come from compile-training CI artifacts
|
||
- cp build-out/training/* ${CI_PROJECT_DIR}/training-bin/ 2>/dev/null || true
|
||
- chmod +x ${CI_PROJECT_DIR}/training-bin/* 2>/dev/null || true
|
||
- export PATH="${CI_PROJECT_DIR}/training-bin:$PATH"
|
||
|
||
train-validate-rl:
|
||
extends: .train-rl-base
|
||
needs:
|
||
- job: compile-training
|
||
optional: true
|
||
script:
|
||
- |
|
||
train_baseline_rl \
|
||
--model both \
|
||
--symbol ES.FUT \
|
||
--data-dir /mnt/training-data/futures-baseline \
|
||
--output-dir ${CI_PROJECT_DIR}/output \
|
||
--max-steps-per-epoch 50 \
|
||
&& TRAIN_OK=1 || TRAIN_OK=0
|
||
echo "=== RL training exit: $TRAIN_OK ==="
|
||
echo "=== Running evaluation ==="
|
||
evaluate_baseline \
|
||
--model both \
|
||
--models-dir ${CI_PROJECT_DIR}/output \
|
||
--data-dir /mnt/training-data/futures-baseline \
|
||
--output ${CI_PROJECT_DIR}/output/rl_eval_report.json \
|
||
--symbol ES.FUT \
|
||
|| true
|
||
echo "=== Eval report ==="
|
||
cat ${CI_PROJECT_DIR}/output/rl_eval_report.json 2>/dev/null || echo "No eval report"
|
||
[ "$TRAIN_OK" = "1" ] || exit 1
|
||
artifacts:
|
||
paths:
|
||
- output/rl_eval_report.json
|
||
when: always
|
||
expire_in: 30 days
|
||
|
||
train-validate-tft:
|
||
extends: .train-validate-base
|
||
needs:
|
||
- job: compile-training
|
||
optional: true
|
||
script:
|
||
- train_baseline_supervised
|
||
--model tft
|
||
--symbol ES.FUT
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output-dir ${CI_PROJECT_DIR}/output
|
||
--max-steps-per-epoch 50
|
||
- echo "=== TFT training complete, running evaluation ==="
|
||
- evaluate_supervised
|
||
--model tft
|
||
--models-dir ${CI_PROJECT_DIR}/output
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output ${CI_PROJECT_DIR}/output/tft_eval_report.json
|
||
--symbol ES.FUT
|
||
- echo "TFT validation + evaluation passed"
|
||
- cat ${CI_PROJECT_DIR}/output/tft_eval_report.json
|
||
artifacts:
|
||
paths:
|
||
- output/tft_eval_report.json
|
||
when: always
|
||
expire_in: 30 days
|
||
|
||
hyperopt-ppo:
|
||
extends: .train-rl-base
|
||
needs:
|
||
- job: compile-training
|
||
optional: true
|
||
timeout: 4h
|
||
script:
|
||
- hyperopt_baseline_rl
|
||
--model ppo
|
||
--trials 20
|
||
--n-initial 5
|
||
--epochs 8
|
||
--parallel 0
|
||
--symbol ES.FUT
|
||
--tx-cost-bps 0.1
|
||
--tick-size 0.25
|
||
--spread-ticks 1.0
|
||
--initial-capital 35000
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--base-dir ${CI_PROJECT_DIR}/output/hyperopt
|
||
--output ${CI_PROJECT_DIR}/output/ppo_hyperopt_results.json
|
||
- echo "=== PPO Hyperopt complete ==="
|
||
- cat ${CI_PROJECT_DIR}/output/ppo_hyperopt_results.json
|
||
- echo "=== Training PPO with best params ==="
|
||
- train_baseline_rl
|
||
--model ppo
|
||
--symbol ES.FUT
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output-dir ${CI_PROJECT_DIR}/output
|
||
--hyperopt-params ${CI_PROJECT_DIR}/output/ppo_hyperopt_results.json
|
||
- echo "=== Evaluating PPO with best params ==="
|
||
- evaluate_baseline
|
||
--model ppo
|
||
--models-dir ${CI_PROJECT_DIR}/output
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output ${CI_PROJECT_DIR}/output/ppo_eval_report.json
|
||
--symbol ES.FUT
|
||
- cat ${CI_PROJECT_DIR}/output/ppo_eval_report.json
|
||
artifacts:
|
||
paths:
|
||
- output/ppo_hyperopt_results.json
|
||
- output/ppo_eval_report.json
|
||
when: always
|
||
expire_in: 90 days
|
||
|
||
# --- Train-validate: remaining supervised models ---
|
||
|
||
train-validate-mamba2:
|
||
extends: .train-validate-base
|
||
needs:
|
||
- job: compile-training
|
||
optional: true
|
||
script:
|
||
- train_baseline_supervised
|
||
--model mamba2
|
||
--symbol ES.FUT
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output-dir ${CI_PROJECT_DIR}/output
|
||
--max-steps-per-epoch 50
|
||
- echo "=== Mamba2 training complete, running evaluation ==="
|
||
- evaluate_supervised
|
||
--model mamba2
|
||
--models-dir ${CI_PROJECT_DIR}/output
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output ${CI_PROJECT_DIR}/output/mamba2_eval_report.json
|
||
--symbol ES.FUT
|
||
- echo "Mamba2 validation + evaluation passed"
|
||
- cat ${CI_PROJECT_DIR}/output/mamba2_eval_report.json
|
||
artifacts:
|
||
paths:
|
||
- output/mamba2_eval_report.json
|
||
when: always
|
||
expire_in: 30 days
|
||
|
||
train-validate-liquid:
|
||
extends: .train-validate-base
|
||
needs:
|
||
- job: compile-training
|
||
optional: true
|
||
script:
|
||
- train_baseline_supervised
|
||
--model liquid
|
||
--symbol ES.FUT
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output-dir ${CI_PROJECT_DIR}/output
|
||
--max-steps-per-epoch 50
|
||
- echo "=== Liquid training complete, running evaluation ==="
|
||
- evaluate_supervised
|
||
--model liquid
|
||
--models-dir ${CI_PROJECT_DIR}/output
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output ${CI_PROJECT_DIR}/output/liquid_eval_report.json
|
||
--symbol ES.FUT
|
||
- echo "Liquid validation + evaluation passed"
|
||
- cat ${CI_PROJECT_DIR}/output/liquid_eval_report.json
|
||
artifacts:
|
||
paths:
|
||
- output/liquid_eval_report.json
|
||
when: always
|
||
expire_in: 30 days
|
||
|
||
train-validate-tggn:
|
||
extends: .train-validate-base
|
||
needs:
|
||
- job: compile-training
|
||
optional: true
|
||
script:
|
||
- train_baseline_supervised
|
||
--model tggn
|
||
--symbol ES.FUT
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output-dir ${CI_PROJECT_DIR}/output
|
||
--max-steps-per-epoch 50
|
||
- echo "=== TGGN training complete, running evaluation ==="
|
||
- evaluate_supervised
|
||
--model tggn
|
||
--models-dir ${CI_PROJECT_DIR}/output
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output ${CI_PROJECT_DIR}/output/tggn_eval_report.json
|
||
--symbol ES.FUT
|
||
- echo "TGGN validation + evaluation passed"
|
||
- cat ${CI_PROJECT_DIR}/output/tggn_eval_report.json
|
||
artifacts:
|
||
paths:
|
||
- output/tggn_eval_report.json
|
||
when: always
|
||
expire_in: 30 days
|
||
|
||
train-validate-tlob:
|
||
extends: .train-validate-base
|
||
needs:
|
||
- job: compile-training
|
||
optional: true
|
||
script:
|
||
- train_baseline_supervised
|
||
--model tlob
|
||
--symbol ES.FUT
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output-dir ${CI_PROJECT_DIR}/output
|
||
--max-steps-per-epoch 50
|
||
- echo "=== TLOB training complete, running evaluation ==="
|
||
- evaluate_supervised
|
||
--model tlob
|
||
--models-dir ${CI_PROJECT_DIR}/output
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output ${CI_PROJECT_DIR}/output/tlob_eval_report.json
|
||
--symbol ES.FUT
|
||
- echo "TLOB validation + evaluation passed"
|
||
- cat ${CI_PROJECT_DIR}/output/tlob_eval_report.json
|
||
artifacts:
|
||
paths:
|
||
- output/tlob_eval_report.json
|
||
when: always
|
||
expire_in: 30 days
|
||
|
||
train-validate-kan:
|
||
extends: .train-validate-base
|
||
needs:
|
||
- job: compile-training
|
||
optional: true
|
||
script:
|
||
- train_baseline_supervised
|
||
--model kan
|
||
--symbol ES.FUT
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output-dir ${CI_PROJECT_DIR}/output
|
||
--max-steps-per-epoch 50
|
||
- echo "=== KAN training complete, running evaluation ==="
|
||
- evaluate_supervised
|
||
--model kan
|
||
--models-dir ${CI_PROJECT_DIR}/output
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output ${CI_PROJECT_DIR}/output/kan_eval_report.json
|
||
--symbol ES.FUT
|
||
- echo "KAN validation + evaluation passed"
|
||
- cat ${CI_PROJECT_DIR}/output/kan_eval_report.json
|
||
artifacts:
|
||
paths:
|
||
- output/kan_eval_report.json
|
||
when: always
|
||
expire_in: 30 days
|
||
|
||
train-validate-xlstm:
|
||
extends: .train-validate-base
|
||
needs:
|
||
- job: compile-training
|
||
optional: true
|
||
script:
|
||
- train_baseline_supervised
|
||
--model xlstm
|
||
--symbol ES.FUT
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output-dir ${CI_PROJECT_DIR}/output
|
||
--max-steps-per-epoch 50
|
||
- echo "=== xLSTM training complete, running evaluation ==="
|
||
- evaluate_supervised
|
||
--model xlstm
|
||
--models-dir ${CI_PROJECT_DIR}/output
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output ${CI_PROJECT_DIR}/output/xlstm_eval_report.json
|
||
--symbol ES.FUT
|
||
- echo "xLSTM validation + evaluation passed"
|
||
- cat ${CI_PROJECT_DIR}/output/xlstm_eval_report.json
|
||
artifacts:
|
||
paths:
|
||
- output/xlstm_eval_report.json
|
||
when: always
|
||
expire_in: 30 days
|
||
|
||
train-validate-diffusion:
|
||
extends: .train-validate-base
|
||
needs:
|
||
- job: compile-training
|
||
optional: true
|
||
script:
|
||
- train_baseline_supervised
|
||
--model diffusion
|
||
--symbol ES.FUT
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output-dir ${CI_PROJECT_DIR}/output
|
||
--max-steps-per-epoch 50
|
||
- echo "=== Diffusion training complete, running evaluation ==="
|
||
- evaluate_supervised
|
||
--model diffusion
|
||
--models-dir ${CI_PROJECT_DIR}/output
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output ${CI_PROJECT_DIR}/output/diffusion_eval_report.json
|
||
--symbol ES.FUT
|
||
- echo "Diffusion validation + evaluation passed"
|
||
- cat ${CI_PROJECT_DIR}/output/diffusion_eval_report.json
|
||
artifacts:
|
||
paths:
|
||
- output/diffusion_eval_report.json
|
||
when: always
|
||
expire_in: 30 days
|
||
|
||
# --- Hyperopt: all models ---
|
||
|
||
hyperopt-dqn:
|
||
extends: .train-rl-base
|
||
needs:
|
||
- job: compile-training
|
||
optional: true
|
||
timeout: 4h
|
||
script:
|
||
- |
|
||
hyperopt_baseline_rl \
|
||
--model dqn \
|
||
--trials 20 \
|
||
--n-initial 5 \
|
||
--epochs 8 \
|
||
--parallel 0 \
|
||
--symbol ES.FUT \
|
||
--tx-cost-bps 0.1 \
|
||
--tick-size 0.25 \
|
||
--spread-ticks 1.0 \
|
||
--initial-capital 35000 \
|
||
--data-dir /mnt/training-data/futures-baseline \
|
||
--base-dir ${CI_PROJECT_DIR}/output/hyperopt \
|
||
--output ${CI_PROJECT_DIR}/output/dqn_hyperopt_results.json \
|
||
&& HYPEROPT_OK=1 || HYPEROPT_OK=0
|
||
echo "=== DQN Hyperopt exit: $HYPEROPT_OK ==="
|
||
cat ${CI_PROJECT_DIR}/output/dqn_hyperopt_results.json 2>/dev/null || echo "No hyperopt results"
|
||
echo "=== Training DQN with best params ==="
|
||
train_baseline_rl \
|
||
--model dqn \
|
||
--symbol ES.FUT \
|
||
--data-dir /mnt/training-data/futures-baseline \
|
||
--output-dir ${CI_PROJECT_DIR}/output \
|
||
--hyperopt-params ${CI_PROJECT_DIR}/output/dqn_hyperopt_results.json \
|
||
&& TRAIN_OK=1 || TRAIN_OK=0
|
||
echo "=== Training exit: $TRAIN_OK ==="
|
||
echo "=== Evaluating DQN ==="
|
||
evaluate_baseline \
|
||
--model dqn \
|
||
--models-dir ${CI_PROJECT_DIR}/output \
|
||
--data-dir /mnt/training-data/futures-baseline \
|
||
--output ${CI_PROJECT_DIR}/output/dqn_eval_report.json \
|
||
--symbol ES.FUT \
|
||
|| true
|
||
echo "=== Eval report ==="
|
||
cat ${CI_PROJECT_DIR}/output/dqn_eval_report.json 2>/dev/null || echo "No eval report"
|
||
[ "$HYPEROPT_OK" = "1" ] && [ "$TRAIN_OK" = "1" ] || exit 1
|
||
artifacts:
|
||
paths:
|
||
- output/dqn_hyperopt_results.json
|
||
- output/dqn_eval_report.json
|
||
when: always
|
||
expire_in: 90 days
|
||
|
||
hyperopt-tft:
|
||
extends: .train-validate-base
|
||
needs:
|
||
- job: compile-training
|
||
optional: true
|
||
timeout: 4h
|
||
script:
|
||
- hyperopt_baseline_supervised
|
||
--model tft
|
||
--trials 20
|
||
--n-initial 5
|
||
--epochs 15
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--base-dir ${CI_PROJECT_DIR}/output/hyperopt
|
||
--output ${CI_PROJECT_DIR}/output/tft_hyperopt_results.json
|
||
- echo "=== TFT Hyperopt complete ==="
|
||
- cat ${CI_PROJECT_DIR}/output/tft_hyperopt_results.json
|
||
- echo "=== Training TFT with best params ==="
|
||
- train_baseline_supervised
|
||
--model tft
|
||
--symbol ES.FUT
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output-dir ${CI_PROJECT_DIR}/output
|
||
--hyperopt-params ${CI_PROJECT_DIR}/output/tft_hyperopt_results.json
|
||
- echo "=== Evaluating TFT with best params ==="
|
||
- evaluate_supervised
|
||
--model tft
|
||
--models-dir ${CI_PROJECT_DIR}/output
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output ${CI_PROJECT_DIR}/output/tft_eval_report.json
|
||
--symbol ES.FUT
|
||
- cat ${CI_PROJECT_DIR}/output/tft_eval_report.json
|
||
artifacts:
|
||
paths:
|
||
- output/tft_hyperopt_results.json
|
||
- output/tft_eval_report.json
|
||
when: always
|
||
expire_in: 90 days
|
||
|
||
hyperopt-mamba2:
|
||
extends: .train-validate-base
|
||
needs:
|
||
- job: compile-training
|
||
optional: true
|
||
timeout: 4h
|
||
script:
|
||
- hyperopt_baseline_supervised
|
||
--model mamba2
|
||
--trials 20
|
||
--n-initial 5
|
||
--epochs 15
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--base-dir ${CI_PROJECT_DIR}/output/hyperopt
|
||
--output ${CI_PROJECT_DIR}/output/mamba2_hyperopt_results.json
|
||
- echo "=== Mamba2 Hyperopt complete ==="
|
||
- cat ${CI_PROJECT_DIR}/output/mamba2_hyperopt_results.json
|
||
- echo "=== Training Mamba2 with best params ==="
|
||
- train_baseline_supervised
|
||
--model mamba2
|
||
--symbol ES.FUT
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output-dir ${CI_PROJECT_DIR}/output
|
||
--hyperopt-params ${CI_PROJECT_DIR}/output/mamba2_hyperopt_results.json
|
||
- echo "=== Evaluating Mamba2 with best params ==="
|
||
- evaluate_supervised
|
||
--model mamba2
|
||
--models-dir ${CI_PROJECT_DIR}/output
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output ${CI_PROJECT_DIR}/output/mamba2_eval_report.json
|
||
--symbol ES.FUT
|
||
- cat ${CI_PROJECT_DIR}/output/mamba2_eval_report.json
|
||
artifacts:
|
||
paths:
|
||
- output/mamba2_hyperopt_results.json
|
||
- output/mamba2_eval_report.json
|
||
when: always
|
||
expire_in: 90 days
|
||
|
||
hyperopt-liquid:
|
||
extends: .train-validate-base
|
||
needs:
|
||
- job: compile-training
|
||
optional: true
|
||
timeout: 4h
|
||
script:
|
||
- hyperopt_baseline_supervised
|
||
--model liquid
|
||
--trials 20
|
||
--n-initial 5
|
||
--epochs 15
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--base-dir ${CI_PROJECT_DIR}/output/hyperopt
|
||
--output ${CI_PROJECT_DIR}/output/liquid_hyperopt_results.json
|
||
- echo "=== Liquid Hyperopt complete ==="
|
||
- cat ${CI_PROJECT_DIR}/output/liquid_hyperopt_results.json
|
||
- echo "=== Training Liquid with best params ==="
|
||
- train_baseline_supervised
|
||
--model liquid
|
||
--symbol ES.FUT
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output-dir ${CI_PROJECT_DIR}/output
|
||
--hyperopt-params ${CI_PROJECT_DIR}/output/liquid_hyperopt_results.json
|
||
- echo "=== Evaluating Liquid with best params ==="
|
||
- evaluate_supervised
|
||
--model liquid
|
||
--models-dir ${CI_PROJECT_DIR}/output
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output ${CI_PROJECT_DIR}/output/liquid_eval_report.json
|
||
--symbol ES.FUT
|
||
- cat ${CI_PROJECT_DIR}/output/liquid_eval_report.json
|
||
artifacts:
|
||
paths:
|
||
- output/liquid_hyperopt_results.json
|
||
- output/liquid_eval_report.json
|
||
when: always
|
||
expire_in: 90 days
|
||
|
||
hyperopt-tggn:
|
||
extends: .train-validate-base
|
||
needs:
|
||
- job: compile-training
|
||
optional: true
|
||
timeout: 4h
|
||
script:
|
||
- hyperopt_baseline_supervised
|
||
--model tggn
|
||
--trials 20
|
||
--n-initial 5
|
||
--epochs 15
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--base-dir ${CI_PROJECT_DIR}/output/hyperopt
|
||
--output ${CI_PROJECT_DIR}/output/tggn_hyperopt_results.json
|
||
- echo "=== TGGN Hyperopt complete ==="
|
||
- cat ${CI_PROJECT_DIR}/output/tggn_hyperopt_results.json
|
||
- echo "=== Training TGGN with best params ==="
|
||
- train_baseline_supervised
|
||
--model tggn
|
||
--symbol ES.FUT
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output-dir ${CI_PROJECT_DIR}/output
|
||
--hyperopt-params ${CI_PROJECT_DIR}/output/tggn_hyperopt_results.json
|
||
- echo "=== Evaluating TGGN with best params ==="
|
||
- evaluate_supervised
|
||
--model tggn
|
||
--models-dir ${CI_PROJECT_DIR}/output
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output ${CI_PROJECT_DIR}/output/tggn_eval_report.json
|
||
--symbol ES.FUT
|
||
- cat ${CI_PROJECT_DIR}/output/tggn_eval_report.json
|
||
artifacts:
|
||
paths:
|
||
- output/tggn_hyperopt_results.json
|
||
- output/tggn_eval_report.json
|
||
when: always
|
||
expire_in: 90 days
|
||
|
||
hyperopt-tlob:
|
||
extends: .train-validate-base
|
||
needs:
|
||
- job: compile-training
|
||
optional: true
|
||
timeout: 4h
|
||
script:
|
||
- hyperopt_baseline_supervised
|
||
--model tlob
|
||
--trials 20
|
||
--n-initial 5
|
||
--epochs 15
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--base-dir ${CI_PROJECT_DIR}/output/hyperopt
|
||
--output ${CI_PROJECT_DIR}/output/tlob_hyperopt_results.json
|
||
- echo "=== TLOB Hyperopt complete ==="
|
||
- cat ${CI_PROJECT_DIR}/output/tlob_hyperopt_results.json
|
||
- echo "=== Training TLOB with best params ==="
|
||
- train_baseline_supervised
|
||
--model tlob
|
||
--symbol ES.FUT
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output-dir ${CI_PROJECT_DIR}/output
|
||
--hyperopt-params ${CI_PROJECT_DIR}/output/tlob_hyperopt_results.json
|
||
- echo "=== Evaluating TLOB with best params ==="
|
||
- evaluate_supervised
|
||
--model tlob
|
||
--models-dir ${CI_PROJECT_DIR}/output
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output ${CI_PROJECT_DIR}/output/tlob_eval_report.json
|
||
--symbol ES.FUT
|
||
- cat ${CI_PROJECT_DIR}/output/tlob_eval_report.json
|
||
artifacts:
|
||
paths:
|
||
- output/tlob_hyperopt_results.json
|
||
- output/tlob_eval_report.json
|
||
when: always
|
||
expire_in: 90 days
|
||
|
||
hyperopt-kan:
|
||
extends: .train-validate-base
|
||
needs:
|
||
- job: compile-training
|
||
optional: true
|
||
timeout: 4h
|
||
script:
|
||
- hyperopt_baseline_supervised
|
||
--model kan
|
||
--trials 20
|
||
--n-initial 5
|
||
--epochs 15
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--base-dir ${CI_PROJECT_DIR}/output/hyperopt
|
||
--output ${CI_PROJECT_DIR}/output/kan_hyperopt_results.json
|
||
- echo "=== KAN Hyperopt complete ==="
|
||
- cat ${CI_PROJECT_DIR}/output/kan_hyperopt_results.json
|
||
- echo "=== Training KAN with best params ==="
|
||
- train_baseline_supervised
|
||
--model kan
|
||
--symbol ES.FUT
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output-dir ${CI_PROJECT_DIR}/output
|
||
--hyperopt-params ${CI_PROJECT_DIR}/output/kan_hyperopt_results.json
|
||
- echo "=== Evaluating KAN with best params ==="
|
||
- evaluate_supervised
|
||
--model kan
|
||
--models-dir ${CI_PROJECT_DIR}/output
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output ${CI_PROJECT_DIR}/output/kan_eval_report.json
|
||
--symbol ES.FUT
|
||
- cat ${CI_PROJECT_DIR}/output/kan_eval_report.json
|
||
artifacts:
|
||
paths:
|
||
- output/kan_hyperopt_results.json
|
||
- output/kan_eval_report.json
|
||
when: always
|
||
expire_in: 90 days
|
||
|
||
hyperopt-xlstm:
|
||
extends: .train-validate-base
|
||
needs:
|
||
- job: compile-training
|
||
optional: true
|
||
timeout: 4h
|
||
script:
|
||
- hyperopt_baseline_supervised
|
||
--model xlstm
|
||
--trials 20
|
||
--n-initial 5
|
||
--epochs 15
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--base-dir ${CI_PROJECT_DIR}/output/hyperopt
|
||
--output ${CI_PROJECT_DIR}/output/xlstm_hyperopt_results.json
|
||
- echo "=== xLSTM Hyperopt complete ==="
|
||
- cat ${CI_PROJECT_DIR}/output/xlstm_hyperopt_results.json
|
||
- echo "=== Training xLSTM with best params ==="
|
||
- train_baseline_supervised
|
||
--model xlstm
|
||
--symbol ES.FUT
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output-dir ${CI_PROJECT_DIR}/output
|
||
--hyperopt-params ${CI_PROJECT_DIR}/output/xlstm_hyperopt_results.json
|
||
- echo "=== Evaluating xLSTM with best params ==="
|
||
- evaluate_supervised
|
||
--model xlstm
|
||
--models-dir ${CI_PROJECT_DIR}/output
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output ${CI_PROJECT_DIR}/output/xlstm_eval_report.json
|
||
--symbol ES.FUT
|
||
- cat ${CI_PROJECT_DIR}/output/xlstm_eval_report.json
|
||
artifacts:
|
||
paths:
|
||
- output/xlstm_hyperopt_results.json
|
||
- output/xlstm_eval_report.json
|
||
when: always
|
||
expire_in: 90 days
|
||
|
||
hyperopt-diffusion:
|
||
extends: .train-validate-base
|
||
needs:
|
||
- job: compile-training
|
||
optional: true
|
||
timeout: 4h
|
||
script:
|
||
- hyperopt_baseline_supervised
|
||
--model diffusion
|
||
--trials 20
|
||
--n-initial 5
|
||
--epochs 15
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--base-dir ${CI_PROJECT_DIR}/output/hyperopt
|
||
--output ${CI_PROJECT_DIR}/output/diffusion_hyperopt_results.json
|
||
- echo "=== Diffusion Hyperopt complete ==="
|
||
- cat ${CI_PROJECT_DIR}/output/diffusion_hyperopt_results.json
|
||
- echo "=== Training Diffusion with best params ==="
|
||
- train_baseline_supervised
|
||
--model diffusion
|
||
--symbol ES.FUT
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output-dir ${CI_PROJECT_DIR}/output
|
||
--hyperopt-params ${CI_PROJECT_DIR}/output/diffusion_hyperopt_results.json
|
||
- echo "=== Evaluating Diffusion with best params ==="
|
||
- evaluate_supervised
|
||
--model diffusion
|
||
--models-dir ${CI_PROJECT_DIR}/output
|
||
--data-dir /mnt/training-data/futures-baseline
|
||
--output ${CI_PROJECT_DIR}/output/diffusion_eval_report.json
|
||
--symbol ES.FUT
|
||
- cat ${CI_PROJECT_DIR}/output/diffusion_eval_report.json
|
||
artifacts:
|
||
paths:
|
||
- output/diffusion_hyperopt_results.json
|
||
- output/diffusion_eval_report.json
|
||
when: always
|
||
expire_in: 90 days
|
||
|
||
# --------------------------------------------------------------------------
|
||
# IaC: Terragrunt plan on MR (runs on gitlab pool)
|
||
# --------------------------------------------------------------------------
|
||
infra-plan:
|
||
stage: test
|
||
image: ${INFRA_RUNNER_IMAGE}
|
||
tags:
|
||
- kapsule
|
||
- docker
|
||
needs: []
|
||
rules:
|
||
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
|
||
changes:
|
||
- infra/**
|
||
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
|
||
changes:
|
||
- infra/**
|
||
before_script:
|
||
- export SCW_ACCESS_KEY=${SCW_ACCESS_KEY}
|
||
- export SCW_SECRET_KEY=${SCW_SECRET_KEY}
|
||
- export SCW_DEFAULT_PROJECT_ID=${SCW_DEFAULT_PROJECT_ID}
|
||
- export AWS_ACCESS_KEY_ID=${SCW_ACCESS_KEY}
|
||
- export AWS_SECRET_ACCESS_KEY=${SCW_SECRET_KEY}
|
||
script:
|
||
- cd infra/live/production
|
||
- terragrunt run-all plan --terragrunt-non-interactive 2>&1 | tee ${CI_PROJECT_DIR}/plan-output.txt
|
||
- echo "Plan completed — review output, then trigger infra-apply manually"
|
||
artifacts:
|
||
paths:
|
||
- plan-output.txt
|
||
when: always
|
||
expire_in: 7 days
|
||
|
||
# --------------------------------------------------------------------------
|
||
# IaC: Terragrunt apply (auto on main push, manual from web/api)
|
||
# --------------------------------------------------------------------------
|
||
infra-apply:
|
||
stage: deploy
|
||
image: ${INFRA_RUNNER_IMAGE}
|
||
tags:
|
||
- kapsule
|
||
- docker
|
||
needs:
|
||
- job: infra-plan
|
||
optional: true
|
||
rules:
|
||
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
|
||
changes:
|
||
- infra/**
|
||
- if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
|
||
when: manual
|
||
before_script:
|
||
- export SCW_ACCESS_KEY=${SCW_ACCESS_KEY}
|
||
- export SCW_SECRET_KEY=${SCW_SECRET_KEY}
|
||
- export SCW_DEFAULT_PROJECT_ID=${SCW_DEFAULT_PROJECT_ID}
|
||
- export AWS_ACCESS_KEY_ID=${SCW_ACCESS_KEY}
|
||
- export AWS_SECRET_ACCESS_KEY=${SCW_SECRET_KEY}
|
||
script:
|
||
- cd infra/live/production
|
||
- terragrunt run-all apply --terragrunt-non-interactive -auto-approve
|
||
environment:
|
||
name: production/infrastructure
|
||
|
||
# --------------------------------------------------------------------------
|
||
# IaC: Weekly drift detection (scheduled pipeline, gitlab pool)
|
||
# --------------------------------------------------------------------------
|
||
infra-drift-check:
|
||
stage: test
|
||
image: ${INFRA_RUNNER_IMAGE}
|
||
tags:
|
||
- kapsule
|
||
- docker
|
||
needs: []
|
||
rules:
|
||
- if: $CI_PIPELINE_SOURCE == "schedule" && $DRIFT_CHECK == "true"
|
||
before_script:
|
||
- export SCW_ACCESS_KEY=${SCW_ACCESS_KEY}
|
||
- export SCW_SECRET_KEY=${SCW_SECRET_KEY}
|
||
- export SCW_DEFAULT_PROJECT_ID=${SCW_DEFAULT_PROJECT_ID}
|
||
- export AWS_ACCESS_KEY_ID=${SCW_ACCESS_KEY}
|
||
- export AWS_SECRET_ACCESS_KEY=${SCW_SECRET_KEY}
|
||
script:
|
||
- cd infra/live/production
|
||
- |
|
||
terragrunt run-all plan -detailed-exitcode --terragrunt-non-interactive \
|
||
2>&1 | tee ${CI_PROJECT_DIR}/drift-output.txt; EXIT_CODE=$?
|
||
if [ "$EXIT_CODE" -eq 2 ]; then
|
||
echo "DRIFT DETECTED — creating GitLab issue"
|
||
glab issue create \
|
||
--title "IaC Drift Detected ($(date +%Y-%m-%d))" \
|
||
--description "$(tail -100 ${CI_PROJECT_DIR}/drift-output.txt)" \
|
||
--label "infrastructure,drift"
|
||
exit 1
|
||
elif [ "$EXIT_CODE" -ne 0 ]; then
|
||
exit $EXIT_CODE
|
||
fi
|
||
echo "No drift detected"
|
||
artifacts:
|
||
paths:
|
||
- drift-output.txt
|
||
when: always
|
||
expire_in: 7 days
|
||
|
||
# --------------------------------------------------------------------------
|
||
# Stage 4: Deploy to Kapsule (main only)
|
||
# --------------------------------------------------------------------------
|
||
deploy:
|
||
stage: deploy
|
||
image: ${INFRA_RUNNER_IMAGE}
|
||
tags:
|
||
- kapsule
|
||
- docker
|
||
variables:
|
||
# Deploy runs on CPU compile pool (lightweight kubectl apply)
|
||
KUBERNETES_CPU_REQUEST: "200m"
|
||
KUBERNETES_CPU_LIMIT: "500m"
|
||
KUBERNETES_MEMORY_REQUEST: "256Mi"
|
||
KUBERNETES_MEMORY_LIMIT: "512Mi"
|
||
needs:
|
||
- job: compile-services
|
||
optional: true
|
||
artifacts: true
|
||
- job: compile-training
|
||
optional: true
|
||
artifacts: true
|
||
- job: build-web-dashboard
|
||
optional: true
|
||
artifacts: true
|
||
rules:
|
||
# Deploy when source code or infra manifests changed
|
||
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
|
||
changes:
|
||
- crates/**
|
||
- bin/**
|
||
- services/**
|
||
- Cargo.toml
|
||
- Cargo.lock
|
||
- infra/k8s/**
|
||
- .gitlab-ci.yml
|
||
- if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
|
||
before_script:
|
||
# Install kubectl if not in image yet (removed once infra-runner is rebuilt)
|
||
- which kubectl || (curl -fsSL "https://dl.k8s.io/release/v1.31.4/bin/linux/amd64/kubectl" -o /usr/local/bin/kubectl && chmod +x /usr/local/bin/kubectl)
|
||
# scw CLI needs these for kubeconfig exec plugin (get-token)
|
||
- export SCW_ACCESS_KEY=${SCW_ACCESS_KEY}
|
||
- export SCW_SECRET_KEY=${SCW_SECRET_KEY}
|
||
- export SCW_DEFAULT_PROJECT_ID=${SCW_DEFAULT_PROJECT_ID}
|
||
- export SCW_DEFAULT_ORGANIZATION_ID=${SCW_DEFAULT_ORGANIZATION_ID:-${SCW_DEFAULT_PROJECT_ID}}
|
||
- mkdir -p ~/.kube
|
||
- echo "$KUBECONFIG_B64" | base64 -d > ~/.kube/config
|
||
- chmod 600 ~/.kube/config
|
||
# Rewrite external API URL to in-cluster endpoint (pod can't reach external API due to CIDR overlap)
|
||
- sed -i 's|https://.*\.api\.k8s\.fr-par\.scw\.cloud:6443|https://kubernetes.default.svc|g' ~/.kube/config
|
||
# Verify cluster connectivity before deploying
|
||
- kubectl cluster-info
|
||
environment:
|
||
name: production
|
||
kubernetes:
|
||
namespace: foxhunt
|
||
script:
|
||
# Shared PVC binary deploy: CI copies binaries to foxhunt-binaries PVC via writer pod.
|
||
# Services mount the PVC read-only — no initContainer, no S3.
|
||
# ORDER MATTERS: write binaries → apply manifests → rollout restart
|
||
- echo "Deploying ${CI_COMMIT_SHA}..."
|
||
# Apply pending database migrations before deploying services
|
||
- |
|
||
echo "Applying database migrations..."
|
||
DB_POD=$(kubectl -n foxhunt get pod -l app.kubernetes.io/name=postgres -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
|
||
if [ -n "$DB_POD" ]; then
|
||
# Create tracking table if it doesn't exist
|
||
kubectl -n foxhunt exec "$DB_POD" -- psql -U foxhunt -d foxhunt -c \
|
||
"CREATE TABLE IF NOT EXISTS _applied_migrations (name TEXT PRIMARY KEY, applied_at TIMESTAMPTZ DEFAULT now())" 2>/dev/null || true
|
||
# Apply each migration in order, skipping already-applied ones
|
||
for f in $(ls migrations/*.sql | grep -v '\.down\.' | sort); do
|
||
MIGRATION_NAME=$(basename "$f")
|
||
ALREADY=$(kubectl -n foxhunt exec "$DB_POD" -- psql -U foxhunt -d foxhunt -tAc \
|
||
"SELECT 1 FROM _applied_migrations WHERE name = '${MIGRATION_NAME}'" 2>/dev/null || true)
|
||
if [ "$ALREADY" = "1" ]; then
|
||
echo " skip: $MIGRATION_NAME (already applied)"
|
||
else
|
||
echo " apply: $MIGRATION_NAME"
|
||
kubectl -n foxhunt exec -i "$DB_POD" -- psql -U foxhunt -d foxhunt < "$f"
|
||
kubectl -n foxhunt exec "$DB_POD" -- psql -U foxhunt -d foxhunt -c \
|
||
"INSERT INTO _applied_migrations (name) VALUES ('${MIGRATION_NAME}')"
|
||
fi
|
||
done
|
||
echo "Migrations complete"
|
||
else
|
||
echo "WARNING: postgres pod not found, skipping migrations"
|
||
fi
|
||
# Deploy monitoring stack (Loki, Tempo, Promtail, node-exporter, kube-state-metrics)
|
||
- kubectl apply -f infra/k8s/monitoring/loki.yaml -f infra/k8s/monitoring/tempo.yaml -f infra/k8s/monitoring/promtail.yaml -f infra/k8s/monitoring/node-exporter.yaml -f infra/k8s/monitoring/kube-state-metrics.yaml
|
||
# Deploy Grafana dashboards as ConfigMaps (sidecar auto-reloads)
|
||
# Resolve datasource variable placeholders to actual UIDs before applying
|
||
- |
|
||
PROM_UID="PBFA97CFB590B2093"
|
||
LOKI_UID="P8E80F9AEF21F6940"
|
||
TEMPO_UID="tempo"
|
||
DASH_DIR="infra/k8s/monitoring/dashboards"
|
||
TMP_DIR=$(mktemp -d)
|
||
for f in "$DASH_DIR"/*.json; do
|
||
python3 -c "
|
||
import json, sys
|
||
with open('$f') as fh:
|
||
d = json.load(fh)
|
||
d.pop('__inputs', None)
|
||
d.pop('__requires', None)
|
||
raw = json.dumps(d)
|
||
raw = raw.replace('\${DS_PROMETHEUS}', '${PROM_UID}')
|
||
raw = raw.replace('\${DS_LOKI}', '${LOKI_UID}')
|
||
raw = raw.replace('\${DS_TEMPO}', '${TEMPO_UID}')
|
||
print(raw)
|
||
" > "$TMP_DIR/$(basename "$f")"
|
||
done
|
||
# Build ConfigMaps grouped by category (POSIX-compatible, no declare -A)
|
||
apply_dashboard_cm() {
|
||
cm_name="$1"; folder="$2"; shift 2
|
||
args=""
|
||
for df in "$@"; do
|
||
[ -f "$TMP_DIR/$df" ] && args="$args --from-file=$df=$TMP_DIR/$df"
|
||
done
|
||
if [ -n "$args" ]; then
|
||
kubectl create configmap "$cm_name" -n foxhunt $args \
|
||
--dry-run=client -o yaml | \
|
||
kubectl label -f - --dry-run=client -o yaml --local grafana_dashboard=1 | \
|
||
kubectl annotate -f - --dry-run=client -o yaml --local "grafana_folder=$folder" | \
|
||
kubectl apply -f -
|
||
echo " Applied ConfigMap: $cm_name (folder: $folder)"
|
||
fi
|
||
}
|
||
apply_dashboard_cm grafana-dashboards-foxhunt Foxhunt \
|
||
foxhunt-logs.json foxhunt-overview.json foxhunt-services.json \
|
||
foxhunt-cockpit.json foxhunt-trading-cockpit.json foxhunt-training-cockpit.json foxhunt-traces.json
|
||
apply_dashboard_cm grafana-dashboards-infra Infrastructure \
|
||
cluster-overview.json gpu-overview.json foxhunt-gpu-training.json foxhunt-infrastructure.json
|
||
apply_dashboard_cm grafana-dashboards-cicd "CI/CD" \
|
||
foxhunt-ci-pipelines.json gitlab-services.json
|
||
rm -rf "$TMP_DIR"
|
||
echo "Dashboard ConfigMaps deployed"
|
||
# Apply proxy (picks up nginx config changes for dashboard.fxhnt.ai routing)
|
||
- kubectl apply -f infra/k8s/gitlab/tailscale-proxy.yaml
|
||
# Apply PVCs (idempotent, must exist before writer pods or services reference them)
|
||
- kubectl apply -f infra/k8s/storage/foxhunt-binaries-pvc.yaml
|
||
- kubectl apply -f infra/k8s/training/training-binaries-pvc.yaml
|
||
# ── Step 1: Copy binaries to PVCs BEFORE applying service manifests ──
|
||
# This ensures binaries exist on disk before any pod tries to exec them.
|
||
- |
|
||
if [ -d "build-out/services" ] || [ -d "build-out/training" ]; then
|
||
set -e
|
||
echo "=== Deploying binaries to PVCs ==="
|
||
# Clean up any stale writer pod from a previous failed deploy
|
||
kubectl delete pod binary-writer -n foxhunt --ignore-not-found --wait=true 2>/dev/null || true
|
||
kubectl delete pod training-binary-writer -n foxhunt --ignore-not-found --wait=true 2>/dev/null || true
|
||
# --- Service binaries → foxhunt-binaries PVC (foxhunt node) ---
|
||
if [ -d "build-out/services" ]; then
|
||
echo "--- Writing service binaries to foxhunt-binaries PVC ---"
|
||
kubectl run binary-writer -n foxhunt \
|
||
--image=busybox:1.36 \
|
||
--restart=Never \
|
||
--command \
|
||
--overrides='{
|
||
"spec": {
|
||
"nodeSelector": {"k8s.scaleway.com/pool-name": "foxhunt"},
|
||
"securityContext": {"runAsUser": 1000, "runAsGroup": 1000, "fsGroup": 1000},
|
||
"volumes": [{"name": "binaries", "persistentVolumeClaim": {"claimName": "foxhunt-binaries"}}],
|
||
"containers": [{"name": "binary-writer", "image": "busybox:1.36", "command": ["sleep", "600"],
|
||
"volumeMounts": [{"name": "binaries", "mountPath": "/binaries"}]}]
|
||
}
|
||
}' \
|
||
-- sleep 600
|
||
kubectl wait --for=condition=Ready pod/binary-writer -n foxhunt --timeout=120s
|
||
for bin in trading_service api_gateway broker_gateway_service \
|
||
ml_training_service backtesting_service \
|
||
trading_agent_service data_acquisition_service web-gateway \
|
||
monitoring_service; do
|
||
if [ -f "build-out/services/$bin" ]; then
|
||
kubectl cp "build-out/services/$bin" foxhunt/binary-writer:/binaries/$bin
|
||
kubectl exec -n foxhunt binary-writer -- chmod +x /binaries/$bin
|
||
echo " Copied $bin ($(stat -c%s "build-out/services/$bin") bytes)"
|
||
else
|
||
echo " WARNING: build-out/services/$bin not found, skipping"
|
||
fi
|
||
done
|
||
# Copy training binaries to service PVC (for ml-training-service to read)
|
||
if [ -d "build-out/training" ]; then
|
||
kubectl exec -n foxhunt binary-writer -- mkdir -p /binaries/training
|
||
for bin in train_baseline_rl train_baseline_supervised evaluate_baseline \
|
||
evaluate_supervised hyperopt_baseline_rl hyperopt_baseline_supervised \
|
||
training_uploader; do
|
||
if [ -f "build-out/training/$bin" ]; then
|
||
kubectl cp "build-out/training/$bin" foxhunt/binary-writer:/binaries/training/$bin
|
||
kubectl exec -n foxhunt binary-writer -- chmod +x /binaries/training/$bin
|
||
echo " Copied training/$bin ($(stat -c%s "build-out/training/$bin") bytes)"
|
||
fi
|
||
done
|
||
fi
|
||
# Copy web dashboard static assets
|
||
if [ -d "web-dashboard/dist" ]; then
|
||
kubectl exec -n foxhunt binary-writer -- mkdir -p /binaries/static
|
||
tar -C web-dashboard/dist -cf - . | kubectl exec -i -n foxhunt binary-writer -- tar -C /binaries/static -xf -
|
||
echo " Copied web-dashboard static assets"
|
||
fi
|
||
kubectl delete pod binary-writer -n foxhunt --wait=false
|
||
echo "Service binaries deployed to foxhunt-binaries PVC"
|
||
fi
|
||
# --- Training binaries → training-binaries PVC (ci-training node) ---
|
||
# Training jobs run on ci-training (GPU) node — separate RWO PVC needed.
|
||
# Best-effort: GPU node may be scaled to zero. CI training uses artifacts anyway.
|
||
if [ -d "build-out/training" ]; then
|
||
set +e # training PVC is non-critical — don't kill deploy if GPU node is down
|
||
echo "--- Writing training binaries to training-binaries PVC (best-effort) ---"
|
||
kubectl run training-binary-writer -n foxhunt \
|
||
--image=busybox:1.36 \
|
||
--restart=Never \
|
||
--command \
|
||
--overrides='{
|
||
"spec": {
|
||
"nodeSelector": {"k8s.scaleway.com/pool-name": "ci-training"},
|
||
"tolerations": [
|
||
{"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"},
|
||
{"key": "node.cilium.io/agent-not-ready", "operator": "Exists", "effect": "NoSchedule"}
|
||
],
|
||
"securityContext": {"runAsUser": 1000, "runAsGroup": 1000, "fsGroup": 1000},
|
||
"volumes": [{"name": "binaries", "persistentVolumeClaim": {"claimName": "training-binaries"}}],
|
||
"containers": [{"name": "training-binary-writer", "image": "busybox:1.36", "command": ["sleep", "600"],
|
||
"volumeMounts": [{"name": "binaries", "mountPath": "/binaries"}]}]
|
||
}
|
||
}' \
|
||
-- sleep 600
|
||
if kubectl wait --for=condition=Ready pod/training-binary-writer -n foxhunt --timeout=300s 2>/dev/null; then
|
||
for bin in train_baseline_rl train_baseline_supervised evaluate_baseline \
|
||
evaluate_supervised hyperopt_baseline_rl hyperopt_baseline_supervised \
|
||
training_uploader; do
|
||
if [ -f "build-out/training/$bin" ]; then
|
||
kubectl cp "build-out/training/$bin" foxhunt/training-binary-writer:/binaries/$bin
|
||
kubectl exec -n foxhunt training-binary-writer -- chmod +x /binaries/$bin
|
||
echo " Copied training/$bin ($(stat -c%s "build-out/training/$bin") bytes)"
|
||
fi
|
||
done
|
||
kubectl delete pod training-binary-writer -n foxhunt --wait=false
|
||
echo "Training binaries deployed to training-binaries PVC"
|
||
else
|
||
echo "WARNING: training-binary-writer did not become ready (GPU node may be scaled to zero)"
|
||
echo "Training binaries NOT deployed — CI training jobs use artifacts, ad-hoc jobs will fail"
|
||
kubectl delete pod training-binary-writer -n foxhunt --ignore-not-found --wait=false 2>/dev/null || true
|
||
fi
|
||
set -e # re-enable for remaining steps
|
||
fi
|
||
echo "=== Binary deployment complete ==="
|
||
else
|
||
echo "No new binaries — compile jobs did not run, PVCs have existing binaries"
|
||
fi
|
||
# ── Step 2: Apply service manifests (binaries already on PVC) ──
|
||
# GPU overlays are in gpu-overlays/ — NOT auto-applied. Apply manually when needed.
|
||
- kubectl apply -f infra/k8s/services/
|
||
# ── Step 3: Rolling restart + wait ──
|
||
- |
|
||
for svc in trading-service api-gateway broker-gateway \
|
||
ml-training-service backtesting-service \
|
||
trading-agent-service data-acquisition-service web-gateway \
|
||
monitoring-service; do
|
||
kubectl -n foxhunt rollout restart deployment/$svc
|
||
echo "Restarted $svc"
|
||
done
|
||
- |
|
||
for svc in trading-service api-gateway broker-gateway \
|
||
ml-training-service backtesting-service \
|
||
trading-agent-service data-acquisition-service web-gateway \
|
||
monitoring-service; do
|
||
kubectl -n foxhunt rollout status deployment/$svc --timeout=300s
|
||
done
|
||
- echo "All services deployed"
|