ce65e4ca42
Split GPU workloads across two pools: - ci-compile (L4-1-24G): for cargo check/test/build — cheaper, same CUDA CC 8.9 as L40S, sufficient 24GB VRAM for compilation - ci-build (L40S-1-48G): for hyperopt + training — 48GB VRAM needed for large model training batches Runner defaults to ci-compile; training jobs override to ci-build via KUBERNETES_NODE_SELECTOR. Both pools autoscale to zero when idle. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1269 lines
46 KiB
YAML
1269 lines
46 KiB
YAML
# GitLab CI/CD — Foxhunt
|
|
# Compile: ci-compile pool (L4, CUDA 8.9) | Training: ci-build pool (L40S, 48GB VRAM)
|
|
# CI builder image: CUDA 12.4 + Rust 1.89 + protoc + sccache (hosted on Scaleway CR)
|
|
# Service images: pushed to Scaleway Container Registry (Kaniko)
|
|
# Note: .gitlab-ci.yml itself is NOT in changes: filters — CI-only edits skip build
|
|
|
|
# Ensure pipeline is always created — individual job rules handle filtering.
|
|
# Without this, GitLab CE may silently drop push pipelines when changes:
|
|
# evaluation fails to match (known issue with path glob evaluation).
|
|
workflow:
|
|
rules:
|
|
- if: $CI_PIPELINE_SOURCE == "push"
|
|
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
|
|
- if: $CI_PIPELINE_SOURCE == "schedule"
|
|
- if: $CI_PIPELINE_SOURCE == "api"
|
|
- if: $CI_PIPELINE_SOURCE == "web"
|
|
|
|
stages:
|
|
- prepare
|
|
- check
|
|
- test
|
|
- compile
|
|
- build
|
|
- train
|
|
- deploy
|
|
|
|
variables:
|
|
SQLX_OFFLINE: "true"
|
|
CARGO_TERM_COLOR: always
|
|
# CUDA_COMPUTE_CAP is set at runtime by detect_gpu() — L4=89, H100=90
|
|
# Do NOT hardcode here; sccache is partitioned by compute cap below
|
|
# Stable JWT secret for api_gateway tests (prevents env-var race between parallel tests)
|
|
JWT_SECRET: "CiTestSecret_Kx7mP9nR2sW5vY8bC3fG6jH1kL4pQ7tZ0uN9dM5eV8xS2wT6yA4zB_64chars_min"
|
|
# Training data on block storage PVC (mounted at /mnt/training-data by runner)
|
|
FOXHUNT_DATA_DIR: "/mnt/training-data/futures-baseline"
|
|
# sccache base dir on PVC — GPU stages override to /mnt/sccache/sm_<cap>
|
|
SCCACHE_DIR: "/mnt/sccache"
|
|
RUSTC_WRAPPER: /usr/local/bin/sccache
|
|
# Build profile: "release" (default, fat LTO) or "dev-release" (thin LTO, fast)
|
|
# Set DEV_RELEASE=true in pipeline vars or CI/CD settings to use fast profile
|
|
CARGO_BUILD_PROFILE: release
|
|
# S3 credentials for Kaniko registry auth and Dockerfile builds
|
|
AWS_ACCESS_KEY_ID: $SCW_ACCESS_KEY
|
|
AWS_SECRET_ACCESS_KEY: $SCW_SECRET_KEY
|
|
# CI builder image on Scaleway Container Registry (reachable by Kapsule nodes)
|
|
CI_BUILDER_IMAGE: rg.fr-par.scw.cloud/foxhunt-ci/ci-builder:latest
|
|
# Scaleway Container Registry for service images
|
|
REGISTRY: rg.fr-par.scw.cloud/foxhunt-ci
|
|
# IaC runner image on Scaleway Container Registry
|
|
INFRA_RUNNER_IMAGE: rg.fr-par.scw.cloud/foxhunt-ci/infra-runner:latest
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Stage 0: Build CI builder image → push to Scaleway CR
|
|
# --------------------------------------------------------------------------
|
|
build-ci-builder:
|
|
stage: prepare
|
|
image:
|
|
name: gcr.io/kaniko-project/executor:debug
|
|
entrypoint: [""]
|
|
tags:
|
|
- kapsule
|
|
- docker
|
|
rules:
|
|
- if: $CI_PIPELINE_SOURCE == "push"
|
|
changes:
|
|
- infra/docker/Dockerfile.ci-builder
|
|
when: on_success
|
|
- when: manual
|
|
allow_failure: true
|
|
before_script:
|
|
- mkdir -p /kaniko/.docker
|
|
- >-
|
|
echo "{\"auths\":{
|
|
\"rg.fr-par.scw.cloud\":{\"username\":\"nologin\",\"password\":\"${SCW_SECRET_KEY}\"},
|
|
\"https://index.docker.io/v1/\":{\"username\":\"${DOCKERHUB_USERNAME}\",\"password\":\"${DOCKERHUB_TOKEN}\"}
|
|
}}" > /kaniko/.docker/config.json
|
|
script:
|
|
- /kaniko/executor
|
|
--context "${CI_PROJECT_DIR}"
|
|
--dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.ci-builder"
|
|
--cache=true --cache-repo="${REGISTRY}/cache"
|
|
--destination "${CI_BUILDER_IMAGE}"
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Stage 0b: Build devcontainer image → push to internal GitLab registry
|
|
# --------------------------------------------------------------------------
|
|
build-devcontainer:
|
|
stage: prepare
|
|
image:
|
|
name: gcr.io/kaniko-project/executor:debug
|
|
entrypoint: [""]
|
|
tags:
|
|
- kapsule
|
|
- docker
|
|
rules:
|
|
- if: $CI_PIPELINE_SOURCE == "push"
|
|
changes:
|
|
- .devcontainer/**
|
|
when: on_success
|
|
- when: manual
|
|
allow_failure: true
|
|
before_script:
|
|
- mkdir -p /kaniko/.docker
|
|
- >-
|
|
echo "{\"auths\":{
|
|
\"rg.fr-par.scw.cloud\":{\"username\":\"nologin\",\"password\":\"${SCW_SECRET_KEY}\"},
|
|
\"https://index.docker.io/v1/\":{\"username\":\"${DOCKERHUB_USERNAME}\",\"password\":\"${DOCKERHUB_TOKEN}\"}
|
|
}}" > /kaniko/.docker/config.json
|
|
script:
|
|
- /kaniko/executor
|
|
--context "${CI_PROJECT_DIR}"
|
|
--dockerfile "${CI_PROJECT_DIR}/.devcontainer/Dockerfile"
|
|
--cache=true --cache-repo="${REGISTRY}/cache"
|
|
--destination "rg.fr-par.scw.cloud/foxhunt-ci/devcontainer:${CI_COMMIT_SHA}"
|
|
--destination "rg.fr-par.scw.cloud/foxhunt-ci/devcontainer:latest"
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Stage 0c: Build infra-runner image → push to Scaleway CR
|
|
# --------------------------------------------------------------------------
|
|
build-infra-runner:
|
|
stage: prepare
|
|
image:
|
|
name: gcr.io/kaniko-project/executor:debug
|
|
entrypoint: [""]
|
|
tags:
|
|
- kapsule
|
|
- docker
|
|
rules:
|
|
- if: $CI_PIPELINE_SOURCE == "push"
|
|
changes:
|
|
- infra/docker/Dockerfile.infra-runner
|
|
when: on_success
|
|
- when: manual
|
|
allow_failure: true
|
|
before_script:
|
|
- mkdir -p /kaniko/.docker
|
|
- >-
|
|
echo "{\"auths\":{
|
|
\"rg.fr-par.scw.cloud\":{\"username\":\"nologin\",\"password\":\"${SCW_SECRET_KEY}\"},
|
|
\"https://index.docker.io/v1/\":{\"username\":\"${DOCKERHUB_USERNAME}\",\"password\":\"${DOCKERHUB_TOKEN}\"}
|
|
}}" > /kaniko/.docker/config.json
|
|
script:
|
|
- /kaniko/executor
|
|
--context "${CI_PROJECT_DIR}"
|
|
--dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.infra-runner"
|
|
--cache=true --cache-repo="${REGISTRY}/cache"
|
|
--destination "${INFRA_RUNNER_IMAGE}"
|
|
|
|
# Base template for Rust jobs — pre-baked CI builder from Scaleway CR
|
|
# Image pre-exists in SCR; rebuild via build-ci-builder when Dockerfile changes
|
|
# Runs on ci-compile pool (L4: 24GB VRAM, CUDA CC 89) by default (runner config)
|
|
# Training jobs override to ci-build pool (L40S: 48GB VRAM) via node_selector
|
|
.rust-base:
|
|
image: ${CI_BUILDER_IMAGE}
|
|
tags:
|
|
- kapsule
|
|
- rust
|
|
variables:
|
|
# Give compile pods all available CPU for fast Rust compilation
|
|
KUBERNETES_CPU_REQUEST: "7000m"
|
|
KUBERNETES_CPU_LIMIT: "7800m"
|
|
KUBERNETES_MEMORY_REQUEST: "16Gi"
|
|
KUBERNETES_MEMORY_LIMIT: "80Gi"
|
|
before_script:
|
|
# Mold linker fallback — .cargo/config.toml uses -fuse-ld=mold via clang.
|
|
# clang's -fuse-ld=mold searches for "ld.mold" (not "mold") in its linker paths.
|
|
# If mold isn't installed, create ld.mold wrapper that delegates to ld.lld.
|
|
- |
|
|
if ! command -v mold &>/dev/null; then
|
|
printf '#!/bin/sh\nexec ld.lld "$@"\n' > /usr/local/bin/ld.mold
|
|
chmod +x /usr/local/bin/ld.mold
|
|
echo "mold not found, falling back to lld via ld.mold wrapper"
|
|
fi
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Stage 1: cargo check + clippy (SKIPPED — test stage already compiles)
|
|
# Uncomment the script lines below to re-enable check+clippy before tests.
|
|
# --------------------------------------------------------------------------
|
|
check:
|
|
extends: .rust-base
|
|
stage: check
|
|
needs: []
|
|
rules:
|
|
- if: $DEV_RELEASE == "true"
|
|
when: never
|
|
- if: $CI_PIPELINE_SOURCE == "push"
|
|
changes:
|
|
- Cargo.toml
|
|
- Cargo.lock
|
|
- crates/**
|
|
- bin/**
|
|
- services/**
|
|
- testing/**
|
|
- infra/docker/Dockerfile.service
|
|
- infra/docker/Dockerfile.web-gateway
|
|
- infra/docker/Dockerfile.training
|
|
- infra/docker/Dockerfile.training-runtime
|
|
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
|
|
changes:
|
|
- Cargo.toml
|
|
- Cargo.lock
|
|
- crates/**
|
|
- bin/**
|
|
- services/**
|
|
- testing/**
|
|
- if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
|
|
script:
|
|
- echo "check+clippy skipped — test stage compiles the workspace"
|
|
# - sccache --zero-stats || true
|
|
# - cargo check --workspace
|
|
# - cargo clippy --workspace -- -D warnings
|
|
# - sccache --show-stats || true
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Stage 2: tests
|
|
# --------------------------------------------------------------------------
|
|
test:
|
|
extends: .rust-base
|
|
stage: test
|
|
needs:
|
|
- job: check
|
|
optional: true # skipped in dev-release mode
|
|
rules:
|
|
- if: $CI_PIPELINE_SOURCE == "push"
|
|
changes:
|
|
- Cargo.toml
|
|
- Cargo.lock
|
|
- crates/**
|
|
- bin/**
|
|
- services/**
|
|
- testing/**
|
|
- infra/docker/Dockerfile.service
|
|
- infra/docker/Dockerfile.web-gateway
|
|
- infra/docker/Dockerfile.training
|
|
- infra/docker/Dockerfile.training-runtime
|
|
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
|
|
changes:
|
|
- Cargo.toml
|
|
- Cargo.lock
|
|
- crates/**
|
|
- bin/**
|
|
- services/**
|
|
- testing/**
|
|
- if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
|
|
services:
|
|
- name: redis:7-alpine
|
|
alias: redis
|
|
script:
|
|
# Remove CUDA stubs from LD_LIBRARY_PATH — nvidia RuntimeClass provides real libcuda.so
|
|
- export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//')
|
|
- nvidia-smi # verify GPU access
|
|
# Detect GPU compute capability and partition sccache per architecture
|
|
- export CUDA_COMPUTE_CAP=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader | head -1 | tr -d '.')
|
|
- echo "Detected CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP"
|
|
- export SCCACHE_DIR="/mnt/sccache/sm_${CUDA_COMPUTE_CAP}"
|
|
- mkdir -p "$SCCACHE_DIR"
|
|
# Wait for Redis sidecar (K8s executor doesn't wait for service readiness)
|
|
- |
|
|
for i in $(seq 1 30); do
|
|
if bash -c 'echo PING > /dev/tcp/localhost/6379' 2>/dev/null; then
|
|
echo "Redis ready"; break
|
|
fi
|
|
echo "Waiting for Redis... ($i/30)"; sleep 1
|
|
done
|
|
- sccache --zero-stats || true
|
|
- cargo test --workspace --lib -- --skip model_loader::tests
|
|
- sccache --show-stats || true
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Stage 3a: Compile all service binaries (single job, PVC sccache)
|
|
# --------------------------------------------------------------------------
|
|
compile-services:
|
|
extends: .rust-base
|
|
stage: compile
|
|
needs: [test]
|
|
rules:
|
|
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
|
|
changes:
|
|
- Cargo.toml
|
|
- Cargo.lock
|
|
- crates/**
|
|
- bin/**
|
|
- services/**
|
|
- testing/**
|
|
- infra/docker/Dockerfile.service
|
|
- infra/docker/Dockerfile.web-gateway
|
|
- infra/docker/Dockerfile.training
|
|
- infra/docker/Dockerfile.training-runtime
|
|
- if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
|
|
script:
|
|
# Detect GPU compute capability and partition sccache per architecture
|
|
- export CUDA_COMPUTE_CAP=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader | head -1 | tr -d '.')
|
|
- echo "Detected CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP"
|
|
- export SCCACHE_DIR="/mnt/sccache/sm_${CUDA_COMPUTE_CAP}"
|
|
- mkdir -p "$SCCACHE_DIR"
|
|
- sccache --zero-stats || true
|
|
# Select build profile: dev-release (fast, thin LTO) or release (full, fat LTO)
|
|
- PROFILE=${DEV_RELEASE:+dev-release}; PROFILE=${PROFILE:-release}
|
|
- echo "Building with --profile $PROFILE"
|
|
- TARGET_DIR="target/$PROFILE"
|
|
# 1) Build service binaries (no CUDA feature)
|
|
- cargo build --profile $PROFILE
|
|
-p trading_service
|
|
-p api_gateway
|
|
-p broker_gateway_service
|
|
-p ml_training_service
|
|
-p backtesting_service
|
|
-p trading_agent_service
|
|
-p data_acquisition_service
|
|
-p web-gateway
|
|
# 2) Build training binaries with CUDA (sccache already warm from step 1)
|
|
- cargo build --profile $PROFILE -p ml --features ml/cuda
|
|
--example train_baseline_rl
|
|
--example train_baseline_supervised
|
|
--example evaluate_baseline
|
|
--example evaluate_supervised
|
|
--example hyperopt_baseline_rl
|
|
--example hyperopt_baseline_supervised
|
|
# 3) Build training uploader sidecar (no CUDA needed)
|
|
- cargo build --profile $PROFILE -p training_uploader
|
|
- sccache --show-stats || true
|
|
- mkdir -p build-out
|
|
- |
|
|
for bin in trading_service api_gateway broker_gateway_service ml_training_service backtesting_service trading_agent_service data_acquisition_service web-gateway; do
|
|
cp $TARGET_DIR/$bin build-out/
|
|
strip build-out/$bin
|
|
done
|
|
for bin in train_baseline_rl train_baseline_supervised evaluate_baseline evaluate_supervised hyperopt_baseline_rl hyperopt_baseline_supervised; do
|
|
cp $TARGET_DIR/examples/$bin build-out/
|
|
strip build-out/$bin
|
|
done
|
|
cp $TARGET_DIR/training_uploader build-out/
|
|
strip build-out/training_uploader
|
|
- ls -lh build-out/
|
|
artifacts:
|
|
paths:
|
|
- build-out/
|
|
expire_in: 1 hour
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Stage 3b: Package service images (Kaniko → Scaleway CR, uses pre-built binaries)
|
|
# --------------------------------------------------------------------------
|
|
.kaniko-base:
|
|
stage: build
|
|
needs: [compile-services]
|
|
image:
|
|
name: gcr.io/kaniko-project/executor:debug
|
|
entrypoint: [""]
|
|
tags:
|
|
- kapsule
|
|
- docker
|
|
rules:
|
|
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
|
|
changes:
|
|
- Cargo.toml
|
|
- Cargo.lock
|
|
- crates/**
|
|
- bin/**
|
|
- services/**
|
|
- testing/**
|
|
- infra/docker/Dockerfile.service
|
|
- infra/docker/Dockerfile.runtime
|
|
- infra/docker/Dockerfile.web-gateway
|
|
- infra/docker/Dockerfile.web-gateway-runtime
|
|
- infra/docker/Dockerfile.training
|
|
- infra/docker/Dockerfile.training-runtime
|
|
- if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
|
|
before_script:
|
|
- mkdir -p /kaniko/.docker
|
|
- >-
|
|
echo "{\"auths\":{
|
|
\"rg.fr-par.scw.cloud\":{\"username\":\"nologin\",\"password\":\"${SCW_SECRET_KEY}\"},
|
|
\"https://index.docker.io/v1/\":{\"username\":\"${DOCKERHUB_USERNAME}\",\"password\":\"${DOCKERHUB_TOKEN}\"}
|
|
}}" > /kaniko/.docker/config.json
|
|
|
|
# 7 services: pre-built binary → Dockerfile.runtime (no Rust toolchain, ~30s each)
|
|
build-trading-service:
|
|
extends: .kaniko-base
|
|
script:
|
|
- /kaniko/executor
|
|
--context dir://${CI_PROJECT_DIR}/build-out
|
|
--dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.runtime"
|
|
--build-arg SERVICE=trading_service
|
|
--destination "${REGISTRY}/trading_service:${CI_COMMIT_SHA}"
|
|
--destination "${REGISTRY}/trading_service:latest"
|
|
|
|
build-api-gateway:
|
|
extends: .kaniko-base
|
|
script:
|
|
- /kaniko/executor
|
|
--context dir://${CI_PROJECT_DIR}/build-out
|
|
--dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.runtime"
|
|
--build-arg SERVICE=api_gateway
|
|
--destination "${REGISTRY}/api_gateway:${CI_COMMIT_SHA}"
|
|
--destination "${REGISTRY}/api_gateway:latest"
|
|
|
|
build-broker-gateway:
|
|
extends: .kaniko-base
|
|
script:
|
|
- /kaniko/executor
|
|
--context dir://${CI_PROJECT_DIR}/build-out
|
|
--dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.runtime"
|
|
--build-arg SERVICE=broker_gateway_service
|
|
--destination "${REGISTRY}/broker_gateway_service:${CI_COMMIT_SHA}"
|
|
--destination "${REGISTRY}/broker_gateway_service:latest"
|
|
|
|
build-ml-training:
|
|
extends: .kaniko-base
|
|
script:
|
|
- /kaniko/executor
|
|
--context dir://${CI_PROJECT_DIR}/build-out
|
|
--dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.runtime"
|
|
--build-arg SERVICE=ml_training_service
|
|
--destination "${REGISTRY}/ml_training_service:${CI_COMMIT_SHA}"
|
|
--destination "${REGISTRY}/ml_training_service:latest"
|
|
|
|
build-backtesting:
|
|
extends: .kaniko-base
|
|
script:
|
|
- /kaniko/executor
|
|
--context dir://${CI_PROJECT_DIR}/build-out
|
|
--dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.runtime"
|
|
--build-arg SERVICE=backtesting_service
|
|
--destination "${REGISTRY}/backtesting_service:${CI_COMMIT_SHA}"
|
|
--destination "${REGISTRY}/backtesting_service:latest"
|
|
|
|
build-trading-agent:
|
|
extends: .kaniko-base
|
|
script:
|
|
- /kaniko/executor
|
|
--context dir://${CI_PROJECT_DIR}/build-out
|
|
--dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.runtime"
|
|
--build-arg SERVICE=trading_agent_service
|
|
--destination "${REGISTRY}/trading_agent_service:${CI_COMMIT_SHA}"
|
|
--destination "${REGISTRY}/trading_agent_service:latest"
|
|
|
|
build-data-acquisition:
|
|
extends: .kaniko-base
|
|
script:
|
|
- /kaniko/executor
|
|
--context dir://${CI_PROJECT_DIR}/build-out
|
|
--dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.runtime"
|
|
--build-arg SERVICE=data_acquisition_service
|
|
--destination "${REGISTRY}/data_acquisition_service:${CI_COMMIT_SHA}"
|
|
--destination "${REGISTRY}/data_acquisition_service:latest"
|
|
|
|
# web-gateway: Node.js dashboard build + pre-built Rust binary
|
|
build-web-gateway:
|
|
extends: .kaniko-base
|
|
script:
|
|
- /kaniko/executor
|
|
--context "${CI_PROJECT_DIR}"
|
|
--dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.web-gateway-runtime"
|
|
--cache=true --cache-repo="${REGISTRY}/cache"
|
|
--destination "${REGISTRY}/web-gateway:${CI_COMMIT_SHA}"
|
|
--destination "${REGISTRY}/web-gateway:latest"
|
|
|
|
# training: pre-built CUDA binaries → CUDA runtime image (no recompilation)
|
|
build-training:
|
|
extends: .kaniko-base
|
|
script:
|
|
- /kaniko/executor
|
|
--context dir://${CI_PROJECT_DIR}/build-out
|
|
--dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.training-runtime"
|
|
--destination "${REGISTRY}/training:${CI_COMMIT_SHA}"
|
|
--destination "${REGISTRY}/training:latest"
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Stage 3c: Training validation — quick smoke test on GPU
|
|
# Runs the training binary for a few steps to verify CUDA + model correctness.
|
|
# Triggered manually or on schedule (full ensemble training uses train.sh).
|
|
# --------------------------------------------------------------------------
|
|
.train-validate-base:
|
|
stage: train
|
|
image: ${REGISTRY}/training:${CI_COMMIT_SHA}
|
|
tags:
|
|
- kapsule
|
|
- gpu
|
|
variables:
|
|
# Route training to L40S pool (48GB VRAM needed for large models)
|
|
# Runner default is ci-compile (L4); override here for training
|
|
"KUBERNETES_NODE_SELECTOR_k8s.scaleway.com/pool-name": "ci-build"
|
|
# Training is GPU-bound — needs minimal CPU. Low request allows 2 concurrent
|
|
# training jobs on same L40S node (7800m allocatable ÷ 2 = 3900m each)
|
|
KUBERNETES_CPU_REQUEST: "2000m"
|
|
KUBERNETES_CPU_LIMIT: "3500m"
|
|
KUBERNETES_MEMORY_REQUEST: "16Gi"
|
|
KUBERNETES_MEMORY_LIMIT: "40Gi"
|
|
rules:
|
|
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
|
|
when: manual
|
|
allow_failure: true
|
|
- if: $CI_PIPELINE_SOURCE == "schedule" && $TRAIN_VALIDATE == "true"
|
|
- if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
|
|
when: manual
|
|
allow_failure: true
|
|
before_script:
|
|
- export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//')
|
|
- nvidia-smi
|
|
- mkdir -p ${CI_PROJECT_DIR}/output
|
|
|
|
train-validate-rl:
|
|
extends: .train-validate-base
|
|
needs: [build-training]
|
|
script:
|
|
- /usr/local/bin/train_baseline_rl
|
|
--model both
|
|
--symbol ES.FUT
|
|
--data-dir /mnt/training-data/futures-baseline
|
|
--output-dir ${CI_PROJECT_DIR}/output
|
|
--max-steps-per-epoch 50
|
|
- echo "=== RL training complete, running evaluation ==="
|
|
- /usr/local/bin/evaluate_baseline
|
|
--model both
|
|
--models-dir ${CI_PROJECT_DIR}/output
|
|
--data-dir /mnt/training-data/futures-baseline
|
|
--output ${CI_PROJECT_DIR}/output/rl_eval_report.json
|
|
--symbol ES.FUT
|
|
- echo "RL validation + evaluation passed"
|
|
- cat ${CI_PROJECT_DIR}/output/rl_eval_report.json
|
|
artifacts:
|
|
paths:
|
|
- output/rl_eval_report.json
|
|
when: always
|
|
expire_in: 30 days
|
|
|
|
train-validate-tft:
|
|
extends: .train-validate-base
|
|
needs: [build-training]
|
|
script:
|
|
- /usr/local/bin/train_baseline_supervised
|
|
--model tft
|
|
--symbol ES.FUT
|
|
--data-dir /mnt/training-data/futures-baseline
|
|
--output-dir ${CI_PROJECT_DIR}/output
|
|
--max-steps-per-epoch 50
|
|
- echo "=== TFT training complete, running evaluation ==="
|
|
- /usr/local/bin/evaluate_supervised
|
|
--model tft
|
|
--models-dir ${CI_PROJECT_DIR}/output
|
|
--data-dir /mnt/training-data/futures-baseline
|
|
--output ${CI_PROJECT_DIR}/output/tft_eval_report.json
|
|
--symbol ES.FUT
|
|
- echo "TFT validation + evaluation passed"
|
|
- cat ${CI_PROJECT_DIR}/output/tft_eval_report.json
|
|
artifacts:
|
|
paths:
|
|
- output/tft_eval_report.json
|
|
when: always
|
|
expire_in: 30 days
|
|
|
|
hyperopt-ppo:
|
|
extends: .train-validate-base
|
|
needs: [build-training]
|
|
timeout: 4h
|
|
script:
|
|
- /usr/local/bin/hyperopt_baseline_rl
|
|
--model ppo
|
|
--trials 20
|
|
--n-initial 5
|
|
--epochs 15
|
|
--symbol ES.FUT
|
|
--data-dir /mnt/training-data/futures-baseline
|
|
--base-dir ${CI_PROJECT_DIR}/output/hyperopt
|
|
--output ${CI_PROJECT_DIR}/output/ppo_hyperopt_results.json
|
|
- echo "=== PPO Hyperopt complete ==="
|
|
- cat ${CI_PROJECT_DIR}/output/ppo_hyperopt_results.json
|
|
- echo "=== Training PPO with best params ==="
|
|
- /usr/local/bin/train_baseline_rl
|
|
--model ppo
|
|
--symbol ES.FUT
|
|
--data-dir /mnt/training-data/futures-baseline
|
|
--output-dir ${CI_PROJECT_DIR}/output
|
|
--hyperopt-params ${CI_PROJECT_DIR}/output/ppo_hyperopt_results.json
|
|
- echo "=== Evaluating PPO with best params ==="
|
|
- /usr/local/bin/evaluate_baseline
|
|
--model ppo
|
|
--models-dir ${CI_PROJECT_DIR}/output
|
|
--data-dir /mnt/training-data/futures-baseline
|
|
--output ${CI_PROJECT_DIR}/output/ppo_eval_report.json
|
|
--symbol ES.FUT
|
|
- cat ${CI_PROJECT_DIR}/output/ppo_eval_report.json
|
|
artifacts:
|
|
paths:
|
|
- output/ppo_hyperopt_results.json
|
|
- output/ppo_eval_report.json
|
|
when: always
|
|
expire_in: 90 days
|
|
|
|
# --- Train-validate: remaining supervised models ---
|
|
|
|
train-validate-mamba2:
|
|
extends: .train-validate-base
|
|
needs: [build-training]
|
|
script:
|
|
- /usr/local/bin/train_baseline_supervised
|
|
--model mamba2
|
|
--symbol ES.FUT
|
|
--data-dir /mnt/training-data/futures-baseline
|
|
--output-dir ${CI_PROJECT_DIR}/output
|
|
--max-steps-per-epoch 50
|
|
- echo "=== Mamba2 training complete, running evaluation ==="
|
|
- /usr/local/bin/evaluate_supervised
|
|
--model mamba2
|
|
--models-dir ${CI_PROJECT_DIR}/output
|
|
--data-dir /mnt/training-data/futures-baseline
|
|
--output ${CI_PROJECT_DIR}/output/mamba2_eval_report.json
|
|
--symbol ES.FUT
|
|
- echo "Mamba2 validation + evaluation passed"
|
|
- cat ${CI_PROJECT_DIR}/output/mamba2_eval_report.json
|
|
artifacts:
|
|
paths:
|
|
- output/mamba2_eval_report.json
|
|
when: always
|
|
expire_in: 30 days
|
|
|
|
train-validate-liquid:
|
|
extends: .train-validate-base
|
|
needs: [build-training]
|
|
script:
|
|
- /usr/local/bin/train_baseline_supervised
|
|
--model liquid
|
|
--symbol ES.FUT
|
|
--data-dir /mnt/training-data/futures-baseline
|
|
--output-dir ${CI_PROJECT_DIR}/output
|
|
--max-steps-per-epoch 50
|
|
- echo "=== Liquid training complete, running evaluation ==="
|
|
- /usr/local/bin/evaluate_supervised
|
|
--model liquid
|
|
--models-dir ${CI_PROJECT_DIR}/output
|
|
--data-dir /mnt/training-data/futures-baseline
|
|
--output ${CI_PROJECT_DIR}/output/liquid_eval_report.json
|
|
--symbol ES.FUT
|
|
- echo "Liquid validation + evaluation passed"
|
|
- cat ${CI_PROJECT_DIR}/output/liquid_eval_report.json
|
|
artifacts:
|
|
paths:
|
|
- output/liquid_eval_report.json
|
|
when: always
|
|
expire_in: 30 days
|
|
|
|
train-validate-tggn:
|
|
extends: .train-validate-base
|
|
needs: [build-training]
|
|
script:
|
|
- /usr/local/bin/train_baseline_supervised
|
|
--model tggn
|
|
--symbol ES.FUT
|
|
--data-dir /mnt/training-data/futures-baseline
|
|
--output-dir ${CI_PROJECT_DIR}/output
|
|
--max-steps-per-epoch 50
|
|
- echo "=== TGGN training complete, running evaluation ==="
|
|
- /usr/local/bin/evaluate_supervised
|
|
--model tggn
|
|
--models-dir ${CI_PROJECT_DIR}/output
|
|
--data-dir /mnt/training-data/futures-baseline
|
|
--output ${CI_PROJECT_DIR}/output/tggn_eval_report.json
|
|
--symbol ES.FUT
|
|
- echo "TGGN validation + evaluation passed"
|
|
- cat ${CI_PROJECT_DIR}/output/tggn_eval_report.json
|
|
artifacts:
|
|
paths:
|
|
- output/tggn_eval_report.json
|
|
when: always
|
|
expire_in: 30 days
|
|
|
|
train-validate-tlob:
|
|
extends: .train-validate-base
|
|
needs: [build-training]
|
|
script:
|
|
- /usr/local/bin/train_baseline_supervised
|
|
--model tlob
|
|
--symbol ES.FUT
|
|
--data-dir /mnt/training-data/futures-baseline
|
|
--output-dir ${CI_PROJECT_DIR}/output
|
|
--max-steps-per-epoch 50
|
|
- echo "=== TLOB training complete, running evaluation ==="
|
|
- /usr/local/bin/evaluate_supervised
|
|
--model tlob
|
|
--models-dir ${CI_PROJECT_DIR}/output
|
|
--data-dir /mnt/training-data/futures-baseline
|
|
--output ${CI_PROJECT_DIR}/output/tlob_eval_report.json
|
|
--symbol ES.FUT
|
|
- echo "TLOB validation + evaluation passed"
|
|
- cat ${CI_PROJECT_DIR}/output/tlob_eval_report.json
|
|
artifacts:
|
|
paths:
|
|
- output/tlob_eval_report.json
|
|
when: always
|
|
expire_in: 30 days
|
|
|
|
train-validate-kan:
|
|
extends: .train-validate-base
|
|
needs: [build-training]
|
|
script:
|
|
- /usr/local/bin/train_baseline_supervised
|
|
--model kan
|
|
--symbol ES.FUT
|
|
--data-dir /mnt/training-data/futures-baseline
|
|
--output-dir ${CI_PROJECT_DIR}/output
|
|
--max-steps-per-epoch 50
|
|
- echo "=== KAN training complete, running evaluation ==="
|
|
- /usr/local/bin/evaluate_supervised
|
|
--model kan
|
|
--models-dir ${CI_PROJECT_DIR}/output
|
|
--data-dir /mnt/training-data/futures-baseline
|
|
--output ${CI_PROJECT_DIR}/output/kan_eval_report.json
|
|
--symbol ES.FUT
|
|
- echo "KAN validation + evaluation passed"
|
|
- cat ${CI_PROJECT_DIR}/output/kan_eval_report.json
|
|
artifacts:
|
|
paths:
|
|
- output/kan_eval_report.json
|
|
when: always
|
|
expire_in: 30 days
|
|
|
|
train-validate-xlstm:
|
|
extends: .train-validate-base
|
|
needs: [build-training]
|
|
script:
|
|
- /usr/local/bin/train_baseline_supervised
|
|
--model xlstm
|
|
--symbol ES.FUT
|
|
--data-dir /mnt/training-data/futures-baseline
|
|
--output-dir ${CI_PROJECT_DIR}/output
|
|
--max-steps-per-epoch 50
|
|
- echo "=== xLSTM training complete, running evaluation ==="
|
|
- /usr/local/bin/evaluate_supervised
|
|
--model xlstm
|
|
--models-dir ${CI_PROJECT_DIR}/output
|
|
--data-dir /mnt/training-data/futures-baseline
|
|
--output ${CI_PROJECT_DIR}/output/xlstm_eval_report.json
|
|
--symbol ES.FUT
|
|
- echo "xLSTM validation + evaluation passed"
|
|
- cat ${CI_PROJECT_DIR}/output/xlstm_eval_report.json
|
|
artifacts:
|
|
paths:
|
|
- output/xlstm_eval_report.json
|
|
when: always
|
|
expire_in: 30 days
|
|
|
|
train-validate-diffusion:
|
|
extends: .train-validate-base
|
|
needs: [build-training]
|
|
script:
|
|
- /usr/local/bin/train_baseline_supervised
|
|
--model diffusion
|
|
--symbol ES.FUT
|
|
--data-dir /mnt/training-data/futures-baseline
|
|
--output-dir ${CI_PROJECT_DIR}/output
|
|
--max-steps-per-epoch 50
|
|
- echo "=== Diffusion training complete, running evaluation ==="
|
|
- /usr/local/bin/evaluate_supervised
|
|
--model diffusion
|
|
--models-dir ${CI_PROJECT_DIR}/output
|
|
--data-dir /mnt/training-data/futures-baseline
|
|
--output ${CI_PROJECT_DIR}/output/diffusion_eval_report.json
|
|
--symbol ES.FUT
|
|
- echo "Diffusion validation + evaluation passed"
|
|
- cat ${CI_PROJECT_DIR}/output/diffusion_eval_report.json
|
|
artifacts:
|
|
paths:
|
|
- output/diffusion_eval_report.json
|
|
when: always
|
|
expire_in: 30 days
|
|
|
|
# --- Hyperopt: all models ---
|
|
|
|
hyperopt-dqn:
|
|
extends: .train-validate-base
|
|
needs: [build-training]
|
|
timeout: 4h
|
|
script:
|
|
- /usr/local/bin/hyperopt_baseline_rl
|
|
--model dqn
|
|
--trials 20
|
|
--n-initial 5
|
|
--epochs 15
|
|
--symbol ES.FUT
|
|
--data-dir /mnt/training-data/futures-baseline
|
|
--base-dir ${CI_PROJECT_DIR}/output/hyperopt
|
|
--output ${CI_PROJECT_DIR}/output/dqn_hyperopt_results.json
|
|
- echo "=== DQN Hyperopt complete ==="
|
|
- cat ${CI_PROJECT_DIR}/output/dqn_hyperopt_results.json
|
|
- echo "=== Training DQN with best params ==="
|
|
- /usr/local/bin/train_baseline_rl
|
|
--model dqn
|
|
--symbol ES.FUT
|
|
--data-dir /mnt/training-data/futures-baseline
|
|
--output-dir ${CI_PROJECT_DIR}/output
|
|
--hyperopt-params ${CI_PROJECT_DIR}/output/dqn_hyperopt_results.json
|
|
- echo "=== Evaluating DQN with best params ==="
|
|
- /usr/local/bin/evaluate_baseline
|
|
--model dqn
|
|
--models-dir ${CI_PROJECT_DIR}/output
|
|
--data-dir /mnt/training-data/futures-baseline
|
|
--output ${CI_PROJECT_DIR}/output/dqn_eval_report.json
|
|
--symbol ES.FUT
|
|
- cat ${CI_PROJECT_DIR}/output/dqn_eval_report.json
|
|
artifacts:
|
|
paths:
|
|
- output/dqn_hyperopt_results.json
|
|
- output/dqn_eval_report.json
|
|
when: always
|
|
expire_in: 90 days
|
|
|
|
hyperopt-tft:
|
|
extends: .train-validate-base
|
|
needs: [build-training]
|
|
timeout: 4h
|
|
script:
|
|
- /usr/local/bin/hyperopt_baseline_supervised
|
|
--model tft
|
|
--trials 20
|
|
--n-initial 5
|
|
--epochs 15
|
|
--data-dir /mnt/training-data/futures-baseline
|
|
--base-dir ${CI_PROJECT_DIR}/output/hyperopt
|
|
--output ${CI_PROJECT_DIR}/output/tft_hyperopt_results.json
|
|
- echo "=== TFT Hyperopt complete ==="
|
|
- cat ${CI_PROJECT_DIR}/output/tft_hyperopt_results.json
|
|
- echo "=== Training TFT with best params ==="
|
|
- /usr/local/bin/train_baseline_supervised
|
|
--model tft
|
|
--symbol ES.FUT
|
|
--data-dir /mnt/training-data/futures-baseline
|
|
--output-dir ${CI_PROJECT_DIR}/output
|
|
--hyperopt-params ${CI_PROJECT_DIR}/output/tft_hyperopt_results.json
|
|
- echo "=== Evaluating TFT with best params ==="
|
|
- /usr/local/bin/evaluate_supervised
|
|
--model tft
|
|
--models-dir ${CI_PROJECT_DIR}/output
|
|
--data-dir /mnt/training-data/futures-baseline
|
|
--output ${CI_PROJECT_DIR}/output/tft_eval_report.json
|
|
--symbol ES.FUT
|
|
- cat ${CI_PROJECT_DIR}/output/tft_eval_report.json
|
|
artifacts:
|
|
paths:
|
|
- output/tft_hyperopt_results.json
|
|
- output/tft_eval_report.json
|
|
when: always
|
|
expire_in: 90 days
|
|
|
|
hyperopt-mamba2:
|
|
extends: .train-validate-base
|
|
needs: [build-training]
|
|
timeout: 4h
|
|
script:
|
|
- /usr/local/bin/hyperopt_baseline_supervised
|
|
--model mamba2
|
|
--trials 20
|
|
--n-initial 5
|
|
--epochs 15
|
|
--data-dir /mnt/training-data/futures-baseline
|
|
--base-dir ${CI_PROJECT_DIR}/output/hyperopt
|
|
--output ${CI_PROJECT_DIR}/output/mamba2_hyperopt_results.json
|
|
- echo "=== Mamba2 Hyperopt complete ==="
|
|
- cat ${CI_PROJECT_DIR}/output/mamba2_hyperopt_results.json
|
|
- echo "=== Training Mamba2 with best params ==="
|
|
- /usr/local/bin/train_baseline_supervised
|
|
--model mamba2
|
|
--symbol ES.FUT
|
|
--data-dir /mnt/training-data/futures-baseline
|
|
--output-dir ${CI_PROJECT_DIR}/output
|
|
--hyperopt-params ${CI_PROJECT_DIR}/output/mamba2_hyperopt_results.json
|
|
- echo "=== Evaluating Mamba2 with best params ==="
|
|
- /usr/local/bin/evaluate_supervised
|
|
--model mamba2
|
|
--models-dir ${CI_PROJECT_DIR}/output
|
|
--data-dir /mnt/training-data/futures-baseline
|
|
--output ${CI_PROJECT_DIR}/output/mamba2_eval_report.json
|
|
--symbol ES.FUT
|
|
- cat ${CI_PROJECT_DIR}/output/mamba2_eval_report.json
|
|
artifacts:
|
|
paths:
|
|
- output/mamba2_hyperopt_results.json
|
|
- output/mamba2_eval_report.json
|
|
when: always
|
|
expire_in: 90 days
|
|
|
|
hyperopt-liquid:
|
|
extends: .train-validate-base
|
|
needs: [build-training]
|
|
timeout: 4h
|
|
script:
|
|
- /usr/local/bin/hyperopt_baseline_supervised
|
|
--model liquid
|
|
--trials 20
|
|
--n-initial 5
|
|
--epochs 15
|
|
--data-dir /mnt/training-data/futures-baseline
|
|
--base-dir ${CI_PROJECT_DIR}/output/hyperopt
|
|
--output ${CI_PROJECT_DIR}/output/liquid_hyperopt_results.json
|
|
- echo "=== Liquid Hyperopt complete ==="
|
|
- cat ${CI_PROJECT_DIR}/output/liquid_hyperopt_results.json
|
|
- echo "=== Training Liquid with best params ==="
|
|
- /usr/local/bin/train_baseline_supervised
|
|
--model liquid
|
|
--symbol ES.FUT
|
|
--data-dir /mnt/training-data/futures-baseline
|
|
--output-dir ${CI_PROJECT_DIR}/output
|
|
--hyperopt-params ${CI_PROJECT_DIR}/output/liquid_hyperopt_results.json
|
|
- echo "=== Evaluating Liquid with best params ==="
|
|
- /usr/local/bin/evaluate_supervised
|
|
--model liquid
|
|
--models-dir ${CI_PROJECT_DIR}/output
|
|
--data-dir /mnt/training-data/futures-baseline
|
|
--output ${CI_PROJECT_DIR}/output/liquid_eval_report.json
|
|
--symbol ES.FUT
|
|
- cat ${CI_PROJECT_DIR}/output/liquid_eval_report.json
|
|
artifacts:
|
|
paths:
|
|
- output/liquid_hyperopt_results.json
|
|
- output/liquid_eval_report.json
|
|
when: always
|
|
expire_in: 90 days
|
|
|
|
hyperopt-tggn:
|
|
extends: .train-validate-base
|
|
needs: [build-training]
|
|
timeout: 4h
|
|
script:
|
|
- /usr/local/bin/hyperopt_baseline_supervised
|
|
--model tggn
|
|
--trials 20
|
|
--n-initial 5
|
|
--epochs 15
|
|
--data-dir /mnt/training-data/futures-baseline
|
|
--base-dir ${CI_PROJECT_DIR}/output/hyperopt
|
|
--output ${CI_PROJECT_DIR}/output/tggn_hyperopt_results.json
|
|
- echo "=== TGGN Hyperopt complete ==="
|
|
- cat ${CI_PROJECT_DIR}/output/tggn_hyperopt_results.json
|
|
- echo "=== Training TGGN with best params ==="
|
|
- /usr/local/bin/train_baseline_supervised
|
|
--model tggn
|
|
--symbol ES.FUT
|
|
--data-dir /mnt/training-data/futures-baseline
|
|
--output-dir ${CI_PROJECT_DIR}/output
|
|
--hyperopt-params ${CI_PROJECT_DIR}/output/tggn_hyperopt_results.json
|
|
- echo "=== Evaluating TGGN with best params ==="
|
|
- /usr/local/bin/evaluate_supervised
|
|
--model tggn
|
|
--models-dir ${CI_PROJECT_DIR}/output
|
|
--data-dir /mnt/training-data/futures-baseline
|
|
--output ${CI_PROJECT_DIR}/output/tggn_eval_report.json
|
|
--symbol ES.FUT
|
|
- cat ${CI_PROJECT_DIR}/output/tggn_eval_report.json
|
|
artifacts:
|
|
paths:
|
|
- output/tggn_hyperopt_results.json
|
|
- output/tggn_eval_report.json
|
|
when: always
|
|
expire_in: 90 days
|
|
|
|
hyperopt-tlob:
|
|
extends: .train-validate-base
|
|
needs: [build-training]
|
|
timeout: 4h
|
|
script:
|
|
- /usr/local/bin/hyperopt_baseline_supervised
|
|
--model tlob
|
|
--trials 20
|
|
--n-initial 5
|
|
--epochs 15
|
|
--data-dir /mnt/training-data/futures-baseline
|
|
--base-dir ${CI_PROJECT_DIR}/output/hyperopt
|
|
--output ${CI_PROJECT_DIR}/output/tlob_hyperopt_results.json
|
|
- echo "=== TLOB Hyperopt complete ==="
|
|
- cat ${CI_PROJECT_DIR}/output/tlob_hyperopt_results.json
|
|
- echo "=== Training TLOB with best params ==="
|
|
- /usr/local/bin/train_baseline_supervised
|
|
--model tlob
|
|
--symbol ES.FUT
|
|
--data-dir /mnt/training-data/futures-baseline
|
|
--output-dir ${CI_PROJECT_DIR}/output
|
|
--hyperopt-params ${CI_PROJECT_DIR}/output/tlob_hyperopt_results.json
|
|
- echo "=== Evaluating TLOB with best params ==="
|
|
- /usr/local/bin/evaluate_supervised
|
|
--model tlob
|
|
--models-dir ${CI_PROJECT_DIR}/output
|
|
--data-dir /mnt/training-data/futures-baseline
|
|
--output ${CI_PROJECT_DIR}/output/tlob_eval_report.json
|
|
--symbol ES.FUT
|
|
- cat ${CI_PROJECT_DIR}/output/tlob_eval_report.json
|
|
artifacts:
|
|
paths:
|
|
- output/tlob_hyperopt_results.json
|
|
- output/tlob_eval_report.json
|
|
when: always
|
|
expire_in: 90 days
|
|
|
|
hyperopt-kan:
|
|
extends: .train-validate-base
|
|
needs: [build-training]
|
|
timeout: 4h
|
|
script:
|
|
- /usr/local/bin/hyperopt_baseline_supervised
|
|
--model kan
|
|
--trials 20
|
|
--n-initial 5
|
|
--epochs 15
|
|
--data-dir /mnt/training-data/futures-baseline
|
|
--base-dir ${CI_PROJECT_DIR}/output/hyperopt
|
|
--output ${CI_PROJECT_DIR}/output/kan_hyperopt_results.json
|
|
- echo "=== KAN Hyperopt complete ==="
|
|
- cat ${CI_PROJECT_DIR}/output/kan_hyperopt_results.json
|
|
- echo "=== Training KAN with best params ==="
|
|
- /usr/local/bin/train_baseline_supervised
|
|
--model kan
|
|
--symbol ES.FUT
|
|
--data-dir /mnt/training-data/futures-baseline
|
|
--output-dir ${CI_PROJECT_DIR}/output
|
|
--hyperopt-params ${CI_PROJECT_DIR}/output/kan_hyperopt_results.json
|
|
- echo "=== Evaluating KAN with best params ==="
|
|
- /usr/local/bin/evaluate_supervised
|
|
--model kan
|
|
--models-dir ${CI_PROJECT_DIR}/output
|
|
--data-dir /mnt/training-data/futures-baseline
|
|
--output ${CI_PROJECT_DIR}/output/kan_eval_report.json
|
|
--symbol ES.FUT
|
|
- cat ${CI_PROJECT_DIR}/output/kan_eval_report.json
|
|
artifacts:
|
|
paths:
|
|
- output/kan_hyperopt_results.json
|
|
- output/kan_eval_report.json
|
|
when: always
|
|
expire_in: 90 days
|
|
|
|
hyperopt-xlstm:
|
|
extends: .train-validate-base
|
|
needs: [build-training]
|
|
timeout: 4h
|
|
script:
|
|
- /usr/local/bin/hyperopt_baseline_supervised
|
|
--model xlstm
|
|
--trials 20
|
|
--n-initial 5
|
|
--epochs 15
|
|
--data-dir /mnt/training-data/futures-baseline
|
|
--base-dir ${CI_PROJECT_DIR}/output/hyperopt
|
|
--output ${CI_PROJECT_DIR}/output/xlstm_hyperopt_results.json
|
|
- echo "=== xLSTM Hyperopt complete ==="
|
|
- cat ${CI_PROJECT_DIR}/output/xlstm_hyperopt_results.json
|
|
- echo "=== Training xLSTM with best params ==="
|
|
- /usr/local/bin/train_baseline_supervised
|
|
--model xlstm
|
|
--symbol ES.FUT
|
|
--data-dir /mnt/training-data/futures-baseline
|
|
--output-dir ${CI_PROJECT_DIR}/output
|
|
--hyperopt-params ${CI_PROJECT_DIR}/output/xlstm_hyperopt_results.json
|
|
- echo "=== Evaluating xLSTM with best params ==="
|
|
- /usr/local/bin/evaluate_supervised
|
|
--model xlstm
|
|
--models-dir ${CI_PROJECT_DIR}/output
|
|
--data-dir /mnt/training-data/futures-baseline
|
|
--output ${CI_PROJECT_DIR}/output/xlstm_eval_report.json
|
|
--symbol ES.FUT
|
|
- cat ${CI_PROJECT_DIR}/output/xlstm_eval_report.json
|
|
artifacts:
|
|
paths:
|
|
- output/xlstm_hyperopt_results.json
|
|
- output/xlstm_eval_report.json
|
|
when: always
|
|
expire_in: 90 days
|
|
|
|
hyperopt-diffusion:
|
|
extends: .train-validate-base
|
|
needs: [build-training]
|
|
timeout: 4h
|
|
script:
|
|
- /usr/local/bin/hyperopt_baseline_supervised
|
|
--model diffusion
|
|
--trials 20
|
|
--n-initial 5
|
|
--epochs 15
|
|
--data-dir /mnt/training-data/futures-baseline
|
|
--base-dir ${CI_PROJECT_DIR}/output/hyperopt
|
|
--output ${CI_PROJECT_DIR}/output/diffusion_hyperopt_results.json
|
|
- echo "=== Diffusion Hyperopt complete ==="
|
|
- cat ${CI_PROJECT_DIR}/output/diffusion_hyperopt_results.json
|
|
- echo "=== Training Diffusion with best params ==="
|
|
- /usr/local/bin/train_baseline_supervised
|
|
--model diffusion
|
|
--symbol ES.FUT
|
|
--data-dir /mnt/training-data/futures-baseline
|
|
--output-dir ${CI_PROJECT_DIR}/output
|
|
--hyperopt-params ${CI_PROJECT_DIR}/output/diffusion_hyperopt_results.json
|
|
- echo "=== Evaluating Diffusion with best params ==="
|
|
- /usr/local/bin/evaluate_supervised
|
|
--model diffusion
|
|
--models-dir ${CI_PROJECT_DIR}/output
|
|
--data-dir /mnt/training-data/futures-baseline
|
|
--output ${CI_PROJECT_DIR}/output/diffusion_eval_report.json
|
|
--symbol ES.FUT
|
|
- cat ${CI_PROJECT_DIR}/output/diffusion_eval_report.json
|
|
artifacts:
|
|
paths:
|
|
- output/diffusion_hyperopt_results.json
|
|
- output/diffusion_eval_report.json
|
|
when: always
|
|
expire_in: 90 days
|
|
|
|
# --------------------------------------------------------------------------
|
|
# IaC: Terragrunt plan on MR (runs on gitlab pool)
|
|
# --------------------------------------------------------------------------
|
|
infra-plan:
|
|
stage: check
|
|
image: ${INFRA_RUNNER_IMAGE}
|
|
tags:
|
|
- kapsule
|
|
needs: []
|
|
rules:
|
|
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
|
|
changes:
|
|
- infra/**
|
|
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
|
|
changes:
|
|
- infra/**
|
|
before_script:
|
|
- export SCW_ACCESS_KEY=${SCW_ACCESS_KEY}
|
|
- export SCW_SECRET_KEY=${SCW_SECRET_KEY}
|
|
- export SCW_DEFAULT_PROJECT_ID=${SCW_DEFAULT_PROJECT_ID}
|
|
- export AWS_ACCESS_KEY_ID=${SCW_ACCESS_KEY}
|
|
- export AWS_SECRET_ACCESS_KEY=${SCW_SECRET_KEY}
|
|
script:
|
|
- cd infra/live/production
|
|
- terragrunt run-all plan --terragrunt-non-interactive 2>&1 | tee ${CI_PROJECT_DIR}/plan-output.txt
|
|
- echo "Plan completed — review output, then trigger infra-apply manually"
|
|
artifacts:
|
|
paths:
|
|
- plan-output.txt
|
|
when: always
|
|
expire_in: 7 days
|
|
|
|
# --------------------------------------------------------------------------
|
|
# IaC: Terragrunt apply (manual trigger after reviewing plan)
|
|
# --------------------------------------------------------------------------
|
|
infra-apply:
|
|
stage: deploy
|
|
image: ${INFRA_RUNNER_IMAGE}
|
|
tags:
|
|
- kapsule
|
|
needs: [infra-plan]
|
|
rules:
|
|
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
|
|
changes:
|
|
- infra/**
|
|
when: manual
|
|
before_script:
|
|
- export SCW_ACCESS_KEY=${SCW_ACCESS_KEY}
|
|
- export SCW_SECRET_KEY=${SCW_SECRET_KEY}
|
|
- export SCW_DEFAULT_PROJECT_ID=${SCW_DEFAULT_PROJECT_ID}
|
|
- export AWS_ACCESS_KEY_ID=${SCW_ACCESS_KEY}
|
|
- export AWS_SECRET_ACCESS_KEY=${SCW_SECRET_KEY}
|
|
script:
|
|
- cd infra/live/production
|
|
- terragrunt run-all apply --terragrunt-non-interactive -auto-approve
|
|
environment:
|
|
name: production/infrastructure
|
|
|
|
# --------------------------------------------------------------------------
|
|
# IaC: Weekly drift detection (scheduled pipeline, gitlab pool)
|
|
# --------------------------------------------------------------------------
|
|
infra-drift-check:
|
|
stage: check
|
|
image: ${INFRA_RUNNER_IMAGE}
|
|
tags:
|
|
- kapsule
|
|
needs: []
|
|
rules:
|
|
- if: $CI_PIPELINE_SOURCE == "schedule" && $DRIFT_CHECK == "true"
|
|
before_script:
|
|
- export SCW_ACCESS_KEY=${SCW_ACCESS_KEY}
|
|
- export SCW_SECRET_KEY=${SCW_SECRET_KEY}
|
|
- export SCW_DEFAULT_PROJECT_ID=${SCW_DEFAULT_PROJECT_ID}
|
|
- export AWS_ACCESS_KEY_ID=${SCW_ACCESS_KEY}
|
|
- export AWS_SECRET_ACCESS_KEY=${SCW_SECRET_KEY}
|
|
script:
|
|
- cd infra/live/production
|
|
- |
|
|
terragrunt run-all plan -detailed-exitcode --terragrunt-non-interactive \
|
|
2>&1 | tee ${CI_PROJECT_DIR}/drift-output.txt; EXIT_CODE=$?
|
|
if [ "$EXIT_CODE" -eq 2 ]; then
|
|
echo "DRIFT DETECTED — creating GitLab issue"
|
|
glab issue create \
|
|
--title "IaC Drift Detected ($(date +%Y-%m-%d))" \
|
|
--description "$(tail -100 ${CI_PROJECT_DIR}/drift-output.txt)" \
|
|
--label "infrastructure,drift"
|
|
exit 1
|
|
elif [ "$EXIT_CODE" -ne 0 ]; then
|
|
exit $EXIT_CODE
|
|
fi
|
|
echo "No drift detected"
|
|
artifacts:
|
|
paths:
|
|
- drift-output.txt
|
|
when: always
|
|
expire_in: 7 days
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Stage 4: Deploy to Kapsule (main only)
|
|
# --------------------------------------------------------------------------
|
|
deploy:
|
|
stage: deploy
|
|
image: ${INFRA_RUNNER_IMAGE}
|
|
tags:
|
|
- kapsule
|
|
needs:
|
|
- build-trading-service
|
|
- build-api-gateway
|
|
- build-broker-gateway
|
|
- build-ml-training
|
|
- build-backtesting
|
|
- build-trading-agent
|
|
- build-data-acquisition
|
|
- build-web-gateway
|
|
- build-training
|
|
rules:
|
|
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
|
|
- if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
|
|
before_script:
|
|
# Install kubectl if not in image yet (removed once infra-runner is rebuilt)
|
|
- which kubectl || (curl -fsSL "https://dl.k8s.io/release/v1.31.4/bin/linux/amd64/kubectl" -o /usr/local/bin/kubectl && chmod +x /usr/local/bin/kubectl)
|
|
# scw CLI needs these for kubeconfig exec plugin (get-token)
|
|
- export SCW_ACCESS_KEY=${SCW_ACCESS_KEY}
|
|
- export SCW_SECRET_KEY=${SCW_SECRET_KEY}
|
|
- export SCW_DEFAULT_PROJECT_ID=${SCW_DEFAULT_PROJECT_ID}
|
|
- export SCW_DEFAULT_ORGANIZATION_ID=${SCW_DEFAULT_ORGANIZATION_ID:-${SCW_DEFAULT_PROJECT_ID}}
|
|
- mkdir -p ~/.kube
|
|
- echo "$KUBECONFIG_B64" | base64 -d > ~/.kube/config
|
|
- chmod 600 ~/.kube/config
|
|
# Rewrite external API URL to in-cluster endpoint (pod can't reach external API due to CIDR overlap)
|
|
- sed -i 's|https://.*\.api\.k8s\.fr-par\.scw\.cloud:6443|https://kubernetes.default.svc|g' ~/.kube/config
|
|
# Verify cluster connectivity before deploying
|
|
- kubectl cluster-info
|
|
environment:
|
|
name: production
|
|
kubernetes:
|
|
namespace: foxhunt
|
|
script:
|
|
# Update images to this commit's build
|
|
# Format: image_repo:deployment_name:container_name
|
|
- |
|
|
SERVICES="
|
|
trading_service:trading-service:trading-service
|
|
api_gateway:api-gateway:api-gateway
|
|
broker_gateway_service:broker-gateway:broker-gateway
|
|
ml_training_service:ml-training-service:ml-training-service
|
|
backtesting_service:backtesting-service:backtesting-service
|
|
trading_agent_service:trading-agent-service:trading-agent-service
|
|
web-gateway:web-gateway:web-gateway
|
|
"
|
|
TOTAL=0
|
|
FAILED=0
|
|
for entry in $SERVICES; do
|
|
IMAGE_REPO=$(echo "$entry" | cut -d: -f1)
|
|
DEPLOY_NAME=$(echo "$entry" | cut -d: -f2)
|
|
CONTAINER_NAME=$(echo "$entry" | cut -d: -f3)
|
|
TOTAL=$((TOTAL + 1))
|
|
echo "Deploying ${DEPLOY_NAME} (image: ${IMAGE_REPO}:${CI_COMMIT_SHA})..."
|
|
if ! kubectl -n foxhunt set image "deployment/${DEPLOY_NAME}" "${CONTAINER_NAME}=${REGISTRY}/${IMAGE_REPO}:${CI_COMMIT_SHA}"; then
|
|
echo "WARNING: failed to update ${DEPLOY_NAME} (deployment may not exist yet)"
|
|
FAILED=$((FAILED + 1))
|
|
fi
|
|
done
|
|
echo "Deploy complete: $((TOTAL - FAILED))/${TOTAL} services updated"
|
|
# Fail if ALL deployments failed (cluster probably unreachable)
|
|
[ "$FAILED" -lt "$TOTAL" ]
|