# GitLab CI/CD — Foxhunt
# Two runners: CPU (compile/build/deploy) and GPU (training)
# CI builder images:
#   ci-builder:     CUDA 12.4 + Rust 1.89 + protoc + sccache (CUDA stubs for compile)
#   ci-builder-cpu: Rust 1.89 + protoc + sccache, no CUDA (for service binaries)
# Service images: pushed to Scaleway Container Registry (Kaniko)
#
# Runner routing (by tags):
#   kapsule,rust → CPU runner → ci-compile-cpu pool (POP2-32C-128G, 32 vCPU)
#   kapsule,l40s → L40S runner → ci-training (L40S 48GB)
#   kapsule,h100 → H100 runner → ci-training-h100 (H100 80GB)
# Override nodeSelector: KUBERNETES_NODE_SELECTOR_k8s.scaleway.com/pool-name: "<name>" (k=v format)
# Pools: ci-compile-cpu (POP2-32C-128G), ci-training (L40S), ci-training-h100 (H100), services, gitlab
# All compilation (services + training) runs on ci-compile-cpu. CUDA stubs in ci-builder image.

# DISABLED 2026-03-04: Migrated to Argo Workflows (infra/k8s/argo/).
# GitLab still serves as git host; Argo Events webhook triggers pipelines.
# To re-enable: remove the "when: never" rule below and restore the original rules.
workflow:
  rules:
    - when: never
    # Original rules (disabled):
    # - if: $CI_PIPELINE_SOURCE == "push"
    # - if: $CI_PIPELINE_SOURCE == "merge_request_event"
    # - if: $CI_PIPELINE_SOURCE == "schedule"
    # - if: $CI_PIPELINE_SOURCE == "api"
    # - if: $CI_PIPELINE_SOURCE == "web"

stages:
  - prepare
  - scan
  - test
  - compile
  - train
  - deploy

variables:
  SQLX_OFFLINE: "true"
  CARGO_TERM_COLOR: always
  # CUDA_COMPUTE_CAP is set at runtime by detect_gpu() — L4=89, H100=90
  # Do NOT hardcode here; sccache is partitioned by compute cap below
  # Stable JWT secret for api tests (prevents env-var race between parallel tests)
  JWT_SECRET: "CiTestSecret_Kx7mP9nR2sW5vY8bC3fG6jH1kL4pQ7tZ0uN9dM5eV8xS2wT6yA4zB_64chars_min"
  # Training data on block storage PVC (mounted at /mnt/training-data by runner)
  FOXHUNT_DATA_DIR: "/mnt/training-data/futures-baseline"
  # sccache backed by MinIO S3 (HTTP) — shared across all nodes (no PVC needed)
  SCCACHE_BUCKET: foxhunt-sccache
  SCCACHE_ENDPOINT: http://minio.foxhunt.svc.cluster.local:9000
  SCCACHE_S3_KEY_PREFIX: cpu
  SCCACHE_S3_USE_SSL: "false"
  SCCACHE_S3_NO_CREDENTIALS: "false"
  RUSTC_WRAPPER: /usr/local/bin/sccache
  # Build profile: "dev-release" (thin LTO, fast) is default for quick iteration
  # Set DEV_RELEASE=false in pipeline vars to use full "release" (fat LTO) profile
  DEV_RELEASE: "true"
  CARGO_BUILD_PROFILE: dev-release
  # S3 credentials for sccache MinIO backend (also used by rclone binary uploads)
  AWS_ACCESS_KEY_ID: $MINIO_ACCESS_KEY
  AWS_SECRET_ACCESS_KEY: $MINIO_SECRET_KEY
  # CI builder images on internal GitLab registry (backed by MinIO)
  CI_BUILDER_IMAGE: localhost:30500/root/foxhunt/ci-builder:latest
  CI_BUILDER_CPU_IMAGE: localhost:30500/root/foxhunt/ci-builder-cpu:latest
  # Container registry for all images
  REGISTRY: localhost:30500/root/foxhunt
  # Infrastructure runner image
  INFRA_RUNNER_IMAGE: localhost:30500/root/foxhunt/infra-runner:latest
  # Generic runtime images for S3 binary share (service pods + training pods + node builder)
  RUNTIME_IMAGE: localhost:30500/root/foxhunt/foxhunt-runtime:latest
  TRAINING_RUNTIME_IMAGE: localhost:30500/root/foxhunt/foxhunt-training-runtime:latest
  # Node builder image eliminated — web dashboard uses node:22-slim + inline rclone

# --------------------------------------------------------------------------
# Stage 0: Build CI builder image → push to Scaleway CR
# --------------------------------------------------------------------------
build-ci-builder:
  stage: prepare
  image:
    name: gcr.io/kaniko-project/executor:debug
    entrypoint: [""]
  tags:
    - kapsule
    - docker
  rules:
    - if: $CI_PIPELINE_SOURCE == "push"
      changes:
        - infra/docker/Dockerfile.ci-builder
      when: on_success
    - when: manual
      allow_failure: true
  before_script:
    - mkdir -p /kaniko/.docker
    - >-
      echo "{\"auths\":{
      \"rg.fr-par.scw.cloud\":{\"username\":\"nologin\",\"password\":\"${SCW_SECRET_KEY}\"},
      \"https://index.docker.io/v1/\":{\"username\":\"${DOCKERHUB_USERNAME}\",\"password\":\"${DOCKERHUB_TOKEN}\"}
      }}" > /kaniko/.docker/config.json
  script:
    - /kaniko/executor
      --context "${CI_PROJECT_DIR}"
      --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.ci-builder"
      --cache=true --cache-repo="${REGISTRY}/cache"
      --destination "${CI_BUILDER_IMAGE}"

# --------------------------------------------------------------------------
# Stage 0a: Build CPU-only CI builder image → push to Scaleway CR
# --------------------------------------------------------------------------
build-ci-builder-cpu:
  stage: prepare
  image:
    name: gcr.io/kaniko-project/executor:debug
    entrypoint: [""]
  tags:
    - kapsule
    - docker
  rules:
    - if: $CI_PIPELINE_SOURCE == "push"
      changes:
        - infra/docker/Dockerfile.ci-builder-cpu
      when: on_success
    - when: manual
      allow_failure: true
  before_script:
    - mkdir -p /kaniko/.docker
    - >-
      echo "{\"auths\":{
      \"rg.fr-par.scw.cloud\":{\"username\":\"nologin\",\"password\":\"${SCW_SECRET_KEY}\"},
      \"https://index.docker.io/v1/\":{\"username\":\"${DOCKERHUB_USERNAME}\",\"password\":\"${DOCKERHUB_TOKEN}\"}
      }}" > /kaniko/.docker/config.json
  script:
    - /kaniko/executor
      --context "${CI_PROJECT_DIR}"
      --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.ci-builder-cpu"
      --cache=true --cache-repo="${REGISTRY}/cache"
      --destination "${CI_BUILDER_CPU_IMAGE}"

# --------------------------------------------------------------------------
# Stage 0b: Build devcontainer image → push to internal GitLab registry
# --------------------------------------------------------------------------
build-devcontainer:
  stage: prepare
  image:
    name: gcr.io/kaniko-project/executor:debug
    entrypoint: [""]
  tags:
    - kapsule
    - docker
  rules:
    - if: $CI_PIPELINE_SOURCE == "push"
      changes:
        - .devcontainer/**
      when: on_success
    - when: manual
      allow_failure: true
  before_script:
    - mkdir -p /kaniko/.docker
    - >-
      echo "{\"auths\":{
      \"rg.fr-par.scw.cloud\":{\"username\":\"nologin\",\"password\":\"${SCW_SECRET_KEY}\"},
      \"https://index.docker.io/v1/\":{\"username\":\"${DOCKERHUB_USERNAME}\",\"password\":\"${DOCKERHUB_TOKEN}\"}
      }}" > /kaniko/.docker/config.json
  script:
    - /kaniko/executor
      --context "${CI_PROJECT_DIR}"
      --dockerfile "${CI_PROJECT_DIR}/.devcontainer/Dockerfile"
      --cache=true --cache-repo="${REGISTRY}/cache"
      --destination "localhost:30500/root/foxhunt/devcontainer:${CI_COMMIT_SHA}"
      --destination "localhost:30500/root/foxhunt/devcontainer:latest"

# --------------------------------------------------------------------------
# Stage 0c: Build infra-runner image → push to Scaleway CR
# --------------------------------------------------------------------------
build-infra-runner:
  stage: prepare
  image:
    name: gcr.io/kaniko-project/executor:debug
    entrypoint: [""]
  tags:
    - kapsule
    - docker
  rules:
    - if: $CI_PIPELINE_SOURCE == "push"
      changes:
        - infra/docker/Dockerfile.infra-runner
      when: on_success
    - when: manual
      allow_failure: true
  before_script:
    - mkdir -p /kaniko/.docker
    - >-
      echo "{\"auths\":{
      \"rg.fr-par.scw.cloud\":{\"username\":\"nologin\",\"password\":\"${SCW_SECRET_KEY}\"},
      \"https://index.docker.io/v1/\":{\"username\":\"${DOCKERHUB_USERNAME}\",\"password\":\"${DOCKERHUB_TOKEN}\"}
      }}" > /kaniko/.docker/config.json
  script:
    - /kaniko/executor
      --context "${CI_PROJECT_DIR}"
      --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.infra-runner"
      --cache=true --cache-repo="${REGISTRY}/cache"
      --destination "${INFRA_RUNNER_IMAGE}"

# --------------------------------------------------------------------------
# Stage 0d: Build generic runtime base images → push to Scaleway CR
# --------------------------------------------------------------------------
build-foxhunt-runtime:
  stage: prepare
  image:
    name: gcr.io/kaniko-project/executor:debug
    entrypoint: [""]
  tags:
    - kapsule
    - docker
  rules:
    - if: $CI_PIPELINE_SOURCE == "push"
      changes:
        - infra/docker/Dockerfile.foxhunt-runtime
      when: on_success
    - when: manual
      allow_failure: true
  before_script:
    - mkdir -p /kaniko/.docker
    - >-
      echo "{\"auths\":{
      \"rg.fr-par.scw.cloud\":{\"username\":\"nologin\",\"password\":\"${SCW_SECRET_KEY}\"},
      \"https://index.docker.io/v1/\":{\"username\":\"${DOCKERHUB_USERNAME}\",\"password\":\"${DOCKERHUB_TOKEN}\"}
      }}" > /kaniko/.docker/config.json
  script:
    - /kaniko/executor
      --context "${CI_PROJECT_DIR}"
      --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.foxhunt-runtime"
      --cache=true --cache-repo="${REGISTRY}/cache"
      --destination "${REGISTRY}/foxhunt-runtime:latest"

build-foxhunt-training-runtime:
  stage: prepare
  image:
    name: gcr.io/kaniko-project/executor:debug
    entrypoint: [""]
  tags:
    - kapsule
    - docker
  rules:
    - if: $CI_PIPELINE_SOURCE == "push"
      changes:
        - infra/docker/Dockerfile.foxhunt-training-runtime
      when: on_success
    - when: manual
      allow_failure: true
  before_script:
    - mkdir -p /kaniko/.docker
    - >-
      echo "{\"auths\":{
      \"rg.fr-par.scw.cloud\":{\"username\":\"nologin\",\"password\":\"${SCW_SECRET_KEY}\"},
      \"https://index.docker.io/v1/\":{\"username\":\"${DOCKERHUB_USERNAME}\",\"password\":\"${DOCKERHUB_TOKEN}\"}
      }}" > /kaniko/.docker/config.json
  script:
    - /kaniko/executor
      --context "${CI_PROJECT_DIR}"
      --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.foxhunt-training-runtime"
      --cache=true --cache-repo="${REGISTRY}/cache"
      --destination "${REGISTRY}/foxhunt-training-runtime:latest"

# --------------------------------------------------------------------------
# Stage 0.5: Scan runtime images for vulnerabilities (Trivy)
# --------------------------------------------------------------------------
scan-runtime-images:
  stage: scan
  image:
    name: aquasec/trivy:latest
    entrypoint: [""]
  tags:
    - kapsule
    - docker
  variables:
    KUBERNETES_CPU_REQUEST: "200m"
    KUBERNETES_CPU_LIMIT: "500m"
    KUBERNETES_MEMORY_REQUEST: "256Mi"
    KUBERNETES_MEMORY_LIMIT: "512Mi"
  needs:
    - job: build-foxhunt-runtime
      optional: true
    - job: build-foxhunt-training-runtime
      optional: true
  rules:
    - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
    - if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
  cache:
    key: trivy-db
    paths:
      - .trivy-cache/
  before_script:
    - export TRIVY_USERNAME=nologin
    - export TRIVY_PASSWORD=${SCW_SECRET_KEY}
  script:
    - echo "Scanning runtime images for CRITICAL/HIGH vulnerabilities..."
    - trivy image
      --cache-dir .trivy-cache
      --exit-code 1
      --severity CRITICAL,HIGH
      --ignore-unfixed
      --no-progress
      "${REGISTRY}/foxhunt-runtime:latest"
    - trivy image
      --cache-dir .trivy-cache
      --exit-code 1
      --severity CRITICAL,HIGH
      --ignore-unfixed
      --no-progress
      "${REGISTRY}/foxhunt-training-runtime:latest"
    - echo "Image scan passed — no fixable CRITICAL/HIGH vulnerabilities."
  allow_failure: false


# Base template for Rust jobs — pre-baked CI builder from Scaleway CR
# Image pre-exists in SCR; rebuild via build-ci-builder when Dockerfile changes
# Runs on ci-compile-cpu pool (POP2-32C-128G: 32 vCPU, 128GB RAM) via CPU runner
# CPU budget: 14 CPUs per job for tests; per-service compile jobs use 4 CPUs each
# (Scaleway quota: 1× POP2-32C-128G, so both jobs must share)
# Base template for jobs needing CUDA builder (test stage only — compile uses .rust-base-cpu)
.rust-base:
  image: ${CI_BUILDER_IMAGE}
  tags:
    - kapsule
    - rust
  variables:
    CUDA_COMPUTE_CAP: "89"
    CARGO_BUILD_JOBS: "14"
    KUBERNETES_CPU_REQUEST: "14000m"
    KUBERNETES_CPU_LIMIT: "15500m"
    KUBERNETES_MEMORY_REQUEST: "24Gi"
    KUBERNETES_MEMORY_LIMIT: "56Gi"
  before_script:
    # Mold linker fallback — .cargo/config.toml uses -fuse-ld=mold via clang.
    # clang's -fuse-ld=mold searches for "ld.mold" (not "mold") in its linker paths.
    # If mold isn't installed, create ld.mold wrapper that delegates to ld.lld.
    - |
      if ! command -v mold &>/dev/null; then
        printf '#!/bin/sh\nexec ld.lld "$@"\n' > /usr/local/bin/ld.mold
        chmod +x /usr/local/bin/ld.mold
        echo "mold not found, falling back to lld via ld.mold wrapper"
      fi

# Base template for CPU-only Rust jobs — lightweight builder without CUDA
# Uses ci-builder-cpu image (rust:1.89-slim, ~2-3GB, no CUDA toolkit)
# Per-service compile jobs override CPU/memory to 4 CPUs each (see .compile-service-base)
.rust-base-cpu:
  image: ${CI_BUILDER_CPU_IMAGE}
  tags:
    - kapsule
    - rust
  variables:
    # No CUDA_COMPUTE_CAP — CPU-only builds don't compile CUDA kernels
    # Request 10 CPUs (burstable to 15.5) — two compile jobs + web-dashboard fit on 1 POP2-32C-128G node
    CARGO_BUILD_JOBS: "10"
    KUBERNETES_NODE_SELECTOR_POOL: "k8s.scaleway.com/pool-name=ci-compile-cpu"
    KUBERNETES_CPU_REQUEST: "10000m"
    KUBERNETES_CPU_LIMIT: "15500m"
    KUBERNETES_MEMORY_REQUEST: "18Gi"
    KUBERNETES_MEMORY_LIMIT: "56Gi"
  before_script:
    # Mold linker fallback — same as .rust-base
    - |
      if ! command -v mold &>/dev/null; then
        printf '#!/bin/sh\nexec ld.lld "$@"\n' > /usr/local/bin/ld.mold
        chmod +x /usr/local/bin/ld.mold
        echo "mold not found, falling back to lld via ld.mold wrapper"
      fi

# Base template for per-service compile jobs (CPU-only, no CUDA)
# Each service gets its own job with dependency-aware changes: filters.
# CARGO_BUILD_JOBS=4 (down from 10) since up to 8 jobs may run on the same node.
.compile-service-base:
  extends: .rust-base-cpu
  stage: compile
  needs: []
  interruptible: true
  variables:
    CARGO_BUILD_JOBS: "4"
    KUBERNETES_CPU_REQUEST: "4000m"
    KUBERNETES_CPU_LIMIT: "6000m"
    KUBERNETES_MEMORY_REQUEST: "8Gi"
    KUBERNETES_MEMORY_LIMIT: "16Gi"
  script:
    - sccache --zero-stats || true
    - PROFILE=${DEV_RELEASE:+dev-release}; PROFILE=${PROFILE:-release}
    - |
      # CalVer: YYYY.MM.<pipeline_iid>
      YEAR=$(date -d "$CI_COMMIT_TIMESTAMP" +%Y 2>/dev/null || date +%Y)
      MONTH=$(date -d "$CI_COMMIT_TIMESTAMP" +%m 2>/dev/null || date +%m)
      export FOXHUNT_BUILD_VERSION="${YEAR}.${MONTH}.${CI_PIPELINE_IID:-0}"
      echo "Build version: $FOXHUNT_BUILD_VERSION"
    - echo "Building ${SERVICE_PACKAGE} with --profile $PROFILE (CPU-only, no CUDA)"
    - TARGET_DIR="target/$PROFILE"
    - cargo build --profile $PROFILE -p ${SERVICE_PACKAGE}
    - sccache --show-stats || true
    - mkdir -p build-out/services
    - |
      cp $TARGET_DIR/${SERVICE_PACKAGE} build-out/services/
      strip build-out/services/${SERVICE_PACKAGE}
    - ls -lh build-out/services/
    - echo "${SERVICE_PACKAGE} binary ready as CI artifact"
  artifacts:
    paths:
      - build-out/services/
    expire_in: 1 day

# --------------------------------------------------------------------------
# Stage 2: tests
# --------------------------------------------------------------------------
test:
  extends: .rust-base
  stage: test
  needs: []
  rules:
    # Tests disabled by default — compile+deploy is the fast path for now.
    # Trigger manually from the pipeline UI when needed.
    - if: $CI_PIPELINE_SOURCE == "push"
      when: manual
      allow_failure: true
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
      when: manual
      allow_failure: true
    - if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
      when: manual
      allow_failure: true
  services:
    - name: redis:7-alpine
      alias: redis
  script:
    # Detect GPU if available (nvidia runtime), otherwise use CUDA_COMPUTE_CAP from variables
    - |
      if command -v nvidia-smi &>/dev/null; then
        export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//')
        export CUDA_COMPUTE_CAP=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader | head -1 | tr -d '.')
        echo "GPU=$(nvidia-smi --query-gpu=name --format=csv,noheader) CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP"
      else
        echo "No nvidia-smi — using CUDA stubs, CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP"
      fi
    - export SCCACHE_S3_KEY_PREFIX="cuda/sm_${CUDA_COMPUTE_CAP}"
    # Wait for Redis sidecar (K8s executor doesn't wait for service readiness)
    - |
      for i in $(seq 1 30); do
        if bash -c 'echo PING > /dev/tcp/localhost/6379' 2>/dev/null; then
          echo "Redis ready"; break
        fi
        echo "Waiting for Redis... ($i/30)"; sleep 1
      done
    - sccache --zero-stats || true
    - cargo test --workspace --lib -- --skip model_loader::tests
    - sccache --show-stats || true

# --------------------------------------------------------------------------
# Stage 3a: Per-service compile jobs (CPU-only, no CUDA)
# Each service compiled independently with dependency-aware changes: filters.
# sccache shared across jobs — incremental crate compilation is cached.
# --------------------------------------------------------------------------
compile-api:
  extends: .compile-service-base
  variables:
    SERVICE_PACKAGE: api
  rules: &rules-api
    - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
      changes:
        - services/api/**
        - crates/common/**
        - crates/config/**
        - crates/trading_engine/**
        - bin/fxt/**
        - Cargo.toml
        - Cargo.lock
    - if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"

compile-trading-service:
  extends: .compile-service-base
  variables:
    SERVICE_PACKAGE: trading-service
  rules: &rules-trading-service
    - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
      changes:
        - services/trading_service/**
        - services/api/**
        - crates/common/**
        - crates/config/**
        - crates/data/**
        - crates/database/**
        - crates/ml/**
        - crates/risk/**
        - crates/storage/**
        - crates/trading_engine/**
        - Cargo.toml
        - Cargo.lock
    - if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"

compile-ml-training-service:
  extends: .compile-service-base
  variables:
    SERVICE_PACKAGE: ml-training-service
  rules: &rules-ml-training-service
    - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
      changes:
        - services/ml_training_service/**
        - crates/common/**
        - crates/config/**
        - crates/data/**
        - crates/ml/**
        - crates/risk/**
        - crates/storage/**
        - crates/trading_engine/**
        - Cargo.toml
        - Cargo.lock
    - if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"

compile-backtesting-service:
  extends: .compile-service-base
  variables:
    SERVICE_PACKAGE: backtesting-service
  rules: &rules-backtesting-service
    - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
      changes:
        - services/backtesting_service/**
        - crates/common/**
        - crates/config/**
        - crates/data/**
        - crates/ml/**
        - crates/model_loader/**
        - crates/risk/**
        - crates/storage/**
        - crates/trading_engine/**
        - bin/fxt/**
        - Cargo.toml
        - Cargo.lock
    - if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"

compile-trading-agent-service:
  extends: .compile-service-base
  variables:
    SERVICE_PACKAGE: trading-agent-service
  rules: &rules-trading-agent-service
    - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
      changes:
        - services/trading_agent_service/**
        - crates/common/**
        - crates/config/**
        - crates/ml/**
        - crates/risk/**
        - Cargo.toml
        - Cargo.lock
    - if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"

compile-broker-gateway:
  extends: .compile-service-base
  variables:
    SERVICE_PACKAGE: broker-gateway
  rules: &rules-broker-gateway
    - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
      changes:
        - services/broker_gateway_service/**
        - crates/common/**
        - crates/config/**
        - crates/trading_engine/**
        - vendor/ctrader-openapi/**
        - Cargo.toml
        - Cargo.lock
    - if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"

compile-data-acquisition-service:
  extends: .compile-service-base
  variables:
    SERVICE_PACKAGE: data-acquisition-service
  rules: &rules-data-acquisition-service
    - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
      changes:
        - services/data_acquisition_service/**
        - crates/common/**
        - crates/config/**
        - crates/storage/**
        - Cargo.toml
        - Cargo.lock
    - if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"

# --------------------------------------------------------------------------
# Stage 3a: Compile training binaries (CPU node, CUDA stubs from ci-builder image)
# --------------------------------------------------------------------------
compile-training:
  extends: .rust-base-cpu
  stage: compile
  image: ${CI_BUILDER_IMAGE}
  needs: []  # parallel with per-service compile jobs — shares 1 POP2-32C-128G node
  resource_group: compile-training
  interruptible: true
  variables:
    # CUDA stubs from ci-builder image — compiles CUDA kernels without GPU hardware.
    # L4 GPU = compute capability 8.9 → candle-kernels generates sm_89 PTX
    CUDA_COMPUTE_CAP: "89"
    # When all 8 service jobs fire (Cargo.toml change), the node is fully loaded.
    # Increase pod scheduling timeout so training waits for service jobs to finish.
    KUBERNETES_POLL_TIMEOUT: "1200"
  rules:
    - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
      changes:
        - Cargo.toml
        - Cargo.lock
        - crates/ml/**
        - crates/trading_engine/**
        - crates/common/**
        - crates/risk/**
        - crates/data/**
        - crates/config/**
        - crates/storage/**
        - services/training_uploader/**
    - if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
  script:
    - echo "CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP"
    - export SCCACHE_S3_KEY_PREFIX="cuda/sm_${CUDA_COMPUTE_CAP}"
    - sccache --zero-stats || true
    - PROFILE=${DEV_RELEASE:+dev-release}; PROFILE=${PROFILE:-release}
    - |
      # CalVer: YYYY.MM.<pipeline_iid>
      YEAR=$(date -d "$CI_COMMIT_TIMESTAMP" +%Y 2>/dev/null || date +%Y)
      MONTH=$(date -d "$CI_COMMIT_TIMESTAMP" +%m 2>/dev/null || date +%m)
      export FOXHUNT_BUILD_VERSION="${YEAR}.${MONTH}.${CI_PIPELINE_IID:-0}"
      echo "Build version: $FOXHUNT_BUILD_VERSION"
    - echo "Building training binaries with --profile $PROFILE (CUDA stubs)"
    - TARGET_DIR="target/$PROFILE"
    - cargo build --profile $PROFILE -p ml --features ml/cuda
        --example train_baseline_rl
        --example train_baseline_supervised
        --example evaluate_baseline
        --example evaluate_supervised
        --example hyperopt_baseline_rl
        --example hyperopt_baseline_supervised
    - cargo build --profile $PROFILE -p training_uploader
    - sccache --show-stats || true
    - mkdir -p build-out/training
    - |
      for bin in train_baseline_rl train_baseline_supervised evaluate_baseline evaluate_supervised hyperopt_baseline_rl hyperopt_baseline_supervised; do
        cp $TARGET_DIR/examples/$bin build-out/training/
        strip build-out/training/$bin
      done
      cp $TARGET_DIR/training_uploader build-out/training/
      strip build-out/training/training_uploader
    - ls -lh build-out/training/
    - echo "Training binaries ready as CI artifacts"
  artifacts:
    paths:
      - build-out/training/
    expire_in: 3 days

# --------------------------------------------------------------------------
# Stage 3c: Build web-dashboard static assets (Vite)
# --------------------------------------------------------------------------
build-web-dashboard:
  stage: compile
  image: node:22-slim
  needs: []
  resource_group: compile-web-dashboard
  interruptible: true
  tags:
    - kapsule
    - docker
  variables:
    KUBERNETES_NODE_SELECTOR_POOL: "k8s.scaleway.com/pool-name=ci-compile-cpu"
    KUBERNETES_CPU_REQUEST: "2000m"
    KUBERNETES_CPU_LIMIT: "4000m"
    KUBERNETES_MEMORY_REQUEST: "2Gi"
    KUBERNETES_MEMORY_LIMIT: "4Gi"
  rules:
    - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
      changes:
        - web-dashboard/**
    - if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
  script:
    - cd web-dashboard
    - npm ci
    - npm run build
    - echo "Web dashboard assets ready as CI artifacts"
  artifacts:
    paths:
      - web-dashboard/dist/
    expire_in: 1 day

# --------------------------------------------------------------------------
# Stage 3c: Training — RL on L4 (CPU-heavy), Supervised on L40S (GPU-heavy)
# --------------------------------------------------------------------------

# --------------------------------------------------------------------------
# RL training base: DQN + PPO → ci-training pool (L40S)
# RL hyperopt runs parallel PSO trials — CPU+GPU bound, low VRAM.
# --------------------------------------------------------------------------
.train-rl-base:
  stage: train
  image: ${TRAINING_RUNTIME_IMAGE}
  tags:
    - kapsule
    - l40s
  variables:
    # Route to ci-training pool (L40S GPU)
    KUBERNETES_NODE_SELECTOR_POOL: "k8s.scaleway.com/pool-name=ci-training"
    KUBERNETES_RUNTIME_CLASS_NAME: "nvidia"
    KUBERNETES_CPU_REQUEST: "7000m"
    KUBERNETES_CPU_LIMIT: "8000m"
    KUBERNETES_MEMORY_REQUEST: "8Gi"
    KUBERNETES_MEMORY_LIMIT: "20Gi"
    CUBLAS_WORKSPACE_CONFIG: ":4096:8"
    OTEL_EXPORTER_OTLP_ENDPOINT: "http://tempo.foxhunt.svc.cluster.local:4317"
    # Expose training metrics to Prometheus (binaries serve /metrics on :9094)
    KUBERNETES_POD_ANNOTATIONS_1: "gitlab.com/prometheus_scrape=true"
    KUBERNETES_POD_ANNOTATIONS_2: "gitlab.com/prometheus_port=9094"
    KUBERNETES_POD_ANNOTATIONS_3: "gitlab.com/prometheus_path=/metrics"
  rules:
    - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
      when: manual
      allow_failure: true
    - if: $CI_PIPELINE_SOURCE == "schedule" && $TRAIN_VALIDATE == "true"
    - if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
      when: manual
      allow_failure: true
  before_script:
    - export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//')
    - nvidia-smi
    - mkdir -p ${CI_PROJECT_DIR}/output ${CI_PROJECT_DIR}/training-bin
    # Fetch training binaries from MinIO (CI artifacts as fallback)
    - |
      if cp build-out/training/* ${CI_PROJECT_DIR}/training-bin/ 2>/dev/null; then
        echo "Using CI artifact binaries"
      else
        echo "Fetching training binaries from MinIO..."
        rclone copy :s3:foxhunt-binaries/training/ ${CI_PROJECT_DIR}/training-bin/ \
          --s3-provider=Minio \
          --s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 \
          --s3-access-key-id="${MINIO_ACCESS_KEY}" \
          --s3-secret-access-key="${MINIO_SECRET_KEY}" \
          --s3-no-check-bucket \
        echo "Fetched $(ls ${CI_PROJECT_DIR}/training-bin/ | wc -l) training binaries from MinIO"
      fi
    - chmod +x ${CI_PROJECT_DIR}/training-bin/* 2>/dev/null || true
    - export PATH="${CI_PROJECT_DIR}/training-bin:$PATH"

# --------------------------------------------------------------------------
# Supervised training base: TFT, Mamba2, TGGN, TLOB, Liquid, KAN, xLSTM, Diffusion → L40S
# Supervised models are GPU-bound — low CPU, high VRAM.
# --------------------------------------------------------------------------
.train-validate-base:
  stage: train
  image: ${TRAINING_RUNTIME_IMAGE}
  tags:
    - kapsule
    - l40s
  variables:
    # nvidia runtime for GPU nodes (ci-training has L40S 48GB)
    KUBERNETES_RUNTIME_CLASS_NAME: "nvidia"
    # Route to L40S pool (48GB VRAM for large supervised models)
    # Overrides runner default to L40S pool
    KUBERNETES_NODE_SELECTOR_POOL: "k8s.scaleway.com/pool-name=ci-training"
    # Supervised is GPU-bound — minimal CPU. Low request allows 3-4 concurrent jobs.
    KUBERNETES_CPU_REQUEST: "1000m"
    KUBERNETES_CPU_LIMIT: "2000m"
    KUBERNETES_MEMORY_REQUEST: "16Gi"
    KUBERNETES_MEMORY_LIMIT: "40Gi"
    OTEL_EXPORTER_OTLP_ENDPOINT: "http://tempo.foxhunt.svc.cluster.local:4317"
    # Expose training metrics to Prometheus (binaries serve /metrics on :9094)
    KUBERNETES_POD_ANNOTATIONS_1: "gitlab.com/prometheus_scrape=true"
    KUBERNETES_POD_ANNOTATIONS_2: "gitlab.com/prometheus_port=9094"
    KUBERNETES_POD_ANNOTATIONS_3: "gitlab.com/prometheus_path=/metrics"
  rules:
    - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
      when: manual
      allow_failure: true
    - if: $CI_PIPELINE_SOURCE == "schedule" && $TRAIN_VALIDATE == "true"
    - if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
      when: manual
      allow_failure: true
  before_script:
    - export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//')
    - nvidia-smi
    - mkdir -p ${CI_PROJECT_DIR}/output ${CI_PROJECT_DIR}/training-bin
    # Fetch training binaries from MinIO (CI artifacts as fallback)
    - |
      if cp build-out/training/* ${CI_PROJECT_DIR}/training-bin/ 2>/dev/null; then
        echo "Using CI artifact binaries"
      else
        echo "Fetching training binaries from MinIO..."
        rclone copy :s3:foxhunt-binaries/training/ ${CI_PROJECT_DIR}/training-bin/ \
          --s3-provider=Minio \
          --s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 \
          --s3-access-key-id="${MINIO_ACCESS_KEY}" \
          --s3-secret-access-key="${MINIO_SECRET_KEY}" \
          --s3-no-check-bucket \
        echo "Fetched $(ls ${CI_PROJECT_DIR}/training-bin/ | wc -l) training binaries from MinIO"
      fi
    - chmod +x ${CI_PROJECT_DIR}/training-bin/* 2>/dev/null || true
    - export PATH="${CI_PROJECT_DIR}/training-bin:$PATH"

train-validate-rl:
  extends: .train-rl-base
  needs:
    - job: compile-training
      optional: true
  script:
    - |
      train_baseline_rl \
        --model both \
        --symbol ES.FUT \
        --data-dir /mnt/training-data/futures-baseline \
        --output-dir ${CI_PROJECT_DIR}/output \
        --max-steps-per-epoch 50 \
      && TRAIN_OK=1 || TRAIN_OK=0
      echo "=== RL training exit: $TRAIN_OK ==="
      echo "=== Running evaluation ==="
      evaluate_baseline \
        --model both \
        --models-dir ${CI_PROJECT_DIR}/output \
        --data-dir /mnt/training-data/futures-baseline \
        --output ${CI_PROJECT_DIR}/output/rl_eval_report.json \
        --symbol ES.FUT \
      || true
      echo "=== Eval report ==="
      cat ${CI_PROJECT_DIR}/output/rl_eval_report.json 2>/dev/null || echo "No eval report"
      [ "$TRAIN_OK" = "1" ] || exit 1
  artifacts:
    paths:
      - output/rl_eval_report.json
    when: always
    expire_in: 30 days

train-validate-tft:
  extends: .train-validate-base
  needs:
    - job: compile-training
      optional: true
  script:
    - train_baseline_supervised
        --model tft
        --symbol ES.FUT
        --data-dir /mnt/training-data/futures-baseline
        --output-dir ${CI_PROJECT_DIR}/output
        --max-steps-per-epoch 50
    - echo "=== TFT training complete, running evaluation ==="
    - evaluate_supervised
        --model tft
        --models-dir ${CI_PROJECT_DIR}/output
        --data-dir /mnt/training-data/futures-baseline
        --output ${CI_PROJECT_DIR}/output/tft_eval_report.json
        --symbol ES.FUT
    - echo "TFT validation + evaluation passed"
    - cat ${CI_PROJECT_DIR}/output/tft_eval_report.json
  artifacts:
    paths:
      - output/tft_eval_report.json
    when: always
    expire_in: 30 days

hyperopt-ppo:
  extends: .train-rl-base
  tags:
    - kapsule
    - h100
  variables:
    KUBERNETES_NODE_SELECTOR_POOL: "k8s.scaleway.com/pool-name=ci-training-h100"
    KUBERNETES_MEMORY_REQUEST: "16Gi"
    KUBERNETES_MEMORY_LIMIT: "48Gi"
  needs:
    - job: compile-training
      optional: true
  timeout: 4h
  script:
    - hyperopt_baseline_rl
        --model ppo
        --trials 20
        --n-initial 5
        --epochs 8
        --parallel 0
        --symbol ES.FUT
        --tx-cost-bps 0.1
        --tick-size 0.25
        --spread-ticks 1.0
        --initial-capital 35000
        --data-dir /mnt/training-data/futures-baseline
        --base-dir ${CI_PROJECT_DIR}/output/hyperopt
        --output ${CI_PROJECT_DIR}/output/ppo_hyperopt_results.json
    - echo "=== PPO Hyperopt complete ==="
    - cat ${CI_PROJECT_DIR}/output/ppo_hyperopt_results.json
    - echo "=== Training PPO with best params ==="
    - train_baseline_rl
        --model ppo
        --symbol ES.FUT
        --data-dir /mnt/training-data/futures-baseline
        --output-dir ${CI_PROJECT_DIR}/output
        --hyperopt-params ${CI_PROJECT_DIR}/output/ppo_hyperopt_results.json
    - echo "=== Evaluating PPO with best params ==="
    - evaluate_baseline
        --model ppo
        --models-dir ${CI_PROJECT_DIR}/output
        --data-dir /mnt/training-data/futures-baseline
        --output ${CI_PROJECT_DIR}/output/ppo_eval_report.json
        --symbol ES.FUT
        --hyperopt-params ${CI_PROJECT_DIR}/output/ppo_hyperopt_results.json
    - cat ${CI_PROJECT_DIR}/output/ppo_eval_report.json
  artifacts:
    paths:
      - output/ppo_hyperopt_results.json
      - output/ppo_eval_report.json
    when: always
    expire_in: 90 days

# --- Train-validate: remaining supervised models ---

train-validate-mamba2:
  extends: .train-validate-base
  needs:
    - job: compile-training
      optional: true
  script:
    - train_baseline_supervised
        --model mamba2
        --symbol ES.FUT
        --data-dir /mnt/training-data/futures-baseline
        --output-dir ${CI_PROJECT_DIR}/output
        --max-steps-per-epoch 50
    - echo "=== Mamba2 training complete, running evaluation ==="
    - evaluate_supervised
        --model mamba2
        --models-dir ${CI_PROJECT_DIR}/output
        --data-dir /mnt/training-data/futures-baseline
        --output ${CI_PROJECT_DIR}/output/mamba2_eval_report.json
        --symbol ES.FUT
    - echo "Mamba2 validation + evaluation passed"
    - cat ${CI_PROJECT_DIR}/output/mamba2_eval_report.json
  artifacts:
    paths:
      - output/mamba2_eval_report.json
    when: always
    expire_in: 30 days

train-validate-liquid:
  extends: .train-validate-base
  needs:
    - job: compile-training
      optional: true
  script:
    - train_baseline_supervised
        --model liquid
        --symbol ES.FUT
        --data-dir /mnt/training-data/futures-baseline
        --output-dir ${CI_PROJECT_DIR}/output
        --max-steps-per-epoch 50
    - echo "=== Liquid training complete, running evaluation ==="
    - evaluate_supervised
        --model liquid
        --models-dir ${CI_PROJECT_DIR}/output
        --data-dir /mnt/training-data/futures-baseline
        --output ${CI_PROJECT_DIR}/output/liquid_eval_report.json
        --symbol ES.FUT
    - echo "Liquid validation + evaluation passed"
    - cat ${CI_PROJECT_DIR}/output/liquid_eval_report.json
  artifacts:
    paths:
      - output/liquid_eval_report.json
    when: always
    expire_in: 30 days

train-validate-tggn:
  extends: .train-validate-base
  needs:
    - job: compile-training
      optional: true
  script:
    - train_baseline_supervised
        --model tggn
        --symbol ES.FUT
        --data-dir /mnt/training-data/futures-baseline
        --output-dir ${CI_PROJECT_DIR}/output
        --max-steps-per-epoch 50
    - echo "=== TGGN training complete, running evaluation ==="
    - evaluate_supervised
        --model tggn
        --models-dir ${CI_PROJECT_DIR}/output
        --data-dir /mnt/training-data/futures-baseline
        --output ${CI_PROJECT_DIR}/output/tggn_eval_report.json
        --symbol ES.FUT
    - echo "TGGN validation + evaluation passed"
    - cat ${CI_PROJECT_DIR}/output/tggn_eval_report.json
  artifacts:
    paths:
      - output/tggn_eval_report.json
    when: always
    expire_in: 30 days

train-validate-tlob:
  extends: .train-validate-base
  needs:
    - job: compile-training
      optional: true
  script:
    - train_baseline_supervised
        --model tlob
        --symbol ES.FUT
        --data-dir /mnt/training-data/futures-baseline
        --output-dir ${CI_PROJECT_DIR}/output
        --max-steps-per-epoch 50
    - echo "=== TLOB training complete, running evaluation ==="
    - evaluate_supervised
        --model tlob
        --models-dir ${CI_PROJECT_DIR}/output
        --data-dir /mnt/training-data/futures-baseline
        --output ${CI_PROJECT_DIR}/output/tlob_eval_report.json
        --symbol ES.FUT
    - echo "TLOB validation + evaluation passed"
    - cat ${CI_PROJECT_DIR}/output/tlob_eval_report.json
  artifacts:
    paths:
      - output/tlob_eval_report.json
    when: always
    expire_in: 30 days

train-validate-kan:
  extends: .train-validate-base
  needs:
    - job: compile-training
      optional: true
  script:
    - train_baseline_supervised
        --model kan
        --symbol ES.FUT
        --data-dir /mnt/training-data/futures-baseline
        --output-dir ${CI_PROJECT_DIR}/output
        --max-steps-per-epoch 50
    - echo "=== KAN training complete, running evaluation ==="
    - evaluate_supervised
        --model kan
        --models-dir ${CI_PROJECT_DIR}/output
        --data-dir /mnt/training-data/futures-baseline
        --output ${CI_PROJECT_DIR}/output/kan_eval_report.json
        --symbol ES.FUT
    - echo "KAN validation + evaluation passed"
    - cat ${CI_PROJECT_DIR}/output/kan_eval_report.json
  artifacts:
    paths:
      - output/kan_eval_report.json
    when: always
    expire_in: 30 days

train-validate-xlstm:
  extends: .train-validate-base
  needs:
    - job: compile-training
      optional: true
  script:
    - train_baseline_supervised
        --model xlstm
        --symbol ES.FUT
        --data-dir /mnt/training-data/futures-baseline
        --output-dir ${CI_PROJECT_DIR}/output
        --max-steps-per-epoch 50
    - echo "=== xLSTM training complete, running evaluation ==="
    - evaluate_supervised
        --model xlstm
        --models-dir ${CI_PROJECT_DIR}/output
        --data-dir /mnt/training-data/futures-baseline
        --output ${CI_PROJECT_DIR}/output/xlstm_eval_report.json
        --symbol ES.FUT
    - echo "xLSTM validation + evaluation passed"
    - cat ${CI_PROJECT_DIR}/output/xlstm_eval_report.json
  artifacts:
    paths:
      - output/xlstm_eval_report.json
    when: always
    expire_in: 30 days

train-validate-diffusion:
  extends: .train-validate-base
  needs:
    - job: compile-training
      optional: true
  script:
    - train_baseline_supervised
        --model diffusion
        --symbol ES.FUT
        --data-dir /mnt/training-data/futures-baseline
        --output-dir ${CI_PROJECT_DIR}/output
        --max-steps-per-epoch 50
    - echo "=== Diffusion training complete, running evaluation ==="
    - evaluate_supervised
        --model diffusion
        --models-dir ${CI_PROJECT_DIR}/output
        --data-dir /mnt/training-data/futures-baseline
        --output ${CI_PROJECT_DIR}/output/diffusion_eval_report.json
        --symbol ES.FUT
    - echo "Diffusion validation + evaluation passed"
    - cat ${CI_PROJECT_DIR}/output/diffusion_eval_report.json
  artifacts:
    paths:
      - output/diffusion_eval_report.json
    when: always
    expire_in: 30 days

# --- Hyperopt: all models ---

hyperopt-dqn:
  extends: .train-rl-base
  tags:
    - kapsule
    - h100
  variables:
    KUBERNETES_NODE_SELECTOR_POOL: "k8s.scaleway.com/pool-name=ci-training-h100"
    KUBERNETES_MEMORY_REQUEST: "16Gi"
    KUBERNETES_MEMORY_LIMIT: "48Gi"
  needs:
    - job: compile-training
      optional: true
  timeout: 4h
  script:
    - |
      hyperopt_baseline_rl \
        --model dqn \
        --trials 20 \
        --n-initial 5 \
        --epochs 8 \
        --parallel 0 \
        --symbol ES.FUT \
        --tx-cost-bps 0.1 \
        --tick-size 0.25 \
        --spread-ticks 1.0 \
        --initial-capital 35000 \
        --data-dir /mnt/training-data/futures-baseline \
        --base-dir ${CI_PROJECT_DIR}/output/hyperopt \
        --output ${CI_PROJECT_DIR}/output/dqn_hyperopt_results.json \
      && HYPEROPT_OK=1 || HYPEROPT_OK=0
      echo "=== DQN Hyperopt exit: $HYPEROPT_OK ==="
      cat ${CI_PROJECT_DIR}/output/dqn_hyperopt_results.json 2>/dev/null || echo "No hyperopt results"
      echo "=== Training DQN with best params ==="
      train_baseline_rl \
        --model dqn \
        --symbol ES.FUT \
        --tx-cost-bps 0.1 \
        --tick-size 0.25 \
        --spread-ticks 1.0 \
        --data-dir /mnt/training-data/futures-baseline \
        --output-dir ${CI_PROJECT_DIR}/output \
        --hyperopt-params ${CI_PROJECT_DIR}/output/dqn_hyperopt_results.json \
      && TRAIN_OK=1 || TRAIN_OK=0
      echo "=== Training exit: $TRAIN_OK ==="
      echo "=== Evaluating DQN ==="
      evaluate_baseline \
        --model dqn \
        --models-dir ${CI_PROJECT_DIR}/output \
        --data-dir /mnt/training-data/futures-baseline \
        --output ${CI_PROJECT_DIR}/output/dqn_eval_report.json \
        --symbol ES.FUT \
        --tx-cost-bps 0.1 \
        --tick-size 0.25 \
        --spread-ticks 1.0 \
        --hyperopt-params ${CI_PROJECT_DIR}/output/dqn_hyperopt_results.json \
      || true
      echo "=== Eval report ==="
      cat ${CI_PROJECT_DIR}/output/dqn_eval_report.json 2>/dev/null || echo "No eval report"
      [ "$HYPEROPT_OK" = "1" ] && [ "$TRAIN_OK" = "1" ] || exit 1
  artifacts:
    paths:
      - output/dqn_hyperopt_results.json
      - output/dqn_eval_report.json
    when: always
    expire_in: 90 days

hyperopt-tft:
  extends: .train-validate-base
  needs:
    - job: compile-training
      optional: true
  timeout: 4h
  script:
    - hyperopt_baseline_supervised
        --model tft
        --trials 20
        --n-initial 5
        --epochs 15
        --data-dir /mnt/training-data/futures-baseline
        --base-dir ${CI_PROJECT_DIR}/output/hyperopt
        --output ${CI_PROJECT_DIR}/output/tft_hyperopt_results.json
    - echo "=== TFT Hyperopt complete ==="
    - cat ${CI_PROJECT_DIR}/output/tft_hyperopt_results.json
    - echo "=== Training TFT with best params ==="
    - train_baseline_supervised
        --model tft
        --symbol ES.FUT
        --data-dir /mnt/training-data/futures-baseline
        --output-dir ${CI_PROJECT_DIR}/output
        --hyperopt-params ${CI_PROJECT_DIR}/output/tft_hyperopt_results.json
    - echo "=== Evaluating TFT with best params ==="
    - evaluate_supervised
        --model tft
        --models-dir ${CI_PROJECT_DIR}/output
        --data-dir /mnt/training-data/futures-baseline
        --output ${CI_PROJECT_DIR}/output/tft_eval_report.json
        --symbol ES.FUT
    - cat ${CI_PROJECT_DIR}/output/tft_eval_report.json
  artifacts:
    paths:
      - output/tft_hyperopt_results.json
      - output/tft_eval_report.json
    when: always
    expire_in: 90 days

hyperopt-mamba2:
  extends: .train-validate-base
  needs:
    - job: compile-training
      optional: true
  timeout: 4h
  script:
    - hyperopt_baseline_supervised
        --model mamba2
        --trials 20
        --n-initial 5
        --epochs 15
        --data-dir /mnt/training-data/futures-baseline
        --base-dir ${CI_PROJECT_DIR}/output/hyperopt
        --output ${CI_PROJECT_DIR}/output/mamba2_hyperopt_results.json
    - echo "=== Mamba2 Hyperopt complete ==="
    - cat ${CI_PROJECT_DIR}/output/mamba2_hyperopt_results.json
    - echo "=== Training Mamba2 with best params ==="
    - train_baseline_supervised
        --model mamba2
        --symbol ES.FUT
        --data-dir /mnt/training-data/futures-baseline
        --output-dir ${CI_PROJECT_DIR}/output
        --hyperopt-params ${CI_PROJECT_DIR}/output/mamba2_hyperopt_results.json
    - echo "=== Evaluating Mamba2 with best params ==="
    - evaluate_supervised
        --model mamba2
        --models-dir ${CI_PROJECT_DIR}/output
        --data-dir /mnt/training-data/futures-baseline
        --output ${CI_PROJECT_DIR}/output/mamba2_eval_report.json
        --symbol ES.FUT
    - cat ${CI_PROJECT_DIR}/output/mamba2_eval_report.json
  artifacts:
    paths:
      - output/mamba2_hyperopt_results.json
      - output/mamba2_eval_report.json
    when: always
    expire_in: 90 days

hyperopt-liquid:
  extends: .train-validate-base
  needs:
    - job: compile-training
      optional: true
  timeout: 4h
  script:
    - hyperopt_baseline_supervised
        --model liquid
        --trials 20
        --n-initial 5
        --epochs 15
        --data-dir /mnt/training-data/futures-baseline
        --base-dir ${CI_PROJECT_DIR}/output/hyperopt
        --output ${CI_PROJECT_DIR}/output/liquid_hyperopt_results.json
    - echo "=== Liquid Hyperopt complete ==="
    - cat ${CI_PROJECT_DIR}/output/liquid_hyperopt_results.json
    - echo "=== Training Liquid with best params ==="
    - train_baseline_supervised
        --model liquid
        --symbol ES.FUT
        --data-dir /mnt/training-data/futures-baseline
        --output-dir ${CI_PROJECT_DIR}/output
        --hyperopt-params ${CI_PROJECT_DIR}/output/liquid_hyperopt_results.json
    - echo "=== Evaluating Liquid with best params ==="
    - evaluate_supervised
        --model liquid
        --models-dir ${CI_PROJECT_DIR}/output
        --data-dir /mnt/training-data/futures-baseline
        --output ${CI_PROJECT_DIR}/output/liquid_eval_report.json
        --symbol ES.FUT
    - cat ${CI_PROJECT_DIR}/output/liquid_eval_report.json
  artifacts:
    paths:
      - output/liquid_hyperopt_results.json
      - output/liquid_eval_report.json
    when: always
    expire_in: 90 days

hyperopt-tggn:
  extends: .train-validate-base
  needs:
    - job: compile-training
      optional: true
  timeout: 4h
  script:
    - hyperopt_baseline_supervised
        --model tggn
        --trials 20
        --n-initial 5
        --epochs 15
        --data-dir /mnt/training-data/futures-baseline
        --base-dir ${CI_PROJECT_DIR}/output/hyperopt
        --output ${CI_PROJECT_DIR}/output/tggn_hyperopt_results.json
    - echo "=== TGGN Hyperopt complete ==="
    - cat ${CI_PROJECT_DIR}/output/tggn_hyperopt_results.json
    - echo "=== Training TGGN with best params ==="
    - train_baseline_supervised
        --model tggn
        --symbol ES.FUT
        --data-dir /mnt/training-data/futures-baseline
        --output-dir ${CI_PROJECT_DIR}/output
        --hyperopt-params ${CI_PROJECT_DIR}/output/tggn_hyperopt_results.json
    - echo "=== Evaluating TGGN with best params ==="
    - evaluate_supervised
        --model tggn
        --models-dir ${CI_PROJECT_DIR}/output
        --data-dir /mnt/training-data/futures-baseline
        --output ${CI_PROJECT_DIR}/output/tggn_eval_report.json
        --symbol ES.FUT
    - cat ${CI_PROJECT_DIR}/output/tggn_eval_report.json
  artifacts:
    paths:
      - output/tggn_hyperopt_results.json
      - output/tggn_eval_report.json
    when: always
    expire_in: 90 days

hyperopt-tlob:
  extends: .train-validate-base
  needs:
    - job: compile-training
      optional: true
  timeout: 4h
  script:
    - hyperopt_baseline_supervised
        --model tlob
        --trials 20
        --n-initial 5
        --epochs 15
        --data-dir /mnt/training-data/futures-baseline
        --base-dir ${CI_PROJECT_DIR}/output/hyperopt
        --output ${CI_PROJECT_DIR}/output/tlob_hyperopt_results.json
    - echo "=== TLOB Hyperopt complete ==="
    - cat ${CI_PROJECT_DIR}/output/tlob_hyperopt_results.json
    - echo "=== Training TLOB with best params ==="
    - train_baseline_supervised
        --model tlob
        --symbol ES.FUT
        --data-dir /mnt/training-data/futures-baseline
        --output-dir ${CI_PROJECT_DIR}/output
        --hyperopt-params ${CI_PROJECT_DIR}/output/tlob_hyperopt_results.json
    - echo "=== Evaluating TLOB with best params ==="
    - evaluate_supervised
        --model tlob
        --models-dir ${CI_PROJECT_DIR}/output
        --data-dir /mnt/training-data/futures-baseline
        --output ${CI_PROJECT_DIR}/output/tlob_eval_report.json
        --symbol ES.FUT
    - cat ${CI_PROJECT_DIR}/output/tlob_eval_report.json
  artifacts:
    paths:
      - output/tlob_hyperopt_results.json
      - output/tlob_eval_report.json
    when: always
    expire_in: 90 days

hyperopt-kan:
  extends: .train-validate-base
  needs:
    - job: compile-training
      optional: true
  timeout: 4h
  script:
    - hyperopt_baseline_supervised
        --model kan
        --trials 20
        --n-initial 5
        --epochs 15
        --data-dir /mnt/training-data/futures-baseline
        --base-dir ${CI_PROJECT_DIR}/output/hyperopt
        --output ${CI_PROJECT_DIR}/output/kan_hyperopt_results.json
    - echo "=== KAN Hyperopt complete ==="
    - cat ${CI_PROJECT_DIR}/output/kan_hyperopt_results.json
    - echo "=== Training KAN with best params ==="
    - train_baseline_supervised
        --model kan
        --symbol ES.FUT
        --data-dir /mnt/training-data/futures-baseline
        --output-dir ${CI_PROJECT_DIR}/output
        --hyperopt-params ${CI_PROJECT_DIR}/output/kan_hyperopt_results.json
    - echo "=== Evaluating KAN with best params ==="
    - evaluate_supervised
        --model kan
        --models-dir ${CI_PROJECT_DIR}/output
        --data-dir /mnt/training-data/futures-baseline
        --output ${CI_PROJECT_DIR}/output/kan_eval_report.json
        --symbol ES.FUT
    - cat ${CI_PROJECT_DIR}/output/kan_eval_report.json
  artifacts:
    paths:
      - output/kan_hyperopt_results.json
      - output/kan_eval_report.json
    when: always
    expire_in: 90 days

hyperopt-xlstm:
  extends: .train-validate-base
  needs:
    - job: compile-training
      optional: true
  timeout: 4h
  script:
    - hyperopt_baseline_supervised
        --model xlstm
        --trials 20
        --n-initial 5
        --epochs 15
        --data-dir /mnt/training-data/futures-baseline
        --base-dir ${CI_PROJECT_DIR}/output/hyperopt
        --output ${CI_PROJECT_DIR}/output/xlstm_hyperopt_results.json
    - echo "=== xLSTM Hyperopt complete ==="
    - cat ${CI_PROJECT_DIR}/output/xlstm_hyperopt_results.json
    - echo "=== Training xLSTM with best params ==="
    - train_baseline_supervised
        --model xlstm
        --symbol ES.FUT
        --data-dir /mnt/training-data/futures-baseline
        --output-dir ${CI_PROJECT_DIR}/output
        --hyperopt-params ${CI_PROJECT_DIR}/output/xlstm_hyperopt_results.json
    - echo "=== Evaluating xLSTM with best params ==="
    - evaluate_supervised
        --model xlstm
        --models-dir ${CI_PROJECT_DIR}/output
        --data-dir /mnt/training-data/futures-baseline
        --output ${CI_PROJECT_DIR}/output/xlstm_eval_report.json
        --symbol ES.FUT
    - cat ${CI_PROJECT_DIR}/output/xlstm_eval_report.json
  artifacts:
    paths:
      - output/xlstm_hyperopt_results.json
      - output/xlstm_eval_report.json
    when: always
    expire_in: 90 days

hyperopt-diffusion:
  extends: .train-validate-base
  needs:
    - job: compile-training
      optional: true
  timeout: 4h
  script:
    - hyperopt_baseline_supervised
        --model diffusion
        --trials 20
        --n-initial 5
        --epochs 15
        --data-dir /mnt/training-data/futures-baseline
        --base-dir ${CI_PROJECT_DIR}/output/hyperopt
        --output ${CI_PROJECT_DIR}/output/diffusion_hyperopt_results.json
    - echo "=== Diffusion Hyperopt complete ==="
    - cat ${CI_PROJECT_DIR}/output/diffusion_hyperopt_results.json
    - echo "=== Training Diffusion with best params ==="
    - train_baseline_supervised
        --model diffusion
        --symbol ES.FUT
        --data-dir /mnt/training-data/futures-baseline
        --output-dir ${CI_PROJECT_DIR}/output
        --hyperopt-params ${CI_PROJECT_DIR}/output/diffusion_hyperopt_results.json
    - echo "=== Evaluating Diffusion with best params ==="
    - evaluate_supervised
        --model diffusion
        --models-dir ${CI_PROJECT_DIR}/output
        --data-dir /mnt/training-data/futures-baseline
        --output ${CI_PROJECT_DIR}/output/diffusion_eval_report.json
        --symbol ES.FUT
    - cat ${CI_PROJECT_DIR}/output/diffusion_eval_report.json
  artifacts:
    paths:
      - output/diffusion_hyperopt_results.json
      - output/diffusion_eval_report.json
    when: always
    expire_in: 90 days

# --------------------------------------------------------------------------
# IaC: Terragrunt plan on MR (runs on gitlab pool)
# --------------------------------------------------------------------------
infra-plan:
  stage: test
  image: ${INFRA_RUNNER_IMAGE}
  tags:
    - kapsule
    - docker
  needs: []
  rules:
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
      changes:
        - infra/**
    - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
      changes:
        - infra/**
  before_script:
    - export SCW_ACCESS_KEY=${SCW_ACCESS_KEY}
    - export SCW_SECRET_KEY=${SCW_SECRET_KEY}
    - export SCW_DEFAULT_PROJECT_ID=${SCW_DEFAULT_PROJECT_ID}
    - export AWS_ACCESS_KEY_ID=${SCW_ACCESS_KEY}
    - export AWS_SECRET_ACCESS_KEY=${SCW_SECRET_KEY}
  script:
    - cd infra/live/production
    - terragrunt run-all plan --terragrunt-non-interactive 2>&1 | tee ${CI_PROJECT_DIR}/plan-output.txt
    - echo "Plan completed — review output, then trigger infra-apply manually"
  artifacts:
    paths:
      - plan-output.txt
    when: always
    expire_in: 7 days

# --------------------------------------------------------------------------
# IaC: Terragrunt apply (auto on main push, manual from web/api)
# --------------------------------------------------------------------------
infra-apply:
  stage: deploy
  image: ${INFRA_RUNNER_IMAGE}
  tags:
    - kapsule
    - docker
  needs:
    - job: infra-plan
      optional: true
  rules:
    - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
      changes:
        - infra/**
    - if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
      when: manual
  before_script:
    - export SCW_ACCESS_KEY=${SCW_ACCESS_KEY}
    - export SCW_SECRET_KEY=${SCW_SECRET_KEY}
    - export SCW_DEFAULT_PROJECT_ID=${SCW_DEFAULT_PROJECT_ID}
    - export AWS_ACCESS_KEY_ID=${SCW_ACCESS_KEY}
    - export AWS_SECRET_ACCESS_KEY=${SCW_SECRET_KEY}
  script:
    - cd infra/live/production
    - terragrunt run-all apply --terragrunt-non-interactive -auto-approve
  environment:
    name: production/infrastructure

# --------------------------------------------------------------------------
# IaC: Weekly drift detection (scheduled pipeline, gitlab pool)
# --------------------------------------------------------------------------
infra-drift-check:
  stage: test
  image: ${INFRA_RUNNER_IMAGE}
  tags:
    - kapsule
    - docker
  needs: []
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule" && $DRIFT_CHECK == "true"
  before_script:
    - export SCW_ACCESS_KEY=${SCW_ACCESS_KEY}
    - export SCW_SECRET_KEY=${SCW_SECRET_KEY}
    - export SCW_DEFAULT_PROJECT_ID=${SCW_DEFAULT_PROJECT_ID}
    - export AWS_ACCESS_KEY_ID=${SCW_ACCESS_KEY}
    - export AWS_SECRET_ACCESS_KEY=${SCW_SECRET_KEY}
  script:
    - cd infra/live/production
    - |
      terragrunt run-all plan -detailed-exitcode --terragrunt-non-interactive \
        2>&1 | tee ${CI_PROJECT_DIR}/drift-output.txt; EXIT_CODE=$?
      if [ "$EXIT_CODE" -eq 2 ]; then
        echo "DRIFT DETECTED — creating GitLab issue"
        glab issue create \
          --title "IaC Drift Detected ($(date +%Y-%m-%d))" \
          --description "$(tail -100 ${CI_PROJECT_DIR}/drift-output.txt)" \
          --label "infrastructure,drift"
        exit 1
      elif [ "$EXIT_CODE" -ne 0 ]; then
        exit $EXIT_CODE
      fi
      echo "No drift detected"
  artifacts:
    paths:
      - drift-output.txt
    when: always
    expire_in: 7 days

# --------------------------------------------------------------------------
# Stage 4: Deploy base templates
# --------------------------------------------------------------------------
# Shared kubeconfig setup for all deploy jobs
.deploy-base:
  stage: deploy
  image: ${INFRA_RUNNER_IMAGE}
  tags:
    - kapsule
    - docker
  variables:
    KUBERNETES_CPU_REQUEST: "200m"
    KUBERNETES_CPU_LIMIT: "500m"
    KUBERNETES_MEMORY_REQUEST: "256Mi"
    KUBERNETES_MEMORY_LIMIT: "512Mi"
  before_script:
    - which kubectl || (curl -fsSL "https://dl.k8s.io/release/v1.31.4/bin/linux/amd64/kubectl" -o /usr/local/bin/kubectl && chmod +x /usr/local/bin/kubectl)
    - export SCW_ACCESS_KEY=${SCW_ACCESS_KEY}
    - export SCW_SECRET_KEY=${SCW_SECRET_KEY}
    - export SCW_DEFAULT_PROJECT_ID=${SCW_DEFAULT_PROJECT_ID}
    - export SCW_DEFAULT_ORGANIZATION_ID=${SCW_DEFAULT_ORGANIZATION_ID:-${SCW_DEFAULT_PROJECT_ID}}
    - mkdir -p ~/.kube
    - echo "$KUBECONFIG_B64" | base64 -d > ~/.kube/config
    - chmod 600 ~/.kube/config
    - sed -i 's|https://.*\.api\.k8s\.fr-par\.scw\.cloud:6443|https://kubernetes.default.svc|g' ~/.kube/config
    - kubectl cluster-info
  environment:
    name: production
    kubernetes:
      namespace: foxhunt

# Per-service deploy: upload binary → apply manifest → rollout restart
# Each service deploys IMMEDIATELY after its compile job finishes.
.deploy-service-base:
  extends: .deploy-base
  script:
    - export AWS_ACCESS_KEY_ID="${MINIO_ACCESS_KEY}"
    - export AWS_SECRET_ACCESS_KEY="${MINIO_SECRET_KEY}"
    - S3="aws s3 --endpoint-url http://minio.foxhunt.svc.cluster.local:9000"
    - echo "Deploying ${SERVICE_PACKAGE}..."
    - $S3 cp build-out/services/${SERVICE_PACKAGE} s3://foxhunt-binaries/services/${SERVICE_PACKAGE}
    - kubectl apply -f infra/k8s/services/${SERVICE_PACKAGE}.yaml
    - kubectl -n foxhunt rollout restart deployment/${SERVICE_PACKAGE}
    - kubectl -n foxhunt rollout status deployment/${SERVICE_PACKAGE} --timeout=300s
    - echo "Deployed ${SERVICE_PACKAGE}"

# --------------------------------------------------------------------------
# Stage 4a: Per-service deploy (fires as each compile finishes)
# Rules use YAML anchors shared with compile jobs — ensures compile+deploy
# are created/skipped together.
# --------------------------------------------------------------------------
deploy-api:
  extends: .deploy-service-base
  needs: [{job: compile-api, artifacts: true}]
  variables:
    SERVICE_PACKAGE: api
  rules: *rules-api

deploy-trading-service:
  extends: .deploy-service-base
  needs: [{job: compile-trading-service, artifacts: true}]
  variables:
    SERVICE_PACKAGE: trading-service
  rules: *rules-trading-service

deploy-ml-training-service:
  extends: .deploy-service-base
  needs: [{job: compile-ml-training-service, artifacts: true}]
  variables:
    SERVICE_PACKAGE: ml-training-service
  rules: *rules-ml-training-service

deploy-backtesting-service:
  extends: .deploy-service-base
  needs: [{job: compile-backtesting-service, artifacts: true}]
  variables:
    SERVICE_PACKAGE: backtesting-service
  rules: *rules-backtesting-service

deploy-trading-agent-service:
  extends: .deploy-service-base
  needs: [{job: compile-trading-agent-service, artifacts: true}]
  variables:
    SERVICE_PACKAGE: trading-agent-service
  rules: *rules-trading-agent-service

deploy-broker-gateway:
  extends: .deploy-service-base
  needs: [{job: compile-broker-gateway, artifacts: true}]
  variables:
    SERVICE_PACKAGE: broker-gateway
  rules: *rules-broker-gateway

deploy-data-acquisition-service:
  extends: .deploy-service-base
  needs: [{job: compile-data-acquisition-service, artifacts: true}]
  variables:
    SERVICE_PACKAGE: data-acquisition-service
  rules: *rules-data-acquisition-service

# --------------------------------------------------------------------------
# Stage 4b: Deploy training binaries to MinIO
# --------------------------------------------------------------------------
deploy-training:
  extends: .deploy-base
  needs:
    - job: compile-training
      artifacts: true
  rules:
    - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
      changes:
        - Cargo.toml
        - Cargo.lock
        - crates/ml/**
        - crates/trading_engine/**
        - crates/common/**
        - crates/risk/**
        - crates/data/**
        - crates/config/**
        - crates/storage/**
        - services/training_uploader/**
    - if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
  script:
    - export AWS_ACCESS_KEY_ID="${MINIO_ACCESS_KEY}"
    - export AWS_SECRET_ACCESS_KEY="${MINIO_SECRET_KEY}"
    - S3="aws s3 --endpoint-url http://minio.foxhunt.svc.cluster.local:9000"
    - $S3 cp build-out/training/ s3://foxhunt-binaries/training/ --recursive
    - echo "Training binaries uploaded to MinIO"

# --------------------------------------------------------------------------
# Stage 4c: Deploy web dashboard assets to MinIO
# --------------------------------------------------------------------------
deploy-web-dashboard:
  extends: .deploy-base
  needs:
    - job: build-web-dashboard
      artifacts: true
  rules:
    - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
      changes:
        - web-dashboard/**
    - if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
  script:
    - export AWS_ACCESS_KEY_ID="${MINIO_ACCESS_KEY}"
    - export AWS_SECRET_ACCESS_KEY="${MINIO_SECRET_KEY}"
    - S3="aws s3 --endpoint-url http://minio.foxhunt.svc.cluster.local:9000"
    - $S3 cp web-dashboard/dist/ s3://foxhunt-binaries/static/ --recursive
    - echo "Web dashboard assets uploaded to MinIO"

# --------------------------------------------------------------------------
# Stage 4d: Deploy infrastructure (migrations, monitoring, dashboards, proxy)
# Runs on every main push — no compile dependency.
# --------------------------------------------------------------------------
deploy-infra:
  extends: .deploy-base
  needs: []
  rules:
    - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
    - if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
  script:
    - echo "Deploying infrastructure ${CI_COMMIT_SHA}..."
    # Apply pending database migrations
    - |
      echo "Applying database migrations..."
      DB_POD=$(kubectl -n foxhunt get pod -l app.kubernetes.io/name=postgres -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)
      if [ -n "$DB_POD" ]; then
        kubectl -n foxhunt exec "$DB_POD" -- psql -U foxhunt -d foxhunt -c \
          "CREATE TABLE IF NOT EXISTS _applied_migrations (name TEXT PRIMARY KEY, applied_at TIMESTAMPTZ DEFAULT now())" 2>/dev/null || true
        for f in $(ls migrations/*.sql | grep -v '\.down\.' | sort); do
          MIGRATION_NAME=$(basename "$f")
          ALREADY=$(kubectl -n foxhunt exec "$DB_POD" -- psql -U foxhunt -d foxhunt -tAc \
            "SELECT 1 FROM _applied_migrations WHERE name = '${MIGRATION_NAME}'" 2>/dev/null || true)
          if [ "$ALREADY" = "1" ]; then
            echo "  skip: $MIGRATION_NAME (already applied)"
          else
            echo "  apply: $MIGRATION_NAME"
            kubectl -n foxhunt exec -i "$DB_POD" -- psql -U foxhunt -d foxhunt < "$f"
            kubectl -n foxhunt exec "$DB_POD" -- psql -U foxhunt -d foxhunt -c \
              "INSERT INTO _applied_migrations (name) VALUES ('${MIGRATION_NAME}')"
          fi
        done
        echo "Migrations complete"
      else
        echo "WARNING: postgres pod not found, skipping migrations"
      fi
    # Deploy monitoring stack
    - kubectl apply -f infra/k8s/monitoring/loki.yaml -f infra/k8s/monitoring/tempo.yaml -f infra/k8s/monitoring/promtail.yaml -f infra/k8s/monitoring/node-exporter.yaml -f infra/k8s/monitoring/kube-state-metrics.yaml
    # Deploy Grafana dashboards
    - |
      PROM_UID="PBFA97CFB590B2093"
      LOKI_UID="P8E80F9AEF21F6940"
      TEMPO_UID="tempo"
      DASH_DIR="infra/k8s/monitoring/dashboards"
      TMP_DIR=$(mktemp -d)
      for f in "$DASH_DIR"/*.json; do
        python3 -c "
      import json, sys
      with open('$f') as fh:
          d = json.load(fh)
      d.pop('__inputs', None)
      d.pop('__requires', None)
      raw = json.dumps(d)
      raw = raw.replace('\${DS_PROMETHEUS}', '${PROM_UID}')
      raw = raw.replace('\${DS_LOKI}', '${LOKI_UID}')
      raw = raw.replace('\${DS_TEMPO}', '${TEMPO_UID}')
      print(raw)
      " > "$TMP_DIR/$(basename "$f")"
      done
      apply_dashboard_cm() {
        cm_name="$1"; folder="$2"; shift 2
        args=""
        for df in "$@"; do
          [ -f "$TMP_DIR/$df" ] && args="$args --from-file=$df=$TMP_DIR/$df"
        done
        if [ -n "$args" ]; then
          kubectl create configmap "$cm_name" -n foxhunt $args \
            --dry-run=client -o yaml | \
            kubectl label -f - --dry-run=client -o yaml --local grafana_dashboard=1 | \
            kubectl annotate -f - --dry-run=client -o yaml --local "grafana_folder=$folder" | \
            kubectl apply -f -
          echo "  Applied ConfigMap: $cm_name (folder: $folder)"
        fi
      }
      apply_dashboard_cm grafana-dashboards-foxhunt Foxhunt \
        foxhunt-logs.json foxhunt-overview.json foxhunt-services.json \
        foxhunt-cockpit.json foxhunt-trading-cockpit.json foxhunt-training-cockpit.json foxhunt-traces.json
      apply_dashboard_cm grafana-dashboards-infra Infrastructure \
        cluster-overview.json gpu-overview.json foxhunt-gpu-training.json foxhunt-infrastructure.json
      apply_dashboard_cm grafana-dashboards-cicd "CI/CD" \
        foxhunt-ci-pipelines.json gitlab-services.json
      rm -rf "$TMP_DIR"
      echo "Dashboard ConfigMaps deployed"
    # Apply proxy + service manifests
    - kubectl apply -f infra/k8s/gitlab/tailscale-proxy.yaml
    - kubectl -n foxhunt rollout restart deployment/tailscale-gitlab-proxy
    - kubectl apply -f infra/k8s/services/
    - echo "Infrastructure deploy complete"