foxhunt/.gitlab-ci.yml

# GitLab CI/CD — Foxhunt
# Runs on gpu-training pool (H100-1-80G: 24 vCPU, 240GB, scale-to-zero)
# CI builder image: CUDA 12.4 + Rust 1.89 + protoc + sccache (hosted on Scaleway CR)
# Service images: pushed to Scaleway Container Registry (Kaniko)

stages:
  - prepare
  - check
  - test
  - build
  - deploy

variables:
  SQLX_OFFLINE: "true"
  CARGO_TERM_COLOR: always
  # H100 = compute capability 9.0 — tells bindgen_cuda to skip nvidia-smi detection
  CUDA_COMPUTE_CAP: "90"
  # Stable JWT secret for api_gateway tests (prevents env-var race between parallel tests)
  JWT_SECRET: "CiTestSecret_Kx7mP9nR2sW5vY8bC3fG6jH1kL4pQ7tZ0uN9dM5eV8xS2wT6yA4zB_64chars_min"
  # Training data on block storage PVC (mounted at /mnt/training-data by runner)
  FOXHUNT_DATA_DIR: "/mnt/training-data/futures-baseline"
  SCCACHE_BUCKET: $SCCACHE_BUCKET
  SCCACHE_ENDPOINT: $SCCACHE_ENDPOINT
  SCCACHE_S3_USE_SSL: "true"
  AWS_ACCESS_KEY_ID: $SCW_ACCESS_KEY
  AWS_SECRET_ACCESS_KEY: $SCW_SECRET_KEY
  RUSTC_WRAPPER: /usr/local/bin/sccache
  # CI builder image on Scaleway Container Registry (reachable by Kapsule nodes)
  CI_BUILDER_IMAGE: rg.fr-par.scw.cloud/foxhunt-ci/ci-builder:latest
  # Scaleway Container Registry for service images
  REGISTRY: rg.fr-par.scw.cloud/foxhunt-ci
  # IaC runner image on Scaleway Container Registry
  INFRA_RUNNER_IMAGE: rg.fr-par.scw.cloud/foxhunt-ci/infra-runner:latest

# --------------------------------------------------------------------------
# Stage 0: Build CI builder image → push to Scaleway CR
# --------------------------------------------------------------------------
build-ci-builder:
  stage: prepare
  image:
    name: gcr.io/kaniko-project/executor:debug
    entrypoint: [""]
  tags:
    - kapsule
    - docker
  rules:
    - if: $CI_PIPELINE_SOURCE == "push"
      changes:
        - infra/docker/Dockerfile.ci-builder
      when: on_success
    - when: manual
      allow_failure: true
  before_script:
    - mkdir -p /kaniko/.docker
    - echo "{\"auths\":{\"rg.fr-par.scw.cloud\":{\"username\":\"nologin\",\"password\":\"${SCW_SECRET_KEY}\"}}}" > /kaniko/.docker/config.json
  script:
    - /kaniko/executor
      --context "${CI_PROJECT_DIR}"
      --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.ci-builder"
      --destination "${CI_BUILDER_IMAGE}"

# --------------------------------------------------------------------------
# Stage 0b: Build devcontainer image → push to internal GitLab registry
# --------------------------------------------------------------------------
build-devcontainer:
  stage: prepare
  image:
    name: gcr.io/kaniko-project/executor:debug
    entrypoint: [""]
  tags:
    - kapsule
    - docker
  rules:
    - if: $CI_PIPELINE_SOURCE == "push"
      changes:
        - .devcontainer/**
      when: on_success
    - when: manual
      allow_failure: true
  before_script:
    - mkdir -p /kaniko/.docker
    - |
      echo "{\"auths\":{\"rg.fr-par.scw.cloud\":{\"username\":\"nologin\",\"password\":\"${SCW_SECRET_KEY}\"}}}" > /kaniko/.docker/config.json
  script:
    - /kaniko/executor
      --context "${CI_PROJECT_DIR}"
      --dockerfile "${CI_PROJECT_DIR}/.devcontainer/Dockerfile"
      --destination "rg.fr-par.scw.cloud/foxhunt-ci/devcontainer:${CI_COMMIT_SHA}"
      --destination "rg.fr-par.scw.cloud/foxhunt-ci/devcontainer:latest"

# --------------------------------------------------------------------------
# Stage 0c: Build infra-runner image → push to Scaleway CR
# --------------------------------------------------------------------------
build-infra-runner:
  stage: prepare
  image:
    name: gcr.io/kaniko-project/executor:debug
    entrypoint: [""]
  tags:
    - kapsule
    - docker
  rules:
    - if: $CI_PIPELINE_SOURCE == "push"
      changes:
        - infra/docker/Dockerfile.infra-runner
      when: on_success
    - when: manual
      allow_failure: true
  before_script:
    - mkdir -p /kaniko/.docker
    - |
      echo "{\"auths\":{\"rg.fr-par.scw.cloud\":{\"username\":\"nologin\",\"password\":\"${SCW_SECRET_KEY}\"}}}" > /kaniko/.docker/config.json
  script:
    - /kaniko/executor
      --context "${CI_PROJECT_DIR}"
      --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.infra-runner"
      --destination "${INFRA_RUNNER_IMAGE}"

# Base template for Rust jobs — pre-baked CI builder from Scaleway CR
# Image pre-exists in SCR; rebuild via build-ci-builder when Dockerfile changes
.rust-base:
  image: ${CI_BUILDER_IMAGE}
  tags:
    - kapsule
    - rust

# --------------------------------------------------------------------------
# Stage 1: cargo check + clippy
# --------------------------------------------------------------------------
check:
  extends: .rust-base
  stage: check
  needs: []
  script:
    - sccache --zero-stats || true
    - cargo check --workspace
    - cargo clippy --workspace -- -D warnings
    - sccache --show-stats || true

# --------------------------------------------------------------------------
# Stage 2: tests
# --------------------------------------------------------------------------
test:
  extends: .rust-base
  stage: test
  needs: [check]
  services:
    - name: redis:7-alpine
      alias: redis
  script:
    # Remove CUDA stubs from LD_LIBRARY_PATH — nvidia RuntimeClass provides real libcuda.so
    - export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//')
    - nvidia-smi  # verify GPU access
    # Wait for Redis sidecar (K8s executor doesn't wait for service readiness)
    - |
      for i in $(seq 1 30); do
        if bash -c 'echo PING > /dev/tcp/localhost/6379' 2>/dev/null; then
          echo "Redis ready"; break
        fi
        echo "Waiting for Redis... ($i/30)"; sleep 1
      done
    - cargo test --workspace --lib -- --skip model_loader::tests

# --------------------------------------------------------------------------
# Stage 3: Build + push service images (main only, Kaniko → Scaleway CR)
# --------------------------------------------------------------------------
.kaniko-base:
  stage: build
  needs: [test]
  image:
    name: gcr.io/kaniko-project/executor:debug
    entrypoint: [""]
  tags:
    - kapsule
    - docker
  rules:
    - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
  before_script:
    - mkdir -p /kaniko/.docker
    - echo "{\"auths\":{\"rg.fr-par.scw.cloud\":{\"username\":\"nologin\",\"password\":\"${SCW_SECRET_KEY}\"}}}" > /kaniko/.docker/config.json

build-trading-service:
  extends: .kaniko-base
  script:
    - /kaniko/executor
      --context "${CI_PROJECT_DIR}"
      --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.service"
      --build-arg SERVICE=trading_service
      --build-arg SCCACHE_BUCKET=${SCCACHE_BUCKET}
      --build-arg AWS_ACCESS_KEY_ID=${SCW_ACCESS_KEY}
      --build-arg AWS_SECRET_ACCESS_KEY=${SCW_SECRET_KEY}
      --build-arg SCCACHE_ENDPOINT=${SCCACHE_ENDPOINT}
      --destination "${REGISTRY}/trading_service:${CI_COMMIT_SHA}"
      --destination "${REGISTRY}/trading_service:latest"

build-api-gateway:
  extends: .kaniko-base
  script:
    - /kaniko/executor
      --context "${CI_PROJECT_DIR}"
      --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.service"
      --build-arg SERVICE=api_gateway
      --build-arg SCCACHE_BUCKET=${SCCACHE_BUCKET}
      --build-arg AWS_ACCESS_KEY_ID=${SCW_ACCESS_KEY}
      --build-arg AWS_SECRET_ACCESS_KEY=${SCW_SECRET_KEY}
      --build-arg SCCACHE_ENDPOINT=${SCCACHE_ENDPOINT}
      --destination "${REGISTRY}/api_gateway:${CI_COMMIT_SHA}"
      --destination "${REGISTRY}/api_gateway:latest"

build-broker-gateway:
  extends: .kaniko-base
  script:
    - /kaniko/executor
      --context "${CI_PROJECT_DIR}"
      --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.service"
      --build-arg SERVICE=broker_gateway_service
      --build-arg SCCACHE_BUCKET=${SCCACHE_BUCKET}
      --build-arg AWS_ACCESS_KEY_ID=${SCW_ACCESS_KEY}
      --build-arg AWS_SECRET_ACCESS_KEY=${SCW_SECRET_KEY}
      --build-arg SCCACHE_ENDPOINT=${SCCACHE_ENDPOINT}
      --destination "${REGISTRY}/broker_gateway_service:${CI_COMMIT_SHA}"
      --destination "${REGISTRY}/broker_gateway_service:latest"

build-ml-training:
  extends: .kaniko-base
  script:
    - /kaniko/executor
      --context "${CI_PROJECT_DIR}"
      --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.service"
      --build-arg SERVICE=ml_training_service
      --build-arg SCCACHE_BUCKET=${SCCACHE_BUCKET}
      --build-arg AWS_ACCESS_KEY_ID=${SCW_ACCESS_KEY}
      --build-arg AWS_SECRET_ACCESS_KEY=${SCW_SECRET_KEY}
      --build-arg SCCACHE_ENDPOINT=${SCCACHE_ENDPOINT}
      --destination "${REGISTRY}/ml_training_service:${CI_COMMIT_SHA}"
      --destination "${REGISTRY}/ml_training_service:latest"

build-backtesting:
  extends: .kaniko-base
  script:
    - /kaniko/executor
      --context "${CI_PROJECT_DIR}"
      --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.service"
      --build-arg SERVICE=backtesting_service
      --build-arg SCCACHE_BUCKET=${SCCACHE_BUCKET}
      --build-arg AWS_ACCESS_KEY_ID=${SCW_ACCESS_KEY}
      --build-arg AWS_SECRET_ACCESS_KEY=${SCW_SECRET_KEY}
      --build-arg SCCACHE_ENDPOINT=${SCCACHE_ENDPOINT}
      --destination "${REGISTRY}/backtesting_service:${CI_COMMIT_SHA}"
      --destination "${REGISTRY}/backtesting_service:latest"

build-trading-agent:
  extends: .kaniko-base
  script:
    - /kaniko/executor
      --context "${CI_PROJECT_DIR}"
      --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.service"
      --build-arg SERVICE=trading_agent_service
      --build-arg SCCACHE_BUCKET=${SCCACHE_BUCKET}
      --build-arg AWS_ACCESS_KEY_ID=${SCW_ACCESS_KEY}
      --build-arg AWS_SECRET_ACCESS_KEY=${SCW_SECRET_KEY}
      --build-arg SCCACHE_ENDPOINT=${SCCACHE_ENDPOINT}
      --destination "${REGISTRY}/trading_agent_service:${CI_COMMIT_SHA}"
      --destination "${REGISTRY}/trading_agent_service:latest"

build-data-acquisition:
  extends: .kaniko-base
  script:
    - /kaniko/executor
      --context "${CI_PROJECT_DIR}"
      --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.service"
      --build-arg SERVICE=data_acquisition_service
      --build-arg SCCACHE_BUCKET=${SCCACHE_BUCKET}
      --build-arg AWS_ACCESS_KEY_ID=${SCW_ACCESS_KEY}
      --build-arg AWS_SECRET_ACCESS_KEY=${SCW_SECRET_KEY}
      --build-arg SCCACHE_ENDPOINT=${SCCACHE_ENDPOINT}
      --destination "${REGISTRY}/data_acquisition_service:${CI_COMMIT_SHA}"
      --destination "${REGISTRY}/data_acquisition_service:latest"

build-web-gateway:
  extends: .kaniko-base
  script:
    - /kaniko/executor
      --context "${CI_PROJECT_DIR}"
      --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.web-gateway"
      --destination "${REGISTRY}/web-gateway:${CI_COMMIT_SHA}"
      --destination "${REGISTRY}/web-gateway:latest"

build-training:
  extends: .kaniko-base
  script:
    - /kaniko/executor
      --context "${CI_PROJECT_DIR}"
      --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.training"
      --build-arg SCCACHE_BUCKET=${SCCACHE_BUCKET}
      --build-arg AWS_ACCESS_KEY_ID=${SCW_ACCESS_KEY}
      --build-arg AWS_SECRET_ACCESS_KEY=${SCW_SECRET_KEY}
      --build-arg SCCACHE_ENDPOINT=${SCCACHE_ENDPOINT}
      --destination "${REGISTRY}/training:${CI_COMMIT_SHA}"
      --destination "${REGISTRY}/training:latest"

# --------------------------------------------------------------------------
# IaC: Terragrunt plan on MR (runs on gitlab pool)
# --------------------------------------------------------------------------
infra-plan:
  stage: check
  image: ${INFRA_RUNNER_IMAGE}
  tags:
    - kapsule
  needs: []
  rules:
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
      changes:
        - infra/**
  before_script:
    - export SCW_ACCESS_KEY=${SCW_ACCESS_KEY}
    - export SCW_SECRET_KEY=${SCW_SECRET_KEY}
    - export SCW_DEFAULT_PROJECT_ID=${SCW_DEFAULT_PROJECT_ID}
    - export AWS_ACCESS_KEY_ID=${SCW_ACCESS_KEY}
    - export AWS_SECRET_ACCESS_KEY=${SCW_SECRET_KEY}
  script:
    - cd infra/live/production
    - terragrunt run-all plan --terragrunt-non-interactive 2>&1 | tee ${CI_PROJECT_DIR}/plan-output.txt
    - echo "Plan completed successfully"
  artifacts:
    paths:
      - plan-output.txt
    when: always
    expire_in: 7 days

# --------------------------------------------------------------------------
# IaC: Terragrunt apply on merge to main (runs on gitlab pool)
# --------------------------------------------------------------------------
infra-apply:
  stage: deploy
  image: ${INFRA_RUNNER_IMAGE}
  tags:
    - kapsule
  needs: []
  rules:
    - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
      changes:
        - infra/**
  before_script:
    - export SCW_ACCESS_KEY=${SCW_ACCESS_KEY}
    - export SCW_SECRET_KEY=${SCW_SECRET_KEY}
    - export SCW_DEFAULT_PROJECT_ID=${SCW_DEFAULT_PROJECT_ID}
    - export AWS_ACCESS_KEY_ID=${SCW_ACCESS_KEY}
    - export AWS_SECRET_ACCESS_KEY=${SCW_SECRET_KEY}
  script:
    - cd infra/live/production
    - terragrunt run-all apply --terragrunt-non-interactive -auto-approve
  environment:
    name: production/infrastructure

# --------------------------------------------------------------------------
# IaC: Weekly drift detection (scheduled pipeline, gitlab pool)
# --------------------------------------------------------------------------
infra-drift-check:
  stage: check
  image: ${INFRA_RUNNER_IMAGE}
  tags:
    - kapsule
  needs: []
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule" && $DRIFT_CHECK == "true"
  before_script:
    - export SCW_ACCESS_KEY=${SCW_ACCESS_KEY}
    - export SCW_SECRET_KEY=${SCW_SECRET_KEY}
    - export SCW_DEFAULT_PROJECT_ID=${SCW_DEFAULT_PROJECT_ID}
    - export AWS_ACCESS_KEY_ID=${SCW_ACCESS_KEY}
    - export AWS_SECRET_ACCESS_KEY=${SCW_SECRET_KEY}
  script:
    - cd infra/live/production
    - |
      terragrunt run-all plan -detailed-exitcode --terragrunt-non-interactive \
        2>&1 | tee ${CI_PROJECT_DIR}/drift-output.txt; EXIT_CODE=$?
      if [ "$EXIT_CODE" -eq 2 ]; then
        echo "DRIFT DETECTED — creating GitLab issue"
        glab issue create \
          --title "IaC Drift Detected ($(date +%Y-%m-%d))" \
          --description "$(tail -100 ${CI_PROJECT_DIR}/drift-output.txt)" \
          --label "infrastructure,drift"
        exit 1
      elif [ "$EXIT_CODE" -ne 0 ]; then
        exit $EXIT_CODE
      fi
      echo "No drift detected"
  artifacts:
    paths:
      - drift-output.txt
    when: always
    expire_in: 7 days

# --------------------------------------------------------------------------
# Stage 4: Deploy to Kapsule (main only)
# --------------------------------------------------------------------------
deploy:
  stage: deploy
  image: bitnami/kubectl:latest
  tags:
    - kapsule
  needs:
    - build-trading-service
    - build-api-gateway
    - build-broker-gateway
    - build-ml-training
    - build-backtesting
    - build-trading-agent
    - build-data-acquisition
    - build-web-gateway
    - build-training
  rules:
    - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
  before_script:
    - mkdir -p ~/.kube
    - echo "$KUBECONFIG_B64" | base64 -d > ~/.kube/config
    - chmod 600 ~/.kube/config
  environment:
    name: production
    kubernetes:
      namespace: foxhunt
  script:
    # Update images to this commit's build
    - |
      for svc in trading_service api_gateway broker_gateway_service ml_training_service backtesting_service trading_agent_service data_acquisition_service; do
        kubectl -n foxhunt set image deployment/${svc} ${svc}=${REGISTRY}/${svc}:${CI_COMMIT_SHA} || true
      done
      kubectl -n foxhunt set image deployment/web-gateway web-gateway=${REGISTRY}/web-gateway:${CI_COMMIT_SHA} || true