foxhunt/.gitlab-ci.yml

# GitLab CI/CD Pipeline for Foxhunt HFT Trading System
# This is an example configuration for teams using GitLab CI instead of GitHub Actions

stages:
  - test
  - security
  - build
  - deploy

variables:
  CARGO_HOME: ${CI_PROJECT_DIR}/.cargo
  RUST_BACKTRACE: "1"
  SQLX_OFFLINE: "true"
  REGISTRY: ${CI_REGISTRY}
  IMAGE_PREFIX: ${CI_REGISTRY_IMAGE}

# Cache Cargo dependencies
.rust_cache:
  cache:
    key: ${CI_COMMIT_REF_SLUG}
    paths:
      - .cargo/
      - target/

# Docker-in-Docker service for building images
.docker_service:
  services:
    - docker:24-dind
  variables:
    DOCKER_HOST: tcp://docker:2376
    DOCKER_TLS_CERTDIR: "/certs"
    DOCKER_TLS_VERIFY: 1
    DOCKER_CERT_PATH: "$DOCKER_TLS_CERTDIR/client"

# ================================
# Test Stage
# ================================

test:unit:
  stage: test
  image: rust:1.75-slim
  extends: .rust_cache
  services:
    - postgres:16-alpine
    - redis:7-alpine
    - vault:latest
  variables:
    POSTGRES_USER: foxhunt
    POSTGRES_PASSWORD: foxhunt_dev_password
    POSTGRES_DB: foxhunt
    DATABASE_URL: postgresql://foxhunt:foxhunt_dev_password@postgres:5432/foxhunt
    REDIS_URL: redis://redis:6379
    VAULT_ADDR: http://vault:8200
    VAULT_TOKEN: foxhunt-dev-root
    VAULT_DEV_ROOT_TOKEN_ID: foxhunt-dev-root
    JWT_SECRET: test_jwt_secret
    RUST_LOG: info
  before_script:
    - apt-get update && apt-get install -y libssl-dev pkg-config postgresql-client
    - cargo install sqlx-cli --no-default-features --features postgres
    - cargo install cargo-llvm-cov
    - cargo sqlx migrate run
  script:
    - cargo fmt --all -- --check
    - cargo clippy --workspace --all-targets -- -D warnings
    - cargo test --workspace --lib --bins
    - cargo test --workspace --test '*' -- --test-threads=1
  timeout: 30m
  artifacts:
    when: always
    paths:
      - target/nextest/
    expire_in: 30 days
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
    - if: '$CI_COMMIT_BRANCH == "main"'
    - if: '$CI_COMMIT_BRANCH == "develop"'
    - if: '$CI_COMMIT_BRANCH =~ /^feature\/.*/'

test:coverage:
  stage: test
  image: rust:1.75-slim
  extends: .rust_cache
  services:
    - postgres:16-alpine
    - redis:7-alpine
    - vault:latest
  variables:
    POSTGRES_USER: foxhunt
    POSTGRES_PASSWORD: foxhunt_dev_password
    POSTGRES_DB: foxhunt
    DATABASE_URL: postgresql://foxhunt:foxhunt_dev_password@postgres:5432/foxhunt
    REDIS_URL: redis://redis:6379
    VAULT_ADDR: http://vault:8200
    VAULT_TOKEN: foxhunt-dev-root
    VAULT_DEV_ROOT_TOKEN_ID: foxhunt-dev-root
    JWT_SECRET: test_jwt_secret
  before_script:
    - apt-get update && apt-get install -y libssl-dev pkg-config postgresql-client
    - cargo install sqlx-cli --no-default-features --features postgres
    - cargo install cargo-llvm-cov
    - cargo sqlx migrate run
  script:
    - cargo llvm-cov --workspace --lcov --output-path lcov.info
    - cargo llvm-cov --workspace --html --output-dir coverage_report
    - |
      COVERAGE=$(cargo llvm-cov --workspace --summary-only | grep -oP 'TOTAL.*\K[0-9.]+(?=%)')
      echo "Coverage: $COVERAGE%"
      if (( $(echo "$COVERAGE < 60.0" | bc -l) )); then
        echo "❌ Coverage $COVERAGE% below threshold 60%"
        exit 1
      fi
      echo "✅ Coverage $COVERAGE% meets threshold"
  coverage: '/TOTAL.*\s+(\d+\.\d+)%/'
  timeout: 30m
  artifacts:
    paths:
      - lcov.info
      - coverage_report/
    expire_in: 30 days
    reports:
      coverage_report:
        coverage_format: cobertura
        path: lcov.info
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
    - if: '$CI_COMMIT_BRANCH == "main"'
    - if: '$CI_COMMIT_BRANCH == "develop"'

# ================================
# Security Stage
# ================================

security:audit:
  stage: security
  image: rust:1.75-slim
  extends: .rust_cache
  before_script:
    - cargo install cargo-audit
    - cargo install cargo-geiger
  script:
    - cargo audit --deny warnings --ignore RUSTSEC-2020-0071
    - cargo geiger --all-features --output-format Json > geiger-report.json
  timeout: 10m
  artifacts:
    paths:
      - geiger-report.json
    expire_in: 30 days
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
    - if: '$CI_COMMIT_BRANCH == "main"'
    - if: '$CI_COMMIT_BRANCH == "develop"'

security:sast:
  stage: security
  image: returntocorp/semgrep:latest
  script:
    - semgrep --config=auto --sarif --output=sast-report.sarif .
  timeout: 10m
  artifacts:
    reports:
      sast: sast-report.sarif
    expire_in: 30 days
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
    - if: '$CI_COMMIT_BRANCH == "main"'

security:secrets:
  stage: security
  image: trufflesecurity/trufflehog:latest
  script:
    - trufflehog filesystem . --json --only-verified > secrets-scan.json
  timeout: 10m
  artifacts:
    paths:
      - secrets-scan.json
    expire_in: 30 days
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
    - if: '$CI_COMMIT_BRANCH == "main"'

# ================================
# Build Stage
# ================================

.build_docker_image:
  stage: build
  extends: .docker_service
  image: docker:24
  before_script:
    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
  script:
    - |
      # Build multi-platform image
      docker buildx create --use --name multiarch --driver docker-container
      docker buildx build \
        --platform linux/amd64,linux/arm64 \
        --file services/${SERVICE}/Dockerfile \
        --tag ${IMAGE_PREFIX}/${SERVICE}:${CI_COMMIT_SHA} \
        --tag ${IMAGE_PREFIX}/${SERVICE}:${CI_COMMIT_REF_SLUG} \
        --push \
        --build-arg SERVICE_NAME=${SERVICE} \
        --build-arg GIT_COMMIT=${CI_COMMIT_SHA} \
        --build-arg GIT_BRANCH=${CI_COMMIT_REF_NAME} \
        --build-arg BUILD_DATE=${CI_COMMIT_TIMESTAMP} \
        .
    - |
      # Scan image for vulnerabilities
      docker pull ${IMAGE_PREFIX}/${SERVICE}:${CI_COMMIT_SHA}
      docker run --rm \
        -v /var/run/docker.sock:/var/run/docker.sock \
        aquasec/trivy:latest \
        image --severity CRITICAL,HIGH \
        --exit-code 0 \
        --format json \
        --output trivy-${SERVICE}.json \
        ${IMAGE_PREFIX}/${SERVICE}:${CI_COMMIT_SHA}
  timeout: 30m
  artifacts:
    paths:
      - trivy-*.json
    expire_in: 30 days
  rules:
    - if: '$CI_COMMIT_BRANCH == "main"'
    - if: '$CI_COMMIT_BRANCH == "develop"'
    - if: '$CI_COMMIT_TAG'

build:api_gateway:
  extends: .build_docker_image
  variables:
    SERVICE: api_gateway

build:trading_service:
  extends: .build_docker_image
  variables:
    SERVICE: trading_service

build:backtesting_service:
  extends: .build_docker_image
  variables:
    SERVICE: backtesting_service

build:ml_training_service:
  extends: .build_docker_image
  variables:
    SERVICE: ml_training_service

build:tli:
  stage: build
  image: rust:1.75-slim
  extends: .rust_cache
  parallel:
    matrix:
      - TARGET: x86_64-unknown-linux-gnu
        PLATFORM: linux-amd64
      - TARGET: x86_64-apple-darwin
        PLATFORM: macos-amd64
      - TARGET: x86_64-pc-windows-msvc
        PLATFORM: windows-amd64
  before_script:
    - apt-get update && apt-get install -y libssl-dev pkg-config
    - rustup target add ${TARGET}
  script:
    - cargo build --release -p tli --target ${TARGET}
    - |
      if [ "${TARGET}" != "x86_64-pc-windows-msvc" ]; then
        strip target/${TARGET}/release/tli
      fi
  timeout: 20m
  artifacts:
    name: "tli-${PLATFORM}"
    paths:
      - target/${TARGET}/release/tli*
    expire_in: 30 days
  rules:
    - if: '$CI_COMMIT_TAG'

# ================================
# Deploy Stage
# ================================

.deploy:
  stage: deploy
  image: bitnami/kubectl:latest
  before_script:
    - kubectl config set-cluster k8s --server="${K8S_API_URL}"
    - kubectl config set-credentials gitlab --token="${K8S_TOKEN}"
    - kubectl config set-context default --cluster=k8s --user=gitlab --namespace=${NAMESPACE}
    - kubectl config use-context default
  script:
    - |
      # Update image tags
      cd k8s/overlays/${ENVIRONMENT}
      kubectl set image deployment/api-gateway api-gateway=${IMAGE_PREFIX}/api_gateway:${CI_COMMIT_SHA}
      kubectl set image deployment/trading-service trading-service=${IMAGE_PREFIX}/trading_service:${CI_COMMIT_SHA}
      kubectl set image deployment/backtesting-service backtesting-service=${IMAGE_PREFIX}/backtesting_service:${CI_COMMIT_SHA}
      kubectl set image deployment/ml-training-service ml-training-service=${IMAGE_PREFIX}/ml_training_service:${CI_COMMIT_SHA}

      # Wait for rollout
      kubectl rollout status deployment/api-gateway --timeout=10m
      kubectl rollout status deployment/trading-service --timeout=10m
      kubectl rollout status deployment/backtesting-service --timeout=10m
      kubectl rollout status deployment/ml-training-service --timeout=10m

      # Health checks
      for SERVICE in api-gateway trading-service backtesting-service ml-training-service; do
        POD=$(kubectl get pod -l app=${SERVICE} -o jsonpath='{.items[0].metadata.name}')
        kubectl exec ${POD} -- grpc_health_probe -addr=:50051
      done
  timeout: 30m

deploy:dev:
  extends: .deploy
  variables:
    ENVIRONMENT: dev
    NAMESPACE: foxhunt-dev
    K8S_API_URL: ${DEV_K8S_API_URL}
    K8S_TOKEN: ${DEV_K8S_TOKEN}
  environment:
    name: development
    url: https://dev.foxhunt.io
    on_stop: stop:dev
  rules:
    - if: '$CI_COMMIT_BRANCH == "develop"'

deploy:staging:
  extends: .deploy
  variables:
    ENVIRONMENT: staging
    NAMESPACE: foxhunt-staging
    K8S_API_URL: ${STAGING_K8S_API_URL}
    K8S_TOKEN: ${STAGING_K8S_TOKEN}
  environment:
    name: staging
    url: https://staging.foxhunt.io
    on_stop: stop:staging
  rules:
    - if: '$CI_COMMIT_BRANCH == "main"'
      when: manual

deploy:production:
  extends: .deploy
  variables:
    ENVIRONMENT: production
    NAMESPACE: foxhunt-prod
    K8S_API_URL: ${PROD_K8S_API_URL}
    K8S_TOKEN: ${PROD_K8S_TOKEN}
  environment:
    name: production
    url: https://foxhunt.io
    on_stop: stop:production
  rules:
    - if: '$CI_COMMIT_TAG'
      when: manual
  needs:
    - build:api_gateway
    - build:trading_service
    - build:backtesting_service
    - build:ml_training_service
    - security:audit

# ================================
# Cleanup
# ================================

.stop_environment:
  stage: deploy
  image: bitnami/kubectl:latest
  when: manual
  script:
    - kubectl scale deployment --all --replicas=0 -n ${NAMESPACE}

stop:dev:
  extends: .stop_environment
  variables:
    NAMESPACE: foxhunt-dev
  environment:
    name: development
    action: stop

stop:staging:
  extends: .stop_environment
  variables:
    NAMESPACE: foxhunt-staging
  environment:
    name: staging
    action: stop

stop:production:
  extends: .stop_environment
  variables:
    NAMESPACE: foxhunt-prod
  environment:
    name: production
    action: stop

# ================================
# Release
# ================================

release:
  stage: deploy
  image: registry.gitlab.com/gitlab-org/release-cli:latest
  script:
    - echo "Creating release for ${CI_COMMIT_TAG}"
  release:
    tag_name: ${CI_COMMIT_TAG}
    name: 'Release ${CI_COMMIT_TAG}'
    description: |
      ## Release ${CI_COMMIT_TAG}

      Deployed to production at ${CI_COMMIT_TIMESTAMP}

      ### Services
      - API Gateway: ${IMAGE_PREFIX}/api_gateway:${CI_COMMIT_TAG}
      - Trading Service: ${IMAGE_PREFIX}/trading_service:${CI_COMMIT_TAG}
      - Backtesting Service: ${IMAGE_PREFIX}/backtesting_service:${CI_COMMIT_TAG}
      - ML Training Service: ${IMAGE_PREFIX}/ml_training_service:${CI_COMMIT_TAG}
  rules:
    - if: '$CI_COMMIT_TAG'
  needs:
    - deploy:production