diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7350cec26..77547c5ed 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,7 +1,7 @@ # GitLab CI/CD — Foxhunt # Runs on gpu-training pool (H100-1-80G: 24 vCPU, 240GB, scale-to-zero) # CI builder image: CUDA 12.4 + Rust 1.89 + protoc + sccache (hosted on Scaleway CR) -# Service images: pushed to internal GitLab registry (Kaniko) +# Service images: pushed to Scaleway Container Registry (Kaniko) stages: - prepare @@ -27,9 +27,8 @@ variables: RUSTC_WRAPPER: /usr/local/bin/sccache # CI builder image on Scaleway Container Registry (reachable by Kapsule nodes) CI_BUILDER_IMAGE: rg.fr-par.scw.cloud/foxhunt-ci/ci-builder:latest - # Internal GitLab registry (HTTP, reachable from pods only — for service images) - INTERNAL_REGISTRY: gitlab-registry.foxhunt.svc.cluster.local:5000 - REGISTRY: ${INTERNAL_REGISTRY}/root/foxhunt + # Scaleway Container Registry for service images + REGISTRY: rg.fr-par.scw.cloud/foxhunt-ci # -------------------------------------------------------------------------- # Stage 0: Build CI builder image → push to Scaleway CR @@ -104,7 +103,7 @@ test: - cargo test --workspace --lib -- --skip model_loader::tests # -------------------------------------------------------------------------- -# Stage 3: Build + push service images (main only, Kaniko → GitLab registry) +# Stage 3: Build + push service images (main only, Kaniko → Scaleway CR) # -------------------------------------------------------------------------- .kaniko-base: stage: build @@ -119,7 +118,7 @@ test: - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push" before_script: - mkdir -p /kaniko/.docker - - echo "{\"auths\":{\"${INTERNAL_REGISTRY}\":{\"username\":\"${CI_REGISTRY_USER}\",\"password\":\"${CI_REGISTRY_PASSWORD}\"}}}" > /kaniko/.docker/config.json + - echo "{\"auths\":{\"rg.fr-par.scw.cloud\":{\"username\":\"nologin\",\"password\":\"${SCW_SECRET_KEY}\"}}}" > /kaniko/.docker/config.json build-trading-service: extends: .kaniko-base @@ -127,7 +126,7 @@ build-trading-service: - /kaniko/executor --context "${CI_PROJECT_DIR}" --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.service" - --insecure-registry "${INTERNAL_REGISTRY}" + --build-arg SERVICE=trading_service --build-arg SCCACHE_BUCKET=${SCCACHE_BUCKET} --build-arg AWS_ACCESS_KEY_ID=${SCW_ACCESS_KEY} @@ -142,7 +141,7 @@ build-api-gateway: - /kaniko/executor --context "${CI_PROJECT_DIR}" --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.service" - --insecure-registry "${INTERNAL_REGISTRY}" + --build-arg SERVICE=api_gateway --build-arg SCCACHE_BUCKET=${SCCACHE_BUCKET} --build-arg AWS_ACCESS_KEY_ID=${SCW_ACCESS_KEY} @@ -157,7 +156,7 @@ build-broker-gateway: - /kaniko/executor --context "${CI_PROJECT_DIR}" --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.service" - --insecure-registry "${INTERNAL_REGISTRY}" + --build-arg SERVICE=broker_gateway_service --build-arg SCCACHE_BUCKET=${SCCACHE_BUCKET} --build-arg AWS_ACCESS_KEY_ID=${SCW_ACCESS_KEY} @@ -172,7 +171,7 @@ build-ml-training: - /kaniko/executor --context "${CI_PROJECT_DIR}" --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.service" - --insecure-registry "${INTERNAL_REGISTRY}" + --build-arg SERVICE=ml_training_service --build-arg SCCACHE_BUCKET=${SCCACHE_BUCKET} --build-arg AWS_ACCESS_KEY_ID=${SCW_ACCESS_KEY} @@ -187,7 +186,7 @@ build-backtesting: - /kaniko/executor --context "${CI_PROJECT_DIR}" --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.service" - --insecure-registry "${INTERNAL_REGISTRY}" + --build-arg SERVICE=backtesting_service --build-arg SCCACHE_BUCKET=${SCCACHE_BUCKET} --build-arg AWS_ACCESS_KEY_ID=${SCW_ACCESS_KEY} @@ -202,7 +201,7 @@ build-trading-agent: - /kaniko/executor --context "${CI_PROJECT_DIR}" --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.service" - --insecure-registry "${INTERNAL_REGISTRY}" + --build-arg SERVICE=trading_agent_service --build-arg SCCACHE_BUCKET=${SCCACHE_BUCKET} --build-arg AWS_ACCESS_KEY_ID=${SCW_ACCESS_KEY} @@ -217,7 +216,7 @@ build-data-acquisition: - /kaniko/executor --context "${CI_PROJECT_DIR}" --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.service" - --insecure-registry "${INTERNAL_REGISTRY}" + --build-arg SERVICE=data_acquisition_service --build-arg SCCACHE_BUCKET=${SCCACHE_BUCKET} --build-arg AWS_ACCESS_KEY_ID=${SCW_ACCESS_KEY} @@ -232,7 +231,7 @@ build-web-gateway: - /kaniko/executor --context "${CI_PROJECT_DIR}" --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.web-gateway" - --insecure-registry "${INTERNAL_REGISTRY}" + --destination "${REGISTRY}/web-gateway:${CI_COMMIT_SHA}" --destination "${REGISTRY}/web-gateway:latest" @@ -242,7 +241,7 @@ build-training: - /kaniko/executor --context "${CI_PROJECT_DIR}" --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.training" - --insecure-registry "${INTERNAL_REGISTRY}" + --build-arg SCCACHE_BUCKET=${SCCACHE_BUCKET} --build-arg AWS_ACCESS_KEY_ID=${SCW_ACCESS_KEY} --build-arg AWS_SECRET_ACCESS_KEY=${SCW_SECRET_KEY} diff --git a/infra/docker/Dockerfile.service b/infra/docker/Dockerfile.service index 05a83a94e..3da94001c 100644 --- a/infra/docker/Dockerfile.service +++ b/infra/docker/Dockerfile.service @@ -49,27 +49,11 @@ WORKDIR /build COPY Cargo.toml Cargo.lock ./ COPY .sqlx ./.sqlx -# Copy all workspace crate directories -COPY trading_engine ./trading_engine -COPY risk ./risk -COPY risk-data ./risk-data -COPY trading-data ./trading-data -COPY fxt ./fxt -COPY ml ./ml -COPY ml-data ./ml-data -COPY data ./data -COPY backtesting ./backtesting -COPY adaptive-strategy ./adaptive-strategy -COPY common ./common -COPY storage ./storage -COPY model_loader ./model_loader -COPY market-data ./market-data -COPY database ./database -COPY config ./config -COPY web-gateway ./web-gateway -COPY ctrader-openapi ./ctrader-openapi +# Copy workspace directories (post-restructure layout) +COPY crates ./crates +COPY bin ./bin COPY services ./services -COPY tests ./tests +COPY testing ./testing ENV SQLX_OFFLINE=true diff --git a/infra/docker/Dockerfile.training b/infra/docker/Dockerfile.training index 6231636f4..542b6a755 100644 --- a/infra/docker/Dockerfile.training +++ b/infra/docker/Dockerfile.training @@ -48,27 +48,11 @@ WORKDIR /build COPY Cargo.toml Cargo.lock ./ COPY .sqlx ./.sqlx -# Copy all workspace crate directories -COPY trading_engine ./trading_engine -COPY risk ./risk -COPY risk-data ./risk-data -COPY trading-data ./trading-data -COPY fxt ./fxt -COPY ml ./ml -COPY ml-data ./ml-data -COPY data ./data -COPY backtesting ./backtesting -COPY adaptive-strategy ./adaptive-strategy -COPY common ./common -COPY storage ./storage -COPY model_loader ./model_loader -COPY market-data ./market-data -COPY database ./database -COPY config ./config -COPY web-gateway ./web-gateway -COPY ctrader-openapi ./ctrader-openapi +# Copy workspace directories (post-restructure layout) +COPY crates ./crates +COPY bin ./bin COPY services ./services -COPY tests ./tests +COPY testing ./testing ENV SQLX_OFFLINE=true ENV CUDA_COMPUTE_CAP=90 diff --git a/infra/docker/Dockerfile.web-gateway b/infra/docker/Dockerfile.web-gateway index 06df7f216..63b7209c0 100644 --- a/infra/docker/Dockerfile.web-gateway +++ b/infra/docker/Dockerfile.web-gateway @@ -34,27 +34,11 @@ WORKDIR /build COPY Cargo.toml Cargo.lock ./ COPY .sqlx ./.sqlx -# Copy all workspace crate directories -COPY trading_engine ./trading_engine -COPY risk ./risk -COPY risk-data ./risk-data -COPY trading-data ./trading-data -COPY fxt ./fxt -COPY ml ./ml -COPY ml-data ./ml-data -COPY data ./data -COPY backtesting ./backtesting -COPY adaptive-strategy ./adaptive-strategy -COPY common ./common -COPY storage ./storage -COPY model_loader ./model_loader -COPY market-data ./market-data -COPY database ./database -COPY config ./config -COPY web-gateway ./web-gateway -COPY ctrader-openapi ./ctrader-openapi +# Copy workspace directories (post-restructure layout) +COPY crates ./crates +COPY bin ./bin COPY services ./services -COPY tests ./tests +COPY testing ./testing ENV SQLX_OFFLINE=true