diff --git a/infra/k8s/argo/ci-pipeline-template.yaml b/infra/k8s/argo/ci-pipeline-template.yaml index d13cb4722..dc05c0c2a 100644 --- a/infra/k8s/argo/ci-pipeline-template.yaml +++ b/infra/k8s/argo/ci-pipeline-template.yaml @@ -16,16 +16,6 @@ spec: app.kubernetes.io/part-of: foxhunt ttlStrategy: secondsAfterCompletion: 3600 - # Volumes for compile-training (called via templateRef — spec-level volumes don't carry over) - volumeClaimTemplates: - - metadata: - name: workspace - spec: - accessModes: [ReadWriteOnce] - resources: - requests: - storage: 20Gi - storageClassName: scw-bssd volumes: - name: sccache persistentVolumeClaim: @@ -41,102 +31,23 @@ spec: - name: commits-json value: "[]" templates: - # ── DAG: orchestrate compile tasks ── + # ── DAG: orchestrate pipeline ── - name: pipeline dag: tasks: - name: detect-changes template: detect-changes - - name: compile-training + - name: compile-all dependencies: [detect-changes] - templateRef: - name: compile-training - template: compile + template: compile-all arguments: parameters: - - name: commit-sha - value: "{{workflow.parameters.commit-sha}}" - - name: cuda-compute-cap - value: "89" - when: "{{tasks.detect-changes.outputs.parameters.training}} == true" - - - name: compile-api - dependencies: [detect-changes] - template: compile-service - arguments: - parameters: - - name: service-package - value: api - - name: commit-sha - value: "{{workflow.parameters.commit-sha}}" - when: "{{tasks.detect-changes.outputs.parameters.api}} == true" - - - name: compile-trading-service - dependencies: [detect-changes] - template: compile-service - arguments: - parameters: - - name: service-package - value: trading-service - - name: commit-sha - value: "{{workflow.parameters.commit-sha}}" - when: "{{tasks.detect-changes.outputs.parameters.trading-service}} == true" - - - name: compile-ml-training-service - dependencies: [detect-changes] - template: compile-service - arguments: - parameters: - - name: service-package - value: ml-training-service - - name: commit-sha - value: "{{workflow.parameters.commit-sha}}" - when: "{{tasks.detect-changes.outputs.parameters.ml-training-service}} == true" - - - name: compile-backtesting-service - dependencies: [detect-changes] - template: compile-service - arguments: - parameters: - - name: service-package - value: backtesting-service - - name: commit-sha - value: "{{workflow.parameters.commit-sha}}" - when: "{{tasks.detect-changes.outputs.parameters.backtesting-service}} == true" - - - name: compile-trading-agent-service - dependencies: [detect-changes] - template: compile-service - arguments: - parameters: - - name: service-package - value: trading-agent-service - - name: commit-sha - value: "{{workflow.parameters.commit-sha}}" - when: "{{tasks.detect-changes.outputs.parameters.trading-agent-service}} == true" - - - name: compile-broker-gateway - dependencies: [detect-changes] - template: compile-service - arguments: - parameters: - - name: service-package - value: broker-gateway - - name: commit-sha - value: "{{workflow.parameters.commit-sha}}" - when: "{{tasks.detect-changes.outputs.parameters.broker-gateway}} == true" - - - name: compile-data-acquisition-service - dependencies: [detect-changes] - template: compile-service - arguments: - parameters: - - name: service-package - value: data-acquisition-service - - name: commit-sha - value: "{{workflow.parameters.commit-sha}}" - when: "{{tasks.detect-changes.outputs.parameters.data-acquisition-service}} == true" + - name: needs-services + value: "{{tasks.detect-changes.outputs.parameters.needs-services}}" + - name: needs-training + value: "{{tasks.detect-changes.outputs.parameters.needs-training}}" + when: "{{tasks.detect-changes.outputs.parameters.needs-services}} == true || {{tasks.detect-changes.outputs.parameters.needs-training}} == true" - name: rebuild-ci-builder dependencies: [detect-changes] @@ -198,15 +109,10 @@ spec: value: foxhunt-training-runtime when: "{{tasks.detect-changes.outputs.parameters.docker-images}} == true" - # ── detect-changes: determine which services need rebuilding ── + # ── detect-changes: simplified to needs-compile + docker-images ── - name: detect-changes nodeSelector: - k8s.scaleway.com/pool-name: platform - tolerations: - - key: gitlab - operator: Equal - value: "true" - effect: NoSchedule + k8s.scaleway.com/pool-name: infra container: image: rg.fr-par.scw.cloud/foxhunt-ci/foxhunt-runtime:latest command: ["/bin/sh", "-c"] @@ -227,18 +133,17 @@ spec: #!/bin/sh set -e - # COMMITS_JSON is passed as env var to avoid shell escaping issues + # COMMITS_JSON is the full webhook payload: {"body":{"commits":[...]}, "header":{...}} + # Extract changed file paths from .body.commits array - # Extract all changed file paths from commits array if command -v jq >/dev/null 2>&1; then CHANGED_FILES=$(echo "$COMMITS_JSON" | jq -r ' - if type == "array" then - [.[].added[], .[].modified[], .[].removed[]] | unique | .[] - elif type == "string" then - (fromjson // []) | [.[].added[], .[].modified[], .[].removed[]] | unique | .[] - else - empty - end + # Handle full webhook payload with .body.commits + (if .body.commits then .body.commits + elif type == "array" then . + else [] + end) | + [.[].added[]?, .[].modified[]?, .[].removed[]?] | unique | .[] ' 2>/dev/null || echo "") else CHANGED_FILES="Cargo.toml" @@ -265,83 +170,69 @@ spec: echo "false" } - API=$(check_paths "services/api/ crates/common/ crates/config/ crates/trading_engine/ bin/fxt/ Cargo.toml Cargo.lock") - TRADING_SERVICE=$(check_paths "services/trading_service/ services/api/ crates/common/ crates/config/ crates/data/ crates/database/ crates/ml/ crates/risk/ crates/storage/ crates/trading_engine/ Cargo.toml Cargo.lock") - ML_TRAINING_SERVICE=$(check_paths "services/ml_training_service/ crates/common/ crates/config/ crates/data/ crates/ml/ crates/risk/ crates/storage/ crates/trading_engine/ Cargo.toml Cargo.lock") - BACKTESTING_SERVICE=$(check_paths "services/backtesting_service/ crates/common/ crates/config/ crates/data/ crates/ml/ crates/model_loader/ crates/risk/ crates/storage/ crates/trading_engine/ bin/fxt/ Cargo.toml Cargo.lock") - TRADING_AGENT_SERVICE=$(check_paths "services/trading_agent_service/ crates/common/ crates/config/ crates/ml/ crates/risk/ Cargo.toml Cargo.lock") - BROKER_GATEWAY=$(check_paths "services/broker_gateway_service/ crates/common/ crates/config/ crates/trading_engine/ vendor/ctrader-openapi/ Cargo.toml Cargo.lock") - DATA_ACQUISITION_SERVICE=$(check_paths "services/data_acquisition_service/ crates/common/ crates/config/ crates/storage/ Cargo.toml Cargo.lock") - TRAINING=$(check_paths "crates/ml/ crates/trading_engine/ crates/common/ crates/risk/ crates/data/ crates/config/ crates/storage/ services/training_uploader/ Cargo.toml Cargo.lock") + # Shared crates trigger both services and training + SHARED=$(check_paths "crates/common/ crates/config/ Cargo.toml Cargo.lock") + + # Service-specific paths + SVC_ONLY=$(check_paths "services/ bin/fxt/ crates/trading_engine/ crates/risk/ crates/data/ crates/storage/ crates/database/ crates/broker crates/ctrader") + + # Training-specific paths (ml crate also used by ml-training-service + trading-service) + ML_CHANGED=$(check_paths "crates/ml/") + + # Docker images DOCKER_IMAGES=$(check_paths "infra/docker/") + # Services rebuild when: shared crates, service code, or ml crate changes + if [ "$SHARED" = "true" ] || [ "$SVC_ONLY" = "true" ] || [ "$ML_CHANGED" = "true" ]; then + NEEDS_SERVICES="true" + else + NEEDS_SERVICES="false" + fi + + # Training rebuilds when: shared crates or ml crate changes + if [ "$SHARED" = "true" ] || [ "$ML_CHANGED" = "true" ]; then + NEEDS_TRAINING="true" + else + NEEDS_TRAINING="false" + fi + echo "=== Build decisions ===" - echo "api: $API" - echo "trading-service: $TRADING_SERVICE" - echo "ml-training-service: $ML_TRAINING_SERVICE" - echo "backtesting-service: $BACKTESTING_SERVICE" - echo "trading-agent-service: $TRADING_AGENT_SERVICE" - echo "broker-gateway: $BROKER_GATEWAY" - echo "data-acquisition-service: $DATA_ACQUISITION_SERVICE" - echo "training: $TRAINING" + echo "needs-services: $NEEDS_SERVICES" + echo "needs-training: $NEEDS_TRAINING" echo "docker-images: $DOCKER_IMAGES" echo "======================" mkdir -p /tmp/outputs - echo -n "$API" > /tmp/outputs/api - echo -n "$TRADING_SERVICE" > /tmp/outputs/trading-service - echo -n "$ML_TRAINING_SERVICE" > /tmp/outputs/ml-training-service - echo -n "$BACKTESTING_SERVICE" > /tmp/outputs/backtesting-service - echo -n "$TRADING_AGENT_SERVICE" > /tmp/outputs/trading-agent-service - echo -n "$BROKER_GATEWAY" > /tmp/outputs/broker-gateway - echo -n "$DATA_ACQUISITION_SERVICE" > /tmp/outputs/data-acquisition-service - echo -n "$TRAINING" > /tmp/outputs/training + echo -n "$NEEDS_SERVICES" > /tmp/outputs/needs-services + echo -n "$NEEDS_TRAINING" > /tmp/outputs/needs-training echo -n "$DOCKER_IMAGES" > /tmp/outputs/docker-images SCRIPT chmod +x /tmp/detect.sh /tmp/detect.sh outputs: parameters: - - name: api + - name: needs-services valueFrom: - path: /tmp/outputs/api - - name: trading-service + path: /tmp/outputs/needs-services + - name: needs-training valueFrom: - path: /tmp/outputs/trading-service - - name: ml-training-service - valueFrom: - path: /tmp/outputs/ml-training-service - - name: backtesting-service - valueFrom: - path: /tmp/outputs/backtesting-service - - name: trading-agent-service - valueFrom: - path: /tmp/outputs/trading-agent-service - - name: broker-gateway - valueFrom: - path: /tmp/outputs/broker-gateway - - name: data-acquisition-service - valueFrom: - path: /tmp/outputs/data-acquisition-service - - name: training - valueFrom: - path: /tmp/outputs/training + path: /tmp/outputs/needs-training - name: docker-images valueFrom: path: /tmp/outputs/docker-images - # ── compile-service: build and upload a single service binary ── - - name: compile-service + # ── compile-all: build and upload service + training binaries (conditionally) ── + - name: compile-all + inputs: + parameters: + - name: needs-services + - name: needs-training nodeSelector: k8s.scaleway.com/pool-name: ci-compile-cpu tolerations: - key: node.cilium.io/agent-not-ready operator: Exists effect: NoSchedule - inputs: - parameters: - - name: service-package - - name: commit-sha container: image: rg.fr-par.scw.cloud/foxhunt-ci/ci-builder:latest command: ["/bin/sh", "-c"] @@ -350,6 +241,10 @@ spec: value: "true" - name: CARGO_TERM_COLOR value: always + - name: NEEDS_SERVICES + value: "{{inputs.parameters.needs-services}}" + - name: NEEDS_TRAINING + value: "{{inputs.parameters.needs-training}}" - name: MINIO_ACCESS_KEY valueFrom: secretKeyRef: @@ -362,11 +257,11 @@ spec: key: secret-key resources: requests: - cpu: "4" - memory: 8Gi + cpu: "14" + memory: 32Gi limits: - cpu: "6" - memory: 16Gi + cpu: "30" + memory: 64Gi volumeMounts: - name: sccache mountPath: /mnt/sccache @@ -376,17 +271,15 @@ spec: args: - | set -e - SERVICE="{{inputs.parameters.service-package}}" - SHA="{{inputs.parameters.commit-sha}}" + SHA="{{workflow.parameters.commit-sha}}" - # Configure SSH for git clone + # ── Git checkout ── mkdir -p ~/.ssh cp /etc/git-ssh/ssh-privatekey ~/.ssh/id_ed25519 chmod 600 ~/.ssh/id_ed25519 printf 'Host *\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n' > ~/.ssh/config chmod 600 ~/.ssh/config - echo "=== Compiling $SERVICE at $SHA ===" REPO="ssh://git@gitlab-gitlab-shell.foxhunt.svc.cluster.local:2222/root/foxhunt.git" if [ "$SHA" = "HEAD" ]; then git clone --depth=1 "$REPO" /workspace/src @@ -399,35 +292,79 @@ spec: cd /workspace/src echo "Checked out $(git rev-parse --short HEAD)" - export SCCACHE_DIR="/mnt/sccache/cpu" - export RUSTC_WRAPPER=sccache - mkdir -p "$SCCACHE_DIR" - sccache --zero-stats || true - YEAR=$(date +%Y) MONTH=$(date +%m) export FOXHUNT_BUILD_VERSION="${YEAR}.${MONTH}.argo" + export RUSTC_WRAPPER=sccache - # Service name matches Cargo.toml package name (all use hyphens) - PKG="$SERVICE" + # ── Build services (skip if only training changed) ── + if [ "$NEEDS_SERVICES" = "true" ]; then + echo "=== Building service binaries ===" + export SCCACHE_DIR="/mnt/sccache/cpu" + mkdir -p "$SCCACHE_DIR" + sccache --zero-stats || true - cargo build --release -p "$PKG" - sccache --show-stats || true + cargo build --release \ + -p api -p trading-service -p ml-training-service \ + -p backtesting-service -p trading-agent-service \ + -p broker-gateway -p data-acquisition-service - mkdir -p /workspace/bin - cp "target/release/$PKG" /workspace/bin/ 2>/dev/null \ - || cp "target/release/${PKG//_/-}" /workspace/bin/ 2>/dev/null \ - || { echo "Binary not found for $PKG"; ls target/release/; exit 1; } - strip /workspace/bin/* + mkdir -p /workspace/bin/services + for pkg in api trading-service ml-training-service backtesting-service trading-agent-service broker-gateway data-acquisition-service; do + bin_name=$(echo "$pkg" | tr '-' '_') + cp "target/release/$pkg" "/workspace/bin/services/" 2>/dev/null \ + || cp "target/release/$bin_name" "/workspace/bin/services/" 2>/dev/null \ + || { echo "Binary not found for $pkg"; ls target/release/; exit 1; } + done + strip /workspace/bin/services/* - echo "=== Uploading $SERVICE binary to MinIO ===" - rclone copy /workspace/bin/ ":s3:foxhunt-binaries/services/" \ - --s3-provider=Minio \ - --s3-endpoint=https://minio.foxhunt.svc.cluster.local:9000 \ - --s3-access-key-id="${MINIO_ACCESS_KEY}" \ - --s3-secret-access-key="${MINIO_SECRET_KEY}" \ - --s3-no-check-bucket \ - --no-check-certificate + echo "=== Service binaries ===" + ls -lh /workspace/bin/services/ + sccache --show-stats || true + else + echo "=== Skipping service binaries (no service changes) ===" + fi - echo "=== $SERVICE compile + upload complete ===" - ls -lh /workspace/bin/ + # ── Build training binaries (skip if only services changed) ── + if [ "$NEEDS_TRAINING" = "true" ]; then + echo "=== Building training binaries (CUDA_COMPUTE_CAP={{workflow.parameters.cuda-compute-cap}}) ===" + export CUDA_COMPUTE_CAP={{workflow.parameters.cuda-compute-cap}} + export SCCACHE_DIR="/mnt/sccache/sm_{{workflow.parameters.cuda-compute-cap}}" + mkdir -p "$SCCACHE_DIR" + sccache --zero-stats || true + + cargo build --release -p ml --features ml/cuda \ + --example train_baseline_rl --example train_baseline_supervised \ + --example evaluate_baseline --example evaluate_supervised \ + --example hyperopt_baseline_rl --example hyperopt_baseline_supervised + + cargo build --release -p training_uploader + + mkdir -p /workspace/bin/training + for bin in train_baseline_rl train_baseline_supervised evaluate_baseline evaluate_supervised hyperopt_baseline_rl hyperopt_baseline_supervised; do + cp target/release/examples/$bin /workspace/bin/training/ + done + cp target/release/training_uploader /workspace/bin/training/ + strip /workspace/bin/training/* + + echo "=== Training binaries ===" + ls -lh /workspace/bin/training/ + sccache --show-stats || true + else + echo "=== Skipping training binaries (no ml changes) ===" + fi + + # ── Upload to MinIO (only what was built) ── + RCLONE_S3="--s3-provider=Minio --s3-endpoint=https://minio.foxhunt.svc.cluster.local:9000 --s3-access-key-id=${MINIO_ACCESS_KEY} --s3-secret-access-key=${MINIO_SECRET_KEY} --s3-no-check-bucket --no-check-certificate" + + if [ "$NEEDS_SERVICES" = "true" ]; then + echo "=== Uploading service binaries ===" + eval rclone copy /workspace/bin/services/ \":s3:foxhunt-binaries/services/\" $RCLONE_S3 + fi + + if [ "$NEEDS_TRAINING" = "true" ]; then + echo "=== Uploading training binaries ===" + eval rclone copy /workspace/bin/training/ \":s3:foxhunt-binaries/training/\" $RCLONE_S3 + fi + + echo "=== Done (services=$NEEDS_SERVICES, training=$NEEDS_TRAINING) ==="