From c06954f471f446b2fee439d2e3a6b1b83eb5a7f7 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Thu, 12 Mar 2026 09:12:40 +0100 Subject: [PATCH] feat(ci): add test-gate step + clean up detect-changes dead outputs Add clippy + cargo test quality gate to CI pipeline DAG. The test-gate runs on ci-compile-cpu nodes with persistent cargo cache PVC for fast incremental builds. Also simplifies detect-changes by removing 5 unused output parameters (service-packages, training-examples, deploy-list, needs-services, needs-training) and ~170 lines of dead shell logic. Co-Authored-By: Claude Opus 4.6 --- infra/k8s/argo/ci-pipeline-template.yaml | 306 ++++++++--------------- 1 file changed, 101 insertions(+), 205 deletions(-) diff --git a/infra/k8s/argo/ci-pipeline-template.yaml b/infra/k8s/argo/ci-pipeline-template.yaml index fb45778b5..2bc1b08ac 100644 --- a/infra/k8s/argo/ci-pipeline-template.yaml +++ b/infra/k8s/argo/ci-pipeline-template.yaml @@ -30,6 +30,9 @@ spec: items: - key: .dockerconfigjson path: config.json + - name: cargo-target-cpu + persistentVolumeClaim: + claimName: cargo-target-cpu arguments: parameters: - name: commit-sha @@ -111,6 +114,15 @@ spec: value: foxhunt-training-runtime when: "{{tasks.detect-changes.outputs.parameters.docker-images}} == true" + - name: test-gate + depends: "detect-changes && (rebuild-ci-builder-cpu.Succeeded || rebuild-ci-builder-cpu.Skipped)" + template: test-gate + arguments: + parameters: + - name: commit-sha + value: "{{workflow.parameters.commit-sha}}" + when: "{{tasks.detect-changes.outputs.parameters.needs-code}} == true" + - name: apply-argo-templates depends: "detect-changes.Succeeded && (rebuild-ci-builder-cpu.Succeeded || rebuild-ci-builder-cpu.Skipped)" template: apply-argo-templates @@ -129,7 +141,7 @@ spec: value: "{{workflow.parameters.commit-sha}}" when: "{{tasks.detect-changes.outputs.parameters.needs-infra}} == true" - # ── detect-changes: granular per-binary change detection ── + # ── detect-changes: classify changed files to gate downstream steps ── - name: detect-changes nodeSelector: k8s.scaleway.com/pool-name: platform @@ -182,238 +194,41 @@ spec: echo "false" } - # --- Dependency tiers --- - # Tier 0: workspace root — rebuilds EVERYTHING - SHARED=$(check_paths "Cargo.toml Cargo.lock crates/common/ crates/config/") - - # Tier 1: proto files used by services - PROTO=$(check_paths "bin/fxt/proto/") - - # Tier 2: domain crates shared by subsets of services - TRADING_ENGINE=$(check_paths "crates/trading_engine/") - RISK=$(check_paths "crates/risk/") - STORAGE=$(check_paths "crates/storage/ crates/database/") - BROKER=$(check_paths "crates/broker crates/ctrader") - DATA=$(check_paths "crates/data/") - - # Tier 3: ML crates — affect ml-training-service, trading-service, AND all training binaries - ML_CHANGED=$(check_paths "crates/ml/ crates/ml-core/ crates/ml-dqn/ crates/ml-ppo/ crates/ml-features/ crates/ml-supervised/ crates/ml-ensemble/ crates/ml-regime/ crates/ml-hyperopt/ crates/ml-data-validation/ crates/ml-labeling/ crates/ml-validation/ crates/ml-checkpoint/ crates/ml-risk/ crates/ml-backtesting/") - - # Tier 4: service-specific directories - SVC_API=$(check_paths "services/api/") - SVC_TRADING=$(check_paths "services/trading_service/") - SVC_ML_TRAIN=$(check_paths "services/ml_training_service/") - SVC_BACKTEST=$(check_paths "services/backtesting_service/") - SVC_AGENT=$(check_paths "services/trading_agent_service/") - SVC_BROKER=$(check_paths "services/broker_gateway/") - SVC_DATA=$(check_paths "services/data_acquisition_service/") - - # Tier 5: training-specific subdirectories within ml crate - ML_TRAINERS=$(check_paths "crates/ml/src/trainers/") - ML_HYPEROPT=$(check_paths "crates/ml/src/hyperopt/") - ML_EVAL=$(check_paths "crates/ml/src/evaluation/") - ML_EXAMPLES=$(check_paths "crates/ml/examples/") - TRAINING_UPLOADER=$(check_paths "crates/training_uploader/") - - # Tier 6: infra and docker - DOCKER_IMAGES=$(check_paths "infra/docker/") + # --- Boolean gates for downstream DAG tasks --- + NEEDS_CODE=$(check_paths "Cargo.toml Cargo.lock crates/ services/ bin/fxt/") NEEDS_DASHBOARD=$(check_paths "web-dashboard/") - INFRA_CI=$(check_paths "infra/k8s/argo/ci-pipeline infra/k8s/services/") + DOCKER_IMAGES=$(check_paths "infra/docker/") NEEDS_INFRA=$(check_paths "infra/live/ infra/modules/") NEEDS_ARGO_TEMPLATES=$(check_paths "infra/k8s/argo/") - # ======================================== - # Build SERVICE package list - # ======================================== - SERVICE_PKGS="" - - add_svc() { - local pkg="$1" - # Deduplicate: only add if not already present - case " $SERVICE_PKGS " in - *" $pkg "*) ;; - *) SERVICE_PKGS="$SERVICE_PKGS $pkg" ;; - esac - } - - if [ "$SHARED" = "true" ] || [ "$PROTO" = "true" ] || [ "$INFRA_CI" = "true" ]; then - # Shared changes → rebuild all services - SERVICE_PKGS="api trading-service ml-training-service backtesting-service trading-agent-service broker-gateway data-acquisition-service" - else - # trading_engine affects: api, trading-service, backtesting-service, trading-agent-service - if [ "$TRADING_ENGINE" = "true" ]; then - add_svc "api"; add_svc "trading-service"; add_svc "backtesting-service"; add_svc "trading-agent-service" - fi - - # risk affects: api, trading-service - if [ "$RISK" = "true" ]; then - add_svc "api"; add_svc "trading-service" - fi - - # storage/database affects: most services that persist data - if [ "$STORAGE" = "true" ]; then - add_svc "api"; add_svc "trading-service"; add_svc "ml-training-service" - add_svc "backtesting-service"; add_svc "trading-agent-service" - fi - - # broker crates affect: broker-gateway, trading-service - if [ "$BROKER" = "true" ]; then - add_svc "broker-gateway"; add_svc "trading-service" - fi - - # data crate affects: data-acquisition-service, backtesting-service - if [ "$DATA" = "true" ]; then - add_svc "data-acquisition-service"; add_svc "backtesting-service" - fi - - # ML crates affect: ml-training-service, trading-service (ensemble inference) - if [ "$ML_CHANGED" = "true" ]; then - add_svc "ml-training-service"; add_svc "trading-service" - fi - - # Service-specific directories - [ "$SVC_API" = "true" ] && add_svc "api" - [ "$SVC_TRADING" = "true" ] && add_svc "trading-service" - [ "$SVC_ML_TRAIN" = "true" ] && add_svc "ml-training-service" - [ "$SVC_BACKTEST" = "true" ] && add_svc "backtesting-service" - [ "$SVC_AGENT" = "true" ] && add_svc "trading-agent-service" - [ "$SVC_BROKER" = "true" ] && add_svc "broker-gateway" - [ "$SVC_DATA" = "true" ] && add_svc "data-acquisition-service" - fi - - # Trim leading space - SERVICE_PKGS=$(echo "$SERVICE_PKGS" | sed 's/^ //') - - # ======================================== - # Build TRAINING example list - # ======================================== - TRAINING_EXAMPLES="" - - add_train() { - local ex="$1" - case " $TRAINING_EXAMPLES " in - *" $ex "*) ;; - *) TRAINING_EXAMPLES="$TRAINING_EXAMPLES $ex" ;; - esac - } - - ALL_TRAIN_EXAMPLES="train_baseline_rl train_baseline_supervised evaluate_baseline evaluate_supervised hyperopt_baseline_rl hyperopt_baseline_supervised" - - if [ "$SHARED" = "true" ] || [ "$ML_CHANGED" = "true" ]; then - # Shared or ML crate changes → rebuild all training binaries - TRAINING_EXAMPLES="$ALL_TRAIN_EXAMPLES" - add_train "training_uploader" - else - # Granular: specific ML subdirectories - if [ "$ML_TRAINERS" = "true" ]; then - add_train "train_baseline_rl"; add_train "train_baseline_supervised" - fi - if [ "$ML_HYPEROPT" = "true" ]; then - add_train "hyperopt_baseline_rl"; add_train "hyperopt_baseline_supervised" - fi - if [ "$ML_EVAL" = "true" ]; then - add_train "evaluate_baseline"; add_train "evaluate_supervised" - fi - # Individual example file changes - if [ "$ML_EXAMPLES" = "true" ]; then - for file in $CHANGED_FILES; do - case "$file" in - crates/ml/examples/*.rs) - # Extract example name from filename (strip path and .rs) - EXAMPLE_NAME=$(basename "$file" .rs) - case "$EXAMPLE_NAME" in - train_baseline_rl|train_baseline_supervised|evaluate_baseline|evaluate_supervised|hyperopt_baseline_rl|hyperopt_baseline_supervised) - add_train "$EXAMPLE_NAME" ;; - esac - ;; - esac - done - fi - if [ "$TRAINING_UPLOADER" = "true" ]; then - add_train "training_uploader" - fi - fi - - # Trim leading space - TRAINING_EXAMPLES=$(echo "$TRAINING_EXAMPLES" | sed 's/^ //') - - # ======================================== - # Build DEPLOY list (service name → K8s deployment name) - # ======================================== - # trading-service excluded from auto-deploy for safety - DEPLOY_LIST="" - for pkg in $SERVICE_PKGS; do - case "$pkg" in - trading-service) ;; # excluded — must be explicitly enabled - *) DEPLOY_LIST="$DEPLOY_LIST $pkg" ;; - esac - done - DEPLOY_LIST=$(echo "$DEPLOY_LIST" | sed 's/^ //') - - # ======================================== - # Compute boolean gates - # ======================================== - if [ -n "$SERVICE_PKGS" ]; then NEEDS_SERVICES="true"; else NEEDS_SERVICES="false"; fi - if [ -n "$TRAINING_EXAMPLES" ]; then NEEDS_TRAINING="true"; else NEEDS_TRAINING="false"; fi - if [ "$NEEDS_SERVICES" = "true" ] || [ "$NEEDS_TRAINING" = "true" ]; then - NEEDS_CODE="true" - else - NEEDS_CODE="false" - fi - echo "=== Build decisions ===" - echo "needs-services: $NEEDS_SERVICES" - echo "needs-training: $NEEDS_TRAINING" echo "needs-code: $NEEDS_CODE" echo "needs-dashboard: $NEEDS_DASHBOARD" + echo "docker-images: $DOCKER_IMAGES" echo "needs-infra: $NEEDS_INFRA" echo "needs-argo: $NEEDS_ARGO_TEMPLATES" - echo "docker-images: $DOCKER_IMAGES" - echo "service-packages: $SERVICE_PKGS" - echo "training-examples: $TRAINING_EXAMPLES" - echo "deploy-list: $DEPLOY_LIST" echo "======================" mkdir -p /tmp/outputs - echo -n "$NEEDS_SERVICES" > /tmp/outputs/needs-services - echo -n "$NEEDS_TRAINING" > /tmp/outputs/needs-training + echo -n "$NEEDS_CODE" > /tmp/outputs/needs-code echo -n "$NEEDS_DASHBOARD" > /tmp/outputs/needs-dashboard echo -n "$DOCKER_IMAGES" > /tmp/outputs/docker-images - echo -n "$NEEDS_CODE" > /tmp/outputs/needs-code echo -n "$NEEDS_INFRA" > /tmp/outputs/needs-infra echo -n "$NEEDS_ARGO_TEMPLATES" > /tmp/outputs/needs-argo-templates - echo -n "$SERVICE_PKGS" > /tmp/outputs/service-packages - echo -n "$TRAINING_EXAMPLES" > /tmp/outputs/training-examples - echo -n "$DEPLOY_LIST" > /tmp/outputs/deploy-list SCRIPT chmod +x /tmp/detect.sh /tmp/detect.sh outputs: parameters: - - name: needs-services + - name: needs-code valueFrom: - path: /tmp/outputs/needs-services - - name: needs-training - valueFrom: - path: /tmp/outputs/needs-training + path: /tmp/outputs/needs-code - name: needs-dashboard valueFrom: path: /tmp/outputs/needs-dashboard - name: docker-images valueFrom: path: /tmp/outputs/docker-images - - name: needs-code - valueFrom: - path: /tmp/outputs/needs-code - - name: service-packages - valueFrom: - path: /tmp/outputs/service-packages - - name: training-examples - valueFrom: - path: /tmp/outputs/training-examples - - name: deploy-list - valueFrom: - path: /tmp/outputs/deploy-list - name: needs-infra valueFrom: path: /tmp/outputs/needs-infra @@ -421,6 +236,87 @@ spec: valueFrom: path: /tmp/outputs/needs-argo-templates + # ── test-gate: clippy + cargo test quality gate ── + - name: test-gate + inputs: + parameters: + - name: commit-sha + nodeSelector: + k8s.scaleway.com/pool-name: ci-compile-cpu + tolerations: + - key: k8s.scaleway.com/pool-name + operator: Equal + value: ci-compile-cpu + effect: NoSchedule + container: + image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/ci-builder-cpu:latest + imagePullPolicy: Always + command: ["/bin/sh", "-c"] + env: + - name: SQLX_OFFLINE + value: "true" + - name: CARGO_TERM_COLOR + value: always + - name: CARGO_TARGET_DIR + value: /cargo-target + - name: CARGO_HOME + value: /cargo-target/cargo-home + resources: + requests: + cpu: "14" + memory: 16Gi + limits: + cpu: "30" + memory: 32Gi + volumeMounts: + - name: git-ssh-key + mountPath: /etc/git-ssh + readOnly: true + - name: cargo-target-cpu + mountPath: /cargo-target + args: + - | + set -e + SHA="{{inputs.parameters.commit-sha}}" + + # SSH setup + mkdir -p ~/.ssh + cp /etc/git-ssh/ssh-privatekey ~/.ssh/id_ed25519 + chmod 600 ~/.ssh/id_ed25519 + printf 'Host *\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n' > ~/.ssh/config + chmod 600 ~/.ssh/config + + REPO="ssh://git@gitlab-gitlab-shell.foxhunt.svc.cluster.local:2222/root/foxhunt.git" + WORKSPACE="/cargo-target/src" + + git config --global --add safe.directory "$WORKSPACE" + + if [ -d "$WORKSPACE/.git" ]; then + cd "$WORKSPACE" + CURRENT=$(git rev-parse HEAD 2>/dev/null || echo "none") + if [ "$CURRENT" = "$SHA" ]; then + echo "=== Already at $SHA ===" + else + git fetch origin + git checkout --force "$SHA" + git clean -fd + fi + else + git clone --filter=blob:none "$REPO" "$WORKSPACE" + cd "$WORKSPACE" + git checkout "$SHA" + fi + + export PATH="${CARGO_HOME}/bin:${PATH}" + + echo "=== Running clippy ===" + cargo clippy --workspace --all-targets -- -D warnings + + echo "=== Running tests (lib only, no integration) ===" + cargo test --workspace --lib + + echo "=== Test gate passed ===" + # ── build-web-dashboard: npm build + upload to MinIO ── - name: build-web-dashboard nodeSelector: