From 3a2923b800903dbde12f9ee7e2b7ab420be2e9ba Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Tue, 10 Mar 2026 16:32:07 +0100 Subject: [PATCH] feat(ci): per-binary selective compilation in Argo pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace monolithic compile-all with granular per-binary change detection. detect-changes now outputs space-separated package/example lists based on a dependency map from source directories to binary targets: - Shared crates (common, config, Cargo.toml) → all binaries - Service-specific dirs → only that service binary - Domain crates (trading_engine, risk) → dependent service subset - ML crates → ml-training-service + trading-service + all training - ML subdirs (trainers/, hyperopt/, evaluation/) → specific training binaries compile-services and compile-training accept package lists and build only affected binaries, saving ~20-30s link time per skipped binary. deploy-services restarts only affected deployments (trading-service excluded from auto-deploy for safety). Fix: 'latest' package update now replaces individual files instead of deleting the entire package, preventing corruption during partial builds. compile-and-train-template: derive needed training binaries from model parameter (3 instead of 7), drop unused training_uploader build. Co-Authored-By: Claude Opus 4.6 --- infra/k8s/argo/ci-pipeline-template.yaml | 366 ++++++++++++++---- .../k8s/argo/compile-and-train-template.yaml | 27 +- 2 files changed, 319 insertions(+), 74 deletions(-) diff --git a/infra/k8s/argo/ci-pipeline-template.yaml b/infra/k8s/argo/ci-pipeline-template.yaml index 0f977ee8c..168f05565 100644 --- a/infra/k8s/argo/ci-pipeline-template.yaml +++ b/infra/k8s/argo/ci-pipeline-template.yaml @@ -67,6 +67,8 @@ spec: parameters: - name: tag value: "{{tasks.create-tag.outputs.parameters.tag}}" + - name: service-packages + value: "{{tasks.detect-changes.outputs.parameters.service-packages}}" when: "{{tasks.detect-changes.outputs.parameters.needs-services}} == true" - name: compile-training @@ -76,6 +78,8 @@ spec: parameters: - name: tag value: "{{tasks.create-tag.outputs.parameters.tag}}" + - name: training-examples + value: "{{tasks.detect-changes.outputs.parameters.training-examples}}" when: "{{tasks.detect-changes.outputs.parameters.needs-training}} == true" - name: gpu-warmup @@ -104,6 +108,8 @@ spec: parameters: - name: tag value: "{{tasks.create-tag.outputs.parameters.tag}}" + - name: deploy-list + value: "{{tasks.detect-changes.outputs.parameters.deploy-list}}" when: "{{tasks.detect-changes.outputs.parameters.needs-services}} == true" - name: rebuild-ci-builder @@ -166,7 +172,7 @@ spec: value: foxhunt-training-runtime when: "{{tasks.detect-changes.outputs.parameters.docker-images}} == true" - # ── detect-changes ── + # ── detect-changes: granular per-binary change detection ── - name: detect-changes nodeSelector: k8s.scaleway.com/pool-name: platform @@ -191,7 +197,6 @@ spec: set -e # Extract changed file paths from webhook JSON using grep+sed (no jq dependency). - # Matches strings inside "added":[...], "modified":[...], "removed":[...] arrays. CHANGED_FILES=$(echo "$COMMITS_JSON" \ | grep -oE '"(added|modified|removed)":\[[^]]*\]' \ | sed 's/"added"://;s/"modified"://;s/"removed"://' \ @@ -199,7 +204,7 @@ spec: || echo "") if [ -z "$CHANGED_FILES" ]; then - echo "No changed files detected — triggering all builds" + echo "No changed files detected — triggering full rebuild" CHANGED_FILES="Cargo.toml" fi @@ -207,6 +212,7 @@ spec: echo "$CHANGED_FILES" echo "====================" + # --- Helper: check if any changed file matches a set of path prefixes --- check_paths() { local patterns="$1" for file in $CHANGED_FILES; do @@ -219,24 +225,177 @@ spec: echo "false" } - SHARED=$(check_paths "crates/common/ crates/config/ Cargo.toml Cargo.lock") - SVC_ONLY=$(check_paths "services/ bin/fxt/ crates/trading_engine/ crates/risk/ crates/data/ crates/storage/ crates/database/ crates/broker crates/ctrader infra/k8s/argo/ci-pipeline infra/k8s/services/") - ML_CHANGED=$(check_paths "crates/ml/ crates/ml-core/ crates/ml-dqn/ crates/ml-ppo/ crates/ml-features/ crates/ml-supervised/ crates/ml-ensemble/ crates/ml-regime/ crates/ml-hyperopt/ crates/ml-data-validation/ crates/ml-labeling/ crates/ml-validation/ crates/ml-checkpoint/ crates/ml-risk/") + # --- Dependency tiers --- + # Tier 0: workspace root — rebuilds EVERYTHING + SHARED=$(check_paths "Cargo.toml Cargo.lock crates/common/ crates/config/") + + # Tier 1: proto files used by services + PROTO=$(check_paths "bin/fxt/proto/") + + # Tier 2: domain crates shared by subsets of services + TRADING_ENGINE=$(check_paths "crates/trading_engine/") + RISK=$(check_paths "crates/risk/") + STORAGE=$(check_paths "crates/storage/ crates/database/") + BROKER=$(check_paths "crates/broker crates/ctrader") + DATA=$(check_paths "crates/data/") + + # Tier 3: ML crates — affect ml-training-service, trading-service, AND all training binaries + ML_CHANGED=$(check_paths "crates/ml/ crates/ml-core/ crates/ml-dqn/ crates/ml-ppo/ crates/ml-features/ crates/ml-supervised/ crates/ml-ensemble/ crates/ml-regime/ crates/ml-hyperopt/ crates/ml-data-validation/ crates/ml-labeling/ crates/ml-validation/ crates/ml-checkpoint/ crates/ml-risk/ crates/ml-backtesting/") + + # Tier 4: service-specific directories + SVC_API=$(check_paths "services/api/") + SVC_TRADING=$(check_paths "services/trading_service/") + SVC_ML_TRAIN=$(check_paths "services/ml_training_service/") + SVC_BACKTEST=$(check_paths "services/backtesting_service/") + SVC_AGENT=$(check_paths "services/trading_agent_service/") + SVC_BROKER=$(check_paths "services/broker_gateway/") + SVC_DATA=$(check_paths "services/data_acquisition_service/") + + # Tier 5: training-specific subdirectories within ml crate + ML_TRAINERS=$(check_paths "crates/ml/src/trainers/") + ML_HYPEROPT=$(check_paths "crates/ml/src/hyperopt/") + ML_EVAL=$(check_paths "crates/ml/src/evaluation/") + ML_EXAMPLES=$(check_paths "crates/ml/examples/") + TRAINING_UPLOADER=$(check_paths "crates/training_uploader/") + + # Tier 6: infra and docker DOCKER_IMAGES=$(check_paths "infra/docker/") NEEDS_DASHBOARD=$(check_paths "web-dashboard/") + INFRA_CI=$(check_paths "infra/k8s/argo/ci-pipeline infra/k8s/services/") - if [ "$SHARED" = "true" ] || [ "$SVC_ONLY" = "true" ] || [ "$ML_CHANGED" = "true" ]; then - NEEDS_SERVICES="true" + # ======================================== + # Build SERVICE package list + # ======================================== + SERVICE_PKGS="" + + add_svc() { + local pkg="$1" + # Deduplicate: only add if not already present + case " $SERVICE_PKGS " in + *" $pkg "*) ;; + *) SERVICE_PKGS="$SERVICE_PKGS $pkg" ;; + esac + } + + if [ "$SHARED" = "true" ] || [ "$PROTO" = "true" ] || [ "$INFRA_CI" = "true" ]; then + # Shared changes → rebuild all services + SERVICE_PKGS="api trading-service ml-training-service backtesting-service trading-agent-service broker-gateway data-acquisition-service" else - NEEDS_SERVICES="false" + # trading_engine affects: api, trading-service, backtesting-service, trading-agent-service + if [ "$TRADING_ENGINE" = "true" ]; then + add_svc "api"; add_svc "trading-service"; add_svc "backtesting-service"; add_svc "trading-agent-service" + fi + + # risk affects: api, trading-service + if [ "$RISK" = "true" ]; then + add_svc "api"; add_svc "trading-service" + fi + + # storage/database affects: most services that persist data + if [ "$STORAGE" = "true" ]; then + add_svc "api"; add_svc "trading-service"; add_svc "ml-training-service" + add_svc "backtesting-service"; add_svc "trading-agent-service" + fi + + # broker crates affect: broker-gateway, trading-service + if [ "$BROKER" = "true" ]; then + add_svc "broker-gateway"; add_svc "trading-service" + fi + + # data crate affects: data-acquisition-service, backtesting-service + if [ "$DATA" = "true" ]; then + add_svc "data-acquisition-service"; add_svc "backtesting-service" + fi + + # ML crates affect: ml-training-service, trading-service (ensemble inference) + if [ "$ML_CHANGED" = "true" ]; then + add_svc "ml-training-service"; add_svc "trading-service" + fi + + # Service-specific directories + [ "$SVC_API" = "true" ] && add_svc "api" + [ "$SVC_TRADING" = "true" ] && add_svc "trading-service" + [ "$SVC_ML_TRAIN" = "true" ] && add_svc "ml-training-service" + [ "$SVC_BACKTEST" = "true" ] && add_svc "backtesting-service" + [ "$SVC_AGENT" = "true" ] && add_svc "trading-agent-service" + [ "$SVC_BROKER" = "true" ] && add_svc "broker-gateway" + [ "$SVC_DATA" = "true" ] && add_svc "data-acquisition-service" fi + # Trim leading space + SERVICE_PKGS=$(echo "$SERVICE_PKGS" | sed 's/^ //') + + # ======================================== + # Build TRAINING example list + # ======================================== + TRAINING_EXAMPLES="" + + add_train() { + local ex="$1" + case " $TRAINING_EXAMPLES " in + *" $ex "*) ;; + *) TRAINING_EXAMPLES="$TRAINING_EXAMPLES $ex" ;; + esac + } + + ALL_TRAIN_EXAMPLES="train_baseline_rl train_baseline_supervised evaluate_baseline evaluate_supervised hyperopt_baseline_rl hyperopt_baseline_supervised" + if [ "$SHARED" = "true" ] || [ "$ML_CHANGED" = "true" ]; then - NEEDS_TRAINING="true" + # Shared or ML crate changes → rebuild all training binaries + TRAINING_EXAMPLES="$ALL_TRAIN_EXAMPLES" + add_train "training_uploader" else - NEEDS_TRAINING="false" + # Granular: specific ML subdirectories + if [ "$ML_TRAINERS" = "true" ]; then + add_train "train_baseline_rl"; add_train "train_baseline_supervised" + fi + if [ "$ML_HYPEROPT" = "true" ]; then + add_train "hyperopt_baseline_rl"; add_train "hyperopt_baseline_supervised" + fi + if [ "$ML_EVAL" = "true" ]; then + add_train "evaluate_baseline"; add_train "evaluate_supervised" + fi + # Individual example file changes + if [ "$ML_EXAMPLES" = "true" ]; then + for file in $CHANGED_FILES; do + case "$file" in + crates/ml/examples/*.rs) + # Extract example name from filename (strip path and .rs) + EXAMPLE_NAME=$(basename "$file" .rs) + case "$EXAMPLE_NAME" in + train_baseline_rl|train_baseline_supervised|evaluate_baseline|evaluate_supervised|hyperopt_baseline_rl|hyperopt_baseline_supervised) + add_train "$EXAMPLE_NAME" ;; + esac + ;; + esac + done + fi + if [ "$TRAINING_UPLOADER" = "true" ]; then + add_train "training_uploader" + fi fi + # Trim leading space + TRAINING_EXAMPLES=$(echo "$TRAINING_EXAMPLES" | sed 's/^ //') + + # ======================================== + # Build DEPLOY list (service name → K8s deployment name) + # ======================================== + # trading-service excluded from auto-deploy for safety + DEPLOY_LIST="" + for pkg in $SERVICE_PKGS; do + case "$pkg" in + trading-service) ;; # excluded — must be explicitly enabled + *) DEPLOY_LIST="$DEPLOY_LIST $pkg" ;; + esac + done + DEPLOY_LIST=$(echo "$DEPLOY_LIST" | sed 's/^ //') + + # ======================================== + # Compute boolean gates + # ======================================== + if [ -n "$SERVICE_PKGS" ]; then NEEDS_SERVICES="true"; else NEEDS_SERVICES="false"; fi + if [ -n "$TRAINING_EXAMPLES" ]; then NEEDS_TRAINING="true"; else NEEDS_TRAINING="false"; fi if [ "$NEEDS_SERVICES" = "true" ] || [ "$NEEDS_TRAINING" = "true" ]; then NEEDS_CODE="true" else @@ -244,19 +403,25 @@ spec: fi echo "=== Build decisions ===" - echo "needs-services: $NEEDS_SERVICES" - echo "needs-training: $NEEDS_TRAINING" - echo "needs-code: $NEEDS_CODE" - echo "needs-dashboard: $NEEDS_DASHBOARD" - echo "docker-images: $DOCKER_IMAGES" + echo "needs-services: $NEEDS_SERVICES" + echo "needs-training: $NEEDS_TRAINING" + echo "needs-code: $NEEDS_CODE" + echo "needs-dashboard: $NEEDS_DASHBOARD" + echo "docker-images: $DOCKER_IMAGES" + echo "service-packages: $SERVICE_PKGS" + echo "training-examples: $TRAINING_EXAMPLES" + echo "deploy-list: $DEPLOY_LIST" echo "======================" mkdir -p /tmp/outputs - echo -n "$NEEDS_SERVICES" > /tmp/outputs/needs-services - echo -n "$NEEDS_TRAINING" > /tmp/outputs/needs-training - echo -n "$NEEDS_DASHBOARD" > /tmp/outputs/needs-dashboard - echo -n "$DOCKER_IMAGES" > /tmp/outputs/docker-images - echo -n "$NEEDS_CODE" > /tmp/outputs/needs-code + echo -n "$NEEDS_SERVICES" > /tmp/outputs/needs-services + echo -n "$NEEDS_TRAINING" > /tmp/outputs/needs-training + echo -n "$NEEDS_DASHBOARD" > /tmp/outputs/needs-dashboard + echo -n "$DOCKER_IMAGES" > /tmp/outputs/docker-images + echo -n "$NEEDS_CODE" > /tmp/outputs/needs-code + echo -n "$SERVICE_PKGS" > /tmp/outputs/service-packages + echo -n "$TRAINING_EXAMPLES" > /tmp/outputs/training-examples + echo -n "$DEPLOY_LIST" > /tmp/outputs/deploy-list SCRIPT chmod +x /tmp/detect.sh /tmp/detect.sh @@ -277,6 +442,15 @@ spec: - name: needs-code valueFrom: path: /tmp/outputs/needs-code + - name: service-packages + valueFrom: + path: /tmp/outputs/service-packages + - name: training-examples + valueFrom: + path: /tmp/outputs/training-examples + - name: deploy-list + valueFrom: + path: /tmp/outputs/deploy-list # ── create-tag: CalVer auto-tag on code changes ── - name: create-tag @@ -411,7 +585,7 @@ spec: echo "=== Web dashboard build + upload done ===" - # ── compile-services: CPU-only, sccache on local RWO PVC ── + # ── compile-services: selective per-binary build, sccache on local RWO PVC ── - name: compile-services metadata: labels: @@ -419,6 +593,7 @@ spec: inputs: parameters: - name: tag + - name: service-packages nodeSelector: k8s.scaleway.com/pool-name: ci-compile-cpu tolerations: @@ -460,6 +635,7 @@ spec: - | set -e SHA="{{workflow.parameters.commit-sha}}" + SERVICE_PKGS="{{inputs.parameters.service-packages}}" # Git clone (self-contained) mkdir -p ~/.ssh @@ -485,14 +661,24 @@ spec: export RUSTC_WRAPPER=sccache sccache --zero-stats || true - echo "=== Building service binaries (sccache → local PVC /sccache) ===" - cargo build --release \ - -p api -p trading-service -p ml-training-service \ - -p backtesting-service -p trading-agent-service \ - -p broker-gateway -p data-acquisition-service + # Guard: empty package list would build entire workspace + if [ -z "$SERVICE_PKGS" ]; then + echo "ERROR: service-packages is empty, refusing to build entire workspace" + exit 1 + fi + # Build only the affected service packages + CARGO_ARGS="" + for pkg in $SERVICE_PKGS; do + CARGO_ARGS="$CARGO_ARGS -p $pkg" + done + + echo "=== Building service binaries: $SERVICE_PKGS ===" + cargo build --release $CARGO_ARGS + + # Collect built binaries mkdir -p "$WORKSPACE/bin/services" - for pkg in api trading-service ml-training-service backtesting-service trading-agent-service broker-gateway data-acquisition-service; do + for pkg in $SERVICE_PKGS; do bin_name=$(echo "$pkg" | tr '-' '_') cp "target/release/$pkg" "$WORKSPACE/bin/services/" 2>/dev/null \ || cp "target/release/$bin_name" "$WORKSPACE/bin/services/" 2>/dev/null \ @@ -515,26 +701,32 @@ spec: "${GITLAB}/api/v4/projects/1/packages/generic/foxhunt-services/${TAG}/${BIN_NAME}" done - # Delete old 'latest' package then re-upload (GitLab doesn't overwrite files) - echo "=== Tagging as 'latest' ===" - OLD_PKG=$(curl -sf -H "PRIVATE-TOKEN: ${GITLAB_PAT}" \ + # Update 'latest' per-file (preserves unbuilt binaries from prior runs) + echo "=== Updating 'latest' package ===" + LATEST_PKG=$(curl -sf -H "PRIVATE-TOKEN: ${GITLAB_PAT}" \ "${GITLAB}/api/v4/projects/1/packages?package_name=foxhunt-services&package_version=latest" \ | grep -oP '"id":\K[0-9]+' | head -1) - if [ -n "$OLD_PKG" ]; then - curl -sf -X DELETE -H "PRIVATE-TOKEN: ${GITLAB_PAT}" \ - "${GITLAB}/api/v4/projects/1/packages/${OLD_PKG}" && \ - echo "Deleted old 'latest' package (ID ${OLD_PKG})" || true - fi for bin in "$WORKSPACE/bin/services/"*; do BIN_NAME=$(basename "$bin") + # Delete existing file by name before uploading replacement + if [ -n "$LATEST_PKG" ]; then + FILE_ID=$(curl -sf -H "PRIVATE-TOKEN: ${GITLAB_PAT}" \ + "${GITLAB}/api/v4/projects/1/packages/${LATEST_PKG}/package_files" \ + | grep -oP "\"id\":([0-9]+),\"package_id\":${LATEST_PKG}[^}]*\"file_name\":\"${BIN_NAME}\"" \ + | grep -oP '"id":\K[0-9]+' | head -1) + if [ -n "$FILE_ID" ]; then + curl -sf -X DELETE -H "PRIVATE-TOKEN: ${GITLAB_PAT}" \ + "${GITLAB}/api/v4/projects/1/packages/${LATEST_PKG}/package_files/${FILE_ID}" || true + fi + fi curl -f --upload-file "$bin" \ -H "PRIVATE-TOKEN: ${GITLAB_PAT}" \ "${GITLAB}/api/v4/projects/1/packages/generic/foxhunt-services/latest/${BIN_NAME}" || true done - echo "=== Service compile + upload done ===" + echo "=== Service compile + upload done ($SERVICE_PKGS) ===" - # ── compile-training: CUDA build, sccache on local RWO PVC ── + # ── compile-training: selective per-binary CUDA build, sccache on local RWO PVC ── - name: compile-training metadata: labels: @@ -542,6 +734,7 @@ spec: inputs: parameters: - name: tag + - name: training-examples nodeSelector: k8s.scaleway.com/pool-name: ci-compile-cpu tolerations: @@ -583,6 +776,7 @@ spec: - | set -e SHA="{{workflow.parameters.commit-sha}}" + TRAINING_EXAMPLES="{{inputs.parameters.training-examples}}" # Git clone (self-contained — ephemeral node) mkdir -p ~/.ssh @@ -609,19 +803,48 @@ spec: export CUDA_COMPUTE_CAP={{workflow.parameters.cuda-compute-cap}} sccache --zero-stats || true - echo "=== Building training binaries (CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP, sccache → local PVC /sccache) ===" - cargo build --release -p ml --features ml/cuda \ - --example train_baseline_rl --example train_baseline_supervised \ - --example evaluate_baseline --example evaluate_supervised \ - --example hyperopt_baseline_rl --example hyperopt_baseline_supervised - - cargo build --release -p training_uploader - - mkdir -p "$WORKSPACE/bin/training" - for bin in train_baseline_rl train_baseline_supervised evaluate_baseline evaluate_supervised hyperopt_baseline_rl hyperopt_baseline_supervised; do - cp target/release/examples/$bin "$WORKSPACE/bin/training/" + # Separate ml examples from training_uploader (different build commands) + ML_EXAMPLE_ARGS="" + BUILD_UPLOADER="false" + for ex in $TRAINING_EXAMPLES; do + case "$ex" in + training_uploader) + BUILD_UPLOADER="true" + ;; + *) + ML_EXAMPLE_ARGS="$ML_EXAMPLE_ARGS --example $ex" + ;; + esac + done + + # Guard: empty example list would be a no-op waste of a compile pod + if [ -z "$TRAINING_EXAMPLES" ]; then + echo "ERROR: training-examples is empty, nothing to build" + exit 1 + fi + + echo "=== Building training binaries: $TRAINING_EXAMPLES (CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP) ===" + + if [ -n "$ML_EXAMPLE_ARGS" ]; then + cargo build --release -p ml --features ml/cuda $ML_EXAMPLE_ARGS + fi + + if [ "$BUILD_UPLOADER" = "true" ]; then + cargo build --release -p training_uploader + fi + + # Collect built binaries + mkdir -p "$WORKSPACE/bin/training" + for ex in $TRAINING_EXAMPLES; do + case "$ex" in + training_uploader) + cp target/release/training_uploader "$WORKSPACE/bin/training/" + ;; + *) + cp target/release/examples/$ex "$WORKSPACE/bin/training/" + ;; + esac done - cp target/release/training_uploader "$WORKSPACE/bin/training/" strip "$WORKSPACE/bin/training/"* echo "=== Training binaries ===" @@ -639,24 +862,29 @@ spec: "${GITLAB}/api/v4/projects/1/packages/generic/foxhunt-training/${TAG}/${BIN_NAME}" done - # Delete old 'latest' package then re-upload (GitLab doesn't overwrite files) - echo "=== Tagging as 'latest' ===" - OLD_PKG=$(curl -sf -H "PRIVATE-TOKEN: ${GITLAB_PAT}" \ + # Update 'latest' per-file (preserves unbuilt binaries from prior runs) + echo "=== Updating 'latest' package ===" + LATEST_PKG=$(curl -sf -H "PRIVATE-TOKEN: ${GITLAB_PAT}" \ "${GITLAB}/api/v4/projects/1/packages?package_name=foxhunt-training&package_version=latest" \ | grep -oP '"id":\K[0-9]+' | head -1) - if [ -n "$OLD_PKG" ]; then - curl -sf -X DELETE -H "PRIVATE-TOKEN: ${GITLAB_PAT}" \ - "${GITLAB}/api/v4/projects/1/packages/${OLD_PKG}" && \ - echo "Deleted old 'latest' package (ID ${OLD_PKG})" || true - fi for bin in "$WORKSPACE/bin/training/"*; do BIN_NAME=$(basename "$bin") + if [ -n "$LATEST_PKG" ]; then + FILE_ID=$(curl -sf -H "PRIVATE-TOKEN: ${GITLAB_PAT}" \ + "${GITLAB}/api/v4/projects/1/packages/${LATEST_PKG}/package_files" \ + | grep -oP "\"id\":([0-9]+),\"package_id\":${LATEST_PKG}[^}]*\"file_name\":\"${BIN_NAME}\"" \ + | grep -oP '"id":\K[0-9]+' | head -1) + if [ -n "$FILE_ID" ]; then + curl -sf -X DELETE -H "PRIVATE-TOKEN: ${GITLAB_PAT}" \ + "${GITLAB}/api/v4/projects/1/packages/${LATEST_PKG}/package_files/${FILE_ID}" || true + fi + fi curl -f --upload-file "$bin" \ -H "PRIVATE-TOKEN: ${GITLAB_PAT}" \ "${GITLAB}/api/v4/projects/1/packages/generic/foxhunt-training/latest/${BIN_NAME}" || true done - echo "=== Training compile + upload done ===" + echo "=== Training compile + upload done ($TRAINING_EXAMPLES) ===" # ── upload-release: create GitLab Release with package links ── - name: upload-release @@ -776,11 +1004,12 @@ spec: cpu: 200m memory: 256Mi - # ── deploy-services: rolling restart after new binaries are uploaded ── + # ── deploy-services: selective rolling restart for affected deployments ── - name: deploy-services inputs: parameters: - name: tag + - name: deploy-list serviceAccountName: argo-workflow nodeSelector: k8s.scaleway.com/pool-name: platform @@ -805,11 +1034,16 @@ spec: fi TAG="{{inputs.parameters.tag}}" - echo "=== Deploying release ${TAG} ===" + DEPLOY_LIST="{{inputs.parameters.deploy-list}}" - # trading-service excluded — must be explicitly enabled for live trading - SERVICES="api ml-training-service backtesting-service trading-agent-service broker-gateway data-acquisition-service" - for svc in $SERVICES; do + if [ -z "$DEPLOY_LIST" ]; then + echo "=== No services to deploy ===" + exit 0 + fi + + echo "=== Deploying release ${TAG}: $DEPLOY_LIST ===" + + for svc in $DEPLOY_LIST; do echo "Patching $svc with FOXHUNT_RELEASE=${TAG}..." kubectl -n foxhunt patch deployment "$svc" -p "{ \"spec\":{\"template\":{ @@ -823,8 +1057,8 @@ spec: done echo "=== Waiting for rollouts ===" - for svc in $SERVICES; do + for svc in $DEPLOY_LIST; do kubectl -n foxhunt rollout status deployment "$svc" --timeout=120s || echo "WARN: $svc rollout timeout" done - echo "=== Deploy ${TAG} complete ===" + echo "=== Deploy ${TAG} complete ($DEPLOY_LIST) ===" diff --git a/infra/k8s/argo/compile-and-train-template.yaml b/infra/k8s/argo/compile-and-train-template.yaml index 02b89a82d..493079fb6 100644 --- a/infra/k8s/argo/compile-and-train-template.yaml +++ b/infra/k8s/argo/compile-and-train-template.yaml @@ -192,19 +192,30 @@ spec: export CUDA_COMPUTE_CAP={{workflow.parameters.cuda-compute-cap}} sccache --zero-stats || true - echo "=== Building training binaries (CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP, sccache → local PVC /sccache) ===" - cargo build --release -p ml --features ml/cuda \ - --example train_baseline_rl --example train_baseline_supervised \ - --example evaluate_baseline --example evaluate_supervised \ - --example hyperopt_baseline_rl --example hyperopt_baseline_supervised + # Derive needed binaries from model parameter (only build what's used) + MODEL="{{workflow.parameters.model}}" + case "$MODEL" in + dqn|ppo) + EXAMPLES="hyperopt_baseline_rl train_baseline_rl evaluate_baseline" + ;; + *) + EXAMPLES="hyperopt_baseline_supervised train_baseline_supervised evaluate_supervised" + ;; + esac - cargo build --release -p training_uploader + ML_EXAMPLE_ARGS="" + for ex in $EXAMPLES; do + ML_EXAMPLE_ARGS="$ML_EXAMPLE_ARGS --example $ex" + done + + echo "=== Building training binaries for $MODEL: $EXAMPLES ===" + echo " CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP, sccache → local PVC /sccache" + cargo build --release -p ml --features ml/cuda $ML_EXAMPLE_ARGS mkdir -p "$BUILD/bin/training" - for bin in train_baseline_rl train_baseline_supervised evaluate_baseline evaluate_supervised hyperopt_baseline_rl hyperopt_baseline_supervised; do + for bin in $EXAMPLES; do cp target/release/examples/$bin "$BUILD/bin/training/" done - cp target/release/training_uploader "$BUILD/bin/training/" strip "$BUILD/bin/training/"* echo "=== Training binaries ==="