feat(ci): per-binary selective compilation in Argo pipeline
Replace monolithic compile-all with granular per-binary change detection. detect-changes now outputs space-separated package/example lists based on a dependency map from source directories to binary targets: - Shared crates (common, config, Cargo.toml) → all binaries - Service-specific dirs → only that service binary - Domain crates (trading_engine, risk) → dependent service subset - ML crates → ml-training-service + trading-service + all training - ML subdirs (trainers/, hyperopt/, evaluation/) → specific training binaries compile-services and compile-training accept package lists and build only affected binaries, saving ~20-30s link time per skipped binary. deploy-services restarts only affected deployments (trading-service excluded from auto-deploy for safety). Fix: 'latest' package update now replaces individual files instead of deleting the entire package, preventing corruption during partial builds. compile-and-train-template: derive needed training binaries from model parameter (3 instead of 7), drop unused training_uploader build. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
@@ -67,6 +67,8 @@ spec:
|
||||
parameters:
|
||||
- name: tag
|
||||
value: "{{tasks.create-tag.outputs.parameters.tag}}"
|
||||
- name: service-packages
|
||||
value: "{{tasks.detect-changes.outputs.parameters.service-packages}}"
|
||||
when: "{{tasks.detect-changes.outputs.parameters.needs-services}} == true"
|
||||
|
||||
- name: compile-training
|
||||
@@ -76,6 +78,8 @@ spec:
|
||||
parameters:
|
||||
- name: tag
|
||||
value: "{{tasks.create-tag.outputs.parameters.tag}}"
|
||||
- name: training-examples
|
||||
value: "{{tasks.detect-changes.outputs.parameters.training-examples}}"
|
||||
when: "{{tasks.detect-changes.outputs.parameters.needs-training}} == true"
|
||||
|
||||
- name: gpu-warmup
|
||||
@@ -104,6 +108,8 @@ spec:
|
||||
parameters:
|
||||
- name: tag
|
||||
value: "{{tasks.create-tag.outputs.parameters.tag}}"
|
||||
- name: deploy-list
|
||||
value: "{{tasks.detect-changes.outputs.parameters.deploy-list}}"
|
||||
when: "{{tasks.detect-changes.outputs.parameters.needs-services}} == true"
|
||||
|
||||
- name: rebuild-ci-builder
|
||||
@@ -166,7 +172,7 @@ spec:
|
||||
value: foxhunt-training-runtime
|
||||
when: "{{tasks.detect-changes.outputs.parameters.docker-images}} == true"
|
||||
|
||||
# ── detect-changes ──
|
||||
# ── detect-changes: granular per-binary change detection ──
|
||||
- name: detect-changes
|
||||
nodeSelector:
|
||||
k8s.scaleway.com/pool-name: platform
|
||||
@@ -191,7 +197,6 @@ spec:
|
||||
set -e
|
||||
|
||||
# Extract changed file paths from webhook JSON using grep+sed (no jq dependency).
|
||||
# Matches strings inside "added":[...], "modified":[...], "removed":[...] arrays.
|
||||
CHANGED_FILES=$(echo "$COMMITS_JSON" \
|
||||
| grep -oE '"(added|modified|removed)":\[[^]]*\]' \
|
||||
| sed 's/"added"://;s/"modified"://;s/"removed"://' \
|
||||
@@ -199,7 +204,7 @@ spec:
|
||||
|| echo "")
|
||||
|
||||
if [ -z "$CHANGED_FILES" ]; then
|
||||
echo "No changed files detected — triggering all builds"
|
||||
echo "No changed files detected — triggering full rebuild"
|
||||
CHANGED_FILES="Cargo.toml"
|
||||
fi
|
||||
|
||||
@@ -207,6 +212,7 @@ spec:
|
||||
echo "$CHANGED_FILES"
|
||||
echo "===================="
|
||||
|
||||
# --- Helper: check if any changed file matches a set of path prefixes ---
|
||||
check_paths() {
|
||||
local patterns="$1"
|
||||
for file in $CHANGED_FILES; do
|
||||
@@ -219,24 +225,177 @@ spec:
|
||||
echo "false"
|
||||
}
|
||||
|
||||
SHARED=$(check_paths "crates/common/ crates/config/ Cargo.toml Cargo.lock")
|
||||
SVC_ONLY=$(check_paths "services/ bin/fxt/ crates/trading_engine/ crates/risk/ crates/data/ crates/storage/ crates/database/ crates/broker crates/ctrader infra/k8s/argo/ci-pipeline infra/k8s/services/")
|
||||
ML_CHANGED=$(check_paths "crates/ml/ crates/ml-core/ crates/ml-dqn/ crates/ml-ppo/ crates/ml-features/ crates/ml-supervised/ crates/ml-ensemble/ crates/ml-regime/ crates/ml-hyperopt/ crates/ml-data-validation/ crates/ml-labeling/ crates/ml-validation/ crates/ml-checkpoint/ crates/ml-risk/")
|
||||
# --- Dependency tiers ---
|
||||
# Tier 0: workspace root — rebuilds EVERYTHING
|
||||
SHARED=$(check_paths "Cargo.toml Cargo.lock crates/common/ crates/config/")
|
||||
|
||||
# Tier 1: proto files used by services
|
||||
PROTO=$(check_paths "bin/fxt/proto/")
|
||||
|
||||
# Tier 2: domain crates shared by subsets of services
|
||||
TRADING_ENGINE=$(check_paths "crates/trading_engine/")
|
||||
RISK=$(check_paths "crates/risk/")
|
||||
STORAGE=$(check_paths "crates/storage/ crates/database/")
|
||||
BROKER=$(check_paths "crates/broker crates/ctrader")
|
||||
DATA=$(check_paths "crates/data/")
|
||||
|
||||
# Tier 3: ML crates — affect ml-training-service, trading-service, AND all training binaries
|
||||
ML_CHANGED=$(check_paths "crates/ml/ crates/ml-core/ crates/ml-dqn/ crates/ml-ppo/ crates/ml-features/ crates/ml-supervised/ crates/ml-ensemble/ crates/ml-regime/ crates/ml-hyperopt/ crates/ml-data-validation/ crates/ml-labeling/ crates/ml-validation/ crates/ml-checkpoint/ crates/ml-risk/ crates/ml-backtesting/")
|
||||
|
||||
# Tier 4: service-specific directories
|
||||
SVC_API=$(check_paths "services/api/")
|
||||
SVC_TRADING=$(check_paths "services/trading_service/")
|
||||
SVC_ML_TRAIN=$(check_paths "services/ml_training_service/")
|
||||
SVC_BACKTEST=$(check_paths "services/backtesting_service/")
|
||||
SVC_AGENT=$(check_paths "services/trading_agent_service/")
|
||||
SVC_BROKER=$(check_paths "services/broker_gateway/")
|
||||
SVC_DATA=$(check_paths "services/data_acquisition_service/")
|
||||
|
||||
# Tier 5: training-specific subdirectories within ml crate
|
||||
ML_TRAINERS=$(check_paths "crates/ml/src/trainers/")
|
||||
ML_HYPEROPT=$(check_paths "crates/ml/src/hyperopt/")
|
||||
ML_EVAL=$(check_paths "crates/ml/src/evaluation/")
|
||||
ML_EXAMPLES=$(check_paths "crates/ml/examples/")
|
||||
TRAINING_UPLOADER=$(check_paths "crates/training_uploader/")
|
||||
|
||||
# Tier 6: infra and docker
|
||||
DOCKER_IMAGES=$(check_paths "infra/docker/")
|
||||
NEEDS_DASHBOARD=$(check_paths "web-dashboard/")
|
||||
INFRA_CI=$(check_paths "infra/k8s/argo/ci-pipeline infra/k8s/services/")
|
||||
|
||||
if [ "$SHARED" = "true" ] || [ "$SVC_ONLY" = "true" ] || [ "$ML_CHANGED" = "true" ]; then
|
||||
NEEDS_SERVICES="true"
|
||||
# ========================================
|
||||
# Build SERVICE package list
|
||||
# ========================================
|
||||
SERVICE_PKGS=""
|
||||
|
||||
add_svc() {
|
||||
local pkg="$1"
|
||||
# Deduplicate: only add if not already present
|
||||
case " $SERVICE_PKGS " in
|
||||
*" $pkg "*) ;;
|
||||
*) SERVICE_PKGS="$SERVICE_PKGS $pkg" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
if [ "$SHARED" = "true" ] || [ "$PROTO" = "true" ] || [ "$INFRA_CI" = "true" ]; then
|
||||
# Shared changes → rebuild all services
|
||||
SERVICE_PKGS="api trading-service ml-training-service backtesting-service trading-agent-service broker-gateway data-acquisition-service"
|
||||
else
|
||||
NEEDS_SERVICES="false"
|
||||
# trading_engine affects: api, trading-service, backtesting-service, trading-agent-service
|
||||
if [ "$TRADING_ENGINE" = "true" ]; then
|
||||
add_svc "api"; add_svc "trading-service"; add_svc "backtesting-service"; add_svc "trading-agent-service"
|
||||
fi
|
||||
|
||||
# risk affects: api, trading-service
|
||||
if [ "$RISK" = "true" ]; then
|
||||
add_svc "api"; add_svc "trading-service"
|
||||
fi
|
||||
|
||||
# storage/database affects: most services that persist data
|
||||
if [ "$STORAGE" = "true" ]; then
|
||||
add_svc "api"; add_svc "trading-service"; add_svc "ml-training-service"
|
||||
add_svc "backtesting-service"; add_svc "trading-agent-service"
|
||||
fi
|
||||
|
||||
# broker crates affect: broker-gateway, trading-service
|
||||
if [ "$BROKER" = "true" ]; then
|
||||
add_svc "broker-gateway"; add_svc "trading-service"
|
||||
fi
|
||||
|
||||
# data crate affects: data-acquisition-service, backtesting-service
|
||||
if [ "$DATA" = "true" ]; then
|
||||
add_svc "data-acquisition-service"; add_svc "backtesting-service"
|
||||
fi
|
||||
|
||||
# ML crates affect: ml-training-service, trading-service (ensemble inference)
|
||||
if [ "$ML_CHANGED" = "true" ]; then
|
||||
add_svc "ml-training-service"; add_svc "trading-service"
|
||||
fi
|
||||
|
||||
# Service-specific directories
|
||||
[ "$SVC_API" = "true" ] && add_svc "api"
|
||||
[ "$SVC_TRADING" = "true" ] && add_svc "trading-service"
|
||||
[ "$SVC_ML_TRAIN" = "true" ] && add_svc "ml-training-service"
|
||||
[ "$SVC_BACKTEST" = "true" ] && add_svc "backtesting-service"
|
||||
[ "$SVC_AGENT" = "true" ] && add_svc "trading-agent-service"
|
||||
[ "$SVC_BROKER" = "true" ] && add_svc "broker-gateway"
|
||||
[ "$SVC_DATA" = "true" ] && add_svc "data-acquisition-service"
|
||||
fi
|
||||
|
||||
# Trim leading space
|
||||
SERVICE_PKGS=$(echo "$SERVICE_PKGS" | sed 's/^ //')
|
||||
|
||||
# ========================================
|
||||
# Build TRAINING example list
|
||||
# ========================================
|
||||
TRAINING_EXAMPLES=""
|
||||
|
||||
add_train() {
|
||||
local ex="$1"
|
||||
case " $TRAINING_EXAMPLES " in
|
||||
*" $ex "*) ;;
|
||||
*) TRAINING_EXAMPLES="$TRAINING_EXAMPLES $ex" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
ALL_TRAIN_EXAMPLES="train_baseline_rl train_baseline_supervised evaluate_baseline evaluate_supervised hyperopt_baseline_rl hyperopt_baseline_supervised"
|
||||
|
||||
if [ "$SHARED" = "true" ] || [ "$ML_CHANGED" = "true" ]; then
|
||||
NEEDS_TRAINING="true"
|
||||
# Shared or ML crate changes → rebuild all training binaries
|
||||
TRAINING_EXAMPLES="$ALL_TRAIN_EXAMPLES"
|
||||
add_train "training_uploader"
|
||||
else
|
||||
NEEDS_TRAINING="false"
|
||||
# Granular: specific ML subdirectories
|
||||
if [ "$ML_TRAINERS" = "true" ]; then
|
||||
add_train "train_baseline_rl"; add_train "train_baseline_supervised"
|
||||
fi
|
||||
if [ "$ML_HYPEROPT" = "true" ]; then
|
||||
add_train "hyperopt_baseline_rl"; add_train "hyperopt_baseline_supervised"
|
||||
fi
|
||||
if [ "$ML_EVAL" = "true" ]; then
|
||||
add_train "evaluate_baseline"; add_train "evaluate_supervised"
|
||||
fi
|
||||
# Individual example file changes
|
||||
if [ "$ML_EXAMPLES" = "true" ]; then
|
||||
for file in $CHANGED_FILES; do
|
||||
case "$file" in
|
||||
crates/ml/examples/*.rs)
|
||||
# Extract example name from filename (strip path and .rs)
|
||||
EXAMPLE_NAME=$(basename "$file" .rs)
|
||||
case "$EXAMPLE_NAME" in
|
||||
train_baseline_rl|train_baseline_supervised|evaluate_baseline|evaluate_supervised|hyperopt_baseline_rl|hyperopt_baseline_supervised)
|
||||
add_train "$EXAMPLE_NAME" ;;
|
||||
esac
|
||||
;;
|
||||
esac
|
||||
done
|
||||
fi
|
||||
if [ "$TRAINING_UPLOADER" = "true" ]; then
|
||||
add_train "training_uploader"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Trim leading space
|
||||
TRAINING_EXAMPLES=$(echo "$TRAINING_EXAMPLES" | sed 's/^ //')
|
||||
|
||||
# ========================================
|
||||
# Build DEPLOY list (service name → K8s deployment name)
|
||||
# ========================================
|
||||
# trading-service excluded from auto-deploy for safety
|
||||
DEPLOY_LIST=""
|
||||
for pkg in $SERVICE_PKGS; do
|
||||
case "$pkg" in
|
||||
trading-service) ;; # excluded — must be explicitly enabled
|
||||
*) DEPLOY_LIST="$DEPLOY_LIST $pkg" ;;
|
||||
esac
|
||||
done
|
||||
DEPLOY_LIST=$(echo "$DEPLOY_LIST" | sed 's/^ //')
|
||||
|
||||
# ========================================
|
||||
# Compute boolean gates
|
||||
# ========================================
|
||||
if [ -n "$SERVICE_PKGS" ]; then NEEDS_SERVICES="true"; else NEEDS_SERVICES="false"; fi
|
||||
if [ -n "$TRAINING_EXAMPLES" ]; then NEEDS_TRAINING="true"; else NEEDS_TRAINING="false"; fi
|
||||
if [ "$NEEDS_SERVICES" = "true" ] || [ "$NEEDS_TRAINING" = "true" ]; then
|
||||
NEEDS_CODE="true"
|
||||
else
|
||||
@@ -244,19 +403,25 @@ spec:
|
||||
fi
|
||||
|
||||
echo "=== Build decisions ==="
|
||||
echo "needs-services: $NEEDS_SERVICES"
|
||||
echo "needs-training: $NEEDS_TRAINING"
|
||||
echo "needs-code: $NEEDS_CODE"
|
||||
echo "needs-dashboard: $NEEDS_DASHBOARD"
|
||||
echo "docker-images: $DOCKER_IMAGES"
|
||||
echo "needs-services: $NEEDS_SERVICES"
|
||||
echo "needs-training: $NEEDS_TRAINING"
|
||||
echo "needs-code: $NEEDS_CODE"
|
||||
echo "needs-dashboard: $NEEDS_DASHBOARD"
|
||||
echo "docker-images: $DOCKER_IMAGES"
|
||||
echo "service-packages: $SERVICE_PKGS"
|
||||
echo "training-examples: $TRAINING_EXAMPLES"
|
||||
echo "deploy-list: $DEPLOY_LIST"
|
||||
echo "======================"
|
||||
|
||||
mkdir -p /tmp/outputs
|
||||
echo -n "$NEEDS_SERVICES" > /tmp/outputs/needs-services
|
||||
echo -n "$NEEDS_TRAINING" > /tmp/outputs/needs-training
|
||||
echo -n "$NEEDS_DASHBOARD" > /tmp/outputs/needs-dashboard
|
||||
echo -n "$DOCKER_IMAGES" > /tmp/outputs/docker-images
|
||||
echo -n "$NEEDS_CODE" > /tmp/outputs/needs-code
|
||||
echo -n "$NEEDS_SERVICES" > /tmp/outputs/needs-services
|
||||
echo -n "$NEEDS_TRAINING" > /tmp/outputs/needs-training
|
||||
echo -n "$NEEDS_DASHBOARD" > /tmp/outputs/needs-dashboard
|
||||
echo -n "$DOCKER_IMAGES" > /tmp/outputs/docker-images
|
||||
echo -n "$NEEDS_CODE" > /tmp/outputs/needs-code
|
||||
echo -n "$SERVICE_PKGS" > /tmp/outputs/service-packages
|
||||
echo -n "$TRAINING_EXAMPLES" > /tmp/outputs/training-examples
|
||||
echo -n "$DEPLOY_LIST" > /tmp/outputs/deploy-list
|
||||
SCRIPT
|
||||
chmod +x /tmp/detect.sh
|
||||
/tmp/detect.sh
|
||||
@@ -277,6 +442,15 @@ spec:
|
||||
- name: needs-code
|
||||
valueFrom:
|
||||
path: /tmp/outputs/needs-code
|
||||
- name: service-packages
|
||||
valueFrom:
|
||||
path: /tmp/outputs/service-packages
|
||||
- name: training-examples
|
||||
valueFrom:
|
||||
path: /tmp/outputs/training-examples
|
||||
- name: deploy-list
|
||||
valueFrom:
|
||||
path: /tmp/outputs/deploy-list
|
||||
|
||||
# ── create-tag: CalVer auto-tag on code changes ──
|
||||
- name: create-tag
|
||||
@@ -411,7 +585,7 @@ spec:
|
||||
|
||||
echo "=== Web dashboard build + upload done ==="
|
||||
|
||||
# ── compile-services: CPU-only, sccache on local RWO PVC ──
|
||||
# ── compile-services: selective per-binary build, sccache on local RWO PVC ──
|
||||
- name: compile-services
|
||||
metadata:
|
||||
labels:
|
||||
@@ -419,6 +593,7 @@ spec:
|
||||
inputs:
|
||||
parameters:
|
||||
- name: tag
|
||||
- name: service-packages
|
||||
nodeSelector:
|
||||
k8s.scaleway.com/pool-name: ci-compile-cpu
|
||||
tolerations:
|
||||
@@ -460,6 +635,7 @@ spec:
|
||||
- |
|
||||
set -e
|
||||
SHA="{{workflow.parameters.commit-sha}}"
|
||||
SERVICE_PKGS="{{inputs.parameters.service-packages}}"
|
||||
|
||||
# Git clone (self-contained)
|
||||
mkdir -p ~/.ssh
|
||||
@@ -485,14 +661,24 @@ spec:
|
||||
export RUSTC_WRAPPER=sccache
|
||||
sccache --zero-stats || true
|
||||
|
||||
echo "=== Building service binaries (sccache → local PVC /sccache) ==="
|
||||
cargo build --release \
|
||||
-p api -p trading-service -p ml-training-service \
|
||||
-p backtesting-service -p trading-agent-service \
|
||||
-p broker-gateway -p data-acquisition-service
|
||||
# Guard: empty package list would build entire workspace
|
||||
if [ -z "$SERVICE_PKGS" ]; then
|
||||
echo "ERROR: service-packages is empty, refusing to build entire workspace"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Build only the affected service packages
|
||||
CARGO_ARGS=""
|
||||
for pkg in $SERVICE_PKGS; do
|
||||
CARGO_ARGS="$CARGO_ARGS -p $pkg"
|
||||
done
|
||||
|
||||
echo "=== Building service binaries: $SERVICE_PKGS ==="
|
||||
cargo build --release $CARGO_ARGS
|
||||
|
||||
# Collect built binaries
|
||||
mkdir -p "$WORKSPACE/bin/services"
|
||||
for pkg in api trading-service ml-training-service backtesting-service trading-agent-service broker-gateway data-acquisition-service; do
|
||||
for pkg in $SERVICE_PKGS; do
|
||||
bin_name=$(echo "$pkg" | tr '-' '_')
|
||||
cp "target/release/$pkg" "$WORKSPACE/bin/services/" 2>/dev/null \
|
||||
|| cp "target/release/$bin_name" "$WORKSPACE/bin/services/" 2>/dev/null \
|
||||
@@ -515,26 +701,32 @@ spec:
|
||||
"${GITLAB}/api/v4/projects/1/packages/generic/foxhunt-services/${TAG}/${BIN_NAME}"
|
||||
done
|
||||
|
||||
# Delete old 'latest' package then re-upload (GitLab doesn't overwrite files)
|
||||
echo "=== Tagging as 'latest' ==="
|
||||
OLD_PKG=$(curl -sf -H "PRIVATE-TOKEN: ${GITLAB_PAT}" \
|
||||
# Update 'latest' per-file (preserves unbuilt binaries from prior runs)
|
||||
echo "=== Updating 'latest' package ==="
|
||||
LATEST_PKG=$(curl -sf -H "PRIVATE-TOKEN: ${GITLAB_PAT}" \
|
||||
"${GITLAB}/api/v4/projects/1/packages?package_name=foxhunt-services&package_version=latest" \
|
||||
| grep -oP '"id":\K[0-9]+' | head -1)
|
||||
if [ -n "$OLD_PKG" ]; then
|
||||
curl -sf -X DELETE -H "PRIVATE-TOKEN: ${GITLAB_PAT}" \
|
||||
"${GITLAB}/api/v4/projects/1/packages/${OLD_PKG}" && \
|
||||
echo "Deleted old 'latest' package (ID ${OLD_PKG})" || true
|
||||
fi
|
||||
for bin in "$WORKSPACE/bin/services/"*; do
|
||||
BIN_NAME=$(basename "$bin")
|
||||
# Delete existing file by name before uploading replacement
|
||||
if [ -n "$LATEST_PKG" ]; then
|
||||
FILE_ID=$(curl -sf -H "PRIVATE-TOKEN: ${GITLAB_PAT}" \
|
||||
"${GITLAB}/api/v4/projects/1/packages/${LATEST_PKG}/package_files" \
|
||||
| grep -oP "\"id\":([0-9]+),\"package_id\":${LATEST_PKG}[^}]*\"file_name\":\"${BIN_NAME}\"" \
|
||||
| grep -oP '"id":\K[0-9]+' | head -1)
|
||||
if [ -n "$FILE_ID" ]; then
|
||||
curl -sf -X DELETE -H "PRIVATE-TOKEN: ${GITLAB_PAT}" \
|
||||
"${GITLAB}/api/v4/projects/1/packages/${LATEST_PKG}/package_files/${FILE_ID}" || true
|
||||
fi
|
||||
fi
|
||||
curl -f --upload-file "$bin" \
|
||||
-H "PRIVATE-TOKEN: ${GITLAB_PAT}" \
|
||||
"${GITLAB}/api/v4/projects/1/packages/generic/foxhunt-services/latest/${BIN_NAME}" || true
|
||||
done
|
||||
|
||||
echo "=== Service compile + upload done ==="
|
||||
echo "=== Service compile + upload done ($SERVICE_PKGS) ==="
|
||||
|
||||
# ── compile-training: CUDA build, sccache on local RWO PVC ──
|
||||
# ── compile-training: selective per-binary CUDA build, sccache on local RWO PVC ──
|
||||
- name: compile-training
|
||||
metadata:
|
||||
labels:
|
||||
@@ -542,6 +734,7 @@ spec:
|
||||
inputs:
|
||||
parameters:
|
||||
- name: tag
|
||||
- name: training-examples
|
||||
nodeSelector:
|
||||
k8s.scaleway.com/pool-name: ci-compile-cpu
|
||||
tolerations:
|
||||
@@ -583,6 +776,7 @@ spec:
|
||||
- |
|
||||
set -e
|
||||
SHA="{{workflow.parameters.commit-sha}}"
|
||||
TRAINING_EXAMPLES="{{inputs.parameters.training-examples}}"
|
||||
|
||||
# Git clone (self-contained — ephemeral node)
|
||||
mkdir -p ~/.ssh
|
||||
@@ -609,19 +803,48 @@ spec:
|
||||
export CUDA_COMPUTE_CAP={{workflow.parameters.cuda-compute-cap}}
|
||||
sccache --zero-stats || true
|
||||
|
||||
echo "=== Building training binaries (CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP, sccache → local PVC /sccache) ==="
|
||||
cargo build --release -p ml --features ml/cuda \
|
||||
--example train_baseline_rl --example train_baseline_supervised \
|
||||
--example evaluate_baseline --example evaluate_supervised \
|
||||
--example hyperopt_baseline_rl --example hyperopt_baseline_supervised
|
||||
|
||||
cargo build --release -p training_uploader
|
||||
|
||||
mkdir -p "$WORKSPACE/bin/training"
|
||||
for bin in train_baseline_rl train_baseline_supervised evaluate_baseline evaluate_supervised hyperopt_baseline_rl hyperopt_baseline_supervised; do
|
||||
cp target/release/examples/$bin "$WORKSPACE/bin/training/"
|
||||
# Separate ml examples from training_uploader (different build commands)
|
||||
ML_EXAMPLE_ARGS=""
|
||||
BUILD_UPLOADER="false"
|
||||
for ex in $TRAINING_EXAMPLES; do
|
||||
case "$ex" in
|
||||
training_uploader)
|
||||
BUILD_UPLOADER="true"
|
||||
;;
|
||||
*)
|
||||
ML_EXAMPLE_ARGS="$ML_EXAMPLE_ARGS --example $ex"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Guard: empty example list would be a no-op waste of a compile pod
|
||||
if [ -z "$TRAINING_EXAMPLES" ]; then
|
||||
echo "ERROR: training-examples is empty, nothing to build"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "=== Building training binaries: $TRAINING_EXAMPLES (CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP) ==="
|
||||
|
||||
if [ -n "$ML_EXAMPLE_ARGS" ]; then
|
||||
cargo build --release -p ml --features ml/cuda $ML_EXAMPLE_ARGS
|
||||
fi
|
||||
|
||||
if [ "$BUILD_UPLOADER" = "true" ]; then
|
||||
cargo build --release -p training_uploader
|
||||
fi
|
||||
|
||||
# Collect built binaries
|
||||
mkdir -p "$WORKSPACE/bin/training"
|
||||
for ex in $TRAINING_EXAMPLES; do
|
||||
case "$ex" in
|
||||
training_uploader)
|
||||
cp target/release/training_uploader "$WORKSPACE/bin/training/"
|
||||
;;
|
||||
*)
|
||||
cp target/release/examples/$ex "$WORKSPACE/bin/training/"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
cp target/release/training_uploader "$WORKSPACE/bin/training/"
|
||||
strip "$WORKSPACE/bin/training/"*
|
||||
|
||||
echo "=== Training binaries ==="
|
||||
@@ -639,24 +862,29 @@ spec:
|
||||
"${GITLAB}/api/v4/projects/1/packages/generic/foxhunt-training/${TAG}/${BIN_NAME}"
|
||||
done
|
||||
|
||||
# Delete old 'latest' package then re-upload (GitLab doesn't overwrite files)
|
||||
echo "=== Tagging as 'latest' ==="
|
||||
OLD_PKG=$(curl -sf -H "PRIVATE-TOKEN: ${GITLAB_PAT}" \
|
||||
# Update 'latest' per-file (preserves unbuilt binaries from prior runs)
|
||||
echo "=== Updating 'latest' package ==="
|
||||
LATEST_PKG=$(curl -sf -H "PRIVATE-TOKEN: ${GITLAB_PAT}" \
|
||||
"${GITLAB}/api/v4/projects/1/packages?package_name=foxhunt-training&package_version=latest" \
|
||||
| grep -oP '"id":\K[0-9]+' | head -1)
|
||||
if [ -n "$OLD_PKG" ]; then
|
||||
curl -sf -X DELETE -H "PRIVATE-TOKEN: ${GITLAB_PAT}" \
|
||||
"${GITLAB}/api/v4/projects/1/packages/${OLD_PKG}" && \
|
||||
echo "Deleted old 'latest' package (ID ${OLD_PKG})" || true
|
||||
fi
|
||||
for bin in "$WORKSPACE/bin/training/"*; do
|
||||
BIN_NAME=$(basename "$bin")
|
||||
if [ -n "$LATEST_PKG" ]; then
|
||||
FILE_ID=$(curl -sf -H "PRIVATE-TOKEN: ${GITLAB_PAT}" \
|
||||
"${GITLAB}/api/v4/projects/1/packages/${LATEST_PKG}/package_files" \
|
||||
| grep -oP "\"id\":([0-9]+),\"package_id\":${LATEST_PKG}[^}]*\"file_name\":\"${BIN_NAME}\"" \
|
||||
| grep -oP '"id":\K[0-9]+' | head -1)
|
||||
if [ -n "$FILE_ID" ]; then
|
||||
curl -sf -X DELETE -H "PRIVATE-TOKEN: ${GITLAB_PAT}" \
|
||||
"${GITLAB}/api/v4/projects/1/packages/${LATEST_PKG}/package_files/${FILE_ID}" || true
|
||||
fi
|
||||
fi
|
||||
curl -f --upload-file "$bin" \
|
||||
-H "PRIVATE-TOKEN: ${GITLAB_PAT}" \
|
||||
"${GITLAB}/api/v4/projects/1/packages/generic/foxhunt-training/latest/${BIN_NAME}" || true
|
||||
done
|
||||
|
||||
echo "=== Training compile + upload done ==="
|
||||
echo "=== Training compile + upload done ($TRAINING_EXAMPLES) ==="
|
||||
|
||||
# ── upload-release: create GitLab Release with package links ──
|
||||
- name: upload-release
|
||||
@@ -776,11 +1004,12 @@ spec:
|
||||
cpu: 200m
|
||||
memory: 256Mi
|
||||
|
||||
# ── deploy-services: rolling restart after new binaries are uploaded ──
|
||||
# ── deploy-services: selective rolling restart for affected deployments ──
|
||||
- name: deploy-services
|
||||
inputs:
|
||||
parameters:
|
||||
- name: tag
|
||||
- name: deploy-list
|
||||
serviceAccountName: argo-workflow
|
||||
nodeSelector:
|
||||
k8s.scaleway.com/pool-name: platform
|
||||
@@ -805,11 +1034,16 @@ spec:
|
||||
fi
|
||||
|
||||
TAG="{{inputs.parameters.tag}}"
|
||||
echo "=== Deploying release ${TAG} ==="
|
||||
DEPLOY_LIST="{{inputs.parameters.deploy-list}}"
|
||||
|
||||
# trading-service excluded — must be explicitly enabled for live trading
|
||||
SERVICES="api ml-training-service backtesting-service trading-agent-service broker-gateway data-acquisition-service"
|
||||
for svc in $SERVICES; do
|
||||
if [ -z "$DEPLOY_LIST" ]; then
|
||||
echo "=== No services to deploy ==="
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "=== Deploying release ${TAG}: $DEPLOY_LIST ==="
|
||||
|
||||
for svc in $DEPLOY_LIST; do
|
||||
echo "Patching $svc with FOXHUNT_RELEASE=${TAG}..."
|
||||
kubectl -n foxhunt patch deployment "$svc" -p "{
|
||||
\"spec\":{\"template\":{
|
||||
@@ -823,8 +1057,8 @@ spec:
|
||||
done
|
||||
|
||||
echo "=== Waiting for rollouts ==="
|
||||
for svc in $SERVICES; do
|
||||
for svc in $DEPLOY_LIST; do
|
||||
kubectl -n foxhunt rollout status deployment "$svc" --timeout=120s || echo "WARN: $svc rollout timeout"
|
||||
done
|
||||
|
||||
echo "=== Deploy ${TAG} complete ==="
|
||||
echo "=== Deploy ${TAG} complete ($DEPLOY_LIST) ==="
|
||||
|
||||
@@ -192,19 +192,30 @@ spec:
|
||||
export CUDA_COMPUTE_CAP={{workflow.parameters.cuda-compute-cap}}
|
||||
sccache --zero-stats || true
|
||||
|
||||
echo "=== Building training binaries (CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP, sccache → local PVC /sccache) ==="
|
||||
cargo build --release -p ml --features ml/cuda \
|
||||
--example train_baseline_rl --example train_baseline_supervised \
|
||||
--example evaluate_baseline --example evaluate_supervised \
|
||||
--example hyperopt_baseline_rl --example hyperopt_baseline_supervised
|
||||
# Derive needed binaries from model parameter (only build what's used)
|
||||
MODEL="{{workflow.parameters.model}}"
|
||||
case "$MODEL" in
|
||||
dqn|ppo)
|
||||
EXAMPLES="hyperopt_baseline_rl train_baseline_rl evaluate_baseline"
|
||||
;;
|
||||
*)
|
||||
EXAMPLES="hyperopt_baseline_supervised train_baseline_supervised evaluate_supervised"
|
||||
;;
|
||||
esac
|
||||
|
||||
cargo build --release -p training_uploader
|
||||
ML_EXAMPLE_ARGS=""
|
||||
for ex in $EXAMPLES; do
|
||||
ML_EXAMPLE_ARGS="$ML_EXAMPLE_ARGS --example $ex"
|
||||
done
|
||||
|
||||
echo "=== Building training binaries for $MODEL: $EXAMPLES ==="
|
||||
echo " CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP, sccache → local PVC /sccache"
|
||||
cargo build --release -p ml --features ml/cuda $ML_EXAMPLE_ARGS
|
||||
|
||||
mkdir -p "$BUILD/bin/training"
|
||||
for bin in train_baseline_rl train_baseline_supervised evaluate_baseline evaluate_supervised hyperopt_baseline_rl hyperopt_baseline_supervised; do
|
||||
for bin in $EXAMPLES; do
|
||||
cp target/release/examples/$bin "$BUILD/bin/training/"
|
||||
done
|
||||
cp target/release/training_uploader "$BUILD/bin/training/"
|
||||
strip "$BUILD/bin/training/"*
|
||||
|
||||
echo "=== Training binaries ==="
|
||||
|
||||
Reference in New Issue
Block a user