23a122607d
- Argo workflow archive via PostgreSQL — persistent logs after pod GC - Allow Argo controller to reach PostgreSQL via network policy - Allow Argo server to reach MinIO for S3 log archival - Replace foxhunt-runtime with lightweight images (alpine:3.21, curlimages/curl) in orchestration steps: fetch-binary, upload-results, detect-changes, gpu-warmup - Use ci-builder for training steps (hyperopt, train-best, evaluate) — nvcc required - Fix --hyperopt-results → --hyperopt-params flag in train-best step - Remove invalid podGCGracePeriod from Argo Helm values - Archive RBAC, kustomization updates Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
495 lines
18 KiB
YAML
495 lines
18 KiB
YAML
# Compile-and-Deploy: manual workflow for building and deploying service binaries.
|
|
#
|
|
# DAG:
|
|
# create-tag ──> compile-services ──> upload-release ──> deploy-services
|
|
#
|
|
# Image builds handled by ci-pipeline, not here.
|
|
# Training compilation lives in compile-and-train-template.yaml.
|
|
---
|
|
apiVersion: argoproj.io/v1alpha1
|
|
kind: WorkflowTemplate
|
|
metadata:
|
|
name: compile-and-deploy
|
|
namespace: foxhunt
|
|
labels:
|
|
app.kubernetes.io/name: compile-and-deploy
|
|
app.kubernetes.io/part-of: foxhunt
|
|
spec:
|
|
entrypoint: pipeline
|
|
onExit: notify-result
|
|
serviceAccountName: argo-workflow
|
|
podMetadata:
|
|
labels:
|
|
app.kubernetes.io/part-of: foxhunt
|
|
app.kubernetes.io/component: compile-and-deploy
|
|
podGC:
|
|
strategy: OnPodCompletion
|
|
ttlStrategy:
|
|
secondsAfterCompletion: 3600
|
|
activeDeadlineSeconds: 3600 # 1 hour (compile + deploy)
|
|
|
|
arguments:
|
|
parameters:
|
|
- name: commit-sha
|
|
value: HEAD
|
|
- name: service-packages
|
|
value: "api trading-service backtesting-service trading-agent-service broker-gateway data-acquisition-service"
|
|
|
|
volumes:
|
|
- name: git-ssh-key
|
|
secret:
|
|
secretName: argo-git-ssh-key
|
|
defaultMode: 256
|
|
- name: cargo-target-cpu
|
|
persistentVolumeClaim:
|
|
claimName: cargo-target-cpu
|
|
|
|
templates:
|
|
- name: pipeline
|
|
dag:
|
|
tasks:
|
|
- name: create-tag
|
|
template: create-tag
|
|
- name: compile-services
|
|
depends: "create-tag"
|
|
template: compile-services
|
|
arguments:
|
|
parameters:
|
|
- name: tag
|
|
value: "{{tasks.create-tag.outputs.parameters.tag}}"
|
|
- name: service-packages
|
|
value: "{{workflow.parameters.service-packages}}"
|
|
- name: upload-release
|
|
depends: "compile-services"
|
|
template: upload-release
|
|
arguments:
|
|
parameters:
|
|
- name: tag
|
|
value: "{{tasks.create-tag.outputs.parameters.tag}}"
|
|
- name: deploy-services
|
|
depends: "upload-release"
|
|
template: deploy-services
|
|
arguments:
|
|
parameters:
|
|
- name: tag
|
|
value: "{{tasks.create-tag.outputs.parameters.tag}}"
|
|
- name: deploy-list
|
|
value: "{{workflow.parameters.service-packages}}"
|
|
|
|
# ── create-tag: CalVer auto-tag on code changes ──
|
|
- name: create-tag
|
|
nodeSelector:
|
|
k8s.scaleway.com/pool-name: platform
|
|
container:
|
|
image: curlimages/curl:8.12.1
|
|
command: ["/bin/sh", "-c"]
|
|
env:
|
|
- name: GITLAB_PAT
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: gitlab-pat
|
|
key: token
|
|
resources:
|
|
requests:
|
|
cpu: 100m
|
|
memory: 64Mi
|
|
limits:
|
|
cpu: 200m
|
|
memory: 128Mi
|
|
args:
|
|
- |
|
|
set -e
|
|
GITLAB="http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181"
|
|
PROJECT_ID=1
|
|
SHA="{{workflow.parameters.commit-sha}}"
|
|
|
|
# Compute CalVer prefix: vYYYY.MM
|
|
PREFIX="v$(date +%Y.%m)"
|
|
|
|
# Query existing tags for this month
|
|
TAGS=$(curl -sf \
|
|
-H "PRIVATE-TOKEN: ${GITLAB_PAT}" \
|
|
"${GITLAB}/api/v4/projects/${PROJECT_ID}/repository/tags?search=${PREFIX}" \
|
|
|| echo "[]")
|
|
|
|
# Parse highest N from vYYYY.MM.N tags
|
|
LAST_N=$(echo "$TAGS" | grep -oP "\"name\":\"${PREFIX}\.\K[0-9]+" | sort -n | tail -1)
|
|
if [ -z "$LAST_N" ]; then
|
|
NEXT_N=1
|
|
else
|
|
NEXT_N=$((LAST_N + 1))
|
|
fi
|
|
|
|
TAG="${PREFIX}.${NEXT_N}"
|
|
echo "Creating tag: ${TAG} at ${SHA}"
|
|
|
|
# Create the tag
|
|
RESULT=$(curl -sf -X POST \
|
|
-H "PRIVATE-TOKEN: ${GITLAB_PAT}" \
|
|
-d "tag_name=${TAG}" \
|
|
-d "ref=${SHA}" \
|
|
"${GITLAB}/api/v4/projects/${PROJECT_ID}/repository/tags")
|
|
|
|
echo "$RESULT" | grep -q "$TAG" || { echo "Tag creation failed: $RESULT"; exit 1; }
|
|
echo "Tag ${TAG} created successfully"
|
|
|
|
mkdir -p /tmp/outputs
|
|
echo -n "$TAG" > /tmp/outputs/tag
|
|
outputs:
|
|
parameters:
|
|
- name: tag
|
|
valueFrom:
|
|
path: /tmp/outputs/tag
|
|
|
|
# ── compile-services: selective per-binary build, incremental on local RWO PVC ──
|
|
- name: compile-services
|
|
metadata:
|
|
labels:
|
|
app.kubernetes.io/component: compile
|
|
inputs:
|
|
parameters:
|
|
- name: tag
|
|
- name: service-packages
|
|
nodeSelector:
|
|
k8s.scaleway.com/pool-name: ci-compile-cpu
|
|
tolerations:
|
|
- key: node.cilium.io/agent-not-ready
|
|
operator: Exists
|
|
effect: NoSchedule
|
|
container:
|
|
image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/ci-builder-cpu:latest
|
|
imagePullPolicy: Always
|
|
command: ["/bin/sh", "-c"]
|
|
env:
|
|
- name: SQLX_OFFLINE
|
|
value: "true"
|
|
- name: CARGO_TERM_COLOR
|
|
value: always
|
|
- name: CARGO_TARGET_DIR
|
|
value: /cargo-target
|
|
- name: CARGO_HOME
|
|
value: /cargo-target/cargo-home
|
|
- name: GITLAB_PAT
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: gitlab-pat
|
|
key: token
|
|
resources:
|
|
requests:
|
|
cpu: "14"
|
|
memory: 16Gi
|
|
limits:
|
|
cpu: "30"
|
|
memory: 32Gi
|
|
volumeMounts:
|
|
- name: git-ssh-key
|
|
mountPath: /etc/git-ssh
|
|
readOnly: true
|
|
- name: cargo-target-cpu
|
|
mountPath: /cargo-target
|
|
args:
|
|
- |
|
|
set -e
|
|
SHA="{{workflow.parameters.commit-sha}}"
|
|
SERVICE_PKGS="{{inputs.parameters.service-packages}}"
|
|
|
|
# SSH setup
|
|
mkdir -p ~/.ssh
|
|
cp /etc/git-ssh/ssh-privatekey ~/.ssh/id_ed25519
|
|
chmod 600 ~/.ssh/id_ed25519
|
|
printf 'Host *\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n' > ~/.ssh/config
|
|
chmod 600 ~/.ssh/config
|
|
|
|
REPO="ssh://git@gitlab-gitlab-shell.foxhunt.svc.cluster.local:2222/root/foxhunt.git"
|
|
WORKSPACE="/cargo-target/src"
|
|
|
|
# PVC may be owned by a different UID from a previous run
|
|
git config --global --add safe.directory "$WORKSPACE"
|
|
|
|
# Persistent checkout on PVC — only changed files get new mtimes,
|
|
# so cargo skips recompiling unchanged workspace crates.
|
|
if [ -d "$WORKSPACE/.git" ]; then
|
|
cd "$WORKSPACE"
|
|
CURRENT=$(git rev-parse HEAD 2>/dev/null || echo "none")
|
|
if [ "$CURRENT" = "$SHA" ]; then
|
|
echo "=== Already at $SHA, skipping checkout ==="
|
|
else
|
|
echo "=== Updating checkout: $(echo $CURRENT | cut -c1-8) -> $(echo $SHA | cut -c1-8) ==="
|
|
git fetch origin
|
|
git checkout --force "$SHA"
|
|
git clean -fd
|
|
fi
|
|
else
|
|
echo "=== Initial clone (first run) ==="
|
|
git clone --filter=blob:none "$REPO" "$WORKSPACE"
|
|
cd "$WORKSPACE"
|
|
git checkout "$SHA"
|
|
fi
|
|
echo "Checked out $(git rev-parse --short HEAD)"
|
|
|
|
# Ensure cargo home registry is on PVC
|
|
export PATH="${CARGO_HOME}/bin:${PATH}"
|
|
|
|
export FOXHUNT_BUILD_VERSION="{{inputs.parameters.tag}}"
|
|
|
|
# Prune build artifacts if PVC exceeds 25GB (prevents unbounded growth)
|
|
TARGET_SIZE_MB=$(du -sm "$CARGO_TARGET_DIR" 2>/dev/null | cut -f1 || echo 0)
|
|
echo "PVC usage: ${TARGET_SIZE_MB}MB"
|
|
if [ "$TARGET_SIZE_MB" -gt 25000 ]; then
|
|
echo "PVC exceeds 25GB limit, pruning build artifacts..."
|
|
rm -rf "$CARGO_TARGET_DIR/release" "$CARGO_TARGET_DIR/debug"
|
|
fi
|
|
|
|
# Guard: empty package list would build entire workspace
|
|
if [ -z "$SERVICE_PKGS" ]; then
|
|
echo "ERROR: service-packages is empty, refusing to build entire workspace"
|
|
exit 1
|
|
fi
|
|
|
|
# Build only the affected service packages (incremental via persistent target dir)
|
|
CARGO_ARGS=""
|
|
for pkg in $SERVICE_PKGS; do
|
|
CARGO_ARGS="$CARGO_ARGS -p $pkg"
|
|
done
|
|
|
|
echo "=== Building service binaries: $SERVICE_PKGS (incremental) ==="
|
|
cargo build --release $CARGO_ARGS
|
|
|
|
# Collect built binaries
|
|
mkdir -p "$WORKSPACE/bin/services"
|
|
for pkg in $SERVICE_PKGS; do
|
|
bin_name=$(echo "$pkg" | tr '-' '_')
|
|
cp "$CARGO_TARGET_DIR/release/$pkg" "$WORKSPACE/bin/services/" 2>/dev/null \
|
|
|| cp "$CARGO_TARGET_DIR/release/$bin_name" "$WORKSPACE/bin/services/" 2>/dev/null \
|
|
|| { echo "Binary not found for $pkg"; ls "$CARGO_TARGET_DIR/release/"; exit 1; }
|
|
done
|
|
strip "$WORKSPACE/bin/services/"*
|
|
|
|
echo "=== Service binaries ==="
|
|
ls -lh "$WORKSPACE/bin/services/"
|
|
|
|
echo "=== Uploading service binaries to GitLab packages ==="
|
|
GITLAB="http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181"
|
|
TAG="${FOXHUNT_BUILD_VERSION}"
|
|
for bin in "$WORKSPACE/bin/services/"*; do
|
|
BIN_NAME=$(basename "$bin")
|
|
echo "Uploading ${BIN_NAME} (${TAG})..."
|
|
curl -f --upload-file "$bin" \
|
|
-H "PRIVATE-TOKEN: ${GITLAB_PAT}" \
|
|
"${GITLAB}/api/v4/projects/1/packages/generic/foxhunt-services/${TAG}/${BIN_NAME}"
|
|
done
|
|
|
|
# Update 'latest' per-file (preserves unbuilt binaries from prior runs)
|
|
echo "=== Updating 'latest' package ==="
|
|
LATEST_PKG=$(curl -sf -H "PRIVATE-TOKEN: ${GITLAB_PAT}" \
|
|
"${GITLAB}/api/v4/projects/1/packages?package_name=foxhunt-services&package_version=latest" \
|
|
| grep -oP '"id":\K[0-9]+' | head -1)
|
|
for bin in "$WORKSPACE/bin/services/"*; do
|
|
BIN_NAME=$(basename "$bin")
|
|
# Delete existing file by name before uploading replacement
|
|
if [ -n "$LATEST_PKG" ]; then
|
|
FILE_ID=$(curl -sf -H "PRIVATE-TOKEN: ${GITLAB_PAT}" \
|
|
"${GITLAB}/api/v4/projects/1/packages/${LATEST_PKG}/package_files" \
|
|
| grep -oP "\"id\":([0-9]+),\"package_id\":${LATEST_PKG}[^}]*\"file_name\":\"${BIN_NAME}\"" \
|
|
| grep -oP '"id":\K[0-9]+' | head -1)
|
|
if [ -n "$FILE_ID" ]; then
|
|
curl -sf -X DELETE -H "PRIVATE-TOKEN: ${GITLAB_PAT}" \
|
|
"${GITLAB}/api/v4/projects/1/packages/${LATEST_PKG}/package_files/${FILE_ID}" || true
|
|
fi
|
|
fi
|
|
curl -f --upload-file "$bin" \
|
|
-H "PRIVATE-TOKEN: ${GITLAB_PAT}" \
|
|
"${GITLAB}/api/v4/projects/1/packages/generic/foxhunt-services/latest/${BIN_NAME}" || true
|
|
done
|
|
|
|
echo "=== Service compile + upload done ($SERVICE_PKGS) ==="
|
|
|
|
# ── upload-release: create GitLab Release with package links ──
|
|
- name: upload-release
|
|
inputs:
|
|
parameters:
|
|
- name: tag
|
|
nodeSelector:
|
|
k8s.scaleway.com/pool-name: platform
|
|
container:
|
|
image: curlimages/curl:8.12.1
|
|
command: ["/bin/sh", "-c"]
|
|
env:
|
|
- name: GITLAB_PAT
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: gitlab-pat
|
|
key: token
|
|
resources:
|
|
requests:
|
|
cpu: 100m
|
|
memory: 64Mi
|
|
limits:
|
|
cpu: 200m
|
|
memory: 128Mi
|
|
args:
|
|
- |
|
|
set -e
|
|
GITLAB="http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181"
|
|
PROJECT_ID=1
|
|
TAG="{{inputs.parameters.tag}}"
|
|
|
|
echo "=== Creating GitLab Release ${TAG} ==="
|
|
|
|
# Get commits since previous tag for release notes
|
|
PREV_TAG=$(curl -sf \
|
|
-H "PRIVATE-TOKEN: ${GITLAB_PAT}" \
|
|
"${GITLAB}/api/v4/projects/${PROJECT_ID}/repository/tags?per_page=2" \
|
|
| grep -oP '"name":"\K[^"]+' | sed -n '2p')
|
|
|
|
if [ -n "$PREV_TAG" ]; then
|
|
COMMITS=$(curl -sf \
|
|
-H "PRIVATE-TOKEN: ${GITLAB_PAT}" \
|
|
"${GITLAB}/api/v4/projects/${PROJECT_ID}/repository/compare?from=${PREV_TAG}&to=${TAG}" \
|
|
| grep -oP '"title":"\K[^"]+' | head -20 \
|
|
| sed 's/^/- /' || echo "- Release ${TAG}")
|
|
DESCRIPTION="## Changes since ${PREV_TAG}\n\n${COMMITS}"
|
|
else
|
|
DESCRIPTION="## Initial release\n\nFirst CalVer release."
|
|
fi
|
|
|
|
# Create the release
|
|
RESPONSE=$(curl -sf -X POST \
|
|
-H "PRIVATE-TOKEN: ${GITLAB_PAT}" \
|
|
-H "Content-Type: application/json" \
|
|
-d "{
|
|
\"tag_name\": \"${TAG}\",
|
|
\"name\": \"${TAG}\",
|
|
\"description\": \"$(printf '%s' "$DESCRIPTION" | sed 's/"/\\"/g')\"
|
|
}" \
|
|
"${GITLAB}/api/v4/projects/${PROJECT_ID}/releases") || {
|
|
echo "WARN: Release creation failed (may already exist)"
|
|
}
|
|
|
|
echo "Release ${TAG} created"
|
|
echo "$RESPONSE" | head -5
|
|
|
|
# Add package links as release assets
|
|
for pkg_name in foxhunt-services foxhunt-training; do
|
|
PKG_CHECK=$(curl -sf \
|
|
-H "PRIVATE-TOKEN: ${GITLAB_PAT}" \
|
|
"${GITLAB}/api/v4/projects/${PROJECT_ID}/packages?package_name=${pkg_name}&package_version=${TAG}" \
|
|
|| echo "[]")
|
|
|
|
if echo "$PKG_CHECK" | grep -q "$TAG"; then
|
|
curl -sf -X POST \
|
|
-H "PRIVATE-TOKEN: ${GITLAB_PAT}" \
|
|
-H "Content-Type: application/json" \
|
|
-d "{
|
|
\"name\": \"${pkg_name}\",
|
|
\"url\": \"${GITLAB}/-/packages?type=generic&search=${pkg_name}&version=${TAG}\",
|
|
\"link_type\": \"package\"
|
|
}" \
|
|
"${GITLAB}/api/v4/projects/${PROJECT_ID}/releases/${TAG}/assets/links" || true
|
|
echo "Linked ${pkg_name} package to release"
|
|
fi
|
|
done
|
|
|
|
echo "=== Release ${TAG} complete ==="
|
|
|
|
# ── deploy-services: selective rolling restart for affected deployments ──
|
|
- name: deploy-services
|
|
inputs:
|
|
parameters:
|
|
- name: tag
|
|
- name: deploy-list
|
|
serviceAccountName: argo-workflow
|
|
nodeSelector:
|
|
k8s.scaleway.com/pool-name: platform
|
|
container:
|
|
image: curlimages/curl:8.12.1
|
|
command: ["/bin/sh", "-c"]
|
|
resources:
|
|
requests:
|
|
cpu: 100m
|
|
memory: 64Mi
|
|
limits:
|
|
cpu: 200m
|
|
memory: 128Mi
|
|
args:
|
|
- |
|
|
set -e
|
|
if ! command -v kubectl >/dev/null 2>&1; then
|
|
echo "=== Installing kubectl ==="
|
|
curl -sLo /tmp/kubectl "https://dl.k8s.io/release/v1.31.4/bin/linux/amd64/kubectl"
|
|
chmod +x /tmp/kubectl
|
|
export PATH="/tmp:$PATH"
|
|
fi
|
|
|
|
TAG="{{inputs.parameters.tag}}"
|
|
DEPLOY_LIST="{{inputs.parameters.deploy-list}}"
|
|
|
|
if [ -z "$DEPLOY_LIST" ]; then
|
|
echo "=== No services to deploy ==="
|
|
exit 0
|
|
fi
|
|
|
|
echo "=== Deploying release ${TAG}: $DEPLOY_LIST ==="
|
|
|
|
for svc in $DEPLOY_LIST; do
|
|
echo "Patching $svc with FOXHUNT_RELEASE=${TAG}..."
|
|
kubectl -n foxhunt patch deployment "$svc" -p "{
|
|
\"spec\":{\"template\":{
|
|
\"metadata\":{\"annotations\":{\"foxhunt.io/release\":\"${TAG}\"}},
|
|
\"spec\":{\"initContainers\":[{
|
|
\"name\":\"fetch-binary\",
|
|
\"env\":[{\"name\":\"FOXHUNT_RELEASE\",\"value\":\"${TAG}\"}]
|
|
}]}
|
|
}}
|
|
}" || echo "WARN: $svc patch failed (may not exist)"
|
|
done
|
|
|
|
echo "=== Waiting for rollouts ==="
|
|
for svc in $DEPLOY_LIST; do
|
|
kubectl -n foxhunt rollout status deployment "$svc" --timeout=120s || echo "WARN: $svc rollout timeout"
|
|
done
|
|
|
|
echo "=== Deploy ${TAG} complete ($DEPLOY_LIST) ==="
|
|
|
|
# ── notify-result: post workflow outcome to Mattermost ──
|
|
- name: notify-result
|
|
nodeSelector:
|
|
k8s.scaleway.com/pool-name: platform
|
|
container:
|
|
image: curlimages/curl:8.12.1
|
|
command: ["/bin/sh", "-c"]
|
|
env:
|
|
- name: WEBHOOK_URL
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: notification-webhook
|
|
key: webhook-url
|
|
optional: true
|
|
resources:
|
|
requests:
|
|
cpu: 50m
|
|
memory: 32Mi
|
|
limits:
|
|
cpu: 100m
|
|
memory: 64Mi
|
|
args:
|
|
- |
|
|
STATUS="{{workflow.status}}"
|
|
NAME="{{workflow.name}}"
|
|
|
|
if [ -z "$WEBHOOK_URL" ] || echo "$WEBHOOK_URL" | grep -q "PLACEHOLDER"; then
|
|
echo "No webhook configured, skipping notification"
|
|
exit 0
|
|
fi
|
|
|
|
if [ "$STATUS" = "Succeeded" ]; then
|
|
EMOJI=":white_check_mark:"
|
|
else
|
|
EMOJI=":x:"
|
|
fi
|
|
|
|
PAYLOAD="{\"username\":\"Argo CI\",\"text\":\"${EMOJI} **${NAME}** — ${STATUS} ({{workflow.duration}}s)\"}"
|
|
|
|
curl -sf -X POST -H 'Content-Type: application/json' \
|
|
-d "$PAYLOAD" "$WEBHOOK_URL" || echo "WARN: webhook post failed"
|