750 lines
26 KiB
YAML
750 lines
26 KiB
YAML
# Unified training workflow — replaces all model-specific training templates.
|
|
#
|
|
# Key innovation: binary caching by commit SHA on PVC.
|
|
# Check /data/bin/$SHA first; compile only on cache miss.
|
|
#
|
|
# DAG:
|
|
# ensure-binary ──┐
|
|
# gpu-warmup ─────┼──> ensure-fxcache ──> hyperopt ──> train-best ──> evaluate ──> upload-results
|
|
#
|
|
# Usage:
|
|
# argo submit -n foxhunt --from=wftmpl/train
|
|
# argo submit -n foxhunt --from=wftmpl/train -p model=ppo -p train-epochs=100
|
|
# argo submit -n foxhunt --from=wftmpl/train -p hyperopt-trials=0 # skip hyperopt
|
|
---
|
|
apiVersion: argoproj.io/v1alpha1
|
|
kind: WorkflowTemplate
|
|
metadata:
|
|
name: train
|
|
namespace: foxhunt
|
|
labels:
|
|
app.kubernetes.io/name: train
|
|
app.kubernetes.io/part-of: foxhunt
|
|
app.kubernetes.io/component: train
|
|
spec:
|
|
entrypoint: pipeline
|
|
onExit: notify-result
|
|
serviceAccountName: argo-workflow
|
|
archiveLogs: true
|
|
podMetadata:
|
|
labels:
|
|
app.kubernetes.io/part-of: foxhunt
|
|
app.kubernetes.io/component: train
|
|
securityContext:
|
|
fsGroup: 0
|
|
ttlStrategy:
|
|
secondsAfterCompletion: 3600
|
|
activeDeadlineSeconds: 28800 # 8 hours
|
|
|
|
arguments:
|
|
parameters:
|
|
- name: commit-sha
|
|
value: HEAD
|
|
- name: git-branch
|
|
value: main
|
|
- name: cuda-compute-cap
|
|
value: "90"
|
|
- name: model
|
|
value: dqn
|
|
- name: gpu-pool
|
|
value: ci-training-h100
|
|
- name: hyperopt-trials
|
|
value: "20"
|
|
- name: hyperopt-epochs
|
|
value: "8"
|
|
- name: train-epochs
|
|
value: "50"
|
|
- name: symbol
|
|
value: ES.FUT
|
|
- name: initial-capital
|
|
value: "35000"
|
|
- name: tx-cost-bps
|
|
value: "0.1"
|
|
- name: tick-size
|
|
value: "0.25"
|
|
- name: spread-ticks
|
|
value: "1.0"
|
|
value: "5"
|
|
- name: sanitizer
|
|
value: "none" # "none", "memcheck", "racecheck", "synccheck"
|
|
|
|
volumes:
|
|
- name: git-ssh-key
|
|
secret:
|
|
secretName: argo-git-ssh-key
|
|
defaultMode: 256
|
|
- name: training-data
|
|
persistentVolumeClaim:
|
|
claimName: training-data-pvc
|
|
- name: cargo-target-cuda
|
|
persistentVolumeClaim:
|
|
claimName: cargo-target-cuda
|
|
- name: feature-cache
|
|
persistentVolumeClaim:
|
|
claimName: feature-cache-pvc
|
|
|
|
volumeClaimTemplates:
|
|
- metadata:
|
|
name: workspace
|
|
spec:
|
|
accessModes: ["ReadWriteOnce"]
|
|
storageClassName: scw-bssd
|
|
resources:
|
|
requests:
|
|
storage: 5Gi
|
|
|
|
templates:
|
|
# ── DAG: orchestrate all steps ──
|
|
- name: pipeline
|
|
dag:
|
|
tasks:
|
|
- name: ensure-binary
|
|
template: ensure-binary
|
|
- name: gpu-warmup
|
|
template: gpu-warmup
|
|
- name: ensure-fxcache
|
|
template: ensure-fxcache
|
|
dependencies: [ensure-binary]
|
|
arguments:
|
|
parameters:
|
|
- name: sha
|
|
value: "{{tasks.ensure-binary.outputs.parameters.sha}}"
|
|
- name: hyperopt
|
|
template: hyperopt
|
|
dependencies: [ensure-binary, gpu-warmup, ensure-fxcache]
|
|
when: "{{workflow.parameters.hyperopt-trials}} != 0"
|
|
arguments:
|
|
parameters:
|
|
- name: sha
|
|
value: "{{tasks.ensure-binary.outputs.parameters.sha}}"
|
|
- name: train-best
|
|
template: train-best
|
|
dependencies: [ensure-binary, gpu-warmup, ensure-fxcache, hyperopt]
|
|
arguments:
|
|
parameters:
|
|
- name: sha
|
|
value: "{{tasks.ensure-binary.outputs.parameters.sha}}"
|
|
- name: evaluate
|
|
template: evaluate
|
|
dependencies: [train-best]
|
|
arguments:
|
|
parameters:
|
|
- name: sha
|
|
value: "{{tasks.ensure-binary.outputs.parameters.sha}}"
|
|
- name: upload-results
|
|
template: upload-results
|
|
dependencies: [evaluate]
|
|
|
|
# ── ensure-binary: cache-or-compile training binaries by commit SHA ──
|
|
- name: ensure-binary
|
|
outputs:
|
|
parameters:
|
|
- name: sha
|
|
valueFrom:
|
|
path: /tmp/sha
|
|
nodeSelector:
|
|
k8s.scaleway.com/pool-name: ci-compile-cpu
|
|
tolerations:
|
|
- key: node.cilium.io/agent-not-ready
|
|
operator: Exists
|
|
effect: NoSchedule
|
|
container:
|
|
image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/ci-builder:latest
|
|
imagePullPolicy: Always
|
|
command: ["/bin/bash", "-c"]
|
|
env:
|
|
- name: SQLX_OFFLINE
|
|
value: "true"
|
|
- name: CARGO_TERM_COLOR
|
|
value: always
|
|
- name: CARGO_TARGET_DIR
|
|
value: /cargo-target
|
|
- name: CARGO_HOME
|
|
value: /cargo-target/cargo-home
|
|
- name: GITLAB_PAT
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: gitlab-pat
|
|
key: token
|
|
resources:
|
|
requests:
|
|
cpu: "14"
|
|
memory: 32Gi
|
|
limits:
|
|
cpu: "30"
|
|
memory: 64Gi
|
|
volumeMounts:
|
|
- name: git-ssh-key
|
|
mountPath: /etc/git-ssh
|
|
readOnly: true
|
|
- name: cargo-target-cuda
|
|
mountPath: /cargo-target
|
|
- name: training-data
|
|
mountPath: /data
|
|
args:
|
|
- |
|
|
set -e
|
|
SHA="{{workflow.parameters.commit-sha}}"
|
|
BRANCH="{{workflow.parameters.git-branch}}"
|
|
MODEL="{{workflow.parameters.model}}"
|
|
|
|
# ── SSH setup ──
|
|
mkdir -p ~/.ssh
|
|
cp /etc/git-ssh/ssh-privatekey ~/.ssh/id_ed25519
|
|
chmod 600 ~/.ssh/id_ed25519
|
|
printf 'Host *\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n' > ~/.ssh/config
|
|
chmod 600 ~/.ssh/config
|
|
|
|
REPO="ssh://git@gitlab-gitlab-shell.foxhunt.svc.cluster.local:2222/root/foxhunt.git"
|
|
BUILD="/cargo-target/src"
|
|
|
|
git config --global --add safe.directory "$BUILD"
|
|
|
|
# ── Resolve SHA ──
|
|
if [ "$SHA" = "HEAD" ]; then
|
|
if [ -d "$BUILD/.git" ]; then
|
|
cd "$BUILD"
|
|
git fetch origin
|
|
SHA=$(git rev-parse "origin/$BRANCH")
|
|
cd /
|
|
else
|
|
git clone --filter=blob:none "$REPO" "$BUILD"
|
|
cd "$BUILD"
|
|
git checkout "origin/$BRANCH"
|
|
SHA=$(git rev-parse HEAD)
|
|
cd /
|
|
fi
|
|
fi
|
|
SHORT_SHA=$(echo "$SHA" | cut -c1-9)
|
|
echo "Resolved SHA: $SHA (short: $SHORT_SHA)"
|
|
|
|
# ── Derive needed binaries from model ──
|
|
case "$MODEL" in
|
|
dqn|ppo)
|
|
BINARIES="hyperopt_baseline_rl train_baseline_rl evaluate_baseline precompute_features"
|
|
;;
|
|
*)
|
|
BINARIES="hyperopt_baseline_supervised train_baseline_supervised evaluate_supervised precompute_features"
|
|
;;
|
|
esac
|
|
|
|
# ── Cache check ──
|
|
BIN_DIR="/data/bin/$SHORT_SHA"
|
|
ALL_CACHED=true
|
|
for bin in $BINARIES; do
|
|
if [ ! -x "$BIN_DIR/$bin" ]; then
|
|
ALL_CACHED=false
|
|
break
|
|
fi
|
|
done
|
|
|
|
if [ "$ALL_CACHED" = "true" ]; then
|
|
echo "=== Cache HIT: all binaries present in $BIN_DIR ==="
|
|
ls -lh "$BIN_DIR/"
|
|
echo "$SHORT_SHA" > /tmp/sha
|
|
exit 0
|
|
fi
|
|
|
|
echo "=== Cache MISS: compiling binaries for $SHORT_SHA ==="
|
|
|
|
# ── Clone / checkout ──
|
|
if [ -d "$BUILD/.git" ]; then
|
|
cd "$BUILD"
|
|
git fetch origin
|
|
CURRENT=$(git rev-parse HEAD 2>/dev/null || echo "none")
|
|
if [ "$CURRENT" != "$SHA" ]; then
|
|
echo "Updating checkout: $(echo $CURRENT | cut -c1-8) -> $SHORT_SHA"
|
|
git checkout --force "$SHA"
|
|
git clean -fd
|
|
fi
|
|
else
|
|
git clone --filter=blob:none "$REPO" "$BUILD"
|
|
cd "$BUILD"
|
|
git checkout "$SHA"
|
|
fi
|
|
|
|
# ── Build ──
|
|
export PATH="${CARGO_HOME}/bin:${PATH}"
|
|
export CUDA_COMPUTE_CAP={{workflow.parameters.cuda-compute-cap}}
|
|
|
|
ML_EXAMPLE_ARGS=""
|
|
for ex in $BINARIES; do
|
|
ML_EXAMPLE_ARGS="$ML_EXAMPLE_ARGS --example $ex"
|
|
done
|
|
|
|
echo "Building: $BINARIES"
|
|
echo " CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP"
|
|
cargo build --release -p ml --features ml/cuda $ML_EXAMPLE_ARGS
|
|
|
|
# ── Install to cache dir ──
|
|
mkdir -p "$BIN_DIR"
|
|
for bin in $BINARIES; do
|
|
cp "$CARGO_TARGET_DIR/release/examples/$bin" "$BIN_DIR/"
|
|
done
|
|
strip "$BIN_DIR/"*
|
|
|
|
echo "=== Cached binaries ==="
|
|
ls -lh "$BIN_DIR/"
|
|
|
|
# ── Prune old SHAs: keep last 5 ──
|
|
cd /data/bin
|
|
ls -1t | tail -n +6 | while read -r old; do
|
|
echo "Pruning old cache: $old"
|
|
rm -rf "$old"
|
|
done
|
|
|
|
echo "$SHORT_SHA" > /tmp/sha
|
|
|
|
# ── ensure-fxcache: precompute feature cache if needed ──
|
|
- name: ensure-fxcache
|
|
inputs:
|
|
parameters:
|
|
- name: sha
|
|
nodeSelector:
|
|
k8s.scaleway.com/pool-name: ci-compile-cpu
|
|
tolerations:
|
|
- key: node.cilium.io/agent-not-ready
|
|
operator: Exists
|
|
effect: NoSchedule
|
|
container:
|
|
image: ubuntu:24.04
|
|
command: ["/bin/bash", "-c"]
|
|
env:
|
|
- name: RUST_LOG
|
|
value: info
|
|
resources:
|
|
requests:
|
|
cpu: "4"
|
|
memory: 16Gi
|
|
limits:
|
|
cpu: "28"
|
|
memory: 56Gi
|
|
volumeMounts:
|
|
- name: training-data
|
|
mountPath: /data
|
|
readOnly: true
|
|
- name: feature-cache
|
|
mountPath: /feature-cache
|
|
args:
|
|
- |
|
|
set -e
|
|
SHA="{{inputs.parameters.sha}}"
|
|
BINARY="/data/bin/$SHA/precompute_features"
|
|
|
|
if [ ! -x "$BINARY" ]; then
|
|
echo "ERROR: precompute_features not found at $BINARY"
|
|
exit 1
|
|
fi
|
|
|
|
# fxcache has built-in version validation (FXCACHE_VERSION in header).
|
|
# The precompute binary checks existing cache — if version matches,
|
|
# it skips regeneration. If version mismatches, it fails and we
|
|
# delete + regenerate. No unconditional rm — cache is reused when valid.
|
|
echo "=== Running precompute_features (SHA: $SHA) ==="
|
|
if $BINARY \
|
|
--data-dir /data/futures-baseline \
|
|
--mbp10-data-dir /data/futures-baseline-mbp10 \
|
|
--trades-data-dir /data/futures-baseline-trades \
|
|
--output-dir /feature-cache \
|
|
--symbol {{workflow.parameters.symbol}} \
|
|
--yes; then
|
|
echo "=== Feature cache ready ==="
|
|
else
|
|
echo "=== Cache stale or missing — regenerating ==="
|
|
rm -f /feature-cache/*.fxcache
|
|
$BINARY \
|
|
--data-dir /data/futures-baseline \
|
|
--mbp10-data-dir /data/futures-baseline-mbp10 \
|
|
--trades-data-dir /data/futures-baseline-trades \
|
|
--output-dir /feature-cache \
|
|
--symbol {{workflow.parameters.symbol}} \
|
|
--yes
|
|
echo "=== Feature cache regenerated ==="
|
|
fi
|
|
|
|
# ── gpu-warmup: trigger GPU node autoscale during compile ──
|
|
- name: gpu-warmup
|
|
nodeSelector:
|
|
k8s.scaleway.com/pool-name: "{{workflow.parameters.gpu-pool}}"
|
|
tolerations:
|
|
- key: nvidia.com/gpu
|
|
operator: Exists
|
|
effect: NoSchedule
|
|
- key: node.cilium.io/agent-not-ready
|
|
operator: Exists
|
|
effect: NoSchedule
|
|
container:
|
|
image: busybox:1.37
|
|
command: ["/bin/sh", "-c"]
|
|
args:
|
|
- |
|
|
echo "GPU warmup: triggering node autoscale..."
|
|
echo "GPU node scheduled, exiting to free resources"
|
|
resources:
|
|
requests:
|
|
nvidia.com/gpu: "1"
|
|
cpu: 100m
|
|
memory: 64Mi
|
|
limits:
|
|
nvidia.com/gpu: "1"
|
|
cpu: 200m
|
|
memory: 128Mi
|
|
|
|
# ── hyperopt: PSO/TPE hyperparameter optimization on GPU ──
|
|
- name: hyperopt
|
|
inputs:
|
|
parameters:
|
|
- name: sha
|
|
nodeSelector:
|
|
k8s.scaleway.com/pool-name: "{{workflow.parameters.gpu-pool}}"
|
|
tolerations:
|
|
- key: nvidia.com/gpu
|
|
operator: Exists
|
|
effect: NoSchedule
|
|
- key: node.cilium.io/agent-not-ready
|
|
operator: Exists
|
|
effect: NoSchedule
|
|
container:
|
|
image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/ci-builder:latest
|
|
imagePullPolicy: IfNotPresent
|
|
command: ["/bin/sh", "-c"]
|
|
env:
|
|
- name: RUST_LOG
|
|
value: info
|
|
- name: SQLX_OFFLINE
|
|
value: "true"
|
|
- name: CUBLAS_WORKSPACE_CONFIG
|
|
value: ":4096:8"
|
|
- name: FOXHUNT_FEATURE_CACHE_DIR
|
|
value: /feature-cache
|
|
resources:
|
|
requests:
|
|
nvidia.com/gpu: "1"
|
|
cpu: "4"
|
|
memory: 32Gi
|
|
limits:
|
|
nvidia.com/gpu: "1"
|
|
cpu: "8"
|
|
memory: 64Gi
|
|
volumeMounts:
|
|
- name: workspace
|
|
mountPath: /workspace
|
|
- name: training-data
|
|
mountPath: /data
|
|
readOnly: true
|
|
- name: feature-cache
|
|
mountPath: /feature-cache
|
|
readOnly: true
|
|
args:
|
|
- |
|
|
set -e
|
|
SHA="{{inputs.parameters.sha}}"
|
|
export PATH="/data/bin/$SHA:$PATH"
|
|
export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//')
|
|
nvidia-smi
|
|
|
|
mkdir -p /workspace/output/hyperopt
|
|
|
|
MODEL="{{workflow.parameters.model}}"
|
|
case "$MODEL" in
|
|
dqn|ppo) BINARY=hyperopt_baseline_rl ;;
|
|
*) BINARY=hyperopt_baseline_supervised ;;
|
|
esac
|
|
|
|
echo "=== Running hyperopt: $MODEL ($BINARY) ==="
|
|
echo " Trials: {{workflow.parameters.hyperopt-trials}}, Epochs: {{workflow.parameters.hyperopt-epochs}}"
|
|
|
|
${BINARY} \
|
|
--model "$MODEL" \
|
|
--phase fast \
|
|
--trials {{workflow.parameters.hyperopt-trials}} \
|
|
--epochs {{workflow.parameters.hyperopt-epochs}} \
|
|
--symbol {{workflow.parameters.symbol}} \
|
|
--tx-cost-bps {{workflow.parameters.tx-cost-bps}} \
|
|
--tick-size {{workflow.parameters.tick-size}} \
|
|
--spread-ticks {{workflow.parameters.spread-ticks}} \
|
|
--initial-capital {{workflow.parameters.initial-capital}} \
|
|
--data-dir /data/futures-baseline \
|
|
--mbp10-data-dir /data/futures-baseline-mbp10 \
|
|
--trades-data-dir /data/futures-baseline-trades \
|
|
--base-dir /workspace/output/hyperopt \
|
|
--output /workspace/output/${MODEL}_hyperopt_results.json
|
|
|
|
echo "=== Hyperopt complete ==="
|
|
cat /workspace/output/${MODEL}_hyperopt_results.json 2>/dev/null || echo "No results file"
|
|
|
|
# ── train-best: full training with best hyperparams ──
|
|
- name: train-best
|
|
inputs:
|
|
parameters:
|
|
- name: sha
|
|
nodeSelector:
|
|
k8s.scaleway.com/pool-name: "{{workflow.parameters.gpu-pool}}"
|
|
tolerations:
|
|
- key: nvidia.com/gpu
|
|
operator: Exists
|
|
effect: NoSchedule
|
|
- key: node.cilium.io/agent-not-ready
|
|
operator: Exists
|
|
effect: NoSchedule
|
|
container:
|
|
image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/ci-builder:latest
|
|
imagePullPolicy: IfNotPresent
|
|
command: ["/bin/sh", "-c"]
|
|
env:
|
|
- name: RUST_LOG
|
|
value: info
|
|
- name: SQLX_OFFLINE
|
|
value: "true"
|
|
- name: CUBLAS_WORKSPACE_CONFIG
|
|
value: ":4096:8"
|
|
- name: FOXHUNT_FEATURE_CACHE_DIR
|
|
value: /feature-cache
|
|
resources:
|
|
requests:
|
|
nvidia.com/gpu: "1"
|
|
cpu: "4"
|
|
memory: 32Gi
|
|
limits:
|
|
nvidia.com/gpu: "1"
|
|
cpu: "8"
|
|
memory: 64Gi
|
|
volumeMounts:
|
|
- name: workspace
|
|
mountPath: /workspace
|
|
- name: training-data
|
|
mountPath: /data
|
|
readOnly: true
|
|
- name: feature-cache
|
|
mountPath: /feature-cache
|
|
readOnly: true
|
|
args:
|
|
- |
|
|
set -e
|
|
SHA="{{inputs.parameters.sha}}"
|
|
export PATH="/data/bin/$SHA:$PATH"
|
|
export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//')
|
|
nvidia-smi
|
|
|
|
MODEL="{{workflow.parameters.model}}"
|
|
case "$MODEL" in
|
|
dqn|ppo) BINARY=train_baseline_rl ;;
|
|
*) BINARY=train_baseline_supervised ;;
|
|
esac
|
|
|
|
HYPEROPT_FLAG=""
|
|
if [ -f "/workspace/output/${MODEL}_hyperopt_results.json" ]; then
|
|
HYPEROPT_FLAG="--hyperopt-params /workspace/output/${MODEL}_hyperopt_results.json"
|
|
echo " Using hyperopt results: ${MODEL}_hyperopt_results.json"
|
|
else
|
|
echo " No hyperopt results — training with default hyperparams"
|
|
fi
|
|
|
|
echo "=== Training: $MODEL ({{workflow.parameters.train-epochs}} epochs) ==="
|
|
|
|
# compute-sanitizer / nsys: optional GPU debugging/profiling
|
|
SANITIZER="{{workflow.parameters.sanitizer}}"
|
|
SANITIZER_PREFIX=""
|
|
if [ "$SANITIZER" = "nsys" ]; then
|
|
NSYS_BIN=$(which nsys 2>/dev/null || echo "/usr/local/cuda/bin/nsys")
|
|
if [ -x "$NSYS_BIN" ]; then
|
|
NSYS_OUT="/feature-cache/nsys_$(date +%Y%m%d_%H%M%S)"
|
|
mkdir -p /feature-cache
|
|
SANITIZER_PREFIX="$NSYS_BIN profile -o $NSYS_OUT --cuda-graph-trace=node --stats=true --show-output=true -f true --duration=60"
|
|
echo " nsys profiling enabled — output: ${NSYS_OUT}.nsys-rep (persistent PVC)"
|
|
echo " --cuda-graph-trace=node: per-kernel timing inside parent graph"
|
|
echo " --cudabacktrace=kernel: kernel source attribution"
|
|
echo " --capture-range=cudaProfilerApi: use cudaProfilerStart/Stop to limit capture"
|
|
echo " WARNING: ~2x slower — use with 1-2 epochs only"
|
|
else
|
|
echo " WARNING: nsys not found at $NSYS_BIN — running without"
|
|
fi
|
|
elif [ "$SANITIZER" != "none" ] && [ -n "$SANITIZER" ]; then
|
|
SANITIZER_BIN=$(which compute-sanitizer 2>/dev/null || echo "/usr/local/cuda/bin/compute-sanitizer")
|
|
if [ -x "$SANITIZER_BIN" ]; then
|
|
SANITIZER_PREFIX="$SANITIZER_BIN --tool $SANITIZER --print-limit 20 --error-exitcode 1"
|
|
echo " compute-sanitizer enabled: --tool $SANITIZER"
|
|
echo " WARNING: 10-100x slower — use with 1-2 epochs only"
|
|
else
|
|
echo " WARNING: compute-sanitizer not found at $SANITIZER_BIN — running without"
|
|
fi
|
|
fi
|
|
|
|
stdbuf -oL $SANITIZER_PREFIX ${BINARY} \
|
|
--model "$MODEL" \
|
|
--symbol {{workflow.parameters.symbol}} \
|
|
--tx-cost-bps {{workflow.parameters.tx-cost-bps}} \
|
|
--tick-size {{workflow.parameters.tick-size}} \
|
|
--spread-ticks {{workflow.parameters.spread-ticks}} \
|
|
--initial-capital {{workflow.parameters.initial-capital}} \
|
|
--data-dir /data/futures-baseline \
|
|
--mbp10-data-dir /data/futures-baseline-mbp10 \
|
|
--trades-data-dir /data/futures-baseline-trades \
|
|
--output-dir /workspace/output \
|
|
--epochs {{workflow.parameters.train-epochs}} \
|
|
$HYPEROPT_FLAG
|
|
|
|
echo "=== Training complete ==="
|
|
|
|
# ── evaluate: run evaluation on trained model ──
|
|
- name: evaluate
|
|
inputs:
|
|
parameters:
|
|
- name: sha
|
|
nodeSelector:
|
|
k8s.scaleway.com/pool-name: "{{workflow.parameters.gpu-pool}}"
|
|
tolerations:
|
|
- key: nvidia.com/gpu
|
|
operator: Exists
|
|
effect: NoSchedule
|
|
- key: node.cilium.io/agent-not-ready
|
|
operator: Exists
|
|
effect: NoSchedule
|
|
container:
|
|
image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/ci-builder:latest
|
|
imagePullPolicy: IfNotPresent
|
|
command: ["/bin/sh", "-c"]
|
|
env:
|
|
- name: RUST_LOG
|
|
value: info
|
|
- name: SQLX_OFFLINE
|
|
value: "true"
|
|
resources:
|
|
requests:
|
|
nvidia.com/gpu: "1"
|
|
cpu: "2"
|
|
memory: 16Gi
|
|
limits:
|
|
nvidia.com/gpu: "1"
|
|
cpu: "4"
|
|
memory: 32Gi
|
|
volumeMounts:
|
|
- name: workspace
|
|
mountPath: /workspace
|
|
- name: training-data
|
|
mountPath: /data
|
|
readOnly: true
|
|
- name: feature-cache
|
|
mountPath: /feature-cache
|
|
readOnly: true
|
|
args:
|
|
- |
|
|
set -e
|
|
SHA="{{inputs.parameters.sha}}"
|
|
export PATH="/data/bin/$SHA:$PATH"
|
|
export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//')
|
|
|
|
MODEL="{{workflow.parameters.model}}"
|
|
case "$MODEL" in
|
|
dqn|ppo) BINARY=evaluate_baseline ;;
|
|
*) BINARY=evaluate_supervised ;;
|
|
esac
|
|
|
|
echo "=== Evaluating: $MODEL ==="
|
|
${BINARY} \
|
|
--model "$MODEL" \
|
|
--symbol {{workflow.parameters.symbol}} \
|
|
--data-dir /data/futures-baseline \
|
|
--checkpoint-dir /workspace/output \
|
|
--output-dir /workspace/output/eval \
|
|
--initial-capital {{workflow.parameters.initial-capital}} \
|
|
--tx-cost-bps {{workflow.parameters.tx-cost-bps}} \
|
|
--tick-size {{workflow.parameters.tick-size}} \
|
|
--spread-ticks {{workflow.parameters.spread-ticks}} || {
|
|
echo "WARN: Evaluation failed, continuing"
|
|
}
|
|
|
|
echo "=== Evaluation complete ==="
|
|
|
|
# ── upload-results: push artifacts to GitLab packages ──
|
|
- name: upload-results
|
|
nodeSelector:
|
|
k8s.scaleway.com/pool-name: platform
|
|
container:
|
|
image: curlimages/curl:8.12.1
|
|
command: ["/bin/sh", "-c"]
|
|
env:
|
|
- name: GITLAB_PAT
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: gitlab-pat
|
|
key: token
|
|
resources:
|
|
requests:
|
|
cpu: 100m
|
|
memory: 128Mi
|
|
limits:
|
|
cpu: 500m
|
|
memory: 256Mi
|
|
volumeMounts:
|
|
- name: workspace
|
|
mountPath: /workspace
|
|
args:
|
|
- |
|
|
set -e
|
|
MODEL="{{workflow.parameters.model}}"
|
|
SYMBOL="{{workflow.parameters.symbol}}"
|
|
TIMESTAMP=$(date +%Y%m%d-%H%M%S)
|
|
GITLAB_API="http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181/api/v4"
|
|
PKG_NAME="foxhunt-training-results"
|
|
PKG_VERSION="${MODEL}-${SYMBOL}-${TIMESTAMP}"
|
|
|
|
echo "Uploading training artifacts: ${PKG_NAME}/${PKG_VERSION}"
|
|
|
|
UPLOADED=0
|
|
find /workspace/output -type f | while read -r file; do
|
|
REL_PATH="${file#/workspace/output/}"
|
|
SAFE_NAME=$(echo "$REL_PATH" | tr '/' '--')
|
|
echo " Uploading ${REL_PATH} as ${SAFE_NAME}..."
|
|
curl -f --upload-file "$file" \
|
|
-H "PRIVATE-TOKEN: ${GITLAB_PAT}" \
|
|
"${GITLAB_API}/projects/1/packages/generic/${PKG_NAME}/${PKG_VERSION}/${SAFE_NAME}" && \
|
|
UPLOADED=$((UPLOADED + 1)) || \
|
|
echo " WARN: Failed to upload ${REL_PATH}"
|
|
done
|
|
|
|
echo "=== Upload complete (${UPLOADED} files) ==="
|
|
echo "Package: ${PKG_NAME}/${PKG_VERSION}"
|
|
|
|
# ── notify-result: post workflow outcome to Mattermost (onExit) ──
|
|
- name: notify-result
|
|
nodeSelector:
|
|
k8s.scaleway.com/pool-name: platform
|
|
container:
|
|
image: curlimages/curl:8.12.1
|
|
command: ["/bin/sh", "-c"]
|
|
env:
|
|
- name: WEBHOOK_URL
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: notification-webhook
|
|
key: webhook-url
|
|
optional: true
|
|
resources:
|
|
requests:
|
|
cpu: 50m
|
|
memory: 32Mi
|
|
limits:
|
|
cpu: 100m
|
|
memory: 64Mi
|
|
args:
|
|
- |
|
|
STATUS="{{workflow.status}}"
|
|
NAME="{{workflow.name}}"
|
|
|
|
if [ -z "$WEBHOOK_URL" ] || echo "$WEBHOOK_URL" | grep -q "PLACEHOLDER"; then
|
|
echo "No webhook configured, skipping notification"
|
|
exit 0
|
|
fi
|
|
|
|
if [ "$STATUS" = "Succeeded" ]; then
|
|
EMOJI=":white_check_mark:"
|
|
else
|
|
EMOJI=":x:"
|
|
fi
|
|
|
|
PAYLOAD="{\"username\":\"Argo CI\",\"text\":\"${EMOJI} **${NAME}** — ${STATUS} ({{workflow.duration}}s)\"}"
|
|
|
|
curl -sf -X POST -H 'Content-Type: application/json' \
|
|
-d "$PAYLOAD" "$WEBHOOK_URL" || echo "WARN: webhook post failed"
|