feat: unified train workflow template — cache-or-compile + cache-or-precompute
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
jgrusewski
679
infra/k8s/argo/train-template.yaml
Normal file
679
infra/k8s/argo/train-template.yaml
Normal file
@@ -0,0 +1,679 @@
|
||||
# Unified training workflow — replaces all model-specific training templates.
|
||||
#
|
||||
# Key innovation: binary caching by commit SHA on PVC.
|
||||
# Check /data/bin/$SHA first; compile only on cache miss.
|
||||
#
|
||||
# DAG:
|
||||
# ensure-binary ──┐
|
||||
# gpu-warmup ─────┼──> ensure-fxcache ──> hyperopt ──> train-best ──> evaluate ──> upload-results
|
||||
#
|
||||
# Usage:
|
||||
# argo submit -n foxhunt --from=wftmpl/train
|
||||
# argo submit -n foxhunt --from=wftmpl/train -p model=ppo -p train-epochs=100
|
||||
# argo submit -n foxhunt --from=wftmpl/train -p hyperopt-trials=0 # skip hyperopt
|
||||
---
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: WorkflowTemplate
|
||||
metadata:
|
||||
name: train
|
||||
namespace: foxhunt
|
||||
labels:
|
||||
app.kubernetes.io/name: train
|
||||
app.kubernetes.io/part-of: foxhunt
|
||||
app.kubernetes.io/component: training
|
||||
spec:
|
||||
entrypoint: pipeline
|
||||
onExit: notify-result
|
||||
serviceAccountName: argo-workflow
|
||||
podMetadata:
|
||||
labels:
|
||||
app.kubernetes.io/part-of: foxhunt
|
||||
app.kubernetes.io/component: training
|
||||
securityContext:
|
||||
fsGroup: 0
|
||||
ttlStrategy:
|
||||
secondsAfterCompletion: 3600
|
||||
activeDeadlineSeconds: 28800 # 8 hours
|
||||
|
||||
arguments:
|
||||
parameters:
|
||||
- name: commit-sha
|
||||
value: HEAD
|
||||
- name: git-branch
|
||||
value: main
|
||||
- name: cuda-compute-cap
|
||||
value: "90"
|
||||
- name: model
|
||||
value: dqn
|
||||
- name: gpu-pool
|
||||
value: ci-training-h100
|
||||
- name: hyperopt-trials
|
||||
value: "20"
|
||||
- name: hyperopt-epochs
|
||||
value: "8"
|
||||
- name: train-epochs
|
||||
value: "50"
|
||||
- name: symbol
|
||||
value: ES.FUT
|
||||
- name: initial-capital
|
||||
value: "35000"
|
||||
- name: tx-cost-bps
|
||||
value: "0.1"
|
||||
- name: tick-size
|
||||
value: "0.25"
|
||||
- name: spread-ticks
|
||||
value: "1.0"
|
||||
- name: min-hold-bars
|
||||
value: "5"
|
||||
|
||||
volumes:
|
||||
- name: git-ssh-key
|
||||
secret:
|
||||
secretName: argo-git-ssh-key
|
||||
defaultMode: 256
|
||||
- name: training-data
|
||||
persistentVolumeClaim:
|
||||
claimName: training-data-pvc
|
||||
- name: cargo-target-cuda
|
||||
persistentVolumeClaim:
|
||||
claimName: cargo-target-cuda
|
||||
- name: feature-cache
|
||||
persistentVolumeClaim:
|
||||
claimName: feature-cache-pvc
|
||||
|
||||
volumeClaimTemplates:
|
||||
- metadata:
|
||||
name: workspace
|
||||
spec:
|
||||
accessModes: ["ReadWriteOnce"]
|
||||
storageClassName: scw-bssd
|
||||
resources:
|
||||
requests:
|
||||
storage: 5Gi
|
||||
|
||||
templates:
|
||||
# ── DAG: orchestrate all steps ──
|
||||
- name: pipeline
|
||||
dag:
|
||||
tasks:
|
||||
- name: ensure-binary
|
||||
template: ensure-binary
|
||||
- name: gpu-warmup
|
||||
template: gpu-warmup
|
||||
- name: ensure-fxcache
|
||||
template: ensure-fxcache
|
||||
dependencies: [ensure-binary]
|
||||
- name: hyperopt
|
||||
template: hyperopt
|
||||
dependencies: [ensure-binary, gpu-warmup, ensure-fxcache]
|
||||
when: "{{workflow.parameters.hyperopt-trials}} != 0"
|
||||
- name: train-best
|
||||
template: train-best
|
||||
dependencies: [ensure-binary, gpu-warmup, ensure-fxcache, hyperopt]
|
||||
- name: evaluate
|
||||
template: evaluate
|
||||
dependencies: [train-best]
|
||||
- name: upload-results
|
||||
template: upload-results
|
||||
dependencies: [evaluate]
|
||||
|
||||
# ── ensure-binary: cache-or-compile training binaries by commit SHA ──
|
||||
- name: ensure-binary
|
||||
outputs:
|
||||
parameters:
|
||||
- name: sha
|
||||
valueFrom:
|
||||
path: /tmp/sha
|
||||
nodeSelector:
|
||||
k8s.scaleway.com/pool-name: ci-compile-cpu
|
||||
tolerations:
|
||||
- key: node.cilium.io/agent-not-ready
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
container:
|
||||
image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/ci-builder:latest
|
||||
imagePullPolicy: Always
|
||||
command: ["/bin/bash", "-c"]
|
||||
env:
|
||||
- name: SQLX_OFFLINE
|
||||
value: "true"
|
||||
- name: CARGO_TERM_COLOR
|
||||
value: always
|
||||
- name: CARGO_TARGET_DIR
|
||||
value: /cargo-target
|
||||
- name: CARGO_HOME
|
||||
value: /cargo-target/cargo-home
|
||||
- name: GITLAB_PAT
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: gitlab-pat
|
||||
key: token
|
||||
resources:
|
||||
requests:
|
||||
cpu: "14"
|
||||
memory: 32Gi
|
||||
limits:
|
||||
cpu: "30"
|
||||
memory: 64Gi
|
||||
volumeMounts:
|
||||
- name: git-ssh-key
|
||||
mountPath: /etc/git-ssh
|
||||
readOnly: true
|
||||
- name: cargo-target-cuda
|
||||
mountPath: /cargo-target
|
||||
- name: training-data
|
||||
mountPath: /data
|
||||
args:
|
||||
- |
|
||||
set -e
|
||||
SHA="{{workflow.parameters.commit-sha}}"
|
||||
BRANCH="{{workflow.parameters.git-branch}}"
|
||||
MODEL="{{workflow.parameters.model}}"
|
||||
|
||||
# ── SSH setup ──
|
||||
mkdir -p ~/.ssh
|
||||
cp /etc/git-ssh/ssh-privatekey ~/.ssh/id_ed25519
|
||||
chmod 600 ~/.ssh/id_ed25519
|
||||
printf 'Host *\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n' > ~/.ssh/config
|
||||
chmod 600 ~/.ssh/config
|
||||
|
||||
REPO="ssh://git@gitlab-gitlab-shell.foxhunt.svc.cluster.local:2222/root/foxhunt.git"
|
||||
BUILD="/cargo-target/src"
|
||||
|
||||
git config --global --add safe.directory "$BUILD"
|
||||
|
||||
# ── Resolve SHA ──
|
||||
if [ "$SHA" = "HEAD" ]; then
|
||||
if [ -d "$BUILD/.git" ]; then
|
||||
cd "$BUILD"
|
||||
git fetch origin
|
||||
SHA=$(git rev-parse "origin/$BRANCH")
|
||||
cd /
|
||||
else
|
||||
git clone --filter=blob:none "$REPO" "$BUILD"
|
||||
cd "$BUILD"
|
||||
git checkout "origin/$BRANCH"
|
||||
SHA=$(git rev-parse HEAD)
|
||||
cd /
|
||||
fi
|
||||
fi
|
||||
SHORT_SHA=$(echo "$SHA" | cut -c1-9)
|
||||
echo "Resolved SHA: $SHA (short: $SHORT_SHA)"
|
||||
|
||||
# ── Derive needed binaries from model ──
|
||||
case "$MODEL" in
|
||||
dqn|ppo)
|
||||
BINARIES="hyperopt_baseline_rl train_baseline_rl evaluate_baseline precompute_features"
|
||||
;;
|
||||
*)
|
||||
BINARIES="hyperopt_baseline_supervised train_baseline_supervised evaluate_supervised precompute_features"
|
||||
;;
|
||||
esac
|
||||
|
||||
# ── Cache check ──
|
||||
BIN_DIR="/data/bin/$SHORT_SHA"
|
||||
ALL_CACHED=true
|
||||
for bin in $BINARIES; do
|
||||
if [ ! -x "$BIN_DIR/$bin" ]; then
|
||||
ALL_CACHED=false
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$ALL_CACHED" = "true" ]; then
|
||||
echo "=== Cache HIT: all binaries present in $BIN_DIR ==="
|
||||
ls -lh "$BIN_DIR/"
|
||||
echo "$SHORT_SHA" > /tmp/sha
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "=== Cache MISS: compiling binaries for $SHORT_SHA ==="
|
||||
|
||||
# ── Clone / checkout ──
|
||||
if [ -d "$BUILD/.git" ]; then
|
||||
cd "$BUILD"
|
||||
git fetch origin
|
||||
CURRENT=$(git rev-parse HEAD 2>/dev/null || echo "none")
|
||||
if [ "$CURRENT" != "$SHA" ]; then
|
||||
echo "Updating checkout: $(echo $CURRENT | cut -c1-8) -> $SHORT_SHA"
|
||||
git checkout --force "$SHA"
|
||||
git clean -fd
|
||||
fi
|
||||
else
|
||||
git clone --filter=blob:none "$REPO" "$BUILD"
|
||||
cd "$BUILD"
|
||||
git checkout "$SHA"
|
||||
fi
|
||||
|
||||
# ── Build ──
|
||||
export PATH="${CARGO_HOME}/bin:${PATH}"
|
||||
export CUDA_COMPUTE_CAP={{workflow.parameters.cuda-compute-cap}}
|
||||
|
||||
ML_EXAMPLE_ARGS=""
|
||||
for ex in $BINARIES; do
|
||||
ML_EXAMPLE_ARGS="$ML_EXAMPLE_ARGS --example $ex"
|
||||
done
|
||||
|
||||
echo "Building: $BINARIES"
|
||||
echo " CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP"
|
||||
cargo build --release -p ml --features ml/cuda $ML_EXAMPLE_ARGS
|
||||
|
||||
# ── Install to cache dir ──
|
||||
mkdir -p "$BIN_DIR"
|
||||
for bin in $BINARIES; do
|
||||
cp "$CARGO_TARGET_DIR/release/examples/$bin" "$BIN_DIR/"
|
||||
done
|
||||
strip "$BIN_DIR/"*
|
||||
|
||||
echo "=== Cached binaries ==="
|
||||
ls -lh "$BIN_DIR/"
|
||||
|
||||
# ── Prune old SHAs: keep last 5 ──
|
||||
cd /data/bin
|
||||
ls -1t | tail -n +6 | while read -r old; do
|
||||
echo "Pruning old cache: $old"
|
||||
rm -rf "$old"
|
||||
done
|
||||
|
||||
echo "$SHORT_SHA" > /tmp/sha
|
||||
|
||||
# ── ensure-fxcache: precompute feature cache if needed ──
|
||||
- name: ensure-fxcache
|
||||
nodeSelector:
|
||||
k8s.scaleway.com/pool-name: ci-compile-cpu
|
||||
tolerations:
|
||||
- key: node.cilium.io/agent-not-ready
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
container:
|
||||
image: ubuntu:24.04
|
||||
command: ["/bin/bash", "-c"]
|
||||
env:
|
||||
- name: RUST_LOG
|
||||
value: info
|
||||
resources:
|
||||
requests:
|
||||
cpu: "4"
|
||||
memory: 16Gi
|
||||
limits:
|
||||
cpu: "28"
|
||||
memory: 56Gi
|
||||
volumeMounts:
|
||||
- name: training-data
|
||||
mountPath: /data
|
||||
readOnly: true
|
||||
- name: feature-cache
|
||||
mountPath: /feature-cache
|
||||
args:
|
||||
- |
|
||||
set -e
|
||||
SHA="{{tasks.ensure-binary.outputs.parameters.sha}}"
|
||||
BINARY="/data/bin/$SHA/precompute_features"
|
||||
|
||||
if [ ! -x "$BINARY" ]; then
|
||||
echo "ERROR: precompute_features not found at $BINARY"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "=== Running precompute_features (SHA: $SHA) ==="
|
||||
$BINARY \
|
||||
--data-dir /data/futures-baseline \
|
||||
--mbp10-data-dir /data/futures-baseline-mbp10 \
|
||||
--trades-data-dir /data/futures-baseline-trades \
|
||||
--output-dir /feature-cache \
|
||||
--symbol {{workflow.parameters.symbol}} \
|
||||
--yes
|
||||
|
||||
echo "=== Feature cache ready ==="
|
||||
|
||||
# ── gpu-warmup: trigger GPU node autoscale during compile ──
|
||||
- name: gpu-warmup
|
||||
nodeSelector:
|
||||
k8s.scaleway.com/pool-name: "{{workflow.parameters.gpu-pool}}"
|
||||
tolerations:
|
||||
- key: nvidia.com/gpu
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
- key: node.cilium.io/agent-not-ready
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
container:
|
||||
image: busybox:1.37
|
||||
command: ["/bin/sh", "-c"]
|
||||
args:
|
||||
- |
|
||||
echo "GPU warmup: triggering node autoscale..."
|
||||
echo "GPU node scheduled, exiting to free resources"
|
||||
resources:
|
||||
requests:
|
||||
nvidia.com/gpu: "1"
|
||||
cpu: 100m
|
||||
memory: 64Mi
|
||||
limits:
|
||||
nvidia.com/gpu: "1"
|
||||
cpu: 200m
|
||||
memory: 128Mi
|
||||
|
||||
# ── hyperopt: PSO/TPE hyperparameter optimization on GPU ──
|
||||
- name: hyperopt
|
||||
nodeSelector:
|
||||
k8s.scaleway.com/pool-name: "{{workflow.parameters.gpu-pool}}"
|
||||
tolerations:
|
||||
- key: nvidia.com/gpu
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
- key: node.cilium.io/agent-not-ready
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
container:
|
||||
image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/ci-builder:latest
|
||||
imagePullPolicy: IfNotPresent
|
||||
command: ["/bin/sh", "-c"]
|
||||
env:
|
||||
- name: RUST_LOG
|
||||
value: info
|
||||
- name: SQLX_OFFLINE
|
||||
value: "true"
|
||||
- name: CUBLAS_WORKSPACE_CONFIG
|
||||
value: ":4096:8"
|
||||
- name: FOXHUNT_FEATURE_CACHE_DIR
|
||||
value: /feature-cache
|
||||
resources:
|
||||
requests:
|
||||
nvidia.com/gpu: "1"
|
||||
cpu: "4"
|
||||
memory: 32Gi
|
||||
limits:
|
||||
nvidia.com/gpu: "1"
|
||||
cpu: "8"
|
||||
memory: 64Gi
|
||||
volumeMounts:
|
||||
- name: workspace
|
||||
mountPath: /workspace
|
||||
- name: training-data
|
||||
mountPath: /data
|
||||
readOnly: true
|
||||
- name: feature-cache
|
||||
mountPath: /feature-cache
|
||||
readOnly: true
|
||||
args:
|
||||
- |
|
||||
set -e
|
||||
SHA="{{tasks.ensure-binary.outputs.parameters.sha}}"
|
||||
export PATH="/data/bin/$SHA:$PATH"
|
||||
export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//')
|
||||
nvidia-smi
|
||||
|
||||
mkdir -p /workspace/output/hyperopt
|
||||
|
||||
MODEL="{{workflow.parameters.model}}"
|
||||
case "$MODEL" in
|
||||
dqn|ppo) BINARY=hyperopt_baseline_rl ;;
|
||||
*) BINARY=hyperopt_baseline_supervised ;;
|
||||
esac
|
||||
|
||||
echo "=== Running hyperopt: $MODEL ($BINARY) ==="
|
||||
echo " Trials: {{workflow.parameters.hyperopt-trials}}, Epochs: {{workflow.parameters.hyperopt-epochs}}"
|
||||
|
||||
${BINARY} \
|
||||
--model "$MODEL" \
|
||||
--phase fast \
|
||||
--trials {{workflow.parameters.hyperopt-trials}} \
|
||||
--epochs {{workflow.parameters.hyperopt-epochs}} \
|
||||
--symbol {{workflow.parameters.symbol}} \
|
||||
--tx-cost-bps {{workflow.parameters.tx-cost-bps}} \
|
||||
--tick-size {{workflow.parameters.tick-size}} \
|
||||
--spread-ticks {{workflow.parameters.spread-ticks}} \
|
||||
--initial-capital {{workflow.parameters.initial-capital}} \
|
||||
--min-hold-bars {{workflow.parameters.min-hold-bars}} \
|
||||
--data-dir /data/futures-baseline \
|
||||
--mbp10-data-dir /data/futures-baseline-mbp10 \
|
||||
--trades-data-dir /data/futures-baseline-trades \
|
||||
--base-dir /workspace/output/hyperopt \
|
||||
--output /workspace/output/${MODEL}_hyperopt_results.json
|
||||
|
||||
echo "=== Hyperopt complete ==="
|
||||
cat /workspace/output/${MODEL}_hyperopt_results.json 2>/dev/null || echo "No results file"
|
||||
|
||||
# ── train-best: full training with best hyperparams ──
|
||||
- name: train-best
|
||||
nodeSelector:
|
||||
k8s.scaleway.com/pool-name: "{{workflow.parameters.gpu-pool}}"
|
||||
tolerations:
|
||||
- key: nvidia.com/gpu
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
- key: node.cilium.io/agent-not-ready
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
container:
|
||||
image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/ci-builder:latest
|
||||
imagePullPolicy: IfNotPresent
|
||||
command: ["/bin/sh", "-c"]
|
||||
env:
|
||||
- name: RUST_LOG
|
||||
value: info
|
||||
- name: SQLX_OFFLINE
|
||||
value: "true"
|
||||
- name: CUBLAS_WORKSPACE_CONFIG
|
||||
value: ":4096:8"
|
||||
- name: FOXHUNT_FEATURE_CACHE_DIR
|
||||
value: /feature-cache
|
||||
resources:
|
||||
requests:
|
||||
nvidia.com/gpu: "1"
|
||||
cpu: "4"
|
||||
memory: 32Gi
|
||||
limits:
|
||||
nvidia.com/gpu: "1"
|
||||
cpu: "8"
|
||||
memory: 64Gi
|
||||
volumeMounts:
|
||||
- name: workspace
|
||||
mountPath: /workspace
|
||||
- name: training-data
|
||||
mountPath: /data
|
||||
readOnly: true
|
||||
- name: feature-cache
|
||||
mountPath: /feature-cache
|
||||
readOnly: true
|
||||
args:
|
||||
- |
|
||||
set -e
|
||||
SHA="{{tasks.ensure-binary.outputs.parameters.sha}}"
|
||||
export PATH="/data/bin/$SHA:$PATH"
|
||||
export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//')
|
||||
nvidia-smi
|
||||
|
||||
MODEL="{{workflow.parameters.model}}"
|
||||
case "$MODEL" in
|
||||
dqn|ppo) BINARY=train_baseline_rl ;;
|
||||
*) BINARY=train_baseline_supervised ;;
|
||||
esac
|
||||
|
||||
HYPEROPT_FLAG=""
|
||||
if [ -f "/workspace/output/${MODEL}_hyperopt_results.json" ]; then
|
||||
HYPEROPT_FLAG="--hyperopt-params /workspace/output/${MODEL}_hyperopt_results.json"
|
||||
echo " Using hyperopt results: ${MODEL}_hyperopt_results.json"
|
||||
else
|
||||
echo " No hyperopt results — training with default hyperparams"
|
||||
fi
|
||||
|
||||
echo "=== Training: $MODEL ({{workflow.parameters.train-epochs}} epochs) ==="
|
||||
|
||||
stdbuf -oL ${BINARY} \
|
||||
--model "$MODEL" \
|
||||
--symbol {{workflow.parameters.symbol}} \
|
||||
--tx-cost-bps {{workflow.parameters.tx-cost-bps}} \
|
||||
--tick-size {{workflow.parameters.tick-size}} \
|
||||
--spread-ticks {{workflow.parameters.spread-ticks}} \
|
||||
--initial-capital {{workflow.parameters.initial-capital}} \
|
||||
--min-hold-bars {{workflow.parameters.min-hold-bars}} \
|
||||
--data-dir /data/futures-baseline \
|
||||
--mbp10-data-dir /data/futures-baseline-mbp10 \
|
||||
--trades-data-dir /data/futures-baseline-trades \
|
||||
--output-dir /workspace/output \
|
||||
--epochs {{workflow.parameters.train-epochs}} \
|
||||
$HYPEROPT_FLAG
|
||||
|
||||
echo "=== Training complete ==="
|
||||
|
||||
# ── evaluate: run evaluation on trained model ──
|
||||
- name: evaluate
|
||||
nodeSelector:
|
||||
k8s.scaleway.com/pool-name: "{{workflow.parameters.gpu-pool}}"
|
||||
tolerations:
|
||||
- key: nvidia.com/gpu
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
- key: node.cilium.io/agent-not-ready
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
container:
|
||||
image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/ci-builder:latest
|
||||
imagePullPolicy: IfNotPresent
|
||||
command: ["/bin/sh", "-c"]
|
||||
env:
|
||||
- name: RUST_LOG
|
||||
value: info
|
||||
- name: SQLX_OFFLINE
|
||||
value: "true"
|
||||
resources:
|
||||
requests:
|
||||
nvidia.com/gpu: "1"
|
||||
cpu: "2"
|
||||
memory: 16Gi
|
||||
limits:
|
||||
nvidia.com/gpu: "1"
|
||||
cpu: "4"
|
||||
memory: 32Gi
|
||||
volumeMounts:
|
||||
- name: workspace
|
||||
mountPath: /workspace
|
||||
- name: training-data
|
||||
mountPath: /data
|
||||
readOnly: true
|
||||
- name: feature-cache
|
||||
mountPath: /feature-cache
|
||||
readOnly: true
|
||||
args:
|
||||
- |
|
||||
set -e
|
||||
SHA="{{tasks.ensure-binary.outputs.parameters.sha}}"
|
||||
export PATH="/data/bin/$SHA:$PATH"
|
||||
export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//')
|
||||
|
||||
MODEL="{{workflow.parameters.model}}"
|
||||
case "$MODEL" in
|
||||
dqn|ppo) BINARY=evaluate_baseline ;;
|
||||
*) BINARY=evaluate_supervised ;;
|
||||
esac
|
||||
|
||||
echo "=== Evaluating: $MODEL ==="
|
||||
${BINARY} \
|
||||
--model "$MODEL" \
|
||||
--symbol {{workflow.parameters.symbol}} \
|
||||
--data-dir /data/futures-baseline \
|
||||
--checkpoint-dir /workspace/output \
|
||||
--output-dir /workspace/output/eval \
|
||||
--initial-capital {{workflow.parameters.initial-capital}} \
|
||||
--min-hold-bars {{workflow.parameters.min-hold-bars}} \
|
||||
--tx-cost-bps {{workflow.parameters.tx-cost-bps}} \
|
||||
--tick-size {{workflow.parameters.tick-size}} \
|
||||
--spread-ticks {{workflow.parameters.spread-ticks}} || {
|
||||
echo "WARN: Evaluation failed, continuing"
|
||||
}
|
||||
|
||||
echo "=== Evaluation complete ==="
|
||||
|
||||
# ── upload-results: push artifacts to GitLab packages ──
|
||||
- name: upload-results
|
||||
nodeSelector:
|
||||
k8s.scaleway.com/pool-name: platform
|
||||
container:
|
||||
image: curlimages/curl:8.12.1
|
||||
command: ["/bin/sh", "-c"]
|
||||
env:
|
||||
- name: GITLAB_PAT
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: gitlab-pat
|
||||
key: token
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 128Mi
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 256Mi
|
||||
volumeMounts:
|
||||
- name: workspace
|
||||
mountPath: /workspace
|
||||
args:
|
||||
- |
|
||||
set -e
|
||||
MODEL="{{workflow.parameters.model}}"
|
||||
SYMBOL="{{workflow.parameters.symbol}}"
|
||||
TIMESTAMP=$(date +%Y%m%d-%H%M%S)
|
||||
GITLAB_API="http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181/api/v4"
|
||||
PKG_NAME="foxhunt-training-results"
|
||||
PKG_VERSION="${MODEL}-${SYMBOL}-${TIMESTAMP}"
|
||||
|
||||
echo "Uploading training artifacts: ${PKG_NAME}/${PKG_VERSION}"
|
||||
|
||||
UPLOADED=0
|
||||
find /workspace/output -type f | while read -r file; do
|
||||
REL_PATH="${file#/workspace/output/}"
|
||||
SAFE_NAME=$(echo "$REL_PATH" | tr '/' '--')
|
||||
echo " Uploading ${REL_PATH} as ${SAFE_NAME}..."
|
||||
curl -f --upload-file "$file" \
|
||||
-H "PRIVATE-TOKEN: ${GITLAB_PAT}" \
|
||||
"${GITLAB_API}/projects/1/packages/generic/${PKG_NAME}/${PKG_VERSION}/${SAFE_NAME}" && \
|
||||
UPLOADED=$((UPLOADED + 1)) || \
|
||||
echo " WARN: Failed to upload ${REL_PATH}"
|
||||
done
|
||||
|
||||
echo "=== Upload complete (${UPLOADED} files) ==="
|
||||
echo "Package: ${PKG_NAME}/${PKG_VERSION}"
|
||||
|
||||
# ── notify-result: post workflow outcome to Mattermost (onExit) ──
|
||||
- name: notify-result
|
||||
nodeSelector:
|
||||
k8s.scaleway.com/pool-name: platform
|
||||
container:
|
||||
image: curlimages/curl:8.12.1
|
||||
command: ["/bin/sh", "-c"]
|
||||
env:
|
||||
- name: WEBHOOK_URL
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: notification-webhook
|
||||
key: webhook-url
|
||||
optional: true
|
||||
resources:
|
||||
requests:
|
||||
cpu: 50m
|
||||
memory: 32Mi
|
||||
limits:
|
||||
cpu: 100m
|
||||
memory: 64Mi
|
||||
args:
|
||||
- |
|
||||
STATUS="{{workflow.status}}"
|
||||
NAME="{{workflow.name}}"
|
||||
|
||||
if [ -z "$WEBHOOK_URL" ] || echo "$WEBHOOK_URL" | grep -q "PLACEHOLDER"; then
|
||||
echo "No webhook configured, skipping notification"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "$STATUS" = "Succeeded" ]; then
|
||||
EMOJI=":white_check_mark:"
|
||||
else
|
||||
EMOJI=":x:"
|
||||
fi
|
||||
|
||||
PAYLOAD="{\"username\":\"Argo CI\",\"text\":\"${EMOJI} **${NAME}** — ${STATUS} ({{workflow.duration}}s)\"}"
|
||||
|
||||
curl -sf -X POST -H 'Content-Type: application/json' \
|
||||
-d "$PAYLOAD" "$WEBHOOK_URL" || echo "WARN: webhook post failed"
|
||||
Reference in New Issue
Block a user