62484e9c59
The feature cache auto-invalidates via content hash — manual rm -f wasted ~2 minutes of MBP-10 + OFI recomputation per trial start. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
725 lines
26 KiB
YAML
725 lines
26 KiB
YAML
# Compile-and-Train: unified workflow for code change → training run.
|
|
#
|
|
# DAG:
|
|
# compile-training ──┐
|
|
# gpu-warmup ────────┼──> fetch-binary ──> hyperopt ──> train-best ──> evaluate ──> upload-results
|
|
#
|
|
# Image builds handled by ci-pipeline, not here.
|
|
# Compile and GPU warmup run in parallel (~5.5 min compile, ~3-5 min autoscale).
|
|
# When both finish, fetch-binary downloads the freshly compiled binaries and
|
|
# training starts immediately — no manual CI wait or package seeding.
|
|
---
|
|
apiVersion: argoproj.io/v1alpha1
|
|
kind: WorkflowTemplate
|
|
metadata:
|
|
name: compile-and-train
|
|
namespace: foxhunt
|
|
labels:
|
|
app.kubernetes.io/name: compile-and-train
|
|
app.kubernetes.io/part-of: foxhunt
|
|
app.kubernetes.io/component: gpu-test
|
|
spec:
|
|
entrypoint: pipeline
|
|
onExit: notify-result
|
|
serviceAccountName: argo-workflow
|
|
podMetadata:
|
|
labels:
|
|
app.kubernetes.io/part-of: foxhunt
|
|
app.kubernetes.io/component: compile-and-train
|
|
# No runAsUser — compile (ci-builder) needs root, training runtime defaults
|
|
# to its image user. fsGroup ensures workspace PVC is writable by all pods.
|
|
securityContext:
|
|
fsGroup: 0
|
|
ttlStrategy:
|
|
secondsAfterCompletion: 3600
|
|
activeDeadlineSeconds: 28800 # 8 hours (compile + full training)
|
|
|
|
arguments:
|
|
parameters:
|
|
# ── Compile ──
|
|
- name: commit-sha
|
|
value: HEAD
|
|
- name: git-branch
|
|
value: main
|
|
- name: cuda-compute-cap
|
|
value: "90"
|
|
# ── Training ──
|
|
- name: model
|
|
value: dqn
|
|
- name: gpu-pool
|
|
value: ci-training-h100
|
|
- name: hyperopt-trials
|
|
value: "20"
|
|
- name: hyperopt-epochs
|
|
value: "8"
|
|
- name: train-epochs
|
|
value: "50"
|
|
- name: symbol
|
|
value: ES.FUT
|
|
- name: data-dir
|
|
value: /data/futures-baseline
|
|
- name: mbp10-data-dir
|
|
value: /data/futures-baseline-mbp10
|
|
- name: trades-data-dir
|
|
value: /data/futures-baseline-trades
|
|
- name: tx-cost-bps
|
|
value: "0.1"
|
|
- name: tick-size
|
|
value: "0.25"
|
|
- name: spread-ticks
|
|
value: "1.0"
|
|
- name: initial-capital
|
|
value: "35000"
|
|
- name: ensemble-top-k
|
|
value: "5"
|
|
|
|
volumes:
|
|
- name: git-ssh-key
|
|
secret:
|
|
secretName: argo-git-ssh-key
|
|
defaultMode: 256
|
|
- name: training-data
|
|
persistentVolumeClaim:
|
|
claimName: training-data-pvc
|
|
readOnly: true
|
|
- name: cargo-target-cuda
|
|
persistentVolumeClaim:
|
|
claimName: cargo-target-cuda
|
|
|
|
volumeClaimTemplates:
|
|
- metadata:
|
|
name: workspace
|
|
spec:
|
|
accessModes: ["ReadWriteOnce"]
|
|
storageClassName: scw-bssd
|
|
resources:
|
|
requests:
|
|
storage: 5Gi
|
|
|
|
templates:
|
|
- name: pipeline
|
|
dag:
|
|
tasks:
|
|
- name: compile-training
|
|
template: compile-training
|
|
- name: gpu-warmup
|
|
template: gpu-warmup
|
|
- name: fetch-binary
|
|
template: fetch-binary
|
|
dependencies: [compile-training]
|
|
arguments:
|
|
parameters:
|
|
- name: tag
|
|
value: "{{tasks.compile-training.outputs.parameters.tag}}"
|
|
- name: hyperopt
|
|
template: hyperopt
|
|
dependencies: [fetch-binary, gpu-warmup]
|
|
when: "{{workflow.parameters.hyperopt-trials}} != 0"
|
|
- name: train-best
|
|
template: train-best
|
|
dependencies: [fetch-binary, gpu-warmup, hyperopt]
|
|
- name: evaluate
|
|
template: evaluate
|
|
dependencies: [train-best]
|
|
- name: upload-results
|
|
template: upload-results
|
|
dependencies: [evaluate]
|
|
|
|
# ── compile-training: build from source on ci-compile-cpu ──
|
|
- name: compile-training
|
|
outputs:
|
|
parameters:
|
|
- name: tag
|
|
valueFrom:
|
|
path: /tmp/tag
|
|
nodeSelector:
|
|
k8s.scaleway.com/pool-name: ci-compile-cpu
|
|
tolerations:
|
|
- key: node.cilium.io/agent-not-ready
|
|
operator: Exists
|
|
effect: NoSchedule
|
|
container:
|
|
image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/ci-builder:latest
|
|
imagePullPolicy: Always
|
|
command: ["/bin/sh", "-c"]
|
|
env:
|
|
- name: SQLX_OFFLINE
|
|
value: "true"
|
|
- name: CARGO_TERM_COLOR
|
|
value: always
|
|
- name: CARGO_TARGET_DIR
|
|
value: /cargo-target
|
|
- name: CARGO_HOME
|
|
value: /cargo-target/cargo-home
|
|
- name: GITLAB_PAT
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: gitlab-pat
|
|
key: token
|
|
resources:
|
|
requests:
|
|
cpu: "14"
|
|
memory: 32Gi
|
|
limits:
|
|
cpu: "30"
|
|
memory: 64Gi
|
|
volumeMounts:
|
|
- name: git-ssh-key
|
|
mountPath: /etc/git-ssh
|
|
readOnly: true
|
|
- name: cargo-target-cuda
|
|
mountPath: /cargo-target
|
|
args:
|
|
- |
|
|
set -e
|
|
SHA="{{workflow.parameters.commit-sha}}"
|
|
BRANCH="{{workflow.parameters.git-branch}}"
|
|
|
|
# SSH setup
|
|
mkdir -p ~/.ssh
|
|
cp /etc/git-ssh/ssh-privatekey ~/.ssh/id_ed25519
|
|
chmod 600 ~/.ssh/id_ed25519
|
|
printf 'Host *\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n' > ~/.ssh/config
|
|
chmod 600 ~/.ssh/config
|
|
|
|
REPO="ssh://git@gitlab-gitlab-shell.foxhunt.svc.cluster.local:2222/root/foxhunt.git"
|
|
BUILD="/cargo-target/src"
|
|
|
|
# PVC may be owned by a different UID from a previous run
|
|
git config --global --add safe.directory "$BUILD"
|
|
|
|
# Persistent checkout on PVC — only changed files get new mtimes,
|
|
# so cargo skips recompiling unchanged workspace crates.
|
|
if [ -d "$BUILD/.git" ]; then
|
|
cd "$BUILD"
|
|
git fetch origin
|
|
# Resolve SHA: "HEAD" means remote branch tip, not local HEAD
|
|
if [ "$SHA" = "HEAD" ]; then
|
|
SHA=$(git rev-parse "origin/$BRANCH")
|
|
fi
|
|
CURRENT=$(git rev-parse HEAD 2>/dev/null || echo "none")
|
|
if [ "$CURRENT" = "$SHA" ]; then
|
|
echo "=== Already at $SHA, skipping checkout ==="
|
|
else
|
|
echo "=== Updating checkout: $(echo $CURRENT | cut -c1-8) -> $(echo $SHA | cut -c1-8) ==="
|
|
git checkout --force "$SHA"
|
|
git clean -fd
|
|
fi
|
|
else
|
|
echo "=== Initial clone (first run) ==="
|
|
git clone --filter=blob:none "$REPO" "$BUILD"
|
|
cd "$BUILD"
|
|
if [ "$SHA" = "HEAD" ]; then
|
|
git checkout "origin/$BRANCH"
|
|
else
|
|
git checkout "$SHA"
|
|
fi
|
|
fi
|
|
SHORT_SHA=$(git rev-parse --short HEAD)
|
|
echo "Checked out ${SHORT_SHA}"
|
|
|
|
# Ensure cargo home registry is on PVC
|
|
export PATH="${CARGO_HOME}/bin:${PATH}"
|
|
|
|
export CUDA_COMPUTE_CAP={{workflow.parameters.cuda-compute-cap}}
|
|
|
|
# Prune build artifacts if PVC exceeds 25GB (prevents unbounded growth)
|
|
TARGET_SIZE_MB=$(du -sm "$CARGO_TARGET_DIR" 2>/dev/null | cut -f1 || echo 0)
|
|
echo "PVC usage: ${TARGET_SIZE_MB}MB"
|
|
if [ "$TARGET_SIZE_MB" -gt 25000 ]; then
|
|
echo "PVC exceeds 25GB limit, pruning build artifacts..."
|
|
rm -rf "$CARGO_TARGET_DIR/release" "$CARGO_TARGET_DIR/debug"
|
|
fi
|
|
|
|
# Derive needed binaries from model parameter (only build what's used)
|
|
MODEL="{{workflow.parameters.model}}"
|
|
case "$MODEL" in
|
|
dqn|ppo)
|
|
EXAMPLES="hyperopt_baseline_rl train_baseline_rl evaluate_baseline"
|
|
;;
|
|
*)
|
|
EXAMPLES="hyperopt_baseline_supervised train_baseline_supervised evaluate_supervised"
|
|
;;
|
|
esac
|
|
|
|
ML_EXAMPLE_ARGS=""
|
|
for ex in $EXAMPLES; do
|
|
ML_EXAMPLE_ARGS="$ML_EXAMPLE_ARGS --example $ex"
|
|
done
|
|
|
|
echo "=== Building training binaries for $MODEL: $EXAMPLES (incremental) ==="
|
|
echo " CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP, target dir on PVC /cargo-target"
|
|
cargo build --release -p ml --features ml/cuda $ML_EXAMPLE_ARGS
|
|
|
|
mkdir -p "$BUILD/bin/training"
|
|
for bin in $EXAMPLES; do
|
|
cp "$CARGO_TARGET_DIR/release/examples/$bin" "$BUILD/bin/training/"
|
|
done
|
|
strip "$BUILD/bin/training/"*
|
|
|
|
echo "=== Training binaries ==="
|
|
ls -lh "$BUILD/bin/training/"
|
|
|
|
# Upload to GitLab packages under commit SHA
|
|
GITLAB="http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181"
|
|
TAG="dev-${SHORT_SHA}"
|
|
echo "=== Uploading as ${TAG} ==="
|
|
for bin in "$BUILD/bin/training/"*; do
|
|
BIN_NAME=$(basename "$bin")
|
|
curl -f --upload-file "$bin" \
|
|
-H "PRIVATE-TOKEN: ${GITLAB_PAT}" \
|
|
"${GITLAB}/api/v4/projects/1/packages/generic/foxhunt-training/${TAG}/${BIN_NAME}"
|
|
done
|
|
|
|
echo "=== Compile + upload done (${TAG}) ==="
|
|
echo "$TAG" > /tmp/tag
|
|
|
|
# ── gpu-warmup: trigger GPU autoscale during compile ──
|
|
- name: gpu-warmup
|
|
nodeSelector:
|
|
k8s.scaleway.com/pool-name: "{{workflow.parameters.gpu-pool}}"
|
|
tolerations:
|
|
- key: nvidia.com/gpu
|
|
operator: Exists
|
|
effect: NoSchedule
|
|
- key: node.cilium.io/agent-not-ready
|
|
operator: Exists
|
|
effect: NoSchedule
|
|
container:
|
|
image: busybox:1.37
|
|
command: ["/bin/sh", "-c"]
|
|
args:
|
|
- |
|
|
echo "GPU warmup: triggering node autoscale..."
|
|
echo "GPU node scheduled, exiting to free resources"
|
|
resources:
|
|
requests:
|
|
nvidia.com/gpu: "1"
|
|
cpu: 100m
|
|
memory: 64Mi
|
|
limits:
|
|
nvidia.com/gpu: "1"
|
|
cpu: 200m
|
|
memory: 128Mi
|
|
|
|
# ── fetch-binary: download freshly compiled binaries to workspace ──
|
|
- name: fetch-binary
|
|
inputs:
|
|
parameters:
|
|
- name: tag
|
|
nodeSelector:
|
|
k8s.scaleway.com/pool-name: platform
|
|
container:
|
|
image: curlimages/curl:8.12.1
|
|
command: ["/bin/sh", "-c"]
|
|
args:
|
|
- |
|
|
set -e
|
|
mkdir -p /workspace/bin
|
|
GITLAB_API="http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181/api/v4"
|
|
|
|
TAG="{{inputs.parameters.tag}}"
|
|
echo "Fetching training binaries from GitLab package: foxhunt-training/${TAG}"
|
|
|
|
BINARIES="hyperopt_baseline_rl hyperopt_baseline_supervised train_baseline_rl train_baseline_supervised evaluate_baseline evaluate_supervised training_uploader"
|
|
for bin in $BINARIES; do
|
|
echo " Downloading ${bin}..."
|
|
curl -fSL -o "/workspace/bin/${bin}" \
|
|
-H "PRIVATE-TOKEN: ${GITLAB_PAT}" \
|
|
"${GITLAB_API}/projects/1/packages/generic/foxhunt-training/${TAG}/${bin}" || {
|
|
echo "WARN: ${bin} not found, skipping"
|
|
continue
|
|
}
|
|
done
|
|
|
|
chmod +x /workspace/bin/*
|
|
echo "Fetched binaries:"
|
|
ls -lh /workspace/bin/
|
|
env:
|
|
- name: GITLAB_PAT
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: gitlab-pat
|
|
key: token
|
|
volumeMounts:
|
|
- name: workspace
|
|
mountPath: /workspace
|
|
resources:
|
|
requests:
|
|
cpu: 100m
|
|
memory: 128Mi
|
|
limits:
|
|
cpu: 500m
|
|
memory: 256Mi
|
|
|
|
# ── hyperopt: PSO/TPE optimization on GPU ──
|
|
- name: hyperopt
|
|
nodeSelector:
|
|
k8s.scaleway.com/pool-name: "{{workflow.parameters.gpu-pool}}"
|
|
tolerations:
|
|
- key: nvidia.com/gpu
|
|
operator: Exists
|
|
effect: NoSchedule
|
|
- key: node.cilium.io/agent-not-ready
|
|
operator: Exists
|
|
effect: NoSchedule
|
|
container:
|
|
image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/ci-builder:latest
|
|
imagePullPolicy: IfNotPresent
|
|
command: ["/bin/sh", "-c"]
|
|
env:
|
|
- name: RUST_LOG
|
|
value: info
|
|
- name: SQLX_OFFLINE
|
|
value: "true"
|
|
- name: CUBLAS_WORKSPACE_CONFIG
|
|
value: ":4096:8"
|
|
- name: CARGO_TARGET_DIR
|
|
value: /workspace
|
|
args:
|
|
- |
|
|
set -e
|
|
export PATH="/workspace/bin:$PATH"
|
|
# Remove CUDA stubs from LD_LIBRARY_PATH — stubs shadow real NVIDIA drivers
|
|
export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//')
|
|
nvidia-smi
|
|
|
|
# Training data available via training-data-pvc mounted at /data
|
|
echo "=== Training data (PVC) ==="
|
|
echo "OHLCV: $(ls {{workflow.parameters.data-dir}}/ES.FUT/*.dbn.zst 2>/dev/null | wc -l) files"
|
|
echo "MBP-10: $(ls {{workflow.parameters.mbp10-data-dir}}/ES.FUT/*.dbn.zst 2>/dev/null | wc -l) files"
|
|
echo "Trades: $(ls {{workflow.parameters.trades-data-dir}}/ES.FUT/*.dbn.zst 2>/dev/null | wc -l) files"
|
|
|
|
mkdir -p /workspace/output/hyperopt
|
|
|
|
# Feature cache auto-invalidates via content hash — no manual clearing needed.
|
|
# The cache key includes data paths + MBP-10 + trades dirs, so stale caches
|
|
# are automatically skipped when data changes.
|
|
echo "=== Feature cache: $(ls /workspace/.foxhunt_feature_cache/*.bin 2>/dev/null | wc -l) cached files ==="
|
|
|
|
MODEL="{{workflow.parameters.model}}"
|
|
case "$MODEL" in
|
|
dqn|ppo) BINARY=hyperopt_baseline_rl ;;
|
|
*) BINARY=hyperopt_baseline_supervised ;;
|
|
esac
|
|
|
|
echo "=== Running hyperopt: $MODEL (${BINARY}) ==="
|
|
echo " Trials: {{workflow.parameters.hyperopt-trials}}, Epochs: {{workflow.parameters.hyperopt-epochs}}"
|
|
|
|
# Phase 1 (fast): fix architecture, search learning dynamics
|
|
${BINARY} \
|
|
--model "$MODEL" \
|
|
--phase fast \
|
|
--trials {{workflow.parameters.hyperopt-trials}} \
|
|
--epochs {{workflow.parameters.hyperopt-epochs}} \
|
|
--symbol {{workflow.parameters.symbol}} \
|
|
--tx-cost-bps {{workflow.parameters.tx-cost-bps}} \
|
|
--tick-size {{workflow.parameters.tick-size}} \
|
|
--spread-ticks {{workflow.parameters.spread-ticks}} \
|
|
--initial-capital {{workflow.parameters.initial-capital}} \
|
|
--data-dir {{workflow.parameters.data-dir}} \
|
|
--mbp10-data-dir {{workflow.parameters.mbp10-data-dir}} \
|
|
--trades-data-dir {{workflow.parameters.trades-data-dir}} \
|
|
--base-dir /workspace/output/hyperopt \
|
|
--output /workspace/output/${MODEL}_phase1_results.json
|
|
|
|
echo "=== Phase 1 complete ==="
|
|
cat /workspace/output/${MODEL}_phase1_results.json 2>/dev/null || echo "No Phase 1 results"
|
|
|
|
# Phase 2 (full): fix dynamics from Phase 1, search architecture
|
|
PHASE2_TRIALS=$(({{workflow.parameters.hyperopt-trials}} / 2))
|
|
[ "$PHASE2_TRIALS" -lt 5 ] && PHASE2_TRIALS=5
|
|
PHASE2_EPOCHS=$(({{workflow.parameters.hyperopt-epochs}} * 2))
|
|
|
|
${BINARY} \
|
|
--model "$MODEL" \
|
|
--phase full \
|
|
--hyperopt-params /workspace/output/${MODEL}_phase1_results.json \
|
|
--trials $PHASE2_TRIALS \
|
|
--epochs $PHASE2_EPOCHS \
|
|
--symbol {{workflow.parameters.symbol}} \
|
|
--tx-cost-bps {{workflow.parameters.tx-cost-bps}} \
|
|
--tick-size {{workflow.parameters.tick-size}} \
|
|
--spread-ticks {{workflow.parameters.spread-ticks}} \
|
|
--initial-capital {{workflow.parameters.initial-capital}} \
|
|
--data-dir {{workflow.parameters.data-dir}} \
|
|
--mbp10-data-dir {{workflow.parameters.mbp10-data-dir}} \
|
|
--trades-data-dir {{workflow.parameters.trades-data-dir}} \
|
|
--base-dir /workspace/output/hyperopt \
|
|
--output /workspace/output/${MODEL}_phase2_results.json
|
|
|
|
echo "=== Phase 2 complete ==="
|
|
cat /workspace/output/${MODEL}_phase2_results.json 2>/dev/null || echo "No Phase 2 results"
|
|
|
|
# Phase 3 (reward): fix dynamics + architecture, search only reward weights (7D)
|
|
PHASE3_TRIALS=$(({{workflow.parameters.hyperopt-trials}} / 2))
|
|
[ "$PHASE3_TRIALS" -lt 5 ] && PHASE3_TRIALS=5
|
|
PHASE3_EPOCHS=$(({{workflow.parameters.hyperopt-epochs}} * 2))
|
|
|
|
${BINARY} \
|
|
--model "$MODEL" \
|
|
--phase reward \
|
|
--hyperopt-params /workspace/output/${MODEL}_phase2_results.json \
|
|
--trials $PHASE3_TRIALS \
|
|
--epochs $PHASE3_EPOCHS \
|
|
--symbol {{workflow.parameters.symbol}} \
|
|
--tx-cost-bps {{workflow.parameters.tx-cost-bps}} \
|
|
--tick-size {{workflow.parameters.tick-size}} \
|
|
--spread-ticks {{workflow.parameters.spread-ticks}} \
|
|
--initial-capital {{workflow.parameters.initial-capital}} \
|
|
--data-dir {{workflow.parameters.data-dir}} \
|
|
--mbp10-data-dir {{workflow.parameters.mbp10-data-dir}} \
|
|
--trades-data-dir {{workflow.parameters.trades-data-dir}} \
|
|
--base-dir /workspace/output/hyperopt \
|
|
--output /workspace/output/${MODEL}_hyperopt_results.json
|
|
|
|
echo "=== Three-phase hyperopt complete ==="
|
|
cat /workspace/output/${MODEL}_hyperopt_results.json 2>/dev/null || echo "No results file"
|
|
resources:
|
|
requests:
|
|
nvidia.com/gpu: "1"
|
|
cpu: "4"
|
|
memory: 32Gi
|
|
limits:
|
|
nvidia.com/gpu: "1"
|
|
cpu: "8"
|
|
memory: 64Gi
|
|
volumeMounts:
|
|
- name: workspace
|
|
mountPath: /workspace
|
|
- name: training-data
|
|
mountPath: /data
|
|
readOnly: true
|
|
|
|
# ── train-best: full training with best hyperparams ──
|
|
- name: train-best
|
|
nodeSelector:
|
|
k8s.scaleway.com/pool-name: "{{workflow.parameters.gpu-pool}}"
|
|
tolerations:
|
|
- key: nvidia.com/gpu
|
|
operator: Exists
|
|
effect: NoSchedule
|
|
- key: node.cilium.io/agent-not-ready
|
|
operator: Exists
|
|
effect: NoSchedule
|
|
container:
|
|
image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/ci-builder:latest
|
|
imagePullPolicy: IfNotPresent
|
|
command: ["/bin/sh", "-c"]
|
|
env:
|
|
- name: RUST_LOG
|
|
value: info
|
|
- name: SQLX_OFFLINE
|
|
value: "true"
|
|
- name: CUBLAS_WORKSPACE_CONFIG
|
|
value: ":4096:8"
|
|
args:
|
|
- |
|
|
set -e
|
|
export PATH="/workspace/bin:$PATH"
|
|
export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//')
|
|
nvidia-smi
|
|
|
|
MODEL="{{workflow.parameters.model}}"
|
|
|
|
# Determine binary and hyperopt flag
|
|
case "$MODEL" in
|
|
dqn|ppo) BINARY=train_baseline_rl ;;
|
|
*) BINARY=train_baseline_supervised ;;
|
|
esac
|
|
|
|
HYPEROPT_FLAG=""
|
|
if [ -f "/workspace/output/${MODEL}_hyperopt_results.json" ]; then
|
|
HYPEROPT_FLAG="--hyperopt-params /workspace/output/${MODEL}_hyperopt_results.json"
|
|
echo " Using hyperopt results: ${MODEL}_hyperopt_results.json"
|
|
else
|
|
echo " No hyperopt results — training with default hyperparams"
|
|
fi
|
|
|
|
echo "=== Training best: $MODEL ({{workflow.parameters.train-epochs}} epochs) ==="
|
|
|
|
${BINARY} \
|
|
--model "$MODEL" \
|
|
--symbol {{workflow.parameters.symbol}} \
|
|
--tx-cost-bps {{workflow.parameters.tx-cost-bps}} \
|
|
--tick-size {{workflow.parameters.tick-size}} \
|
|
--spread-ticks {{workflow.parameters.spread-ticks}} \
|
|
--data-dir {{workflow.parameters.data-dir}} \
|
|
--mbp10-data-dir {{workflow.parameters.mbp10-data-dir}} \
|
|
--trades-data-dir {{workflow.parameters.trades-data-dir}} \
|
|
--output-dir /workspace/output \
|
|
--max-steps-per-epoch {{workflow.parameters.train-epochs}} \
|
|
$HYPEROPT_FLAG
|
|
|
|
echo "=== Training complete ==="
|
|
resources:
|
|
requests:
|
|
nvidia.com/gpu: "1"
|
|
cpu: "4"
|
|
memory: 32Gi
|
|
limits:
|
|
nvidia.com/gpu: "1"
|
|
cpu: "8"
|
|
memory: 64Gi
|
|
volumeMounts:
|
|
- name: workspace
|
|
mountPath: /workspace
|
|
- name: training-data
|
|
mountPath: /data
|
|
readOnly: true
|
|
|
|
# ── evaluate: run evaluation on trained model ──
|
|
- name: evaluate
|
|
nodeSelector:
|
|
k8s.scaleway.com/pool-name: "{{workflow.parameters.gpu-pool}}"
|
|
tolerations:
|
|
- key: nvidia.com/gpu
|
|
operator: Exists
|
|
effect: NoSchedule
|
|
- key: node.cilium.io/agent-not-ready
|
|
operator: Exists
|
|
effect: NoSchedule
|
|
container:
|
|
image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/ci-builder:latest
|
|
imagePullPolicy: IfNotPresent
|
|
command: ["/bin/sh", "-c"]
|
|
env:
|
|
- name: RUST_LOG
|
|
value: info
|
|
- name: SQLX_OFFLINE
|
|
value: "true"
|
|
args:
|
|
- |
|
|
set -e
|
|
export PATH="/workspace/bin:$PATH"
|
|
export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//')
|
|
|
|
MODEL="{{workflow.parameters.model}}"
|
|
case "$MODEL" in
|
|
dqn|ppo) BINARY=evaluate_baseline ;;
|
|
*) BINARY=evaluate_supervised ;;
|
|
esac
|
|
|
|
echo "=== Evaluating: $MODEL ==="
|
|
${BINARY} \
|
|
--model "$MODEL" \
|
|
--symbol {{workflow.parameters.symbol}} \
|
|
--data-dir {{workflow.parameters.data-dir}} \
|
|
--checkpoint-dir /workspace/output \
|
|
--output-dir /workspace/output/eval \
|
|
--initial-capital {{workflow.parameters.initial-capital}} \
|
|
--tx-cost-bps {{workflow.parameters.tx-cost-bps}} \
|
|
--tick-size {{workflow.parameters.tick-size}} \
|
|
--spread-ticks {{workflow.parameters.spread-ticks}} || {
|
|
echo "WARN: Evaluation failed, continuing to upload available results"
|
|
}
|
|
|
|
echo "=== Evaluation complete ==="
|
|
resources:
|
|
requests:
|
|
nvidia.com/gpu: "1"
|
|
cpu: "2"
|
|
memory: 16Gi
|
|
limits:
|
|
nvidia.com/gpu: "1"
|
|
cpu: "4"
|
|
memory: 32Gi
|
|
volumeMounts:
|
|
- name: workspace
|
|
mountPath: /workspace
|
|
- name: training-data
|
|
mountPath: /data
|
|
readOnly: true
|
|
|
|
# ── upload-results: push artifacts to GitLab packages ──
|
|
- name: upload-results
|
|
nodeSelector:
|
|
k8s.scaleway.com/pool-name: platform
|
|
container:
|
|
image: curlimages/curl:8.12.1
|
|
command: ["/bin/sh", "-c"]
|
|
args:
|
|
- |
|
|
set -e
|
|
MODEL="{{workflow.parameters.model}}"
|
|
SYMBOL="{{workflow.parameters.symbol}}"
|
|
TIMESTAMP=$(date +%Y%m%d-%H%M%S)
|
|
GITLAB_API="http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181/api/v4"
|
|
PKG_NAME="foxhunt-training-results"
|
|
PKG_VERSION="${MODEL}-${SYMBOL}-${TIMESTAMP}"
|
|
|
|
echo "Uploading training artifacts to GitLab packages: ${PKG_NAME}/${PKG_VERSION}"
|
|
|
|
UPLOADED=0
|
|
find /workspace/output -type f | while read -r file; do
|
|
REL_PATH="${file#/workspace/output/}"
|
|
SAFE_NAME=$(echo "$REL_PATH" | tr '/' '--')
|
|
echo " Uploading ${REL_PATH} as ${SAFE_NAME}..."
|
|
curl -f --upload-file "$file" \
|
|
-H "PRIVATE-TOKEN: ${GITLAB_PAT}" \
|
|
"${GITLAB_API}/projects/1/packages/generic/${PKG_NAME}/${PKG_VERSION}/${SAFE_NAME}" && \
|
|
UPLOADED=$((UPLOADED + 1)) || \
|
|
echo " WARN: Failed to upload ${REL_PATH}"
|
|
done
|
|
|
|
echo "=== Upload complete (${UPLOADED} files) ==="
|
|
echo "Package: ${PKG_NAME}/${PKG_VERSION}"
|
|
env:
|
|
- name: GITLAB_PAT
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: gitlab-pat
|
|
key: token
|
|
volumeMounts:
|
|
- name: workspace
|
|
mountPath: /workspace
|
|
resources:
|
|
requests:
|
|
cpu: 100m
|
|
memory: 128Mi
|
|
limits:
|
|
cpu: 500m
|
|
memory: 256Mi
|
|
|
|
# ── notify-result: post workflow outcome to Mattermost ──
|
|
- name: notify-result
|
|
nodeSelector:
|
|
k8s.scaleway.com/pool-name: platform
|
|
container:
|
|
image: curlimages/curl:8.12.1
|
|
command: ["/bin/sh", "-c"]
|
|
env:
|
|
- name: WEBHOOK_URL
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: notification-webhook
|
|
key: webhook-url
|
|
optional: true
|
|
resources:
|
|
requests:
|
|
cpu: 50m
|
|
memory: 32Mi
|
|
limits:
|
|
cpu: 100m
|
|
memory: 64Mi
|
|
args:
|
|
- |
|
|
STATUS="{{workflow.status}}"
|
|
NAME="{{workflow.name}}"
|
|
|
|
if [ -z "$WEBHOOK_URL" ] || echo "$WEBHOOK_URL" | grep -q "PLACEHOLDER"; then
|
|
echo "No webhook configured, skipping notification"
|
|
exit 0
|
|
fi
|
|
|
|
if [ "$STATUS" = "Succeeded" ]; then
|
|
EMOJI=":white_check_mark:"
|
|
else
|
|
EMOJI=":x:"
|
|
fi
|
|
|
|
PAYLOAD="{\"username\":\"Argo CI\",\"text\":\"${EMOJI} **${NAME}** — ${STATUS} ({{workflow.duration}}s)\"}"
|
|
|
|
curl -sf -X POST -H 'Content-Type: application/json' \
|
|
-d "$PAYLOAD" "$WEBHOOK_URL" || echo "WARN: webhook post failed"
|