gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: unified Argo train workflow — replace 7 templates with 1

Browse Source 
New: train-template.yaml with smart caching:
  - ensure-binary: checks /data/bin/{sha}/ on PVC, compiles only on cache miss
  - ensure-fxcache: runs precompute only if cache doesn't exist
  - gpu-warmup: parallel autoscale during compile
  - hyperopt → train-best → evaluate → upload-results

Deleted 7 redundant templates:
  - compile-and-train-template.yaml
  - train-dqn-template.yaml
  - train-baseline-rl-template.yaml
  - train-supervised-template.yaml
  - training-workflow-template.yaml
  - precompute-features-template.yaml
  - train-ppo-template.yaml

Deleted: scripts/argo-precompute.sh (absorbed into ensure-fxcache step)
Rewritten: scripts/argo-train.sh (single template, --sha for commit pinning)
Updated: kustomization.yaml

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-04-10 20:36:03 +02:00
parent f8e0f459fc
commit 445c2197f2
11 changed files with 688 additions and 1928 deletions
Download Patch File Download Diff File Expand all files Collapse all files

70
scripts/argo-precompute.sh
View File

@@ -1,70 +0,0 @@
#!/usr/bin/env bash
# Pre-compute DQN features via Argo Workflows.
#
# Usage:
# ./scripts/argo-precompute.sh ES.FUT
# ./scripts/argo-precompute.sh ES.FUT --pool platform --watch
#
# Requires: argo CLI
set -euo pipefail
TEMPLATE="precompute-features"
POOL=""
DATA_DIR=""
MBP10_DIR=""
TRADES_DIR=""
OUTPUT_DIR=""
WATCH=false
usage() {
cat <<EOF
Usage: $(basename "$0") <symbol> [OPTIONS]
Options:
--pool <name> Node pool (default: ci-compile-cpu)
--data-dir <dir> OHLCV data dir (default: /data/futures-baseline)
--mbp10-dir <dir> MBP-10 data dir (default: /data/futures-baseline-mbp10)
--trades-dir <dir> Trades data dir (default: /data/futures-baseline-trades)
--output-dir <dir> Output dir (default: /data/feature-cache)
--watch Follow workflow logs after submission
-h, --help Show this help
EOF
exit 0
}
[[ $# -eq 0 ]] && { echo "Error: symbol argument required"; usage; }
[[ "$1" == "-h" || "$1" == "--help" ]] && usage
SYMBOL="$1"; shift
while [[ $# -gt 0 ]]; do
case $1 in
--pool) POOL="$2"; shift 2 ;;
--data-dir) DATA_DIR="$2"; shift 2 ;;
--mbp10-dir) MBP10_DIR="$2"; shift 2 ;;
--trades-dir) TRADES_DIR="$2"; shift 2 ;;
--output-dir) OUTPUT_DIR="$2"; shift 2 ;;
--watch) WATCH=true; shift ;;
-h|--help) usage ;;
*) echo "Unknown option: $1"; usage ;;
esac
done
CMD="argo submit -n foxhunt --from=wftmpl/$TEMPLATE"
CMD="$CMD -p symbol=$SYMBOL"
[[ -n "$POOL" ]] && CMD="$CMD -p node-pool=$POOL"
[[ -n "$DATA_DIR" ]] && CMD="$CMD -p data-dir=$DATA_DIR"
[[ -n "$MBP10_DIR" ]] && CMD="$CMD -p mbp10-dir=$MBP10_DIR"
[[ -n "$TRADES_DIR" ]] && CMD="$CMD -p trades-dir=$TRADES_DIR"
[[ -n "$OUTPUT_DIR" ]] && CMD="$CMD -p output-dir=$OUTPUT_DIR"
if $WATCH; then
CMD="$CMD --watch"
fi
echo "Submitting feature pre-compute workflow..."
echo " symbol: $SYMBOL"
[[ -n "$POOL" ]] && echo " pool: $POOL"
echo ""
eval "$CMD"

111
scripts/argo-train.sh
View File

@@ -1,29 +1,30 @@
#!/usr/bin/env bash
# Train a model via Argo Workflows (DQN, PPO, or any supervised model).
# Train a model via Argo Workflows.
#
# Usage:
# ./scripts/argo-train.sh dqn # DQN with defaults
# ./scripts/argo-train.sh ppo --tag dev-abc1234 # PPO with specific binary
# ./scripts/argo-train.sh tft --trials 40 --epochs 100 # supervised TFT
# ./scripts/argo-train.sh mamba2 --webhook # trigger via webhook
# ./scripts/argo-train.sh dqn # defaults: HEAD, H100, 50 epochs
# ./scripts/argo-train.sh dqn --sha abc1234 # specific commit
# ./scripts/argo-train.sh dqn --epochs 100 --trials 40 # override training params
# ./scripts/argo-train.sh ppo --gpu-pool ci-training # L40S instead of H100
# ./scripts/argo-train.sh dqn --baseline # skip hyperopt
# ./scripts/argo-train.sh dqn --watch # follow logs
#
# Supported models:
# RL: dqn, ppo
# Supervised: tft, mamba2, tggn, tlob, liquid, kan, xlstm, diffusion
#
# Requires: argo CLI (default) or curl (webhook mode)
# Requires: argo CLI
set -euo pipefail
TAG="latest"
SHA="HEAD"
BRANCH="main"
TRIALS=""
EPOCHS=""
GPU_POOL=""
SYMBOL=""
WEBHOOK=false
WEBHOOK_BASE="http://workflow-trigger-eventsource-svc.foxhunt:12001"
WATCH=false
BASELINE=false
CACHE_DIR=""
CAPITAL=""
usage() {
cat <<EOF
@@ -31,19 +32,19 @@ Usage: $(basename "$0") <model> [OPTIONS]
Models:
dqn, ppo (RL)
tft, mamba2, tggn, tlob, liquid, kan, xlstm, diffusion (supervised)
tft, mamba2, tggn, tlob, liquid, kan, xlstm (supervised)
Options:
--tag <binary-tag> Binary tag to use (default: latest)
--trials <n> Hyperopt trials (default: model-specific)
--epochs <n> Training epochs (default: model-specific)
--gpu-pool <pool> GPU node pool (default: ci-training-h100)
--symbol <sym> Trading symbol (default: ES.FUT)
--baseline Skip hyperopt (set trials=0), train with defaults
--cache-dir <path> Override feature cache directory
--webhook Trigger via webhook instead of argo CLI
--watch Follow workflow logs after submission
-h, --help Show this help
--sha <commit> Git commit SHA (default: HEAD)
--branch <branch> Git branch (default: main)
--trials <n> Hyperopt trials (default: 20)
--epochs <n> Training epochs (default: 50)
--gpu-pool <pool> GPU node pool (default: ci-training-h100)
--symbol <sym> Trading symbol (default: ES.FUT)
--capital <n> Initial capital (default: 35000)
--baseline Skip hyperopt (trials=0)
--watch Follow workflow logs
-h, --help Show this help
EOF
exit 0
}
@@ -52,7 +53,6 @@ EOF
MODEL="$1"; shift
# Validate model name
case "$MODEL" in
dqn|ppo|tft|mamba2|tggn|tlob|liquid|kan|xlstm|diffusion) ;;
*) echo "Error: unknown model '$MODEL'"; usage ;;
@@ -60,72 +60,41 @@ esac
while [[ $# -gt 0 ]]; do
case $1 in
--tag) TAG="$2"; shift 2 ;;
--sha) SHA="$2"; shift 2 ;;
--branch) BRANCH="$2"; shift 2 ;;
--trials) TRIALS="$2"; shift 2 ;;
--epochs) EPOCHS="$2"; shift 2 ;;
--gpu-pool) GPU_POOL="$2"; shift 2 ;;
--symbol) SYMBOL="$2"; shift 2 ;;
--capital) CAPITAL="$2"; shift 2 ;;
--baseline) BASELINE=true; shift ;;
--cache-dir) CACHE_DIR="$2"; shift 2 ;;
--webhook) WEBHOOK=true; shift ;;
--watch) WATCH=true; shift ;;
-h|--help) usage ;;
*) echo "Unknown option: $1"; usage ;;
esac
done
# Determine workflow template name
case "$MODEL" in
dqn) TEMPLATE="train-dqn" ;;
ppo) TEMPLATE="train-ppo" ;;
*) TEMPLATE="train-supervised" ;;
esac
if $WEBHOOK; then
case "$MODEL" in
dqn) ENDPOINT="/train/dqn"; PAYLOAD="{\"binary_tag\": \"${TAG}\"}" ;;
ppo) ENDPOINT="/train/ppo"; PAYLOAD="{\"binary_tag\": \"${TAG}\"}" ;;
*) ENDPOINT="/train/supervised"; PAYLOAD="{\"model\": \"${MODEL}\", \"binary_tag\": \"${TAG}\"}" ;;
esac
echo "Triggering $MODEL training via webhook..."
curl -sf -X POST "${WEBHOOK_BASE}${ENDPOINT}" \
-H 'Content-Type: application/json' \
-d "$PAYLOAD"
echo "Webhook sent: $PAYLOAD"
exit 0
fi
# Build argo submit command
CMD="argo submit -n foxhunt --from=wftmpl/$TEMPLATE"
CMD="$CMD -p binary-tag=$TAG"
# Supervised models need --model param
if [[ "$MODEL" != "dqn" && "$MODEL" != "ppo" ]]; then
CMD="$CMD -p model=$MODEL"
fi
CMD="argo submit -n foxhunt --from=wftmpl/train"
CMD="$CMD -p commit-sha=$SHA"
CMD="$CMD -p git-branch=$BRANCH"
CMD="$CMD -p model=$MODEL"
[[ -n "$TRIALS" ]] && CMD="$CMD -p hyperopt-trials=$TRIALS"
[[ -n "$EPOCHS" ]] && CMD="$CMD -p train-epochs=$EPOCHS"
[[ -n "$GPU_POOL" ]] && CMD="$CMD -p gpu-pool=$GPU_POOL"
[[ -n "$SYMBOL" ]] && CMD="$CMD -p symbol=$SYMBOL"
[[ -n "$CACHE_DIR" ]] && CMD="$CMD -p feature-cache-dir=$CACHE_DIR"
[[ -n "$CAPITAL" ]] && CMD="$CMD -p initial-capital=$CAPITAL"
if $BASELINE; then
CMD="$CMD -p hyperopt-trials=0"
fi
$BASELINE && CMD="$CMD -p hyperopt-trials=0"
$WATCH && CMD="$CMD --watch"
if $WATCH; then
CMD="$CMD --watch"
fi
echo "Submitting $MODEL training workflow ($TEMPLATE)..."
echo " binary-tag: $TAG"
$BASELINE && echo " mode: baseline (hyperopt skipped)"
[[ -n "$TRIALS" ]] && echo " trials: $TRIALS"
[[ -n "$EPOCHS" ]] && echo " epochs: $EPOCHS"
[[ -n "$GPU_POOL" ]] && echo " gpu-pool: $GPU_POOL"
[[ -n "$SYMBOL" ]] && echo " symbol: $SYMBOL"
[[ -n "$CACHE_DIR" ]] && echo " cache-dir: $CACHE_DIR"
echo "Submitting $MODEL training workflow..."
echo " sha: $SHA"
echo " branch: $BRANCH"
echo " model: $MODEL"
$BASELINE && echo " mode: baseline (no hyperopt)"
[[ -n "$TRIALS" ]] && echo " trials: $TRIALS"
[[ -n "$EPOCHS" ]] && echo " epochs: $EPOCHS"
[[ -n "$GPU_POOL" ]] && echo " gpu: $GPU_POOL"
echo ""
eval "$CMD"