445c2197f2
New: train-template.yaml with smart caching:
- ensure-binary: checks /data/bin/{sha}/ on PVC, compiles only on cache miss
- ensure-fxcache: runs precompute only if cache doesn't exist
- gpu-warmup: parallel autoscale during compile
- hyperopt → train-best → evaluate → upload-results
Deleted 7 redundant templates:
- compile-and-train-template.yaml
- train-dqn-template.yaml
- train-baseline-rl-template.yaml
- train-supervised-template.yaml
- training-workflow-template.yaml
- precompute-features-template.yaml
- train-ppo-template.yaml
Deleted: scripts/argo-precompute.sh (absorbed into ensure-fxcache step)
Rewritten: scripts/argo-train.sh (single template, --sha for commit pinning)
Updated: kustomization.yaml
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
101 lines
3.1 KiB
Bash
Executable File
101 lines
3.1 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Train a model via Argo Workflows.
|
|
#
|
|
# Usage:
|
|
# ./scripts/argo-train.sh dqn # defaults: HEAD, H100, 50 epochs
|
|
# ./scripts/argo-train.sh dqn --sha abc1234 # specific commit
|
|
# ./scripts/argo-train.sh dqn --epochs 100 --trials 40 # override training params
|
|
# ./scripts/argo-train.sh ppo --gpu-pool ci-training # L40S instead of H100
|
|
# ./scripts/argo-train.sh dqn --baseline # skip hyperopt
|
|
# ./scripts/argo-train.sh dqn --watch # follow logs
|
|
#
|
|
# Supported models:
|
|
# RL: dqn, ppo
|
|
# Supervised: tft, mamba2, tggn, tlob, liquid, kan, xlstm, diffusion
|
|
#
|
|
# Requires: argo CLI
|
|
set -euo pipefail
|
|
|
|
SHA="HEAD"
|
|
BRANCH="main"
|
|
TRIALS=""
|
|
EPOCHS=""
|
|
GPU_POOL=""
|
|
SYMBOL=""
|
|
WATCH=false
|
|
BASELINE=false
|
|
CAPITAL=""
|
|
|
|
usage() {
|
|
cat <<EOF
|
|
Usage: $(basename "$0") <model> [OPTIONS]
|
|
|
|
Models:
|
|
dqn, ppo (RL)
|
|
tft, mamba2, tggn, tlob, liquid, kan, xlstm (supervised)
|
|
|
|
Options:
|
|
--sha <commit> Git commit SHA (default: HEAD)
|
|
--branch <branch> Git branch (default: main)
|
|
--trials <n> Hyperopt trials (default: 20)
|
|
--epochs <n> Training epochs (default: 50)
|
|
--gpu-pool <pool> GPU node pool (default: ci-training-h100)
|
|
--symbol <sym> Trading symbol (default: ES.FUT)
|
|
--capital <n> Initial capital (default: 35000)
|
|
--baseline Skip hyperopt (trials=0)
|
|
--watch Follow workflow logs
|
|
-h, --help Show this help
|
|
EOF
|
|
exit 0
|
|
}
|
|
|
|
[[ $# -eq 0 ]] && { echo "Error: model argument required"; usage; }
|
|
|
|
MODEL="$1"; shift
|
|
|
|
case "$MODEL" in
|
|
dqn|ppo|tft|mamba2|tggn|tlob|liquid|kan|xlstm|diffusion) ;;
|
|
*) echo "Error: unknown model '$MODEL'"; usage ;;
|
|
esac
|
|
|
|
while [[ $# -gt 0 ]]; do
|
|
case $1 in
|
|
--sha) SHA="$2"; shift 2 ;;
|
|
--branch) BRANCH="$2"; shift 2 ;;
|
|
--trials) TRIALS="$2"; shift 2 ;;
|
|
--epochs) EPOCHS="$2"; shift 2 ;;
|
|
--gpu-pool) GPU_POOL="$2"; shift 2 ;;
|
|
--symbol) SYMBOL="$2"; shift 2 ;;
|
|
--capital) CAPITAL="$2"; shift 2 ;;
|
|
--baseline) BASELINE=true; shift ;;
|
|
--watch) WATCH=true; shift ;;
|
|
-h|--help) usage ;;
|
|
*) echo "Unknown option: $1"; usage ;;
|
|
esac
|
|
done
|
|
|
|
CMD="argo submit -n foxhunt --from=wftmpl/train"
|
|
CMD="$CMD -p commit-sha=$SHA"
|
|
CMD="$CMD -p git-branch=$BRANCH"
|
|
CMD="$CMD -p model=$MODEL"
|
|
|
|
[[ -n "$TRIALS" ]] && CMD="$CMD -p hyperopt-trials=$TRIALS"
|
|
[[ -n "$EPOCHS" ]] && CMD="$CMD -p train-epochs=$EPOCHS"
|
|
[[ -n "$GPU_POOL" ]] && CMD="$CMD -p gpu-pool=$GPU_POOL"
|
|
[[ -n "$SYMBOL" ]] && CMD="$CMD -p symbol=$SYMBOL"
|
|
[[ -n "$CAPITAL" ]] && CMD="$CMD -p initial-capital=$CAPITAL"
|
|
|
|
$BASELINE && CMD="$CMD -p hyperopt-trials=0"
|
|
$WATCH && CMD="$CMD --watch"
|
|
|
|
echo "Submitting $MODEL training workflow..."
|
|
echo " sha: $SHA"
|
|
echo " branch: $BRANCH"
|
|
echo " model: $MODEL"
|
|
$BASELINE && echo " mode: baseline (no hyperopt)"
|
|
[[ -n "$TRIALS" ]] && echo " trials: $TRIALS"
|
|
[[ -n "$EPOCHS" ]] && echo " epochs: $EPOCHS"
|
|
[[ -n "$GPU_POOL" ]] && echo " gpu: $GPU_POOL"
|
|
echo ""
|
|
eval "$CMD"
|