feat(infra): optional nsys profiling in Argo template (-p nsys-profile=true)
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -59,6 +59,8 @@ spec:
|
||||
value: "1"
|
||||
- name: n-eval-steps
|
||||
value: "0"
|
||||
- name: nsys-profile
|
||||
value: "false"
|
||||
|
||||
volumes:
|
||||
- name: git-ssh-key
|
||||
@@ -164,7 +166,13 @@ spec:
|
||||
OUT="/feature-cache/alpha-rl-runs/$SHA/fold{{workflow.parameters.fold-idx}}"
|
||||
mkdir -p "$OUT"
|
||||
echo "=== Train on $(nvidia-smi --query-gpu=name --format=csv,noheader) ==="
|
||||
stdbuf -oL ${CARGO_TARGET_DIR}/release/examples/alpha_rl_train \
|
||||
TRAIN_BIN="${CARGO_TARGET_DIR}/release/examples/alpha_rl_train"
|
||||
if [ "{{workflow.parameters.nsys-profile}}" = "true" ]; then
|
||||
TRAIN_CMD="nsys profile -o $OUT/nsys_trace --stats=true --force-overwrite=true $TRAIN_BIN"
|
||||
else
|
||||
TRAIN_CMD="stdbuf -oL $TRAIN_BIN"
|
||||
fi
|
||||
$TRAIN_CMD \
|
||||
--mbp10-data-dir /data/futures-baseline-mbp10/ES.FUT \
|
||||
--predecoded-dir /feature-cache/predecoded \
|
||||
--out "$OUT" \
|
||||
|
||||
Reference in New Issue
Block a user