feat: Argo precompute-features template + argo-precompute.sh CLI
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
jgrusewski
123
infra/k8s/argo/precompute-features-template.yaml
Normal file
123
infra/k8s/argo/precompute-features-template.yaml
Normal file
@@ -0,0 +1,123 @@
|
||||
# Pre-compute DQN training features to .fxcache format.
|
||||
#
|
||||
# Runs after data downloads complete. Reads DBN files from training-data PVC,
|
||||
# extracts all features (OHLCV 42-dim + targets 4-dim + OFI 8-dim), writes
|
||||
# flat binary cache for zero-overhead GPU loading.
|
||||
#
|
||||
# Usage:
|
||||
# argo submit -n foxhunt --from=wftmpl/precompute-features
|
||||
# ./scripts/argo-precompute.sh ES.FUT --bf16
|
||||
---
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: WorkflowTemplate
|
||||
metadata:
|
||||
name: precompute-features
|
||||
namespace: foxhunt
|
||||
labels:
|
||||
app.kubernetes.io/name: precompute-features
|
||||
app.kubernetes.io/part-of: foxhunt
|
||||
spec:
|
||||
entrypoint: precompute
|
||||
serviceAccountName: argo-workflow
|
||||
podMetadata:
|
||||
labels:
|
||||
app.kubernetes.io/part-of: foxhunt
|
||||
app.kubernetes.io/component: precompute-features
|
||||
securityContext:
|
||||
fsGroup: 1000
|
||||
ttlStrategy:
|
||||
secondsAfterCompletion: 3600
|
||||
activeDeadlineSeconds: 7200
|
||||
|
||||
arguments:
|
||||
parameters:
|
||||
- name: symbol
|
||||
value: ES.FUT
|
||||
- name: data-dir
|
||||
value: /data/futures-baseline
|
||||
- name: mbp10-dir
|
||||
value: /data/futures-baseline-mbp10
|
||||
- name: trades-dir
|
||||
value: /data/futures-baseline-trades
|
||||
- name: output-dir
|
||||
value: /data/feature-cache
|
||||
- name: node-pool
|
||||
value: ci-compile-cpu
|
||||
- name: bf16
|
||||
value: "false"
|
||||
|
||||
volumes:
|
||||
- name: training-data
|
||||
persistentVolumeClaim:
|
||||
claimName: training-data-pvc
|
||||
|
||||
templates:
|
||||
- name: precompute
|
||||
inputs:
|
||||
parameters:
|
||||
- name: symbol
|
||||
- name: data-dir
|
||||
- name: mbp10-dir
|
||||
- name: trades-dir
|
||||
- name: output-dir
|
||||
- name: node-pool
|
||||
- name: bf16
|
||||
nodeSelector:
|
||||
k8s.scaleway.com/pool-name: "{{inputs.parameters.node-pool}}"
|
||||
container:
|
||||
image: ubuntu:24.04
|
||||
command: ["/bin/bash", "-c"]
|
||||
env:
|
||||
- name: RUST_LOG
|
||||
value: info
|
||||
resources:
|
||||
requests:
|
||||
cpu: "4"
|
||||
memory: 8Gi
|
||||
limits:
|
||||
cpu: "16"
|
||||
memory: 32Gi
|
||||
volumeMounts:
|
||||
- name: training-data
|
||||
mountPath: /data
|
||||
args:
|
||||
- |
|
||||
set -e
|
||||
BINARY=/data/bin/precompute_features
|
||||
chmod +x "$BINARY"
|
||||
|
||||
SYMBOL="{{inputs.parameters.symbol}}"
|
||||
DATA_DIR="{{inputs.parameters.data-dir}}"
|
||||
MBP10_DIR="{{inputs.parameters.mbp10-dir}}"
|
||||
TRADES_DIR="{{inputs.parameters.trades-dir}}"
|
||||
OUTPUT_DIR="{{inputs.parameters.output-dir}}"
|
||||
BF16="{{inputs.parameters.bf16}}"
|
||||
|
||||
BF16_FLAG=""
|
||||
if [ "$BF16" = "true" ]; then
|
||||
BF16_FLAG="--bf16"
|
||||
fi
|
||||
|
||||
echo "=== Feature Pre-Compute ==="
|
||||
echo "Symbol: $SYMBOL"
|
||||
echo "OHLCV: $DATA_DIR"
|
||||
echo "MBP-10: $MBP10_DIR"
|
||||
echo "Trades: $TRADES_DIR"
|
||||
echo "Output: $OUTPUT_DIR"
|
||||
echo "BF16: $BF16"
|
||||
echo "Node: $(hostname), CPUs: $(nproc), RAM: $(free -g | awk '/Mem:/{print $2}')G"
|
||||
echo ""
|
||||
|
||||
$BINARY \
|
||||
--data-dir "$DATA_DIR" \
|
||||
--mbp10-data-dir "$MBP10_DIR" \
|
||||
--trades-data-dir "$TRADES_DIR" \
|
||||
--output-dir "$OUTPUT_DIR" \
|
||||
--symbol "$SYMBOL" \
|
||||
$BF16_FLAG \
|
||||
--yes
|
||||
|
||||
echo ""
|
||||
echo "=== Output ==="
|
||||
find "$OUTPUT_DIR" -name '*.fxcache' -exec ls -lh {} \;
|
||||
du -sh "$OUTPUT_DIR"
|
||||
76
scripts/argo-precompute.sh
Executable file
76
scripts/argo-precompute.sh
Executable file
@@ -0,0 +1,76 @@
|
||||
#!/usr/bin/env bash
|
||||
# Pre-compute DQN features via Argo Workflows.
|
||||
#
|
||||
# Usage:
|
||||
# ./scripts/argo-precompute.sh ES.FUT
|
||||
# ./scripts/argo-precompute.sh ES.FUT --bf16
|
||||
# ./scripts/argo-precompute.sh ES.FUT --pool platform --watch
|
||||
#
|
||||
# Requires: argo CLI
|
||||
set -euo pipefail
|
||||
|
||||
TEMPLATE="precompute-features"
|
||||
POOL=""
|
||||
BF16=""
|
||||
DATA_DIR=""
|
||||
MBP10_DIR=""
|
||||
TRADES_DIR=""
|
||||
OUTPUT_DIR=""
|
||||
WATCH=false
|
||||
|
||||
usage() {
|
||||
cat <<EOF
|
||||
Usage: $(basename "$0") <symbol> [OPTIONS]
|
||||
|
||||
Options:
|
||||
--bf16 Write bf16 format (version 2)
|
||||
--pool <name> Node pool (default: ci-compile-cpu)
|
||||
--data-dir <dir> OHLCV data dir (default: /data/futures-baseline)
|
||||
--mbp10-dir <dir> MBP-10 data dir (default: /data/futures-baseline-mbp10)
|
||||
--trades-dir <dir> Trades data dir (default: /data/futures-baseline-trades)
|
||||
--output-dir <dir> Output dir (default: /data/feature-cache)
|
||||
--watch Follow workflow logs after submission
|
||||
-h, --help Show this help
|
||||
EOF
|
||||
exit 0
|
||||
}
|
||||
|
||||
[[ $# -eq 0 ]] && { echo "Error: symbol argument required"; usage; }
|
||||
[[ "$1" == "-h" || "$1" == "--help" ]] && usage
|
||||
|
||||
SYMBOL="$1"; shift
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--bf16) BF16="true"; shift ;;
|
||||
--pool) POOL="$2"; shift 2 ;;
|
||||
--data-dir) DATA_DIR="$2"; shift 2 ;;
|
||||
--mbp10-dir) MBP10_DIR="$2"; shift 2 ;;
|
||||
--trades-dir) TRADES_DIR="$2"; shift 2 ;;
|
||||
--output-dir) OUTPUT_DIR="$2"; shift 2 ;;
|
||||
--watch) WATCH=true; shift ;;
|
||||
-h|--help) usage ;;
|
||||
*) echo "Unknown option: $1"; usage ;;
|
||||
esac
|
||||
done
|
||||
|
||||
CMD="argo submit -n foxhunt --from=wftmpl/$TEMPLATE"
|
||||
CMD="$CMD -p symbol=$SYMBOL"
|
||||
|
||||
[[ -n "$BF16" ]] && CMD="$CMD -p bf16=$BF16"
|
||||
[[ -n "$POOL" ]] && CMD="$CMD -p node-pool=$POOL"
|
||||
[[ -n "$DATA_DIR" ]] && CMD="$CMD -p data-dir=$DATA_DIR"
|
||||
[[ -n "$MBP10_DIR" ]] && CMD="$CMD -p mbp10-dir=$MBP10_DIR"
|
||||
[[ -n "$TRADES_DIR" ]] && CMD="$CMD -p trades-dir=$TRADES_DIR"
|
||||
[[ -n "$OUTPUT_DIR" ]] && CMD="$CMD -p output-dir=$OUTPUT_DIR"
|
||||
|
||||
if $WATCH; then
|
||||
CMD="$CMD --watch"
|
||||
fi
|
||||
|
||||
echo "Submitting feature pre-compute workflow..."
|
||||
echo " symbol: $SYMBOL"
|
||||
[[ -n "$BF16" ]] && echo " bf16: $BF16"
|
||||
[[ -n "$POOL" ]] && echo " pool: $POOL"
|
||||
echo ""
|
||||
eval "$CMD"
|
||||
Reference in New Issue
Block a user