135 lines
3.9 KiB
YAML
135 lines
3.9 KiB
YAML
# Pre-compute DQN training features to .fxcache format.
|
|
#
|
|
# Runs after data downloads complete. Reads DBN files from training-data PVC,
|
|
# extracts all features (OHLCV 42-dim + targets 4-dim + OFI 8-dim), writes
|
|
# flat binary cache for zero-overhead GPU loading.
|
|
#
|
|
# Usage:
|
|
# argo submit -n foxhunt --from=wftmpl/precompute-features
|
|
# ./scripts/argo-precompute.sh ES.FUT --bf16
|
|
---
|
|
apiVersion: argoproj.io/v1alpha1
|
|
kind: WorkflowTemplate
|
|
metadata:
|
|
name: precompute-features
|
|
namespace: foxhunt
|
|
labels:
|
|
app.kubernetes.io/name: precompute-features
|
|
app.kubernetes.io/part-of: foxhunt
|
|
spec:
|
|
entrypoint: precompute
|
|
serviceAccountName: argo-workflow
|
|
podMetadata:
|
|
labels:
|
|
app.kubernetes.io/part-of: foxhunt
|
|
app.kubernetes.io/component: precompute-features
|
|
securityContext:
|
|
fsGroup: 1000
|
|
ttlStrategy:
|
|
secondsAfterCompletion: 3600
|
|
activeDeadlineSeconds: 7200
|
|
|
|
arguments:
|
|
parameters:
|
|
- name: symbol
|
|
value: ES.FUT
|
|
- name: data-dir
|
|
value: /data/futures-baseline
|
|
- name: mbp10-dir
|
|
value: /data/futures-baseline-mbp10
|
|
- name: trades-dir
|
|
value: /data/futures-baseline-trades
|
|
- name: output-dir
|
|
value: /feature-cache
|
|
- name: node-pool
|
|
value: ci-compile-cpu
|
|
- name: bf16
|
|
value: "false"
|
|
|
|
volumes:
|
|
- name: training-data
|
|
persistentVolumeClaim:
|
|
claimName: training-data-pvc
|
|
- name: feature-cache
|
|
persistentVolumeClaim:
|
|
claimName: feature-cache-pvc
|
|
|
|
templates:
|
|
- name: precompute
|
|
inputs:
|
|
parameters:
|
|
- name: symbol
|
|
- name: data-dir
|
|
- name: mbp10-dir
|
|
- name: trades-dir
|
|
- name: output-dir
|
|
- name: node-pool
|
|
- name: bf16
|
|
nodeSelector:
|
|
k8s.scaleway.com/pool-name: "{{inputs.parameters.node-pool}}"
|
|
container:
|
|
image: ubuntu:24.04
|
|
command: ["/bin/bash", "-c"]
|
|
env:
|
|
- name: RUST_LOG
|
|
value: info
|
|
resources:
|
|
requests:
|
|
cpu: "4"
|
|
memory: 16Gi
|
|
limits:
|
|
cpu: "28"
|
|
memory: 56Gi
|
|
volumeMounts:
|
|
- name: training-data
|
|
mountPath: /data
|
|
readOnly: true
|
|
- name: feature-cache
|
|
mountPath: /feature-cache
|
|
args:
|
|
- |
|
|
set -e
|
|
# Binary deployed to PVC by compile-and-train workflow
|
|
BINARY=/data/bin/precompute_features
|
|
if [ ! -x "$BINARY" ]; then
|
|
echo "ERROR: $BINARY not found or not executable. Run compile-and-train first."
|
|
exit 1
|
|
fi
|
|
|
|
SYMBOL="{{inputs.parameters.symbol}}"
|
|
# --symbol flag filters to the correct subdirectory (collect_dbn_files_filtered)
|
|
DATA_DIR="{{inputs.parameters.data-dir}}"
|
|
MBP10_DIR="{{inputs.parameters.mbp10-dir}}"
|
|
TRADES_DIR="{{inputs.parameters.trades-dir}}"
|
|
OUTPUT_DIR="{{inputs.parameters.output-dir}}"
|
|
BF16="{{inputs.parameters.bf16}}"
|
|
|
|
BF16_FLAG=""
|
|
if [ "$BF16" = "true" ]; then
|
|
BF16_FLAG="--bf16"
|
|
fi
|
|
|
|
echo "=== Feature Pre-Compute ==="
|
|
echo "Symbol: $SYMBOL"
|
|
echo "OHLCV: $DATA_DIR"
|
|
echo "MBP-10: $MBP10_DIR"
|
|
echo "Trades: $TRADES_DIR"
|
|
echo "Output: $OUTPUT_DIR"
|
|
echo "BF16: $BF16"
|
|
echo "Node: $(hostname), CPUs: $(nproc), RAM: $(free -g | awk '/Mem:/{print $2}')G"
|
|
echo ""
|
|
|
|
$BINARY \
|
|
--data-dir "$DATA_DIR" \
|
|
--mbp10-data-dir "$MBP10_DIR" \
|
|
--trades-data-dir "$TRADES_DIR" \
|
|
--output-dir "$OUTPUT_DIR" \
|
|
--symbol "$SYMBOL" \
|
|
$BF16_FLAG \
|
|
--yes
|
|
|
|
echo ""
|
|
echo "=== Output ==="
|
|
find "$OUTPUT_DIR" -name '*.fxcache' -exec ls -lh {} \;
|
|
du -sh "$OUTPUT_DIR"
|