diff --git a/infra/k8s/argo/precompute-features-template.yaml b/infra/k8s/argo/precompute-features-template.yaml new file mode 100644 index 000000000..272ddfd5b --- /dev/null +++ b/infra/k8s/argo/precompute-features-template.yaml @@ -0,0 +1,123 @@ +# Pre-compute DQN training features to .fxcache format. +# +# Runs after data downloads complete. Reads DBN files from training-data PVC, +# extracts all features (OHLCV 42-dim + targets 4-dim + OFI 8-dim), writes +# flat binary cache for zero-overhead GPU loading. +# +# Usage: +# argo submit -n foxhunt --from=wftmpl/precompute-features +# ./scripts/argo-precompute.sh ES.FUT --bf16 +--- +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: precompute-features + namespace: foxhunt + labels: + app.kubernetes.io/name: precompute-features + app.kubernetes.io/part-of: foxhunt +spec: + entrypoint: precompute + serviceAccountName: argo-workflow + podMetadata: + labels: + app.kubernetes.io/part-of: foxhunt + app.kubernetes.io/component: precompute-features + securityContext: + fsGroup: 1000 + ttlStrategy: + secondsAfterCompletion: 3600 + activeDeadlineSeconds: 7200 + + arguments: + parameters: + - name: symbol + value: ES.FUT + - name: data-dir + value: /data/futures-baseline + - name: mbp10-dir + value: /data/futures-baseline-mbp10 + - name: trades-dir + value: /data/futures-baseline-trades + - name: output-dir + value: /data/feature-cache + - name: node-pool + value: ci-compile-cpu + - name: bf16 + value: "false" + + volumes: + - name: training-data + persistentVolumeClaim: + claimName: training-data-pvc + + templates: + - name: precompute + inputs: + parameters: + - name: symbol + - name: data-dir + - name: mbp10-dir + - name: trades-dir + - name: output-dir + - name: node-pool + - name: bf16 + nodeSelector: + k8s.scaleway.com/pool-name: "{{inputs.parameters.node-pool}}" + container: + image: ubuntu:24.04 + command: ["/bin/bash", "-c"] + env: + - name: RUST_LOG + value: info + resources: + requests: + cpu: "4" + memory: 8Gi + limits: + cpu: "16" + memory: 32Gi + volumeMounts: + - name: training-data + mountPath: /data + args: + - | + set -e + BINARY=/data/bin/precompute_features + chmod +x "$BINARY" + + SYMBOL="{{inputs.parameters.symbol}}" + DATA_DIR="{{inputs.parameters.data-dir}}" + MBP10_DIR="{{inputs.parameters.mbp10-dir}}" + TRADES_DIR="{{inputs.parameters.trades-dir}}" + OUTPUT_DIR="{{inputs.parameters.output-dir}}" + BF16="{{inputs.parameters.bf16}}" + + BF16_FLAG="" + if [ "$BF16" = "true" ]; then + BF16_FLAG="--bf16" + fi + + echo "=== Feature Pre-Compute ===" + echo "Symbol: $SYMBOL" + echo "OHLCV: $DATA_DIR" + echo "MBP-10: $MBP10_DIR" + echo "Trades: $TRADES_DIR" + echo "Output: $OUTPUT_DIR" + echo "BF16: $BF16" + echo "Node: $(hostname), CPUs: $(nproc), RAM: $(free -g | awk '/Mem:/{print $2}')G" + echo "" + + $BINARY \ + --data-dir "$DATA_DIR" \ + --mbp10-data-dir "$MBP10_DIR" \ + --trades-data-dir "$TRADES_DIR" \ + --output-dir "$OUTPUT_DIR" \ + --symbol "$SYMBOL" \ + $BF16_FLAG \ + --yes + + echo "" + echo "=== Output ===" + find "$OUTPUT_DIR" -name '*.fxcache' -exec ls -lh {} \; + du -sh "$OUTPUT_DIR" diff --git a/scripts/argo-precompute.sh b/scripts/argo-precompute.sh new file mode 100755 index 000000000..82bf4dcf0 --- /dev/null +++ b/scripts/argo-precompute.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# Pre-compute DQN features via Argo Workflows. +# +# Usage: +# ./scripts/argo-precompute.sh ES.FUT +# ./scripts/argo-precompute.sh ES.FUT --bf16 +# ./scripts/argo-precompute.sh ES.FUT --pool platform --watch +# +# Requires: argo CLI +set -euo pipefail + +TEMPLATE="precompute-features" +POOL="" +BF16="" +DATA_DIR="" +MBP10_DIR="" +TRADES_DIR="" +OUTPUT_DIR="" +WATCH=false + +usage() { + cat < [OPTIONS] + +Options: + --bf16 Write bf16 format (version 2) + --pool Node pool (default: ci-compile-cpu) + --data-dir OHLCV data dir (default: /data/futures-baseline) + --mbp10-dir MBP-10 data dir (default: /data/futures-baseline-mbp10) + --trades-dir Trades data dir (default: /data/futures-baseline-trades) + --output-dir Output dir (default: /data/feature-cache) + --watch Follow workflow logs after submission + -h, --help Show this help +EOF + exit 0 +} + +[[ $# -eq 0 ]] && { echo "Error: symbol argument required"; usage; } +[[ "$1" == "-h" || "$1" == "--help" ]] && usage + +SYMBOL="$1"; shift + +while [[ $# -gt 0 ]]; do + case $1 in + --bf16) BF16="true"; shift ;; + --pool) POOL="$2"; shift 2 ;; + --data-dir) DATA_DIR="$2"; shift 2 ;; + --mbp10-dir) MBP10_DIR="$2"; shift 2 ;; + --trades-dir) TRADES_DIR="$2"; shift 2 ;; + --output-dir) OUTPUT_DIR="$2"; shift 2 ;; + --watch) WATCH=true; shift ;; + -h|--help) usage ;; + *) echo "Unknown option: $1"; usage ;; + esac +done + +CMD="argo submit -n foxhunt --from=wftmpl/$TEMPLATE" +CMD="$CMD -p symbol=$SYMBOL" + +[[ -n "$BF16" ]] && CMD="$CMD -p bf16=$BF16" +[[ -n "$POOL" ]] && CMD="$CMD -p node-pool=$POOL" +[[ -n "$DATA_DIR" ]] && CMD="$CMD -p data-dir=$DATA_DIR" +[[ -n "$MBP10_DIR" ]] && CMD="$CMD -p mbp10-dir=$MBP10_DIR" +[[ -n "$TRADES_DIR" ]] && CMD="$CMD -p trades-dir=$TRADES_DIR" +[[ -n "$OUTPUT_DIR" ]] && CMD="$CMD -p output-dir=$OUTPUT_DIR" + +if $WATCH; then + CMD="$CMD --watch" +fi + +echo "Submitting feature pre-compute workflow..." +echo " symbol: $SYMBOL" +[[ -n "$BF16" ]] && echo " bf16: $BF16" +[[ -n "$POOL" ]] && echo " pool: $POOL" +echo "" +eval "$CMD"