242c18f4ab
Usage: ./scripts/argo-precompute.sh [--branch main] [--watch] Deletes old fxcache, builds precompute binary, runs with MBP-10 + trades data. OFI_DIM=20 (20 microstructure features). ~5-10 min. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
78 lines
2.3 KiB
Bash
Executable File
78 lines
2.3 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Regenerate fxcache on PVC from MBP-10 + trades data.
|
|
# Usage: ./scripts/argo-precompute.sh [--branch main] [--watch]
|
|
set -euo pipefail
|
|
|
|
BRANCH="main"
|
|
WATCH=false
|
|
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--branch) BRANCH="$2"; shift 2;;
|
|
--watch) WATCH=true; shift;;
|
|
-h|--help)
|
|
echo "Usage: $0 [--branch <branch>] [--watch]"
|
|
echo " Regenerates fxcache on PVC with OFI_DIM=20 (20 microstructure features)"
|
|
exit 0;;
|
|
*) echo "Unknown option: $1"; exit 1;;
|
|
esac
|
|
done
|
|
|
|
echo "=== Argo Precompute fxcache ==="
|
|
echo " branch: $BRANCH"
|
|
echo ""
|
|
|
|
WORKFLOW=$(cat <<EOF | kubectl -n foxhunt create -f - -o name
|
|
apiVersion: argoproj.io/v1alpha1
|
|
kind: Workflow
|
|
metadata:
|
|
generateName: precompute-fxcache-
|
|
spec:
|
|
entrypoint: precompute
|
|
volumes:
|
|
- name: training-data
|
|
persistentVolumeClaim:
|
|
claimName: training-data-pvc
|
|
templates:
|
|
- name: precompute
|
|
nodeSelector:
|
|
k8s.scaleway.com/pool-name: ci-compile-cpu
|
|
container:
|
|
image: rg.fr-par.scw.cloud/foxhunt/ci-builder:latest
|
|
command: ["/bin/bash", "-c"]
|
|
args:
|
|
- |
|
|
set -ex
|
|
cd /workspace
|
|
git clone --depth 1 --branch ${BRANCH} ssh://root@100.90.76.85:2222/root/foxhunt.git .
|
|
rm -f /mnt/training-data/feature-cache/*.fxcache
|
|
SQLX_OFFLINE=true cargo build -p ml --release --example precompute_features
|
|
echo "yes" | SQLX_OFFLINE=true ./target/release/examples/precompute_features \\
|
|
--data-dir /mnt/training-data/futures-baseline \\
|
|
--mbp10-data-dir /mnt/training-data/futures-baseline-mbp10 \\
|
|
--trades-data-dir /mnt/training-data/futures-baseline-trades
|
|
echo "=== Precompute complete ==="
|
|
ls -lh /mnt/training-data/feature-cache/*.fxcache
|
|
xxd /mnt/training-data/feature-cache/*.fxcache | head -2
|
|
volumeMounts:
|
|
- name: training-data
|
|
mountPath: /mnt/training-data
|
|
resources:
|
|
requests:
|
|
cpu: "4"
|
|
memory: "16Gi"
|
|
limits:
|
|
cpu: "8"
|
|
memory: "32Gi"
|
|
timeout: "30m"
|
|
EOF
|
|
)
|
|
|
|
echo "Submitted: $WORKFLOW"
|
|
|
|
if $WATCH; then
|
|
echo "Watching logs..."
|
|
sleep 5
|
|
kubectl -n foxhunt logs -f "$WORKFLOW" --tail=50
|
|
fi
|