922fd5e93e
GPU test pipeline: - Add perf-benchmark step after compile-and-test - Runs DQN training on 3Q ES.FUT (OHLCV + MBP-10 + trades) - Reports epoch time (ms), fails if > 500ms (H100 regression guard) - batch_size=1024, 5 epochs × 100 steps, skips epoch 1 (init) Populate test data job: - Copy 3 quarters per symbol (was 1) for all data types - OHLCV: ~6MB/symbol for walk-forward + perf benchmarks - MBP-10: 3Q for OFI feature pipeline testing - Trades: 3Q for trade-flow feature testing Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
111 lines
4.0 KiB
YAML
111 lines
4.0 KiB
YAML
# infra/k8s/training/populate-test-data-job.yaml
|
|
apiVersion: batch/v1
|
|
kind: Job
|
|
metadata:
|
|
generateName: populate-test-data-
|
|
namespace: foxhunt
|
|
labels:
|
|
app.kubernetes.io/name: populate-test-data
|
|
app.kubernetes.io/part-of: foxhunt
|
|
spec:
|
|
backoffLimit: 1
|
|
ttlSecondsAfterFinished: 300
|
|
template:
|
|
spec:
|
|
restartPolicy: Never
|
|
nodeSelector:
|
|
k8s.scaleway.com/pool-name: ci-training-h100
|
|
tolerations:
|
|
- key: nvidia.com/gpu
|
|
operator: Exists
|
|
effect: NoSchedule
|
|
- key: node.cilium.io/agent-not-ready
|
|
operator: Exists
|
|
effect: NoSchedule
|
|
containers:
|
|
- name: populate
|
|
image: busybox:1.37
|
|
command: ["/bin/sh", "-c"]
|
|
args:
|
|
- |
|
|
set -e
|
|
echo "=== Populating test data PVC ==="
|
|
|
|
# All symbols used in CI tests (4 symbols from futures-baseline)
|
|
SYMBOLS="ES.FUT 6E.FUT ZN.FUT NQ.FUT"
|
|
|
|
for SYM in $SYMBOLS; do
|
|
echo "--- $SYM ---"
|
|
|
|
# OHLCV (used by DQN/PPO training tests and walk-forward validation)
|
|
# Copy 3 quarters per symbol: enough for --train-months 3 --val-months 1 --test-months 1
|
|
# (5 months window), perf benchmarks, and walk-forward tests. ~6MB/symbol compressed.
|
|
mkdir -p /test-data/ohlcv/$SYM
|
|
if ls /training-data/futures-baseline/$SYM/*.dbn.zst >/dev/null 2>&1; then
|
|
COUNT=0
|
|
for F in $(ls /training-data/futures-baseline/$SYM/*.dbn.zst 2>/dev/null | head -3); do
|
|
cp "$F" /test-data/ohlcv/$SYM/
|
|
echo " OHLCV: copied $(basename $F)"
|
|
COUNT=$((COUNT + 1))
|
|
done
|
|
echo " OHLCV: $COUNT quarters copied"
|
|
else
|
|
echo " OHLCV: no source data for $SYM"
|
|
fi
|
|
|
|
# MBP-10 (used by OFI feature loading)
|
|
mkdir -p /test-data/mbp10/$SYM
|
|
if ls /training-data/futures-baseline-mbp10/$SYM/*.dbn.zst >/dev/null 2>&1; then
|
|
COUNT=0
|
|
for F in $(ls /training-data/futures-baseline-mbp10/$SYM/*.dbn.zst 2>/dev/null | head -3); do
|
|
cp "$F" /test-data/mbp10/$SYM/
|
|
echo " MBP-10: copied $(basename $F)"
|
|
COUNT=$((COUNT + 1))
|
|
done
|
|
else
|
|
echo " MBP-10: no source data for $SYM"
|
|
fi
|
|
|
|
# Trades
|
|
mkdir -p /test-data/trades/$SYM
|
|
if ls /training-data/futures-baseline-trades/$SYM/*.dbn.zst >/dev/null 2>&1; then
|
|
COUNT=0
|
|
for F in $(ls /training-data/futures-baseline-trades/$SYM/*.dbn.zst 2>/dev/null | head -3); do
|
|
cp "$F" /test-data/trades/$SYM/
|
|
echo " Trades: copied $(basename $F)"
|
|
COUNT=$((COUNT + 1))
|
|
done
|
|
else
|
|
echo " Trades: no source data for $SYM"
|
|
fi
|
|
done
|
|
|
|
echo ""
|
|
echo "=== Test data contents ==="
|
|
find /test-data -type f -exec ls -lh {} \;
|
|
echo "Total: $(du -sh /test-data | cut -f1)"
|
|
echo "=== Done ==="
|
|
resources:
|
|
requests:
|
|
nvidia.com/gpu: "1"
|
|
cpu: 100m
|
|
memory: 128Mi
|
|
limits:
|
|
nvidia.com/gpu: "1"
|
|
cpu: 500m
|
|
memory: 256Mi
|
|
volumeMounts:
|
|
- name: training-data
|
|
mountPath: /training-data
|
|
readOnly: true
|
|
- name: test-data
|
|
mountPath: /test-data
|
|
volumes:
|
|
- name: training-data
|
|
persistentVolumeClaim:
|
|
claimName: training-data-pvc
|
|
readOnly: true
|
|
- name: test-data
|
|
persistentVolumeClaim:
|
|
claimName: test-data-pvc
|