fix(k8s): remove data sync initContainer + fix --output flag
Training data is already on the PVC — the rclone sync step was wasting 2-3 minutes per job. Also fixes --output-dir → --output to match the actual CLI flag. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -72,45 +72,6 @@ spec:
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 128Mi
|
||||
# 2. Sync training data from MinIO → PVC (delta-only, fast on repeat runs)
|
||||
- name: sync-training-data
|
||||
image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-training-runtime:latest
|
||||
command: ["/bin/sh", "-c"]
|
||||
args:
|
||||
- |
|
||||
set -e
|
||||
RCLONE_FLAGS="--s3-provider=Minio --s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 --s3-access-key-id=${MINIO_ACCESS_KEY} --s3-secret-access-key=${MINIO_SECRET_KEY} --s3-no-check-bucket"
|
||||
|
||||
echo "Syncing OHLCV data..."
|
||||
rclone sync ":s3:foxhunt-training-data/futures-baseline" /data/futures-baseline $RCLONE_FLAGS --stats-one-line -v
|
||||
echo "Syncing MBP-10 data..."
|
||||
rclone sync ":s3:foxhunt-training-data/futures-baseline-mbp10" /data/futures-baseline-mbp10 $RCLONE_FLAGS --stats-one-line -v
|
||||
echo "Syncing trades data..."
|
||||
rclone sync ":s3:foxhunt-training-data/futures-baseline-trades" /data/futures-baseline-trades $RCLONE_FLAGS --stats-one-line -v
|
||||
|
||||
echo "Data sync complete:"
|
||||
du -sh /data/futures-baseline/ /data/futures-baseline-mbp10/ /data/futures-baseline-trades/ 2>/dev/null || true
|
||||
env:
|
||||
- name: MINIO_ACCESS_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: minio-credentials
|
||||
key: access-key
|
||||
- name: MINIO_SECRET_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: minio-credentials
|
||||
key: secret-key
|
||||
volumeMounts:
|
||||
- name: training-data
|
||||
mountPath: /data
|
||||
resources:
|
||||
requests:
|
||||
cpu: "1"
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: "2"
|
||||
memory: 1Gi
|
||||
containers:
|
||||
- name: training
|
||||
image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-training-runtime:latest
|
||||
@@ -126,7 +87,7 @@ spec:
|
||||
- "--data-dir=/data/futures-baseline"
|
||||
- "--mbp10-data-dir=/data/futures-baseline-mbp10"
|
||||
- "--trades-data-dir=/data/futures-baseline-trades"
|
||||
- "--output-dir=/output"
|
||||
- "--output=/output"
|
||||
env:
|
||||
- name: TRAINING_BINARY
|
||||
value: train_baseline_supervised
|
||||
|
||||
Reference in New Issue
Block a user