b31329931f
- Remove all HTTPS/TLS from MinIO (plain HTTP for internal cluster traffic) - Fix sccache 0% cache hit rate (rustls rejected self-signed MinIO cert) - Remove hardcoded URLs from k8s_dispatcher.rs (S3_ENDPOINT, TRAINING_RUNTIME_IMAGE, CALLBACK_ENDPOINT now required env vars) - Update GitLab registry S3 credentials to HTTP endpoint - Fix PVC manifest (20Gi → 100Gi to match cluster) - Fix nodeSelector: infra/foxhunt → platform (match actual node pool) - Fix rclone trailing backslash causing chmod to be parsed as rclone args - Remove minio-ca-cert ConfigMap references from all manifests - Update trading-service GPU overlay to l40s pool 20 files changed, -118 lines net Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
130 lines
4.4 KiB
YAML
130 lines
4.4 KiB
YAML
apiVersion: batch/v1
|
||
kind: Job
|
||
metadata:
|
||
generateName: training-
|
||
namespace: foxhunt
|
||
labels:
|
||
app.kubernetes.io/name: training
|
||
app.kubernetes.io/part-of: foxhunt
|
||
foxhunt/job-type: training
|
||
spec:
|
||
backoffLimit: 1
|
||
activeDeadlineSeconds: 21600 # 6 hours — hyperopt runs 20 trials × 8 epochs
|
||
ttlSecondsAfterFinished: 600
|
||
template:
|
||
metadata:
|
||
annotations:
|
||
gitlab.com/prometheus_scrape: "true"
|
||
gitlab.com/prometheus_port: "9094"
|
||
gitlab.com/prometheus_path: "/metrics"
|
||
labels:
|
||
app.kubernetes.io/name: training
|
||
foxhunt/job-type: training
|
||
spec:
|
||
nodeSelector:
|
||
k8s.scaleway.com/pool-name: ci-training
|
||
tolerations:
|
||
- key: nvidia.com/gpu
|
||
operator: Exists
|
||
effect: NoSchedule
|
||
# Cilium CNI agent takes ~30s to initialize on fresh scale-from-zero nodes
|
||
- key: node.cilium.io/agent-not-ready
|
||
operator: Exists
|
||
effect: NoSchedule
|
||
imagePullSecrets:
|
||
- name: gitlab-registry
|
||
restartPolicy: Never
|
||
# Fetch training binary from MinIO at job startup via rclone.
|
||
# Replaces PVC-based copy — no node affinity constraint, no L40S autoscale for writes.
|
||
initContainers:
|
||
- name: fetch-binary
|
||
image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-training-runtime:latest
|
||
command: ["/bin/sh", "-c"]
|
||
args:
|
||
- |
|
||
set -e
|
||
rclone copyto \
|
||
":s3:foxhunt-binaries/training/$(TRAINING_BINARY)" \
|
||
"/binaries/$(TRAINING_BINARY)" \
|
||
--s3-provider=Minio \
|
||
--s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 \
|
||
--s3-access-key-id="${MINIO_ACCESS_KEY}" \
|
||
--s3-secret-access-key="${MINIO_SECRET_KEY}" \
|
||
--s3-no-check-bucket \
|
||
chmod +x "/binaries/$(TRAINING_BINARY)"
|
||
echo "Fetched $(TRAINING_BINARY) ($(stat -c%s /binaries/$(TRAINING_BINARY)) bytes)"
|
||
env:
|
||
- name: TRAINING_BINARY
|
||
value: train_baseline_supervised
|
||
- name: MINIO_ACCESS_KEY
|
||
valueFrom:
|
||
secretKeyRef:
|
||
name: minio-credentials
|
||
key: access-key
|
||
- name: MINIO_SECRET_KEY
|
||
valueFrom:
|
||
secretKeyRef:
|
||
name: minio-credentials
|
||
key: secret-key
|
||
volumeMounts:
|
||
- name: binaries
|
||
mountPath: /binaries
|
||
resources:
|
||
requests:
|
||
cpu: 100m
|
||
memory: 64Mi
|
||
limits:
|
||
cpu: 500m
|
||
memory: 128Mi
|
||
containers:
|
||
- name: training
|
||
image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-training-runtime:latest
|
||
# Available binaries (copied by initContainer):
|
||
# train_baseline_rl (for dqn, ppo)
|
||
# train_baseline_supervised (for tft, mamba2, tggn, tlob, liquid, kan, xlstm, diffusion)
|
||
# evaluate_baseline
|
||
# hyperopt_baseline_rl (for dqn, ppo)
|
||
# hyperopt_baseline_supervised (for tft, mamba2)
|
||
command: ["/binaries/$(TRAINING_BINARY)"]
|
||
args:
|
||
- "--symbol=ES.FUT"
|
||
- "--data-dir=/data/futures-baseline"
|
||
- "--output-dir=/output"
|
||
env:
|
||
- name: TRAINING_BINARY
|
||
value: train_baseline_supervised
|
||
- name: RUST_LOG
|
||
value: info
|
||
- name: SQLX_OFFLINE
|
||
value: "true"
|
||
- name: OTEL_EXPORTER_OTLP_ENDPOINT
|
||
value: "http://tempo.foxhunt.svc.cluster.local:4317"
|
||
volumeMounts:
|
||
- name: training-data
|
||
mountPath: /data
|
||
readOnly: true
|
||
- name: output
|
||
mountPath: /output
|
||
- name: binaries
|
||
mountPath: /binaries
|
||
readOnly: true
|
||
resources:
|
||
requests:
|
||
nvidia.com/gpu: "1"
|
||
cpu: "4"
|
||
memory: 16Gi
|
||
limits:
|
||
nvidia.com/gpu: "1"
|
||
cpu: "8"
|
||
memory: 32Gi
|
||
volumes:
|
||
- name: training-data
|
||
persistentVolumeClaim:
|
||
claimName: training-data-pvc
|
||
- name: output
|
||
emptyDir:
|
||
sizeLimit: 2Gi
|
||
- name: binaries
|
||
emptyDir:
|
||
sizeLimit: 500Mi
|