a75a98bd0d
Training Profile Loader: - 3-tier resolution: $FOXHUNT_TRAINING_PROFILE > filesystem > embedded defaults - DqnTrainingProfile with 10 sections, all Option<T> for sparse profiles - apply_to() applies only Some fields, preserving struct defaults - 11 unit tests, all passing TOML Profiles (config/training/): - dqn-production.toml: full Rainbow DQN (40+ params) - dqn-smoketest.toml: CI fast path (sparse, 8 overrides) - dqn-hyperopt.toml: PSO search space ranges + fixed flags - ppo-production.toml, ppo-smoketest.toml - supervised-production.toml, supervised-smoketest.toml - walk-forward.toml: window sizes CLI Integration: - train_baseline_rl: --training-profile (default: dqn-production) - train_baseline_supervised: --training-profile (default: supervised-production) - Merge priority: CLI args > TOML profile > GPU profile > struct defaults Smoke Tests: - smoke_params() now loads dqn-smoketest.toml instead of hardcoding - Production features set as manual overrides (testing flags, not config) Infrastructure: - K8s job-template.yaml: TRAINING_PROFILE env var + --training-profile arg - Delete old config/ml/training.toml (replaced, zero callers) Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
131 lines
4.5 KiB
YAML
131 lines
4.5 KiB
YAML
apiVersion: batch/v1
|
||
kind: Job
|
||
metadata:
|
||
generateName: training-
|
||
namespace: foxhunt
|
||
labels:
|
||
app.kubernetes.io/name: training
|
||
app.kubernetes.io/part-of: foxhunt
|
||
foxhunt/job-type: training
|
||
spec:
|
||
backoffLimit: 1
|
||
activeDeadlineSeconds: 21600 # 6 hours — hyperopt runs 20 trials × 8 epochs
|
||
ttlSecondsAfterFinished: 600
|
||
template:
|
||
metadata:
|
||
annotations:
|
||
prometheus.io/scrape: "true"
|
||
prometheus.io/port: "9094"
|
||
prometheus.io/path: "/metrics"
|
||
labels:
|
||
app.kubernetes.io/name: training
|
||
app.kubernetes.io/component: training-workflow
|
||
app.kubernetes.io/part-of: foxhunt
|
||
foxhunt/job-type: training
|
||
spec:
|
||
nodeSelector:
|
||
k8s.scaleway.com/pool-name: ci-training-h100
|
||
tolerations:
|
||
- key: nvidia.com/gpu
|
||
operator: Exists
|
||
effect: NoSchedule
|
||
# Cilium CNI agent takes ~30s to initialize on fresh scale-from-zero nodes
|
||
- key: node.cilium.io/agent-not-ready
|
||
operator: Exists
|
||
effect: NoSchedule
|
||
imagePullSecrets:
|
||
- name: gitlab-registry
|
||
restartPolicy: Never
|
||
initContainers:
|
||
# 1. Fetch training binary from GitLab Generic Package Registry
|
||
- name: fetch-binary
|
||
image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-training-runtime:latest
|
||
command: ["/bin/sh", "-c"]
|
||
args:
|
||
- |
|
||
set -e
|
||
BINARY="$(TRAINING_BINARY)"
|
||
curl -fSL -o "/binaries/${BINARY}" \
|
||
--header "DEPLOY-TOKEN: ${GITLAB_DEPLOY_TOKEN}" \
|
||
"${GITLAB_API}/projects/1/packages/generic/foxhunt-training/${FOXHUNT_RELEASE}/${BINARY}"
|
||
chmod +x "/binaries/${BINARY}"
|
||
echo "Fetched ${BINARY} ${FOXHUNT_RELEASE} ($(stat -c%s /binaries/${BINARY}) bytes)"
|
||
env:
|
||
- name: TRAINING_BINARY
|
||
value: train_baseline_supervised
|
||
- name: GITLAB_DEPLOY_TOKEN
|
||
valueFrom:
|
||
secretKeyRef:
|
||
name: gitlab-deploy-token
|
||
key: token
|
||
- name: GITLAB_API
|
||
value: "http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181/api/v4"
|
||
- name: FOXHUNT_RELEASE
|
||
value: "latest"
|
||
volumeMounts:
|
||
- name: binaries
|
||
mountPath: /binaries
|
||
resources:
|
||
requests:
|
||
cpu: 100m
|
||
memory: 64Mi
|
||
limits:
|
||
cpu: 500m
|
||
memory: 128Mi
|
||
containers:
|
||
- name: training
|
||
image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-training-runtime:latest
|
||
# Available binaries (copied by initContainer):
|
||
# train_baseline_rl (for dqn, ppo)
|
||
# train_baseline_supervised (for tft, mamba2, tggn, tlob, liquid, kan, xlstm, diffusion)
|
||
# evaluate_baseline
|
||
# hyperopt_baseline_rl (for dqn, ppo)
|
||
# hyperopt_baseline_supervised (for tft, mamba2)
|
||
command: ["/binaries/$(TRAINING_BINARY)"]
|
||
args:
|
||
- "--symbol=ES.FUT"
|
||
- "--data-dir=/data/futures-baseline"
|
||
- "--mbp10-data-dir=/data/futures-baseline-mbp10"
|
||
- "--trades-data-dir=/data/futures-baseline-trades"
|
||
- "--training-profile=$(TRAINING_PROFILE)"
|
||
- "--output=/output"
|
||
env:
|
||
- name: TRAINING_BINARY
|
||
value: train_baseline_supervised
|
||
- name: TRAINING_PROFILE
|
||
value: "dqn-production"
|
||
- name: RUST_LOG
|
||
value: info
|
||
- name: SQLX_OFFLINE
|
||
value: "true"
|
||
- name: OTEL_EXPORTER_OTLP_ENDPOINT
|
||
value: "http://tempo.foxhunt.svc.cluster.local:4317"
|
||
volumeMounts:
|
||
- name: training-data
|
||
mountPath: /data
|
||
readOnly: true
|
||
- name: output
|
||
mountPath: /output
|
||
- name: binaries
|
||
mountPath: /binaries
|
||
readOnly: true
|
||
resources:
|
||
requests:
|
||
nvidia.com/gpu: "1"
|
||
cpu: "4"
|
||
memory: 16Gi
|
||
limits:
|
||
nvidia.com/gpu: "1"
|
||
cpu: "8"
|
||
memory: 32Gi
|
||
volumes:
|
||
- name: training-data
|
||
persistentVolumeClaim:
|
||
claimName: training-data-pvc
|
||
- name: output
|
||
emptyDir:
|
||
sizeLimit: 2Gi
|
||
- name: binaries
|
||
emptyDir:
|
||
sizeLimit: 500Mi
|