infra: add training orchestrator and smoke test scripts
- train.sh: submit GPU training K8s Jobs with presets (quick-test, single-model, full-ensemble, hyperopt) - smoke-test.sh: end-to-end verification of cluster, networking, databases, services, and node pools Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
147
infra/scripts/smoke-test.sh
Executable file
147
infra/scripts/smoke-test.sh
Executable file
@@ -0,0 +1,147 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# smoke-test.sh - End-to-end smoke test for the Foxhunt Kapsule cluster
|
||||
# ---------------------------------------------------------------------------
|
||||
# Runs a series of checks and prints OK/FAIL for each.
|
||||
# Exits 0 if everything passes, 1 with failure count otherwise.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
NAMESPACE="foxhunt"
|
||||
TAILSCALE_NS="tailscale"
|
||||
KAPSULE_NODE_NAME="foxhunt-kapsule"
|
||||
CLUSTER_ID="${SCW_KAPSULE_CLUSTER_ID:-}"
|
||||
SCW_REGION="${SCW_REGION:-nl-ams}"
|
||||
|
||||
GRPC_SERVICES=(
|
||||
trading-engine
|
||||
trading-agent
|
||||
ml-training
|
||||
risk-manager
|
||||
data-acquisition
|
||||
broker-gateway
|
||||
)
|
||||
|
||||
PASS=0
|
||||
FAIL=0
|
||||
FAILURES=()
|
||||
|
||||
# --- Helpers --------------------------------------------------------------
|
||||
|
||||
check() {
|
||||
local description="$1"
|
||||
shift
|
||||
if "$@" >/dev/null 2>&1; then
|
||||
printf " [OK] %s\n" "$description"
|
||||
(( PASS++ ))
|
||||
else
|
||||
printf " [FAIL] %s\n" "$description"
|
||||
(( FAIL++ ))
|
||||
FAILURES+=("$description")
|
||||
fi
|
||||
}
|
||||
|
||||
section() {
|
||||
echo ""
|
||||
echo "=== $1 ==="
|
||||
}
|
||||
|
||||
# --- 1. Cluster -----------------------------------------------------------
|
||||
section "Cluster"
|
||||
|
||||
check "kubectl can reach nodes" \
|
||||
kubectl get nodes --no-headers
|
||||
|
||||
check "Namespace '${NAMESPACE}' exists" \
|
||||
kubectl get namespace "$NAMESPACE"
|
||||
|
||||
# --- 2. Tailscale ---------------------------------------------------------
|
||||
section "Tailscale"
|
||||
|
||||
check "Tailscale pod running in '${TAILSCALE_NS}' namespace" \
|
||||
bash -c "kubectl -n ${TAILSCALE_NS} get pods --field-selector=status.phase=Running --no-headers 2>/dev/null | grep -q ."
|
||||
|
||||
check "Tailscale status shows '${KAPSULE_NODE_NAME}'" \
|
||||
bash -c "tailscale status 2>/dev/null | grep -q '${KAPSULE_NODE_NAME}'"
|
||||
|
||||
# --- 3. Databases ---------------------------------------------------------
|
||||
section "Databases"
|
||||
|
||||
check "PostgreSQL accepts connections" \
|
||||
bash -c "
|
||||
PG_POD=\$(kubectl -n ${NAMESPACE} get pod -l app=postgresql -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)
|
||||
kubectl -n ${NAMESPACE} exec \"\$PG_POD\" -- pg_isready -U foxhunt
|
||||
"
|
||||
|
||||
check "Redis responds to PING" \
|
||||
bash -c "
|
||||
REDIS_POD=\$(kubectl -n ${NAMESPACE} get pod -l app=redis -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)
|
||||
kubectl -n ${NAMESPACE} exec \"\$REDIS_POD\" -- redis-cli ping | grep -q PONG
|
||||
"
|
||||
|
||||
check "QuestDB HTTP health" \
|
||||
bash -c "
|
||||
QUEST_POD=\$(kubectl -n ${NAMESPACE} get pod -l app=questdb -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)
|
||||
kubectl -n ${NAMESPACE} exec \"\$QUEST_POD\" -- curl -sf http://localhost:9000/
|
||||
"
|
||||
|
||||
# --- 4. Services ----------------------------------------------------------
|
||||
section "Services (gRPC + web-gateway)"
|
||||
|
||||
for svc in "${GRPC_SERVICES[@]}"; do
|
||||
check "Pod '${svc}' is Running" \
|
||||
bash -c "kubectl -n ${NAMESPACE} get pods -l app=${svc} --field-selector=status.phase=Running --no-headers 2>/dev/null | grep -q ."
|
||||
done
|
||||
|
||||
check "Pod 'web-gateway' is Running" \
|
||||
bash -c "kubectl -n ${NAMESPACE} get pods -l app=web-gateway --field-selector=status.phase=Running --no-headers 2>/dev/null | grep -q ."
|
||||
|
||||
# --- 5. Web gateway health ------------------------------------------------
|
||||
section "Web Gateway Health"
|
||||
|
||||
check "web-gateway /health returns 200" \
|
||||
bash -c "
|
||||
CLUSTER_IP=\$(kubectl -n ${NAMESPACE} get svc web-gateway -o jsonpath='{.spec.clusterIP}' 2>/dev/null)
|
||||
PORT=\$(kubectl -n ${NAMESPACE} get svc web-gateway -o jsonpath='{.spec.ports[0].port}' 2>/dev/null)
|
||||
kubectl -n ${NAMESPACE} run smoke-curl --rm -i --restart=Never --image=curlimages/curl -- \
|
||||
curl -sf http://\${CLUSTER_IP}:\${PORT}/health
|
||||
"
|
||||
|
||||
# --- 6. Node pools -------------------------------------------------------
|
||||
section "Node Pools"
|
||||
|
||||
check "Always-on node is Ready" \
|
||||
bash -c "kubectl get nodes -l k8s.scaleway.com/pool-name=always-on --no-headers 2>/dev/null | grep -q ' Ready'"
|
||||
|
||||
check "CI node pool exists" \
|
||||
bash -c "
|
||||
[[ -z '${CLUSTER_ID}' ]] && exit 1
|
||||
scw k8s pool list cluster-id=${CLUSTER_ID} region=${SCW_REGION} -o json 2>/dev/null | grep -q '\"ci\"'
|
||||
"
|
||||
|
||||
check "GPU node pool exists" \
|
||||
bash -c "
|
||||
[[ -z '${CLUSTER_ID}' ]] && exit 1
|
||||
scw k8s pool list cluster-id=${CLUSTER_ID} region=${SCW_REGION} -o json 2>/dev/null | grep -q '\"gpu\"'
|
||||
"
|
||||
|
||||
# --- Summary --------------------------------------------------------------
|
||||
echo ""
|
||||
echo "======================================================================"
|
||||
printf " Results: %d passed, %d failed\n" "$PASS" "$FAIL"
|
||||
echo "======================================================================"
|
||||
|
||||
if [[ $FAIL -gt 0 ]]; then
|
||||
echo ""
|
||||
echo " Failed checks:"
|
||||
for f in "${FAILURES[@]}"; do
|
||||
echo " - ${f}"
|
||||
done
|
||||
echo ""
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo " All checks passed."
|
||||
exit 0
|
||||
238
infra/scripts/train.sh
Executable file
238
infra/scripts/train.sh
Executable file
@@ -0,0 +1,238 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# train.sh - Training orchestrator: submits K8s Jobs to the GPU pool
|
||||
# ---------------------------------------------------------------------------
|
||||
# Usage:
|
||||
# ./infra/scripts/train.sh <preset> [--model MODEL] [--symbol SYMBOL]
|
||||
# [--trials N] [--max-steps N] [--timeout N]
|
||||
#
|
||||
# Presets:
|
||||
# quick-test - fast sanity check (max-steps=500, requires --model)
|
||||
# single-model - full training run for one model (requires --model)
|
||||
# full-ensemble - trains all 4 models (dqn ppo tft mamba2) in parallel
|
||||
# hyperopt - hyperparameter optimisation (requires --model, uses --trials)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# --- Defaults -------------------------------------------------------------
|
||||
SYMBOL="ES.FUT"
|
||||
TRIALS=20
|
||||
MAX_STEPS=2000
|
||||
TIMEOUT=3600
|
||||
DATA_DIR="/data/futures-baseline"
|
||||
NAMESPACE="foxhunt"
|
||||
IMAGE="rg.nl-ams.scw.cloud/foxhunt/training:latest"
|
||||
MODEL=""
|
||||
PRESET=""
|
||||
TIMESTAMP="$(date +%s)"
|
||||
|
||||
ALL_MODELS=(dqn ppo tft mamba2)
|
||||
|
||||
# --- Model-to-binary mapping ----------------------------------------------
|
||||
declare -A MODEL_BINARY=(
|
||||
[dqn]=train_dqn
|
||||
[ppo]=train_ppo_parquet
|
||||
[tft]=hyperopt_tft_demo
|
||||
[mamba2]=hyperopt_mamba2_demo
|
||||
)
|
||||
|
||||
# --- Helpers --------------------------------------------------------------
|
||||
usage() {
|
||||
cat <<'USAGE'
|
||||
Usage: ./infra/scripts/train.sh <preset> [OPTIONS]
|
||||
|
||||
Presets:
|
||||
quick-test Fast sanity check (requires --model, max-steps=500)
|
||||
single-model Full training run (requires --model)
|
||||
full-ensemble Trains dqn, ppo, tft, mamba2 in parallel Jobs
|
||||
hyperopt Hyperparameter search (requires --model, uses --trials)
|
||||
|
||||
Options:
|
||||
--model MODEL Model name: dqn | ppo | tft | mamba2
|
||||
--symbol SYMBOL Trading symbol (default: ES.FUT)
|
||||
--trials N Hyperopt trial count (default: 20)
|
||||
--max-steps N Max steps per epoch (default: 2000)
|
||||
--timeout N Job timeout in seconds (default: 3600)
|
||||
-h, --help Show this help message
|
||||
USAGE
|
||||
exit 0
|
||||
}
|
||||
|
||||
die() { echo "ERROR: $*" >&2; exit 1; }
|
||||
|
||||
validate_model() {
|
||||
local m="$1"
|
||||
[[ -n "${MODEL_BINARY[$m]+x}" ]] || die "Unknown model '${m}'. Valid: ${!MODEL_BINARY[*]}"
|
||||
}
|
||||
|
||||
# --- Parse arguments -------------------------------------------------------
|
||||
[[ $# -eq 0 ]] && usage
|
||||
|
||||
PRESET="$1"; shift
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--model) MODEL="$2"; shift 2 ;;
|
||||
--symbol) SYMBOL="$2"; shift 2 ;;
|
||||
--trials) TRIALS="$2"; shift 2 ;;
|
||||
--max-steps) MAX_STEPS="$2"; shift 2 ;;
|
||||
--timeout) TIMEOUT="$2"; shift 2 ;;
|
||||
-h|--help) usage ;;
|
||||
*) die "Unknown option: $1" ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# --- Preset validation -----------------------------------------------------
|
||||
case "$PRESET" in
|
||||
quick-test)
|
||||
[[ -z "$MODEL" ]] && die "quick-test preset requires --model"
|
||||
validate_model "$MODEL"
|
||||
MAX_STEPS=500
|
||||
;;
|
||||
single-model)
|
||||
[[ -z "$MODEL" ]] && die "single-model preset requires --model"
|
||||
validate_model "$MODEL"
|
||||
;;
|
||||
full-ensemble)
|
||||
# Trains all models; --model is ignored if provided
|
||||
MODEL=""
|
||||
;;
|
||||
hyperopt)
|
||||
[[ -z "$MODEL" ]] && die "hyperopt preset requires --model"
|
||||
validate_model "$MODEL"
|
||||
;;
|
||||
*)
|
||||
die "Unknown preset '${PRESET}'. Valid: quick-test, single-model, full-ensemble, hyperopt"
|
||||
;;
|
||||
esac
|
||||
|
||||
# --- Build training arguments for a given model ----------------------------
|
||||
build_args() {
|
||||
local model="$1"
|
||||
local binary="${MODEL_BINARY[$model]}"
|
||||
local args=("$binary")
|
||||
|
||||
args+=("--symbol" "$SYMBOL")
|
||||
args+=("--max-steps-per-epoch" "$MAX_STEPS")
|
||||
|
||||
if [[ "$PRESET" == "hyperopt" ]]; then
|
||||
args+=("--trials" "$TRIALS")
|
||||
fi
|
||||
|
||||
echo "${args[*]}"
|
||||
}
|
||||
|
||||
# --- Generate K8s Job manifest --------------------------------------------
|
||||
generate_job_manifest() {
|
||||
local model="$1"
|
||||
local job_name="train-${model}-${TIMESTAMP}"
|
||||
local train_args
|
||||
train_args="$(build_args "$model")"
|
||||
|
||||
cat <<EOF
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: ${job_name}
|
||||
namespace: ${NAMESPACE}
|
||||
labels:
|
||||
foxhunt/job-type: training
|
||||
foxhunt/model: ${model}
|
||||
foxhunt/preset: ${PRESET}
|
||||
spec:
|
||||
activeDeadlineSeconds: ${TIMEOUT}
|
||||
backoffLimit: 0
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
foxhunt/job-type: training
|
||||
foxhunt/model: ${model}
|
||||
foxhunt/preset: ${PRESET}
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
nodeSelector:
|
||||
k8s.scaleway.com/pool-name: gpu
|
||||
tolerations:
|
||||
- key: nvidia.com/gpu
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
imagePullSecrets:
|
||||
- name: scw-registry
|
||||
containers:
|
||||
- name: training
|
||||
image: ${IMAGE}
|
||||
command: ["/bin/sh", "-c"]
|
||||
args:
|
||||
- "${train_args}"
|
||||
resources:
|
||||
requests:
|
||||
nvidia.com/gpu: "1"
|
||||
cpu: "4"
|
||||
memory: "16Gi"
|
||||
limits:
|
||||
nvidia.com/gpu: "1"
|
||||
cpu: "8"
|
||||
memory: "32Gi"
|
||||
volumeMounts:
|
||||
- name: training-data
|
||||
mountPath: /data
|
||||
readOnly: true
|
||||
- name: output
|
||||
mountPath: /output
|
||||
volumes:
|
||||
- name: training-data
|
||||
persistentVolumeClaim:
|
||||
claimName: training-data-pvc
|
||||
- name: output
|
||||
emptyDir: {}
|
||||
EOF
|
||||
}
|
||||
|
||||
# --- Submit a single job and collect its name ------------------------------
|
||||
submitted_jobs=()
|
||||
|
||||
submit_job() {
|
||||
local model="$1"
|
||||
local job_name="train-${model}-${TIMESTAMP}"
|
||||
|
||||
echo "--- Submitting job: ${job_name} (model=${model}, preset=${PRESET})"
|
||||
generate_job_manifest "$model" | kubectl apply -f -
|
||||
submitted_jobs+=("$job_name")
|
||||
}
|
||||
|
||||
# --- Main dispatch ---------------------------------------------------------
|
||||
echo "======================================================================"
|
||||
echo " Foxhunt Training Orchestrator"
|
||||
echo " Preset : ${PRESET}"
|
||||
echo " Symbol : ${SYMBOL}"
|
||||
echo " Image : ${IMAGE}"
|
||||
echo " NS : ${NAMESPACE}"
|
||||
echo "======================================================================"
|
||||
echo ""
|
||||
|
||||
case "$PRESET" in
|
||||
quick-test|single-model|hyperopt)
|
||||
submit_job "$MODEL"
|
||||
;;
|
||||
full-ensemble)
|
||||
for m in "${ALL_MODELS[@]}"; do
|
||||
submit_job "$m"
|
||||
done
|
||||
;;
|
||||
esac
|
||||
|
||||
echo ""
|
||||
echo "======================================================================"
|
||||
echo " All jobs submitted."
|
||||
echo "======================================================================"
|
||||
echo ""
|
||||
echo "Monitor with:"
|
||||
echo ""
|
||||
for jn in "${submitted_jobs[@]}"; do
|
||||
echo " kubectl -n ${NAMESPACE} get job ${jn}"
|
||||
echo " kubectl -n ${NAMESPACE} logs job/${jn} -f"
|
||||
echo ""
|
||||
done
|
||||
echo " kubectl -n ${NAMESPACE} get jobs -l foxhunt/preset=${PRESET}"
|
||||
echo " kubectl -n ${NAMESPACE} get pods -l foxhunt/job-type=training --watch"
|
||||
Reference in New Issue
Block a user