diff --git a/infra/scripts/smoke-test.sh b/infra/scripts/smoke-test.sh new file mode 100755 index 000000000..a638d6446 --- /dev/null +++ b/infra/scripts/smoke-test.sh @@ -0,0 +1,147 @@ +#!/usr/bin/env bash +set -euo pipefail + +# --------------------------------------------------------------------------- +# smoke-test.sh - End-to-end smoke test for the Foxhunt Kapsule cluster +# --------------------------------------------------------------------------- +# Runs a series of checks and prints OK/FAIL for each. +# Exits 0 if everything passes, 1 with failure count otherwise. +# --------------------------------------------------------------------------- + +NAMESPACE="foxhunt" +TAILSCALE_NS="tailscale" +KAPSULE_NODE_NAME="foxhunt-kapsule" +CLUSTER_ID="${SCW_KAPSULE_CLUSTER_ID:-}" +SCW_REGION="${SCW_REGION:-nl-ams}" + +GRPC_SERVICES=( + trading-engine + trading-agent + ml-training + risk-manager + data-acquisition + broker-gateway +) + +PASS=0 +FAIL=0 +FAILURES=() + +# --- Helpers -------------------------------------------------------------- + +check() { + local description="$1" + shift + if "$@" >/dev/null 2>&1; then + printf " [OK] %s\n" "$description" + (( PASS++ )) + else + printf " [FAIL] %s\n" "$description" + (( FAIL++ )) + FAILURES+=("$description") + fi +} + +section() { + echo "" + echo "=== $1 ===" +} + +# --- 1. Cluster ----------------------------------------------------------- +section "Cluster" + +check "kubectl can reach nodes" \ + kubectl get nodes --no-headers + +check "Namespace '${NAMESPACE}' exists" \ + kubectl get namespace "$NAMESPACE" + +# --- 2. Tailscale --------------------------------------------------------- +section "Tailscale" + +check "Tailscale pod running in '${TAILSCALE_NS}' namespace" \ + bash -c "kubectl -n ${TAILSCALE_NS} get pods --field-selector=status.phase=Running --no-headers 2>/dev/null | grep -q ." + +check "Tailscale status shows '${KAPSULE_NODE_NAME}'" \ + bash -c "tailscale status 2>/dev/null | grep -q '${KAPSULE_NODE_NAME}'" + +# --- 3. Databases --------------------------------------------------------- +section "Databases" + +check "PostgreSQL accepts connections" \ + bash -c " + PG_POD=\$(kubectl -n ${NAMESPACE} get pod -l app=postgresql -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) + kubectl -n ${NAMESPACE} exec \"\$PG_POD\" -- pg_isready -U foxhunt + " + +check "Redis responds to PING" \ + bash -c " + REDIS_POD=\$(kubectl -n ${NAMESPACE} get pod -l app=redis -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) + kubectl -n ${NAMESPACE} exec \"\$REDIS_POD\" -- redis-cli ping | grep -q PONG + " + +check "QuestDB HTTP health" \ + bash -c " + QUEST_POD=\$(kubectl -n ${NAMESPACE} get pod -l app=questdb -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) + kubectl -n ${NAMESPACE} exec \"\$QUEST_POD\" -- curl -sf http://localhost:9000/ + " + +# --- 4. Services ---------------------------------------------------------- +section "Services (gRPC + web-gateway)" + +for svc in "${GRPC_SERVICES[@]}"; do + check "Pod '${svc}' is Running" \ + bash -c "kubectl -n ${NAMESPACE} get pods -l app=${svc} --field-selector=status.phase=Running --no-headers 2>/dev/null | grep -q ." +done + +check "Pod 'web-gateway' is Running" \ + bash -c "kubectl -n ${NAMESPACE} get pods -l app=web-gateway --field-selector=status.phase=Running --no-headers 2>/dev/null | grep -q ." + +# --- 5. Web gateway health ------------------------------------------------ +section "Web Gateway Health" + +check "web-gateway /health returns 200" \ + bash -c " + CLUSTER_IP=\$(kubectl -n ${NAMESPACE} get svc web-gateway -o jsonpath='{.spec.clusterIP}' 2>/dev/null) + PORT=\$(kubectl -n ${NAMESPACE} get svc web-gateway -o jsonpath='{.spec.ports[0].port}' 2>/dev/null) + kubectl -n ${NAMESPACE} run smoke-curl --rm -i --restart=Never --image=curlimages/curl -- \ + curl -sf http://\${CLUSTER_IP}:\${PORT}/health + " + +# --- 6. Node pools ------------------------------------------------------- +section "Node Pools" + +check "Always-on node is Ready" \ + bash -c "kubectl get nodes -l k8s.scaleway.com/pool-name=always-on --no-headers 2>/dev/null | grep -q ' Ready'" + +check "CI node pool exists" \ + bash -c " + [[ -z '${CLUSTER_ID}' ]] && exit 1 + scw k8s pool list cluster-id=${CLUSTER_ID} region=${SCW_REGION} -o json 2>/dev/null | grep -q '\"ci\"' + " + +check "GPU node pool exists" \ + bash -c " + [[ -z '${CLUSTER_ID}' ]] && exit 1 + scw k8s pool list cluster-id=${CLUSTER_ID} region=${SCW_REGION} -o json 2>/dev/null | grep -q '\"gpu\"' + " + +# --- Summary -------------------------------------------------------------- +echo "" +echo "======================================================================" +printf " Results: %d passed, %d failed\n" "$PASS" "$FAIL" +echo "======================================================================" + +if [[ $FAIL -gt 0 ]]; then + echo "" + echo " Failed checks:" + for f in "${FAILURES[@]}"; do + echo " - ${f}" + done + echo "" + exit 1 +fi + +echo "" +echo " All checks passed." +exit 0 diff --git a/infra/scripts/train.sh b/infra/scripts/train.sh new file mode 100755 index 000000000..40c690816 --- /dev/null +++ b/infra/scripts/train.sh @@ -0,0 +1,238 @@ +#!/usr/bin/env bash +set -euo pipefail + +# --------------------------------------------------------------------------- +# train.sh - Training orchestrator: submits K8s Jobs to the GPU pool +# --------------------------------------------------------------------------- +# Usage: +# ./infra/scripts/train.sh [--model MODEL] [--symbol SYMBOL] +# [--trials N] [--max-steps N] [--timeout N] +# +# Presets: +# quick-test - fast sanity check (max-steps=500, requires --model) +# single-model - full training run for one model (requires --model) +# full-ensemble - trains all 4 models (dqn ppo tft mamba2) in parallel +# hyperopt - hyperparameter optimisation (requires --model, uses --trials) +# --------------------------------------------------------------------------- + +# --- Defaults ------------------------------------------------------------- +SYMBOL="ES.FUT" +TRIALS=20 +MAX_STEPS=2000 +TIMEOUT=3600 +DATA_DIR="/data/futures-baseline" +NAMESPACE="foxhunt" +IMAGE="rg.nl-ams.scw.cloud/foxhunt/training:latest" +MODEL="" +PRESET="" +TIMESTAMP="$(date +%s)" + +ALL_MODELS=(dqn ppo tft mamba2) + +# --- Model-to-binary mapping ---------------------------------------------- +declare -A MODEL_BINARY=( + [dqn]=train_dqn + [ppo]=train_ppo_parquet + [tft]=hyperopt_tft_demo + [mamba2]=hyperopt_mamba2_demo +) + +# --- Helpers -------------------------------------------------------------- +usage() { + cat <<'USAGE' +Usage: ./infra/scripts/train.sh [OPTIONS] + +Presets: + quick-test Fast sanity check (requires --model, max-steps=500) + single-model Full training run (requires --model) + full-ensemble Trains dqn, ppo, tft, mamba2 in parallel Jobs + hyperopt Hyperparameter search (requires --model, uses --trials) + +Options: + --model MODEL Model name: dqn | ppo | tft | mamba2 + --symbol SYMBOL Trading symbol (default: ES.FUT) + --trials N Hyperopt trial count (default: 20) + --max-steps N Max steps per epoch (default: 2000) + --timeout N Job timeout in seconds (default: 3600) + -h, --help Show this help message +USAGE + exit 0 +} + +die() { echo "ERROR: $*" >&2; exit 1; } + +validate_model() { + local m="$1" + [[ -n "${MODEL_BINARY[$m]+x}" ]] || die "Unknown model '${m}'. Valid: ${!MODEL_BINARY[*]}" +} + +# --- Parse arguments ------------------------------------------------------- +[[ $# -eq 0 ]] && usage + +PRESET="$1"; shift + +while [[ $# -gt 0 ]]; do + case "$1" in + --model) MODEL="$2"; shift 2 ;; + --symbol) SYMBOL="$2"; shift 2 ;; + --trials) TRIALS="$2"; shift 2 ;; + --max-steps) MAX_STEPS="$2"; shift 2 ;; + --timeout) TIMEOUT="$2"; shift 2 ;; + -h|--help) usage ;; + *) die "Unknown option: $1" ;; + esac +done + +# --- Preset validation ----------------------------------------------------- +case "$PRESET" in + quick-test) + [[ -z "$MODEL" ]] && die "quick-test preset requires --model" + validate_model "$MODEL" + MAX_STEPS=500 + ;; + single-model) + [[ -z "$MODEL" ]] && die "single-model preset requires --model" + validate_model "$MODEL" + ;; + full-ensemble) + # Trains all models; --model is ignored if provided + MODEL="" + ;; + hyperopt) + [[ -z "$MODEL" ]] && die "hyperopt preset requires --model" + validate_model "$MODEL" + ;; + *) + die "Unknown preset '${PRESET}'. Valid: quick-test, single-model, full-ensemble, hyperopt" + ;; +esac + +# --- Build training arguments for a given model ---------------------------- +build_args() { + local model="$1" + local binary="${MODEL_BINARY[$model]}" + local args=("$binary") + + args+=("--symbol" "$SYMBOL") + args+=("--max-steps-per-epoch" "$MAX_STEPS") + + if [[ "$PRESET" == "hyperopt" ]]; then + args+=("--trials" "$TRIALS") + fi + + echo "${args[*]}" +} + +# --- Generate K8s Job manifest -------------------------------------------- +generate_job_manifest() { + local model="$1" + local job_name="train-${model}-${TIMESTAMP}" + local train_args + train_args="$(build_args "$model")" + + cat <