foxhunt/infra/scripts/train.sh

#!/usr/bin/env bash
set -euo pipefail

# ---------------------------------------------------------------------------
# train.sh - Training orchestrator: submits K8s Jobs to the GPU pool
# ---------------------------------------------------------------------------
# Usage:
#   ./infra/scripts/train.sh <preset> [--model MODEL] [--symbol SYMBOL]
#                                      [--trials N] [--max-steps N] [--timeout N]
#                                      [--commit SHA] [--poll-interval N]
#
# Presets:
#   quick-test    - fast sanity check (max-steps=500, requires --model)
#   single-model  - full training run for one model (requires --model)
#   full-ensemble - trains all 10 models sequentially (dqn ppo tft mamba2 tggn tlob liquid kan xlstm diffusion)
#   hyperopt      - hyperparameter optimisation (requires --model, uses --trials)
#   ci-train      - wait for Argo CI to pass, then auto-submit training (requires --model)
# ---------------------------------------------------------------------------

# --- Defaults -------------------------------------------------------------
SYMBOL="ES.FUT"
TRIALS=20
MAX_STEPS=2000
TIMEOUT=3600  # default; hyperopt preset overrides to 6h below
DATA_DIR="/data/futures-baseline"
NAMESPACE="foxhunt"
IMAGE="gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-training-runtime:latest"
MODEL=""
PRESET=""
COMMIT=""
POLL_INTERVAL=30
TRAIN_PRESET="hyperopt"
TIMESTAMP="$(date +%s)"
S3_BUCKET="foxhunt-models"
MINIO_ENDPOINT="http://minio.foxhunt.svc.cluster.local:9000"
RUN_ID="$(date +%Y%m%d-%H%M%S)"

ALL_MODELS=(dqn ppo tft mamba2 tggn tlob liquid kan xlstm diffusion)

# --- Model-to-binary mapping ----------------------------------------------
declare -A MODEL_BINARY=(
  [dqn]=train_baseline_rl
  [ppo]=train_baseline_rl
  [tft]=train_baseline_supervised
  [mamba2]=train_baseline_supervised
  [tggn]=train_baseline_supervised
  [tlob]=train_baseline_supervised
  [liquid]=train_baseline_supervised
  [kan]=train_baseline_supervised
  [xlstm]=train_baseline_supervised
  [diffusion]=train_baseline_supervised
)

# Hyperopt uses separate binaries
declare -A HYPEROPT_BINARY=(
  [dqn]=hyperopt_baseline_rl
  [ppo]=hyperopt_baseline_rl
  [tft]=hyperopt_baseline_supervised
  [mamba2]=hyperopt_baseline_supervised
  [tggn]=hyperopt_baseline_supervised
  [tlob]=hyperopt_baseline_supervised
  [liquid]=hyperopt_baseline_supervised
  [kan]=hyperopt_baseline_supervised
  [xlstm]=hyperopt_baseline_supervised
  [diffusion]=hyperopt_baseline_supervised
)

EVAL_BINARY="evaluate_baseline"

# --- Helpers --------------------------------------------------------------
usage() {
  cat <<'USAGE'
Usage: ./infra/scripts/train.sh <preset> [OPTIONS]

Presets:
  quick-test      Fast sanity check (requires --model, max-steps=500)
  single-model    Full training run (requires --model)
  full-ensemble   Trains all 10 models sequentially (dqn ppo tft mamba2 tggn tlob liquid kan xlstm diffusion)
  hyperopt        Hyperparameter search (requires --model, uses --trials)
  evaluate        Evaluate trained models (requires --run-id from prior training run)
  ci-train        Wait for Argo CI pipeline, then auto-submit training (requires --model)

Options:
  --model MODEL       Model name: dqn | ppo | tft | mamba2 | tggn | tlob | liquid | kan | xlstm | diffusion
  --symbol SYMBOL     Trading symbol            (default: ES.FUT)
  --trials N          Hyperopt trial count      (default: 20)
  --max-steps N       Max steps per epoch       (default: 2000)
  --timeout N         Job timeout in seconds    (default: 3600)
  --commit SHA        Git commit to track CI for (default: HEAD of main)
  --poll-interval N   CI poll interval seconds  (default: 30)
  --train-preset P    Training preset after CI  (default: hyperopt; for ci-train)
  -h, --help          Show this help message
USAGE
  exit 0
}

die() { echo "ERROR: $*" >&2; exit 1; }

validate_model() {
  local m="$1"
  [[ -n "${MODEL_BINARY[$m]+x}" ]] || die "Unknown model '${m}'. Valid: ${!MODEL_BINARY[*]}"
}

# --- Parse arguments -------------------------------------------------------
[[ $# -eq 0 ]] && usage
[[ "$1" == "-h" || "$1" == "--help" ]] && usage

PRESET="$1"; shift

while [[ $# -gt 0 ]]; do
  case "$1" in
    --model)    MODEL="$2";     shift 2 ;;
    --symbol)   SYMBOL="$2";    shift 2 ;;
    --trials)   TRIALS="$2";    shift 2 ;;
    --max-steps) MAX_STEPS="$2"; shift 2 ;;
    --timeout)  TIMEOUT="$2";   shift 2 ;;
    --run-id)   RUN_ID="$2";    shift 2 ;;
    --commit)   COMMIT="$2";   shift 2 ;;
    --poll-interval) POLL_INTERVAL="$2"; shift 2 ;;
    --train-preset)  TRAIN_PRESET="$2";  shift 2 ;;
    -h|--help)  usage ;;
    *) die "Unknown option: $1" ;;
  esac
done

# --- Preset validation -----------------------------------------------------
case "$PRESET" in
  quick-test)
    [[ -z "$MODEL" ]] && die "quick-test preset requires --model"
    validate_model "$MODEL"
    MAX_STEPS=500
    ;;
  single-model)
    [[ -z "$MODEL" ]] && die "single-model preset requires --model"
    validate_model "$MODEL"
    ;;
  full-ensemble)
    # Trains all models; --model is ignored if provided
    MODEL=""
    ;;
  hyperopt)
    [[ -z "$MODEL" ]] && die "hyperopt preset requires --model"
    validate_model "$MODEL"
    # Hyperopt runs 20 trials × 8 epochs — needs 6h unless user specified --timeout
    if [[ "$TIMEOUT" -eq 3600 ]]; then
      TIMEOUT=21600
    fi
    ;;
  evaluate)
    [[ -z "$RUN_ID" ]] && die "evaluate preset requires --run-id (from a previous training run)"
    ;;
  ci-train)
    [[ -z "$MODEL" ]] && die "ci-train preset requires --model"
    validate_model "$MODEL"
    # Resolve commit SHA if not provided
    if [[ -z "$COMMIT" ]]; then
      COMMIT="$(git rev-parse HEAD 2>/dev/null || echo "")"
      [[ -z "$COMMIT" ]] && die "Could not determine commit SHA. Use --commit SHA"
    fi
    ;;
  *)
    die "Unknown preset '${PRESET}'. Valid: quick-test, single-model, full-ensemble, hyperopt, evaluate, ci-train"
    ;;
esac


# --- CI Monitoring --------------------------------------------------------
# Find the Argo Workflow for a given commit SHA.
# Argo CI workflows are submitted with parameter commit-sha=<sha>.
# We search for workflows matching the ci-pipeline template.
find_ci_workflow() {
  local sha="$1"
  local short_sha="${sha:0:8}"
  # List recent ci-pipeline workflows and find the one matching our commit.
  # Argo Events labels workflows with events.argoproj.io/trigger (not
  # workflows.argoproj.io/workflow-template), so use that selector.
  kubectl -n "${NAMESPACE}" get workflows \
    -l "events.argoproj.io/trigger=ci-pipeline" \
    -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.phase}{"\t"}{range .spec.arguments.parameters[*]}{.name}={.value}{","}{end}{"\n"}{end}' \
    2>/dev/null \
    | grep "commit-sha=${sha}\|commit-sha=${short_sha}" \
    | head -1 \
    | cut -f1
}

# Get workflow phase: Pending, Running, Succeeded, Failed, Error
get_workflow_phase() {
  local wf_name="$1"
  kubectl -n "${NAMESPACE}" get workflow "${wf_name}" \
    -o jsonpath='{.status.phase}' 2>/dev/null
}

# Get workflow node statuses (one-liner summary)
get_workflow_summary() {
  local wf_name="$1"
  kubectl -n "${NAMESPACE}" get workflow "${wf_name}" \
    -o jsonpath='{range .status.nodes[*]}{.displayName}: {.phase}  {end}' 2>/dev/null
}

# Get workflow start time and duration
get_workflow_timing() {
  local wf_name="$1"
  local started finished
  started=$(kubectl -n "${NAMESPACE}" get workflow "${wf_name}" \
    -o jsonpath='{.status.startedAt}' 2>/dev/null)
  finished=$(kubectl -n "${NAMESPACE}" get workflow "${wf_name}" \
    -o jsonpath='{.status.finishedAt}' 2>/dev/null)
  if [[ -n "$finished" && "$finished" != "null" ]]; then
    echo "started=${started} finished=${finished}"
  elif [[ -n "$started" ]]; then
    echo "started=${started} (still running)"
  else
    echo "(not started yet)"
  fi
}

# Print Prometheus / log links for CI monitoring
print_monitor_links() {
  local wf_name="$1"
  echo ""
  echo "  Argo UI:    kubectl -n ${NAMESPACE} get workflow ${wf_name} -o yaml"
  echo "  Logs:       kubectl -n ${NAMESPACE} logs -l workflows.argoproj.io/workflow=${wf_name} --prefix -f"
  echo "  Prometheus: http://prometheus-stack-kube-prom-prometheus.foxhunt.svc.cluster.local:9090/graph?g0.expr=argo_workflow_status_phase"
  echo ""
}

# Wait for CI workflow to complete. Returns 0 on success, 1 on failure.
wait_for_ci() {
  local sha="$1"
  local short_sha="${sha:0:8}"
  local wf_name=""
  local phase=""
  local attempt=0
  local max_wait_for_wf=60  # max attempts to find the workflow (60 * poll = 30 min)

  echo "======================================================================"
  echo "  Waiting for Argo CI pipeline to complete"
  echo "  Commit : ${sha} (${short_sha})"
  echo "  Poll   : every ${POLL_INTERVAL}s"
  echo "======================================================================"
  echo ""

  # Phase 1: Find the workflow (it may not exist yet if webhook is delayed)
  echo "--- Looking for CI workflow for commit ${short_sha}..."
  while [[ -z "$wf_name" ]]; do
    wf_name=$(find_ci_workflow "$sha")
    if [[ -n "$wf_name" ]]; then
      echo "  Found workflow: ${wf_name}"
      print_monitor_links "$wf_name"
      break
    fi
    attempt=$((attempt + 1))
    if [[ $attempt -ge $max_wait_for_wf ]]; then
      echo "ERROR: No CI workflow found for commit ${short_sha} after $((attempt * POLL_INTERVAL))s"
      echo "  Verify the GitLab webhook triggered. Check:"
      echo "    kubectl -n ${NAMESPACE} get workflows -l workflows.argoproj.io/workflow-template=ci-pipeline"
      return 1
    fi
    echo "  Workflow not found yet (attempt ${attempt}/${max_wait_for_wf}), retrying in ${POLL_INTERVAL}s..."
    sleep "${POLL_INTERVAL}"
  done

  # Phase 2: Poll until terminal phase
  echo "--- Monitoring workflow ${wf_name}..."
  while true; do
    phase=$(get_workflow_phase "$wf_name")
    local timing
    timing=$(get_workflow_timing "$wf_name")

    case "$phase" in
      Succeeded)
        echo ""
        echo "  CI PASSED  (${timing})"
        echo ""
        return 0
        ;;
      Failed|Error)
        echo ""
        echo "  CI FAILED  phase=${phase} (${timing})"
        echo ""
        echo "  Workflow summary:"
        get_workflow_summary "$wf_name" | tr '  ' '\n' | sed 's/^/    /'
        echo ""
        echo "  Logs: kubectl -n ${NAMESPACE} logs -l workflows.argoproj.io/workflow=${wf_name} --prefix"
        return 1
        ;;
      Running|Pending|"")
        local step_info
        step_info=$(get_workflow_summary "$wf_name" 2>/dev/null | tr '  ' '\n' | grep -c "Running" || echo "0")
        printf "  [%s] phase=%-8s  running_steps=%s  %s\n" \
          "$(date +%H:%M:%S)" "${phase:-Pending}" "${step_info}" "${timing}"
        ;;
      *)
        echo "  [$(date +%H:%M:%S)] phase=${phase} (unexpected)"
        ;;
    esac

    sleep "${POLL_INTERVAL}"
  done
}

# --- Generate K8s Job manifest --------------------------------------------
generate_job_manifest() {
  local model="$1"
  local job_name="train-${model}-${TIMESTAMP}"
  local binary
  if [[ "$PRESET" == "hyperopt" ]]; then
    binary="${HYPEROPT_BINARY[$model]}"
  else
    binary="${MODEL_BINARY[$model]}"
  fi

  # Build CLI args (excluding binary name)
  local cli_args=("--model" "$model" "--symbol" "$SYMBOL" "--data-dir" "/data/futures-baseline")
  cli_args+=("--mbp10-data-dir" "/data/futures-baseline-mbp10")
  cli_args+=("--trades-data-dir" "/data/futures-baseline-trades")
  if [[ "$PRESET" == "hyperopt" ]]; then
    # hyperopt binary: --output is a JSON file path, --trials count
    cli_args+=("--output" "/output/hyperopt_results.json")
    cli_args+=("--trials" "$TRIALS")
  else
    # training binary: --output-dir is a directory for checkpoints
    cli_args+=("--output-dir" "/output")
  fi

  # Convert args array to YAML list items
  local args_yaml=""
  for arg in "${cli_args[@]}"; do
    args_yaml="${args_yaml}
            - \"${arg}\""
  done

  cat <<EOF
apiVersion: batch/v1
kind: Job
metadata:
  name: ${job_name}
  namespace: ${NAMESPACE}
  labels:
    foxhunt/job-type: training
    foxhunt/model: ${model}
    foxhunt/preset: ${PRESET}
    foxhunt/run-id: "${RUN_ID}"
spec:
  activeDeadlineSeconds: ${TIMEOUT}
  backoffLimit: 1
  ttlSecondsAfterFinished: 600
  template:
    metadata:
      annotations:
        prometheus.io/scrape: "true"
        prometheus.io/port: "9094"
        prometheus.io/path: "/metrics"
      labels:
        app.kubernetes.io/part-of: foxhunt
        app.kubernetes.io/component: training-workflow
        foxhunt/job-type: training
        foxhunt/model: ${model}
        foxhunt/preset: ${PRESET}
    spec:
      restartPolicy: Never
      nodeSelector:
        k8s.scaleway.com/pool-name: ci-training-h100
      tolerations:
        - key: nvidia.com/gpu
          operator: Exists
          effect: NoSchedule
        - key: node.cilium.io/agent-not-ready
          operator: Exists
          effect: NoSchedule
      imagePullSecrets:
        - name: gitlab-registry
      initContainers:
        - name: fetch-binary
          image: ${IMAGE}
          command: ["/bin/sh", "-c"]
          args:
            - |
              set -e
              BINARY="${binary}"
              curl -fSL -o "/binaries/\${BINARY}" \\
                --header "DEPLOY-TOKEN: \${GITLAB_DEPLOY_TOKEN}" \\
                "\${GITLAB_API}/projects/1/packages/generic/foxhunt-training/\${FOXHUNT_RELEASE}/\${BINARY}"
              chmod +x "/binaries/\${BINARY}"
              echo "Fetched \${BINARY} \${FOXHUNT_RELEASE} (\$(stat -c%s /binaries/\${BINARY}) bytes)"
          env:
            - name: GITLAB_DEPLOY_TOKEN
              valueFrom:
                secretKeyRef:
                  name: gitlab-deploy-token
                  key: token
            - name: GITLAB_API
              value: "http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181/api/v4"
            - name: FOXHUNT_RELEASE
              value: "latest"
          volumeMounts:
            - name: binaries
              mountPath: /binaries
          resources:
            requests:
              cpu: 100m
              memory: 64Mi
            limits:
              cpu: 500m
              memory: 128Mi
        - name: sync-training-data
          image: ${IMAGE}
          command: ["/bin/sh", "-c"]
          args:
            - |
              set -e
              RCLONE_FLAGS="--s3-provider=Minio --s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 --s3-access-key-id=\${MINIO_ACCESS_KEY} --s3-secret-access-key=\${MINIO_SECRET_KEY} --s3-no-check-bucket"
              echo "Syncing OHLCV data..."
              rclone sync ":s3:foxhunt-training-data/futures-baseline" /data/futures-baseline \$RCLONE_FLAGS --stats-one-line -v
              echo "Syncing MBP-10 data..."
              rclone sync ":s3:foxhunt-training-data/futures-baseline-mbp10" /data/futures-baseline-mbp10 \$RCLONE_FLAGS --stats-one-line -v
              echo "Syncing trades data..."
              rclone sync ":s3:foxhunt-training-data/futures-baseline-trades" /data/futures-baseline-trades \$RCLONE_FLAGS --stats-one-line -v
              echo "Data sync complete:"
              du -sh /data/futures-baseline/ /data/futures-baseline-mbp10/ /data/futures-baseline-trades/ 2>/dev/null || true
          env:
            - name: MINIO_ACCESS_KEY
              valueFrom:
                secretKeyRef:
                  name: minio-credentials
                  key: access-key
            - name: MINIO_SECRET_KEY
              valueFrom:
                secretKeyRef:
                  name: minio-credentials
                  key: secret-key
          volumeMounts:
            - name: training-data
              mountPath: /data
          resources:
            requests:
              cpu: "1"
              memory: 512Mi
            limits:
              cpu: "2"
              memory: 1Gi
      containers:
        - name: training
          image: ${IMAGE}
          command: ["/binaries/${binary}"]
          args:${args_yaml}
          env:
            - name: RUST_LOG
              value: info
            - name: SQLX_OFFLINE
              value: "true"
            - name: OTEL_EXPORTER_OTLP_ENDPOINT
              value: "http://tempo.foxhunt.svc.cluster.local:4317"
          resources:
            requests:
              nvidia.com/gpu: "1"
              cpu: "4"
              memory: "16Gi"
            limits:
              nvidia.com/gpu: "1"
              cpu: "8"
              memory: "32Gi"
          volumeMounts:
            - name: training-data
              mountPath: /data
              readOnly: true
            - name: output
              mountPath: /output
            - name: binaries
              mountPath: /binaries
              readOnly: true
      volumes:
        - name: training-data
          persistentVolumeClaim:
            claimName: training-data-pvc
        - name: output
          emptyDir:
            sizeLimit: 2Gi
        - name: binaries
          emptyDir:
            sizeLimit: 500Mi
EOF
}

generate_eval_manifest() {
  local job_name="eval-${RUN_ID}-${TIMESTAMP}"

  cat <<EOF
apiVersion: batch/v1
kind: Job
metadata:
  name: ${job_name}
  namespace: ${NAMESPACE}
  labels:
    foxhunt/job-type: evaluate
    foxhunt/run-id: "${RUN_ID}"
spec:
  activeDeadlineSeconds: ${TIMEOUT}
  backoffLimit: 1
  ttlSecondsAfterFinished: 600
  template:
    metadata:
      labels:
        app.kubernetes.io/part-of: foxhunt
        app.kubernetes.io/component: training-workflow
        foxhunt/job-type: evaluate
      annotations:
        prometheus.io/scrape: "true"
        prometheus.io/port: "9094"
        prometheus.io/path: "/metrics"
    spec:
      restartPolicy: Never
      nodeSelector:
        k8s.scaleway.com/pool-name: ci-training-h100
      tolerations:
        - key: nvidia.com/gpu
          operator: Exists
          effect: NoSchedule
        - key: node.cilium.io/agent-not-ready
          operator: Exists
          effect: NoSchedule
      imagePullSecrets:
        - name: gitlab-registry
      initContainers:
        - name: fetch-binary
          image: ${IMAGE}
          command: ["/bin/sh", "-c"]
          args:
            - |
              set -e
              BINARY="${EVAL_BINARY}"
              curl -fSL -o "/binaries/\${BINARY}" \\
                --header "DEPLOY-TOKEN: \${GITLAB_DEPLOY_TOKEN}" \\
                "\${GITLAB_API}/projects/1/packages/generic/foxhunt-training/\${FOXHUNT_RELEASE}/\${BINARY}"
              chmod +x "/binaries/\${BINARY}"
              echo "Fetched \${BINARY} \${FOXHUNT_RELEASE} (\$(stat -c%s /binaries/\${BINARY}) bytes)"
          env:
            - name: GITLAB_DEPLOY_TOKEN
              valueFrom:
                secretKeyRef:
                  name: gitlab-deploy-token
                  key: token
            - name: GITLAB_API
              value: "http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181/api/v4"
            - name: FOXHUNT_RELEASE
              value: "latest"
          volumeMounts:
            - name: binaries
              mountPath: /binaries
          resources:
            requests:
              cpu: 100m
              memory: 64Mi
            limits:
              cpu: 500m
              memory: 128Mi
        - name: sync-training-data
          image: ${IMAGE}
          command: ["/bin/sh", "-c"]
          args:
            - |
              set -e
              RCLONE_FLAGS="--s3-provider=Minio --s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 --s3-access-key-id=\${MINIO_ACCESS_KEY} --s3-secret-access-key=\${MINIO_SECRET_KEY} --s3-no-check-bucket"
              echo "Syncing OHLCV data..."
              rclone sync ":s3:foxhunt-training-data/futures-baseline" /data/futures-baseline \$RCLONE_FLAGS --stats-one-line -v
              echo "Syncing MBP-10 data..."
              rclone sync ":s3:foxhunt-training-data/futures-baseline-mbp10" /data/futures-baseline-mbp10 \$RCLONE_FLAGS --stats-one-line -v
              echo "Syncing trades data..."
              rclone sync ":s3:foxhunt-training-data/futures-baseline-trades" /data/futures-baseline-trades \$RCLONE_FLAGS --stats-one-line -v
              echo "Data sync complete:"
              du -sh /data/futures-baseline/ /data/futures-baseline-mbp10/ /data/futures-baseline-trades/ 2>/dev/null || true
          env:
            - name: MINIO_ACCESS_KEY
              valueFrom:
                secretKeyRef:
                  name: minio-credentials
                  key: access-key
            - name: MINIO_SECRET_KEY
              valueFrom:
                secretKeyRef:
                  name: minio-credentials
                  key: secret-key
          volumeMounts:
            - name: training-data
              mountPath: /data
          resources:
            requests:
              cpu: "1"
              memory: 512Mi
            limits:
              cpu: "2"
              memory: 1Gi
      containers:
        - name: evaluate
          image: ${IMAGE}
          command: ["/binaries/${EVAL_BINARY}"]
          args:
            - "--model=both"
            - "--data-dir=/data/futures-baseline"
            - "--models-dir=/output/models"
          env:
            - name: RUST_LOG
              value: info
            - name: SQLX_OFFLINE
              value: "true"
          resources:
            requests:
              nvidia.com/gpu: "1"
              cpu: "4"
              memory: "16Gi"
            limits:
              nvidia.com/gpu: "1"
              cpu: "8"
              memory: "32Gi"
          volumeMounts:
            - name: training-data
              mountPath: /data
              readOnly: true
            - name: output
              mountPath: /output
            - name: binaries
              mountPath: /binaries
              readOnly: true
      volumes:
        - name: training-data
          persistentVolumeClaim:
            claimName: training-data-pvc
        - name: output
          emptyDir:
            sizeLimit: 2Gi
        - name: binaries
          emptyDir:
            sizeLimit: 500Mi
EOF
}

# --- Submit a single job and collect its name ------------------------------
submitted_jobs=()

submit_job() {
  local model="$1"
  local job_name="train-${model}-${TIMESTAMP}"

  echo "--- Submitting job: ${job_name} (model=${model}, preset=${PRESET})"
  generate_job_manifest "$model" | kubectl apply -f -
  submitted_jobs+=("$job_name")
}

# --- Main dispatch ---------------------------------------------------------
echo "======================================================================"
echo "  Foxhunt Training Orchestrator"
echo "  Preset : ${PRESET}"
if [[ "$PRESET" == "ci-train" ]]; then
echo "  Commit : ${COMMIT:0:12}"
echo "  After  : ${TRAIN_PRESET} --model ${MODEL}"
echo "  Poll   : every ${POLL_INTERVAL}s"
fi
echo "  Symbol : ${SYMBOL}"
echo "  Image  : ${IMAGE}"
echo "  NS     : ${NAMESPACE}"
echo "  Run ID : ${RUN_ID}"
echo "  MinIO  : minio://${S3_BUCKET}/runs/${RUN_ID}/"
echo "======================================================================"
echo ""

case "$PRESET" in
  quick-test|single-model|hyperopt)
    submit_job "$MODEL"
    ;;
  full-ensemble)
    for m in "${ALL_MODELS[@]}"; do
      submit_job "$m"
      echo "  Waiting for ${m} to complete before next model..."
      kubectl -n "${NAMESPACE}" wait --for=condition=complete "job/train-${m}-${TIMESTAMP}" --timeout="${TIMEOUT}s" 2>/dev/null || true
    done
    ;;
  evaluate)
    echo "--- Submitting evaluation job for run ${RUN_ID}"
    generate_eval_manifest | kubectl apply -f -
    submitted_jobs+=("eval-${RUN_ID}-${TIMESTAMP}")
    ;;
  ci-train)
    echo "--- CI-Train: monitoring CI for commit ${COMMIT:0:8}, then auto-submitting ${TRAIN_PRESET} job"
    echo ""
    if wait_for_ci "$COMMIT"; then
      echo "======================================================================"
      echo "  CI passed — submitting ${TRAIN_PRESET} training for model=${MODEL}"
      echo "======================================================================"
      echo ""
      # Switch to the training preset for job generation
      PRESET="$TRAIN_PRESET"
      # Apply hyperopt timeout if needed
      if [[ "$TRAIN_PRESET" == "hyperopt" && "$TIMEOUT" -eq 3600 ]]; then
        TIMEOUT=21600
      fi
      submit_job "$MODEL"
    else
      echo "======================================================================"
      echo "  CI failed — NOT submitting training job"
      echo "======================================================================"
      exit 1
    fi
    ;;
esac

echo ""
echo "======================================================================"
echo "  All jobs submitted."
echo "======================================================================"
echo ""
echo "Monitor with:"
echo ""
for jn in "${submitted_jobs[@]}"; do
  echo "  kubectl -n ${NAMESPACE} get job ${jn}"
  echo "  kubectl -n ${NAMESPACE} logs job/${jn} -f"
  echo ""
done
echo "  kubectl -n ${NAMESPACE} get jobs -l foxhunt/preset=${PRESET}"
echo "  kubectl -n ${NAMESPACE} get pods -l foxhunt/job-type=training --watch"