foxhunt/scripts/argo-alpha-rl.sh

#!/usr/bin/env bash
# Submit the alpha-rl (integrated RL trainer) workflow.
#
# Drives `IntegratedTrainer::step_with_lobsim` (DQN + PPO on shared
# Mamba2 → CfC encoder, R7d off-policy replay) against MBP-10 from the
# training-data PVC, using LobSimCuda as the synthetic-book environment.
# Emits alpha_rl_train_summary.json to the feature-cache PVC.
#
# Guards (enforced by the rebuild plan A9 feedback list):
#   * `feedback_default_to_l40s_pool` — defaults to ci-training-l40s;
#     H100 (sm_90) is opt-in for scale-up sweeps only.
#   * `feedback_push_before_deploy` — verifies local HEAD == origin
#     HEAD before submission, refuses to submit otherwise.
#   * `feedback_argo_template_must_apply` — kubectl-applies the local
#     alpha-rl-template.yaml BEFORE `argo submit --from=wftmpl/alpha-rl`
#     (the submit reads the cluster CRD, NOT the on-disk YAML, so
#     unknown -p params silently no-op without the apply).
#
# Usage:
#   ./scripts/argo-alpha-rl.sh                                  # 1k step smoke on HEAD
#   ./scripts/argo-alpha-rl.sh --branch main
#   ./scripts/argo-alpha-rl.sh --n-steps 50000 --n-backtests 32 # production sweep
#   ./scripts/argo-alpha-rl.sh --gpu-pool ci-training-h100      # opt into H100
#   ./scripts/argo-alpha-rl.sh --watch
#
# Plan reference: docs/superpowers/plans/2026-05-23-integrated-rl-trainer-rebuild.md

set -euo pipefail

SHA=HEAD
BRANCH=$(git symbolic-ref --short HEAD 2>/dev/null || echo "ml-alpha-phase-a")
GPU_POOL=ci-training-l40s
N_STEPS=1000
SEQ_LEN=32
# Default 16 — fixes b_size=1 signal starvation per
# `pearl_b_size_1_signal_starvation_blocks_q_learning`. 16 parallel
# transitions per Adam step give Q proper gradient variance reduction.
# Override with `--n-backtests N` for sweeps.
N_BACKTESTS=16
PER_CAPACITY=4096
SEED=16962
INSTRUMENT_MODE=front-month
FOLD_IDX=0
N_FOLDS=1
N_EVAL_STEPS=0
WATCH=false
SKIP_PUSH_CHECK=false
SKIP_TEMPLATE_APPLY=false
USE_MULTI_HEAD_POLICY=0
BAND_ENABLED=0

usage() {
  cat <<EOF
Usage: $0 [OPTIONS]
  --sha <commit>            Git SHA (default: HEAD on the current branch)
  --branch <branch>         Git branch (default: $BRANCH)
  --gpu-pool <pool>         GPU pool (default: $GPU_POOL).
                            Per feedback_default_to_l40s_pool: L40S is
                            the default. Opt into ci-training-h100 only
                            for scale-up runs that need sm_90 / 80GB.
  --n-steps <n>             step_with_lobsim count (default: $N_STEPS).
                            R9 smoke = 1000; production = 50000+.
  --seq-len <n>             Encoder sequence length (default: $SEQ_LEN).
  --n-backtests <n>         Parallel LobSim instances per step
                            (default: $N_BACKTESTS).
  --per-capacity <n>        PER buffer capacity (default: $PER_CAPACITY).
  --seed <n>                Random seed (default: $SEED).
  --instrument-mode <m>     all | front-month | id=<N> (default: $INSTRUMENT_MODE)
  --fold-idx <k>            Walk-forward fold index (0-based) for the
                            multi-fold G8 gate (default: $FOLD_IDX). Slices
                            the MBP-10 file list into n_folds blocks; fold k
                            trains on blocks [0..=k] and evals on block [k+1].
                            Requires --n-folds ≥ 3 and --n-eval-steps > 0.
  --n-folds <K>             Total fold count (default: $N_FOLDS). K=1 disables
                            the eval split (single-window mode). K≥3 enables
                            walk-forward; submit K-1 workflows (one per
                            fold_idx) and aggregate the per-fold profit_factor
                            for the G8 gate per
                            \`pearl_single_window_oos_is_not_oos\`.
  --n-eval-steps <n>        Eval-phase step count (default: $N_EVAL_STEPS).
                            After the train phase, runs n_eval_steps additional
                            steps on the held-out fold. LobSim trade records
                            from the eval phase are isolated via a pre-eval
                            head checkpoint; \`eval_summary.json\` lands in
                            OUT_DIR alongside \`alpha_rl_train_summary.json\`.
  --watch                   Follow logs via argo watch
  --skip-push-check         Bypass the feedback_push_before_deploy guard
                            (rare; use only when submitting against a
                            previously-pushed SHA via --sha).
  --skip-template-apply     Bypass the feedback_argo_template_must_apply
                            guard (rare; only when you've manually
                            kubectl applied the template already).
  -h, --help                Show this help
EOF
}

while [[ $# -gt 0 ]]; do
  case "$1" in
    --sha)                 SHA="$2"; shift 2 ;;
    --branch)              BRANCH="$2"; shift 2 ;;
    --gpu-pool)            GPU_POOL="$2"; shift 2 ;;
    --n-steps)             N_STEPS="$2"; shift 2 ;;
    --seq-len)             SEQ_LEN="$2"; shift 2 ;;
    --n-backtests)         N_BACKTESTS="$2"; shift 2 ;;
    --per-capacity)        PER_CAPACITY="$2"; shift 2 ;;
    --seed)                SEED="$2"; shift 2 ;;
    --instrument-mode)     INSTRUMENT_MODE="$2"; shift 2 ;;
    --fold-idx)            FOLD_IDX="$2"; shift 2 ;;
    --n-folds)             N_FOLDS="$2"; shift 2 ;;
    --n-eval-steps)        N_EVAL_STEPS="$2"; shift 2 ;;
    --watch)               WATCH=true; shift ;;
    --skip-push-check)     SKIP_PUSH_CHECK=true; shift ;;
    --skip-template-apply) SKIP_TEMPLATE_APPLY=true; shift ;;
    --use-multi-head-policy) USE_MULTI_HEAD_POLICY="$2"; shift 2 ;;
    --band-enabled)        BAND_ENABLED="$2"; shift 2 ;;
    -h|--help)             usage; exit 0 ;;
    *) echo "Unknown option: $1"; usage; exit 1 ;;
  esac
done

# ── GPU pool → CUDA_COMPUTE_CAP ─────────────────────────────────────
# Per `feedback_default_to_l40s_pool` (2026-05-09): SP-chain training
# defaults to L40S (sm_89). H100 (sm_90) is opt-in for scale-up only.
# Cubins must match the device, so the dispatcher derives the cap from
# the pool name and threads it into the workflow params.
case "$GPU_POOL" in
  ci-training-l40s|ci-training)
    SM=89 ;;
  ci-training-h100)
    SM=90 ;;
  *)
    echo "ERROR: unknown gpu-pool: $GPU_POOL"
    echo "       Supported: ci-training-l40s, ci-training-h100"
    exit 1 ;;
esac

# ── Guard 1: feedback_argo_template_must_apply ───────────────────────
# `argo submit --from=wftmpl/<name>` reads the cluster CRD, NOT the
# on-disk YAML. If the template has been edited locally but never
# `kubectl apply -f`'d, the submit silently uses the stale cluster
# version + drops any newly-added parameters. Pearl ref: 2026-05-21
# canonical incident.
TEMPLATE_FILE="infra/k8s/argo/alpha-rl-template.yaml"
if [[ ! -f "$TEMPLATE_FILE" ]]; then
  echo "ERROR: template not found at $TEMPLATE_FILE (run from repo root)"
  exit 1
fi
if [[ "$SKIP_TEMPLATE_APPLY" == "false" ]]; then
  echo "Applying alpha-rl WorkflowTemplate to cluster..."
  kubectl apply -n foxhunt -f "$TEMPLATE_FILE" >/dev/null
  echo "  applied: $TEMPLATE_FILE"
else
  echo "WARNING: --skip-template-apply set — argo will use whatever's"
  echo "         already in the cluster CRD (may be stale)."
fi

# ── Guard 2: feedback_push_before_deploy ─────────────────────────────
# Per the 2026-05-20 incident: cluster pods pull source from origin
# `<BRANCH>`, NOT from the local working tree. Submitting before
# `git push` deploys whatever was last on origin, which can lag the
# local diff by N commits. Verify HEAD == origin/<branch> before
# submission unless explicitly bypassed.
if [[ "$SKIP_PUSH_CHECK" == "false" ]]; then
  echo "Verifying local HEAD == origin/$BRANCH (feedback_push_before_deploy)..."
  if ! git rev-parse --git-dir >/dev/null 2>&1; then
    echo "ERROR: not in a git repo; cannot verify push state"
    exit 1
  fi
  git fetch --quiet origin "$BRANCH" 2>/dev/null || true
  LOCAL_HEAD=$(git rev-parse HEAD)
  ORIGIN_HEAD=$(git rev-parse "origin/$BRANCH" 2>/dev/null || echo "MISSING")
  if [[ "$ORIGIN_HEAD" == "MISSING" ]]; then
    echo "ERROR: origin/$BRANCH does not exist — push the branch first:"
    echo "       git push -u origin $BRANCH"
    exit 1
  fi
  if [[ "$LOCAL_HEAD" != "$ORIGIN_HEAD" ]]; then
    echo "ERROR: local HEAD ($LOCAL_HEAD) != origin/$BRANCH ($ORIGIN_HEAD)"
    echo "       Push first:"
    echo "         git push origin $BRANCH"
    echo "       Or override with --skip-push-check if you're knowingly"
    echo "       deploying a previously-pushed SHA via --sha."
    exit 1
  fi
  echo "  HEAD verified: $LOCAL_HEAD == origin/$BRANCH"
fi

# Resolve SHA=HEAD to a concrete SHA before submission so the in-cluster
# check-cache pod doesn't need git access. Mirrors alpha-perception.
if [ "$SHA" = "HEAD" ]; then
  SHA=$(git rev-parse "origin/$BRANCH" 2>/dev/null \
        || git rev-parse "$BRANCH" 2>/dev/null \
        || git rev-parse HEAD)
  echo "Resolved HEAD → $SHA"
fi

echo ""
echo "Submitting alpha-rl workflow..."
echo "  branch:        $BRANCH"
echo "  sha:           $SHA"
echo "  gpu-pool:      $GPU_POOL (sm_$SM)"
echo "  n-steps:       $N_STEPS"
echo "  seq-len:       $SEQ_LEN"
echo "  n-backtests:   $N_BACKTESTS"
echo "  per-capacity:  $PER_CAPACITY"
echo "  seed:          $SEED"
echo "  instrument:    $INSTRUMENT_MODE"

WATCH_FLAG=""
if [[ "$WATCH" == "true" ]]; then
  WATCH_FLAG="--watch"
fi

argo submit -n foxhunt --from=wftmpl/alpha-rl \
  -p commit-sha="$SHA" \
  -p git-branch="$BRANCH" \
  -p cuda-compute-cap="$SM" \
  -p gpu-pool="$GPU_POOL" \
  -p n-steps="$N_STEPS" \
  -p seq-len="$SEQ_LEN" \
  -p n-backtests="$N_BACKTESTS" \
  -p per-capacity="$PER_CAPACITY" \
  -p seed="$SEED" \
  -p instrument-mode="$INSTRUMENT_MODE" \
  -p fold-idx="$FOLD_IDX" \
  -p n-folds="$N_FOLDS" \
  -p n-eval-steps="$N_EVAL_STEPS" \
  -p use-multi-head-policy="$USE_MULTI_HEAD_POLICY" \
  -p band-enabled="$BAND_ENABLED" \
  $WATCH_FLAG