From 90d14aaba5afdcf5e2271e0677cfa4c312948ff6 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sat, 16 May 2026 20:27:25 +0200 Subject: [PATCH] spec(ml-alpha): GPU/CPU contract, CUDA Graphs, cold-start, disconnect handling Critical-review pass on the CfC+PPO design adds: - Public API: mapped-pinned ingress slots, single DtoH per decision - Section 5.5: CUDA Graph capture topology (perception / policy / training), kernel inventory, cuBLAS Lt epilogue fusion, persistent ring layouts, determinism mode, build-time cubin compilation, perf budget - Section 5.6: Databento/IBKR disconnect + failure response, never-do list - Section 6 cold-start: 4-state bootstrap, Phase B walk-forward stats as pre-deployment kill-switch baseline, always-live ISV controllers Co-Authored-By: Claude Opus 4.7 --- .../2026-05-16-ml-alpha-cfc-ppo-design.md | 357 ++++++++++++++++-- 1 file changed, 323 insertions(+), 34 deletions(-) diff --git a/docs/superpowers/specs/2026-05-16-ml-alpha-cfc-ppo-design.md b/docs/superpowers/specs/2026-05-16-ml-alpha-cfc-ppo-design.md index 362212370..0d72546de 100644 --- a/docs/superpowers/specs/2026-05-16-ml-alpha-cfc-ppo-design.md +++ b/docs/superpowers/specs/2026-05-16-ml-alpha-cfc-ppo-design.md @@ -44,45 +44,102 @@ A snapshot-level CfC perception trunk (Closed-form Continuous-time recurrent net ### Public API of the model library ```rust -// crates/ml-alpha/src/lib.rs (public surface — total ~5 types, ~10 methods) +// crates/ml-alpha/src/lib.rs (public surface) -pub struct AlphaModel { /* CfC weights + heads + policy + EWC anchor + Adam state, all GPU-resident */ } +pub struct AlphaModel { + // CfC weights + heads + policy + EWC anchor + Adam state — all GPU-resident. + // Position state, PnL state, last-segment-open-mid — also GPU-resident, owned here. + // Pre-allocated mapped-pinned ingress slots for snapshot bytes and fill events. +} impl AlphaModel { pub fn load(weights_path: &Path) -> Result; pub fn save(&self, weights_path: &Path) -> Result<()>; // Hot path — called per snapshot during live. Target latency p99 ≤ 500μs. - pub fn perception_forward(&mut self, snapshot_pinned: &MappedPinnedSnapshot) -> AlphaProbsDevicePtr; + // The snapshot has already been written into AlphaModel's mapped-pinned ingress + // slot by the caller (zero HtoD). This call launches the captured CUDA Graph + // for: snap_feature_assemble → CfC step → multi-horizon heads → projection. + // No return value: outputs are on-device in ISV slots 0..13. The caller does + // NOT read those slots — only the policy_action and update paths consume them. + pub fn perception_forward(&mut self); - // Decision point — called every decision_stride snapshots. - pub fn policy_action(&self, state_gpu: &AlphaStateGpu) -> (Action, LogProb, Value); + // Decision point — called every decision_stride snapshots, also via captured + // CUDA Graph. Internally: + // 1. build_state kernel reads ISV slots 0..13 + position/PnL/time state + + // market-context features → writes 20-dim AlphaState into next replay + // buffer slot (no temporary tensor) + // 2. policy_forward kernel: state → CfC actor → 21 logits + log_softmax, + // CfC critic → value; logit-mask outside ±max_live; sample via cuRAND; + // write (action_idx, log_prob, value) into the same replay buffer slot + // 3. cudaMemcpyAsync the single action_idx byte to a mapped-pinned readback + // slot; cudaStreamSynchronize once at the end so the strategy can read + // the action enum on CPU. Total round-trip ≈ 50-100μs. + // Returns ONLY the action (an i8 enum). log_prob and value live in the buffer + // on-device — never seen by CPU. + pub fn policy_action(&mut self) -> Action; + + // Called from on_fill — the fill event has been written into the model's + // mapped-pinned fill-ingress slot. Updates position state, PnL state, then + // BACK-FILLS the reward into the prior replay buffer slot's reward field + // (kernel reads pinned fill, updates state, computes ΔPnL, writes reward). + // No CPU compute, no plain HtoD. + pub fn ingest_fill(&mut self); // Slow path — periodic online update; ~30s wall on L40S. pub fn update(&mut self, batch: &MiniBatch) -> UpdateStatus; + + // Mapped-pinned ingress accessors — strategy writes via volatile stores. + pub fn snap_pinned_slot_mut(&mut self) -> &mut MappedPinnedSnapshotSlot; + pub fn fill_pinned_slot_mut(&mut self) -> &mut MappedPinnedFillSlot; + + // Audit / observability — slow path, slow snapshot of ISV slots to a small + // mapped-pinned readback buffer. Called from metrics task on a timer, never + // from the hot path. The 1-shot DtoH copy is wrapped in mapped-pinned per + // feedback_no_htod_htoh_only_mapped_pinned.md (slow path is not exempt). + pub fn read_isv_snapshot(&self) -> IsvSnapshot; } -pub struct ReplayBuffer { /* offline + live trajectories, GPU-resident */ } +pub struct ReplayBuffer { /* offline + live trajectories, all GPU-resident */ } impl ReplayBuffer { - pub fn record_step(&mut self, transition: Transition); - pub fn complete_last_segment_reward(&mut self, reward: f32); + // No record_step / complete_last_segment_reward in the public API — + // those happen INSIDE AlphaModel kernels (policy_action writes the + // forward fields; ingest_fill writes the reward). The replay buffer + // is opaque from CPU. pub fn sample_mixed(&self, n: usize, offline_frac: f32) -> MiniBatch; pub fn load_offline_from(&mut self, path: &Path) -> Result<()>; } pub enum UpdateStatus { - Applied { metrics: TrainingMetrics }, + Applied { metrics: TrainingMetrics }, RolledBack { reason: String, deviating_metric: String, sigma: f32 }, } -pub struct AlphaState { /* 20-dim vector, constructed by trading_agent */ } - #[repr(i8)] pub enum Action { /* target_position ∈ {-10, ..., +10} */ } ``` -GPU constraint: weights, hidden states, replay buffer, Adam state, and EWC anchor all live on the GPU. CPU only touches: snapshot bytes from Databento socket (via mapped-pinned), the chosen Action enum (~1 byte), the UpdateStatus enum, and serialized weights at save/load. No CPU shadow buffers. No GPU→CPU roundtrips on the hot path. Mirrors the foxhunt rules: `feedback_cpu_is_read_only.md`, `feedback_no_htod_htoh_only_mapped_pinned.md`, `feedback_no_atomicadd.md`, `feedback_no_nvrtc.md`. +#### GPU/CPU contract (NON-NEGOTIABLE) + +| Quantity | Lives on | Movement | +|---|---|---| +| Model weights, anchor, Fisher, Adam state, hidden states | **GPU** | DtoD only; serialized to disk only at save/load (slow path) | +| Snapshot bytes from Databento socket | **CPU** → **mapped-pinned** → **GPU reads directly** | No plain HtoD anywhere | +| Fill events from broker | **CPU** → **mapped-pinned** → **GPU reads directly** | No plain HtoD anywhere | +| Position state, PnL state, time-since-entry, segment-open-mid | **GPU** | Updated by GPU kernels triggered by snapshot or fill ingress | +| AlphaState (20-dim) | **GPU** | Assembled by `build_state` kernel directly into replay slot | +| log_prob, value, reward in replay buffer | **GPU** | Written by `policy_action` and `ingest_fill` kernels; never crosses CPU | +| Chosen `Action` enum (i8) | **GPU → mapped-pinned readback → CPU** | One byte per decision, the only DtoH on the hot path | +| `UpdateStatus`, `TrainingMetrics`, `IsvSnapshot` | **GPU → mapped-pinned readback → CPU** | Slow path only, ≤1 KB | + +Foxhunt rules enforced: +- `feedback_cpu_is_read_only.md` — no CPU arithmetic on model tensors. Counter increments and config branches in the strategy are control flow, not compute. +- `feedback_no_htod_htoh_only_mapped_pinned.md` — every CPU→GPU transfer (snapshot, fill, sampled-action seed) goes via mapped-pinned. No plain `cuMemcpyHtoD`. +- `feedback_no_atomicadd.md` — block tree-reduce for all reductions (BCE loss, PPO loss, EWC term, advantage normalisation, GAE return, mean reward EMA). Never `atomicAdd` in our kernels. +- `feedback_no_nvrtc.md` — every CUDA kernel pre-compiled to a cubin via `build.rs`. No runtime nvrtc. CfC step kernel, build_state, policy_forward, BCE loss, PPO loss + EWC term, AdamW, sample_mixed, GAE return — all cubins. +- `feedback_cudarc_f64_f32_abi.md` — Rust callers cast f64 → f32 explicitly at the FFI boundary before invoking float kernels. +- All GPU buffers pre-allocated at `AlphaModel::load`. No allocations on the hot path. Replay buffer ring is fixed size at startup. --- @@ -336,57 +393,92 @@ data_acquisition_service trading_agent_service IBKR TWS / Gateway ``` -### AlphaPpoStrategy lifecycle (sketch) +### AlphaPpoStrategy lifecycle (sketch — revised, GPU-resident state) ```rust pub struct AlphaPpoStrategy { - model: AlphaModel, // CfC + policy + EWC anchor + Adam, all GPU - buffer: ReplayBuffer, // offline + live trajectories, GPU - snap_pinned: MappedPinnedSnapshot, // CPU→GPU staging for one snapshot - decision_counter: usize, - last_target: i8, - last_state_gpu: AlphaStateGpu, - config: AlphaConfig, // max_live, decision_stride, update_cadence, ... - metrics: AlphaMetrics, - update_clock: UpdateClock, + model: AlphaModel, // owns ALL state on GPU: weights, anchor, hidden, + // position, PnL, replay ring + decision_counter: usize, // CPU control flow only + last_target: i8, // CPU mirror of GPU position state (read-only sentinel + // for emit-signal-only-on-change branch) + config: AlphaConfig, // CPU config (max_live, decision_stride, ...) + metrics: AlphaMetrics, // CPU bookkeeping + update_clock: UpdateClock, // CPU schedule } impl Strategy for AlphaPpoStrategy { + /// Hot path: called per snapshot. Target p99 ≤ 500μs. + /// Captured-CUDA-Graph fast path on every call. fn on_snapshot(&mut self, snap: &Mbp10Snapshot) -> Option { - self.snap_pinned.write_volatile(snap); - self.model.perception_forward(&self.snap_pinned); + // 1. CPU writes the snapshot bytes into model's mapped-pinned slot. + // No HtoD copy — the GPU reads directly from this pinned region. + self.model.snap_pinned_slot_mut().write_volatile(snap); + + // 2. Replay captured CUDA Graph: snap_feature_assemble → CfC step → + // multi-horizon heads → projection. Outputs land in ISV[0..13] on-device. + self.model.perception_forward(); + + // 3. CPU control flow only. self.decision_counter += 1; - if self.decision_counter % self.config.decision_stride != 0 { return None; } + if self.decision_counter % self.config.decision_stride != 0 { + return None; + } - self.build_state_gpu(snap); - let (action, log_prob, value) = self.model.policy_action(&self.last_state_gpu); + // 4. Decision point. Captured CUDA Graph: build_state → policy_forward → + // sample_action → write (state, action_idx, log_prob, value) to next + // replay slot. ONLY the action_idx (i8) crosses DtoH (mapped-pinned). + let action = self.model.policy_action(); let target = action.clamp_to(self.config.max_live); - let signal = (target != self.last_target).then(|| Signal::SetTargetPosition { contracts: target }); - self.buffer.record_step(Transition { state: self.last_state_gpu.clone(), action: target, log_prob, value }); - self.last_target = target; + let signal = if target != self.last_target { + self.last_target = target; + Some(Signal::SetTargetPosition { contracts: target }) + } else { + None + }; + // 5. Periodic update (slow path). if self.update_clock.tick() { - let batch = self.buffer.sample_mixed(8192, /* offline_frac */ 0.8); + let batch = self.model.replay_buffer().sample_mixed(8192, /*offline_frac*/ 0.8); match self.model.update(&batch) { UpdateStatus::Applied { metrics } => self.metrics.update_applied(metrics), UpdateStatus::RolledBack { reason, .. } => { self.metrics.update_rolled_back(&reason); - self.config.online_updates_enabled = false; // auto-disable; manual ack required + self.config.online_updates_enabled = false; // auto-disable; manual ack tracing::error!(reason, "online update rolled back; auto-updates DISABLED"); } } } + signal } + /// Fill ingress: ALSO via mapped-pinned, then a single GPU kernel + /// updates position_state, pnl_state, computes ΔPnL for the open segment, + /// and writes the reward back into the prior replay slot. + /// No CPU PnL math, no plain HtoD. fn on_fill(&mut self, fill: &Fill) { - self.update_pnl_and_position(fill); - self.buffer.complete_last_segment_reward(self.pnl_delta(fill)); + self.model.fill_pinned_slot_mut().write_volatile(fill); + self.model.ingest_fill(); // GPU kernel: updates state + back-fills reward } } ``` +### Why this structure satisfies the GPU/CPU contract + +| Per-snapshot CPU work | Per-snapshot GPU work | +|---|---| +| Volatile write of snapshot to mapped-pinned slot (≈70 bytes) | snap_feature_assemble kernel (computes Δt, log-returns, OFI per level) | +| `decision_counter += 1` | CfC step kernel (128 cells × per-cell time-constant integration) | +| Modulo branch | Multi-horizon heads (5 linears + sigmoid; block tree-reduce) | +| If decision: read mapped-pinned readback byte for `action` | Projection 128→8 | +| Compare `target != last_target` and emit signal | (Decision-only) build_state kernel → 20-dim AlphaState directly into replay slot | +| (slow path) Schedule check, branch into update | (Decision-only) policy_forward kernel: actor + critic + logit mask + cuRAND sample → writes (action_idx, log_prob, value) to replay slot | +| | DtoH async memcpy of single `action_idx` byte to mapped-pinned readback; one `cudaStreamSynchronize` at end of decision step | + +Zero CPU arithmetic on model tensors. Zero plain HtoD. Zero atomicAdd. All reductions are block tree-reduce. + ### Persistence and restart semantics | Element | Persistence | Restart behavior | @@ -407,6 +499,188 @@ impl Strategy for AlphaPpoStrategy { --- +## Section 5.5 — CUDA Graphs and max-performance kernel design + +Maximum performance is non-negotiable on the hot path. NVIDIA's reference patterns for sub-millisecond inference are: pre-compiled cubins, captured CUDA Graphs for fixed-shape work, persistent pinned-memory rings, cuBLAS Lt with epilogue fusion, and single sync per decision. This section enumerates exactly which kernels exist and how they're orchestrated. + +### CUDA Graph capture topology + +Three captured graphs replay over a per-stream + per-instance lifetime: + +**Graph A — per-snapshot perception (captured once at model load):** +``` +[snap_feature_assemble] + ↓ +[CfC step] ← reads h_old, writes h_new (ping-pong device buffers) + ↓ +[multi-horizon heads (fused: 5 linears into one 5×128 GEMM via cuBLAS Lt)] + ↓ +[projection 128→8] + ↓ +[layer-norm + write to ISV slots 0..13] +``` +Single graph replay per snapshot. Inputs pointer-stable (mapped-pinned snapshot slot, ping-pong h buffer); kernel args are device pointers, not values, so the graph stays valid across replays. + +**Graph B — per-decision policy + replay write (captured once at model load):** +``` +[build_state] ← reads ISV[0..13] + position/pnl state → 20-dim AlphaState + ↓ +[policy_forward (fused: actor CfC + critic CfC + logit-mask + log_softmax)] + ↓ +[sample_action (cuRAND device API, no host sync)] + ↓ +[write_replay_slot (writes state, action_idx, log_prob, value to ring)] + ↓ +[async DtoH 1 byte → mapped-pinned readback] +``` +Single graph replay per decision. The `max_live` mask threshold is read from device memory (ISV slot 28, written by config-update path), so it stays inside the graph. + +**Graph C — per-update training step (captured once at model load):** +``` +[sample minibatch indices (cuRAND, device-side)] + ↓ +[gather state/action/reward/value/log_prob from replay (device gather)] + ↓ +[compute GAE returns + advantages (sequential prefix scan, single-block cooperative)] + ↓ +[advantage normalization (block tree-reduce mean/std, then z-score)] + ↓ +[policy_forward + critic_forward (fused, batched, cuBLAS Lt GEMMs with bias-epilogue)] + ↓ +[ppo_loss + value_loss + entropy_bonus + EWC_term (fused into one kernel — all reductions block tree-reduce)] + ↓ +[backward through fused forward (cuBLAS Lt grad GEMMs)] + ↓ +[AdamW step (per-parameter-group lr scaling, in-place flat-buffer update)] + ↓ +[update ISV slots 13..24 with diagnostics] +``` +One graph per PPO minibatch. Capture once at startup; replay K=4 epochs × N minibatches per update. Mirrors the `fused_training.rs` pattern from `gpu_dqn_trainer.rs` — replaces 2,000+ Candle dispatches per training step with one captured graph. + +### Kernel inventory and which one calls cuBLAS + +| Kernel (NVRTC pre-compiled cubin) | Purpose | Reduction strategy | cuBLAS path | +|---|---|---|---| +| `snap_feature_assemble.cu` | MBP-10 levels → 32-dim feature vector (mid log-return, spread, depth, OFI per level, trade flow, Δt) | None (element-wise) | None | +| `cfc_step.cu` | One CfC time step: `h_new = h_old·exp(-Δt/τ) + (1-exp(-Δt/τ))·tanh(W_in·x + W_rec·h + b)` | None | **Yes** — `W_in @ x` and `W_rec @ h_old` as fused 2-matrix GEMM via cuBLAS Lt with bias + tanh epilogue | +| `multi_horizon_heads.cu` | 128-dim hidden → 5 logits + sigmoid | None | **Yes** — single 5×128 GEMM via cuBLAS Lt with sigmoid epilogue | +| `projection.cu` | 128 → 8 with layer-norm | Block tree-reduce for mean/std | **Yes** — 8×128 GEMM | +| `build_state.cu` | Compose 20-dim AlphaState from ISV slots + position/PnL state | None (element-wise) | None | +| `policy_forward.cu` | Actor CfC + critic CfC, fused 4 GEMMs | Block tree-reduce for log_softmax denom | **Yes** — 4 fused GEMMs (state→hidden actor, hidden→logits, state→hidden critic, hidden→value) | +| `sample_action.cu` | cuRAND categorical sample with logit mask | Block tree-reduce for cumulative sum | None | +| `write_replay_slot.cu` | Write (state, action_idx, log_prob, value) to ring | None | None | +| `ingest_fill.cu` | Read mapped-pinned fill, update position/PnL state, back-fill reward into prior replay slot | None | None | +| `gae_returns.cu` | Sequential prefix scan over rollout for returns + advantages | Single-block cooperative scan | None | +| `advantage_normalize.cu` | z-score advantages across minibatch | Block tree-reduce mean + std | None | +| `ppo_loss.cu` | Fused: PPO clip + value MSE + entropy + EWC term + backward through policy_forward | Block tree-reduce for each loss term | **Yes** — grad GEMMs via cuBLAS Lt | +| `adamw_step.cu` | Per-parameter-group AdamW with weight decay | None (element-wise on flat buffer) | None | +| `bce_loss_multi_horizon.cu` | Phase A only: 5-horizon BCE + backward | Block tree-reduce | **Yes** — grad GEMMs | +| `kill_switch_check.cu` | Compute σ-distance for each metric against rolling baseline; return verdict | Block tree-reduce | None | + +All reductions use the standard NVIDIA cooperative block tree-reduce pattern from `cooperative_groups.h` — single warp per block does the final reduce, no `atomicAdd`. Per `feedback_no_atomicadd.md`. + +### cuBLAS Lt epilogue fusion patterns used + +| Pattern | Where | Saving | +|---|---|---| +| `GEMM + bias + tanh` | CfC step (`W_in @ x + b` then tanh) | 2 kernels → 1 | +| `GEMM + bias + sigmoid` | Multi-horizon heads | 2 kernels → 1 | +| `GEMM + bias + ReLU` (if used) | Policy hidden layers | 2 kernels → 1 | +| `Batched GEMM` | Phase A perception over `seq_len=64` per batch | N kernel launches → 1 | +| `Strided batched GEMM` | Multi-env rollout during Phase B (`n_par=64` envs) | N kernel launches → 1 | + +These mirror the patterns in `pearl_cublas_lt_vs_classic_sgemm.md`. + +### Persistent allocations and ring layouts + +- **Snapshot ingress slot**: 1 × (32 floats + 1 u64 timestamp) ≈ 136 bytes, mapped-pinned, host-write + device-read +- **Fill ingress slot**: 1 × (8 floats: fill_price, fill_qty, side, commission, ts, ...) ≈ 32 bytes, mapped-pinned +- **Action readback slot**: 1 × i8 ≈ 1 byte, mapped-pinned, device-write + host-read +- **CfC hidden ping-pong**: 2 × 128 × f32 = 1 KB device-resident (no pinned) +- **ISV bus**: 32 × f32 = 128 bytes device-resident, accessible by all kernels +- **Replay ring**: ring of `MAX_LIVE_TRANSITIONS` × `(state[20] + action[1 i8 + 3 pad] + log_prob[1] + value[1] + reward[1] + done[1] = 28 floats + 4 bytes)` ≈ 116 bytes × 10K = 1.16 MB +- **Offline replay**: 50K × 116 = 5.8 MB, GPU-resident, never re-uploaded +- **Per-minibatch scratch (training)**: 8192 × 116 ≈ 950 KB (sampled batch); plus gradient buffers ≈ 280 KB (35K params × 2 for momentum + variance) ≈ 1.5 MB total + +All allocated once at `AlphaModel::load`. Replay ring uses a single-writer device-atomic ring index (CAS, no atomicAdd — uses `atomicCAS` for the write head only, which is per-decision overhead ≈ ns). + +### Single-stream design (low-latency hot path) + +Hot path is serial — perception depends on snapshot, policy depends on perception. No multi-stream benefit. Use a single dedicated `CUDA_STREAM_NON_BLOCKING` stream for the hot path. + +Slow path (training updates) uses a separate stream to keep hot-path latency clean during overlapping update windows. The atomic weights swap synchronizes both streams at the boundary. + +### Determinism for tests + +- `cublasLtMatmulPreference` set to `CUBLAS_GEMM_DEFAULT_DEDUCED` (deterministic algorithm) +- `cublasSetMathMode(handle, CUBLAS_PEDANTIC_MATH)` in test builds +- Bit-equivalence tests target ε ≤ 1e-5 relative error across CPU oracle vs GPU +- Production builds use `CUBLAS_TF32_TENSOR_OP_MATH` for max throughput; accept the bit-divergence (still ε-stable) + +### Build-time kernel compilation (no NVRTC) + +All .cu files compiled to cubins by `build.rs`: +- `nvcc --gpu-architecture=compute_89 --gpu-code=sm_89,sm_90 --cubin --use_fast_math` +- `sm_86` (RTX 3050) also targeted for local dev +- Cubins embedded in the binary via `include_bytes!` so no runtime file IO +- Per `feedback_no_nvrtc.md` — zero runtime kernel compilation + +### Performance budget (target on L40S, after CUDA Graph capture amortized) + +| Operation | Budget | Realistic | +|---|---|---| +| Per-snapshot perception (Graph A replay) | ≤100μs | ~30-50μs (small model, single graph replay) | +| Per-decision policy (Graph B replay) | ≤200μs | ~80-120μs (includes single DtoH for action byte) | +| Per-update full PPO step (Graph C × K=4 × N_minibatch) | ≤30s | ~15-25s on L40S for 8192-batch × K=4 | +| Cold start (model load + buffer load) | ≤30s | dominated by offline replay load from PVC | +| Kernel build (build.rs) | ≤2min | one-time per release build | + +--- + +## Section 5.6 — Disconnect, failure, and degraded-mode handling + +The hot path has two external dependencies — Databento (snapshot stream) and IBKR (fills, orders). Either can drop. The default safe state is **flat + paused**. + +### Databento stream loss + +Triggered by: gap in snapshot timestamps > `max_snapshot_gap_ms` (default 1000 ms), connection drop, or malformed payload. + +| Step | Action | +|---|---| +| 1 | Mark strategy state `Degraded::DataStale` (no new `on_snapshot` calls processed) | +| 2 | Issue `Signal::SetTargetPosition { contracts: 0 }` to broker gateway immediately | +| 3 | CfC perception state `h` is **frozen** (no zero-reset — discontinuity should not corrupt the memory) | +| 4 | Reconnect with exponential backoff (1s, 2s, 5s, 10s, 30s, cap) | +| 5 | On reconnect: continue feeding `on_snapshot`, but `decision_counter` skips `decision_stride` ticks of warm-up before first action (CfC needs to re-prime) | + +If gap > `max_recovery_window_s` (default 300s), force a full model reload (state vector and CfC `h` re-initialized from disk snapshot). + +### IBKR API failure modes + +| Failure | Detection | Response | +|---|---|---| +| Order rejected (margin, contract spec, throttle) | `OrderRejected` event | Log + Prometheus counter; do NOT retry blindly. If throttle: backoff per IBKR pacing rules (see Section 3.5). If margin: assume risk-system breach, target=0, halt | +| Fill stream stalled (no fills despite known orders) | No `Fill` events for `max_fill_silence_s` (default 60s) | Query IBKR for open orders; reconcile broker state against local `position` | +| TCP socket closed | `Disconnected` event from `ibapi` | Same as Databento: flat + paused + reconnect with backoff | +| Position drift (broker says X, local thinks Y) | Periodic reconcile (every 60s) | Hard error: alert + halt; never silently re-sync (state corruption from a missed fill must be diagnosed, not papered over) | + +### What is NEVER done + +- **No retry-on-error of the same action** without first reconciling external state +- **No "if disconnected, keep predicting from last known state"** — the policy can't trade if data is stale +- **No silent re-init of CfC hidden state** mid-day — a discontinuity in `h` is a bug to log, not a recovery action + +### Restart semantics (operator-driven) + +`alpha-control restart` writes a clean shutdown signal: +1. Issue target=0, wait for fill confirmation +2. Persist `(weights, h, position, replay_ring, ewc_anchor, kill_switch_state, isv_state)` to disk +3. Acknowledge shutdown to operator + +Cold-start on next launch reloads all of the above. See Section 6 cold-start subsection for what happens before enough live data accumulates. + +--- + ## Section 6 — Online learning mechanics ### EWC anchor — what gets frozen, what gets learned @@ -462,6 +736,21 @@ fn update(&mut self, batch: &MiniBatch) -> UpdateStatus { } ``` +### Cold-start bootstrap (first hours after deploy) + +The kill-switch baseline needs ≥20 successful updates before its σ-bounds are meaningful. Until then, the system runs in **bootstrap mode**. + +| State | Live trajectories collected | Behavior | +|---|---|---| +| **Bootstrap-init** | < 1,000 live decisions | Replay sampler returns 100% offline (live ring is too sparse to mix in without bias). No online updates yet. | +| **Bootstrap-warmup** | 1,000 – 10,000 | Replay sampler ramps from 100% offline → 80/20 linearly. Online updates allowed but kill-switch uses a *conservative* baseline: reject if any metric is 2σ worse than the Phase B walk-forward metrics (not the rolling live baseline, which doesn't exist yet). | +| **Bootstrap-baseline** | 10,000 + 20 successful updates landed | Switch to rolling-baseline kill-switch as documented below. | +| **Steady-state** | After bootstrap-baseline cleared | Full online learning with rolling σ-bounds. | + +Bootstrap-mode kill-switch uses **Phase B walk-forward statistics as the reference distribution** — `mean_reward`, `loss_ppo`, `hit_rate`, `kl_div`, `entropy` from the final fold of Phase B CV serve as the pre-deployment baseline. These get written to a `phase_b_baseline.json` artifact alongside the weights snapshot at the end of Phase B training. + +ISV updates are **always live** — controllers don't need a warmup period because their per-signal EMAs auto-bootstrap on first observation (per `pearl_first_observation_bootstrap.md`). + ### Kill-switch criteria Rolling baseline tracked from last 20 successful updates.