From a27cb40a9a9a6f404e99cff6393f7ea0d6453440 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sat, 16 May 2026 10:05:25 +0200 Subject: [PATCH] fix(precompute): consume DbnTrade Vec during Mbp10Trade conversion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 9-quarter precompute_features run OOM-killed at ~56Gi on the ci-compile-cpu pool (POP2-HC-32C-64G) on 2026-05-16. Root cause: lines 672-684's `t.iter().map(...).collect()` borrows the source DbnTrade Vec while building the Mbp10Trade Vec — both alive simultaneously, transient peak ~25GB just from this transformation for the 199M-trade dataset. `.into_iter()` consumes the source element-by-element so the allocation drops as the destination grows, capping peak at the larger of the two Vecs (~15GB) rather than their sum. Should let the 9-quarter precompute fit comfortably on the existing 64GB ci-compile-cpu pool without provisioning a high-memory node. Co-Authored-By: Claude Opus 4.7 --- crates/ml/examples/precompute_features.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/crates/ml/examples/precompute_features.rs b/crates/ml/examples/precompute_features.rs index c61961e2a..1a6f70291 100644 --- a/crates/ml/examples/precompute_features.rs +++ b/crates/ml/examples/precompute_features.rs @@ -671,8 +671,17 @@ async fn main() -> Result<()> { // f64 volume for size-weighted features. alpha_snapshots = all_snapshots; if let Some(t) = all_trades { + // CONSUME the original DbnTrade Vec while building the + // Mbp10Trade Vec. Using `.iter()` here previously kept + // the original ~10GB Vec alive for the duration of the + // transformation, so peak memory transiently doubled + // to ~25GB (DbnTrade + Mbp10Trade Vecs simultaneously) + // and OOM-killed the 9-quarter precompute at 56Gi + // (2026-05-16). `.into_iter()` drops the source + // allocation element-by-element as the destination + // builds, capping peak at the larger of the two. alpha_trades = t - .iter() + .into_iter() .map(|dbn| ml::features::Mbp10Trade { price: dbn.price, volume: dbn.volume as f64,