From 322c43db44590e19f6e775c3d287f7c7a5eec489 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sat, 21 Feb 2026 23:26:40 +0100 Subject: [PATCH] fix(ml): recover from mutex poisoning in Rayon quantization with unwrap_or_else MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace all 8 `.lock().unwrap()` calls in the Rayon parallel closure of `quantize_varmap_parallel` with `.lock().unwrap_or_else(|e| e.into_inner())` so that mutex poisoning (caused by a panicking Rayon thread) does not cascade and crash all other worker threads — the guard is recovered from the poisoned state and processing continues safely. Co-Authored-By: Claude Opus 4.6 --- ml/src/tft/varmap_quantization.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/ml/src/tft/varmap_quantization.rs b/ml/src/tft/varmap_quantization.rs index f26f4d09d..bdec5d66c 100644 --- a/ml/src/tft/varmap_quantization.rs +++ b/ml/src/tft/varmap_quantization.rs @@ -341,12 +341,12 @@ pub fn quantize_varmap_parallel( // Validate and quantize match validate_tensor_for_quantization(tensor, name) { Ok(_) => { - let mut quantizer_guard = quantizer.lock().unwrap(); + let mut quantizer_guard = quantizer.lock().unwrap_or_else(|e| e.into_inner()); match quantizer_guard.quantize_tensor(tensor, name) { Ok(quantized) => Some(quantized), Err(e) => { warn!("Failed to quantize {}: {}", name, e); - let mut skip_counters = skipped_counter.lock().unwrap(); + let mut skip_counters = skipped_counter.lock().unwrap_or_else(|e| e.into_inner()); skip_counters.3 += 1; // error count None }, @@ -354,7 +354,7 @@ pub fn quantize_varmap_parallel( }, Err(e) => { warn!("Skipping invalid tensor {}: {}", name, e); - let mut skip_counters = skipped_counter.lock().unwrap(); + let mut skip_counters = skipped_counter.lock().unwrap_or_else(|e| e.into_inner()); skip_counters.3 += 1; // error count None }, @@ -362,26 +362,26 @@ pub fn quantize_varmap_parallel( }, TensorCategory::Bias => { debug!("Skipping bias tensor: {}", name); - let mut skip_counters = skipped_counter.lock().unwrap(); + let mut skip_counters = skipped_counter.lock().unwrap_or_else(|e| e.into_inner()); skip_counters.0 += 1; // bias count None }, TensorCategory::LayerNorm => { debug!("Skipping LayerNorm tensor: {}", name); - let mut skip_counters = skipped_counter.lock().unwrap(); + let mut skip_counters = skipped_counter.lock().unwrap_or_else(|e| e.into_inner()); skip_counters.1 += 1; // layernorm count None }, TensorCategory::Small => { debug!("Skipping small tensor: {}", name); - let mut skip_counters = skipped_counter.lock().unwrap(); + let mut skip_counters = skipped_counter.lock().unwrap_or_else(|e| e.into_inner()); skip_counters.2 += 1; // small count None }, }; // Update progress counter - let mut counter = progress_counter.lock().unwrap(); + let mut counter = progress_counter.lock().unwrap_or_else(|e| e.into_inner()); *counter += 1; let current_progress = *counter; drop(counter); @@ -419,7 +419,7 @@ pub fn quantize_varmap_parallel( let elapsed = start_time.elapsed().as_secs_f32(); let rate = total_tensors as f32 / elapsed; - let skip_counters = skipped_counter.lock().unwrap(); + let skip_counters = skipped_counter.lock().unwrap_or_else(|e| e.into_inner()); let (bias_count, layernorm_count, small_count, error_count) = *skip_counters; info!(