From 899ade8dfb618cc034151162a6f3d7bc0a292462 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Thu, 2 Apr 2026 18:09:34 +0200 Subject: [PATCH] =?UTF-8?q?fix:=20fxcache=20key=20uses=20mtime=20which=20c?= =?UTF-8?q?hanges=20on=20PVC=20remount=20=E2=80=94=20use=20size=20only?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cache key hashed filename + size + mtime. PVC remounts on different nodes change file timestamps without changing data, producing a different key every pod boot. Result: fxcache MISS on every H100 run, forcing 10+ min DBN feature extraction from 148GB raw MBP-10 data. Fix: hash filename + size only. Deterministic across remounts. Note: existing cache must be rebuilt once (key format changed). Co-Authored-By: Claude Opus 4.6 (1M context) --- crates/ml/src/feature_cache.rs | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/crates/ml/src/feature_cache.rs b/crates/ml/src/feature_cache.rs index fd62592ec..cdee0b517 100644 --- a/crates/ml/src/feature_cache.rs +++ b/crates/ml/src/feature_cache.rs @@ -7,7 +7,6 @@ use anyhow::{Context, Result}; use sha2::{Digest, Sha256}; use std::path::{Path, PathBuf}; -use std::time::SystemTime; /// Calculate a cache key for a DBN data directory. /// @@ -52,7 +51,9 @@ pub fn calculate_dbn_cache_key_full( } for path in &files { - // Hash filename (not full path) + size + mtime + // Hash filename (not full path) + size. + // NOT mtime: PVC remounts and node reboots change file timestamps + // without changing data, causing cache misses on every new pod. if let Some(name) = path.file_name() { hasher.update(name.to_string_lossy().as_bytes()); } @@ -60,13 +61,6 @@ pub fn calculate_dbn_cache_key_full( .metadata() .with_context(|| format!("Failed to stat {:?}", path))?; hasher.update(&meta.len().to_le_bytes()); - let mtime = meta - .modified() - .context("mtime unavailable")? - .duration_since(SystemTime::UNIX_EPOCH) - .context("mtime before epoch")? - .as_secs(); - hasher.update(&mtime.to_le_bytes()); } Ok(format!("{:x}", hasher.finalize()))