From e3d08296809770f08ebdf0c9fced2061ac85e0f8 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sun, 3 May 2026 18:05:41 +0200 Subject: [PATCH] fix(build): cargo:rerun-if-env-changed=CUDA_COMPUTE_CAP for cubin arch Without this trigger, cargo treated CUDA_COMPUTE_CAP-driven cubin compilation as cached output of a non-tracked env var. The cargo-target PVC is shared across H100 (sm_90) and L40S (sm_89) training jobs, so swapping --gpu-pool between archs left stale cubins from the previous build cached for any future sm_X variation. Symptom (T10 train-multi-seed-rn559 today): Failed to create DQN: rmsnorm cubin load: DriverError( CUDA_ERROR_NO_BINARY_FOR_GPU, "no kernel image is available for execution on the device") The training pod scheduled on L40S (sm_89), the binary cache served sm_90 cubins from a prior H100 build, and the rmsnorm cubin had no sm_89 entrypoint. Fix is one line: tell cargo to invalidate when CUDA_COMPUTE_CAP changes. First post-fix build will rebuild all cubins (cargo conservatively reruns when a new rerun-if-env-changed trigger is added without prior state). Subsequent builds rebuild only on actual env changes. Generalises the same lesson as the recent SP7 host-branch-in-captured-graph fix: env-var-conditional code that doesn't declare its dependencies freezes at first observation regardless of runtime input. --- crates/ml/build.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/crates/ml/build.rs b/crates/ml/build.rs index 3870f7bb2..57b88f195 100644 --- a/crates/ml/build.rs +++ b/crates/ml/build.rs @@ -36,7 +36,12 @@ fn main() { println!("cargo:rerun-if-changed={}", trade_physics_header.display()); println!("cargo:rerun-if-changed={}", sp4_histogram_p99_header.display()); - // Detect GPU architecture from env or default to sm_80 + // Detect GPU architecture from env or default to sm_80. + // `rerun-if-env-changed` is mandatory: the cargo-target PVC is shared + // across H100 (sm_90) and L40S (sm_89) jobs, and without this trigger + // cargo serves stale cubins compiled for the previous arch on every + // subsequent build, producing CUDA_ERROR_NO_BINARY_FOR_GPU at runtime. + println!("cargo:rerun-if-env-changed=CUDA_COMPUTE_CAP"); let cuda_compute_cap = std::env::var("CUDA_COMPUTE_CAP").unwrap_or_else(|_| "80".to_string()); let arch = format!("sm_{cuda_compute_cap}");