Per docs/superpowers/specs/2026-05-17-kloop-parallelization-design.md. cfc_step_batched (fwd + bwd) refactored from grid=(1,1,1) with internal n_batch loop to grid=(B,1,1) — each block handles one batch. Removes the single-SM bottleneck on the K-loop's most-called kernel (64×/step). Param-grad accumulation moves to per-batch scratch: cfc_grad_w_in_scratch_d [B, n_hid, n_in] cfc_grad_w_rec_scratch_d [B, n_hid, n_hid] cfc_grad_b_scratch_d [B, n_hid] cfc_grad_tau_scratch_d [B, n_hid] Zeroed once per training step, K-loop's 64 bwd calls += into them, then 4 reduce_axis0 launches collapse B → final grad buffers (OVERWRITE) before AdamW. New AdamW-after-reducer invariant: final grads are meaningful only after the reducer has run in the current step. New reduce_axis0 kernel: single parameterised reducer [B, N] → [N] via block tree-reduce (no atomicAdd per feedback_no_atomicadd.md). Same pattern as layer_norm_reduce_param_grads — CUDA-Graph-safe. cfc_step_backward_batched shared-mem dropped from (B+1)*n_hid*4 to 2*n_hid*4 bytes per block (only one row of sd_pre needed per block bi). Tests: - New stacked_trainer_loss_shrinks_at_batch_32: FIRST test that actually exercises the cross-batch reduction code path; existing perception_overfit suite was all B=1. Initial 0.24 → final 0.00. - Scratch-clears test removed (explanatory comment kept): structurally hard to assert directly due to begin_capture/end_capture not executing kernels; the B=32 convergence smoke implicitly validates scratch zeroing since divergence would otherwise be immediate. All 9 perception_overfit smokes + 4 backward_finite_diff tests pass. build.rs: - KERNELS list adds "reduce_axis0" - Cache-bust → v11 Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
116 lines
3.7 KiB
Rust
116 lines
3.7 KiB
Rust
//! Pre-compile all ml-alpha CUDA kernels into arch-specific cubins.
|
|
//!
|
|
//! Per `feedback_no_nvrtc.md`: no runtime kernel compilation.
|
|
//! Per `pearl_build_rs_rerun_if_env_changed.md`: every `std::env::var`
|
|
//! is paired with `cargo:rerun-if-env-changed`.
|
|
|
|
use std::path::{Path, PathBuf};
|
|
use std::process::Command;
|
|
|
|
const KERNELS: &[&str] = &[
|
|
"mamba2_alpha_kernel", // Mamba2 SSM scan kernel (used by PerceptionTrainer's encoder prefix)
|
|
"snap_feature_assemble",
|
|
"cfc_step",
|
|
"multi_horizon_heads",
|
|
"projection",
|
|
"bce_loss_multi_horizon",
|
|
"adamw_step",
|
|
"grad_norm",
|
|
"horizon_lambda", // ISV-driven per-horizon gradient scaler (EMA + lambda)
|
|
"layer_norm", // Phase 1: trunk pre-CfC normalisation
|
|
"variable_selection", // Phase 2D: TFT-style per-feature gating
|
|
"attention_pool", // Phase 3: learned context summary at CfC k=0
|
|
"reduce_axis0", // Phase B: cross-batch param-grad reducer
|
|
];
|
|
|
|
// Cache bust v11 (2026-05-17): K-loop parallelization Phase B —
|
|
// new reduce_axis0.cu kernel + block-per-batch refactor of
|
|
// cfc_step_batched (fwd+bwd). Old cubins don't have the new symbols.
|
|
// Force fresh nvcc compile.
|
|
|
|
fn main() {
|
|
println!("cargo:rerun-if-changed=build.rs");
|
|
|
|
println!("cargo:rerun-if-env-changed=CARGO_FEATURE_CUDA");
|
|
if std::env::var("CARGO_FEATURE_CUDA").is_err() {
|
|
eprintln!(" ml-alpha: cuda feature disabled, skipping kernel build");
|
|
return;
|
|
}
|
|
|
|
println!("cargo:rerun-if-env-changed=CUDA_COMPUTE_CAP");
|
|
println!("cargo:rerun-if-env-changed=CUDA_HOME");
|
|
let cap = std::env::var("CUDA_COMPUTE_CAP").unwrap_or_else(|_| "80".to_string());
|
|
let arch = format!("sm_{cap}");
|
|
|
|
let nvcc = match find_nvcc() {
|
|
Some(p) => p,
|
|
None => {
|
|
eprintln!(" ml-alpha: nvcc not found, skipping kernel build (set CUDA_HOME or install CUDA toolkit)");
|
|
return;
|
|
}
|
|
};
|
|
|
|
let out = PathBuf::from(std::env::var("OUT_DIR").expect("OUT_DIR not set by cargo"));
|
|
|
|
for k in KERNELS {
|
|
let src = PathBuf::from(format!("cuda/{k}.cu"));
|
|
if !src.exists() {
|
|
eprintln!(" ml-alpha: skipping {k} — source not yet present");
|
|
continue;
|
|
}
|
|
println!("cargo:rerun-if-changed={}", src.display());
|
|
let cubin = out.join(format!("{k}.cubin"));
|
|
compile(&nvcc, &src, &cubin, &arch);
|
|
}
|
|
}
|
|
|
|
fn compile(nvcc: &Path, src: &Path, cubin: &Path, arch: &str) {
|
|
let status = Command::new(nvcc)
|
|
.args([
|
|
"-cubin",
|
|
&format!("-arch={arch}"),
|
|
"-O3",
|
|
"--use_fast_math",
|
|
"--ftz=true",
|
|
"--fmad=true",
|
|
"-o",
|
|
cubin.to_str().unwrap(),
|
|
src.to_str().unwrap(),
|
|
])
|
|
.status()
|
|
.unwrap_or_else(|e| panic!("nvcc spawn failed for {}: {e}", src.display()));
|
|
if !status.success() {
|
|
panic!(
|
|
"nvcc failed for {} (exit {})",
|
|
src.display(),
|
|
status.code().unwrap_or(-1)
|
|
);
|
|
}
|
|
eprintln!(
|
|
" ml-alpha: compiled {} -> {} ({arch})",
|
|
src.display(),
|
|
cubin.display()
|
|
);
|
|
}
|
|
|
|
fn find_nvcc() -> Option<PathBuf> {
|
|
if let Ok(home) = std::env::var("CUDA_HOME") {
|
|
let p = PathBuf::from(home).join("bin/nvcc");
|
|
if p.exists() {
|
|
return Some(p);
|
|
}
|
|
}
|
|
for cand in ["/usr/local/cuda/bin/nvcc", "/usr/bin/nvcc"] {
|
|
let p = PathBuf::from(cand);
|
|
if p.exists() {
|
|
return Some(p);
|
|
}
|
|
}
|
|
Command::new("nvcc")
|
|
.arg("--version")
|
|
.output()
|
|
.ok()
|
|
.filter(|o| o.status.success())
|
|
.map(|_| PathBuf::from("nvcc"))
|
|
}
|