Files
foxhunt/crates/ml-alpha/build.rs
jgrusewski 494a2e4827 perf(ml-alpha): block-per-batch cfc_step + reduce_axis0 reducer (Phase B commit 1)
Per docs/superpowers/specs/2026-05-17-kloop-parallelization-design.md.

cfc_step_batched (fwd + bwd) refactored from grid=(1,1,1) with internal
n_batch loop to grid=(B,1,1) — each block handles one batch. Removes
the single-SM bottleneck on the K-loop's most-called kernel (64×/step).

Param-grad accumulation moves to per-batch scratch:
  cfc_grad_w_in_scratch_d  [B, n_hid, n_in]
  cfc_grad_w_rec_scratch_d [B, n_hid, n_hid]
  cfc_grad_b_scratch_d     [B, n_hid]
  cfc_grad_tau_scratch_d   [B, n_hid]

Zeroed once per training step, K-loop's 64 bwd calls += into them, then
4 reduce_axis0 launches collapse B → final grad buffers (OVERWRITE)
before AdamW. New AdamW-after-reducer invariant: final grads are
meaningful only after the reducer has run in the current step.

New reduce_axis0 kernel: single parameterised reducer [B, N] → [N] via
block tree-reduce (no atomicAdd per feedback_no_atomicadd.md). Same
pattern as layer_norm_reduce_param_grads — CUDA-Graph-safe.

cfc_step_backward_batched shared-mem dropped from (B+1)*n_hid*4 to
2*n_hid*4 bytes per block (only one row of sd_pre needed per block bi).

Tests:
- New stacked_trainer_loss_shrinks_at_batch_32: FIRST test that
  actually exercises the cross-batch reduction code path; existing
  perception_overfit suite was all B=1. Initial 0.24 → final 0.00.
- Scratch-clears test removed (explanatory comment kept): structurally
  hard to assert directly due to begin_capture/end_capture not
  executing kernels; the B=32 convergence smoke implicitly validates
  scratch zeroing since divergence would otherwise be immediate.

All 9 perception_overfit smokes + 4 backward_finite_diff tests pass.

build.rs:
- KERNELS list adds "reduce_axis0"
- Cache-bust → v11

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
2026-05-17 23:47:40 +02:00

116 lines
3.7 KiB
Rust

//! Pre-compile all ml-alpha CUDA kernels into arch-specific cubins.
//!
//! Per `feedback_no_nvrtc.md`: no runtime kernel compilation.
//! Per `pearl_build_rs_rerun_if_env_changed.md`: every `std::env::var`
//! is paired with `cargo:rerun-if-env-changed`.
use std::path::{Path, PathBuf};
use std::process::Command;
const KERNELS: &[&str] = &[
"mamba2_alpha_kernel", // Mamba2 SSM scan kernel (used by PerceptionTrainer's encoder prefix)
"snap_feature_assemble",
"cfc_step",
"multi_horizon_heads",
"projection",
"bce_loss_multi_horizon",
"adamw_step",
"grad_norm",
"horizon_lambda", // ISV-driven per-horizon gradient scaler (EMA + lambda)
"layer_norm", // Phase 1: trunk pre-CfC normalisation
"variable_selection", // Phase 2D: TFT-style per-feature gating
"attention_pool", // Phase 3: learned context summary at CfC k=0
"reduce_axis0", // Phase B: cross-batch param-grad reducer
];
// Cache bust v11 (2026-05-17): K-loop parallelization Phase B —
// new reduce_axis0.cu kernel + block-per-batch refactor of
// cfc_step_batched (fwd+bwd). Old cubins don't have the new symbols.
// Force fresh nvcc compile.
fn main() {
println!("cargo:rerun-if-changed=build.rs");
println!("cargo:rerun-if-env-changed=CARGO_FEATURE_CUDA");
if std::env::var("CARGO_FEATURE_CUDA").is_err() {
eprintln!(" ml-alpha: cuda feature disabled, skipping kernel build");
return;
}
println!("cargo:rerun-if-env-changed=CUDA_COMPUTE_CAP");
println!("cargo:rerun-if-env-changed=CUDA_HOME");
let cap = std::env::var("CUDA_COMPUTE_CAP").unwrap_or_else(|_| "80".to_string());
let arch = format!("sm_{cap}");
let nvcc = match find_nvcc() {
Some(p) => p,
None => {
eprintln!(" ml-alpha: nvcc not found, skipping kernel build (set CUDA_HOME or install CUDA toolkit)");
return;
}
};
let out = PathBuf::from(std::env::var("OUT_DIR").expect("OUT_DIR not set by cargo"));
for k in KERNELS {
let src = PathBuf::from(format!("cuda/{k}.cu"));
if !src.exists() {
eprintln!(" ml-alpha: skipping {k} — source not yet present");
continue;
}
println!("cargo:rerun-if-changed={}", src.display());
let cubin = out.join(format!("{k}.cubin"));
compile(&nvcc, &src, &cubin, &arch);
}
}
fn compile(nvcc: &Path, src: &Path, cubin: &Path, arch: &str) {
let status = Command::new(nvcc)
.args([
"-cubin",
&format!("-arch={arch}"),
"-O3",
"--use_fast_math",
"--ftz=true",
"--fmad=true",
"-o",
cubin.to_str().unwrap(),
src.to_str().unwrap(),
])
.status()
.unwrap_or_else(|e| panic!("nvcc spawn failed for {}: {e}", src.display()));
if !status.success() {
panic!(
"nvcc failed for {} (exit {})",
src.display(),
status.code().unwrap_or(-1)
);
}
eprintln!(
" ml-alpha: compiled {} -> {} ({arch})",
src.display(),
cubin.display()
);
}
fn find_nvcc() -> Option<PathBuf> {
if let Ok(home) = std::env::var("CUDA_HOME") {
let p = PathBuf::from(home).join("bin/nvcc");
if p.exists() {
return Some(p);
}
}
for cand in ["/usr/local/cuda/bin/nvcc", "/usr/bin/nvcc"] {
let p = PathBuf::from(cand);
if p.exists() {
return Some(p);
}
}
Command::new("nvcc")
.arg("--version")
.output()
.ok()
.filter(|o| o.status.success())
.map(|_| PathBuf::from("nvcc"))
}