Caps any branch's weight-gradient L2 norm at num_branches × median
(median across the 4 branches' norms). Scales the offending branch's
gradient down to the cap; healthy branches pass through unchanged.
Fixes the observed pathology in L40S train-mdh86: grad_ratio_mag_dir
was 15k–26k× for 6 consecutive epochs, then collapsed to ~100× in a
single step at epoch 7 and destabilised learning (Sharpe flipped +34
→ -67, never recovered). Symmetric per-branch capping at
`num_branches × median` prevents the swing at both ends without
requiring a global ratio bound.
No tuned knobs: `num_branches = 4` is architectural (factored action
space: direction × magnitude × order × urgency), `median_branch_norm`
is a per-step statistical reference that tracks the current gradient
regime, and the product is fully adaptive. Per
feedback_adaptive_not_tuned.md, the only static value is the
architectural axis count; medians and derived caps are signal-driven.
Implementation — two CUDA kernel launches in
`branch_grad_balance_kernel.cu`:
branch_grad_norm_reduce: grid=(4,1,1), block=(256,1,1). One block
per branch; sum-of-squares via shared-mem
tree reduce writes `branch_norms_dev[4]`.
No atomicAdd (one-block-per-branch, single
writer per slot).
branch_grad_rescale: grid=(max_blocks, 4, 1), block=(256,1,1).
Each block caches the 4 branch norms into
shared memory, computes the median via a
5-comparator sorting network + two-element
average (branch-deterministic, no reduction
primitive), derives the 4 per-branch scales
`scale[d] = min(1, 4×median/norm[d])`, then
threads multiply their slice element by the
owning branch's scale. No atomicAdd (each
thread writes one distinct element).
Insertion point: inside the `adam_grad_child` graph between the aux
phase and `compute_grad_norm_for_adam`, so Adam's global clip and the
Adam update both observe the rebalanced gradient. Also wired into the
ungraphed fallback paths so no code path can skip the cap. The kernels
have fixed launch configs, no host syncs, no dynamic allocations —
safe to capture.
Per-branch slice metadata (starts/lens for each of the 4 contiguous
4-tensor branch slices in `grad_buf`) is precomputed from
`compute_param_sizes` at trainer construction and uploaded once to
device i32 buffers, matching the existing `grad_decomp_kernel` layout
convention.
Smoke tests (local RTX 3050 Ti, 4 GB):
magnitude_distribution: PASS (MAG_DIST Q=0.637 H=0.114 F=0.249,
EVAL_DIST Q=0.153 H=0.255 F=0.592)
multi_fold_convergence: PASS (3/3 folds produce best-checkpoint;
fold Sharpes +57.8 / +55.6 / +119.3)
grad_ratio_mag_dir trajectory (mag_dist smoke, first fold, first 5
epochs) — pre-fix values from /tmp/l40s_diag/health.log (L40S
train-mdh86):
pre-fix: 14793, 11934, 12858, 16406, 15141 (×1000 regime)
post-fix: 55, 78, 35, 11, 10 (×10-100 regime)
Three+ orders of magnitude reduction. The residual ratio can still
exceed `num_branches = 4` when the direction branch's norm sits below
the median — the cap bounds each branch's absolute norm (≤ 4×median),
not the pairwise ratio, by design (direction-outlier smallness is a
separate pathology that would be masked by a ratio bound).
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
193 lines
6.4 KiB
Rust
193 lines
6.4 KiB
Rust
use std::path::{Path, PathBuf};
|
|
use std::process::Command;
|
|
|
|
fn main() {
|
|
println!("cargo:rerun-if-changed=build.rs");
|
|
|
|
// Only compile CUDA kernels when the cuda feature is enabled
|
|
if std::env::var("CARGO_FEATURE_CUDA").is_err() {
|
|
return;
|
|
}
|
|
|
|
let out_dir = PathBuf::from(std::env::var("OUT_DIR").unwrap());
|
|
let kernel_dir = Path::new("src/cuda_pipeline");
|
|
let common_header = kernel_dir.join("common_device_functions.cuh");
|
|
let trade_physics_header = kernel_dir.join("trade_physics.cuh");
|
|
|
|
// Rebuild cubins when shared headers change
|
|
println!("cargo:rerun-if-changed={}", trade_physics_header.display());
|
|
|
|
// Detect GPU architecture from env or default to sm_80
|
|
let cuda_compute_cap = std::env::var("CUDA_COMPUTE_CAP").unwrap_or_else(|_| "80".to_string());
|
|
let arch = format!("sm_{cuda_compute_cap}");
|
|
|
|
// Check if nvcc is available -- gracefully skip if not (non-CUDA builds)
|
|
let nvcc_path = find_nvcc();
|
|
let nvcc = match nvcc_path {
|
|
Some(p) => p,
|
|
None => {
|
|
eprintln!(" warning: nvcc not found, skipping CUDA kernel precompilation");
|
|
eprintln!(" Install CUDA toolkit or set CUDA_HOME for GPU builds");
|
|
return;
|
|
}
|
|
};
|
|
|
|
// Read common header once
|
|
let common_src = std::fs::read_to_string(&common_header)
|
|
.unwrap_or_else(|e| panic!("Failed to read {}: {e}", common_header.display()));
|
|
println!("cargo:rerun-if-changed={}", common_header.display());
|
|
|
|
// All kernels to precompile.
|
|
// Kernels marked "standalone" have their own helpers and don't need common header.
|
|
// All others get common_device_functions.cuh prepended.
|
|
let kernels_with_common = [
|
|
// Original 24 kernels
|
|
"epsilon_greedy_kernel.cu",
|
|
"backtest_env_kernel.cu",
|
|
"backtest_forward_ppo_kernel.cu",
|
|
"backtest_forward_supervised_kernel.cu",
|
|
"backtest_metrics_kernel.cu",
|
|
"dt_kernels.cu",
|
|
"ensemble_kernels.cu",
|
|
"her_episode_kernel.cu",
|
|
"her_relabel_kernel.cu",
|
|
"signal_adapter_kernel.cu",
|
|
"statistics_kernel.cu",
|
|
"training_guard_kernel.cu",
|
|
"c51_loss_kernel.cu",
|
|
"mse_loss_kernel.cu",
|
|
"curiosity_training_kernel.cu",
|
|
"curiosity_inference_kernel.cu",
|
|
"dqn_utility_kernels.cu",
|
|
"attention_kernel.cu",
|
|
"attention_backward_kernel.cu",
|
|
"iql_value_kernel.cu",
|
|
"iqn_dual_head_kernel.cu",
|
|
"monitoring_kernel.cu",
|
|
"nstep_kernel.cu",
|
|
"reward_shaping_kernel.cu",
|
|
"ppo_experience_kernel.cu",
|
|
"bias_kernels.cu",
|
|
// Formerly standalone — now need common header for BF16 types
|
|
"experience_kernels.cu",
|
|
"ema_kernel.cu",
|
|
"relu_mask_kernel.cu",
|
|
"c51_grad_kernel.cu",
|
|
"mse_grad_kernel.cu",
|
|
"q_stats_kernel.cu",
|
|
"cql_grad_kernel.cu",
|
|
"trade_stats_kernel.cu",
|
|
"backward_kernels.cu",
|
|
"iqn_cvar_kernel.cu",
|
|
"mamba2_temporal_kernel.cu",
|
|
"graph_utility_kernels.cu",
|
|
"grad_decomp_kernel.cu",
|
|
"branch_grad_balance_kernel.cu",
|
|
];
|
|
|
|
// ALL kernels get common header (BF16 types + wrappers)
|
|
let mut failed: Vec<&str> = Vec::new();
|
|
for kernel_name in &kernels_with_common {
|
|
if !try_compile_kernel(
|
|
&nvcc, kernel_dir, kernel_name, &arch, &out_dir, Some(&common_src),
|
|
) {
|
|
failed.push(kernel_name);
|
|
}
|
|
}
|
|
let passed = kernels_with_common.len() - failed.len();
|
|
eprintln!(" Precompiled {passed}/{} CUDA kernels ({arch}) — f32/TF32",
|
|
kernels_with_common.len());
|
|
if !failed.is_empty() {
|
|
eprintln!(" FAILED: {}", failed.join(", "));
|
|
panic!("nvcc failed to compile {} kernel(s): {}", failed.len(), failed.join(", "));
|
|
}
|
|
}
|
|
|
|
/// Compile a single .cu kernel file to a .cubin via nvcc.
|
|
///
|
|
/// If `common_header` is Some, it is prepended to the kernel source.
|
|
fn try_compile_kernel(
|
|
nvcc: &Path,
|
|
kernel_dir: &Path,
|
|
kernel_name: &str,
|
|
arch: &str,
|
|
out_dir: &Path,
|
|
common_header: Option<&str>,
|
|
) -> bool {
|
|
let kernel_path = kernel_dir.join(kernel_name);
|
|
let cubin_name = kernel_name.replace(".cu", ".cubin");
|
|
let cubin_path = out_dir.join(&cubin_name);
|
|
|
|
println!("cargo:rerun-if-changed={}", kernel_path.display());
|
|
|
|
let kernel_src = std::fs::read_to_string(&kernel_path)
|
|
.unwrap_or_else(|e| panic!("Failed to read {}: {e}", kernel_path.display()));
|
|
|
|
// Compose source: optional common header + kernel
|
|
let full_source = match common_header {
|
|
Some(header) => format!("{header}\n{kernel_src}"),
|
|
None => kernel_src,
|
|
};
|
|
|
|
// Write composed source to temp file
|
|
let tmp_src = out_dir.join(format!("_{kernel_name}"));
|
|
std::fs::write(&tmp_src, &full_source).unwrap();
|
|
|
|
// Compile with nvcc (-I kernel_dir for shared .cuh includes)
|
|
let include_flag = format!("-I{}", kernel_dir.display());
|
|
let status = Command::new(nvcc)
|
|
.args([
|
|
"-cubin",
|
|
&format!("-arch={arch}"),
|
|
"-O3",
|
|
"--ftz=true",
|
|
"--fmad=true",
|
|
"--prec-div=true",
|
|
"--prec-sqrt=true",
|
|
&include_flag,
|
|
"-o", cubin_path.to_str().unwrap(),
|
|
tmp_src.to_str().unwrap(),
|
|
])
|
|
.status();
|
|
|
|
match status {
|
|
Ok(s) if s.success() => {
|
|
eprintln!(" Compiled {kernel_name} -> {cubin_name} ({arch})");
|
|
true
|
|
}
|
|
Ok(s) => {
|
|
eprintln!(" FAILED: {kernel_name} (exit={})", s.code().unwrap_or(-1));
|
|
false
|
|
}
|
|
Err(e) => {
|
|
eprintln!(" FAILED: {kernel_name} (nvcc error: {e})");
|
|
false
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Find nvcc: prefer $CUDA_HOME/bin/nvcc, then check PATH
|
|
fn find_nvcc() -> Option<PathBuf> {
|
|
// Try $CUDA_HOME/bin/nvcc first
|
|
if let Ok(home) = std::env::var("CUDA_HOME") {
|
|
let nvcc = PathBuf::from(home).join("bin/nvcc");
|
|
if nvcc.exists() {
|
|
return Some(nvcc);
|
|
}
|
|
}
|
|
|
|
// Try common CUDA paths
|
|
for path in &["/usr/local/cuda/bin/nvcc", "/usr/bin/nvcc"] {
|
|
let p = PathBuf::from(path);
|
|
if p.exists() {
|
|
return Some(p);
|
|
}
|
|
}
|
|
|
|
// Check if nvcc is in PATH
|
|
match Command::new("nvcc").arg("--version").output() {
|
|
Ok(output) if output.status.success() => Some(PathBuf::from("nvcc")),
|
|
_ => None,
|
|
}
|
|
}
|