Files
foxhunt/crates/ml/build.rs
jgrusewski 6cb2257163 fix(dqn): adaptive per-branch gradient-norm balancer
Caps any branch's weight-gradient L2 norm at num_branches × median
(median across the 4 branches' norms). Scales the offending branch's
gradient down to the cap; healthy branches pass through unchanged.

Fixes the observed pathology in L40S train-mdh86: grad_ratio_mag_dir
was 15k–26k× for 6 consecutive epochs, then collapsed to ~100× in a
single step at epoch 7 and destabilised learning (Sharpe flipped +34
→ -67, never recovered). Symmetric per-branch capping at
`num_branches × median` prevents the swing at both ends without
requiring a global ratio bound.

No tuned knobs: `num_branches = 4` is architectural (factored action
space: direction × magnitude × order × urgency), `median_branch_norm`
is a per-step statistical reference that tracks the current gradient
regime, and the product is fully adaptive. Per
feedback_adaptive_not_tuned.md, the only static value is the
architectural axis count; medians and derived caps are signal-driven.

Implementation — two CUDA kernel launches in
`branch_grad_balance_kernel.cu`:

  branch_grad_norm_reduce:  grid=(4,1,1), block=(256,1,1). One block
                            per branch; sum-of-squares via shared-mem
                            tree reduce writes `branch_norms_dev[4]`.
                            No atomicAdd (one-block-per-branch, single
                            writer per slot).

  branch_grad_rescale:      grid=(max_blocks, 4, 1), block=(256,1,1).
                            Each block caches the 4 branch norms into
                            shared memory, computes the median via a
                            5-comparator sorting network + two-element
                            average (branch-deterministic, no reduction
                            primitive), derives the 4 per-branch scales
                            `scale[d] = min(1, 4×median/norm[d])`, then
                            threads multiply their slice element by the
                            owning branch's scale. No atomicAdd (each
                            thread writes one distinct element).

Insertion point: inside the `adam_grad_child` graph between the aux
phase and `compute_grad_norm_for_adam`, so Adam's global clip and the
Adam update both observe the rebalanced gradient. Also wired into the
ungraphed fallback paths so no code path can skip the cap. The kernels
have fixed launch configs, no host syncs, no dynamic allocations —
safe to capture.

Per-branch slice metadata (starts/lens for each of the 4 contiguous
4-tensor branch slices in `grad_buf`) is precomputed from
`compute_param_sizes` at trainer construction and uploaded once to
device i32 buffers, matching the existing `grad_decomp_kernel` layout
convention.

Smoke tests (local RTX 3050 Ti, 4 GB):
  magnitude_distribution:  PASS  (MAG_DIST Q=0.637 H=0.114 F=0.249,
                                  EVAL_DIST Q=0.153 H=0.255 F=0.592)
  multi_fold_convergence:  PASS  (3/3 folds produce best-checkpoint;
                                  fold Sharpes +57.8 / +55.6 / +119.3)

grad_ratio_mag_dir trajectory (mag_dist smoke, first fold, first 5
epochs) — pre-fix values from /tmp/l40s_diag/health.log (L40S
train-mdh86):

  pre-fix:  14793, 11934, 12858, 16406, 15141  (×1000 regime)
  post-fix:    55,    78,    35,    11,    10  (×10-100 regime)

Three+ orders of magnitude reduction. The residual ratio can still
exceed `num_branches = 4` when the direction branch's norm sits below
the median — the cap bounds each branch's absolute norm (≤ 4×median),
not the pairwise ratio, by design (direction-outlier smallness is a
separate pathology that would be masked by a ratio bound).

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-04-23 09:16:11 +02:00

193 lines
6.4 KiB
Rust

use std::path::{Path, PathBuf};
use std::process::Command;
fn main() {
println!("cargo:rerun-if-changed=build.rs");
// Only compile CUDA kernels when the cuda feature is enabled
if std::env::var("CARGO_FEATURE_CUDA").is_err() {
return;
}
let out_dir = PathBuf::from(std::env::var("OUT_DIR").unwrap());
let kernel_dir = Path::new("src/cuda_pipeline");
let common_header = kernel_dir.join("common_device_functions.cuh");
let trade_physics_header = kernel_dir.join("trade_physics.cuh");
// Rebuild cubins when shared headers change
println!("cargo:rerun-if-changed={}", trade_physics_header.display());
// Detect GPU architecture from env or default to sm_80
let cuda_compute_cap = std::env::var("CUDA_COMPUTE_CAP").unwrap_or_else(|_| "80".to_string());
let arch = format!("sm_{cuda_compute_cap}");
// Check if nvcc is available -- gracefully skip if not (non-CUDA builds)
let nvcc_path = find_nvcc();
let nvcc = match nvcc_path {
Some(p) => p,
None => {
eprintln!(" warning: nvcc not found, skipping CUDA kernel precompilation");
eprintln!(" Install CUDA toolkit or set CUDA_HOME for GPU builds");
return;
}
};
// Read common header once
let common_src = std::fs::read_to_string(&common_header)
.unwrap_or_else(|e| panic!("Failed to read {}: {e}", common_header.display()));
println!("cargo:rerun-if-changed={}", common_header.display());
// All kernels to precompile.
// Kernels marked "standalone" have their own helpers and don't need common header.
// All others get common_device_functions.cuh prepended.
let kernels_with_common = [
// Original 24 kernels
"epsilon_greedy_kernel.cu",
"backtest_env_kernel.cu",
"backtest_forward_ppo_kernel.cu",
"backtest_forward_supervised_kernel.cu",
"backtest_metrics_kernel.cu",
"dt_kernels.cu",
"ensemble_kernels.cu",
"her_episode_kernel.cu",
"her_relabel_kernel.cu",
"signal_adapter_kernel.cu",
"statistics_kernel.cu",
"training_guard_kernel.cu",
"c51_loss_kernel.cu",
"mse_loss_kernel.cu",
"curiosity_training_kernel.cu",
"curiosity_inference_kernel.cu",
"dqn_utility_kernels.cu",
"attention_kernel.cu",
"attention_backward_kernel.cu",
"iql_value_kernel.cu",
"iqn_dual_head_kernel.cu",
"monitoring_kernel.cu",
"nstep_kernel.cu",
"reward_shaping_kernel.cu",
"ppo_experience_kernel.cu",
"bias_kernels.cu",
// Formerly standalone — now need common header for BF16 types
"experience_kernels.cu",
"ema_kernel.cu",
"relu_mask_kernel.cu",
"c51_grad_kernel.cu",
"mse_grad_kernel.cu",
"q_stats_kernel.cu",
"cql_grad_kernel.cu",
"trade_stats_kernel.cu",
"backward_kernels.cu",
"iqn_cvar_kernel.cu",
"mamba2_temporal_kernel.cu",
"graph_utility_kernels.cu",
"grad_decomp_kernel.cu",
"branch_grad_balance_kernel.cu",
];
// ALL kernels get common header (BF16 types + wrappers)
let mut failed: Vec<&str> = Vec::new();
for kernel_name in &kernels_with_common {
if !try_compile_kernel(
&nvcc, kernel_dir, kernel_name, &arch, &out_dir, Some(&common_src),
) {
failed.push(kernel_name);
}
}
let passed = kernels_with_common.len() - failed.len();
eprintln!(" Precompiled {passed}/{} CUDA kernels ({arch}) — f32/TF32",
kernels_with_common.len());
if !failed.is_empty() {
eprintln!(" FAILED: {}", failed.join(", "));
panic!("nvcc failed to compile {} kernel(s): {}", failed.len(), failed.join(", "));
}
}
/// Compile a single .cu kernel file to a .cubin via nvcc.
///
/// If `common_header` is Some, it is prepended to the kernel source.
fn try_compile_kernel(
nvcc: &Path,
kernel_dir: &Path,
kernel_name: &str,
arch: &str,
out_dir: &Path,
common_header: Option<&str>,
) -> bool {
let kernel_path = kernel_dir.join(kernel_name);
let cubin_name = kernel_name.replace(".cu", ".cubin");
let cubin_path = out_dir.join(&cubin_name);
println!("cargo:rerun-if-changed={}", kernel_path.display());
let kernel_src = std::fs::read_to_string(&kernel_path)
.unwrap_or_else(|e| panic!("Failed to read {}: {e}", kernel_path.display()));
// Compose source: optional common header + kernel
let full_source = match common_header {
Some(header) => format!("{header}\n{kernel_src}"),
None => kernel_src,
};
// Write composed source to temp file
let tmp_src = out_dir.join(format!("_{kernel_name}"));
std::fs::write(&tmp_src, &full_source).unwrap();
// Compile with nvcc (-I kernel_dir for shared .cuh includes)
let include_flag = format!("-I{}", kernel_dir.display());
let status = Command::new(nvcc)
.args([
"-cubin",
&format!("-arch={arch}"),
"-O3",
"--ftz=true",
"--fmad=true",
"--prec-div=true",
"--prec-sqrt=true",
&include_flag,
"-o", cubin_path.to_str().unwrap(),
tmp_src.to_str().unwrap(),
])
.status();
match status {
Ok(s) if s.success() => {
eprintln!(" Compiled {kernel_name} -> {cubin_name} ({arch})");
true
}
Ok(s) => {
eprintln!(" FAILED: {kernel_name} (exit={})", s.code().unwrap_or(-1));
false
}
Err(e) => {
eprintln!(" FAILED: {kernel_name} (nvcc error: {e})");
false
}
}
}
/// Find nvcc: prefer $CUDA_HOME/bin/nvcc, then check PATH
fn find_nvcc() -> Option<PathBuf> {
// Try $CUDA_HOME/bin/nvcc first
if let Ok(home) = std::env::var("CUDA_HOME") {
let nvcc = PathBuf::from(home).join("bin/nvcc");
if nvcc.exists() {
return Some(nvcc);
}
}
// Try common CUDA paths
for path in &["/usr/local/cuda/bin/nvcc", "/usr/bin/nvcc"] {
let p = PathBuf::from(path);
if p.exists() {
return Some(p);
}
}
// Check if nvcc is in PATH
match Command::new("nvcc").arg("--version").output() {
Ok(output) if output.status.success() => Some(PathBuf::from("nvcc")),
_ => None,
}
}