diff --git a/crates/ml/src/cuda_pipeline/gpu_tlob.rs b/crates/ml/src/cuda_pipeline/gpu_tlob.rs index 51300778f..b7a895494 100644 --- a/crates/ml/src/cuda_pipeline/gpu_tlob.rs +++ b/crates/ml/src/cuda_pipeline/gpu_tlob.rs @@ -229,8 +229,8 @@ impl GpuTlob { .map_err(|e| MLError::ModelError(format!("attn_grad_norm_phase1 load: {e}")))?; let grad_norm_phase2_kernel = bwd_module.load_function("attn_grad_norm_phase2") .map_err(|e| MLError::ModelError(format!("attn_grad_norm_phase2 load: {e}")))?; - let adam_kernel = bwd_module.load_function("attn_adam_update") - .map_err(|e| MLError::ModelError(format!("attn_adam_update load: {e}")))?; + let adam_kernel = bwd_module.load_function("attn_adam_kernel") + .map_err(|e| MLError::ModelError(format!("attn_adam_kernel load: {e}")))?; let bias_grad_reduce_p1_kernel = bwd_module.load_function("attn_bias_grad_reduce_p1") .map_err(|e| MLError::ModelError(format!("attn_bias_grad_reduce_p1 load: {e}")))?; let bias_grad_reduce_p2_kernel = bwd_module.load_function("attn_bias_grad_reduce_p2") diff --git a/docs/dqn-wire-up-audit.md b/docs/dqn-wire-up-audit.md index 0c486ef92..976f1b473 100644 --- a/docs/dqn-wire-up-audit.md +++ b/docs/dqn-wire-up-audit.md @@ -2,6 +2,23 @@ **Status:** Populated during Plan 1 Task 6 (A.5 orphan audit). Updated on every commit per Invariant 7. +TLOB symbol-rename fix (2026-04-28): `gpu_tlob.rs:232` was loading +`attn_adam_update` from `attention_backward_kernel.cubin`, but the +kernel is defined as `attn_adam_kernel` +(`attention_backward_kernel.cu:70`); `gpu_attention.rs:281` already +uses the correct symbol. Mismatch caused `GpuTlob::new` to fail with +"named symbol not found" on every workflow init, suppressed by the +"training continues without TLOB" graceful-degrade warning. Per +`feedback_no_hiding.md`, graceful-degraded modules ARE the ghost-feature +pattern; the warning was hiding a real wire-up bug. Renamed the symbol +to match. With the symbol now loading, a separate cuBLAS-Lt heuristic +failure surfaced (`tlob heuristic (fwd_q): no algo found`) — TLOB GEMM +shape `M=16, K=32, N=batch` is too narrow for cuBLAS-Lt's heuristic +search. Tracked as a follow-up; TLOB remains graceful-degraded until +that's resolved (either pad TLOB dims to alignment-friendly sizes, fall +back to classic cuBLAS sgemm for these shapes, or write a custom small-M +kernel). + adaptive MoE load-balance λ controller (2026-04-27): replaces static `moe_lambda=0.01` with a GPU-driven controller that scales λ inversely with observed gate-entropy. New kernel