rename(ml-alpha): MultiHorizonAttention (no version suffix in identifiers)
Removes the version suffix from the trainer-state bundle introduced
in 5aad1eb84:
perception_v2_state.rs → multi_horizon_attention.rs
PerceptionV2State → MultiHorizonAttention
V2_HIDDEN_DIM → MHA_HIDDEN_DIM
V2_N_HORIZONS → MHA_N_HORIZONS
V2_N_EXPERTS → MHA_N_EXPERTS
V2_REGIME_DIM → MHA_REGIME_DIM
Per the new memory pearl
feedback_single_source_of_truth_no_duplicates: source identifiers
never carry version suffixes — pick the proper conceptual name.
Compile-clean. The actual integration (replace legacy attention_pool
with this bundle as the SINGLE attention path) is the next commit;
this rename eliminates the naming violation in isolation so the
integration commit can focus on the structural replacement.
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
@@ -6,4 +6,4 @@ pub mod loss;
|
||||
pub mod loss_sigma;
|
||||
pub mod optim;
|
||||
pub mod perception;
|
||||
pub mod perception_v2_state;
|
||||
pub mod multi_horizon_attention;
|
||||
|
||||
@@ -1,12 +1,15 @@
|
||||
//! PerceptionV2State — bundles the v2 axis-A/B/C/D/E components into
|
||||
//! one cohesive trainer field. (V9 commit.)
|
||||
//! MultiHorizonAttention — single source of truth for the ml-alpha
|
||||
//! attention path. Replaces the legacy single-Q `attention_pool`
|
||||
//! entirely.
|
||||
//!
|
||||
//! Owns:
|
||||
//! - horizon_tokens (axis C) + their init snapshot for anchor (axis B)
|
||||
//! - inv-attention saved attn buffer (axis E)
|
||||
//! - MoE experts + gate (axis D)
|
||||
//! - Kendall σ logarithm (axis A)
|
||||
//! - Anchor controller (axis B Wiener-α)
|
||||
//! - horizon_tokens + their init snapshot (for L2 anchor)
|
||||
//! - shared Q + its init snapshot
|
||||
//! - horizon-token attention pool kernel binding
|
||||
//! - inverted (cross-variate) attention pool + saved buffers
|
||||
//! - regime-MoE gate + expert weights + init snapshots
|
||||
//! - Kendall σ logarithm (`log_sigma_h`) for the multi-horizon BCE
|
||||
//! - Anchor L2 kernel + the Wiener-α `AnchorController`
|
||||
//!
|
||||
//! All kernel calls go through the bound modules; nothing in this
|
||||
//! file does host-side compute or sync inside captured regions.
|
||||
@@ -27,12 +30,12 @@ use crate::regime_moe_gate::{RegimeMoeGate, MOE_N_EXPERTS};
|
||||
use crate::trainer::anchor_controller::AnchorController;
|
||||
use crate::trainer::optim::AdamW;
|
||||
|
||||
pub const V2_HIDDEN_DIM: usize = HTAP_HIDDEN_DIM;
|
||||
pub const V2_N_HORIZONS: usize = HTAP_N_HORIZONS;
|
||||
pub const V2_N_EXPERTS: usize = MOE_N_EXPERTS;
|
||||
pub const V2_REGIME_DIM: usize = 12; // spread(5) + vol(3) + tod(4); see spec §3.3
|
||||
pub const MHA_HIDDEN_DIM: usize = HTAP_HIDDEN_DIM;
|
||||
pub const MHA_N_HORIZONS: usize = HTAP_N_HORIZONS;
|
||||
pub const MHA_N_EXPERTS: usize = MOE_N_EXPERTS;
|
||||
pub const MHA_REGIME_DIM: usize = 12; // spread(5) + vol(3) + tod(4)
|
||||
|
||||
pub struct PerceptionV2State {
|
||||
pub struct MultiHorizonAttention {
|
||||
// ── (C) horizon-token attention pool ──
|
||||
pub pool: HorizonTokenAttentionPool,
|
||||
pub horizon_tokens_d: CudaSlice<f32>, // [N_H, H] learnable
|
||||
@@ -98,7 +101,7 @@ pub struct PerceptionV2State {
|
||||
stream: Arc<CudaStream>,
|
||||
}
|
||||
|
||||
impl PerceptionV2State {
|
||||
impl MultiHorizonAttention {
|
||||
pub fn new(dev: &MlDevice, n_batch: usize, k_seq: usize, lr: f32, seed: u64) -> Result<Self> {
|
||||
anyhow::ensure!(n_batch > 0 && k_seq > 0, "n_batch and k_seq must be > 0");
|
||||
let stream = dev.cuda_stream().context("v2 stream")?.clone();
|
||||
@@ -113,15 +116,15 @@ impl PerceptionV2State {
|
||||
let anchor_l2 = AnchorL2::new(ctx, stream.clone()).context("AnchorL2")?;
|
||||
|
||||
let mut rng = ChaCha8Rng::seed_from_u64(seed);
|
||||
let scale = (1.0_f32 / V2_HIDDEN_DIM as f32).sqrt();
|
||||
let scale = (1.0_f32 / MHA_HIDDEN_DIM as f32).sqrt();
|
||||
|
||||
let init_horizon: Vec<f32> = (0..V2_N_HORIZONS * V2_HIDDEN_DIM)
|
||||
let init_horizon: Vec<f32> = (0..MHA_N_HORIZONS * MHA_HIDDEN_DIM)
|
||||
.map(|_| rng.gen_range(-scale..scale)).collect();
|
||||
let init_q: Vec<f32> = (0..V2_HIDDEN_DIM)
|
||||
let init_q: Vec<f32> = (0..MHA_HIDDEN_DIM)
|
||||
.map(|_| rng.gen_range(-scale..scale)).collect();
|
||||
let init_w_gate: Vec<f32> = (0..V2_N_EXPERTS * V2_REGIME_DIM)
|
||||
let init_w_gate: Vec<f32> = (0..MHA_N_EXPERTS * MHA_REGIME_DIM)
|
||||
.map(|_| rng.gen_range(-0.1..0.1)).collect();
|
||||
let init_experts_w: Vec<f32> = (0..V2_N_EXPERTS * V2_HIDDEN_DIM * V2_HIDDEN_DIM)
|
||||
let init_experts_w: Vec<f32> = (0..MHA_N_EXPERTS * MHA_HIDDEN_DIM * MHA_HIDDEN_DIM)
|
||||
.map(|_| rng.gen_range(-scale..scale)).collect();
|
||||
|
||||
let horizon_tokens_d = upload(&stream, &init_horizon)?;
|
||||
@@ -130,11 +133,11 @@ impl PerceptionV2State {
|
||||
let q_init_d = upload(&stream, &init_q)?;
|
||||
let w_gate_d = upload(&stream, &init_w_gate)?;
|
||||
let experts_w_d = upload(&stream, &init_experts_w)?;
|
||||
let experts_b_d = stream.alloc_zeros::<f32>(V2_N_EXPERTS * V2_HIDDEN_DIM)?;
|
||||
let experts_b_d = stream.alloc_zeros::<f32>(MHA_N_EXPERTS * MHA_HIDDEN_DIM)?;
|
||||
let experts_w_init_d = upload(&stream, &init_experts_w)?;
|
||||
let experts_b_init_d = stream.alloc_zeros::<f32>(V2_N_EXPERTS * V2_HIDDEN_DIM)?;
|
||||
let experts_b_init_d = stream.alloc_zeros::<f32>(MHA_N_EXPERTS * MHA_HIDDEN_DIM)?;
|
||||
|
||||
let n_ext = V2_N_HORIZONS + k_seq;
|
||||
let n_ext = MHA_N_HORIZONS + k_seq;
|
||||
|
||||
let s = &stream;
|
||||
let state = Self {
|
||||
@@ -143,44 +146,44 @@ impl PerceptionV2State {
|
||||
horizon_tokens_init_d,
|
||||
q_d,
|
||||
q_init_d,
|
||||
grad_horizon_tokens_d: s.alloc_zeros::<f32>(V2_N_HORIZONS * V2_HIDDEN_DIM)?,
|
||||
grad_horizon_tokens_scratch_d:s.alloc_zeros::<f32>(n_batch * V2_N_HORIZONS * V2_HIDDEN_DIM)?,
|
||||
grad_q_d: s.alloc_zeros::<f32>(V2_HIDDEN_DIM)?,
|
||||
grad_q_scratch_d: s.alloc_zeros::<f32>(n_batch * V2_HIDDEN_DIM)?,
|
||||
ctx_h_d: s.alloc_zeros::<f32>(n_batch * V2_N_HORIZONS * V2_HIDDEN_DIM)?,
|
||||
grad_horizon_tokens_d: s.alloc_zeros::<f32>(MHA_N_HORIZONS * MHA_HIDDEN_DIM)?,
|
||||
grad_horizon_tokens_scratch_d:s.alloc_zeros::<f32>(n_batch * MHA_N_HORIZONS * MHA_HIDDEN_DIM)?,
|
||||
grad_q_d: s.alloc_zeros::<f32>(MHA_HIDDEN_DIM)?,
|
||||
grad_q_scratch_d: s.alloc_zeros::<f32>(n_batch * MHA_HIDDEN_DIM)?,
|
||||
ctx_h_d: s.alloc_zeros::<f32>(n_batch * MHA_N_HORIZONS * MHA_HIDDEN_DIM)?,
|
||||
attn_h_d: s.alloc_zeros::<f32>(n_batch * n_ext)?,
|
||||
opt_horizon_tokens: AdamW::new(dev, V2_N_HORIZONS * V2_HIDDEN_DIM, lr)?,
|
||||
opt_q: AdamW::new(dev, V2_HIDDEN_DIM, lr)?,
|
||||
opt_horizon_tokens: AdamW::new(dev, MHA_N_HORIZONS * MHA_HIDDEN_DIM, lr)?,
|
||||
opt_q: AdamW::new(dev, MHA_HIDDEN_DIM, lr)?,
|
||||
|
||||
inv_pool,
|
||||
inv_pooled_d: s.alloc_zeros::<f32>(n_batch * V2_HIDDEN_DIM)?,
|
||||
inv_attn_d: s.alloc_zeros::<f32>(n_batch * V2_HIDDEN_DIM * V2_HIDDEN_DIM)?,
|
||||
inv_d_scores_scratch_d: s.alloc_zeros::<f32>(n_batch * V2_HIDDEN_DIM * V2_HIDDEN_DIM)?,
|
||||
inv_pooled_d: s.alloc_zeros::<f32>(n_batch * MHA_HIDDEN_DIM)?,
|
||||
inv_attn_d: s.alloc_zeros::<f32>(n_batch * MHA_HIDDEN_DIM * MHA_HIDDEN_DIM)?,
|
||||
inv_d_scores_scratch_d: s.alloc_zeros::<f32>(n_batch * MHA_HIDDEN_DIM * MHA_HIDDEN_DIM)?,
|
||||
|
||||
moe,
|
||||
gate_logits_d: s.alloc_zeros::<f32>(n_batch * V2_N_EXPERTS)?,
|
||||
gate_logits_d: s.alloc_zeros::<f32>(n_batch * MHA_N_EXPERTS)?,
|
||||
w_gate_d,
|
||||
experts_w_d,
|
||||
experts_b_d,
|
||||
experts_w_init_d,
|
||||
experts_b_init_d,
|
||||
top_e_d: s.alloc_zeros::<i32>(n_batch)?,
|
||||
gate_probs_d: s.alloc_zeros::<f32>(n_batch * V2_N_EXPERTS)?,
|
||||
gate_probs_d: s.alloc_zeros::<f32>(n_batch * MHA_N_EXPERTS)?,
|
||||
aux_loss_d: s.alloc_zeros::<f32>(1)?,
|
||||
routed_ctx_d: s.alloc_zeros::<f32>(n_batch * V2_N_HORIZONS * V2_HIDDEN_DIM)?,
|
||||
grad_routed_d: s.alloc_zeros::<f32>(n_batch * V2_N_HORIZONS * V2_HIDDEN_DIM)?,
|
||||
grad_w_gate_d: s.alloc_zeros::<f32>(V2_N_EXPERTS * V2_REGIME_DIM)?,
|
||||
grad_experts_w_d: s.alloc_zeros::<f32>(V2_N_EXPERTS * V2_HIDDEN_DIM * V2_HIDDEN_DIM)?,
|
||||
grad_experts_b_d: s.alloc_zeros::<f32>(V2_N_EXPERTS * V2_HIDDEN_DIM)?,
|
||||
grad_w_scratch_d: s.alloc_zeros::<f32>(n_batch * V2_N_HORIZONS * V2_N_EXPERTS * V2_HIDDEN_DIM * V2_HIDDEN_DIM)?,
|
||||
grad_b_scratch_d: s.alloc_zeros::<f32>(n_batch * V2_N_HORIZONS * V2_N_EXPERTS * V2_HIDDEN_DIM)?,
|
||||
opt_w_gate: AdamW::new(dev, V2_N_EXPERTS * V2_REGIME_DIM, lr)?,
|
||||
opt_experts_w: AdamW::new(dev, V2_N_EXPERTS * V2_HIDDEN_DIM * V2_HIDDEN_DIM, lr)?,
|
||||
opt_experts_b: AdamW::new(dev, V2_N_EXPERTS * V2_HIDDEN_DIM, lr)?,
|
||||
routed_ctx_d: s.alloc_zeros::<f32>(n_batch * MHA_N_HORIZONS * MHA_HIDDEN_DIM)?,
|
||||
grad_routed_d: s.alloc_zeros::<f32>(n_batch * MHA_N_HORIZONS * MHA_HIDDEN_DIM)?,
|
||||
grad_w_gate_d: s.alloc_zeros::<f32>(MHA_N_EXPERTS * MHA_REGIME_DIM)?,
|
||||
grad_experts_w_d: s.alloc_zeros::<f32>(MHA_N_EXPERTS * MHA_HIDDEN_DIM * MHA_HIDDEN_DIM)?,
|
||||
grad_experts_b_d: s.alloc_zeros::<f32>(MHA_N_EXPERTS * MHA_HIDDEN_DIM)?,
|
||||
grad_w_scratch_d: s.alloc_zeros::<f32>(n_batch * MHA_N_HORIZONS * MHA_N_EXPERTS * MHA_HIDDEN_DIM * MHA_HIDDEN_DIM)?,
|
||||
grad_b_scratch_d: s.alloc_zeros::<f32>(n_batch * MHA_N_HORIZONS * MHA_N_EXPERTS * MHA_HIDDEN_DIM)?,
|
||||
opt_w_gate: AdamW::new(dev, MHA_N_EXPERTS * MHA_REGIME_DIM, lr)?,
|
||||
opt_experts_w: AdamW::new(dev, MHA_N_EXPERTS * MHA_HIDDEN_DIM * MHA_HIDDEN_DIM, lr)?,
|
||||
opt_experts_b: AdamW::new(dev, MHA_N_EXPERTS * MHA_HIDDEN_DIM, lr)?,
|
||||
|
||||
log_sigma_h_d: s.alloc_zeros::<f32>(V2_N_HORIZONS)?,
|
||||
grad_log_sigma_h_d: s.alloc_zeros::<f32>(V2_N_HORIZONS)?,
|
||||
opt_log_sigma: AdamW::new(dev, V2_N_HORIZONS, lr * 0.25)?, // slow update
|
||||
log_sigma_h_d: s.alloc_zeros::<f32>(MHA_N_HORIZONS)?,
|
||||
grad_log_sigma_h_d: s.alloc_zeros::<f32>(MHA_N_HORIZONS)?,
|
||||
opt_log_sigma: AdamW::new(dev, MHA_N_HORIZONS, lr * 0.25)?, // slow update
|
||||
|
||||
anchor_l2,
|
||||
anchor: AnchorController::new(
|
||||
Reference in New Issue
Block a user