ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);

ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead);

ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);

ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func, false);

ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false);

ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm, false);

// Laguna ships one shared expert and stores its size directly (routed and

// shared experts may differ), so read the size from expert_shared_feed_forward_length.

// The count is not in the config; default to 1 but read the key if present.

hparams.n_expert_shared = 1;

ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared, false);

ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, false);

if (hparams.n_ff_shexp == 0) {

// Weightless fixtures (test-llama-archs) omit this key; derive a nonzero

// size so the shared expert is still built. Real GGUFs always carry the

// exact value (routed and shared FF lengths may differ).

hparams.n_ff_shexp = hparams.n_ff_exp() * hparams.n_expert_shared;

}

// Sliding-window attention is OPTIONAL. XS.2 is hybrid (full / SWA / SWA /

// SWA repeating, period 4 starting with full); M.1 has no sliding window

// (all layers full attention). When sliding_window is absent or zero we

// leave swa_type = NONE and skip the SWA-specific per-layer-type RoPE.

hparams.n_swa = 0;

ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa, false);

if (hparams.n_swa > 0) {

hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;

load_swa_pattern(ml, 4, /*dense_first=*/true); // XS.2: FULL at il%4==0

// Per-layer-type RoPE: full layers use YaRN θ=500000 over 64 dims;

// SWA layers use default RoPE θ=10000 over 128 dims. Base load_hparams

// already reads ROPE_FREQ_BASE and ROPE_DIMENSION_COUNT into the

// non-SWA fields; we explicitly pull the SWA mirrors here.

hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train;

hparams.rope_freq_scale_train_swa = 1.0f; // SWA uses plain RoPE (no YaRN scaling); do NOT inherit full layers 1/factor

ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false);

ml.get_key(LLM_KV_ROPE_DIMENSION_COUNT_SWA, hparams.n_rot_swa, false);

}

// Default the expert gating function to SIGMOID when the key is absent

// (matches the HF reference).

if (hparams.expert_gating_func == LLAMA_EXPERT_GATING_FUNC_TYPE_NONE) {

hparams.expert_gating_func = LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID;

}

switch (hparams.n_layer()) {

case 40: type = LLM_TYPE_30B_A3B; break; // Laguna-XS.2

case 48: type = LLM_TYPE_118B_A8B; break; // Laguna-S.2

case 70: type = LLM_TYPE_230B_A10B; break; // Laguna-M.1

default: type = LLM_TYPE_UNKNOWN;

}

LLAMA_LOAD_LOCALS;

tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);

output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);

output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);

if (output == NULL) {

// tied embeddings fallback

output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED);

}

const int64_t n_ff_exp = hparams.n_ff_exp();

const int64_t n_ff_shexp = hparams.n_ff_shexp;

for (int i = 0; i < n_layer; ++i) {

auto & layer = layers[i];

// Per-layer head count — Laguna varies n_head between full and SWA

// layers (48 vs 64 in XS.2). KV head count is uniform.

const int64_t n_head_il = hparams.n_head(i);

const int64_t n_head_kv_il = hparams.n_head_kv(i);

const int64_t n_embd_q_il = n_embd_head_k * n_head_il;

const int64_t n_embd_k_il = n_embd_head_k * n_head_kv_il;

const int64_t n_embd_v_il = n_embd_head_v * n_head_kv_il;

layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);

create_tensor_qkv(layer, i, n_embd, n_embd_q_il, n_embd_k_il, n_embd_v_il, 0);

layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_q_il, n_embd}, 0);

layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {n_embd_head_k}, 0);

layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd_head_k}, 0);

// Attention output gate. XS.2 is per-head (g_proj -> n_head, one scalar

// per head broadcast over head_dim at multiply time); M.1 is per-element

// (g_proj -> n_head*head_dim, like afmoe). Detect from the stored tensor

// shape so a single arch handles both; the graph mirrors this check.

// Gate width selects per-head vs per-element. Real GGUFs always carry the

// gate tensor, so read the width from it and require EXACTLY one of the two

// valid widths -- never guess between them. Weightless fixtures

// (test-llama-archs) have no gate tensor; fall back to the per-head layout so

// the per-head reshape path is still exercised.

const int64_t n_gate_per_head = n_head_il;

const int64_t n_gate_per_elem = n_embd_head_k * n_head_il;

const ggml_tensor * gate_meta = ml.get_tensor_meta(tn(LLM_TENSOR_ATTN_GATE, "weight", i).str().c_str());

int64_t n_gate_out;

if (gate_meta != nullptr) {

n_gate_out = gate_meta->ne[1];

if (n_gate_out != n_gate_per_head && n_gate_out != n_gate_per_elem) {

GGML_ABORT("Laguna: unexpected attention gate width %lld at layer %d "

"(expected %lld per-head or %lld per-element)",

(long long) n_gate_out, i, (long long) n_gate_per_head, (long long) n_gate_per_elem);

}

} else {

n_gate_out = n_gate_per_head;

}

layer.wqkv_gate = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "weight", i), {n_embd, n_gate_out}, 0);

layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);

if ((uint32_t)i >= hparams.n_layer_dense_lead) {

// MoE layer

layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0);

layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, 0);

layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, 0);

layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, 0);

layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, 0);

// Always-on shared expert.

layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), {n_embd, n_ff_shexp}, 0);

layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", i), {n_embd, n_ff_shexp}, 0);

layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), {n_ff_shexp, n_embd}, 0);

} else {

// Dense layer (the leading n_layer_dense_lead layers)

layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0);

layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0);

layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, 0);

}

}