diff --git a/common/arg.cpp b/common/arg.cpp index 92b4e55..9480068 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -2631,7 +2631,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex ).set_env("LLAMA_ARG_LOAD_MODE")); add_opt(common_arg( {"--longhaul"}, - "stream routed Qwen3.5 MoE, Laguna, or Inkling experts through a bounded CPU or Metal cache", + "stream routed Qwen3.5 MoE, Laguna, Inkling, or Ling 3.0 experts through a bounded CPU or Metal cache", [](common_params & params) { params.load_mode = LLAMA_LOAD_MODE_LONGHAUL; } diff --git a/conversion/__init__.py b/conversion/__init__.py index 1a47b85..87167cd 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -27,6 +27,7 @@ TEXT_MODEL_MAP: dict[str, str] = { "BaichuanForCausalLM": "baichuan", "BailingMoeForCausalLM": "bailingmoe", "BailingMoeV2ForCausalLM": "bailingmoe", + "BailingMoeV3ForCausalLM": "bailingmoe", "BambaForCausalLM": "granite", "BertForMaskedLM": "bert", "BertForSequenceClassification": "bert", diff --git a/conversion/bailingmoe.py b/conversion/bailingmoe.py index 2c6425c..aa22112 100644 --- a/conversion/bailingmoe.py +++ b/conversion/bailingmoe.py @@ -1,5 +1,7 @@ from __future__ import annotations +import math + from typing import Callable, Iterable, TYPE_CHECKING import torch @@ -188,6 +190,184 @@ class BailingMoeV2Model(TextModel): raise ValueError(f"Unprocessed experts: {experts}") +@ModelBase.register("BailingMoeV3ForCausalLM") +class BailingMoeV3Model(TextModel): + """Ling 3.0 - hybrid KDA (linear) + gated MLA (full) attention with a bailing MoE""" + model_arch = gguf.MODEL_ARCH.BAILINGMOE3 + + _experts: list[dict[str, Tensor]] | None = None + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # the MTP block sits right after the last decoder layer and is not converted + self.block_count = self.hparams["num_hidden_layers"] + self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count) + + def is_mla_layer(self, il: int) -> bool: + # every layer_group_size-th layer is a full attention (MLA) layer, the rest are KDA + return (il + 1) % self.hparams["layer_group_size"] == 0 + + def set_vocab(self): + # identical tokenizer to Ling 2.0, the bailingmoe2 pre-tokenizer hash matches + self._set_vocab_gpt2() + + def set_gguf_parameters(self): + hparams = self.hparams + + # note: to enable the MLA KV cache, attention is converted into MQA (ie: GQA with 1 group) + hparams["num_key_value_heads"] = 1 + + super().set_gguf_parameters() + + self.gguf_writer.add_vocab_size(hparams["vocab_size"]) + + # n_head_kv == 0 marks a KDA (recurrent) layer, > 0 marks an MLA (attention) layer + self.gguf_writer.add_head_count_kv([ + 1 if self.is_mla_layer(il) else 0 for il in range(self.block_count) + ]) + + # KDA + self.gguf_writer.add_ssm_conv_kernel(hparams["short_conv_kernel_size"]) + self.gguf_writer.add_kda_head_dim(hparams["head_dim"]) + # safe gate: g = lower_bound * sigmoid(exp(A_log) * (f_proj(x) + dt_bias)) + assert hparams.get("kda_safe_gate", False), "only the safe gate form is implemented" + self.gguf_writer.add_kda_gate_lower_bound(hparams["kda_lower_bound"]) + + # MLA - converted into MQA with larger heads, then decompressed to MHA + kv_lora_rank = hparams["kv_lora_rank"] + qk_rope_head_dim = hparams["qk_rope_head_dim"] + self.gguf_writer.add_kv_lora_rank(kv_lora_rank) + self.gguf_writer.add_key_length(kv_lora_rank + qk_rope_head_dim) + self.gguf_writer.add_value_length(kv_lora_rank) + self.gguf_writer.add_key_length_mla(hparams["qk_nope_head_dim"] + qk_rope_head_dim) + self.gguf_writer.add_value_length_mla(hparams["v_head_dim"]) + self.gguf_writer.add_rope_dimension_count(qk_rope_head_dim) + + # MoE + self.gguf_writer.add_leading_dense_block_count(hparams["first_k_dense_replace"]) + self.gguf_writer.add_expert_feed_forward_length(hparams["moe_intermediate_size"]) + self.gguf_writer.add_expert_shared_feed_forward_length( + hparams["moe_shared_expert_intermediate_size"] * hparams["num_shared_experts"]) + self.gguf_writer.add_expert_shared_count(hparams["num_shared_experts"]) + self.gguf_writer.add_expert_weights_scale(hparams["routed_scaling_factor"]) + self.gguf_writer.add_expert_weights_norm(hparams["norm_topk_prob"]) + self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SIGMOID) + self.gguf_writer.add_expert_group_count(hparams["n_group"]) + self.gguf_writer.add_expert_group_used_count(hparams["topk_group"]) + + # Optional per-layer SwiGLU clamps (vLLM SwigluStepAndMul): + # out = silu(gate).clamp(max=limit) * up.clamp(-limit, limit) + # 0.0 (or a missing/null entry) means no clamping for that layer. + def _clamp_limits(key: str) -> list[float] | None: + if (limits := hparams.get(key)) is None: + return None + limits = [0.0 if v is None else float(v) for v in limits[:self.block_count]] + return limits + [0.0] * (self.block_count - len(limits)) + + if (limits := _clamp_limits("expert_swiglu_limit_list")) is not None: + self.gguf_writer.add_swiglu_clamp_exp(limits) + if (limits := _clamp_limits("share_expert_swiglu_limit_list")) is not None: + self.gguf_writer.add_swiglu_clamp_shexp(limits) + + if (nextn_layers := hparams.get("num_nextn_predict_layers")) is not None: + self.gguf_writer.add_nextn_predict_layers(nextn_layers) + + @classmethod + def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: + name, gen = item + + if name.endswith(".expert_bias"): + name = name.replace(".expert_bias", ".expert_bias.bias") + + return super().filter_tensors((name, gen)) + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + # drop the MTP block + if bid is not None and bid >= self.block_count: + return + + n_head = self.hparams["num_attention_heads"] + head_dim = self.hparams["head_dim"] + + if name.endswith(".A_log"): + # the safe gate uses -exp(A_log) only through exp(A_log), see the graph: + # g = kda_lower_bound * sigmoid(exp(A_log) * (f_proj(x) + dt_bias)) + # {n_head} -> ggml ne = [1, n_head] + data_torch = torch.exp(data_torch.float()) + data_torch = data_torch.reshape(-1, 1) + elif name.endswith(".dt_bias"): + name = name.rpartition(".dt_bias")[0] + ".dt_proj.bias" + elif name.endswith((".q_conv1d.weight", ".k_conv1d.weight", ".v_conv1d.weight")): + # HF {d_inner, [1,] d_conv} -> numpy (1, d_inner, 1, d_conv) -> ggml ne = [d_conv, 1, d_inner, 1] + d_conv = data_torch.shape[-1] + d_inner = math.prod(data_torch.shape[:-1]) + data_torch = data_torch.reshape(1, d_inner, 1, d_conv) + elif name.endswith("attention.g_proj.weight"): + assert bid is not None + # the very same HF name is the KDA output gate on linear layers and the + # attention output gate on MLA layers + tensor = gguf.MODEL_TENSOR.ATTN_GATE if self.is_mla_layer(bid) else gguf.MODEL_TENSOR.SSM_G + yield from super().modify_tensors(data_torch, self.format_tensor_name(tensor, bid), bid) + return + elif name.endswith("attention.kv_b_proj.weight"): + assert bid is not None + # MLA with the absorption optimization needs these two split and k_b transposed + v_head_dim = self.hparams["v_head_dim"] + qk_nope_head_dim = self.hparams["qk_nope_head_dim"] + assert data_torch.shape[0] == n_head * (v_head_dim + qk_nope_head_dim) + + kv_b = data_torch.view(n_head, qk_nope_head_dim + v_head_dim, data_torch.shape[-1]) + k_b, v_b = torch.split(kv_b, [qk_nope_head_dim, v_head_dim], dim=1) + k_b = k_b.transpose(1, 2) + + yield from super().modify_tensors(k_b, self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_K_B, bid), bid) + yield from super().modify_tensors(v_b, self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_V_B, bid), bid) + return + elif name.endswith("attention.dense.weight"): + assert bid is not None + # MLA output projection (KDA layers call it o_proj) + yield from super().modify_tensors(data_torch, self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_OUT, bid), bid) + return + elif "mlp.experts" in name: + n_experts = self.hparams["num_experts"] + assert bid is not None + + if self._experts is None: + self._experts = [{} for _ in range(self.block_count)] + + self._experts[bid][name] = data_torch + + if len(self._experts[bid]) >= n_experts * 3: + # merge the experts into a single 3d tensor + for w_name in ["down_proj", "gate_proj", "up_proj"]: + datas: list[Tensor] = [] + + for xid in range(n_experts): + ename = f"model.layers.{bid}.mlp.experts.{xid}.{w_name}.weight" + datas.append(self._experts[bid][ename]) + del self._experts[bid][ename] + + data_torch = torch.stack(datas, dim=0) + + merged_name = f"model.layers.{bid}.mlp.experts.{w_name}.weight" + + yield from super().modify_tensors(data_torch, merged_name, bid) + return + + del head_dim # only used for the asserts above + + yield from super().modify_tensors(data_torch, name, bid) + + def prepare_tensors(self): + super().prepare_tensors() + + if self._experts is not None: + experts = [k for d in self._experts for k in d.keys()] + if len(experts) > 0: + raise ValueError(f"Unprocessed experts: {experts}") + + + @ModelBase.register("SarvamMoEForCausalLM", "modeling_sarvam_moe.SarvamMoEForCausalLM") class SarvamMoEModel(BailingMoeV2Model): model_arch = gguf.MODEL_ARCH.BAILINGMOE2 diff --git a/docs/longhaul.md b/docs/longhaul.md index e211fc2..6e2efc5 100644 --- a/docs/longhaul.md +++ b/docs/longhaul.md @@ -31,7 +31,7 @@ The normal startup warmup is skipped automatically in longhaul mode. Routed expe Longhaul currently requires: - all repeating layers on CPU, or all repeating layers on Metal -- Qwen3.5 MoE, Laguna, or Inkling architecture +- Qwen3.5 MoE, Laguna, Inkling, or Ling 3.0 (`bailingmoe3`) architecture - text generation without embeddings or LoRA adapters CPU mode is available wherever the CPU backend is supported. Metal mode requires @@ -63,6 +63,15 @@ once, and independent expert slices are read concurrently where the platform supports positional reads. CPU and shared Metal buffers are populated directly; private Metal buffers use a staged fallback. +Ling 3.0 support covers its hybrid KDA/MLA transformer and sigmoid-routed MoE. +For Ling-3.0-flash, the router, score-correction bias, and shared expert remain +resident while the 512 routed experts are streamed. Each token selects eight +routed experts, so cache budgets with fewer than eight slots execute an MoE +layer in multiple stages. AtomicChat's Ling GGUFs can be loaded directly, and +the HF converter also recognizes `BailingMoeV3ForCausalLM` checkpoints. The +converter omits the auxiliary MTP tensor block because Longhaul does not support +MTP tensors. + Inkling keeps its two shared experts resident and streams only the routed 256-expert banks. Inkling-Small selects six routed experts per token. In the seven-shard Q8_0 model, one cache slot across all 40 MoE layers uses about diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 124ea28..0c3fd26 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -244,7 +244,8 @@ class Keys: DT_B_C_RMS = "{arch}.ssm.dt_b_c_rms" class KDA: - HEAD_DIM = "{arch}.kda.head_dim" + HEAD_DIM = "{arch}.kda.head_dim" + GATE_LOWER_BOUND = "{arch}.kda.gate_lower_bound" class WKV: HEAD_SIZE = "{arch}.wkv.head_size" @@ -517,6 +518,7 @@ class MODEL_ARCH(IntEnum): PLM = auto() BAILINGMOE = auto() BAILINGMOE2 = auto() + BAILINGMOE3 = auto() DOTS1 = auto() ARCEE = auto() AFMOE = auto() @@ -664,6 +666,8 @@ class MODEL_TENSOR(IntEnum): SSM_CONV1D_Q = auto() # Kimi Linear SSM_CONV1D_K = auto() # Kimi Linear SSM_CONV1D_V = auto() # Kimi Linear + SSM_F = auto() # bailingmoe3 (no_kda_lora) + SSM_G = auto() # bailingmoe3 (no_kda_lora) SSM_F_A = auto() # Kimi Linear SSM_F_B = auto() # Kimi Linear SSM_BETA = auto() # Kimi Linear qwen3.5 @@ -1128,6 +1132,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = { MODEL_ARCH.PLM: "plm", MODEL_ARCH.BAILINGMOE: "bailingmoe", MODEL_ARCH.BAILINGMOE2: "bailingmoe2", + MODEL_ARCH.BAILINGMOE3: "bailingmoe3", MODEL_ARCH.DOTS1: "dots1", MODEL_ARCH.ARCEE: "arcee", MODEL_ARCH.AFMOE: "afmoe", @@ -1274,6 +1279,8 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = { MODEL_TENSOR.SSM_CONV1D_Q: "blk.{bid}.ssm_conv1d_q", # Kimi Linear MODEL_TENSOR.SSM_CONV1D_K: "blk.{bid}.ssm_conv1d_k", # Kimi Linear MODEL_TENSOR.SSM_CONV1D_V: "blk.{bid}.ssm_conv1d_v", # Kimi Linear + MODEL_TENSOR.SSM_F: "blk.{bid}.ssm_f", # bailingmoe3 + MODEL_TENSOR.SSM_G: "blk.{bid}.ssm_g", # bailingmoe3 MODEL_TENSOR.SSM_F_A: "blk.{bid}.ssm_f_a", # Kimi Linear MODEL_TENSOR.SSM_F_B: "blk.{bid}.ssm_f_b", # Kimi Linear MODEL_TENSOR.SSM_BETA: "blk.{bid}.ssm_beta", # Kimi Linear qwen3.5 @@ -3849,6 +3856,43 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM, MODEL_TENSOR.LAYER_OUT_NORM, ], + MODEL_ARCH.BAILINGMOE3: [ + MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.OUTPUT, + MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_Q, + MODEL_TENSOR.ATTN_K, + MODEL_TENSOR.ATTN_V, + MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.ATTN_GATE, + MODEL_TENSOR.ATTN_KV_A_MQA, + MODEL_TENSOR.ATTN_KV_B, + MODEL_TENSOR.ATTN_K_B, + MODEL_TENSOR.ATTN_V_B, + MODEL_TENSOR.ATTN_KV_A_NORM, + MODEL_TENSOR.SSM_CONV1D_Q, + MODEL_TENSOR.SSM_CONV1D_K, + MODEL_TENSOR.SSM_CONV1D_V, + MODEL_TENSOR.SSM_F, + MODEL_TENSOR.SSM_G, + MODEL_TENSOR.SSM_BETA, + MODEL_TENSOR.SSM_A, + MODEL_TENSOR.SSM_DT, + MODEL_TENSOR.SSM_NORM, + MODEL_TENSOR.FFN_NORM, + MODEL_TENSOR.FFN_GATE, + MODEL_TENSOR.FFN_DOWN, + MODEL_TENSOR.FFN_UP, + MODEL_TENSOR.FFN_GATE_INP, + MODEL_TENSOR.FFN_EXP_PROBS_B, + MODEL_TENSOR.FFN_GATE_EXP, + MODEL_TENSOR.FFN_DOWN_EXP, + MODEL_TENSOR.FFN_UP_EXP, + MODEL_TENSOR.FFN_GATE_SHEXP, + MODEL_TENSOR.FFN_DOWN_SHEXP, + MODEL_TENSOR.FFN_UP_SHEXP, + ], MODEL_ARCH.DOTS1: [ MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index 3aa4f04..0e83d49 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -1087,6 +1087,9 @@ class GGUFWriter: def add_kda_head_dim(self, value: int) -> None: self.add_uint32(Keys.KDA.HEAD_DIM.format(arch=self.arch), value) + def add_kda_gate_lower_bound(self, value: float) -> None: + self.add_float32(Keys.KDA.GATE_LOWER_BOUND.format(arch=self.arch), value) + def add_tokenizer_model(self, model: str) -> None: self.add_string(Keys.Tokenizer.MODEL, model) diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py index 1e991b8..9d500ee 100644 --- a/gguf-py/gguf/tensor_mapping.py +++ b/gguf-py/gguf/tensor_mapping.py @@ -253,6 +253,7 @@ class TensorNameMap: # Attention query MODEL_TENSOR.ATTN_Q: ( + "model.layers.{bid}.attention.q_proj", # bailingmoe3 "model.layers.{bid}.self_attn.q_proj", # llama-hf nemotron olmoe olmo2 phimoe "layers.{bid}.self_attn.q_proj", # embeddinggemma "model.layers.{bid}.self_attn.q_proj_no_perm", # llama-custom @@ -273,6 +274,7 @@ class TensorNameMap: # Attention key MODEL_TENSOR.ATTN_K: ( + "model.layers.{bid}.attention.k_proj", # bailingmoe3 "model.layers.{bid}.self_attn.k_proj", # llama-hf nemotron olmoe olmo2 phimoe "layers.{bid}.self_attn.k_proj", # embeddinggemma "model.layers.{bid}.self_attn.k_proj_no_perm", # llama-custom @@ -294,6 +296,7 @@ class TensorNameMap: # Attention value MODEL_TENSOR.ATTN_V: ( + "model.layers.{bid}.attention.v_proj", # bailingmoe3 "model.layers.{bid}.self_attn.v_proj", # llama-hf nemotron olmoe olmo2 phimoe "layers.{bid}.self_attn.v_proj", # embeddinggemma "layers.{bid}.attention.wv", # llama-pth @@ -314,6 +317,7 @@ class TensorNameMap: # Attention output MODEL_TENSOR.ATTN_OUT: ( + "model.layers.{bid}.attention.o_proj", # bailingmoe3 "gpt_neox.layers.{bid}.attention.dense", # gptneox "transformer.h.{bid}.attn.c_proj", # gpt2 refact qwen jais "transformer.blocks.{bid}.attn.out_proj", # mpt @@ -825,6 +829,7 @@ class TensorNameMap: ), MODEL_TENSOR.SSM_DT: ( + "model.layers.{bid}.attention.dt_proj", # bailingmoe3 "model.layers.{bid}.dt_proj", # mamba-hf "backbone.layers.{bid}.mixer.dt_proj", # mamba "model.layers.{bid}.mamba.dt_proj", # jamba falcon-h1 granite-hybrid @@ -840,6 +845,7 @@ class TensorNameMap: ), MODEL_TENSOR.SSM_A: ( + "model.layers.{bid}.attention.A_log", # bailingmoe3 "model.layers.{bid}.A_log", # mamba-hf "backbone.layers.{bid}.mixer.A_log", # mamba "model.layers.{bid}.mamba.A_log", # jamba falcon-h1 granite-hybrid @@ -872,6 +878,7 @@ class TensorNameMap: "model.layers.{bid}.linear_attn.norm", # qwen3next "backbone.layers.{bid}.mixer.norm", # mamba2 "model.layers.{bid}.self_attn.o_norm", # kimi + "model.layers.{bid}.attention.o_norm", # bailingmoe3 ), MODEL_TENSOR.SSM_OUT: ( @@ -893,12 +900,21 @@ class TensorNameMap: # Kimi Linear KDA (using SSM_ prefix for consistency) MODEL_TENSOR.SSM_CONV1D_Q: ( "model.layers.{bid}.self_attn.q_conv1d", + "model.layers.{bid}.attention.q_conv1d", # bailingmoe3 ), MODEL_TENSOR.SSM_CONV1D_K: ( "model.layers.{bid}.self_attn.k_conv1d", + "model.layers.{bid}.attention.k_conv1d", # bailingmoe3 ), MODEL_TENSOR.SSM_CONV1D_V: ( "model.layers.{bid}.self_attn.v_conv1d", + "model.layers.{bid}.attention.v_conv1d", # bailingmoe3 + ), + MODEL_TENSOR.SSM_F: ( + "model.layers.{bid}.attention.f_proj", # bailingmoe3 + ), + MODEL_TENSOR.SSM_G: ( + "model.layers.{bid}.attention.g_proj", # bailingmoe3 ), MODEL_TENSOR.SSM_F_A: ( "model.layers.{bid}.self_attn.f_a_proj", @@ -909,6 +925,7 @@ class TensorNameMap: MODEL_TENSOR.SSM_BETA: ( "model.layers.{bid}.linear_attn.in_proj_b", # qwen3.5 "model.layers.{bid}.self_attn.b_proj", # Kimi Linear + "model.layers.{bid}.attention.b_proj", # bailingmoe3 ), MODEL_TENSOR.SSM_G_A: ( "model.layers.{bid}.self_attn.g_a_proj", @@ -1097,6 +1114,7 @@ class TensorNameMap: ), MODEL_TENSOR.ATTN_KV_A_MQA: ( + "model.layers.{bid}.attention.kv_a_proj_with_mqa", # bailingmoe3 "model.layers.{bid}.self_attn.kv_a_proj_with_mqa", # deepseek2 "layers.{bid}.attention.wkv_a_with_mqa", # mistral-large ), @@ -1121,6 +1139,7 @@ class TensorNameMap: ), MODEL_TENSOR.ATTN_KV_A_NORM: ( + "model.layers.{bid}.attention.kv_a_layernorm", # bailingmoe3 "model.layers.{bid}.self_attn.kv_a_layernorm", # deepseek2 "layers.{bid}.attention.kv_a_norm", # mistral-large ), diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index dac987f..bbda180 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -105,6 +105,7 @@ static const std::map LLM_ARCH_NAMES = { { LLM_ARCH_PLM, "plm" }, { LLM_ARCH_BAILINGMOE, "bailingmoe" }, { LLM_ARCH_BAILINGMOE2, "bailingmoe2" }, + { LLM_ARCH_BAILINGMOE3, "bailingmoe3" }, { LLM_ARCH_DOTS1, "dots1" }, { LLM_ARCH_ARCEE, "arcee" }, { LLM_ARCH_AFMOE, "afmoe" }, @@ -303,7 +304,8 @@ static const std::map LLM_KV_NAMES = { { LLM_KV_SSM_GROUP_COUNT, "%s.ssm.group_count" }, { LLM_KV_SSM_DT_B_C_RMS, "%s.ssm.dt_b_c_rms" }, - { LLM_KV_KDA_HEAD_DIM, "%s.kda.head_dim" }, + { LLM_KV_KDA_HEAD_DIM, "%s.kda.head_dim" }, + { LLM_KV_KDA_GATE_LOWER_BOUND, "%s.kda.gate_lower_bound" }, { LLM_KV_WKV_HEAD_SIZE, "%s.wkv.head_size" }, @@ -459,6 +461,8 @@ static const std::map LLM_TENSOR_NAMES = { { LLM_TENSOR_SSM_CONV1D_Q, "blk.%d.ssm_conv1d_q" }, { LLM_TENSOR_SSM_CONV1D_K, "blk.%d.ssm_conv1d_k" }, { LLM_TENSOR_SSM_CONV1D_V, "blk.%d.ssm_conv1d_v" }, + { LLM_TENSOR_SSM_F, "blk.%d.ssm_f" }, + { LLM_TENSOR_SSM_G, "blk.%d.ssm_g" }, { LLM_TENSOR_SSM_F_A, "blk.%d.ssm_f_a" }, { LLM_TENSOR_SSM_F_B, "blk.%d.ssm_f_b" }, { LLM_TENSOR_SSM_BETA, "blk.%d.ssm_beta" }, @@ -762,6 +766,8 @@ static const std::map LLM_TENSOR_INFOS = { {LLM_TENSOR_SSM_CONV1D_Q, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, {LLM_TENSOR_SSM_CONV1D_K, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, {LLM_TENSOR_SSM_CONV1D_V, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, + {LLM_TENSOR_SSM_F, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_SSM_G, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, {LLM_TENSOR_SSM_F_A, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, {LLM_TENSOR_SSM_F_B, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, {LLM_TENSOR_SSM_BETA, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, @@ -995,6 +1001,7 @@ bool llm_arch_is_hybrid(const llm_arch & arch) { case LLM_ARCH_NEMOTRON_H: case LLM_ARCH_NEMOTRON_H_MOE: case LLM_ARCH_QWEN3NEXT: + case LLM_ARCH_BAILINGMOE3: case LLM_ARCH_KIMI_LINEAR: case LLM_ARCH_QWEN35: case LLM_ARCH_QWEN35MOE: @@ -1054,6 +1061,7 @@ bool llm_arch_supports_sm_tensor(const llm_arch & arch) { case LLM_ARCH_MINIMAX_M2: case LLM_ARCH_MINIMAX_M3: case LLM_ARCH_MISTRAL4: + case LLM_ARCH_BAILINGMOE3: case LLM_ARCH_KIMI_LINEAR: case LLM_ARCH_INKLING: return false; diff --git a/src/llama-arch.h b/src/llama-arch.h index 0ba69e9..d411059 100644 --- a/src/llama-arch.h +++ b/src/llama-arch.h @@ -110,6 +110,7 @@ enum llm_arch { LLM_ARCH_PLM, LLM_ARCH_BAILINGMOE, LLM_ARCH_BAILINGMOE2, + LLM_ARCH_BAILINGMOE3, LLM_ARCH_DOTS1, LLM_ARCH_ARCEE, LLM_ARCH_AFMOE, @@ -309,6 +310,7 @@ enum llm_kv { LLM_KV_SSM_DT_B_C_RMS, LLM_KV_KDA_HEAD_DIM, + LLM_KV_KDA_GATE_LOWER_BOUND, LLM_KV_WKV_HEAD_SIZE, @@ -490,6 +492,8 @@ enum llm_tensor { LLM_TENSOR_SSM_CONV1D_Q, // kimi: Q conv1d weight LLM_TENSOR_SSM_CONV1D_K, // kimi: K conv1d weight LLM_TENSOR_SSM_CONV1D_V, // kimi: V conv1d weight + LLM_TENSOR_SSM_F, // bailingmoe3: full-rank forget gate projection + LLM_TENSOR_SSM_G, // bailingmoe3: full-rank output gate projection LLM_TENSOR_SSM_F_A, // kimi: forget gate projection A LLM_TENSOR_SSM_F_B, // kimi: forget gate projection B LLM_TENSOR_SSM_BETA, // kimi: beta mixing coefficient and qwen3.5 diff --git a/src/llama-hparams.h b/src/llama-hparams.h index 3f7338d..aececf6 100644 --- a/src/llama-hparams.h +++ b/src/llama-hparams.h @@ -174,6 +174,7 @@ struct llama_hparams { // for Kimi Linear KDA uint32_t n_embd_head_kda = 0; + float kda_gate_lower_bound = 0.0f; bool ssm_dt_b_c_rms = false; diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cpp index 264b0b5..1bf526c 100644 --- a/src/llama-model-saver.cpp +++ b/src/llama-model-saver.cpp @@ -121,6 +121,7 @@ void llama_model_saver::add_kv(const enum llm_kv key, const Container & value, c } // instantiate for external usage: template void llama_model_saver::add_kv>(const enum llm_kv, const std::vector &, const bool); +template void llama_model_saver::add_kv>(const enum llm_kv, const std::vector &, const bool); void llama_model_saver::add_kv(const enum llm_kv key, const std::vector & value) { std::vector tmp(value.size()); @@ -215,8 +216,8 @@ void llama_model_saver::add_kv_from_model() { add_kv(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp); add_kv(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp); add_kv(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_chexp); - add_kv(LLM_KV_SWIGLU_CLAMP_EXP, hparams.swiglu_clamp_exp); - add_kv(LLM_KV_SWIGLU_CLAMP_SHEXP, hparams.swiglu_clamp_shexp); + add_kv(LLM_KV_SWIGLU_CLAMP_EXP, hparams.swiglu_clamp_exp, true); + add_kv(LLM_KV_SWIGLU_CLAMP_SHEXP, hparams.swiglu_clamp_shexp, true); add_kv(LLM_KV_USE_PARALLEL_RESIDUAL, hparams.use_par_res); // add_kv(LLM_KV_TENSOR_DATA_LAYOUT, ???); add_kv(LLM_KV_EXPERT_COUNT, hparams.n_expert); @@ -319,6 +320,7 @@ void llama_model_saver::add_kv_from_model() { add_kv(LLM_KV_SSM_DT_B_C_RMS, hparams.ssm_dt_b_c_rms); add_kv(LLM_KV_KDA_HEAD_DIM, hparams.n_embd_head_kda); + add_kv(LLM_KV_KDA_GATE_LOWER_BOUND, hparams.kda_gate_lower_bound); add_kv(LLM_KV_WKV_HEAD_SIZE, hparams.wkv_head_size); diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 7acab67..76bd22c 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -308,6 +308,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params return new llama_model_dflash(params); case LLM_ARCH_MIMO2: return new llama_model_mimo2(params); + case LLM_ARCH_BAILINGMOE3: + return new llama_model_bailingmoe3(params); case LLM_ARCH_KIMI_LINEAR: return new llama_model_kimi_linear(params); case LLM_ARCH_STEP35: @@ -814,6 +816,7 @@ const char * llm_type_name(llm_type type) { case LLM_TYPE_24B_A2B: return "24B.A2B"; case LLM_TYPE_26B_A4B: return "26B.A4B"; case LLM_TYPE_30B_A3B: return "30B.A3B"; + case LLM_TYPE_124B_A5B: return "124B.A5B"; case LLM_TYPE_31B_A3_5B: return "31B.A3.5B"; case LLM_TYPE_35B_A3B: return "35B.A3B"; case LLM_TYPE_48B_A3B: return "48B.A3B"; @@ -1358,8 +1361,8 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) { } if (params.load_mode == LLAMA_LOAD_MODE_LONGHAUL) { - if (arch != LLM_ARCH_QWEN35MOE && arch != LLM_ARCH_LAGUNA && arch != LLM_ARCH_INKLING) { - throw std::runtime_error("longhaul currently requires a qwen35moe, laguna, or inkling model"); + if (arch != LLM_ARCH_QWEN35MOE && arch != LLM_ARCH_LAGUNA && arch != LLM_ARCH_INKLING && arch != LLM_ARCH_BAILINGMOE3) { + throw std::runtime_error("longhaul currently requires a qwen35moe, laguna, inkling, or bailingmoe3 model"); } if (hparams.n_layer_nextn != 0) { throw std::runtime_error("longhaul does not support MTP tensors"); @@ -2538,6 +2541,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) { case LLM_ARCH_GRANITE_HYBRID: case LLM_ARCH_CHAMELEON: case LLM_ARCH_BAILINGMOE: + case LLM_ARCH_BAILINGMOE3: case LLM_ARCH_NEO_BERT: case LLM_ARCH_SMOLLM3: case LLM_ARCH_ARCEE: diff --git a/src/llama-model.h b/src/llama-model.h index 4e6edc5..6a9a2d6 100644 --- a/src/llama-model.h +++ b/src/llama-model.h @@ -124,6 +124,7 @@ enum llm_type { LLM_TYPE_24B_A2B, // lfm2moe LLM_TYPE_26B_A4B, // Gemma4 LLM_TYPE_30B_A3B, + LLM_TYPE_124B_A5B, // Ling 3.0 flash LLM_TYPE_31B_A3_5B, LLM_TYPE_35B_A3B, // Qwen3.5 LLM_TYPE_48B_A3B, // Kimi Linear @@ -504,6 +505,8 @@ struct llama_layer { struct ggml_tensor * ssm_q_conv = nullptr; struct ggml_tensor * ssm_k_conv = nullptr; struct ggml_tensor * ssm_v_conv = nullptr; + struct ggml_tensor * ssm_f = nullptr; + struct ggml_tensor * ssm_g = nullptr; struct ggml_tensor * ssm_f_a = nullptr; struct ggml_tensor * ssm_f_b = nullptr; struct ggml_tensor * ssm_beta = nullptr; diff --git a/src/models/bailingmoe3.cpp b/src/models/bailingmoe3.cpp new file mode 100644 index 0000000..a54e638 --- /dev/null +++ b/src/models/bailingmoe3.cpp @@ -0,0 +1,461 @@ +#include "models.h" +#include "llama-memory-recurrent.h" + +// Ling 3.0 (BailingMoeV3): hybrid linear architecture, 5 KDA layers per 1 gated MLA layer, +// on top of a bailing MoE (sigmoid router with expert bias and group-limited routing). +// +// Differences to Kimi-Linear, which shares the KDA + MLA layout: +// - KDA forget/output gates are full-rank projections (config: no_kda_lora = true) +// - the KDA gate uses the safe-gate form (config: kda_safe_gate, kda_lower_bound): +// lower_bound * sigmoid(exp(A_log) * (f_proj(x) + dt_bias)) instead of +// -exp(A_log) * softplus(f_proj(x) + dt_bias) +// - MLA layers do use RoPE (partial, interleaved -> ggml NORM) and carry a head-wise +// sigmoid output gate applied before the output projection + +void llama_model_bailingmoe3::load_arch_hparams(llama_model_loader & ml) { + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); + ml.get_key(LLM_KV_ATTENTION_KEY_LENGTH_MLA, hparams.n_embd_head_k_mla_impl); + ml.get_key(LLM_KV_ATTENTION_VALUE_LENGTH_MLA, hparams.n_embd_head_v_mla_impl); + ml.get_key(LLM_KV_ATTENTION_KV_LORA_RANK, hparams.n_lora_kv); + ml.get_key(LLM_KV_SSM_CONV_KERNEL, hparams.ssm_d_conv); + ml.get_key(LLM_KV_KDA_HEAD_DIM, hparams.n_embd_head_kda); + ml.get_key(LLM_KV_KDA_GATE_LOWER_BOUND, hparams.kda_gate_lower_bound); + + // n_head_kv == 0 marks the KDA (recurrent) layers, the rest are full attention + for (uint32_t i = 0; i < hparams.n_layer(); ++i) { + hparams.is_recr_impl[i] = hparams.n_head_kv(i) == 0; + } + + ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp); + ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, false); + ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared); + ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead, false); + ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false); + ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm, false); + ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func); + ml.get_key(LLM_KV_EXPERT_GROUP_COUNT, hparams.n_expert_groups, false); + ml.get_key(LLM_KV_EXPERT_GROUP_USED_COUNT, hparams.n_group_used, false); + + // trained per-layer SwiGLU clamps (config: expert_swiglu_limit_list / share_expert_swiglu_limit_list) + ml.get_key_or_arr(LLM_KV_SWIGLU_CLAMP_EXP, hparams.swiglu_clamp_exp, hparams.n_layer_all, false); + ml.get_key_or_arr(LLM_KV_SWIGLU_CLAMP_SHEXP, hparams.swiglu_clamp_shexp, hparams.n_layer_all, false); + + switch (hparams.n_layer()) { + case 42: type = LLM_TYPE_124B_A5B; break; // Ling-3.0-flash + default: type = LLM_TYPE_UNKNOWN; + } +} + +void llama_model_bailingmoe3::load_arch_tensors(llama_model_loader &) { + LLAMA_LOAD_LOCALS; + + tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); + + output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0); + output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, 0); + + const int64_t head_dim_kda = hparams.n_embd_head_kda; + const int64_t d_inner = head_dim_kda * n_head; + const int64_t ssm_d_conv = hparams.ssm_d_conv; + const int expert_flags = params.load_mode == LLAMA_LOAD_MODE_LONGHAUL ? TENSOR_LONGHAUL : 0; + + for (int i = 0; i < n_layer; ++i) { + auto & layer = layers[i]; + + layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0); + + if (hparams.is_recr(i)) { + // === KDA (linear attention) layer === + // conv1d weights are 4D in GGUF, quantization may drop the trailing 1 + layer.ssm_q_conv = create_tensor(tn(LLM_TENSOR_SSM_CONV1D_Q, "weight", i), {ssm_d_conv, 1, d_inner, 1}, TENSOR_NOT_REQUIRED); + if (!layer.ssm_q_conv) { + layer.ssm_q_conv = create_tensor(tn(LLM_TENSOR_SSM_CONV1D_Q, "weight", i), {ssm_d_conv, 1, d_inner}, 0); + } + layer.ssm_k_conv = create_tensor(tn(LLM_TENSOR_SSM_CONV1D_K, "weight", i), {ssm_d_conv, 1, d_inner, 1}, TENSOR_NOT_REQUIRED); + if (!layer.ssm_k_conv) { + layer.ssm_k_conv = create_tensor(tn(LLM_TENSOR_SSM_CONV1D_K, "weight", i), {ssm_d_conv, 1, d_inner}, 0); + } + layer.ssm_v_conv = create_tensor(tn(LLM_TENSOR_SSM_CONV1D_V, "weight", i), {ssm_d_conv, 1, d_inner, 1}, TENSOR_NOT_REQUIRED); + if (!layer.ssm_v_conv) { + layer.ssm_v_conv = create_tensor(tn(LLM_TENSOR_SSM_CONV1D_V, "weight", i), {ssm_d_conv, 1, d_inner}, 0); + } + + create_tensor_qkv(layer, i, n_embd, d_inner, d_inner, d_inner, 0); + + // full-rank forget/output gate projections (no_kda_lora) + layer.ssm_f = create_tensor(tn(LLM_TENSOR_SSM_F, "weight", i), {n_embd, d_inner}, 0); + layer.ssm_g = create_tensor(tn(LLM_TENSOR_SSM_G, "weight", i), {n_embd, d_inner}, 0); + + layer.ssm_beta = create_tensor(tn(LLM_TENSOR_SSM_BETA, "weight", i), {n_embd, n_head}, 0); + + // note: the conversion script stores exp(A_log) + layer.ssm_a = create_tensor(tn(LLM_TENSOR_SSM_A, i), {1, n_head, 1, 1}, TENSOR_NOT_REQUIRED); + if (!layer.ssm_a) { + layer.ssm_a = create_tensor(tn(LLM_TENSOR_SSM_A, i), {1, n_head}, 0); + } + + layer.ssm_dt_b = create_tensor(tn(LLM_TENSOR_SSM_DT, "bias", i), {d_inner}, 0); + layer.ssm_o_norm = create_tensor(tn(LLM_TENSOR_SSM_NORM, "weight", i), {head_dim_kda}, 0); + + layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {d_inner, n_embd}, 0); + } else { + // === gated MLA layer === + const int64_t kv_lora_rank = hparams.n_lora_kv; + const int64_t n_embd_head_k_mla = hparams.n_embd_head_k_mla(); + const int64_t n_embd_head_v_mla = hparams.n_embd_head_v_mla(); + const int64_t qk_rope_head_dim = hparams.n_rot(); + + // Ling 3.0 has no query compression (q_lora_rank = null) + layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_head * n_embd_head_k_mla}, 0); + + layer.attn_kv_a_norm = create_tensor(tn(LLM_TENSOR_ATTN_KV_A_NORM, "weight", i), {kv_lora_rank}, 0); + layer.wkv_a_mqa = create_tensor(tn(LLM_TENSOR_ATTN_KV_A_MQA, "weight", i), {n_embd, kv_lora_rank + qk_rope_head_dim}, 0); + + layer.wkv_b = create_tensor(tn(LLM_TENSOR_ATTN_KV_B, "weight", i), + {kv_lora_rank, n_head * (n_embd_head_k_mla - qk_rope_head_dim + n_embd_head_v_mla)}, TENSOR_NOT_REQUIRED | TENSOR_SKIP_IF_VIRTUAL); + if (!layer.wkv_b) { // MLA KV cache enabled + layer.wk_b = create_tensor(tn(LLM_TENSOR_ATTN_K_B, "weight", i), {n_embd_head_k_mla - qk_rope_head_dim, kv_lora_rank, n_head}, 0); + layer.wv_b = create_tensor(tn(LLM_TENSOR_ATTN_V_B, "weight", i), {kv_lora_rank, n_embd_head_v_mla, n_head}, 0); + } + + // head-wise output gate (gated_attention_proj_granularity_type = "head_wise") + layer.wqkv_gate = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "weight", i), {n_embd, n_head}, 0); + + layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_head * n_embd_head_v_mla, n_embd}, 0); + } + + layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0); + + if (i < (int) hparams.n_layer_dense_lead) { + layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0); + layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, 0); + layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0); + } else { + const int64_t n_ff_exp = hparams.n_ff_exp; + const int64_t n_ff_shexp = hparams.n_ff_shexp > 0 ? hparams.n_ff_shexp : n_ff_exp * hparams.n_expert_shared; + + layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0); + layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, 0); + + layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, expert_flags); + layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, expert_flags); + layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, expert_flags); + + layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), {n_embd, n_ff_shexp}, 0); + layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), {n_ff_shexp, n_embd}, 0); + layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", i), {n_embd, n_ff_shexp}, 0); + } + } +} + +std::unique_ptr llama_model_bailingmoe3::build_arch_graph(const llm_graph_params & params) const { + return std::make_unique(*this, params); +} + +// projection + causal conv1d + silu for one of Q, K, V (qkv: 0 = Q, 1 = K, 2 = V) +static ggml_tensor * bailingmoe3_causal_conv1d( + ggml_cgraph * gf, ggml_context * ctx0, + ggml_tensor * conv_states_all, ggml_tensor * conv_state_all, + int64_t qkv, ggml_tensor * x, ggml_tensor * proj_w, ggml_tensor * conv_w, + int64_t d_conv, int64_t head_dim, int64_t n_head, + int64_t n_seq_tokens, int64_t n_seqs, int64_t n_tokens, int64_t kv_head) { + const int64_t d_inner = head_dim * n_head; + const int64_t conv_state_size = (d_conv - 1) * d_inner; + const int64_t n_embd_r_total = 3 * conv_state_size; // Q + K + V + + ggml_tensor * conv_state_x = ggml_view_3d(ctx0, conv_state_all, d_conv - 1, d_inner, n_seqs, + (d_conv - 1) * ggml_element_size(conv_state_all), + n_embd_r_total * ggml_element_size(conv_state_all), + qkv * conv_state_size * ggml_element_size(conv_state_all)); + + ggml_tensor * x_proj = ggml_mul_mat(ctx0, proj_w, x); + ggml_tensor * x_3d = ggml_reshape_3d(ctx0, x_proj, d_inner, n_seq_tokens, n_seqs); + + ggml_tensor * conv_x = ggml_concat(ctx0, conv_state_x, ggml_transpose(ctx0, x_3d), 0); + + ggml_tensor * last_conv_x = ggml_view_3d(ctx0, conv_x, d_conv - 1, d_inner, n_seqs, + conv_x->nb[1], conv_x->nb[2], n_seq_tokens * conv_x->nb[0]); + ggml_build_forward_expand(gf, + ggml_cpy(ctx0, last_conv_x, + ggml_view_3d(ctx0, conv_states_all, + d_conv - 1, d_inner, n_seqs, + (d_conv - 1) * ggml_element_size(conv_states_all), + n_embd_r_total * ggml_element_size(conv_states_all), + (kv_head * n_embd_r_total + qkv * conv_state_size) * ggml_element_size(conv_states_all)))); + + ggml_tensor * conv_weight = ggml_reshape_2d(ctx0, conv_w, d_conv, d_inner); + + ggml_tensor * Xcur = ggml_ssm_conv(ctx0, conv_x, conv_weight); + Xcur = ggml_reshape_2d(ctx0, Xcur, d_inner, n_tokens); + Xcur = ggml_silu(ctx0, Xcur); + + return ggml_reshape_4d(ctx0, Xcur, head_dim, n_head, n_seq_tokens, n_seqs); +} + +llama_model_bailingmoe3::graph::graph(const llama_model & model, const llm_graph_params & params) : + llm_build_delta_net_base(params), model(model) { + ggml_tensor * cur; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + cb(inpL, "inp_embd", -1); + + // MLA layers use RoPE, so positions are needed + ggml_tensor * inp_pos = build_inp_pos(); + + auto * inp_kv = !hparams.is_mla() ? build_inp_mem_hybrid() : nullptr; + auto * inp_k = hparams.is_mla() ? build_inp_mem_hybrid_k() : nullptr; + auto * inp_rs = hparams.is_mla() ? inp_k->get_recr() : inp_kv->get_recr(); + auto * inp_attn_kv = !hparams.is_mla() ? inp_kv->get_attn() : nullptr; + auto * inp_attn_k = hparams.is_mla() ? inp_k->get_attn() : nullptr; + + ggml_tensor * inp_out_ids = build_inp_out_ids(); + + const int64_t n_head = hparams.n_head(); + const int64_t head_dim = hparams.n_embd_head_kda; + const int64_t d_conv = hparams.ssm_d_conv; + const int64_t d_inner = n_head * head_dim; + const int64_t n_seqs = ubatch.n_seqs; + const int64_t n_seq_tokens = ubatch.n_seq_tokens; + + GGML_ASSERT(n_seqs != 0); + GGML_ASSERT(ubatch.equal_seqs()); + GGML_ASSERT(ubatch.n_tokens == n_seq_tokens * n_seqs); + + const int64_t n_embd_head_k_mla = hparams.n_embd_head_k_mla(); + const int64_t n_embd_head_v_mla = hparams.n_embd_head_v_mla(); + const int64_t kv_lora_rank = hparams.n_lora_kv; + const int64_t n_embd_head_qk_rope = hparams.n_rot(); + const int64_t n_embd_head_qk_nope = n_embd_head_k_mla - n_embd_head_qk_rope; + + const float kq_scale_mla = 1.0f / sqrtf(float(n_embd_head_k_mla)); + + for (int il = 0; il < n_layer; ++il) { + const auto & layer = model.layers[il]; + + ggml_tensor * inpSA = inpL; + + cur = build_norm(inpL, layer.attn_norm, NULL, LLM_NORM_RMS, il); + cb(cur, "attn_norm", il); + + // the MLA output gate is computed from this same normed hidden state + ggml_tensor * attn_inp = cur; + + if (hparams.is_recr(il)) { + // === KDA === + const auto * mctx_cur = inp_rs->mctx; + const auto kv_head = mctx_cur->get_head(); + + ggml_tensor * conv_states_all = mctx_cur->get_r_l(il); + ggml_tensor * conv_state_all = build_rs(inp_rs, conv_states_all, hparams.n_embd_r(), n_seqs); + + ggml_tensor * Qcur = bailingmoe3_causal_conv1d(gf, ctx0, conv_states_all, conv_state_all, 0, cur, layer.wq, layer.ssm_q_conv, d_conv, head_dim, n_head, n_seq_tokens, n_seqs, n_tokens, kv_head); + ggml_tensor * Kcur = bailingmoe3_causal_conv1d(gf, ctx0, conv_states_all, conv_state_all, 1, cur, layer.wk, layer.ssm_k_conv, d_conv, head_dim, n_head, n_seq_tokens, n_seqs, n_tokens, kv_head); + ggml_tensor * Vcur = bailingmoe3_causal_conv1d(gf, ctx0, conv_states_all, conv_state_all, 2, cur, layer.wv, layer.ssm_v_conv, d_conv, head_dim, n_head, n_seq_tokens, n_seqs, n_tokens, kv_head); + + // safe gate: g1 = lower_bound * sigmoid(exp(A_log) * (f_proj(x) + dt_bias)) + // note: ssm_a holds exp(A_log), applied by the conversion script + ggml_tensor * g1 = ggml_mul_mat(ctx0, layer.ssm_f, cur); + g1 = ggml_add(ctx0, g1, layer.ssm_dt_b); + g1 = ggml_reshape_3d(ctx0, g1, head_dim, n_head, n_tokens); + + ggml_tensor * A = ggml_reshape_3d(ctx0, layer.ssm_a, 1, n_head, 1); + g1 = ggml_mul(ctx0, g1, A); + g1 = ggml_sigmoid(ctx0, g1); + g1 = ggml_scale(ctx0, g1, hparams.kda_gate_lower_bound); + cb(g1, "kda_g1", il); + + g1 = ggml_reshape_4d(ctx0, g1, head_dim, n_head, n_seq_tokens, n_seqs); + + ggml_tensor * beta = ggml_mul_mat(ctx0, layer.ssm_beta, cur); + beta = ggml_reshape_4d(ctx0, beta, 1, n_head, n_seq_tokens, n_seqs); + beta = ggml_sigmoid(ctx0, beta); + cb(beta, "kda_beta", il); + + cur = ggml_reshape_3d(ctx0, cur, cur->ne[0], n_seq_tokens, n_seqs); + + ggml_tensor * ssm_states_all = mctx_cur->get_s_l(il); + ggml_tensor * state = build_rs(inp_rs, ssm_states_all, hparams.n_embd_s(), n_seqs); + state = ggml_reshape_4d(ctx0, state, head_dim, head_dim, n_head, n_seqs); + + const float eps_norm = hparams.f_norm_rms_eps; + + Qcur = ggml_l2_norm(ctx0, Qcur, eps_norm); + Kcur = ggml_l2_norm(ctx0, Kcur, eps_norm); + + auto attn_out = build_delta_net(Qcur, Kcur, Vcur, g1, beta, state, il); + + ggml_tensor * output = ggml_cont(ctx0, attn_out.first); + ggml_tensor * new_state = attn_out.second; + + ggml_build_forward_expand(gf, + ggml_cpy(ctx0, new_state, + ggml_view_1d(ctx0, ssm_states_all, hparams.n_embd_s() * n_seqs, + kv_head * hparams.n_embd_s() * ggml_element_size(ssm_states_all)))); + + // output gate: RMSNorm(o) * sigmoid(g_proj(x)) + ggml_tensor * cur_2d = ggml_reshape_2d(ctx0, cur, cur->ne[0], n_seq_tokens * n_seqs); + ggml_tensor * g2 = ggml_mul_mat(ctx0, layer.ssm_g, cur_2d); + g2 = ggml_reshape_3d(ctx0, g2, head_dim, n_head, n_seq_tokens * n_seqs); + + ggml_tensor * attn_out_final = ggml_reshape_3d(ctx0, output, head_dim, n_head, n_seq_tokens * n_seqs); + ggml_tensor * normed = build_norm(attn_out_final, layer.ssm_o_norm, nullptr, LLM_NORM_RMS, il); + ggml_tensor * gated = ggml_mul(ctx0, normed, ggml_sigmoid(ctx0, g2)); + + gated = ggml_cont_2d(ctx0, gated, d_inner, n_tokens); + cur = ggml_mul_mat(ctx0, layer.wo, gated); + cb(cur, "kda_out", il); + } else { + // === gated MLA === + ggml_tensor * q = ggml_mul_mat(ctx0, layer.wq, cur); + cb(q, "q", il); + + ggml_tensor * q_nope = ggml_view_3d(ctx0, q, n_embd_head_qk_nope, n_head, n_tokens, + ggml_row_size(q->type, n_embd_head_k_mla), + ggml_row_size(q->type, n_embd_head_k_mla) * n_head, 0); + ggml_tensor * q_pe = ggml_view_3d(ctx0, q, n_embd_head_qk_rope, n_head, n_tokens, + ggml_row_size(q->type, n_embd_head_k_mla), + ggml_row_size(q->type, n_embd_head_k_mla) * n_head, + ggml_row_size(q->type, n_embd_head_qk_nope)); + + ggml_tensor * kv_cmpr_pe = ggml_mul_mat(ctx0, layer.wkv_a_mqa, cur); + + ggml_tensor * kv_cmpr = ggml_view_2d(ctx0, kv_cmpr_pe, kv_lora_rank, n_tokens, + ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope), 0); + ggml_tensor * k_pe = ggml_view_3d(ctx0, kv_cmpr_pe, n_embd_head_qk_rope, 1, n_tokens, + ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope), + ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope), + ggml_row_size(kv_cmpr_pe->type, kv_lora_rank)); + + // note: HF applies rope on de-interleaved pairs, which is ggml's NORM rope + q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, + freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); + cb(q_pe, "q_pe", il); + + k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, + freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); + cb(k_pe, "k_pe", il); + + kv_cmpr = build_norm(kv_cmpr, layer.attn_kv_a_norm, nullptr, LLM_NORM_RMS, il); + cb(kv_cmpr, "kv_cmpr", il); + + if (layer.wk_b && layer.wv_b) { // MLA KV cache enabled + q_nope = ggml_permute(ctx0, q_nope, 0, 2, 1, 3); + + ggml_tensor * q_nope_absorbed = ggml_mul_mat(ctx0, layer.wk_b, q_nope); + q_nope_absorbed = ggml_permute(ctx0, q_nope_absorbed, 0, 2, 1, 3); + + // note: rope must go first for in-place context shifting in build_rope_shift() + ggml_tensor * Qcur = ggml_concat(ctx0, q_nope_absorbed, q_pe, 0); + + kv_cmpr = ggml_reshape_3d(ctx0, kv_cmpr, kv_lora_rank, 1, n_tokens); + + ggml_tensor * Kcur = ggml_concat(ctx0, kv_cmpr, k_pe, 0); + ggml_tensor * Vcur = kv_cmpr; + + // no output projection here - the gate is applied first + cur = build_attn(inp_attn_k, nullptr, NULL, nullptr, + Qcur, Kcur, Vcur, nullptr, nullptr, layer.wv_b, kq_scale_mla, il); + } else { // MLA KV cache disabled, fall back to MHA + ggml_tensor * Qcur = ggml_reshape_3d(ctx0, ggml_cont(ctx0, q), n_embd_head_k_mla, n_head, n_tokens); + + ggml_tensor * kv = ggml_mul_mat(ctx0, layer.wkv_b, kv_cmpr); + const int64_t kv_per_head = n_embd_head_qk_nope + n_embd_head_v_mla; + + ggml_tensor * k_nope = ggml_view_3d(ctx0, kv, n_embd_head_qk_nope, n_head, n_tokens, + ggml_row_size(kv->type, kv_per_head), + ggml_row_size(kv->type, kv_per_head * n_head), 0); + ggml_tensor * Vcur = ggml_view_3d(ctx0, kv, n_embd_head_v_mla, n_head, n_tokens, + ggml_row_size(kv->type, kv_per_head), + ggml_row_size(kv->type, kv_per_head * n_head), + ggml_row_size(kv->type, n_embd_head_qk_nope)); + Vcur = ggml_cont(ctx0, Vcur); + + ggml_tensor * k_pe_target = ggml_new_tensor_3d(ctx0, k_pe->type, n_embd_head_qk_rope, n_head, n_tokens); + ggml_tensor * k_pe_repeated = ggml_repeat(ctx0, k_pe, k_pe_target); + ggml_tensor * Kcur = ggml_concat(ctx0, k_nope, k_pe_repeated, 0); + + cur = build_attn(inp_attn_kv, nullptr, NULL, nullptr, + Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale_mla, il); + } + cb(cur, "attn_out_pre_gate", il); + + // head-wise sigmoid gate, computed from the same normed hidden state as q/kv + ggml_tensor * gate = ggml_mul_mat(ctx0, layer.wqkv_gate, attn_inp); + gate = ggml_sigmoid(ctx0, gate); + cb(gate, "attn_gate", il); + + cur = ggml_reshape_3d(ctx0, cur, n_embd_head_v_mla, n_head, n_tokens); + gate = ggml_reshape_3d(ctx0, gate, 1, n_head, n_tokens); + cur = ggml_mul(ctx0, cur, gate); + cur = ggml_reshape_2d(ctx0, cur, n_embd_head_v_mla * n_head, n_tokens); + cb(cur, "attn_gated", il); + + cur = ggml_mul_mat(ctx0, layer.wo, cur); + cb(cur, "attn_out", il); + } + + if (il == n_layer - 1 && inp_out_ids) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } + + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); + cb(ffn_inp, "ffn_inp", il); + + cur = build_norm(ffn_inp, layer.ffn_norm, NULL, LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + if ((uint32_t) il < hparams.n_layer_dense_lead) { + cur = build_ffn(cur, + layer.ffn_up, NULL, NULL, + layer.ffn_gate, NULL, NULL, + layer.ffn_down, NULL, NULL, + NULL, LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(cur, "ffn_out", il); + } else { + ggml_tensor * moe_out = build_moe_ffn(cur, + layer.ffn_gate_inp, + layer.ffn_up_exps, + layer.ffn_gate_exps, + layer.ffn_down_exps, + layer.ffn_exp_probs_b, + hparams.n_expert, + hparams.n_expert_used, + LLM_FFN_SILU, hparams.expert_weights_norm, + hparams.expert_weights_scale, + (llama_expert_gating_func_type) hparams.expert_gating_func, + il); + cb(moe_out, "ffn_moe_out", il); + + ggml_tensor * ffn_shexp = build_ffn(cur, + layer.ffn_up_shexp, NULL, NULL, + layer.ffn_gate_shexp, NULL, NULL, + layer.ffn_down_shexp, NULL, NULL, + NULL, LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(ffn_shexp, "ffn_shexp", il); + + cur = ggml_add(ctx0, moe_out, ffn_shexp); + cb(cur, "ffn_out", il); + } + + cur = ggml_add(ctx0, cur, ffn_inp); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + inpL = cur; + } + + cur = inpL; + + cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1); + cb(cur, "result_norm", -1); + res->t_embd = cur; + + cur = ggml_mul_mat(ctx0, model.output, cur); + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); +} diff --git a/src/models/models.h b/src/models/models.h index 737b13d..3d16e53 100644 --- a/src/models/models.h +++ b/src/models/models.h @@ -2139,6 +2139,21 @@ struct llama_model_mimo2 : public llama_model_base { }; +struct llama_model_bailingmoe3 : public llama_model_base { + llama_model_bailingmoe3(const struct llama_model_params & params) : llama_model_base(params) {} + void load_arch_hparams(llama_model_loader & ml) override; + void load_arch_tensors(llama_model_loader & ml) override; + + struct graph : public llm_build_delta_net_base { + graph(const llama_model & model, const llm_graph_params & params); + + const llama_model & model; + }; + + std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; +}; + + struct llama_model_kimi_linear : public llama_model_base { llama_model_kimi_linear(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index a1649bb..0a2c883 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -291,6 +291,22 @@ llama_test( set_tests_properties(test-longhaul-cpu-inkling-six-slots PROPERTIES FIXTURES_REQUIRED generate-models ) +llama_test( + test-longhaul + NAME test-longhaul-cpu-bailingmoe3 + ARGS --cpu-model "${MODEL_DIR}/bailingmoe3-moe.gguf" 49152 1 +) +set_tests_properties(test-longhaul-cpu-bailingmoe3 PROPERTIES + FIXTURES_REQUIRED generate-models +) +llama_test( + test-longhaul + NAME test-longhaul-cpu-bailingmoe3-two-slots + ARGS --cpu-model "${MODEL_DIR}/bailingmoe3-moe.gguf" 98304 2 +) +set_tests_properties(test-longhaul-cpu-bailingmoe3-two-slots PROPERTIES + FIXTURES_REQUIRED generate-models +) if (APPLE AND GGML_METAL) llama_test( test-longhaul @@ -308,6 +324,14 @@ if (APPLE AND GGML_METAL) set_tests_properties(test-longhaul-metal-inkling-six-slots PROPERTIES FIXTURES_REQUIRED generate-models ) + llama_test( + test-longhaul + NAME test-longhaul-metal-bailingmoe3 + ARGS --metal-model "${MODEL_DIR}/bailingmoe3-moe.gguf" 49152 1 + ) + set_tests_properties(test-longhaul-metal-bailingmoe3 PROPERTIES + FIXTURES_REQUIRED generate-models + ) endif() llama_build_and_test(test-token-cache.cpp) target_include_directories(test-token-cache PRIVATE ${PROJECT_SOURCE_DIR}/src) diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp index 5001b44..e3e767a 100644 --- a/tests/test-llama-archs.cpp +++ b/tests/test-llama-archs.cpp @@ -119,6 +119,11 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { n_head = 2; n_ff = 96; n_layer = 2; + } else if (arch == LLM_ARCH_BAILINGMOE3) { + n_embd = 64; + n_head = 2; + n_ff = 96; + n_layer = 2; } const uint32_t n_embd_head = n_embd / n_head; @@ -129,7 +134,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { ms.add_kv(LLM_KV_EMBEDDING_LENGTH, n_embd); ms.add_kv(LLM_KV_FEATURES_LENGTH, n_embd); ms.add_kv(LLM_KV_BLOCK_COUNT, n_layer); - ms.add_kv(LLM_KV_LEADING_DENSE_BLOCK_COUNT, uint32_t(1)); + ms.add_kv(LLM_KV_LEADING_DENSE_BLOCK_COUNT, arch == LLM_ARCH_BAILINGMOE3 ? uint32_t(0) : uint32_t(1)); if (arch == LLM_ARCH_NEMOTRON_H || arch == LLM_ARCH_NEMOTRON_H_MOE) { std::vector n_ff_per_layer; @@ -148,7 +153,10 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { ms.add_kv(LLM_KV_TIME_DECAY_EXTRA_DIM, uint32_t(128)); ms.add_kv(LLM_KV_FULL_ATTENTION_INTERVAL, uint32_t(2)); - if (arch == LLM_ARCH_PLAMO2 || arch == LLM_ARCH_JAMBA || arch == LLM_ARCH_NEMOTRON_H || arch == LLM_ARCH_NEMOTRON_H_MOE || + if (arch == LLM_ARCH_BAILINGMOE3) { + ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT, std::vector(n_layer, n_head)); + ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT_KV, std::vector({0, 1})); + } else if (arch == LLM_ARCH_PLAMO2 || arch == LLM_ARCH_JAMBA || arch == LLM_ARCH_NEMOTRON_H || arch == LLM_ARCH_NEMOTRON_H_MOE || arch == LLM_ARCH_GRANITE_HYBRID || arch == LLM_ARCH_LFM2 || arch == LLM_ARCH_LFM2MOE || arch == LLM_ARCH_KIMI_LINEAR) { GGML_ASSERT(n_layer >= 2); std::vector n_head_per_layer; @@ -164,7 +172,13 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { } ms.add_kv(LLM_KV_ATTENTION_MAX_ALIBI_BIAS, 8.0f); - if (arch == LLM_ARCH_DEEPSEEK2 + if (arch == LLM_ARCH_BAILINGMOE3) { + ms.add_kv(LLM_KV_ATTENTION_KEY_LENGTH, uint32_t(16)); + ms.add_kv(LLM_KV_ATTENTION_VALUE_LENGTH, uint32_t(8)); + ms.add_kv(LLM_KV_ROPE_DIMENSION_COUNT, uint32_t(8)); + ms.add_kv(LLM_KV_ATTENTION_KEY_LENGTH_MLA, uint32_t(16)); + ms.add_kv(LLM_KV_ATTENTION_VALUE_LENGTH_MLA, uint32_t(8)); + } else if (arch == LLM_ARCH_DEEPSEEK2 || arch == LLM_ARCH_DEEPSEEK32 || arch == LLM_ARCH_GLM_DSA || arch == LLM_ARCH_KIMI_LINEAR @@ -184,7 +198,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { ms.add_kv(LLM_KV_ATTENTION_GROUPNORM_EPS, 1e-5f); ms.add_kv(LLM_KV_ATTENTION_GROUPNORM_GROUPS, uint32_t(8)); ms.add_kv(LLM_KV_ATTENTION_Q_LORA_RANK, uint32_t(512)); - ms.add_kv(LLM_KV_ATTENTION_KV_LORA_RANK, uint32_t(512)); + ms.add_kv(LLM_KV_ATTENTION_KV_LORA_RANK, arch == LLM_ARCH_BAILINGMOE3 ? uint32_t(8) : uint32_t(512)); ms.add_kv(LLM_KV_ATTENTION_RELATIVE_BUCKETS_COUNT, uint32_t(8)); ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW, n_ctx/8); @@ -224,16 +238,21 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { // ms.add_kv(LLM_KV_DENSE_3_FEAT_IN, n_embd); if (moe) { - ms.add_kv(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, n_ff); + ms.add_kv(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, arch == LLM_ARCH_BAILINGMOE3 ? uint32_t(32) : n_ff); ms.add_kv(LLM_KV_INTERLEAVE_MOE_LAYER_STEP, uint32_t(2)); - ms.add_kv(LLM_KV_EXPERT_COUNT, arch == LLM_ARCH_INKLING ? uint32_t(8) : uint32_t(2)); - ms.add_kv(LLM_KV_EXPERT_USED_COUNT, arch == LLM_ARCH_INKLING ? uint32_t(6) : uint32_t(1)); + ms.add_kv(LLM_KV_EXPERT_COUNT, arch == LLM_ARCH_INKLING || arch == LLM_ARCH_BAILINGMOE3 ? uint32_t(8) : uint32_t(2)); + ms.add_kv(LLM_KV_EXPERT_USED_COUNT, arch == LLM_ARCH_INKLING ? uint32_t(6) : arch == LLM_ARCH_BAILINGMOE3 ? uint32_t(2) : uint32_t(1)); ms.add_kv(LLM_KV_EXPERT_SHARED_COUNT, arch == LLM_ARCH_INKLING ? uint32_t(2) : uint32_t(1)); ms.add_kv(LLM_KV_EXPERT_GATING_FUNC, uint32_t(2)); // sigmoid ms.add_kv(LLM_KV_EXPERT_GROUP_SCALE, 1.0f); ms.add_kv(LLM_KV_EXPERTS_PER_GROUP, uint32_t(1)); if (arch == LLM_ARCH_INKLING) { ms.add_kv(LLM_KV_EXPERT_WEIGHTS_SCALE, 1.0f); + } else if (arch == LLM_ARCH_BAILINGMOE3) { + ms.add_kv(LLM_KV_EXPERT_WEIGHTS_SCALE, 1.0f); + ms.add_kv(LLM_KV_EXPERT_WEIGHTS_NORM, true); + ms.add_kv(LLM_KV_SWIGLU_CLAMP_EXP, std::vector({4.0f, 4.0f})); + ms.add_kv(LLM_KV_SWIGLU_CLAMP_SHEXP, std::vector({5.0f, 5.0f})); } } @@ -262,7 +281,10 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { ms.add_kv(LLM_KV_SSM_STATE_SIZE, uint32_t(128)); ms.add_kv(LLM_KV_SSM_TIME_STEP_RANK, n_head); ms.add_kv(LLM_KV_SSM_GROUP_COUNT, arch == LLM_ARCH_PLAMO2 ? 0 : uint32_t(2)); - ms.add_kv(LLM_KV_KDA_HEAD_DIM, uint32_t(128)); + ms.add_kv(LLM_KV_KDA_HEAD_DIM, arch == LLM_ARCH_BAILINGMOE3 ? uint32_t(16) : uint32_t(128)); + if (arch == LLM_ARCH_BAILINGMOE3) { + ms.add_kv(LLM_KV_KDA_GATE_LOWER_BOUND, -5.0f); + } ms.add_kv(LLM_KV_WKV_HEAD_SIZE, n_embd/n_head); ms.add_kv(LLM_KV_SHORTCONV_L_CACHE, uint32_t(3)); @@ -375,6 +397,7 @@ static bool moe_mandatory(const llm_arch arch) { case LLM_ARCH_EXAONE_MOE: case LLM_ARCH_BAILINGMOE: case LLM_ARCH_BAILINGMOE2: + case LLM_ARCH_BAILINGMOE3: case LLM_ARCH_DOTS1: case LLM_ARCH_AFMOE: case LLM_ARCH_ERNIE4_5: diff --git a/tools/cli/README.md b/tools/cli/README.md index e6300cf..57ce641 100644 --- a/tools/cli/README.md +++ b/tools/cli/README.md @@ -61,7 +61,7 @@ | `--mmap, --no-mmap` | DEPRECATED in favor of `--load-mode`: whether to memory-map model. (if mmap disabled, slower load but may reduce pageouts if not using mlock)
(env: LLAMA_ARG_MMAP) | | `-dio, --direct-io, -ndio, --no-direct-io` | DEPRECATED in favor of `--load-mode`: use DirectIO if available
(env: LLAMA_ARG_DIO) | | `-lm, --load-mode MODE` | model loading mode (default: mmap)
- none: no special loading mode
- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)
- mlock: force system to keep model in RAM rather than swapping or compressing
- mmap+mlock: mmap + force system to keep model in RAM rather than swapping or compressing
- dio: use DirectIO if available
- longhaul: stream routed MoE experts through a bounded cache

(env: LLAMA_ARG_LOAD_MODE) | -| `--longhaul` | stream routed Qwen3.5 MoE, Laguna, or Inkling experts through a bounded CPU or Metal cache
(env: LLAMA_ARG_LONGHAUL) | +| `--longhaul` | stream routed Qwen3.5 MoE, Laguna, Inkling, or Ling 3.0 experts through a bounded CPU or Metal cache
(env: LLAMA_ARG_LONGHAUL) | | `--longhaul-cache N` | longhaul expert cache size in GiB
(env: LLAMA_ARG_LONGHAUL_CACHE) | | `--numa TYPE` | attempt optimizations that help on some NUMA systems
- distribute: spread execution evenly over all nodes
- isolate: only spawn threads on CPUs on the node that execution started on
- numactl: use the CPU map provided by numactl
if run without this previously, it is recommended to drop the system page cache before using this
see https://github.com/ggml-org/llama.cpp/issues/1437
(env: LLAMA_ARG_NUMA) | | `-dev, --device ` | comma-separated list of devices to use for offloading (none = don't offload)
use --list-devices to see a list of available devices
(env: LLAMA_ARG_DEVICE) | diff --git a/tools/completion/README.md b/tools/completion/README.md index fe5978e..5ef1445 100644 --- a/tools/completion/README.md +++ b/tools/completion/README.md @@ -144,7 +144,7 @@ llama-completion.exe -m models\gemma-1.1-7b-it.Q4_K_M.gguf --ignore-eos -n -1 | `--mmap, --no-mmap` | DEPRECATED in favor of `--load-mode`: whether to memory-map model. (if mmap disabled, slower load but may reduce pageouts if not using mlock)
(env: LLAMA_ARG_MMAP) | | `-dio, --direct-io, -ndio, --no-direct-io` | DEPRECATED in favor of `--load-mode`: use DirectIO if available
(env: LLAMA_ARG_DIO) | | `-lm, --load-mode MODE` | model loading mode (default: mmap)
- none: no special loading mode
- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)
- mlock: force system to keep model in RAM rather than swapping or compressing
- mmap+mlock: mmap + force system to keep model in RAM rather than swapping or compressing
- dio: use DirectIO if available
- longhaul: stream routed MoE experts through a bounded cache

(env: LLAMA_ARG_LOAD_MODE) | -| `--longhaul` | stream routed Qwen3.5 MoE, Laguna, or Inkling experts through a bounded CPU or Metal cache
(env: LLAMA_ARG_LONGHAUL) | +| `--longhaul` | stream routed Qwen3.5 MoE, Laguna, Inkling, or Ling 3.0 experts through a bounded CPU or Metal cache
(env: LLAMA_ARG_LONGHAUL) | | `--longhaul-cache N` | longhaul expert cache size in GiB
(env: LLAMA_ARG_LONGHAUL_CACHE) | | `--numa TYPE` | attempt optimizations that help on some NUMA systems
- distribute: spread execution evenly over all nodes
- isolate: only spawn threads on CPUs on the node that execution started on
- numactl: use the CPU map provided by numactl
if run without this previously, it is recommended to drop the system page cache before using this
see https://github.com/ggml-org/llama.cpp/issues/1437
(env: LLAMA_ARG_NUMA) | | `-dev, --device ` | comma-separated list of devices to use for offloading (none = don't offload)
use --list-devices to see a list of available devices
(env: LLAMA_ARG_DEVICE) | diff --git a/tools/server/README.md b/tools/server/README.md index 21d8f23..70c4427 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -78,7 +78,7 @@ For the full list of features, please refer to [server's changelog](https://gith | `--mmap, --no-mmap` | DEPRECATED in favor of `--load-mode`: whether to memory-map model. (if mmap disabled, slower load but may reduce pageouts if not using mlock)
(env: LLAMA_ARG_MMAP) | | `-dio, --direct-io, -ndio, --no-direct-io` | DEPRECATED in favor of `--load-mode`: use DirectIO if available
(env: LLAMA_ARG_DIO) | | `-lm, --load-mode MODE` | model loading mode (default: mmap)
- none: no special loading mode
- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)
- mlock: force system to keep model in RAM rather than swapping or compressing
- mmap+mlock: mmap + force system to keep model in RAM rather than swapping or compressing
- dio: use DirectIO if available
- longhaul: stream routed MoE experts through a bounded cache

(env: LLAMA_ARG_LOAD_MODE) | -| `--longhaul` | stream routed Qwen3.5 MoE, Laguna, or Inkling experts through a bounded CPU or Metal cache
(env: LLAMA_ARG_LONGHAUL) | +| `--longhaul` | stream routed Qwen3.5 MoE, Laguna, Inkling, or Ling 3.0 experts through a bounded CPU or Metal cache
(env: LLAMA_ARG_LONGHAUL) | | `--longhaul-cache N` | longhaul expert cache size in GiB
(env: LLAMA_ARG_LONGHAUL_CACHE) | | `--numa TYPE` | attempt optimizations that help on some NUMA systems
- distribute: spread execution evenly over all nodes
- isolate: only spawn threads on CPUs on the node that execution started on
- numactl: use the CPU map provided by numactl
if run without this previously, it is recommended to drop the system page cache before using this
see https://github.com/ggml-org/llama.cpp/issues/1437
(env: LLAMA_ARG_NUMA) | | `-dev, --device ` | comma-separated list of devices to use for offloading (none = don't offload)
use --list-devices to see a list of available devices
(env: LLAMA_ARG_DEVICE) |