feat: add Ling 3.0 LongHaul support

common/arg.cpp

@@ -2631,7 +2631,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
     ).set_env("LLAMA_ARG_LOAD_MODE"));
     add_opt(common_arg(
         {"--longhaul"},
-        "stream routed Qwen3.5 MoE, Laguna, or Inkling experts through a bounded CPU or Metal cache",
+        "stream routed Qwen3.5 MoE, Laguna, Inkling, or Ling 3.0 experts through a bounded CPU or Metal cache",
         [](common_params & params) {
             params.load_mode = LLAMA_LOAD_MODE_LONGHAUL;
         }

conversion/__init__.py

@@ -27,6 +27,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
     "BaichuanForCausalLM": "baichuan",
     "BailingMoeForCausalLM": "bailingmoe",
     "BailingMoeV2ForCausalLM": "bailingmoe",
+    "BailingMoeV3ForCausalLM": "bailingmoe",
     "BambaForCausalLM": "granite",
     "BertForMaskedLM": "bert",
     "BertForSequenceClassification": "bert",

conversion/bailingmoe.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+import math
+
 from typing import Callable, Iterable, TYPE_CHECKING
 
 import torch
@@ -188,6 +190,184 @@ class BailingMoeV2Model(TextModel):
                 raise ValueError(f"Unprocessed experts: {experts}")
 
 
+@ModelBase.register("BailingMoeV3ForCausalLM")
+class BailingMoeV3Model(TextModel):
+    """Ling 3.0 - hybrid KDA (linear) + gated MLA (full) attention with a bailing MoE"""
+    model_arch = gguf.MODEL_ARCH.BAILINGMOE3
+
+    _experts: list[dict[str, Tensor]] | None = None
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        # the MTP block sits right after the last decoder layer and is not converted
+        self.block_count = self.hparams["num_hidden_layers"]
+        self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count)
+
+    def is_mla_layer(self, il: int) -> bool:
+        # every layer_group_size-th layer is a full attention (MLA) layer, the rest are KDA
+        return (il + 1) % self.hparams["layer_group_size"] == 0
+
+    def set_vocab(self):
+        # identical tokenizer to Ling 2.0, the bailingmoe2 pre-tokenizer hash matches
+        self._set_vocab_gpt2()
+
+    def set_gguf_parameters(self):
+        hparams = self.hparams
+
+        # note: to enable the MLA KV cache, attention is converted into MQA (ie: GQA with 1 group)
+        hparams["num_key_value_heads"] = 1
+
+        super().set_gguf_parameters()
+
+        self.gguf_writer.add_vocab_size(hparams["vocab_size"])
+
+        # n_head_kv == 0 marks a KDA (recurrent) layer, > 0 marks an MLA (attention) layer
+        self.gguf_writer.add_head_count_kv([
+            1 if self.is_mla_layer(il) else 0 for il in range(self.block_count)
+        ])
+
+        # KDA
+        self.gguf_writer.add_ssm_conv_kernel(hparams["short_conv_kernel_size"])
+        self.gguf_writer.add_kda_head_dim(hparams["head_dim"])
+        # safe gate: g = lower_bound * sigmoid(exp(A_log) * (f_proj(x) + dt_bias))
+        assert hparams.get("kda_safe_gate", False), "only the safe gate form is implemented"
+        self.gguf_writer.add_kda_gate_lower_bound(hparams["kda_lower_bound"])
+
+        # MLA - converted into MQA with larger heads, then decompressed to MHA
+        kv_lora_rank = hparams["kv_lora_rank"]
+        qk_rope_head_dim = hparams["qk_rope_head_dim"]
+        self.gguf_writer.add_kv_lora_rank(kv_lora_rank)
+        self.gguf_writer.add_key_length(kv_lora_rank + qk_rope_head_dim)
+        self.gguf_writer.add_value_length(kv_lora_rank)
+        self.gguf_writer.add_key_length_mla(hparams["qk_nope_head_dim"] + qk_rope_head_dim)
+        self.gguf_writer.add_value_length_mla(hparams["v_head_dim"])
+        self.gguf_writer.add_rope_dimension_count(qk_rope_head_dim)
+
+        # MoE
+        self.gguf_writer.add_leading_dense_block_count(hparams["first_k_dense_replace"])
+        self.gguf_writer.add_expert_feed_forward_length(hparams["moe_intermediate_size"])
+        self.gguf_writer.add_expert_shared_feed_forward_length(
+            hparams["moe_shared_expert_intermediate_size"] * hparams["num_shared_experts"])
+        self.gguf_writer.add_expert_shared_count(hparams["num_shared_experts"])
+        self.gguf_writer.add_expert_weights_scale(hparams["routed_scaling_factor"])
+        self.gguf_writer.add_expert_weights_norm(hparams["norm_topk_prob"])
+        self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SIGMOID)
+        self.gguf_writer.add_expert_group_count(hparams["n_group"])
+        self.gguf_writer.add_expert_group_used_count(hparams["topk_group"])
+
+        # Optional per-layer SwiGLU clamps (vLLM SwigluStepAndMul):
+        #   out = silu(gate).clamp(max=limit) * up.clamp(-limit, limit)
+        # 0.0 (or a missing/null entry) means no clamping for that layer.
+        def _clamp_limits(key: str) -> list[float] | None:
+            if (limits := hparams.get(key)) is None:
+                return None
+            limits = [0.0 if v is None else float(v) for v in limits[:self.block_count]]
+            return limits + [0.0] * (self.block_count - len(limits))
+
+        if (limits := _clamp_limits("expert_swiglu_limit_list")) is not None:
+            self.gguf_writer.add_swiglu_clamp_exp(limits)
+        if (limits := _clamp_limits("share_expert_swiglu_limit_list")) is not None:
+            self.gguf_writer.add_swiglu_clamp_shexp(limits)
+
+        if (nextn_layers := hparams.get("num_nextn_predict_layers")) is not None:
+            self.gguf_writer.add_nextn_predict_layers(nextn_layers)
+
+    @classmethod
+    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
+        name, gen = item
+
+        if name.endswith(".expert_bias"):
+            name = name.replace(".expert_bias", ".expert_bias.bias")
+
+        return super().filter_tensors((name, gen))
+
+    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+        # drop the MTP block
+        if bid is not None and bid >= self.block_count:
+            return
+
+        n_head = self.hparams["num_attention_heads"]
+        head_dim = self.hparams["head_dim"]
+
+        if name.endswith(".A_log"):
+            # the safe gate uses -exp(A_log) only through exp(A_log), see the graph:
+            #   g = kda_lower_bound * sigmoid(exp(A_log) * (f_proj(x) + dt_bias))
+            # {n_head} -> ggml ne = [1, n_head]
+            data_torch = torch.exp(data_torch.float())
+            data_torch = data_torch.reshape(-1, 1)
+        elif name.endswith(".dt_bias"):
+            name = name.rpartition(".dt_bias")[0] + ".dt_proj.bias"
+        elif name.endswith((".q_conv1d.weight", ".k_conv1d.weight", ".v_conv1d.weight")):
+            # HF {d_inner, [1,] d_conv} -> numpy (1, d_inner, 1, d_conv) -> ggml ne = [d_conv, 1, d_inner, 1]
+            d_conv = data_torch.shape[-1]
+            d_inner = math.prod(data_torch.shape[:-1])
+            data_torch = data_torch.reshape(1, d_inner, 1, d_conv)
+        elif name.endswith("attention.g_proj.weight"):
+            assert bid is not None
+            # the very same HF name is the KDA output gate on linear layers and the
+            # attention output gate on MLA layers
+            tensor = gguf.MODEL_TENSOR.ATTN_GATE if self.is_mla_layer(bid) else gguf.MODEL_TENSOR.SSM_G
+            yield from super().modify_tensors(data_torch, self.format_tensor_name(tensor, bid), bid)
+            return
+        elif name.endswith("attention.kv_b_proj.weight"):
+            assert bid is not None
+            # MLA with the absorption optimization needs these two split and k_b transposed
+            v_head_dim = self.hparams["v_head_dim"]
+            qk_nope_head_dim = self.hparams["qk_nope_head_dim"]
+            assert data_torch.shape[0] == n_head * (v_head_dim + qk_nope_head_dim)
+
+            kv_b = data_torch.view(n_head, qk_nope_head_dim + v_head_dim, data_torch.shape[-1])
+            k_b, v_b = torch.split(kv_b, [qk_nope_head_dim, v_head_dim], dim=1)
+            k_b = k_b.transpose(1, 2)
+
+            yield from super().modify_tensors(k_b, self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_K_B, bid), bid)
+            yield from super().modify_tensors(v_b, self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_V_B, bid), bid)
+            return
+        elif name.endswith("attention.dense.weight"):
+            assert bid is not None
+            # MLA output projection (KDA layers call it o_proj)
+            yield from super().modify_tensors(data_torch, self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_OUT, bid), bid)
+            return
+        elif "mlp.experts" in name:
+            n_experts = self.hparams["num_experts"]
+            assert bid is not None
+
+            if self._experts is None:
+                self._experts = [{} for _ in range(self.block_count)]
+
+            self._experts[bid][name] = data_torch
+
+            if len(self._experts[bid]) >= n_experts * 3:
+                # merge the experts into a single 3d tensor
+                for w_name in ["down_proj", "gate_proj", "up_proj"]:
+                    datas: list[Tensor] = []
+
+                    for xid in range(n_experts):
+                        ename = f"model.layers.{bid}.mlp.experts.{xid}.{w_name}.weight"
+                        datas.append(self._experts[bid][ename])
+                        del self._experts[bid][ename]
+
+                    data_torch = torch.stack(datas, dim=0)
+
+                    merged_name = f"model.layers.{bid}.mlp.experts.{w_name}.weight"
+
+                    yield from super().modify_tensors(data_torch, merged_name, bid)
+            return
+
+        del head_dim  # only used for the asserts above
+
+        yield from super().modify_tensors(data_torch, name, bid)
+
+    def prepare_tensors(self):
+        super().prepare_tensors()
+
+        if self._experts is not None:
+            experts = [k for d in self._experts for k in d.keys()]
+            if len(experts) > 0:
+                raise ValueError(f"Unprocessed experts: {experts}")
+
+
+
 @ModelBase.register("SarvamMoEForCausalLM", "modeling_sarvam_moe.SarvamMoEForCausalLM")
 class SarvamMoEModel(BailingMoeV2Model):
     model_arch = gguf.MODEL_ARCH.BAILINGMOE2

docs/longhaul.md

@@ -31,7 +31,7 @@ The normal startup warmup is skipped automatically in longhaul mode. Routed expe
 Longhaul currently requires:
 
 - all repeating layers on CPU, or all repeating layers on Metal
-- Qwen3.5 MoE, Laguna, or Inkling architecture
+- Qwen3.5 MoE, Laguna, Inkling, or Ling 3.0 (`bailingmoe3`) architecture
 - text generation without embeddings or LoRA adapters
 
 CPU mode is available wherever the CPU backend is supported. Metal mode requires
@@ -63,6 +63,15 @@ once, and independent expert slices are read concurrently where the platform
 supports positional reads. CPU and shared Metal buffers are populated directly;
 private Metal buffers use a staged fallback.
 
+Ling 3.0 support covers its hybrid KDA/MLA transformer and sigmoid-routed MoE.
+For Ling-3.0-flash, the router, score-correction bias, and shared expert remain
+resident while the 512 routed experts are streamed. Each token selects eight
+routed experts, so cache budgets with fewer than eight slots execute an MoE
+layer in multiple stages. AtomicChat's Ling GGUFs can be loaded directly, and
+the HF converter also recognizes `BailingMoeV3ForCausalLM` checkpoints. The
+converter omits the auxiliary MTP tensor block because Longhaul does not support
+MTP tensors.
+
 Inkling keeps its two shared experts resident and streams only the routed
 256-expert banks. Inkling-Small selects six routed experts per token. In the
 seven-shard Q8_0 model, one cache slot across all 40 MoE layers uses about

gguf-py/gguf/constants.py

@@ -244,7 +244,8 @@ class Keys:
         DT_B_C_RMS     = "{arch}.ssm.dt_b_c_rms"
 
     class KDA:
-        HEAD_DIM = "{arch}.kda.head_dim"
+        HEAD_DIM         = "{arch}.kda.head_dim"
+        GATE_LOWER_BOUND = "{arch}.kda.gate_lower_bound"
 
     class WKV:
         HEAD_SIZE = "{arch}.wkv.head_size"
@@ -517,6 +518,7 @@ class MODEL_ARCH(IntEnum):
     PLM              = auto()
     BAILINGMOE       = auto()
     BAILINGMOE2      = auto()
+    BAILINGMOE3      = auto()
     DOTS1            = auto()
     ARCEE            = auto()
     AFMOE            = auto()
@@ -664,6 +666,8 @@ class MODEL_TENSOR(IntEnum):
     SSM_CONV1D_Q         = auto() # Kimi Linear
     SSM_CONV1D_K         = auto() # Kimi Linear
     SSM_CONV1D_V         = auto() # Kimi Linear
+    SSM_F                = auto() # bailingmoe3 (no_kda_lora)
+    SSM_G                = auto() # bailingmoe3 (no_kda_lora)
     SSM_F_A              = auto() # Kimi Linear
     SSM_F_B              = auto() # Kimi Linear
     SSM_BETA             = auto() # Kimi Linear qwen3.5
@@ -1128,6 +1132,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = {
     MODEL_ARCH.PLM:              "plm",
     MODEL_ARCH.BAILINGMOE:       "bailingmoe",
     MODEL_ARCH.BAILINGMOE2:      "bailingmoe2",
+    MODEL_ARCH.BAILINGMOE3:      "bailingmoe3",
     MODEL_ARCH.DOTS1:            "dots1",
     MODEL_ARCH.ARCEE:            "arcee",
     MODEL_ARCH.AFMOE:            "afmoe",
@@ -1274,6 +1279,8 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
     MODEL_TENSOR.SSM_CONV1D_Q:              "blk.{bid}.ssm_conv1d_q",         # Kimi Linear
     MODEL_TENSOR.SSM_CONV1D_K:              "blk.{bid}.ssm_conv1d_k",         # Kimi Linear
     MODEL_TENSOR.SSM_CONV1D_V:              "blk.{bid}.ssm_conv1d_v",         # Kimi Linear
+    MODEL_TENSOR.SSM_F:                     "blk.{bid}.ssm_f",                # bailingmoe3
+    MODEL_TENSOR.SSM_G:                     "blk.{bid}.ssm_g",                # bailingmoe3
     MODEL_TENSOR.SSM_F_A:                   "blk.{bid}.ssm_f_a",              # Kimi Linear
     MODEL_TENSOR.SSM_F_B:                   "blk.{bid}.ssm_f_b",              # Kimi Linear
     MODEL_TENSOR.SSM_BETA:                  "blk.{bid}.ssm_beta",             # Kimi Linear qwen3.5
@@ -3849,6 +3856,43 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
         MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,
         MODEL_TENSOR.LAYER_OUT_NORM,
     ],
+    MODEL_ARCH.BAILINGMOE3: [
+        MODEL_TENSOR.TOKEN_EMBD,
+        MODEL_TENSOR.OUTPUT_NORM,
+        MODEL_TENSOR.OUTPUT,
+        MODEL_TENSOR.ATTN_NORM,
+        MODEL_TENSOR.ATTN_Q,
+        MODEL_TENSOR.ATTN_K,
+        MODEL_TENSOR.ATTN_V,
+        MODEL_TENSOR.ATTN_OUT,
+        MODEL_TENSOR.ATTN_GATE,
+        MODEL_TENSOR.ATTN_KV_A_MQA,
+        MODEL_TENSOR.ATTN_KV_B,
+        MODEL_TENSOR.ATTN_K_B,
+        MODEL_TENSOR.ATTN_V_B,
+        MODEL_TENSOR.ATTN_KV_A_NORM,
+        MODEL_TENSOR.SSM_CONV1D_Q,
+        MODEL_TENSOR.SSM_CONV1D_K,
+        MODEL_TENSOR.SSM_CONV1D_V,
+        MODEL_TENSOR.SSM_F,
+        MODEL_TENSOR.SSM_G,
+        MODEL_TENSOR.SSM_BETA,
+        MODEL_TENSOR.SSM_A,
+        MODEL_TENSOR.SSM_DT,
+        MODEL_TENSOR.SSM_NORM,
+        MODEL_TENSOR.FFN_NORM,
+        MODEL_TENSOR.FFN_GATE,
+        MODEL_TENSOR.FFN_DOWN,
+        MODEL_TENSOR.FFN_UP,
+        MODEL_TENSOR.FFN_GATE_INP,
+        MODEL_TENSOR.FFN_EXP_PROBS_B,
+        MODEL_TENSOR.FFN_GATE_EXP,
+        MODEL_TENSOR.FFN_DOWN_EXP,
+        MODEL_TENSOR.FFN_UP_EXP,
+        MODEL_TENSOR.FFN_GATE_SHEXP,
+        MODEL_TENSOR.FFN_DOWN_SHEXP,
+        MODEL_TENSOR.FFN_UP_SHEXP,
+    ],
     MODEL_ARCH.DOTS1: [
         MODEL_TENSOR.TOKEN_EMBD,
         MODEL_TENSOR.OUTPUT_NORM,

gguf-py/gguf/gguf_writer.py

@@ -1087,6 +1087,9 @@ class GGUFWriter:
     def add_kda_head_dim(self, value: int) -> None:
         self.add_uint32(Keys.KDA.HEAD_DIM.format(arch=self.arch), value)
 
+    def add_kda_gate_lower_bound(self, value: float) -> None:
+        self.add_float32(Keys.KDA.GATE_LOWER_BOUND.format(arch=self.arch), value)
+
     def add_tokenizer_model(self, model: str) -> None:
         self.add_string(Keys.Tokenizer.MODEL, model)
 

gguf-py/gguf/tensor_mapping.py

@@ -253,6 +253,7 @@ class TensorNameMap:
 
         # Attention query
         MODEL_TENSOR.ATTN_Q: (
+            "model.layers.{bid}.attention.q_proj",                       # bailingmoe3
             "model.layers.{bid}.self_attn.q_proj",                       # llama-hf nemotron olmoe olmo2 phimoe
             "layers.{bid}.self_attn.q_proj",                             # embeddinggemma
             "model.layers.{bid}.self_attn.q_proj_no_perm",               # llama-custom
@@ -273,6 +274,7 @@ class TensorNameMap:
 
         # Attention key
         MODEL_TENSOR.ATTN_K: (
+            "model.layers.{bid}.attention.k_proj",                     # bailingmoe3
             "model.layers.{bid}.self_attn.k_proj",                     # llama-hf nemotron olmoe olmo2 phimoe
             "layers.{bid}.self_attn.k_proj",                           # embeddinggemma
             "model.layers.{bid}.self_attn.k_proj_no_perm",             # llama-custom
@@ -294,6 +296,7 @@ class TensorNameMap:
 
         # Attention value
         MODEL_TENSOR.ATTN_V: (
+            "model.layers.{bid}.attention.v_proj",                       # bailingmoe3
             "model.layers.{bid}.self_attn.v_proj",                       # llama-hf nemotron olmoe olmo2 phimoe
             "layers.{bid}.self_attn.v_proj",                             # embeddinggemma
             "layers.{bid}.attention.wv",                                 # llama-pth
@@ -314,6 +317,7 @@ class TensorNameMap:
 
         # Attention output
         MODEL_TENSOR.ATTN_OUT: (
+            "model.layers.{bid}.attention.o_proj",                         # bailingmoe3
             "gpt_neox.layers.{bid}.attention.dense",                        # gptneox
             "transformer.h.{bid}.attn.c_proj",                              # gpt2 refact qwen jais
             "transformer.blocks.{bid}.attn.out_proj",                       # mpt
@@ -825,6 +829,7 @@ class TensorNameMap:
         ),
 
         MODEL_TENSOR.SSM_DT: (
+            "model.layers.{bid}.attention.dt_proj",     # bailingmoe3
             "model.layers.{bid}.dt_proj",               # mamba-hf
             "backbone.layers.{bid}.mixer.dt_proj",      # mamba
             "model.layers.{bid}.mamba.dt_proj",         # jamba falcon-h1 granite-hybrid
@@ -840,6 +845,7 @@ class TensorNameMap:
         ),
 
         MODEL_TENSOR.SSM_A: (
+            "model.layers.{bid}.attention.A_log",       # bailingmoe3
             "model.layers.{bid}.A_log",               # mamba-hf
             "backbone.layers.{bid}.mixer.A_log",      # mamba
             "model.layers.{bid}.mamba.A_log",         # jamba falcon-h1 granite-hybrid
@@ -872,6 +878,7 @@ class TensorNameMap:
             "model.layers.{bid}.linear_attn.norm",  # qwen3next
             "backbone.layers.{bid}.mixer.norm",     # mamba2
             "model.layers.{bid}.self_attn.o_norm",  # kimi
+            "model.layers.{bid}.attention.o_norm",  # bailingmoe3
         ),
 
         MODEL_TENSOR.SSM_OUT: (
@@ -893,12 +900,21 @@ class TensorNameMap:
         # Kimi Linear KDA (using SSM_ prefix for consistency)
         MODEL_TENSOR.SSM_CONV1D_Q: (
             "model.layers.{bid}.self_attn.q_conv1d",
+            "model.layers.{bid}.attention.q_conv1d",  # bailingmoe3
         ),
         MODEL_TENSOR.SSM_CONV1D_K: (
             "model.layers.{bid}.self_attn.k_conv1d",
+            "model.layers.{bid}.attention.k_conv1d",  # bailingmoe3
         ),
         MODEL_TENSOR.SSM_CONV1D_V: (
             "model.layers.{bid}.self_attn.v_conv1d",
+            "model.layers.{bid}.attention.v_conv1d",  # bailingmoe3
+        ),
+        MODEL_TENSOR.SSM_F: (
+            "model.layers.{bid}.attention.f_proj",  # bailingmoe3
+        ),
+        MODEL_TENSOR.SSM_G: (
+            "model.layers.{bid}.attention.g_proj",  # bailingmoe3
         ),
         MODEL_TENSOR.SSM_F_A: (
             "model.layers.{bid}.self_attn.f_a_proj",
@@ -909,6 +925,7 @@ class TensorNameMap:
         MODEL_TENSOR.SSM_BETA: (
             "model.layers.{bid}.linear_attn.in_proj_b",  # qwen3.5
             "model.layers.{bid}.self_attn.b_proj",       # Kimi Linear
+            "model.layers.{bid}.attention.b_proj",       # bailingmoe3
         ),
         MODEL_TENSOR.SSM_G_A: (
             "model.layers.{bid}.self_attn.g_a_proj",
@@ -1097,6 +1114,7 @@ class TensorNameMap:
         ),
 
         MODEL_TENSOR.ATTN_KV_A_MQA: (
+            "model.layers.{bid}.attention.kv_a_proj_with_mqa", # bailingmoe3
             "model.layers.{bid}.self_attn.kv_a_proj_with_mqa", # deepseek2
             "layers.{bid}.attention.wkv_a_with_mqa",           # mistral-large
         ),
@@ -1121,6 +1139,7 @@ class TensorNameMap:
         ),
 
         MODEL_TENSOR.ATTN_KV_A_NORM: (
+            "model.layers.{bid}.attention.kv_a_layernorm", # bailingmoe3
             "model.layers.{bid}.self_attn.kv_a_layernorm", # deepseek2
             "layers.{bid}.attention.kv_a_norm",            # mistral-large
         ),

src/llama-arch.cpp

@@ -105,6 +105,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
     { LLM_ARCH_PLM,              "plm"              },
     { LLM_ARCH_BAILINGMOE,       "bailingmoe"       },
     { LLM_ARCH_BAILINGMOE2,      "bailingmoe2"      },
+    { LLM_ARCH_BAILINGMOE3,      "bailingmoe3"      },
     { LLM_ARCH_DOTS1,            "dots1"            },
     { LLM_ARCH_ARCEE,            "arcee"            },
     { LLM_ARCH_AFMOE,            "afmoe"            },
@@ -303,7 +304,8 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
     { LLM_KV_SSM_GROUP_COUNT,    "%s.ssm.group_count"    },
     { LLM_KV_SSM_DT_B_C_RMS,     "%s.ssm.dt_b_c_rms"     },
 
-    { LLM_KV_KDA_HEAD_DIM, "%s.kda.head_dim" },
+    { LLM_KV_KDA_HEAD_DIM,         "%s.kda.head_dim"         },
+    { LLM_KV_KDA_GATE_LOWER_BOUND, "%s.kda.gate_lower_bound" },
 
     { LLM_KV_WKV_HEAD_SIZE, "%s.wkv.head_size" },
 
@@ -459,6 +461,8 @@ static const std::map<llm_tensor, const char *> LLM_TENSOR_NAMES = {
     { LLM_TENSOR_SSM_CONV1D_Q,                           "blk.%d.ssm_conv1d_q" },
     { LLM_TENSOR_SSM_CONV1D_K,                           "blk.%d.ssm_conv1d_k" },
     { LLM_TENSOR_SSM_CONV1D_V,                           "blk.%d.ssm_conv1d_v" },
+    { LLM_TENSOR_SSM_F,                                  "blk.%d.ssm_f" },
+    { LLM_TENSOR_SSM_G,                                  "blk.%d.ssm_g" },
     { LLM_TENSOR_SSM_F_A,                                "blk.%d.ssm_f_a" },
     { LLM_TENSOR_SSM_F_B,                                "blk.%d.ssm_f_b" },
     { LLM_TENSOR_SSM_BETA,                               "blk.%d.ssm_beta" },
@@ -762,6 +766,8 @@ static const std::map<llm_tensor, llm_tensor_info> LLM_TENSOR_INFOS = {
     {LLM_TENSOR_SSM_CONV1D_Q,               {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
     {LLM_TENSOR_SSM_CONV1D_K,               {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
     {LLM_TENSOR_SSM_CONV1D_V,               {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
+    {LLM_TENSOR_SSM_F,                      {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
+    {LLM_TENSOR_SSM_G,                      {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
     {LLM_TENSOR_SSM_F_A,                    {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
     {LLM_TENSOR_SSM_F_B,                    {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
     {LLM_TENSOR_SSM_BETA,                   {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
@@ -995,6 +1001,7 @@ bool llm_arch_is_hybrid(const llm_arch & arch) {
         case LLM_ARCH_NEMOTRON_H:
         case LLM_ARCH_NEMOTRON_H_MOE:
         case LLM_ARCH_QWEN3NEXT:
+        case LLM_ARCH_BAILINGMOE3:
         case LLM_ARCH_KIMI_LINEAR:
         case LLM_ARCH_QWEN35:
         case LLM_ARCH_QWEN35MOE:
@@ -1054,6 +1061,7 @@ bool llm_arch_supports_sm_tensor(const llm_arch & arch) {
         case LLM_ARCH_MINIMAX_M2:
         case LLM_ARCH_MINIMAX_M3:
         case LLM_ARCH_MISTRAL4:
+        case LLM_ARCH_BAILINGMOE3:
         case LLM_ARCH_KIMI_LINEAR:
         case LLM_ARCH_INKLING:
             return false;

src/llama-arch.h

@@ -110,6 +110,7 @@ enum llm_arch {
     LLM_ARCH_PLM,
     LLM_ARCH_BAILINGMOE,
     LLM_ARCH_BAILINGMOE2,
+    LLM_ARCH_BAILINGMOE3,
     LLM_ARCH_DOTS1,
     LLM_ARCH_ARCEE,
     LLM_ARCH_AFMOE,
@@ -309,6 +310,7 @@ enum llm_kv {
     LLM_KV_SSM_DT_B_C_RMS,
 
     LLM_KV_KDA_HEAD_DIM,
+    LLM_KV_KDA_GATE_LOWER_BOUND,
 
     LLM_KV_WKV_HEAD_SIZE,
 
@@ -490,6 +492,8 @@ enum llm_tensor {
     LLM_TENSOR_SSM_CONV1D_Q,        // kimi: Q conv1d weight
     LLM_TENSOR_SSM_CONV1D_K,        // kimi: K conv1d weight
     LLM_TENSOR_SSM_CONV1D_V,        // kimi: V conv1d weight
+    LLM_TENSOR_SSM_F,               // bailingmoe3: full-rank forget gate projection
+    LLM_TENSOR_SSM_G,               // bailingmoe3: full-rank output gate projection
     LLM_TENSOR_SSM_F_A,             // kimi: forget gate projection A
     LLM_TENSOR_SSM_F_B,             // kimi: forget gate projection B
     LLM_TENSOR_SSM_BETA,            // kimi: beta mixing coefficient and qwen3.5

src/llama-hparams.h

@@ -174,6 +174,7 @@ struct llama_hparams {
 
     // for Kimi Linear KDA
     uint32_t n_embd_head_kda = 0;
+    float    kda_gate_lower_bound = 0.0f;
 
     bool ssm_dt_b_c_rms = false;
 

src/llama-model-saver.cpp

@@ -121,6 +121,7 @@ void llama_model_saver::add_kv(const enum llm_kv key, const Container & value, c
 }
 // instantiate for external usage:
 template void llama_model_saver::add_kv<std::vector<uint32_t>>(const enum llm_kv, const std::vector<uint32_t> &, const bool);
+template void llama_model_saver::add_kv<std::vector<float>>(const enum llm_kv, const std::vector<float> &, const bool);
 
 void llama_model_saver::add_kv(const enum llm_kv key, const std::vector<std::string> & value) {
     std::vector<const char *> tmp(value.size());
@@ -215,8 +216,8 @@ void llama_model_saver::add_kv_from_model() {
     add_kv(LLM_KV_EXPERT_FEED_FORWARD_LENGTH,        hparams.n_ff_exp);
     add_kv(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp);
     add_kv(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_chexp);
-    add_kv(LLM_KV_SWIGLU_CLAMP_EXP,                  hparams.swiglu_clamp_exp);
+    add_kv(LLM_KV_SWIGLU_CLAMP_EXP,                  hparams.swiglu_clamp_exp, true);
-    add_kv(LLM_KV_SWIGLU_CLAMP_SHEXP,                hparams.swiglu_clamp_shexp);
+    add_kv(LLM_KV_SWIGLU_CLAMP_SHEXP,                hparams.swiglu_clamp_shexp, true);
     add_kv(LLM_KV_USE_PARALLEL_RESIDUAL,             hparams.use_par_res);
     // add_kv(LLM_KV_TENSOR_DATA_LAYOUT,                ???);
     add_kv(LLM_KV_EXPERT_COUNT,                      hparams.n_expert);
@@ -319,6 +320,7 @@ void llama_model_saver::add_kv_from_model() {
     add_kv(LLM_KV_SSM_DT_B_C_RMS,                    hparams.ssm_dt_b_c_rms);
 
     add_kv(LLM_KV_KDA_HEAD_DIM,                      hparams.n_embd_head_kda);
+    add_kv(LLM_KV_KDA_GATE_LOWER_BOUND,              hparams.kda_gate_lower_bound);
 
     add_kv(LLM_KV_WKV_HEAD_SIZE,                     hparams.wkv_head_size);
 

src/llama-model.cpp

@@ -308,6 +308,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params
             return new llama_model_dflash(params);
         case LLM_ARCH_MIMO2:
             return new llama_model_mimo2(params);
+        case LLM_ARCH_BAILINGMOE3:
+            return new llama_model_bailingmoe3(params);
         case LLM_ARCH_KIMI_LINEAR:
             return new llama_model_kimi_linear(params);
         case LLM_ARCH_STEP35:
@@ -814,6 +816,7 @@ const char * llm_type_name(llm_type type) {
         case LLM_TYPE_24B_A2B:       return "24B.A2B";
         case LLM_TYPE_26B_A4B:       return "26B.A4B";
         case LLM_TYPE_30B_A3B:       return "30B.A3B";
+        case LLM_TYPE_124B_A5B:      return "124B.A5B";
         case LLM_TYPE_31B_A3_5B:     return "31B.A3.5B";
         case LLM_TYPE_35B_A3B:       return "35B.A3B";
         case LLM_TYPE_48B_A3B:       return "48B.A3B";
@@ -1358,8 +1361,8 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
         }
 
         if (params.load_mode == LLAMA_LOAD_MODE_LONGHAUL) {
-            if (arch != LLM_ARCH_QWEN35MOE && arch != LLM_ARCH_LAGUNA && arch != LLM_ARCH_INKLING) {
+            if (arch != LLM_ARCH_QWEN35MOE && arch != LLM_ARCH_LAGUNA && arch != LLM_ARCH_INKLING && arch != LLM_ARCH_BAILINGMOE3) {
-                throw std::runtime_error("longhaul currently requires a qwen35moe, laguna, or inkling model");
+                throw std::runtime_error("longhaul currently requires a qwen35moe, laguna, inkling, or bailingmoe3 model");
             }
             if (hparams.n_layer_nextn != 0) {
                 throw std::runtime_error("longhaul does not support MTP tensors");
@@ -2538,6 +2541,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
         case LLM_ARCH_GRANITE_HYBRID:
         case LLM_ARCH_CHAMELEON:
         case LLM_ARCH_BAILINGMOE:
+        case LLM_ARCH_BAILINGMOE3:
         case LLM_ARCH_NEO_BERT:
         case LLM_ARCH_SMOLLM3:
         case LLM_ARCH_ARCEE:

src/llama-model.h

@@ -124,6 +124,7 @@ enum llm_type {
     LLM_TYPE_24B_A2B, // lfm2moe
     LLM_TYPE_26B_A4B, // Gemma4
     LLM_TYPE_30B_A3B,
+    LLM_TYPE_124B_A5B, // Ling 3.0 flash
     LLM_TYPE_31B_A3_5B,
     LLM_TYPE_35B_A3B, // Qwen3.5
     LLM_TYPE_48B_A3B, // Kimi Linear
@@ -504,6 +505,8 @@ struct llama_layer {
     struct ggml_tensor * ssm_q_conv = nullptr;
     struct ggml_tensor * ssm_k_conv = nullptr;
     struct ggml_tensor * ssm_v_conv = nullptr;
+    struct ggml_tensor * ssm_f      = nullptr;
+    struct ggml_tensor * ssm_g      = nullptr;
     struct ggml_tensor * ssm_f_a    = nullptr;
     struct ggml_tensor * ssm_f_b    = nullptr;
     struct ggml_tensor * ssm_beta   = nullptr;

src/models/bailingmoe3.cpp

@@ -0,0 +1,461 @@
+#include "models.h"
+#include "llama-memory-recurrent.h"
+
+// Ling 3.0 (BailingMoeV3): hybrid linear architecture, 5 KDA layers per 1 gated MLA layer,
+// on top of a bailing MoE (sigmoid router with expert bias and group-limited routing).
+//
+// Differences to Kimi-Linear, which shares the KDA + MLA layout:
+//   - KDA forget/output gates are full-rank projections (config: no_kda_lora = true)
+//   - the KDA gate uses the safe-gate form (config: kda_safe_gate, kda_lower_bound):
+//     lower_bound * sigmoid(exp(A_log) * (f_proj(x) + dt_bias)) instead of
+//     -exp(A_log) * softplus(f_proj(x) + dt_bias)
+//   - MLA layers do use RoPE (partial, interleaved -> ggml NORM) and carry a head-wise
+//     sigmoid output gate applied before the output projection
+
+void llama_model_bailingmoe3::load_arch_hparams(llama_model_loader & ml) {
+    ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
+    ml.get_key(LLM_KV_ATTENTION_KEY_LENGTH_MLA,    hparams.n_embd_head_k_mla_impl);
+    ml.get_key(LLM_KV_ATTENTION_VALUE_LENGTH_MLA,  hparams.n_embd_head_v_mla_impl);
+    ml.get_key(LLM_KV_ATTENTION_KV_LORA_RANK,      hparams.n_lora_kv);
+    ml.get_key(LLM_KV_SSM_CONV_KERNEL,             hparams.ssm_d_conv);
+    ml.get_key(LLM_KV_KDA_HEAD_DIM,                hparams.n_embd_head_kda);
+    ml.get_key(LLM_KV_KDA_GATE_LOWER_BOUND,        hparams.kda_gate_lower_bound);
+
+    // n_head_kv == 0 marks the KDA (recurrent) layers, the rest are full attention
+    for (uint32_t i = 0; i < hparams.n_layer(); ++i) {
+        hparams.is_recr_impl[i] = hparams.n_head_kv(i) == 0;
+    }
+
+    ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH,        hparams.n_ff_exp);
+    ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, false);
+    ml.get_key(LLM_KV_EXPERT_SHARED_COUNT,               hparams.n_expert_shared);
+    ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT,         hparams.n_layer_dense_lead, false);
+    ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE,              hparams.expert_weights_scale, false);
+    ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM,               hparams.expert_weights_norm, false);
+    ml.get_key(LLM_KV_EXPERT_GATING_FUNC,                hparams.expert_gating_func);
+    ml.get_key(LLM_KV_EXPERT_GROUP_COUNT,                hparams.n_expert_groups, false);
+    ml.get_key(LLM_KV_EXPERT_GROUP_USED_COUNT,           hparams.n_group_used, false);
+
+    // trained per-layer SwiGLU clamps (config: expert_swiglu_limit_list / share_expert_swiglu_limit_list)
+    ml.get_key_or_arr(LLM_KV_SWIGLU_CLAMP_EXP,   hparams.swiglu_clamp_exp,   hparams.n_layer_all, false);
+    ml.get_key_or_arr(LLM_KV_SWIGLU_CLAMP_SHEXP, hparams.swiglu_clamp_shexp, hparams.n_layer_all, false);
+
+    switch (hparams.n_layer()) {
+        case 42: type = LLM_TYPE_124B_A5B; break; // Ling-3.0-flash
+        default: type = LLM_TYPE_UNKNOWN;
+    }
+}
+
+void llama_model_bailingmoe3::load_arch_tensors(llama_model_loader &) {
+    LLAMA_LOAD_LOCALS;
+
+    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
+
+    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);
+    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      "weight"), {n_embd, n_vocab}, 0);
+
+    const int64_t head_dim_kda = hparams.n_embd_head_kda;
+    const int64_t d_inner      = head_dim_kda * n_head;
+    const int64_t ssm_d_conv   = hparams.ssm_d_conv;
+    const int expert_flags = params.load_mode == LLAMA_LOAD_MODE_LONGHAUL ? TENSOR_LONGHAUL : 0;
+
+    for (int i = 0; i < n_layer; ++i) {
+        auto & layer = layers[i];
+
+        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
+
+        if (hparams.is_recr(i)) {
+            // === KDA (linear attention) layer ===
+            // conv1d weights are 4D in GGUF, quantization may drop the trailing 1
+            layer.ssm_q_conv = create_tensor(tn(LLM_TENSOR_SSM_CONV1D_Q, "weight", i), {ssm_d_conv, 1, d_inner, 1}, TENSOR_NOT_REQUIRED);
+            if (!layer.ssm_q_conv) {
+                layer.ssm_q_conv = create_tensor(tn(LLM_TENSOR_SSM_CONV1D_Q, "weight", i), {ssm_d_conv, 1, d_inner}, 0);
+            }
+            layer.ssm_k_conv = create_tensor(tn(LLM_TENSOR_SSM_CONV1D_K, "weight", i), {ssm_d_conv, 1, d_inner, 1}, TENSOR_NOT_REQUIRED);
+            if (!layer.ssm_k_conv) {
+                layer.ssm_k_conv = create_tensor(tn(LLM_TENSOR_SSM_CONV1D_K, "weight", i), {ssm_d_conv, 1, d_inner}, 0);
+            }
+            layer.ssm_v_conv = create_tensor(tn(LLM_TENSOR_SSM_CONV1D_V, "weight", i), {ssm_d_conv, 1, d_inner, 1}, TENSOR_NOT_REQUIRED);
+            if (!layer.ssm_v_conv) {
+                layer.ssm_v_conv = create_tensor(tn(LLM_TENSOR_SSM_CONV1D_V, "weight", i), {ssm_d_conv, 1, d_inner}, 0);
+            }
+
+            create_tensor_qkv(layer, i, n_embd, d_inner, d_inner, d_inner, 0);
+
+            // full-rank forget/output gate projections (no_kda_lora)
+            layer.ssm_f = create_tensor(tn(LLM_TENSOR_SSM_F, "weight", i), {n_embd, d_inner}, 0);
+            layer.ssm_g = create_tensor(tn(LLM_TENSOR_SSM_G, "weight", i), {n_embd, d_inner}, 0);
+
+            layer.ssm_beta = create_tensor(tn(LLM_TENSOR_SSM_BETA, "weight", i), {n_embd, n_head}, 0);
+
+            // note: the conversion script stores exp(A_log)
+            layer.ssm_a = create_tensor(tn(LLM_TENSOR_SSM_A, i), {1, n_head, 1, 1}, TENSOR_NOT_REQUIRED);
+            if (!layer.ssm_a) {
+                layer.ssm_a = create_tensor(tn(LLM_TENSOR_SSM_A, i), {1, n_head}, 0);
+            }
+
+            layer.ssm_dt_b   = create_tensor(tn(LLM_TENSOR_SSM_DT,   "bias",   i), {d_inner}, 0);
+            layer.ssm_o_norm = create_tensor(tn(LLM_TENSOR_SSM_NORM, "weight", i), {head_dim_kda}, 0);
+
+            layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {d_inner, n_embd}, 0);
+        } else {
+            // === gated MLA layer ===
+            const int64_t kv_lora_rank      = hparams.n_lora_kv;
+            const int64_t n_embd_head_k_mla = hparams.n_embd_head_k_mla();
+            const int64_t n_embd_head_v_mla = hparams.n_embd_head_v_mla();
+            const int64_t qk_rope_head_dim  = hparams.n_rot();
+
+            // Ling 3.0 has no query compression (q_lora_rank = null)
+            layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_head * n_embd_head_k_mla}, 0);
+
+            layer.attn_kv_a_norm = create_tensor(tn(LLM_TENSOR_ATTN_KV_A_NORM, "weight", i), {kv_lora_rank}, 0);
+            layer.wkv_a_mqa      = create_tensor(tn(LLM_TENSOR_ATTN_KV_A_MQA,  "weight", i), {n_embd, kv_lora_rank + qk_rope_head_dim}, 0);
+
+            layer.wkv_b = create_tensor(tn(LLM_TENSOR_ATTN_KV_B, "weight", i),
+                {kv_lora_rank, n_head * (n_embd_head_k_mla - qk_rope_head_dim + n_embd_head_v_mla)}, TENSOR_NOT_REQUIRED | TENSOR_SKIP_IF_VIRTUAL);
+            if (!layer.wkv_b) { // MLA KV cache enabled
+                layer.wk_b = create_tensor(tn(LLM_TENSOR_ATTN_K_B, "weight", i), {n_embd_head_k_mla - qk_rope_head_dim, kv_lora_rank, n_head}, 0);
+                layer.wv_b = create_tensor(tn(LLM_TENSOR_ATTN_V_B, "weight", i), {kv_lora_rank, n_embd_head_v_mla, n_head}, 0);
+            }
+
+            // head-wise output gate (gated_attention_proj_granularity_type = "head_wise")
+            layer.wqkv_gate = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "weight", i), {n_embd, n_head}, 0);
+
+            layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_head * n_embd_head_v_mla, n_embd}, 0);
+        }
+
+        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
+
+        if (i < (int) hparams.n_layer_dense_lead) {
+            layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0);
+            layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, 0);
+            layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   "weight", i), {n_embd, n_ff}, 0);
+        } else {
+            const int64_t n_ff_exp   = hparams.n_ff_exp;
+            const int64_t n_ff_shexp = hparams.n_ff_shexp > 0 ? hparams.n_ff_shexp : n_ff_exp * hparams.n_expert_shared;
+
+            layer.ffn_gate_inp    = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP,    "weight", i), {n_embd, n_expert}, 0);
+            layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias",   i), {n_expert}, 0);
+
+            layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, expert_flags);
+            layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, expert_flags);
+            layer.ffn_up_exps   = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS,   "weight", i), {n_embd, n_ff_exp, n_expert}, expert_flags);
+
+            layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), {n_embd, n_ff_shexp}, 0);
+            layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), {n_ff_shexp, n_embd}, 0);
+            layer.ffn_up_shexp   = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP,   "weight", i), {n_embd, n_ff_shexp}, 0);
+        }
+    }
+}
+
+std::unique_ptr<llm_graph_context> llama_model_bailingmoe3::build_arch_graph(const llm_graph_params & params) const {
+    return std::make_unique<graph>(*this, params);
+}
+
+// projection + causal conv1d + silu for one of Q, K, V (qkv: 0 = Q, 1 = K, 2 = V)
+static ggml_tensor * bailingmoe3_causal_conv1d(
+        ggml_cgraph * gf, ggml_context * ctx0,
+        ggml_tensor * conv_states_all, ggml_tensor * conv_state_all,
+        int64_t qkv, ggml_tensor * x, ggml_tensor * proj_w, ggml_tensor * conv_w,
+        int64_t d_conv, int64_t head_dim, int64_t n_head,
+        int64_t n_seq_tokens, int64_t n_seqs, int64_t n_tokens, int64_t kv_head) {
+    const int64_t d_inner         = head_dim * n_head;
+    const int64_t conv_state_size = (d_conv - 1) * d_inner;
+    const int64_t n_embd_r_total  = 3 * conv_state_size; // Q + K + V
+
+    ggml_tensor * conv_state_x = ggml_view_3d(ctx0, conv_state_all, d_conv - 1, d_inner, n_seqs,
+        (d_conv - 1) * ggml_element_size(conv_state_all),
+        n_embd_r_total * ggml_element_size(conv_state_all),
+        qkv * conv_state_size * ggml_element_size(conv_state_all));
+
+    ggml_tensor * x_proj = ggml_mul_mat(ctx0, proj_w, x);
+    ggml_tensor * x_3d   = ggml_reshape_3d(ctx0, x_proj, d_inner, n_seq_tokens, n_seqs);
+
+    ggml_tensor * conv_x = ggml_concat(ctx0, conv_state_x, ggml_transpose(ctx0, x_3d), 0);
+
+    ggml_tensor * last_conv_x = ggml_view_3d(ctx0, conv_x, d_conv - 1, d_inner, n_seqs,
+        conv_x->nb[1], conv_x->nb[2], n_seq_tokens * conv_x->nb[0]);
+    ggml_build_forward_expand(gf,
+        ggml_cpy(ctx0, last_conv_x,
+            ggml_view_3d(ctx0, conv_states_all,
+                d_conv - 1, d_inner, n_seqs,
+                (d_conv - 1) * ggml_element_size(conv_states_all),
+                n_embd_r_total * ggml_element_size(conv_states_all),
+                (kv_head * n_embd_r_total + qkv * conv_state_size) * ggml_element_size(conv_states_all))));
+
+    ggml_tensor * conv_weight = ggml_reshape_2d(ctx0, conv_w, d_conv, d_inner);
+
+    ggml_tensor * Xcur = ggml_ssm_conv(ctx0, conv_x, conv_weight);
+    Xcur = ggml_reshape_2d(ctx0, Xcur, d_inner, n_tokens);
+    Xcur = ggml_silu(ctx0, Xcur);
+
+    return ggml_reshape_4d(ctx0, Xcur, head_dim, n_head, n_seq_tokens, n_seqs);
+}
+
+llama_model_bailingmoe3::graph::graph(const llama_model & model, const llm_graph_params & params) :
+    llm_build_delta_net_base(params), model(model) {
+    ggml_tensor * cur;
+    ggml_tensor * inpL;
+
+    inpL = build_inp_embd(model.tok_embd);
+    cb(inpL, "inp_embd", -1);
+
+    // MLA layers use RoPE, so positions are needed
+    ggml_tensor * inp_pos = build_inp_pos();
+
+    auto * inp_kv      = !hparams.is_mla() ? build_inp_mem_hybrid()   : nullptr;
+    auto * inp_k       =  hparams.is_mla() ? build_inp_mem_hybrid_k() : nullptr;
+    auto * inp_rs      =  hparams.is_mla() ? inp_k->get_recr() : inp_kv->get_recr();
+    auto * inp_attn_kv = !hparams.is_mla() ? inp_kv->get_attn() : nullptr;
+    auto * inp_attn_k  =  hparams.is_mla() ? inp_k->get_attn()  : nullptr;
+
+    ggml_tensor * inp_out_ids = build_inp_out_ids();
+
+    const int64_t n_head      = hparams.n_head();
+    const int64_t head_dim    = hparams.n_embd_head_kda;
+    const int64_t d_conv      = hparams.ssm_d_conv;
+    const int64_t d_inner     = n_head * head_dim;
+    const int64_t n_seqs      = ubatch.n_seqs;
+    const int64_t n_seq_tokens = ubatch.n_seq_tokens;
+
+    GGML_ASSERT(n_seqs != 0);
+    GGML_ASSERT(ubatch.equal_seqs());
+    GGML_ASSERT(ubatch.n_tokens == n_seq_tokens * n_seqs);
+
+    const int64_t n_embd_head_k_mla  = hparams.n_embd_head_k_mla();
+    const int64_t n_embd_head_v_mla  = hparams.n_embd_head_v_mla();
+    const int64_t kv_lora_rank       = hparams.n_lora_kv;
+    const int64_t n_embd_head_qk_rope = hparams.n_rot();
+    const int64_t n_embd_head_qk_nope = n_embd_head_k_mla - n_embd_head_qk_rope;
+
+    const float kq_scale_mla = 1.0f / sqrtf(float(n_embd_head_k_mla));
+
+    for (int il = 0; il < n_layer; ++il) {
+        const auto & layer = model.layers[il];
+
+        ggml_tensor * inpSA = inpL;
+
+        cur = build_norm(inpL, layer.attn_norm, NULL, LLM_NORM_RMS, il);
+        cb(cur, "attn_norm", il);
+
+        // the MLA output gate is computed from this same normed hidden state
+        ggml_tensor * attn_inp = cur;
+
+        if (hparams.is_recr(il)) {
+            // === KDA ===
+            const auto * mctx_cur = inp_rs->mctx;
+            const auto kv_head = mctx_cur->get_head();
+
+            ggml_tensor * conv_states_all = mctx_cur->get_r_l(il);
+            ggml_tensor * conv_state_all  = build_rs(inp_rs, conv_states_all, hparams.n_embd_r(), n_seqs);
+
+            ggml_tensor * Qcur = bailingmoe3_causal_conv1d(gf, ctx0, conv_states_all, conv_state_all, 0, cur, layer.wq, layer.ssm_q_conv, d_conv, head_dim, n_head, n_seq_tokens, n_seqs, n_tokens, kv_head);
+            ggml_tensor * Kcur = bailingmoe3_causal_conv1d(gf, ctx0, conv_states_all, conv_state_all, 1, cur, layer.wk, layer.ssm_k_conv, d_conv, head_dim, n_head, n_seq_tokens, n_seqs, n_tokens, kv_head);
+            ggml_tensor * Vcur = bailingmoe3_causal_conv1d(gf, ctx0, conv_states_all, conv_state_all, 2, cur, layer.wv, layer.ssm_v_conv, d_conv, head_dim, n_head, n_seq_tokens, n_seqs, n_tokens, kv_head);
+
+            // safe gate: g1 = lower_bound * sigmoid(exp(A_log) * (f_proj(x) + dt_bias))
+            // note: ssm_a holds exp(A_log), applied by the conversion script
+            ggml_tensor * g1 = ggml_mul_mat(ctx0, layer.ssm_f, cur);
+            g1 = ggml_add(ctx0, g1, layer.ssm_dt_b);
+            g1 = ggml_reshape_3d(ctx0, g1, head_dim, n_head, n_tokens);
+
+            ggml_tensor * A = ggml_reshape_3d(ctx0, layer.ssm_a, 1, n_head, 1);
+            g1 = ggml_mul(ctx0, g1, A);
+            g1 = ggml_sigmoid(ctx0, g1);
+            g1 = ggml_scale(ctx0, g1, hparams.kda_gate_lower_bound);
+            cb(g1, "kda_g1", il);
+
+            g1 = ggml_reshape_4d(ctx0, g1, head_dim, n_head, n_seq_tokens, n_seqs);
+
+            ggml_tensor * beta = ggml_mul_mat(ctx0, layer.ssm_beta, cur);
+            beta = ggml_reshape_4d(ctx0, beta, 1, n_head, n_seq_tokens, n_seqs);
+            beta = ggml_sigmoid(ctx0, beta);
+            cb(beta, "kda_beta", il);
+
+            cur = ggml_reshape_3d(ctx0, cur, cur->ne[0], n_seq_tokens, n_seqs);
+
+            ggml_tensor * ssm_states_all = mctx_cur->get_s_l(il);
+            ggml_tensor * state = build_rs(inp_rs, ssm_states_all, hparams.n_embd_s(), n_seqs);
+            state = ggml_reshape_4d(ctx0, state, head_dim, head_dim, n_head, n_seqs);
+
+            const float eps_norm = hparams.f_norm_rms_eps;
+
+            Qcur = ggml_l2_norm(ctx0, Qcur, eps_norm);
+            Kcur = ggml_l2_norm(ctx0, Kcur, eps_norm);
+
+            auto attn_out = build_delta_net(Qcur, Kcur, Vcur, g1, beta, state, il);
+
+            ggml_tensor * output    = ggml_cont(ctx0, attn_out.first);
+            ggml_tensor * new_state = attn_out.second;
+
+            ggml_build_forward_expand(gf,
+                ggml_cpy(ctx0, new_state,
+                    ggml_view_1d(ctx0, ssm_states_all, hparams.n_embd_s() * n_seqs,
+                                 kv_head * hparams.n_embd_s() * ggml_element_size(ssm_states_all))));
+
+            // output gate: RMSNorm(o) * sigmoid(g_proj(x))
+            ggml_tensor * cur_2d = ggml_reshape_2d(ctx0, cur, cur->ne[0], n_seq_tokens * n_seqs);
+            ggml_tensor * g2 = ggml_mul_mat(ctx0, layer.ssm_g, cur_2d);
+            g2 = ggml_reshape_3d(ctx0, g2, head_dim, n_head, n_seq_tokens * n_seqs);
+
+            ggml_tensor * attn_out_final = ggml_reshape_3d(ctx0, output, head_dim, n_head, n_seq_tokens * n_seqs);
+            ggml_tensor * normed = build_norm(attn_out_final, layer.ssm_o_norm, nullptr, LLM_NORM_RMS, il);
+            ggml_tensor * gated  = ggml_mul(ctx0, normed, ggml_sigmoid(ctx0, g2));
+
+            gated = ggml_cont_2d(ctx0, gated, d_inner, n_tokens);
+            cur = ggml_mul_mat(ctx0, layer.wo, gated);
+            cb(cur, "kda_out", il);
+        } else {
+            // === gated MLA ===
+            ggml_tensor * q = ggml_mul_mat(ctx0, layer.wq, cur);
+            cb(q, "q", il);
+
+            ggml_tensor * q_nope = ggml_view_3d(ctx0, q, n_embd_head_qk_nope, n_head, n_tokens,
+                ggml_row_size(q->type, n_embd_head_k_mla),
+                ggml_row_size(q->type, n_embd_head_k_mla) * n_head, 0);
+            ggml_tensor * q_pe = ggml_view_3d(ctx0, q, n_embd_head_qk_rope, n_head, n_tokens,
+                ggml_row_size(q->type, n_embd_head_k_mla),
+                ggml_row_size(q->type, n_embd_head_k_mla) * n_head,
+                ggml_row_size(q->type, n_embd_head_qk_nope));
+
+            ggml_tensor * kv_cmpr_pe = ggml_mul_mat(ctx0, layer.wkv_a_mqa, cur);
+
+            ggml_tensor * kv_cmpr = ggml_view_2d(ctx0, kv_cmpr_pe, kv_lora_rank, n_tokens,
+                ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope), 0);
+            ggml_tensor * k_pe = ggml_view_3d(ctx0, kv_cmpr_pe, n_embd_head_qk_rope, 1, n_tokens,
+                ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope),
+                ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope),
+                ggml_row_size(kv_cmpr_pe->type, kv_lora_rank));
+
+            // note: HF applies rope on de-interleaved pairs, which is ggml's NORM rope
+            q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig,
+                                 freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);
+            cb(q_pe, "q_pe", il);
+
+            k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig,
+                                 freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow);
+            cb(k_pe, "k_pe", il);
+
+            kv_cmpr = build_norm(kv_cmpr, layer.attn_kv_a_norm, nullptr, LLM_NORM_RMS, il);
+            cb(kv_cmpr, "kv_cmpr", il);
+
+            if (layer.wk_b && layer.wv_b) { // MLA KV cache enabled
+                q_nope = ggml_permute(ctx0, q_nope, 0, 2, 1, 3);
+
+                ggml_tensor * q_nope_absorbed = ggml_mul_mat(ctx0, layer.wk_b, q_nope);
+                q_nope_absorbed = ggml_permute(ctx0, q_nope_absorbed, 0, 2, 1, 3);
+
+                // note: rope must go first for in-place context shifting in build_rope_shift()
+                ggml_tensor * Qcur = ggml_concat(ctx0, q_nope_absorbed, q_pe, 0);
+
+                kv_cmpr = ggml_reshape_3d(ctx0, kv_cmpr, kv_lora_rank, 1, n_tokens);
+
+                ggml_tensor * Kcur = ggml_concat(ctx0, kv_cmpr, k_pe, 0);
+                ggml_tensor * Vcur = kv_cmpr;
+
+                // no output projection here - the gate is applied first
+                cur = build_attn(inp_attn_k, nullptr, NULL, nullptr,
+                                 Qcur, Kcur, Vcur, nullptr, nullptr, layer.wv_b, kq_scale_mla, il);
+            } else { // MLA KV cache disabled, fall back to MHA
+                ggml_tensor * Qcur = ggml_reshape_3d(ctx0, ggml_cont(ctx0, q), n_embd_head_k_mla, n_head, n_tokens);
+
+                ggml_tensor * kv = ggml_mul_mat(ctx0, layer.wkv_b, kv_cmpr);
+                const int64_t kv_per_head = n_embd_head_qk_nope + n_embd_head_v_mla;
+
+                ggml_tensor * k_nope = ggml_view_3d(ctx0, kv, n_embd_head_qk_nope, n_head, n_tokens,
+                    ggml_row_size(kv->type, kv_per_head),
+                    ggml_row_size(kv->type, kv_per_head * n_head), 0);
+                ggml_tensor * Vcur = ggml_view_3d(ctx0, kv, n_embd_head_v_mla, n_head, n_tokens,
+                    ggml_row_size(kv->type, kv_per_head),
+                    ggml_row_size(kv->type, kv_per_head * n_head),
+                    ggml_row_size(kv->type, n_embd_head_qk_nope));
+                Vcur = ggml_cont(ctx0, Vcur);
+
+                ggml_tensor * k_pe_target   = ggml_new_tensor_3d(ctx0, k_pe->type, n_embd_head_qk_rope, n_head, n_tokens);
+                ggml_tensor * k_pe_repeated = ggml_repeat(ctx0, k_pe, k_pe_target);
+                ggml_tensor * Kcur = ggml_concat(ctx0, k_nope, k_pe_repeated, 0);
+
+                cur = build_attn(inp_attn_kv, nullptr, NULL, nullptr,
+                                 Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale_mla, il);
+            }
+            cb(cur, "attn_out_pre_gate", il);
+
+            // head-wise sigmoid gate, computed from the same normed hidden state as q/kv
+            ggml_tensor * gate = ggml_mul_mat(ctx0, layer.wqkv_gate, attn_inp);
+            gate = ggml_sigmoid(ctx0, gate);
+            cb(gate, "attn_gate", il);
+
+            cur  = ggml_reshape_3d(ctx0, cur,  n_embd_head_v_mla, n_head, n_tokens);
+            gate = ggml_reshape_3d(ctx0, gate, 1,                 n_head, n_tokens);
+            cur  = ggml_mul(ctx0, cur, gate);
+            cur  = ggml_reshape_2d(ctx0, cur, n_embd_head_v_mla * n_head, n_tokens);
+            cb(cur, "attn_gated", il);
+
+            cur = ggml_mul_mat(ctx0, layer.wo, cur);
+            cb(cur, "attn_out", il);
+        }
+
+        if (il == n_layer - 1 && inp_out_ids) {
+            cur   = ggml_get_rows(ctx0, cur,   inp_out_ids);
+            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
+        }
+
+        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);
+        cb(ffn_inp, "ffn_inp", il);
+
+        cur = build_norm(ffn_inp, layer.ffn_norm, NULL, LLM_NORM_RMS, il);
+        cb(cur, "ffn_norm", il);
+
+        if ((uint32_t) il < hparams.n_layer_dense_lead) {
+            cur = build_ffn(cur,
+                layer.ffn_up,   NULL, NULL,
+                layer.ffn_gate, NULL, NULL,
+                layer.ffn_down, NULL, NULL,
+                NULL, LLM_FFN_SILU, LLM_FFN_PAR, il);
+            cb(cur, "ffn_out", il);
+        } else {
+            ggml_tensor * moe_out = build_moe_ffn(cur,
+                layer.ffn_gate_inp,
+                layer.ffn_up_exps,
+                layer.ffn_gate_exps,
+                layer.ffn_down_exps,
+                layer.ffn_exp_probs_b,
+                hparams.n_expert,
+                hparams.n_expert_used,
+                LLM_FFN_SILU, hparams.expert_weights_norm,
+                hparams.expert_weights_scale,
+                (llama_expert_gating_func_type) hparams.expert_gating_func,
+                il);
+            cb(moe_out, "ffn_moe_out", il);
+
+            ggml_tensor * ffn_shexp = build_ffn(cur,
+                    layer.ffn_up_shexp,   NULL, NULL,
+                    layer.ffn_gate_shexp, NULL, NULL,
+                    layer.ffn_down_shexp, NULL, NULL,
+                    NULL, LLM_FFN_SILU, LLM_FFN_PAR, il);
+            cb(ffn_shexp, "ffn_shexp", il);
+
+            cur = ggml_add(ctx0, moe_out, ffn_shexp);
+            cb(cur, "ffn_out", il);
+        }
+
+        cur = ggml_add(ctx0, cur, ffn_inp);
+
+        cur = build_cvec(cur, il);
+        cb(cur, "l_out", il);
+
+        inpL = cur;
+    }
+
+    cur = inpL;
+
+    cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);
+    cb(cur, "result_norm", -1);
+    res->t_embd = cur;
+
+    cur = ggml_mul_mat(ctx0, model.output, cur);
+    cb(cur, "result_output", -1);
+    res->t_logits = cur;
+
+    ggml_build_forward_expand(gf, cur);
+}

src/models/models.h

@@ -2139,6 +2139,21 @@ struct llama_model_mimo2 : public llama_model_base {
 };
 
 
+struct llama_model_bailingmoe3 : public llama_model_base {
+    llama_model_bailingmoe3(const struct llama_model_params & params) : llama_model_base(params) {}
+    void load_arch_hparams(llama_model_loader & ml) override;
+    void load_arch_tensors(llama_model_loader & ml) override;
+
+    struct graph : public llm_build_delta_net_base {
+        graph(const llama_model & model, const llm_graph_params & params);
+
+        const llama_model & model;
+    };
+
+    std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
+};
+
+
 struct llama_model_kimi_linear : public llama_model_base {
     llama_model_kimi_linear(const struct llama_model_params & params) : llama_model_base(params) {}
     void load_arch_hparams(llama_model_loader & ml) override;

tests/CMakeLists.txt

@@ -291,6 +291,22 @@ llama_test(
 set_tests_properties(test-longhaul-cpu-inkling-six-slots PROPERTIES
     FIXTURES_REQUIRED generate-models
 )
+llama_test(
+    test-longhaul
+    NAME test-longhaul-cpu-bailingmoe3
+    ARGS --cpu-model "${MODEL_DIR}/bailingmoe3-moe.gguf" 49152 1
+)
+set_tests_properties(test-longhaul-cpu-bailingmoe3 PROPERTIES
+    FIXTURES_REQUIRED generate-models
+)
+llama_test(
+    test-longhaul
+    NAME test-longhaul-cpu-bailingmoe3-two-slots
+    ARGS --cpu-model "${MODEL_DIR}/bailingmoe3-moe.gguf" 98304 2
+)
+set_tests_properties(test-longhaul-cpu-bailingmoe3-two-slots PROPERTIES
+    FIXTURES_REQUIRED generate-models
+)
 if (APPLE AND GGML_METAL)
     llama_test(
         test-longhaul
@@ -308,6 +324,14 @@ if (APPLE AND GGML_METAL)
     set_tests_properties(test-longhaul-metal-inkling-six-slots PROPERTIES
         FIXTURES_REQUIRED generate-models
     )
+    llama_test(
+        test-longhaul
+        NAME test-longhaul-metal-bailingmoe3
+        ARGS --metal-model "${MODEL_DIR}/bailingmoe3-moe.gguf" 49152 1
+    )
+    set_tests_properties(test-longhaul-metal-bailingmoe3 PROPERTIES
+        FIXTURES_REQUIRED generate-models
+    )
 endif()
 llama_build_and_test(test-token-cache.cpp)
 target_include_directories(test-token-cache PRIVATE ${PROJECT_SOURCE_DIR}/src)

tests/test-llama-archs.cpp

@@ -119,6 +119,11 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
         n_head  = 2;
         n_ff    = 96;
         n_layer = 2;
+    } else if (arch == LLM_ARCH_BAILINGMOE3) {
+        n_embd  = 64;
+        n_head  = 2;
+        n_ff    = 96;
+        n_layer = 2;
     }
 
     const uint32_t n_embd_head = n_embd / n_head;
@@ -129,7 +134,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
     ms.add_kv(LLM_KV_EMBEDDING_LENGTH,          n_embd);
     ms.add_kv(LLM_KV_FEATURES_LENGTH,           n_embd);
     ms.add_kv(LLM_KV_BLOCK_COUNT,               n_layer);
-    ms.add_kv(LLM_KV_LEADING_DENSE_BLOCK_COUNT, uint32_t(1));
+    ms.add_kv(LLM_KV_LEADING_DENSE_BLOCK_COUNT, arch == LLM_ARCH_BAILINGMOE3 ? uint32_t(0) : uint32_t(1));
 
     if (arch == LLM_ARCH_NEMOTRON_H || arch == LLM_ARCH_NEMOTRON_H_MOE) {
         std::vector<uint32_t> n_ff_per_layer;
@@ -148,7 +153,10 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
     ms.add_kv(LLM_KV_TIME_DECAY_EXTRA_DIM,    uint32_t(128));
     ms.add_kv(LLM_KV_FULL_ATTENTION_INTERVAL, uint32_t(2));
 
-    if (arch == LLM_ARCH_PLAMO2 || arch == LLM_ARCH_JAMBA || arch == LLM_ARCH_NEMOTRON_H || arch == LLM_ARCH_NEMOTRON_H_MOE ||
+    if (arch == LLM_ARCH_BAILINGMOE3) {
+        ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT, std::vector<uint32_t>(n_layer, n_head));
+        ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT_KV, std::vector<uint32_t>({0, 1}));
+    } else if (arch == LLM_ARCH_PLAMO2 || arch == LLM_ARCH_JAMBA || arch == LLM_ARCH_NEMOTRON_H || arch == LLM_ARCH_NEMOTRON_H_MOE ||
             arch == LLM_ARCH_GRANITE_HYBRID || arch == LLM_ARCH_LFM2 || arch == LLM_ARCH_LFM2MOE || arch == LLM_ARCH_KIMI_LINEAR) {
         GGML_ASSERT(n_layer >= 2);
         std::vector<uint32_t> n_head_per_layer;
@@ -164,7 +172,13 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
     }
 
     ms.add_kv(LLM_KV_ATTENTION_MAX_ALIBI_BIAS, 8.0f);
-    if (arch == LLM_ARCH_DEEPSEEK2
+    if (arch == LLM_ARCH_BAILINGMOE3) {
+        ms.add_kv(LLM_KV_ATTENTION_KEY_LENGTH,       uint32_t(16));
+        ms.add_kv(LLM_KV_ATTENTION_VALUE_LENGTH,     uint32_t(8));
+        ms.add_kv(LLM_KV_ROPE_DIMENSION_COUNT,       uint32_t(8));
+        ms.add_kv(LLM_KV_ATTENTION_KEY_LENGTH_MLA,   uint32_t(16));
+        ms.add_kv(LLM_KV_ATTENTION_VALUE_LENGTH_MLA, uint32_t(8));
+    } else if (arch == LLM_ARCH_DEEPSEEK2
             || arch == LLM_ARCH_DEEPSEEK32
             || arch == LLM_ARCH_GLM_DSA
             || arch == LLM_ARCH_KIMI_LINEAR
@@ -184,7 +198,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
     ms.add_kv(LLM_KV_ATTENTION_GROUPNORM_EPS,          1e-5f);
     ms.add_kv(LLM_KV_ATTENTION_GROUPNORM_GROUPS,       uint32_t(8));
     ms.add_kv(LLM_KV_ATTENTION_Q_LORA_RANK,            uint32_t(512));
-    ms.add_kv(LLM_KV_ATTENTION_KV_LORA_RANK,           uint32_t(512));
+    ms.add_kv(LLM_KV_ATTENTION_KV_LORA_RANK,           arch == LLM_ARCH_BAILINGMOE3 ? uint32_t(8) : uint32_t(512));
     ms.add_kv(LLM_KV_ATTENTION_RELATIVE_BUCKETS_COUNT, uint32_t(8));
     ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW,         n_ctx/8);
 
@@ -224,16 +238,21 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
     // ms.add_kv(LLM_KV_DENSE_3_FEAT_IN,      n_embd);
 
     if (moe) {
-        ms.add_kv(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, n_ff);
+        ms.add_kv(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, arch == LLM_ARCH_BAILINGMOE3 ? uint32_t(32) : n_ff);
         ms.add_kv(LLM_KV_INTERLEAVE_MOE_LAYER_STEP,  uint32_t(2));
-        ms.add_kv(LLM_KV_EXPERT_COUNT,               arch == LLM_ARCH_INKLING ? uint32_t(8) : uint32_t(2));
+        ms.add_kv(LLM_KV_EXPERT_COUNT,               arch == LLM_ARCH_INKLING || arch == LLM_ARCH_BAILINGMOE3 ? uint32_t(8) : uint32_t(2));
-        ms.add_kv(LLM_KV_EXPERT_USED_COUNT,          arch == LLM_ARCH_INKLING ? uint32_t(6) : uint32_t(1));
+        ms.add_kv(LLM_KV_EXPERT_USED_COUNT,          arch == LLM_ARCH_INKLING ? uint32_t(6) : arch == LLM_ARCH_BAILINGMOE3 ? uint32_t(2) : uint32_t(1));
         ms.add_kv(LLM_KV_EXPERT_SHARED_COUNT,        arch == LLM_ARCH_INKLING ? uint32_t(2) : uint32_t(1));
         ms.add_kv(LLM_KV_EXPERT_GATING_FUNC,         uint32_t(2)); // sigmoid
         ms.add_kv(LLM_KV_EXPERT_GROUP_SCALE,         1.0f);
         ms.add_kv(LLM_KV_EXPERTS_PER_GROUP,          uint32_t(1));
         if (arch == LLM_ARCH_INKLING) {
             ms.add_kv(LLM_KV_EXPERT_WEIGHTS_SCALE, 1.0f);
+        } else if (arch == LLM_ARCH_BAILINGMOE3) {
+            ms.add_kv(LLM_KV_EXPERT_WEIGHTS_SCALE, 1.0f);
+            ms.add_kv(LLM_KV_EXPERT_WEIGHTS_NORM, true);
+            ms.add_kv(LLM_KV_SWIGLU_CLAMP_EXP, std::vector<float>({4.0f, 4.0f}));
+            ms.add_kv(LLM_KV_SWIGLU_CLAMP_SHEXP, std::vector<float>({5.0f, 5.0f}));
         }
     }
 
@@ -262,7 +281,10 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
     ms.add_kv(LLM_KV_SSM_STATE_SIZE,            uint32_t(128));
     ms.add_kv(LLM_KV_SSM_TIME_STEP_RANK,        n_head);
     ms.add_kv(LLM_KV_SSM_GROUP_COUNT,           arch == LLM_ARCH_PLAMO2 ? 0 : uint32_t(2));
-    ms.add_kv(LLM_KV_KDA_HEAD_DIM,              uint32_t(128));
+    ms.add_kv(LLM_KV_KDA_HEAD_DIM,              arch == LLM_ARCH_BAILINGMOE3 ? uint32_t(16) : uint32_t(128));
+    if (arch == LLM_ARCH_BAILINGMOE3) {
+        ms.add_kv(LLM_KV_KDA_GATE_LOWER_BOUND, -5.0f);
+    }
     ms.add_kv(LLM_KV_WKV_HEAD_SIZE,             n_embd/n_head);
     ms.add_kv(LLM_KV_SHORTCONV_L_CACHE,         uint32_t(3));
 
@@ -375,6 +397,7 @@ static bool moe_mandatory(const llm_arch arch) {
         case LLM_ARCH_EXAONE_MOE:
         case LLM_ARCH_BAILINGMOE:
         case LLM_ARCH_BAILINGMOE2:
+        case LLM_ARCH_BAILINGMOE3:
         case LLM_ARCH_DOTS1:
         case LLM_ARCH_AFMOE:
         case LLM_ARCH_ERNIE4_5:

tools/cli/README.md

@@ -61,7 +61,7 @@
 | `--mmap, --no-mmap` | DEPRECATED in favor of `--load-mode`: whether to memory-map model. (if mmap disabled, slower load but may reduce pageouts if not using mlock)<br/>(env: LLAMA_ARG_MMAP) |
 | `-dio, --direct-io, -ndio, --no-direct-io` | DEPRECATED in favor of `--load-mode`: use DirectIO if available<br/>(env: LLAMA_ARG_DIO) |
 | `-lm, --load-mode MODE` | model loading mode (default: mmap)<br/>- none: no special loading mode<br/>- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)<br/>- mlock: force system to keep model in RAM rather than swapping or compressing<br/>- mmap+mlock: mmap + force system to keep model in RAM rather than swapping or compressing<br/>- dio: use DirectIO if available<br/>- longhaul: stream routed MoE experts through a bounded cache<br/><br/>(env: LLAMA_ARG_LOAD_MODE) |
-| `--longhaul` | stream routed Qwen3.5 MoE, Laguna, or Inkling experts through a bounded CPU or Metal cache<br/>(env: LLAMA_ARG_LONGHAUL) |
+| `--longhaul` | stream routed Qwen3.5 MoE, Laguna, Inkling, or Ling 3.0 experts through a bounded CPU or Metal cache<br/>(env: LLAMA_ARG_LONGHAUL) |
 | `--longhaul-cache N` | longhaul expert cache size in GiB<br/>(env: LLAMA_ARG_LONGHAUL_CACHE) |
 | `--numa TYPE` | attempt optimizations that help on some NUMA systems<br/>- distribute: spread execution evenly over all nodes<br/>- isolate: only spawn threads on CPUs on the node that execution started on<br/>- numactl: use the CPU map provided by numactl<br/>if run without this previously, it is recommended to drop the system page cache before using this<br/>see https://github.com/ggml-org/llama.cpp/issues/1437<br/>(env: LLAMA_ARG_NUMA) |
 | `-dev, --device <dev1,dev2,..>` | comma-separated list of devices to use for offloading (none = don't offload)<br/>use --list-devices to see a list of available devices<br/>(env: LLAMA_ARG_DEVICE) |

tools/completion/README.md

@@ -144,7 +144,7 @@ llama-completion.exe -m models\gemma-1.1-7b-it.Q4_K_M.gguf --ignore-eos -n -1
 | `--mmap, --no-mmap` | DEPRECATED in favor of `--load-mode`: whether to memory-map model. (if mmap disabled, slower load but may reduce pageouts if not using mlock)<br/>(env: LLAMA_ARG_MMAP) |
 | `-dio, --direct-io, -ndio, --no-direct-io` | DEPRECATED in favor of `--load-mode`: use DirectIO if available<br/>(env: LLAMA_ARG_DIO) |
 | `-lm, --load-mode MODE` | model loading mode (default: mmap)<br/>- none: no special loading mode<br/>- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)<br/>- mlock: force system to keep model in RAM rather than swapping or compressing<br/>- mmap+mlock: mmap + force system to keep model in RAM rather than swapping or compressing<br/>- dio: use DirectIO if available<br/>- longhaul: stream routed MoE experts through a bounded cache<br/><br/>(env: LLAMA_ARG_LOAD_MODE) |
-| `--longhaul` | stream routed Qwen3.5 MoE, Laguna, or Inkling experts through a bounded CPU or Metal cache<br/>(env: LLAMA_ARG_LONGHAUL) |
+| `--longhaul` | stream routed Qwen3.5 MoE, Laguna, Inkling, or Ling 3.0 experts through a bounded CPU or Metal cache<br/>(env: LLAMA_ARG_LONGHAUL) |
 | `--longhaul-cache N` | longhaul expert cache size in GiB<br/>(env: LLAMA_ARG_LONGHAUL_CACHE) |
 | `--numa TYPE` | attempt optimizations that help on some NUMA systems<br/>- distribute: spread execution evenly over all nodes<br/>- isolate: only spawn threads on CPUs on the node that execution started on<br/>- numactl: use the CPU map provided by numactl<br/>if run without this previously, it is recommended to drop the system page cache before using this<br/>see https://github.com/ggml-org/llama.cpp/issues/1437<br/>(env: LLAMA_ARG_NUMA) |
 | `-dev, --device <dev1,dev2,..>` | comma-separated list of devices to use for offloading (none = don't offload)<br/>use --list-devices to see a list of available devices<br/>(env: LLAMA_ARG_DEVICE) |

tools/server/README.md

@@ -78,7 +78,7 @@ For the full list of features, please refer to [server's changelog](https://gith
 | `--mmap, --no-mmap` | DEPRECATED in favor of `--load-mode`: whether to memory-map model. (if mmap disabled, slower load but may reduce pageouts if not using mlock)<br/>(env: LLAMA_ARG_MMAP) |
 | `-dio, --direct-io, -ndio, --no-direct-io` | DEPRECATED in favor of `--load-mode`: use DirectIO if available<br/>(env: LLAMA_ARG_DIO) |
 | `-lm, --load-mode MODE` | model loading mode (default: mmap)<br/>- none: no special loading mode<br/>- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)<br/>- mlock: force system to keep model in RAM rather than swapping or compressing<br/>- mmap+mlock: mmap + force system to keep model in RAM rather than swapping or compressing<br/>- dio: use DirectIO if available<br/>- longhaul: stream routed MoE experts through a bounded cache<br/><br/>(env: LLAMA_ARG_LOAD_MODE) |
-| `--longhaul` | stream routed Qwen3.5 MoE, Laguna, or Inkling experts through a bounded CPU or Metal cache<br/>(env: LLAMA_ARG_LONGHAUL) |
+| `--longhaul` | stream routed Qwen3.5 MoE, Laguna, Inkling, or Ling 3.0 experts through a bounded CPU or Metal cache<br/>(env: LLAMA_ARG_LONGHAUL) |
 | `--longhaul-cache N` | longhaul expert cache size in GiB<br/>(env: LLAMA_ARG_LONGHAUL_CACHE) |
 | `--numa TYPE` | attempt optimizations that help on some NUMA systems<br/>- distribute: spread execution evenly over all nodes<br/>- isolate: only spawn threads on CPUs on the node that execution started on<br/>- numactl: use the CPU map provided by numactl<br/>if run without this previously, it is recommended to drop the system page cache before using this<br/>see https://github.com/ggml-org/llama.cpp/issues/1437<br/>(env: LLAMA_ARG_NUMA) |
 | `-dev, --device <dev1,dev2,..>` | comma-separated list of devices to use for offloading (none = don't offload)<br/>use --list-devices to see a list of available devices<br/>(env: LLAMA_ARG_DEVICE) |