owen/longhaul.cpp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: add Ling 3.0 LongHaul support

Browse Source
This commit is contained in:
Owen Qwen
2026-08-09 09:57:32 -05:00
parent b26b176575
commit d74649d438
20 changed files with 820 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/CMakeLists.txt
+24
View File
@@ -291,6 +291,22 @@ llama_test(
set_tests_properties(test-longhaul-cpu-inkling-six-slots PROPERTIES
FIXTURES_REQUIRED generate-models
)
llama_test(
test-longhaul
NAME test-longhaul-cpu-bailingmoe3
ARGS --cpu-model "${MODEL_DIR}/bailingmoe3-moe.gguf" 49152 1
)
set_tests_properties(test-longhaul-cpu-bailingmoe3 PROPERTIES
FIXTURES_REQUIRED generate-models
)
llama_test(
test-longhaul
NAME test-longhaul-cpu-bailingmoe3-two-slots
ARGS --cpu-model "${MODEL_DIR}/bailingmoe3-moe.gguf" 98304 2
)
set_tests_properties(test-longhaul-cpu-bailingmoe3-two-slots PROPERTIES
FIXTURES_REQUIRED generate-models
)
if (APPLE AND GGML_METAL)
llama_test(
test-longhaul
@@ -308,6 +324,14 @@ if (APPLE AND GGML_METAL)
set_tests_properties(test-longhaul-metal-inkling-six-slots PROPERTIES
FIXTURES_REQUIRED generate-models
)
llama_test(
test-longhaul
NAME test-longhaul-metal-bailingmoe3
ARGS --metal-model "${MODEL_DIR}/bailingmoe3-moe.gguf" 49152 1
)
set_tests_properties(test-longhaul-metal-bailingmoe3 PROPERTIES
FIXTURES_REQUIRED generate-models
)
endif()
llama_build_and_test(test-token-cache.cpp)
target_include_directories(test-token-cache PRIVATE ${PROJECT_SOURCE_DIR}/src)
tests/test-llama-archs.cpp
+31 -8
View File
@@ -119,6 +119,11 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
n_head = 2;
n_ff = 96;
n_layer = 2;
} else if (arch == LLM_ARCH_BAILINGMOE3) {
n_embd = 64;
n_head = 2;
n_ff = 96;
n_layer = 2;
}
const uint32_t n_embd_head = n_embd / n_head;
@@ -129,7 +134,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
ms.add_kv(LLM_KV_EMBEDDING_LENGTH, n_embd);
ms.add_kv(LLM_KV_FEATURES_LENGTH, n_embd);
ms.add_kv(LLM_KV_BLOCK_COUNT, n_layer);
ms.add_kv(LLM_KV_LEADING_DENSE_BLOCK_COUNT, uint32_t(1));
ms.add_kv(LLM_KV_LEADING_DENSE_BLOCK_COUNT, arch == LLM_ARCH_BAILINGMOE3 ? uint32_t(0) : uint32_t(1));
if (arch == LLM_ARCH_NEMOTRON_H || arch == LLM_ARCH_NEMOTRON_H_MOE) {
std::vector<uint32_t> n_ff_per_layer;
@@ -148,7 +153,10 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
ms.add_kv(LLM_KV_TIME_DECAY_EXTRA_DIM, uint32_t(128));
ms.add_kv(LLM_KV_FULL_ATTENTION_INTERVAL, uint32_t(2));
if (arch == LLM_ARCH_PLAMO2 || arch == LLM_ARCH_JAMBA || arch == LLM_ARCH_NEMOTRON_H || arch == LLM_ARCH_NEMOTRON_H_MOE ||
if (arch == LLM_ARCH_BAILINGMOE3) {
ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT, std::vector<uint32_t>(n_layer, n_head));
ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT_KV, std::vector<uint32_t>({0, 1}));
} else if (arch == LLM_ARCH_PLAMO2 || arch == LLM_ARCH_JAMBA || arch == LLM_ARCH_NEMOTRON_H || arch == LLM_ARCH_NEMOTRON_H_MOE ||
arch == LLM_ARCH_GRANITE_HYBRID || arch == LLM_ARCH_LFM2 || arch == LLM_ARCH_LFM2MOE || arch == LLM_ARCH_KIMI_LINEAR) {
GGML_ASSERT(n_layer >= 2);
std::vector<uint32_t> n_head_per_layer;
@@ -164,7 +172,13 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
}
ms.add_kv(LLM_KV_ATTENTION_MAX_ALIBI_BIAS, 8.0f);
if (arch == LLM_ARCH_DEEPSEEK2
if (arch == LLM_ARCH_BAILINGMOE3) {
ms.add_kv(LLM_KV_ATTENTION_KEY_LENGTH, uint32_t(16));
ms.add_kv(LLM_KV_ATTENTION_VALUE_LENGTH, uint32_t(8));
ms.add_kv(LLM_KV_ROPE_DIMENSION_COUNT, uint32_t(8));
ms.add_kv(LLM_KV_ATTENTION_KEY_LENGTH_MLA, uint32_t(16));
ms.add_kv(LLM_KV_ATTENTION_VALUE_LENGTH_MLA, uint32_t(8));
} else if (arch == LLM_ARCH_DEEPSEEK2
|| arch == LLM_ARCH_DEEPSEEK32
|| arch == LLM_ARCH_GLM_DSA
|| arch == LLM_ARCH_KIMI_LINEAR
@@ -184,7 +198,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
ms.add_kv(LLM_KV_ATTENTION_GROUPNORM_EPS, 1e-5f);
ms.add_kv(LLM_KV_ATTENTION_GROUPNORM_GROUPS, uint32_t(8));
ms.add_kv(LLM_KV_ATTENTION_Q_LORA_RANK, uint32_t(512));
ms.add_kv(LLM_KV_ATTENTION_KV_LORA_RANK, uint32_t(512));
ms.add_kv(LLM_KV_ATTENTION_KV_LORA_RANK, arch == LLM_ARCH_BAILINGMOE3 ? uint32_t(8) : uint32_t(512));
ms.add_kv(LLM_KV_ATTENTION_RELATIVE_BUCKETS_COUNT, uint32_t(8));
ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW, n_ctx/8);
@@ -224,16 +238,21 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
// ms.add_kv(LLM_KV_DENSE_3_FEAT_IN, n_embd);
if (moe) {
ms.add_kv(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, n_ff);
ms.add_kv(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, arch == LLM_ARCH_BAILINGMOE3 ? uint32_t(32) : n_ff);
ms.add_kv(LLM_KV_INTERLEAVE_MOE_LAYER_STEP, uint32_t(2));
ms.add_kv(LLM_KV_EXPERT_COUNT, arch == LLM_ARCH_INKLING ? uint32_t(8) : uint32_t(2));
ms.add_kv(LLM_KV_EXPERT_USED_COUNT, arch == LLM_ARCH_INKLING ? uint32_t(6) : uint32_t(1));
ms.add_kv(LLM_KV_EXPERT_COUNT, arch == LLM_ARCH_INKLING || arch == LLM_ARCH_BAILINGMOE3 ? uint32_t(8) : uint32_t(2));
ms.add_kv(LLM_KV_EXPERT_USED_COUNT, arch == LLM_ARCH_INKLING ? uint32_t(6) : arch == LLM_ARCH_BAILINGMOE3 ? uint32_t(2) : uint32_t(1));
ms.add_kv(LLM_KV_EXPERT_SHARED_COUNT, arch == LLM_ARCH_INKLING ? uint32_t(2) : uint32_t(1));
ms.add_kv(LLM_KV_EXPERT_GATING_FUNC, uint32_t(2)); // sigmoid
ms.add_kv(LLM_KV_EXPERT_GROUP_SCALE, 1.0f);
ms.add_kv(LLM_KV_EXPERTS_PER_GROUP, uint32_t(1));
if (arch == LLM_ARCH_INKLING) {
ms.add_kv(LLM_KV_EXPERT_WEIGHTS_SCALE, 1.0f);
} else if (arch == LLM_ARCH_BAILINGMOE3) {
ms.add_kv(LLM_KV_EXPERT_WEIGHTS_SCALE, 1.0f);
ms.add_kv(LLM_KV_EXPERT_WEIGHTS_NORM, true);
ms.add_kv(LLM_KV_SWIGLU_CLAMP_EXP, std::vector<float>({4.0f, 4.0f}));
ms.add_kv(LLM_KV_SWIGLU_CLAMP_SHEXP, std::vector<float>({5.0f, 5.0f}));
}
}
@@ -262,7 +281,10 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
ms.add_kv(LLM_KV_SSM_STATE_SIZE, uint32_t(128));
ms.add_kv(LLM_KV_SSM_TIME_STEP_RANK, n_head);
ms.add_kv(LLM_KV_SSM_GROUP_COUNT, arch == LLM_ARCH_PLAMO2 ? 0 : uint32_t(2));
ms.add_kv(LLM_KV_KDA_HEAD_DIM, uint32_t(128));
ms.add_kv(LLM_KV_KDA_HEAD_DIM, arch == LLM_ARCH_BAILINGMOE3 ? uint32_t(16) : uint32_t(128));
if (arch == LLM_ARCH_BAILINGMOE3) {
ms.add_kv(LLM_KV_KDA_GATE_LOWER_BOUND, -5.0f);
}
ms.add_kv(LLM_KV_WKV_HEAD_SIZE, n_embd/n_head);
ms.add_kv(LLM_KV_SHORTCONV_L_CACHE, uint32_t(3));
@@ -375,6 +397,7 @@ static bool moe_mandatory(const llm_arch arch) {
case LLM_ARCH_EXAONE_MOE:
case LLM_ARCH_BAILINGMOE:
case LLM_ARCH_BAILINGMOE2:
case LLM_ARCH_BAILINGMOE3:
case LLM_ARCH_DOTS1:
case LLM_ARCH_AFMOE:
case LLM_ARCH_ERNIE4_5: