owen/longhaul.cpp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

progress

Browse Source
This commit is contained in:
Owen Qwen
2026-07-30 21:20:05 -05:00
parent d5458aa44e
commit 22f42628dd
25 changed files with 1038 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test-llama-archs.cpp
+40 -7
View File
@@ -114,6 +114,11 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
n_layer = 3;
} else if (arch == LLM_ARCH_CHAMELEON) {
n_vocab = 10240;
} else if (arch == LLM_ARCH_INKLING) {
n_embd = 64;
n_head = 2;
n_ff = 96;
n_layer = 2;
}
const uint32_t n_embd_head = n_embd / n_head;
@@ -198,6 +203,10 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
pattern.push_back(il % 2);
}
ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, pattern);
} else if (arch == LLM_ARCH_INKLING) {
ms.add_kv(
LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN,
std::vector<uint32_t>({1, 0}));
} else {
ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, uint32_t(2));
}
@@ -217,12 +226,27 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
if (moe) {
ms.add_kv(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, n_ff);
ms.add_kv(LLM_KV_INTERLEAVE_MOE_LAYER_STEP, uint32_t(2));
ms.add_kv(LLM_KV_EXPERT_COUNT, uint32_t(2));
ms.add_kv(LLM_KV_EXPERT_USED_COUNT, uint32_t(1));
ms.add_kv(LLM_KV_EXPERT_SHARED_COUNT, uint32_t(1));
ms.add_kv(LLM_KV_EXPERT_COUNT, arch == LLM_ARCH_INKLING ? uint32_t(8) : uint32_t(2));
ms.add_kv(LLM_KV_EXPERT_USED_COUNT, arch == LLM_ARCH_INKLING ? uint32_t(6) : uint32_t(1));
ms.add_kv(LLM_KV_EXPERT_SHARED_COUNT, arch == LLM_ARCH_INKLING ? uint32_t(2) : uint32_t(1));
ms.add_kv(LLM_KV_EXPERT_GATING_FUNC, uint32_t(2)); // sigmoid
ms.add_kv(LLM_KV_EXPERT_GROUP_SCALE, 1.0f);
ms.add_kv(LLM_KV_EXPERTS_PER_GROUP, uint32_t(1));
if (arch == LLM_ARCH_INKLING) {
ms.add_kv(LLM_KV_EXPERT_WEIGHTS_SCALE, 1.0f);
}
}
if (arch == LLM_ARCH_INKLING) {
ms.add_kv(LLM_KV_INKLING_D_REL, uint32_t(8));
ms.add_kv(LLM_KV_INKLING_REL_EXTENT, uint32_t(32));
ms.add_kv(LLM_KV_INKLING_REL_EXTENT_SWA, uint32_t(16));
ms.add_kv(LLM_KV_INKLING_SHORTCONV_KERNEL, uint32_t(4));
ms.add_kv(LLM_KV_INKLING_DENSE_BLOCK_COUNT, uint32_t(1));
ms.add_kv(LLM_KV_INKLING_LOGIT_SCALE_DENOM, 1.0f);
ms.add_kv(LLM_KV_INKLING_LOG_SCALING_N_FLOOR, uint32_t(16));
ms.add_kv(LLM_KV_INKLING_LOG_SCALING_ALPHA, 0.1f);
ms.add_kv(LLM_KV_INKLING_UNPADDED_VOCAB_SIZE, n_vocab);
}
ms.add_kv(LLM_KV_POSNET_EMBEDDING_LENGTH, n_embd);
@@ -372,6 +396,7 @@ static bool moe_mandatory(const llm_arch arch) {
case LLM_ARCH_MISTRAL4:
case LLM_ARCH_MELLUM:
case LLM_ARCH_LAGUNA:
case LLM_ARCH_INKLING:
return true;
default:
return false;
@@ -478,7 +503,9 @@ static int save_models(const llm_arch target_arch, const size_t seed, const ggml
continue;
}
const bool can_save_fixture =
llama_model_saver_supports_arch(arch) || arch == LLM_ARCH_LAGUNA;
llama_model_saver_supports_arch(arch) ||
arch == LLM_ARCH_LAGUNA ||
arch == LLM_ARCH_INKLING;
if (!can_save_fixture || !arch_supported(arch)) {
LOG_INF("%s: %s model (%s) is unsupported, skipping\n", __func__, llm_arch_name(arch), moe ? "MoE" : "dense");
continue;
@@ -487,15 +514,21 @@ static int save_models(const llm_arch target_arch, const size_t seed, const ggml
auto model_and_ctx = get_model_and_ctx(gguf_ctx.get(), nullptr, seed, {});
const std::string path = dir + "/" + llm_arch_name(arch) + (moe ? "-moe.gguf" : "-dense.gguf");
LOG_INF("%s: Saving %s model (%s) to %s...\n", __func__, llm_arch_name(arch), moe ? "MoE" : "dense", path.c_str());
if (llama_model_saver_supports_arch(arch)) {
const bool private_fixture =
arch == LLM_ARCH_LAGUNA || arch == LLM_ARCH_INKLING;
if (llama_model_saver_supports_arch(arch) && !private_fixture) {
llama_model_save_to_file(model_and_ctx.first.get(), path.c_str());
} else {
// Laguna is not supported by the production model saver yet.
// Laguna and Inkling are not supported by the production
// model saver yet.
// Preserve the exact synthetic fixture metadata and attach the
// initialized model tensors so longhaul can be tested from a
// real file without broadening the saver API.
gguf_context_ptr fixture(gguf_init_empty());
gguf_set_kv(fixture.get(), gguf_ctx.get());
// Model initialization normalizes the caller's GGUF context,
// so regenerate the exact private metadata for the file.
gguf_context_ptr fixture_metadata = get_gguf_ctx(arch, moe);
gguf_set_kv(fixture.get(), fixture_metadata.get());
for (const auto & item : llama_internal_get_tensor_map(model_and_ctx.first.get())) {
gguf_add_tensor(fixture.get(), item.second);
}