owen/longhaul.cpp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

progress

Browse Source
This commit is contained in:
Owen Qwen
2026-07-30 21:20:05 -05:00
parent d5458aa44e
commit 22f42628dd
25 changed files with 1038 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/CMakeLists.txt
+18
View File
@@ -275,6 +275,24 @@ llama_test(
set_tests_properties(test-longhaul-cpu-laguna PROPERTIES
FIXTURES_REQUIRED generate-models
)
llama_test(
test-longhaul
NAME test-longhaul-cpu-inkling
ARGS --cpu-model "${MODEL_DIR}/inkling-moe.gguf" 73728
)
set_tests_properties(test-longhaul-cpu-inkling PROPERTIES
FIXTURES_REQUIRED generate-models
)
if (APPLE AND GGML_METAL)
llama_test(
test-longhaul
NAME test-longhaul-metal-inkling
ARGS --metal-model "${MODEL_DIR}/inkling-moe.gguf" 73728
)
set_tests_properties(test-longhaul-metal-inkling PROPERTIES
FIXTURES_REQUIRED generate-models
)
endif()
llama_build_and_test(test-token-cache.cpp)
target_include_directories(test-token-cache PRIVATE ${PROJECT_SOURCE_DIR}/src)
tests/test-chat-auto-parser.cpp
+49 -1
View File
@@ -91,6 +91,7 @@ static void test_normalize_quotes_with_embedded_quotes(testing & t);
static void test_tagged_args_with_embedded_quotes(testing & t);
static void test_role_markers_all_templates(testing & t);
static void test_inkling_tml_parser(testing & t);
int main(int argc, char * argv[]) {
testing t(std::cout);
@@ -117,6 +118,7 @@ int main(int argc, char * argv[]) {
t.test("standard_json_tools", test_standard_json_tools_formats);
t.test("normalize_quotes_to_json", test_normalize_quotes_to_json);
t.test("tagged_args_embedded_quotes", test_tagged_args_with_embedded_quotes);
t.test("inkling_tml", test_inkling_tml_parser);
t.test("role_markers_all_templates", test_role_markers_all_templates);
return t.summary();
@@ -2075,6 +2077,53 @@ static void test_role_markers_all_templates(testing & t) {
}
}
static void test_inkling_tml_parser(testing & t) {
const std::string tmpl_src =
"{# <|content_thinking|><|content_text|> #}"
"{%- for message in messages -%}"
"<|message_user|><|content_text|>{{ message.content }}<|end_message|>"
"{%- endfor -%}"
"{%- if add_generation_prompt -%}<|message_model|>{%- endif -%}";
common_chat_templates_ptr tmpls =
common_chat_templates_init(nullptr, tmpl_src);
common_chat_templates_inputs inputs;
common_chat_msg user_message;
user_message.role = "user";
user_message.content = "Hello";
inputs.messages = { user_message };
inputs.add_generation_prompt = true;
inputs.reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK;
inputs.use_jinja = true;
const common_chat_params params =
common_chat_templates_apply(tmpls.get(), inputs);
t.assert_equal(
"Inkling uses the native PEG parser",
COMMON_CHAT_FORMAT_PEG_NATIVE,
params.format);
t.assert_true("Inkling exposes thinking blocks", params.supports_thinking);
t.assert_equal(
"Inkling generation marker",
"<|message_model|>",
params.generation_prompt);
common_peg_arena arena;
arena.load(params.parser);
common_chat_parser_params parser_params(params);
parser_params.reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK;
const common_chat_msg parsed = common_chat_peg_parse(
arena,
"<|content_thinking|>plan<|end_message|>"
"<|message_model|><|content_text|>answer<|end_message|>"
"<|content_model_end_sampling|>",
false,
parser_params);
t.assert_equal("reasoning block", "plan", parsed.reasoning_content);
t.assert_equal("text block", "answer", parsed.content);
}
// Test that reproduces the Seed-OSS template issue with embedded quotes
static void test_tagged_args_with_embedded_quotes(testing & t) {
json tools = build_edit_tool();
@@ -2192,4 +2241,3 @@ static void test_tagged_args_with_embedded_quotes(testing & t) {
}
}
}
tests/test-llama-archs.cpp
+40 -7
View File
@@ -114,6 +114,11 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
n_layer = 3;
} else if (arch == LLM_ARCH_CHAMELEON) {
n_vocab = 10240;
} else if (arch == LLM_ARCH_INKLING) {
n_embd = 64;
n_head = 2;
n_ff = 96;
n_layer = 2;
}
const uint32_t n_embd_head = n_embd / n_head;
@@ -198,6 +203,10 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
pattern.push_back(il % 2);
}
ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, pattern);
} else if (arch == LLM_ARCH_INKLING) {
ms.add_kv(
LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN,
std::vector<uint32_t>({1, 0}));
} else {
ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, uint32_t(2));
}
@@ -217,12 +226,27 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
if (moe) {
ms.add_kv(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, n_ff);
ms.add_kv(LLM_KV_INTERLEAVE_MOE_LAYER_STEP, uint32_t(2));
ms.add_kv(LLM_KV_EXPERT_COUNT, uint32_t(2));
ms.add_kv(LLM_KV_EXPERT_USED_COUNT, uint32_t(1));
ms.add_kv(LLM_KV_EXPERT_SHARED_COUNT, uint32_t(1));
ms.add_kv(LLM_KV_EXPERT_COUNT, arch == LLM_ARCH_INKLING ? uint32_t(8) : uint32_t(2));
ms.add_kv(LLM_KV_EXPERT_USED_COUNT, arch == LLM_ARCH_INKLING ? uint32_t(6) : uint32_t(1));
ms.add_kv(LLM_KV_EXPERT_SHARED_COUNT, arch == LLM_ARCH_INKLING ? uint32_t(2) : uint32_t(1));
ms.add_kv(LLM_KV_EXPERT_GATING_FUNC, uint32_t(2)); // sigmoid
ms.add_kv(LLM_KV_EXPERT_GROUP_SCALE, 1.0f);
ms.add_kv(LLM_KV_EXPERTS_PER_GROUP, uint32_t(1));
if (arch == LLM_ARCH_INKLING) {
ms.add_kv(LLM_KV_EXPERT_WEIGHTS_SCALE, 1.0f);
}
}
if (arch == LLM_ARCH_INKLING) {
ms.add_kv(LLM_KV_INKLING_D_REL, uint32_t(8));
ms.add_kv(LLM_KV_INKLING_REL_EXTENT, uint32_t(32));
ms.add_kv(LLM_KV_INKLING_REL_EXTENT_SWA, uint32_t(16));
ms.add_kv(LLM_KV_INKLING_SHORTCONV_KERNEL, uint32_t(4));
ms.add_kv(LLM_KV_INKLING_DENSE_BLOCK_COUNT, uint32_t(1));
ms.add_kv(LLM_KV_INKLING_LOGIT_SCALE_DENOM, 1.0f);
ms.add_kv(LLM_KV_INKLING_LOG_SCALING_N_FLOOR, uint32_t(16));
ms.add_kv(LLM_KV_INKLING_LOG_SCALING_ALPHA, 0.1f);
ms.add_kv(LLM_KV_INKLING_UNPADDED_VOCAB_SIZE, n_vocab);
}
ms.add_kv(LLM_KV_POSNET_EMBEDDING_LENGTH, n_embd);
@@ -372,6 +396,7 @@ static bool moe_mandatory(const llm_arch arch) {
case LLM_ARCH_MISTRAL4:
case LLM_ARCH_MELLUM:
case LLM_ARCH_LAGUNA:
case LLM_ARCH_INKLING:
return true;
default:
return false;
@@ -478,7 +503,9 @@ static int save_models(const llm_arch target_arch, const size_t seed, const ggml
continue;
}
const bool can_save_fixture =
llama_model_saver_supports_arch(arch) || arch == LLM_ARCH_LAGUNA;
llama_model_saver_supports_arch(arch) ||
arch == LLM_ARCH_LAGUNA ||
arch == LLM_ARCH_INKLING;
if (!can_save_fixture || !arch_supported(arch)) {
LOG_INF("%s: %s model (%s) is unsupported, skipping\n", __func__, llm_arch_name(arch), moe ? "MoE" : "dense");
continue;
@@ -487,15 +514,21 @@ static int save_models(const llm_arch target_arch, const size_t seed, const ggml
auto model_and_ctx = get_model_and_ctx(gguf_ctx.get(), nullptr, seed, {});
const std::string path = dir + "/" + llm_arch_name(arch) + (moe ? "-moe.gguf" : "-dense.gguf");
LOG_INF("%s: Saving %s model (%s) to %s...\n", __func__, llm_arch_name(arch), moe ? "MoE" : "dense", path.c_str());
if (llama_model_saver_supports_arch(arch)) {
const bool private_fixture =
arch == LLM_ARCH_LAGUNA || arch == LLM_ARCH_INKLING;
if (llama_model_saver_supports_arch(arch) && !private_fixture) {
llama_model_save_to_file(model_and_ctx.first.get(), path.c_str());
} else {
// Laguna is not supported by the production model saver yet.
// Laguna and Inkling are not supported by the production
// model saver yet.
// Preserve the exact synthetic fixture metadata and attach the
// initialized model tensors so longhaul can be tested from a
// real file without broadening the saver API.
gguf_context_ptr fixture(gguf_init_empty());
gguf_set_kv(fixture.get(), gguf_ctx.get());
// Model initialization normalizes the caller's GGUF context,
// so regenerate the exact private metadata for the file.
gguf_context_ptr fixture_metadata = get_gguf_ctx(arch, moe);
gguf_set_kv(fixture.get(), fixture_metadata.get());
for (const auto & item : llama_internal_get_tensor_map(model_and_ctx.first.get())) {
gguf_add_tensor(fixture.get(), item.second);
}
tests/test-longhaul.cpp
+20 -11
View File
@@ -191,12 +191,13 @@ struct cpu_decode_result {
bool expert_buffer_is_host = false;
};
static cpu_decode_result decode_cpu_model(
static cpu_decode_result decode_model(
const std::string & path,
llama_load_mode load_mode,
uint64_t cache_bytes) {
uint64_t cache_bytes,
int32_t n_gpu_layers) {
llama_model_params model_params = llama_model_default_params();
model_params.n_gpu_layers = 0;
model_params.n_gpu_layers = n_gpu_layers;
model_params.load_mode = load_mode;
model_params.longhaul_cache_bytes = cache_bytes;
model_params.use_extra_bufts = true;
@@ -256,19 +257,24 @@ static cpu_decode_result decode_cpu_model(
return result;
}
static void test_cpu_model(
static void test_model(
testing & t,
const std::string & path,
uint64_t cache_bytes) {
uint64_t cache_bytes,
int32_t n_gpu_layers) {
const cpu_decode_result regular =
decode_cpu_model(path, LLAMA_LOAD_MODE_NONE, 0);
decode_model(path, LLAMA_LOAD_MODE_NONE, 0, n_gpu_layers);
const cpu_decode_result longhaul =
decode_cpu_model(path, LLAMA_LOAD_MODE_LONGHAUL, cache_bytes);
decode_model(path, LLAMA_LOAD_MODE_LONGHAUL, cache_bytes, n_gpu_layers);
t.assert_equal(1u, longhaul.capacity);
t.assert_true("longhaul loaded at least one expert", longhaul.misses > 0);
t.assert_true("longhaul read expert bytes", longhaul.bytes_read > 0);
t.assert_true("streamed experts use a directly writable host buffer", longhaul.expert_buffer_is_host);
if (n_gpu_layers == 0) {
t.assert_true(
"streamed CPU experts use a directly writable host buffer",
longhaul.expert_buffer_is_host);
}
t.assert_equal(regular.logits.size(), longhaul.logits.size());
double squared_error = 0.0;
@@ -296,11 +302,14 @@ int main(int argc, char ** argv) {
llama_log_set([](ggml_log_level, const char *, void *) {}, nullptr);
}
if (argc == 4 && std::string(argv[1]) == "--cpu-model") {
if (argc == 4 &&
(std::string(argv[1]) == "--cpu-model" ||
std::string(argv[1]) == "--metal-model")) {
const std::string path = argv[2];
const uint64_t cache_bytes = std::stoull(argv[3]);
t.test("cpu_model", [&](testing & current) {
test_cpu_model(current, path, cache_bytes);
const bool metal = std::string(argv[1]) == "--metal-model";
t.test(metal ? "metal_model" : "cpu_model", [&](testing & current) {
test_model(current, path, cache_bytes, metal ? 99 : 0);
});
const int result = t.summary();
llama_backend_free();