owen/longhaul.cpp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

progress

Browse Source
This commit is contained in:
Owen Qwen
2026-07-30 21:20:05 -05:00
parent d5458aa44e
commit 22f42628dd
25 changed files with 1038 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/arg.cpp
+1 -1
View File
@@ -2631,7 +2631,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
).set_env("LLAMA_ARG_LOAD_MODE"));
add_opt(common_arg(
{"--longhaul"},
"stream routed Qwen3.5 MoE or Laguna experts through a bounded CPU or Metal cache",
"stream routed Qwen3.5 MoE, Laguna, or Inkling experts through a bounded CPU or Metal cache",
[](common_params & params) {
params.load_mode = LLAMA_LOAD_MODE_LONGHAUL;
}
common/chat.cpp
+119
View File
@@ -2530,6 +2530,118 @@ static common_chat_params common_chat_params_init_minimax_m3(const common_chat_t
return data;
}
// Inkling's TML format can emit multiple typed content blocks in one turn.
// <|end_message|> closes a block while content_model_end_sampling closes the
// generation, so the generic single-block parser is not sufficient.
static common_chat_params common_chat_params_init_inkling(
const common_chat_template & tmpl,
const autoparser::generation_params & inputs) {
common_chat_params data;
const std::string MSG_MODEL = "<|message_model|>";
const std::string MSG_USER = "<|message_user|>";
const std::string MSG_SYSTEM = "<|message_system|>";
const std::string MSG_TOOL = "<|message_tool|>";
const std::string THINK = "<|content_thinking|>";
const std::string TEXT = "<|content_text|>";
const std::string END_MESSAGE = "<|end_message|>";
const std::string END_SAMPLING = "<|content_model_end_sampling|>";
const std::string INVOKE_TOOL = "<|content_invoke_tool_json|>";
data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);
data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);
data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
data.supports_thinking = true;
data.thinking_start_tag = THINK;
data.thinking_end_tags = {END_MESSAGE};
data.preserved_tokens = {
MSG_MODEL, MSG_USER, MSG_SYSTEM, MSG_TOOL,
THINK, TEXT, END_MESSAGE, END_SAMPLING, INVOKE_TOOL,
};
data.message_delimiters = {
{ COMMON_CHAT_ROLE_ASSISTANT, MSG_MODEL },
{ COMMON_CHAT_ROLE_USER, MSG_USER },
{ COMMON_CHAT_ROLE_SYSTEM, MSG_SYSTEM },
{ COMMON_CHAT_ROLE_TOOL, MSG_TOOL },
};
const bool has_tools = inputs.tools.is_array() && !inputs.tools.empty();
const bool extract_reasoning =
inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
if (inputs.has_continuation()) {
const auto & msg = inputs.continue_msg;
data.generation_prompt = MSG_MODEL + THINK + msg.reasoning_content;
if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
data.generation_prompt += END_MESSAGE + TEXT + msg.render_content();
}
data.prompt += data.generation_prompt;
}
auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
auto generation_prompt = p.literal(MSG_MODEL);
auto end = p.end();
common_peg_parser reasoning_block = p.eps();
if (extract_reasoning) {
reasoning_block =
p.literal(THINK) +
p.reasoning(p.until_one_of({ END_MESSAGE, TEXT, END_SAMPLING })) +
p.optional(p.literal(END_MESSAGE));
} else {
reasoning_block = p.content(
p.literal(THINK) +
p.until_one_of({ END_MESSAGE, TEXT, END_SAMPLING }) +
p.optional(p.literal(END_MESSAGE)));
}
auto reasoning = p.optional(reasoning_block);
auto text_block =
p.optional(p.literal(MSG_MODEL)) +
p.optional(p.literal(TEXT)) +
p.content(p.until_one_of({ THINK, END_MESSAGE, END_SAMPLING })) +
p.optional(p.literal(END_MESSAGE));
auto text_content =
p.one_or_more(p.choice({ reasoning_block, text_block }));
if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) {
return generation_prompt + reasoning + text_content +
p.optional(p.literal(END_SAMPLING)) + end;
}
auto tool_section = p.standard_json_tools(
INVOKE_TOOL, END_MESSAGE, inputs.tools,
/* parallel_tool_calls = */ false,
/* force_tool_calls = */ true,
/* name_key = */ "name",
/* args_key = */ "args",
/* array_wrapped = */ false,
/* function_is_key = */ false,
/* call_id_key = */ "",
/* gen_call_id_key = */ "",
/* parameters_order = */ {},
/* accept_openai_wrapper = */ false);
auto tool_block =
p.optional(p.literal(MSG_MODEL)) +
p.until_one_of({ INVOKE_TOOL, TEXT, THINK, END_MESSAGE, END_SAMPLING }) +
tool_section;
auto tool_calls = inputs.parallel_tool_calls
? p.one_or_more(tool_block)
: tool_block;
auto mixed_body =
p.one_or_more(p.choice({ tool_block, reasoning_block, text_block }));
auto body = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED
? tool_calls
: mixed_body;
return generation_prompt + reasoning + body +
p.optional(p.literal(END_SAMPLING)) + end;
});
data.parser = parser.save();
return data;
}
namespace workaround {
static void map_developer_role_to_system(json & messages) {
@@ -2947,6 +3059,13 @@ std::optional<common_chat_params> common_chat_try_specialized_template(
return common_chat_params_init_cohere2moe(tmpl, params);
}
if (src.find("<|content_thinking|>") != std::string::npos &&
src.find("<|content_text|>") != std::string::npos &&
src.find("<|message_model|>") != std::string::npos) {
LOG_DBG("Using specialized template: Inkling\n");
return common_chat_params_init_inkling(tmpl, params);
}
if (is_lfm2_template(src)) {
LOG_DBG("Using specialized template: LFM2\n");
return common_chat_params_init_lfm2(tmpl, params, /* tool_list_tokens = */ true);