owen/longhaul.cpp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add portable CPU longhaul support

Browse Source
This commit is contained in:
Owen Qwen
2026-07-30 16:59:40 -05:00
parent a673afad85
commit d5458aa44e
11 changed files with 277 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/llama-model.cpp
+18 -7
View File
@@ -1356,9 +1356,6 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
}
if (params.load_mode == LLAMA_LOAD_MODE_LONGHAUL) {
#if !defined(__APPLE__)
throw std::runtime_error("longhaul is only supported on macOS");
#endif
if (arch != LLM_ARCH_QWEN35MOE && arch != LLM_ARCH_LAGUNA) {
throw std::runtime_error("longhaul currently requires a qwen35moe or laguna model");
}
@@ -1368,13 +1365,27 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
if (params.check_tensors || params.no_alloc || params.vocab_only) {
throw std::runtime_error("longhaul does not support check-tensors, no-alloc, or vocab-only loading");
}
bool all_cpu = true;
bool all_metal = true;
for (int il = 0; il < n_layer_all; ++il) {
ggml_backend_dev_t dev = pimpl->dev_layer[il].dev;
if (ggml_backend_dev_type(dev) != GGML_BACKEND_DEVICE_TYPE_GPU ||
strcmp(ggml_backend_reg_name(ggml_backend_dev_backend_reg(dev)), "MTL") != 0) {
throw std::runtime_error("longhaul requires every model layer on Metal");
}
const char * backend = ggml_backend_reg_name(ggml_backend_dev_backend_reg(dev));
all_cpu = all_cpu && ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_CPU;
all_metal = all_metal &&
ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_GPU &&
strcmp(backend, "MTL") == 0;
}
if (!all_cpu && !all_metal) {
throw std::runtime_error(
"longhaul requires every repeating model layer on CPU or Metal; "
"use --n-gpu-layers 0 for CPU mode");
}
#if !defined(__APPLE__)
if (all_metal) {
throw std::runtime_error("longhaul Metal mode is only supported on macOS");
}
#endif
LLAMA_LOG_INFO("%s: longhaul using %s expert cache\n", __func__, all_cpu ? "CPU" : "Metal");
ml.configure_longhaul(params.longhaul_cache_bytes, n_expert, n_layer_all);
if (ml.longhaul_slots < (size_t) n_expert_used) {
LLAMA_LOG_WARN("%s: longhaul cache has %zu slots for %lld selected experts; "