Implementing support for laguna arch

docs/longhaul.md

@@ -16,7 +16,7 @@ llama-cli \
 
 `--longhaul-cache` is the expert cache budget in GiB. It is required when `--longhaul` is used. The same options are accepted by `llama-server`.
 
-Longhaul may reduce `--ubatch-size` so that every expert selected by one graph segment can be present in the cache at the same time. The effective value is logged during context creation.
+Longhaul may reduce `--ubatch-size` so that every expert selected by one graph segment can be present in the cache at the same time. The effective value is logged during context creation. If one token selects more experts than the cache has slots, the routed MoE computation is split into multiple stages and the partial results are summed. This permits smaller caches at the cost of additional graph work.
 
 The normal startup warmup is skipped automatically in longhaul mode. Routed expert weights are not read until the first real decode.
 
@@ -25,14 +25,13 @@ The normal startup warmup is skipped automatically in longhaul mode. Routed expe
 Longhaul currently requires:
 
 - macOS with the Metal backend
-- Qwen3.5 MoE architecture
+- Qwen3.5 MoE or Laguna architecture
-- a single-file GGUF
 - all repeating model layers assigned to Metal
 - text generation without embeddings or LoRA adapters
 
 Longhaul does not restrict the GGUF quantization type. Individual tensor types must still be supported by Metal.
 
-MTP/speculative decoding, split GGUF files, tensor validation during loading, vocabulary-only loading, and CPU or mixed CPU/Metal layer placement are not supported.
+MTP/speculative decoding, tensor validation during loading, vocabulary-only loading, and CPU or mixed CPU/Metal layer placement are not supported. Both single-file and split GGUF models are supported; routed expert tensors are read from the shard that owns each tensor.
 
 The cache budget covers the compact routed-expert tensors. It does not include dense weights, attention weights, the KV cache, graph allocations, or the temporary buffer used for one expert read. Disk reads bypass the macOS unified file cache where supported, avoiding a second long-lived copy of streamed weights in system RAM.
 

src/llama-context.cpp

@@ -2353,7 +2353,8 @@ void llama_context::output_reorder() {
 //
 
 uint32_t llama_context::graph_max_nodes(uint32_t n_tokens) const {
-    if (model.arch == LLM_ARCH_QWEN3NEXT ||
+    if (has_longhaul() ||
+        model.arch == LLM_ARCH_QWEN3NEXT ||
         model.arch == LLM_ARCH_KIMI_LINEAR ||
         model.arch == LLM_ARCH_QWEN35 ||
         model.arch == LLM_ARCH_QWEN35MOE ||

src/llama-graph.cpp

@@ -9,6 +9,7 @@
 #include "llama-kv-cache-iswa.h"
 #include "llama-kv-cache-dsa.h"
 #include "llama-kv-cache-dsv4.h"
+#include "llama-longhaul.h"
 #include "llama-memory-hybrid.h"
 #include "llama-memory-hybrid-iswa.h"
 #include "llama-memory-recurrent.h"
@@ -1972,20 +1973,28 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
     //call early so that topk-moe can be used
     ggml_build_forward_expand(gf, weights);
 
-    ggml_tensor * selected_experts_moe = selected_experts;
+    ggml_tensor * moe_inp = ggml_reshape_3d(ctx0, cur, n_embd, 1, n_tokens);
+
+    struct expert_chunk_result {
+        ggml_tensor * output;
+        ggml_tensor * release;
+    };
+
+    auto build_expert_chunk = [&](ggml_tensor * selected_chunk, ggml_tensor * weights_chunk,
+                                  int64_t n_expert_chunk, int chunk) -> expert_chunk_result {
+        ggml_tensor * selected_experts_moe = selected_chunk;
         if (longhaul) {
-        selected_experts_moe = ggml_dup(ctx0, selected_experts);
+            selected_experts_moe = ggml_dup(ctx0, selected_chunk);
-        ggml_format_name(selected_experts_moe, "longhaul.remap.%d", il);
+            ggml_format_name(selected_experts_moe, "longhaul.remap.%d.%d", il, chunk);
             ggml_build_forward_expand(gf, selected_experts_moe);
         }
 
-    cur = ggml_reshape_3d(ctx0, cur, n_embd, 1, n_tokens);
+        ggml_tensor * chunk_cur = moe_inp;
-
         if (weight_before_ffn) {
-        // repeat cur to [n_embd, n_expert_used, n_tokens]
+            // repeat cur to [n_embd, n_expert_chunk, n_tokens]
-        ggml_tensor * repeated = ggml_repeat_4d(ctx0, cur, n_embd, n_expert_used, n_tokens, 1);
+            ggml_tensor * repeated = ggml_repeat_4d(ctx0, chunk_cur, n_embd, n_expert_chunk, n_tokens, 1);
-        cur = ggml_mul(ctx0, repeated, weights);
+            chunk_cur = ggml_mul(ctx0, repeated, weights_chunk);
-        cb(cur, "ffn_moe_weighted", il);
+            cb(chunk_cur, "ffn_moe_weighted", il);
         }
 
         ggml_tensor * up = nullptr;
@@ -1993,7 +2002,7 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
 
         if (gate_up_exps) {
             // merged gate_up path: one mul_mat_id, then split into gate and up views
-        ggml_tensor * gate_up = build_lora_mm_id(gate_up_exps, cur, selected_experts_moe, up_exps_s); // [n_ff*2, n_expert_used, n_tokens]
+            ggml_tensor * gate_up = build_lora_mm_id(gate_up_exps, chunk_cur, selected_experts_moe, up_exps_s);
             cb(gate_up, "ffn_moe_gate_up", il);
 
             if (up_exps_s) {
@@ -2006,13 +2015,15 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
             }
 
             const int64_t n_ff = gate_up->ne[0] / 2;
-        cur = ggml_view_3d(ctx0, gate_up, n_ff, gate_up->ne[1], gate_up->ne[2], gate_up->nb[1], gate_up->nb[2], 0);
+            chunk_cur = ggml_view_3d(ctx0, gate_up, n_ff, gate_up->ne[1], gate_up->ne[2],
-        cb(cur, "ffn_moe_gate", il);
+                    gate_up->nb[1], gate_up->nb[2], 0);
-        up  = ggml_view_3d(ctx0, gate_up, n_ff, gate_up->ne[1], gate_up->ne[2], gate_up->nb[1], gate_up->nb[2], n_ff * gate_up->nb[0]);
+            cb(chunk_cur, "ffn_moe_gate", il);
+            up = ggml_view_3d(ctx0, gate_up, n_ff, gate_up->ne[1], gate_up->ne[2],
+                    gate_up->nb[1], gate_up->nb[2], n_ff * gate_up->nb[0]);
             cb(up, "ffn_moe_up", il);
         } else {
             // separate gate and up path
-        up = build_lora_mm_id(up_exps, cur, selected_experts_moe, up_exps_s); // [n_ff, n_expert_used, n_tokens]
+            up = build_lora_mm_id(up_exps, chunk_cur, selected_experts_moe, up_exps_s);
             cb(up, "ffn_moe_up", il);
 
             if (up_exps_s) {
@@ -2025,19 +2036,19 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
             }
 
             if (gate_exps) {
-            cur = build_lora_mm_id(gate_exps, cur, selected_experts_moe, gate_exps_s); // [n_ff, n_expert_used, n_tokens]
+                chunk_cur = build_lora_mm_id(gate_exps, chunk_cur, selected_experts_moe, gate_exps_s);
-            cb(cur, "ffn_moe_gate", il);
+                cb(chunk_cur, "ffn_moe_gate", il);
             } else {
-            cur = up;
+                chunk_cur = up;
             }
 
             if (gate_exps_s) {
-            cb(cur, "ffn_moe_gate_scaled", il);
+                cb(chunk_cur, "ffn_moe_gate_scaled", il);
             }
 
             if (gate_exps_b) {
-            cur = ggml_add_id(ctx0, cur, gate_exps_b, selected_experts_moe);
+                chunk_cur = ggml_add_id(ctx0, chunk_cur, gate_exps_b, selected_experts_moe);
-            cb(cur, "ffn_moe_gate_biased", il);
+                cb(chunk_cur, "ffn_moe_gate_biased", il);
             }
         }
 
@@ -2054,67 +2065,67 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
                             cb(up, "ffn_moe_up_clamped", il);
 
                             if (arch == LLM_ARCH_DEEPSEEK4) {
-                            cur = ggml_clamp(ctx0, cur, -INFINITY, limit);
+                                chunk_cur = ggml_clamp(ctx0, chunk_cur, -INFINITY, limit);
-                            cb(cur, "ffn_moe_gate_clamped", il);
+                                cb(chunk_cur, "ffn_moe_gate_clamped", il);
-                            cur = ggml_swiglu_split(ctx0, cur, up);
+                                chunk_cur = ggml_swiglu_split(ctx0, chunk_cur, up);
                             } else {
-                            ggml_tensor * gate_act = ggml_silu(ctx0, cur);
+                                ggml_tensor * gate_act = ggml_silu(ctx0, chunk_cur);
                                 cb(gate_act, "ffn_moe_silu", il);
                                 gate_act = ggml_clamp(ctx0, gate_act, -INFINITY, limit);
                                 cb(gate_act, "ffn_moe_silu_clamped", il);
-                            cur = ggml_mul(ctx0, gate_act, up);
+                                chunk_cur = ggml_mul(ctx0, gate_act, up);
                             }
-                        cb(cur, "ffn_moe_swiglu_limited", il);
+                            cb(chunk_cur, "ffn_moe_swiglu_limited", il);
                             break;
                         }
                     }
                 }
 
                 if (has_gate) {
-                cur = ggml_swiglu_split(ctx0, cur, up);
+                    chunk_cur = ggml_swiglu_split(ctx0, chunk_cur, up);
-                cb(cur, "ffn_moe_swiglu", il);
+                    cb(chunk_cur, "ffn_moe_swiglu", il);
                 } else {
-                cur = ggml_silu(ctx0, cur);
+                    chunk_cur = ggml_silu(ctx0, chunk_cur);
-                cb(cur, "ffn_moe_silu", il);
+                    cb(chunk_cur, "ffn_moe_silu", il);
                 } break;
             case LLM_FFN_GELU:
                 if (has_gate) {
-                cur = ggml_geglu_split(ctx0, cur, up);
+                    chunk_cur = ggml_geglu_split(ctx0, chunk_cur, up);
-                cb(cur, "ffn_moe_geglu", il);
+                    cb(chunk_cur, "ffn_moe_geglu", il);
                 } else {
-                cur = ggml_gelu(ctx0, cur);
+                    chunk_cur = ggml_gelu(ctx0, chunk_cur);
-                cb(cur, "ffn_moe_gelu", il);
+                    cb(chunk_cur, "ffn_moe_gelu", il);
                 } break;
             case LLM_FFN_SWIGLU_OAI_MOE:
                 {
                     // TODO: move to hparams?
                     constexpr float alpha = 1.702f;
                     constexpr float limit = 7.0f;
-                cur = ggml_swiglu_oai(ctx0, cur, up, alpha, limit);
+                    chunk_cur = ggml_swiglu_oai(ctx0, chunk_cur, up, alpha, limit);
-                cb(cur, "ffn_moe_swiglu_oai", il);
+                    cb(chunk_cur, "ffn_moe_swiglu_oai", il);
                 } break;
             case LLM_FFN_RELU:
                 if (has_gate) {
-                cur = ggml_reglu_split(ctx0, cur, up);
+                    chunk_cur = ggml_reglu_split(ctx0, chunk_cur, up);
-                cb(cur, "ffn_moe_reglu", il);
+                    cb(chunk_cur, "ffn_moe_reglu", il);
                 } else {
-                cur = ggml_relu(ctx0, cur);
+                    chunk_cur = ggml_relu(ctx0, chunk_cur);
-                cb(cur, "ffn_moe_relu", il);
+                    cb(chunk_cur, "ffn_moe_relu", il);
                 } break;
             case LLM_FFN_RELU_SQR:
                 if (has_gate) {
                     // TODO: add support for gated squared relu
                     GGML_ABORT("fatal error: gated squared relu not implemented");
                 } else {
-                cur = ggml_relu(ctx0, cur);
+                    chunk_cur = ggml_relu(ctx0, chunk_cur);
-                cur = ggml_sqr(ctx0, cur);
+                    chunk_cur = ggml_sqr(ctx0, chunk_cur);
-                cb(cur, "ffn_moe_relu_sqr", il);
+                    cb(chunk_cur, "ffn_moe_relu_sqr", il);
                 } break;
             default:
                 GGML_ABORT("fatal error");
         }
 
-    experts = build_lora_mm_id(down_exps, cur, selected_experts_moe, down_exps_s); // [n_embd, n_expert_used, n_tokens]
+        experts = build_lora_mm_id(down_exps, chunk_cur, selected_experts_moe, down_exps_s);
         cb(experts, "ffn_moe_down", il);
 
         if (down_exps_s) {
@@ -2127,52 +2138,80 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
         }
 
         if (!weight_before_ffn) {
-        experts = ggml_mul(ctx0, experts, weights);
+            experts = ggml_mul(ctx0, experts, weights_chunk);
             cb(experts, "ffn_moe_weighted", il);
         }
 
         ggml_build_forward_expand(gf, experts);
 
         ggml_tensor * cur_experts[LLAMA_MAX_EXPERTS] = { nullptr };
-
+        const int64_t n_expert_aggregate = longhaul
-    assert(n_expert_used > 0);
+            ? n_expert_chunk
+            : std::min<int64_t>(n_expert_chunk, hparams.n_expert_used);
+        GGML_ASSERT(n_expert_aggregate > 0 && n_expert_aggregate <= LLAMA_MAX_EXPERTS);
 
         // order the views before the adds
-    for (uint32_t i = 0; i < hparams.n_expert_used; ++i) {
+        for (int64_t i = 0; i < n_expert_aggregate; ++i) {
-        cur_experts[i] = ggml_view_2d(ctx0, experts, n_embd, n_tokens, experts->nb[2], i*experts->nb[1]);
+            cur_experts[i] = ggml_view_2d(ctx0, experts, n_embd, n_tokens,
-
+                    experts->nb[2], i * experts->nb[1]);
             ggml_build_forward_expand(gf, cur_experts[i]);
         }
 
-    // aggregate experts
+        ggml_tensor * chunk_out = cur_experts[0];
-    // note: here we explicitly use hparams.n_expert_used instead of n_expert_used
+        for (int64_t i = 1; i < n_expert_aggregate; ++i) {
-    //       to avoid potentially a large number of add nodes during warmup
+            chunk_out = ggml_add(ctx0, chunk_out, cur_experts[i]);
-    //       ref: https://github.com/ggml-org/llama.cpp/pull/14753
+            ggml_build_forward_expand(gf, chunk_out);
-    ggml_tensor * moe_out = cur_experts[0];
-
-    for (uint32_t i = 1; i < hparams.n_expert_used; ++i) {
-        moe_out = ggml_add(ctx0, moe_out, cur_experts[i]);
-
-        ggml_build_forward_expand(gf, moe_out);
         }
 
-    if (hparams.n_expert_used == 1) {
+        if (n_expert_aggregate == 1) {
             // avoid returning a non-contiguous tensor
-        moe_out = ggml_cont(ctx0, moe_out);
+            chunk_out = ggml_cont(ctx0, chunk_out);
         }
 
-    cb(moe_out, "ffn_moe_out", il);
+        ggml_tensor * release = nullptr;
-
         if (longhaul) {
-        // Keep the release marker separate from the returned MoE output.
+            // Keep the marker on a separate tensor so callers can rename the
-        // Architecture graph builders commonly rename the returned tensor,
+            // chunk output without preventing the release callback.
-        // which would otherwise erase the callback marker and leave the
+            release = ggml_cont(ctx0, chunk_out);
-        // longhaul cache mutex locked at the next layer.
+            ggml_format_name(release, "longhaul.release.%d.%d", il, chunk);
-        ggml_tensor * release = ggml_cont(ctx0, moe_out);
-        ggml_format_name(release, "longhaul.release.%d", il);
             ggml_build_forward_expand(gf, release);
         }
 
+        return { chunk_out, release };
+    };
+
+    const int64_t chunk_size = longhaul
+        ? std::min<int64_t>(n_expert_used, longhaul->capacity())
+        : n_expert_used;
+    GGML_ASSERT(chunk_size > 0);
+
+    ggml_tensor * moe_out = nullptr;
+    int chunk = 0;
+    for (int64_t first = 0; first < n_expert_used; first += chunk_size, ++chunk) {
+        const int64_t count = std::min<int64_t>(chunk_size, n_expert_used - first);
+
+        ggml_tensor * selected_chunk = selected_experts;
+        ggml_tensor * weights_chunk = weights;
+        if (count != n_expert_used) {
+            selected_chunk = ggml_view_2d(ctx0, selected_experts, count, n_tokens,
+                    selected_experts->nb[1], first * selected_experts->nb[0]);
+            weights_chunk = ggml_view_3d(ctx0, weights, 1, count, n_tokens,
+                    weights->nb[1], weights->nb[2], first * weights->nb[1]);
+        }
+
+        const expert_chunk_result result =
+            build_expert_chunk(selected_chunk, weights_chunk, count, chunk);
+        // For staged execution, consume the release marker in the accumulated
+        // output. This preserves the marker name and orders cache reuse before
+        // the final MoE result can be produced.
+        ggml_tensor * contribution = result.release && chunk_size < n_expert_used
+            ? result.release
+            : result.output;
+        moe_out = moe_out == nullptr ? contribution : ggml_add(ctx0, moe_out, contribution);
+        ggml_build_forward_expand(gf, moe_out);
+    }
+
+    cb(moe_out, "ffn_moe_out", il);
     return moe_out;
 }
 

src/llama-longhaul.cpp

@@ -113,6 +113,10 @@ void llama_longhaul_cache::release(int layer) {
     }
 }
 
+uint32_t llama_longhaul_cache::capacity() const {
+    return n_slots;
+}
+
 uint32_t llama_longhaul_cache::max_ubatch(uint32_t n_expert_used) const {
     return std::max<uint32_t>(1, n_slots / n_expert_used);
 }

src/llama-longhaul.h

@@ -20,6 +20,7 @@ struct llama_longhaul_cache {
     bool remap(int layer, ggml_tensor * ids);
     void release(int layer);
 
+    uint32_t capacity() const;
     uint32_t max_ubatch(uint32_t n_expert_used) const;
     const std::string & error() const;
     bool failed() const;

src/llama-model.cpp

@@ -1359,8 +1359,8 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
 #if !defined(__APPLE__)
             throw std::runtime_error("longhaul is only supported on macOS");
 #endif
-            if (arch != LLM_ARCH_QWEN35MOE) {
+            if (arch != LLM_ARCH_QWEN35MOE && arch != LLM_ARCH_LAGUNA) {
-                throw std::runtime_error("longhaul currently requires a qwen35moe model");
+                throw std::runtime_error("longhaul currently requires a qwen35moe or laguna model");
             }
             if (hparams.n_layer_nextn != 0) {
                 throw std::runtime_error("longhaul does not support MTP tensors");
@@ -1368,9 +1368,6 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
             if (params.check_tensors || params.no_alloc || params.vocab_only) {
                 throw std::runtime_error("longhaul does not support check-tensors, no-alloc, or vocab-only loading");
             }
-            if (ml.files.size() != 1) {
-                throw std::runtime_error("longhaul currently requires a single-file GGUF");
-            }
             for (int il = 0; il < n_layer_all; ++il) {
                 ggml_backend_dev_t dev = pimpl->dev_layer[il].dev;
                 if (ggml_backend_dev_type(dev) != GGML_BACKEND_DEVICE_TYPE_GPU ||
@@ -1380,8 +1377,10 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
             }
             ml.configure_longhaul(params.longhaul_cache_bytes, n_expert, n_layer_all);
             if (ml.longhaul_slots < (size_t) n_expert_used) {
-                throw std::runtime_error(format("longhaul cache has %zu slots per layer, but the model uses %lld experts",
+                LLAMA_LOG_WARN("%s: longhaul cache has %zu slots for %lld selected experts; "
-                    ml.longhaul_slots, (long long) n_expert_used));
+                    "each MoE layer will run in %zu stages\n",
+                    __func__, ml.longhaul_slots, (long long) n_expert_used,
+                    ((size_t) n_expert_used + ml.longhaul_slots - 1) / ml.longhaul_slots);
             }
         }
 

src/models/laguna.cpp

@@ -127,12 +127,14 @@ void llama_model_laguna::load_arch_tensors(llama_model_loader & ml) {
 
         if ((uint32_t)i >= hparams.n_layer_dense_lead) {
             // MoE layer
+            const int expert_flags = params.load_mode == LLAMA_LOAD_MODE_LONGHAUL ? TENSOR_LONGHAUL : 0;
+
             layer.ffn_gate_inp    = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP,    "weight", i), {n_embd, n_expert}, 0);
             layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias",   i), {n_expert}, 0);
 
-            layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd,   n_ff_exp, n_expert}, 0);
+            layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd,   n_ff_exp, n_expert}, expert_flags);
-            layer.ffn_up_exps   = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS,   "weight", i), {n_embd,   n_ff_exp, n_expert}, 0);
+            layer.ffn_up_exps   = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS,   "weight", i), {n_embd,   n_ff_exp, n_expert}, expert_flags);
-            layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd,   n_expert}, 0);
+            layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd,   n_expert}, expert_flags);
 
             // Always-on shared expert.
             layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), {n_embd,    n_ff_shexp}, 0);