diff --git a/docs/longhaul.md b/docs/longhaul.md index 5e33d93..63add34 100644 --- a/docs/longhaul.md +++ b/docs/longhaul.md @@ -16,7 +16,7 @@ llama-cli \ `--longhaul-cache` is the expert cache budget in GiB. It is required when `--longhaul` is used. The same options are accepted by `llama-server`. -Longhaul may reduce `--ubatch-size` so that every expert selected by one graph segment can be present in the cache at the same time. The effective value is logged during context creation. +Longhaul may reduce `--ubatch-size` so that every expert selected by one graph segment can be present in the cache at the same time. The effective value is logged during context creation. If one token selects more experts than the cache has slots, the routed MoE computation is split into multiple stages and the partial results are summed. This permits smaller caches at the cost of additional graph work. The normal startup warmup is skipped automatically in longhaul mode. Routed expert weights are not read until the first real decode. @@ -25,14 +25,13 @@ The normal startup warmup is skipped automatically in longhaul mode. Routed expe Longhaul currently requires: - macOS with the Metal backend -- Qwen3.5 MoE architecture -- a single-file GGUF +- Qwen3.5 MoE or Laguna architecture - all repeating model layers assigned to Metal - text generation without embeddings or LoRA adapters Longhaul does not restrict the GGUF quantization type. Individual tensor types must still be supported by Metal. -MTP/speculative decoding, split GGUF files, tensor validation during loading, vocabulary-only loading, and CPU or mixed CPU/Metal layer placement are not supported. +MTP/speculative decoding, tensor validation during loading, vocabulary-only loading, and CPU or mixed CPU/Metal layer placement are not supported. Both single-file and split GGUF models are supported; routed expert tensors are read from the shard that owns each tensor. The cache budget covers the compact routed-expert tensors. It does not include dense weights, attention weights, the KV cache, graph allocations, or the temporary buffer used for one expert read. Disk reads bypass the macOS unified file cache where supported, avoiding a second long-lived copy of streamed weights in system RAM. diff --git a/src/llama-context.cpp b/src/llama-context.cpp index 0f21717..e78aa3f 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -2353,7 +2353,8 @@ void llama_context::output_reorder() { // uint32_t llama_context::graph_max_nodes(uint32_t n_tokens) const { - if (model.arch == LLM_ARCH_QWEN3NEXT || + if (has_longhaul() || + model.arch == LLM_ARCH_QWEN3NEXT || model.arch == LLM_ARCH_KIMI_LINEAR || model.arch == LLM_ARCH_QWEN35 || model.arch == LLM_ARCH_QWEN35MOE || diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index 91996dd..fd7fb22 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -9,6 +9,7 @@ #include "llama-kv-cache-iswa.h" #include "llama-kv-cache-dsa.h" #include "llama-kv-cache-dsv4.h" +#include "llama-longhaul.h" #include "llama-memory-hybrid.h" #include "llama-memory-hybrid-iswa.h" #include "llama-memory-recurrent.h" @@ -1972,207 +1973,245 @@ ggml_tensor * llm_graph_context::build_moe_ffn( //call early so that topk-moe can be used ggml_build_forward_expand(gf, weights); - ggml_tensor * selected_experts_moe = selected_experts; - if (longhaul) { - selected_experts_moe = ggml_dup(ctx0, selected_experts); - ggml_format_name(selected_experts_moe, "longhaul.remap.%d", il); - ggml_build_forward_expand(gf, selected_experts_moe); - } + ggml_tensor * moe_inp = ggml_reshape_3d(ctx0, cur, n_embd, 1, n_tokens); - cur = ggml_reshape_3d(ctx0, cur, n_embd, 1, n_tokens); + struct expert_chunk_result { + ggml_tensor * output; + ggml_tensor * release; + }; - if (weight_before_ffn) { - // repeat cur to [n_embd, n_expert_used, n_tokens] - ggml_tensor * repeated = ggml_repeat_4d(ctx0, cur, n_embd, n_expert_used, n_tokens, 1); - cur = ggml_mul(ctx0, repeated, weights); - cb(cur, "ffn_moe_weighted", il); - } - - ggml_tensor * up = nullptr; - ggml_tensor * experts = nullptr; - - if (gate_up_exps) { - // merged gate_up path: one mul_mat_id, then split into gate and up views - ggml_tensor * gate_up = build_lora_mm_id(gate_up_exps, cur, selected_experts_moe, up_exps_s); // [n_ff*2, n_expert_used, n_tokens] - cb(gate_up, "ffn_moe_gate_up", il); - - if (up_exps_s) { - cb(gate_up, "ffn_moe_gate_up_scaled", il); + auto build_expert_chunk = [&](ggml_tensor * selected_chunk, ggml_tensor * weights_chunk, + int64_t n_expert_chunk, int chunk) -> expert_chunk_result { + ggml_tensor * selected_experts_moe = selected_chunk; + if (longhaul) { + selected_experts_moe = ggml_dup(ctx0, selected_chunk); + ggml_format_name(selected_experts_moe, "longhaul.remap.%d.%d", il, chunk); + ggml_build_forward_expand(gf, selected_experts_moe); } - if (gate_up_exps_b) { - gate_up = ggml_add_id(ctx0, gate_up, gate_up_exps_b, selected_experts_moe); - cb(gate_up, "ffn_moe_gate_up_biased", il); + ggml_tensor * chunk_cur = moe_inp; + if (weight_before_ffn) { + // repeat cur to [n_embd, n_expert_chunk, n_tokens] + ggml_tensor * repeated = ggml_repeat_4d(ctx0, chunk_cur, n_embd, n_expert_chunk, n_tokens, 1); + chunk_cur = ggml_mul(ctx0, repeated, weights_chunk); + cb(chunk_cur, "ffn_moe_weighted", il); } - const int64_t n_ff = gate_up->ne[0] / 2; - cur = ggml_view_3d(ctx0, gate_up, n_ff, gate_up->ne[1], gate_up->ne[2], gate_up->nb[1], gate_up->nb[2], 0); - cb(cur, "ffn_moe_gate", il); - up = ggml_view_3d(ctx0, gate_up, n_ff, gate_up->ne[1], gate_up->ne[2], gate_up->nb[1], gate_up->nb[2], n_ff * gate_up->nb[0]); - cb(up, "ffn_moe_up", il); - } else { - // separate gate and up path - up = build_lora_mm_id(up_exps, cur, selected_experts_moe, up_exps_s); // [n_ff, n_expert_used, n_tokens] - cb(up, "ffn_moe_up", il); + ggml_tensor * up = nullptr; + ggml_tensor * experts = nullptr; - if (up_exps_s) { - cb(up, "ffn_moe_up_scaled", il); - } + if (gate_up_exps) { + // merged gate_up path: one mul_mat_id, then split into gate and up views + ggml_tensor * gate_up = build_lora_mm_id(gate_up_exps, chunk_cur, selected_experts_moe, up_exps_s); + cb(gate_up, "ffn_moe_gate_up", il); - if (up_exps_b) { - up = ggml_add_id(ctx0, up, up_exps_b, selected_experts_moe); - cb(up, "ffn_moe_up_biased", il); - } - - if (gate_exps) { - cur = build_lora_mm_id(gate_exps, cur, selected_experts_moe, gate_exps_s); // [n_ff, n_expert_used, n_tokens] - cb(cur, "ffn_moe_gate", il); - } else { - cur = up; - } - - if (gate_exps_s) { - cb(cur, "ffn_moe_gate_scaled", il); - } - - if (gate_exps_b) { - cur = ggml_add_id(ctx0, cur, gate_exps_b, selected_experts_moe); - cb(cur, "ffn_moe_gate_biased", il); - } - } - - const bool has_gate = gate_exps || gate_up_exps; - - switch (type_op) { - case LLM_FFN_SILU: - if (gate_exps) { - if (il >= 0) { - const float limit = hparams.swiglu_clamp_exp[il]; - constexpr float eps = 1e-6f; - if (limit > eps) { - up = ggml_clamp(ctx0, up, -limit, limit); - cb(up, "ffn_moe_up_clamped", il); - - if (arch == LLM_ARCH_DEEPSEEK4) { - cur = ggml_clamp(ctx0, cur, -INFINITY, limit); - cb(cur, "ffn_moe_gate_clamped", il); - cur = ggml_swiglu_split(ctx0, cur, up); - } else { - ggml_tensor * gate_act = ggml_silu(ctx0, cur); - cb(gate_act, "ffn_moe_silu", il); - gate_act = ggml_clamp(ctx0, gate_act, -INFINITY, limit); - cb(gate_act, "ffn_moe_silu_clamped", il); - cur = ggml_mul(ctx0, gate_act, up); - } - cb(cur, "ffn_moe_swiglu_limited", il); - break; - } - } + if (up_exps_s) { + cb(gate_up, "ffn_moe_gate_up_scaled", il); } - if (has_gate) { - cur = ggml_swiglu_split(ctx0, cur, up); - cb(cur, "ffn_moe_swiglu", il); + if (gate_up_exps_b) { + gate_up = ggml_add_id(ctx0, gate_up, gate_up_exps_b, selected_experts_moe); + cb(gate_up, "ffn_moe_gate_up_biased", il); + } + + const int64_t n_ff = gate_up->ne[0] / 2; + chunk_cur = ggml_view_3d(ctx0, gate_up, n_ff, gate_up->ne[1], gate_up->ne[2], + gate_up->nb[1], gate_up->nb[2], 0); + cb(chunk_cur, "ffn_moe_gate", il); + up = ggml_view_3d(ctx0, gate_up, n_ff, gate_up->ne[1], gate_up->ne[2], + gate_up->nb[1], gate_up->nb[2], n_ff * gate_up->nb[0]); + cb(up, "ffn_moe_up", il); + } else { + // separate gate and up path + up = build_lora_mm_id(up_exps, chunk_cur, selected_experts_moe, up_exps_s); + cb(up, "ffn_moe_up", il); + + if (up_exps_s) { + cb(up, "ffn_moe_up_scaled", il); + } + + if (up_exps_b) { + up = ggml_add_id(ctx0, up, up_exps_b, selected_experts_moe); + cb(up, "ffn_moe_up_biased", il); + } + + if (gate_exps) { + chunk_cur = build_lora_mm_id(gate_exps, chunk_cur, selected_experts_moe, gate_exps_s); + cb(chunk_cur, "ffn_moe_gate", il); } else { - cur = ggml_silu(ctx0, cur); - cb(cur, "ffn_moe_silu", il); - } break; - case LLM_FFN_GELU: - if (has_gate) { - cur = ggml_geglu_split(ctx0, cur, up); - cb(cur, "ffn_moe_geglu", il); - } else { - cur = ggml_gelu(ctx0, cur); - cb(cur, "ffn_moe_gelu", il); - } break; - case LLM_FFN_SWIGLU_OAI_MOE: - { - // TODO: move to hparams? - constexpr float alpha = 1.702f; - constexpr float limit = 7.0f; - cur = ggml_swiglu_oai(ctx0, cur, up, alpha, limit); - cb(cur, "ffn_moe_swiglu_oai", il); - } break; - case LLM_FFN_RELU: - if (has_gate) { - cur = ggml_reglu_split(ctx0, cur, up); - cb(cur, "ffn_moe_reglu", il); - } else { - cur = ggml_relu(ctx0, cur); - cb(cur, "ffn_moe_relu", il); - } break; - case LLM_FFN_RELU_SQR: - if (has_gate) { - // TODO: add support for gated squared relu - GGML_ABORT("fatal error: gated squared relu not implemented"); - } else { - cur = ggml_relu(ctx0, cur); - cur = ggml_sqr(ctx0, cur); - cb(cur, "ffn_moe_relu_sqr", il); - } break; - default: - GGML_ABORT("fatal error"); - } + chunk_cur = up; + } - experts = build_lora_mm_id(down_exps, cur, selected_experts_moe, down_exps_s); // [n_embd, n_expert_used, n_tokens] - cb(experts, "ffn_moe_down", il); + if (gate_exps_s) { + cb(chunk_cur, "ffn_moe_gate_scaled", il); + } - if (down_exps_s) { - cb(experts, "ffn_moe_down_scaled", il); - } + if (gate_exps_b) { + chunk_cur = ggml_add_id(ctx0, chunk_cur, gate_exps_b, selected_experts_moe); + cb(chunk_cur, "ffn_moe_gate_biased", il); + } + } - if (down_exps_b) { - experts = ggml_add_id(ctx0, experts, down_exps_b, selected_experts_moe); - cb(experts, "ffn_moe_down_biased", il); - } + const bool has_gate = gate_exps || gate_up_exps; - if (!weight_before_ffn) { - experts = ggml_mul(ctx0, experts, weights); - cb(experts, "ffn_moe_weighted", il); - } + switch (type_op) { + case LLM_FFN_SILU: + if (gate_exps) { + if (il >= 0) { + const float limit = hparams.swiglu_clamp_exp[il]; + constexpr float eps = 1e-6f; + if (limit > eps) { + up = ggml_clamp(ctx0, up, -limit, limit); + cb(up, "ffn_moe_up_clamped", il); - ggml_build_forward_expand(gf, experts); + if (arch == LLM_ARCH_DEEPSEEK4) { + chunk_cur = ggml_clamp(ctx0, chunk_cur, -INFINITY, limit); + cb(chunk_cur, "ffn_moe_gate_clamped", il); + chunk_cur = ggml_swiglu_split(ctx0, chunk_cur, up); + } else { + ggml_tensor * gate_act = ggml_silu(ctx0, chunk_cur); + cb(gate_act, "ffn_moe_silu", il); + gate_act = ggml_clamp(ctx0, gate_act, -INFINITY, limit); + cb(gate_act, "ffn_moe_silu_clamped", il); + chunk_cur = ggml_mul(ctx0, gate_act, up); + } + cb(chunk_cur, "ffn_moe_swiglu_limited", il); + break; + } + } + } - ggml_tensor * cur_experts[LLAMA_MAX_EXPERTS] = { nullptr }; + if (has_gate) { + chunk_cur = ggml_swiglu_split(ctx0, chunk_cur, up); + cb(chunk_cur, "ffn_moe_swiglu", il); + } else { + chunk_cur = ggml_silu(ctx0, chunk_cur); + cb(chunk_cur, "ffn_moe_silu", il); + } break; + case LLM_FFN_GELU: + if (has_gate) { + chunk_cur = ggml_geglu_split(ctx0, chunk_cur, up); + cb(chunk_cur, "ffn_moe_geglu", il); + } else { + chunk_cur = ggml_gelu(ctx0, chunk_cur); + cb(chunk_cur, "ffn_moe_gelu", il); + } break; + case LLM_FFN_SWIGLU_OAI_MOE: + { + // TODO: move to hparams? + constexpr float alpha = 1.702f; + constexpr float limit = 7.0f; + chunk_cur = ggml_swiglu_oai(ctx0, chunk_cur, up, alpha, limit); + cb(chunk_cur, "ffn_moe_swiglu_oai", il); + } break; + case LLM_FFN_RELU: + if (has_gate) { + chunk_cur = ggml_reglu_split(ctx0, chunk_cur, up); + cb(chunk_cur, "ffn_moe_reglu", il); + } else { + chunk_cur = ggml_relu(ctx0, chunk_cur); + cb(chunk_cur, "ffn_moe_relu", il); + } break; + case LLM_FFN_RELU_SQR: + if (has_gate) { + // TODO: add support for gated squared relu + GGML_ABORT("fatal error: gated squared relu not implemented"); + } else { + chunk_cur = ggml_relu(ctx0, chunk_cur); + chunk_cur = ggml_sqr(ctx0, chunk_cur); + cb(chunk_cur, "ffn_moe_relu_sqr", il); + } break; + default: + GGML_ABORT("fatal error"); + } - assert(n_expert_used > 0); + experts = build_lora_mm_id(down_exps, chunk_cur, selected_experts_moe, down_exps_s); + cb(experts, "ffn_moe_down", il); - // order the views before the adds - for (uint32_t i = 0; i < hparams.n_expert_used; ++i) { - cur_experts[i] = ggml_view_2d(ctx0, experts, n_embd, n_tokens, experts->nb[2], i*experts->nb[1]); + if (down_exps_s) { + cb(experts, "ffn_moe_down_scaled", il); + } - ggml_build_forward_expand(gf, cur_experts[i]); - } + if (down_exps_b) { + experts = ggml_add_id(ctx0, experts, down_exps_b, selected_experts_moe); + cb(experts, "ffn_moe_down_biased", il); + } - // aggregate experts - // note: here we explicitly use hparams.n_expert_used instead of n_expert_used - // to avoid potentially a large number of add nodes during warmup - // ref: https://github.com/ggml-org/llama.cpp/pull/14753 - ggml_tensor * moe_out = cur_experts[0]; + if (!weight_before_ffn) { + experts = ggml_mul(ctx0, experts, weights_chunk); + cb(experts, "ffn_moe_weighted", il); + } - for (uint32_t i = 1; i < hparams.n_expert_used; ++i) { - moe_out = ggml_add(ctx0, moe_out, cur_experts[i]); + ggml_build_forward_expand(gf, experts); + ggml_tensor * cur_experts[LLAMA_MAX_EXPERTS] = { nullptr }; + const int64_t n_expert_aggregate = longhaul + ? n_expert_chunk + : std::min(n_expert_chunk, hparams.n_expert_used); + GGML_ASSERT(n_expert_aggregate > 0 && n_expert_aggregate <= LLAMA_MAX_EXPERTS); + + // order the views before the adds + for (int64_t i = 0; i < n_expert_aggregate; ++i) { + cur_experts[i] = ggml_view_2d(ctx0, experts, n_embd, n_tokens, + experts->nb[2], i * experts->nb[1]); + ggml_build_forward_expand(gf, cur_experts[i]); + } + + ggml_tensor * chunk_out = cur_experts[0]; + for (int64_t i = 1; i < n_expert_aggregate; ++i) { + chunk_out = ggml_add(ctx0, chunk_out, cur_experts[i]); + ggml_build_forward_expand(gf, chunk_out); + } + + if (n_expert_aggregate == 1) { + // avoid returning a non-contiguous tensor + chunk_out = ggml_cont(ctx0, chunk_out); + } + + ggml_tensor * release = nullptr; + if (longhaul) { + // Keep the marker on a separate tensor so callers can rename the + // chunk output without preventing the release callback. + release = ggml_cont(ctx0, chunk_out); + ggml_format_name(release, "longhaul.release.%d.%d", il, chunk); + ggml_build_forward_expand(gf, release); + } + + return { chunk_out, release }; + }; + + const int64_t chunk_size = longhaul + ? std::min(n_expert_used, longhaul->capacity()) + : n_expert_used; + GGML_ASSERT(chunk_size > 0); + + ggml_tensor * moe_out = nullptr; + int chunk = 0; + for (int64_t first = 0; first < n_expert_used; first += chunk_size, ++chunk) { + const int64_t count = std::min(chunk_size, n_expert_used - first); + + ggml_tensor * selected_chunk = selected_experts; + ggml_tensor * weights_chunk = weights; + if (count != n_expert_used) { + selected_chunk = ggml_view_2d(ctx0, selected_experts, count, n_tokens, + selected_experts->nb[1], first * selected_experts->nb[0]); + weights_chunk = ggml_view_3d(ctx0, weights, 1, count, n_tokens, + weights->nb[1], weights->nb[2], first * weights->nb[1]); + } + + const expert_chunk_result result = + build_expert_chunk(selected_chunk, weights_chunk, count, chunk); + // For staged execution, consume the release marker in the accumulated + // output. This preserves the marker name and orders cache reuse before + // the final MoE result can be produced. + ggml_tensor * contribution = result.release && chunk_size < n_expert_used + ? result.release + : result.output; + moe_out = moe_out == nullptr ? contribution : ggml_add(ctx0, moe_out, contribution); ggml_build_forward_expand(gf, moe_out); } - if (hparams.n_expert_used == 1) { - // avoid returning a non-contiguous tensor - moe_out = ggml_cont(ctx0, moe_out); - } - cb(moe_out, "ffn_moe_out", il); - - if (longhaul) { - // Keep the release marker separate from the returned MoE output. - // Architecture graph builders commonly rename the returned tensor, - // which would otherwise erase the callback marker and leave the - // longhaul cache mutex locked at the next layer. - ggml_tensor * release = ggml_cont(ctx0, moe_out); - ggml_format_name(release, "longhaul.release.%d", il); - ggml_build_forward_expand(gf, release); - } - return moe_out; } diff --git a/src/llama-longhaul.cpp b/src/llama-longhaul.cpp index 39525f0..96d78b8 100644 --- a/src/llama-longhaul.cpp +++ b/src/llama-longhaul.cpp @@ -113,6 +113,10 @@ void llama_longhaul_cache::release(int layer) { } } +uint32_t llama_longhaul_cache::capacity() const { + return n_slots; +} + uint32_t llama_longhaul_cache::max_ubatch(uint32_t n_expert_used) const { return std::max(1, n_slots / n_expert_used); } diff --git a/src/llama-longhaul.h b/src/llama-longhaul.h index 1f7052f..5206eaa 100644 --- a/src/llama-longhaul.h +++ b/src/llama-longhaul.h @@ -20,6 +20,7 @@ struct llama_longhaul_cache { bool remap(int layer, ggml_tensor * ids); void release(int layer); + uint32_t capacity() const; uint32_t max_ubatch(uint32_t n_expert_used) const; const std::string & error() const; bool failed() const; diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 4be3d9d..67ee21e 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -1359,8 +1359,8 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) { #if !defined(__APPLE__) throw std::runtime_error("longhaul is only supported on macOS"); #endif - if (arch != LLM_ARCH_QWEN35MOE) { - throw std::runtime_error("longhaul currently requires a qwen35moe model"); + if (arch != LLM_ARCH_QWEN35MOE && arch != LLM_ARCH_LAGUNA) { + throw std::runtime_error("longhaul currently requires a qwen35moe or laguna model"); } if (hparams.n_layer_nextn != 0) { throw std::runtime_error("longhaul does not support MTP tensors"); @@ -1368,9 +1368,6 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) { if (params.check_tensors || params.no_alloc || params.vocab_only) { throw std::runtime_error("longhaul does not support check-tensors, no-alloc, or vocab-only loading"); } - if (ml.files.size() != 1) { - throw std::runtime_error("longhaul currently requires a single-file GGUF"); - } for (int il = 0; il < n_layer_all; ++il) { ggml_backend_dev_t dev = pimpl->dev_layer[il].dev; if (ggml_backend_dev_type(dev) != GGML_BACKEND_DEVICE_TYPE_GPU || @@ -1380,8 +1377,10 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) { } ml.configure_longhaul(params.longhaul_cache_bytes, n_expert, n_layer_all); if (ml.longhaul_slots < (size_t) n_expert_used) { - throw std::runtime_error(format("longhaul cache has %zu slots per layer, but the model uses %lld experts", - ml.longhaul_slots, (long long) n_expert_used)); + LLAMA_LOG_WARN("%s: longhaul cache has %zu slots for %lld selected experts; " + "each MoE layer will run in %zu stages\n", + __func__, ml.longhaul_slots, (long long) n_expert_used, + ((size_t) n_expert_used + ml.longhaul_slots - 1) / ml.longhaul_slots); } } diff --git a/src/models/laguna.cpp b/src/models/laguna.cpp index 82c9a95..e44c990 100644 --- a/src/models/laguna.cpp +++ b/src/models/laguna.cpp @@ -127,12 +127,14 @@ void llama_model_laguna::load_arch_tensors(llama_model_loader & ml) { if ((uint32_t)i >= hparams.n_layer_dense_lead) { // MoE layer + const int expert_flags = params.load_mode == LLAMA_LOAD_MODE_LONGHAUL ? TENSOR_LONGHAUL : 0; + layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0); layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, 0); - layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, 0); - layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, 0); - layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, 0); + layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, expert_flags); + layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, expert_flags); + layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, expert_flags); // Always-on shared expert. layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), {n_embd, n_ff_shexp}, 0); diff --git a/tests/.DS_Store b/tests/.DS_Store new file mode 100644 index 0000000..171fa06 Binary files /dev/null and b/tests/.DS_Store differ