owen/longhaul.cpp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implementing support for laguna arch

Browse Source
This commit is contained in:
Owen Qwen
2026-07-29 11:46:09 -05:00
parent 0bcb7ca59a
commit 68f5d104a8
8 changed files with 234 additions and 189 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/longhaul.md
+3 -4
View File
@@ -16,7 +16,7 @@ llama-cli \
`--longhaul-cache` is the expert cache budget in GiB. It is required when `--longhaul` is used. The same options are accepted by `llama-server`.
Longhaul may reduce `--ubatch-size` so that every expert selected by one graph segment can be present in the cache at the same time. The effective value is logged during context creation.
Longhaul may reduce `--ubatch-size` so that every expert selected by one graph segment can be present in the cache at the same time. The effective value is logged during context creation. If one token selects more experts than the cache has slots, the routed MoE computation is split into multiple stages and the partial results are summed. This permits smaller caches at the cost of additional graph work.
The normal startup warmup is skipped automatically in longhaul mode. Routed expert weights are not read until the first real decode.
@@ -25,14 +25,13 @@ The normal startup warmup is skipped automatically in longhaul mode. Routed expe
Longhaul currently requires:
- macOS with the Metal backend
- Qwen3.5 MoE architecture
- a single-file GGUF
- Qwen3.5 MoE or Laguna architecture
- all repeating model layers assigned to Metal
- text generation without embeddings or LoRA adapters
Longhaul does not restrict the GGUF quantization type. Individual tensor types must still be supported by Metal.
MTP/speculative decoding, split GGUF files, tensor validation during loading, vocabulary-only loading, and CPU or mixed CPU/Metal layer placement are not supported.
MTP/speculative decoding, tensor validation during loading, vocabulary-only loading, and CPU or mixed CPU/Metal layer placement are not supported. Both single-file and split GGUF models are supported; routed expert tensors are read from the shard that owns each tensor.
The cache budget covers the compact routed-expert tensors. It does not include dense weights, attention weights, the KV cache, graph allocations, or the temporary buffer used for one expert read. Disk reads bypass the macOS unified file cache where supported, avoiding a second long-lived copy of streamed weights in system RAM.
src/llama-context.cpp
+2 -1
View File
@@ -2353,7 +2353,8 @@ void llama_context::output_reorder() {
//
uint32_t llama_context::graph_max_nodes(uint32_t n_tokens) const {
if (model.arch == LLM_ARCH_QWEN3NEXT ||
if (has_longhaul() ||
model.arch == LLM_ARCH_QWEN3NEXT ||
model.arch == LLM_ARCH_KIMI_LINEAR ||
model.arch == LLM_ARCH_QWEN35 ||
model.arch == LLM_ARCH_QWEN35MOE ||
src/llama-graph.cpp
+213 -174
View File
@@ -9,6 +9,7 @@
#include "llama-kv-cache-iswa.h"
#include "llama-kv-cache-dsa.h"
#include "llama-kv-cache-dsv4.h"
#include "llama-longhaul.h"
#include "llama-memory-hybrid.h"
#include "llama-memory-hybrid-iswa.h"
#include "llama-memory-recurrent.h"
@@ -1972,207 +1973,245 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
//call early so that topk-moe can be used
ggml_build_forward_expand(gf, weights);
ggml_tensor * selected_experts_moe = selected_experts;
if (longhaul) {
selected_experts_moe = ggml_dup(ctx0, selected_experts);
ggml_format_name(selected_experts_moe, "longhaul.remap.%d", il);
ggml_build_forward_expand(gf, selected_experts_moe);
}
ggml_tensor * moe_inp = ggml_reshape_3d(ctx0, cur, n_embd, 1, n_tokens);
cur = ggml_reshape_3d(ctx0, cur, n_embd, 1, n_tokens);
struct expert_chunk_result {
ggml_tensor * output;
ggml_tensor * release;
};
if (weight_before_ffn) {
// repeat cur to [n_embd, n_expert_used, n_tokens]
ggml_tensor * repeated = ggml_repeat_4d(ctx0, cur, n_embd, n_expert_used, n_tokens, 1);
cur = ggml_mul(ctx0, repeated, weights);
cb(cur, "ffn_moe_weighted", il);
}
ggml_tensor * up = nullptr;
ggml_tensor * experts = nullptr;
if (gate_up_exps) {
// merged gate_up path: one mul_mat_id, then split into gate and up views
ggml_tensor * gate_up = build_lora_mm_id(gate_up_exps, cur, selected_experts_moe, up_exps_s); // [n_ff*2, n_expert_used, n_tokens]
cb(gate_up, "ffn_moe_gate_up", il);
if (up_exps_s) {
cb(gate_up, "ffn_moe_gate_up_scaled", il);
auto build_expert_chunk = [&](ggml_tensor * selected_chunk, ggml_tensor * weights_chunk,
int64_t n_expert_chunk, int chunk) -> expert_chunk_result {
ggml_tensor * selected_experts_moe = selected_chunk;
if (longhaul) {
selected_experts_moe = ggml_dup(ctx0, selected_chunk);
ggml_format_name(selected_experts_moe, "longhaul.remap.%d.%d", il, chunk);
ggml_build_forward_expand(gf, selected_experts_moe);
}
if (gate_up_exps_b) {
gate_up = ggml_add_id(ctx0, gate_up, gate_up_exps_b, selected_experts_moe);
cb(gate_up, "ffn_moe_gate_up_biased", il);
ggml_tensor * chunk_cur = moe_inp;
if (weight_before_ffn) {
// repeat cur to [n_embd, n_expert_chunk, n_tokens]
ggml_tensor * repeated = ggml_repeat_4d(ctx0, chunk_cur, n_embd, n_expert_chunk, n_tokens, 1);
chunk_cur = ggml_mul(ctx0, repeated, weights_chunk);
cb(chunk_cur, "ffn_moe_weighted", il);
}
const int64_t n_ff = gate_up->ne[0] / 2;
cur = ggml_view_3d(ctx0, gate_up, n_ff, gate_up->ne[1], gate_up->ne[2], gate_up->nb[1], gate_up->nb[2], 0);
cb(cur, "ffn_moe_gate", il);
up = ggml_view_3d(ctx0, gate_up, n_ff, gate_up->ne[1], gate_up->ne[2], gate_up->nb[1], gate_up->nb[2], n_ff * gate_up->nb[0]);
cb(up, "ffn_moe_up", il);
} else {
// separate gate and up path
up = build_lora_mm_id(up_exps, cur, selected_experts_moe, up_exps_s); // [n_ff, n_expert_used, n_tokens]
cb(up, "ffn_moe_up", il);
ggml_tensor * up = nullptr;
ggml_tensor * experts = nullptr;
if (up_exps_s) {
cb(up, "ffn_moe_up_scaled", il);
}
if (gate_up_exps) {
// merged gate_up path: one mul_mat_id, then split into gate and up views
ggml_tensor * gate_up = build_lora_mm_id(gate_up_exps, chunk_cur, selected_experts_moe, up_exps_s);
cb(gate_up, "ffn_moe_gate_up", il);
if (up_exps_b) {
up = ggml_add_id(ctx0, up, up_exps_b, selected_experts_moe);
cb(up, "ffn_moe_up_biased", il);
}
if (gate_exps) {
cur = build_lora_mm_id(gate_exps, cur, selected_experts_moe, gate_exps_s); // [n_ff, n_expert_used, n_tokens]
cb(cur, "ffn_moe_gate", il);
} else {
cur = up;
}
if (gate_exps_s) {
cb(cur, "ffn_moe_gate_scaled", il);
}
if (gate_exps_b) {
cur = ggml_add_id(ctx0, cur, gate_exps_b, selected_experts_moe);
cb(cur, "ffn_moe_gate_biased", il);
}
}
const bool has_gate = gate_exps || gate_up_exps;
switch (type_op) {
case LLM_FFN_SILU:
if (gate_exps) {
if (il >= 0) {
const float limit = hparams.swiglu_clamp_exp[il];
constexpr float eps = 1e-6f;
if (limit > eps) {
up = ggml_clamp(ctx0, up, -limit, limit);
cb(up, "ffn_moe_up_clamped", il);
if (arch == LLM_ARCH_DEEPSEEK4) {
cur = ggml_clamp(ctx0, cur, -INFINITY, limit);
cb(cur, "ffn_moe_gate_clamped", il);
cur = ggml_swiglu_split(ctx0, cur, up);
} else {
ggml_tensor * gate_act = ggml_silu(ctx0, cur);
cb(gate_act, "ffn_moe_silu", il);
gate_act = ggml_clamp(ctx0, gate_act, -INFINITY, limit);
cb(gate_act, "ffn_moe_silu_clamped", il);
cur = ggml_mul(ctx0, gate_act, up);
}
cb(cur, "ffn_moe_swiglu_limited", il);
break;
}
}
if (up_exps_s) {
cb(gate_up, "ffn_moe_gate_up_scaled", il);
}
if (has_gate) {
cur = ggml_swiglu_split(ctx0, cur, up);
cb(cur, "ffn_moe_swiglu", il);
if (gate_up_exps_b) {
gate_up = ggml_add_id(ctx0, gate_up, gate_up_exps_b, selected_experts_moe);
cb(gate_up, "ffn_moe_gate_up_biased", il);
}
const int64_t n_ff = gate_up->ne[0] / 2;
chunk_cur = ggml_view_3d(ctx0, gate_up, n_ff, gate_up->ne[1], gate_up->ne[2],
gate_up->nb[1], gate_up->nb[2], 0);
cb(chunk_cur, "ffn_moe_gate", il);
up = ggml_view_3d(ctx0, gate_up, n_ff, gate_up->ne[1], gate_up->ne[2],
gate_up->nb[1], gate_up->nb[2], n_ff * gate_up->nb[0]);
cb(up, "ffn_moe_up", il);
} else {
// separate gate and up path
up = build_lora_mm_id(up_exps, chunk_cur, selected_experts_moe, up_exps_s);
cb(up, "ffn_moe_up", il);
if (up_exps_s) {
cb(up, "ffn_moe_up_scaled", il);
}
if (up_exps_b) {
up = ggml_add_id(ctx0, up, up_exps_b, selected_experts_moe);
cb(up, "ffn_moe_up_biased", il);
}
if (gate_exps) {
chunk_cur = build_lora_mm_id(gate_exps, chunk_cur, selected_experts_moe, gate_exps_s);
cb(chunk_cur, "ffn_moe_gate", il);
} else {
cur = ggml_silu(ctx0, cur);
cb(cur, "ffn_moe_silu", il);
} break;
case LLM_FFN_GELU:
if (has_gate) {
cur = ggml_geglu_split(ctx0, cur, up);
cb(cur, "ffn_moe_geglu", il);
} else {
cur = ggml_gelu(ctx0, cur);
cb(cur, "ffn_moe_gelu", il);
} break;
case LLM_FFN_SWIGLU_OAI_MOE:
{
// TODO: move to hparams?
constexpr float alpha = 1.702f;
constexpr float limit = 7.0f;
cur = ggml_swiglu_oai(ctx0, cur, up, alpha, limit);
cb(cur, "ffn_moe_swiglu_oai", il);
} break;
case LLM_FFN_RELU:
if (has_gate) {
cur = ggml_reglu_split(ctx0, cur, up);
cb(cur, "ffn_moe_reglu", il);
} else {
cur = ggml_relu(ctx0, cur);
cb(cur, "ffn_moe_relu", il);
} break;
case LLM_FFN_RELU_SQR:
if (has_gate) {
// TODO: add support for gated squared relu
GGML_ABORT("fatal error: gated squared relu not implemented");
} else {
cur = ggml_relu(ctx0, cur);
cur = ggml_sqr(ctx0, cur);
cb(cur, "ffn_moe_relu_sqr", il);
} break;
default:
GGML_ABORT("fatal error");
}
chunk_cur = up;
}
experts = build_lora_mm_id(down_exps, cur, selected_experts_moe, down_exps_s); // [n_embd, n_expert_used, n_tokens]
cb(experts, "ffn_moe_down", il);
if (gate_exps_s) {
cb(chunk_cur, "ffn_moe_gate_scaled", il);
}
if (down_exps_s) {
cb(experts, "ffn_moe_down_scaled", il);
}
if (gate_exps_b) {
chunk_cur = ggml_add_id(ctx0, chunk_cur, gate_exps_b, selected_experts_moe);
cb(chunk_cur, "ffn_moe_gate_biased", il);
}
}
if (down_exps_b) {
experts = ggml_add_id(ctx0, experts, down_exps_b, selected_experts_moe);
cb(experts, "ffn_moe_down_biased", il);
}
const bool has_gate = gate_exps || gate_up_exps;
if (!weight_before_ffn) {
experts = ggml_mul(ctx0, experts, weights);
cb(experts, "ffn_moe_weighted", il);
}
switch (type_op) {
case LLM_FFN_SILU:
if (gate_exps) {
if (il >= 0) {
const float limit = hparams.swiglu_clamp_exp[il];
constexpr float eps = 1e-6f;
if (limit > eps) {
up = ggml_clamp(ctx0, up, -limit, limit);
cb(up, "ffn_moe_up_clamped", il);
ggml_build_forward_expand(gf, experts);
if (arch == LLM_ARCH_DEEPSEEK4) {
chunk_cur = ggml_clamp(ctx0, chunk_cur, -INFINITY, limit);
cb(chunk_cur, "ffn_moe_gate_clamped", il);
chunk_cur = ggml_swiglu_split(ctx0, chunk_cur, up);
} else {
ggml_tensor * gate_act = ggml_silu(ctx0, chunk_cur);
cb(gate_act, "ffn_moe_silu", il);
gate_act = ggml_clamp(ctx0, gate_act, -INFINITY, limit);
cb(gate_act, "ffn_moe_silu_clamped", il);
chunk_cur = ggml_mul(ctx0, gate_act, up);
}
cb(chunk_cur, "ffn_moe_swiglu_limited", il);
break;
}
}
}
ggml_tensor * cur_experts[LLAMA_MAX_EXPERTS] = { nullptr };
if (has_gate) {
chunk_cur = ggml_swiglu_split(ctx0, chunk_cur, up);
cb(chunk_cur, "ffn_moe_swiglu", il);
} else {
chunk_cur = ggml_silu(ctx0, chunk_cur);
cb(chunk_cur, "ffn_moe_silu", il);
} break;
case LLM_FFN_GELU:
if (has_gate) {
chunk_cur = ggml_geglu_split(ctx0, chunk_cur, up);
cb(chunk_cur, "ffn_moe_geglu", il);
} else {
chunk_cur = ggml_gelu(ctx0, chunk_cur);
cb(chunk_cur, "ffn_moe_gelu", il);
} break;
case LLM_FFN_SWIGLU_OAI_MOE:
{
// TODO: move to hparams?
constexpr float alpha = 1.702f;
constexpr float limit = 7.0f;
chunk_cur = ggml_swiglu_oai(ctx0, chunk_cur, up, alpha, limit);
cb(chunk_cur, "ffn_moe_swiglu_oai", il);
} break;
case LLM_FFN_RELU:
if (has_gate) {
chunk_cur = ggml_reglu_split(ctx0, chunk_cur, up);
cb(chunk_cur, "ffn_moe_reglu", il);
} else {
chunk_cur = ggml_relu(ctx0, chunk_cur);
cb(chunk_cur, "ffn_moe_relu", il);
} break;
case LLM_FFN_RELU_SQR:
if (has_gate) {
// TODO: add support for gated squared relu
GGML_ABORT("fatal error: gated squared relu not implemented");
} else {
chunk_cur = ggml_relu(ctx0, chunk_cur);
chunk_cur = ggml_sqr(ctx0, chunk_cur);
cb(chunk_cur, "ffn_moe_relu_sqr", il);
} break;
default:
GGML_ABORT("fatal error");
}
assert(n_expert_used > 0);
experts = build_lora_mm_id(down_exps, chunk_cur, selected_experts_moe, down_exps_s);
cb(experts, "ffn_moe_down", il);
// order the views before the adds
for (uint32_t i = 0; i < hparams.n_expert_used; ++i) {
cur_experts[i] = ggml_view_2d(ctx0, experts, n_embd, n_tokens, experts->nb[2], i*experts->nb[1]);
if (down_exps_s) {
cb(experts, "ffn_moe_down_scaled", il);
}
ggml_build_forward_expand(gf, cur_experts[i]);
}
if (down_exps_b) {
experts = ggml_add_id(ctx0, experts, down_exps_b, selected_experts_moe);
cb(experts, "ffn_moe_down_biased", il);
}
// aggregate experts
// note: here we explicitly use hparams.n_expert_used instead of n_expert_used
// to avoid potentially a large number of add nodes during warmup
// ref: https://github.com/ggml-org/llama.cpp/pull/14753
ggml_tensor * moe_out = cur_experts[0];
if (!weight_before_ffn) {
experts = ggml_mul(ctx0, experts, weights_chunk);
cb(experts, "ffn_moe_weighted", il);
}
for (uint32_t i = 1; i < hparams.n_expert_used; ++i) {
moe_out = ggml_add(ctx0, moe_out, cur_experts[i]);
ggml_build_forward_expand(gf, experts);
ggml_tensor * cur_experts[LLAMA_MAX_EXPERTS] = { nullptr };
const int64_t n_expert_aggregate = longhaul
? n_expert_chunk
: std::min<int64_t>(n_expert_chunk, hparams.n_expert_used);
GGML_ASSERT(n_expert_aggregate > 0 && n_expert_aggregate <= LLAMA_MAX_EXPERTS);
// order the views before the adds
for (int64_t i = 0; i < n_expert_aggregate; ++i) {
cur_experts[i] = ggml_view_2d(ctx0, experts, n_embd, n_tokens,
experts->nb[2], i * experts->nb[1]);
ggml_build_forward_expand(gf, cur_experts[i]);
}
ggml_tensor * chunk_out = cur_experts[0];
for (int64_t i = 1; i < n_expert_aggregate; ++i) {
chunk_out = ggml_add(ctx0, chunk_out, cur_experts[i]);
ggml_build_forward_expand(gf, chunk_out);
}
if (n_expert_aggregate == 1) {
// avoid returning a non-contiguous tensor
chunk_out = ggml_cont(ctx0, chunk_out);
}
ggml_tensor * release = nullptr;
if (longhaul) {
// Keep the marker on a separate tensor so callers can rename the
// chunk output without preventing the release callback.
release = ggml_cont(ctx0, chunk_out);
ggml_format_name(release, "longhaul.release.%d.%d", il, chunk);
ggml_build_forward_expand(gf, release);
}
return { chunk_out, release };
};
const int64_t chunk_size = longhaul
? std::min<int64_t>(n_expert_used, longhaul->capacity())
: n_expert_used;
GGML_ASSERT(chunk_size > 0);
ggml_tensor * moe_out = nullptr;
int chunk = 0;
for (int64_t first = 0; first < n_expert_used; first += chunk_size, ++chunk) {
const int64_t count = std::min<int64_t>(chunk_size, n_expert_used - first);
ggml_tensor * selected_chunk = selected_experts;
ggml_tensor * weights_chunk = weights;
if (count != n_expert_used) {
selected_chunk = ggml_view_2d(ctx0, selected_experts, count, n_tokens,
selected_experts->nb[1], first * selected_experts->nb[0]);
weights_chunk = ggml_view_3d(ctx0, weights, 1, count, n_tokens,
weights->nb[1], weights->nb[2], first * weights->nb[1]);
}
const expert_chunk_result result =
build_expert_chunk(selected_chunk, weights_chunk, count, chunk);
// For staged execution, consume the release marker in the accumulated
// output. This preserves the marker name and orders cache reuse before
// the final MoE result can be produced.
ggml_tensor * contribution = result.release && chunk_size < n_expert_used
? result.release
: result.output;
moe_out = moe_out == nullptr ? contribution : ggml_add(ctx0, moe_out, contribution);
ggml_build_forward_expand(gf, moe_out);
}
if (hparams.n_expert_used == 1) {
// avoid returning a non-contiguous tensor
moe_out = ggml_cont(ctx0, moe_out);
}
cb(moe_out, "ffn_moe_out", il);
if (longhaul) {
// Keep the release marker separate from the returned MoE output.
// Architecture graph builders commonly rename the returned tensor,
// which would otherwise erase the callback marker and leave the
// longhaul cache mutex locked at the next layer.
ggml_tensor * release = ggml_cont(ctx0, moe_out);
ggml_format_name(release, "longhaul.release.%d", il);
ggml_build_forward_expand(gf, release);
}
return moe_out;
}
src/llama-longhaul.cpp
+4
View File
@@ -113,6 +113,10 @@ void llama_longhaul_cache::release(int layer) {
}
}
uint32_t llama_longhaul_cache::capacity() const {
return n_slots;
}
uint32_t llama_longhaul_cache::max_ubatch(uint32_t n_expert_used) const {
return std::max<uint32_t>(1, n_slots / n_expert_used);
}
src/llama-longhaul.h
+1
View File
@@ -20,6 +20,7 @@ struct llama_longhaul_cache {
bool remap(int layer, ggml_tensor * ids);
void release(int layer);
uint32_t capacity() const;
uint32_t max_ubatch(uint32_t n_expert_used) const;
const std::string & error() const;
bool failed() const;
src/llama-model.cpp
+6 -7
View File
@@ -1359,8 +1359,8 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
#if !defined(__APPLE__)
throw std::runtime_error("longhaul is only supported on macOS");
#endif
if (arch != LLM_ARCH_QWEN35MOE) {
throw std::runtime_error("longhaul currently requires a qwen35moe model");
if (arch != LLM_ARCH_QWEN35MOE && arch != LLM_ARCH_LAGUNA) {
throw std::runtime_error("longhaul currently requires a qwen35moe or laguna model");
}
if (hparams.n_layer_nextn != 0) {
throw std::runtime_error("longhaul does not support MTP tensors");
@@ -1368,9 +1368,6 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
if (params.check_tensors || params.no_alloc || params.vocab_only) {
throw std::runtime_error("longhaul does not support check-tensors, no-alloc, or vocab-only loading");
}
if (ml.files.size() != 1) {
throw std::runtime_error("longhaul currently requires a single-file GGUF");
}
for (int il = 0; il < n_layer_all; ++il) {
ggml_backend_dev_t dev = pimpl->dev_layer[il].dev;
if (ggml_backend_dev_type(dev) != GGML_BACKEND_DEVICE_TYPE_GPU ||
@@ -1380,8 +1377,10 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
}
ml.configure_longhaul(params.longhaul_cache_bytes, n_expert, n_layer_all);
if (ml.longhaul_slots < (size_t) n_expert_used) {
throw std::runtime_error(format("longhaul cache has %zu slots per layer, but the model uses %lld experts",
ml.longhaul_slots, (long long) n_expert_used));
LLAMA_LOG_WARN("%s: longhaul cache has %zu slots for %lld selected experts; "
"each MoE layer will run in %zu stages\n",
__func__, ml.longhaul_slots, (long long) n_expert_used,
((size_t) n_expert_used + ml.longhaul_slots - 1) / ml.longhaul_slots);
}
}
src/models/laguna.cpp
+5 -3
View File
@@ -127,12 +127,14 @@ void llama_model_laguna::load_arch_tensors(llama_model_loader & ml) {
if ((uint32_t)i >= hparams.n_layer_dense_lead) {
// MoE layer
const int expert_flags = params.load_mode == LLAMA_LOAD_MODE_LONGHAUL ? TENSOR_LONGHAUL : 0;
layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0);
layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, 0);
layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, 0);
layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, 0);
layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, 0);
layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, expert_flags);
layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, expert_flags);
layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, expert_flags);
// Always-on shared expert.
layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), {n_embd, n_ff_shexp}, 0);
tests/.DS_Store
Vendored
BIN
View File
Binary file not shown.