Implementing support for laguna arch
This commit is contained in:
Owen Qwen
@@ -2353,7 +2353,8 @@ void llama_context::output_reorder() {
|
||||
//
|
||||
|
||||
uint32_t llama_context::graph_max_nodes(uint32_t n_tokens) const {
|
||||
if (model.arch == LLM_ARCH_QWEN3NEXT ||
|
||||
if (has_longhaul() ||
|
||||
model.arch == LLM_ARCH_QWEN3NEXT ||
|
||||
model.arch == LLM_ARCH_KIMI_LINEAR ||
|
||||
model.arch == LLM_ARCH_QWEN35 ||
|
||||
model.arch == LLM_ARCH_QWEN35MOE ||
|
||||
|
||||
+213
-174
@@ -9,6 +9,7 @@
|
||||
#include "llama-kv-cache-iswa.h"
|
||||
#include "llama-kv-cache-dsa.h"
|
||||
#include "llama-kv-cache-dsv4.h"
|
||||
#include "llama-longhaul.h"
|
||||
#include "llama-memory-hybrid.h"
|
||||
#include "llama-memory-hybrid-iswa.h"
|
||||
#include "llama-memory-recurrent.h"
|
||||
@@ -1972,207 +1973,245 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
|
||||
//call early so that topk-moe can be used
|
||||
ggml_build_forward_expand(gf, weights);
|
||||
|
||||
ggml_tensor * selected_experts_moe = selected_experts;
|
||||
if (longhaul) {
|
||||
selected_experts_moe = ggml_dup(ctx0, selected_experts);
|
||||
ggml_format_name(selected_experts_moe, "longhaul.remap.%d", il);
|
||||
ggml_build_forward_expand(gf, selected_experts_moe);
|
||||
}
|
||||
ggml_tensor * moe_inp = ggml_reshape_3d(ctx0, cur, n_embd, 1, n_tokens);
|
||||
|
||||
cur = ggml_reshape_3d(ctx0, cur, n_embd, 1, n_tokens);
|
||||
struct expert_chunk_result {
|
||||
ggml_tensor * output;
|
||||
ggml_tensor * release;
|
||||
};
|
||||
|
||||
if (weight_before_ffn) {
|
||||
// repeat cur to [n_embd, n_expert_used, n_tokens]
|
||||
ggml_tensor * repeated = ggml_repeat_4d(ctx0, cur, n_embd, n_expert_used, n_tokens, 1);
|
||||
cur = ggml_mul(ctx0, repeated, weights);
|
||||
cb(cur, "ffn_moe_weighted", il);
|
||||
}
|
||||
|
||||
ggml_tensor * up = nullptr;
|
||||
ggml_tensor * experts = nullptr;
|
||||
|
||||
if (gate_up_exps) {
|
||||
// merged gate_up path: one mul_mat_id, then split into gate and up views
|
||||
ggml_tensor * gate_up = build_lora_mm_id(gate_up_exps, cur, selected_experts_moe, up_exps_s); // [n_ff*2, n_expert_used, n_tokens]
|
||||
cb(gate_up, "ffn_moe_gate_up", il);
|
||||
|
||||
if (up_exps_s) {
|
||||
cb(gate_up, "ffn_moe_gate_up_scaled", il);
|
||||
auto build_expert_chunk = [&](ggml_tensor * selected_chunk, ggml_tensor * weights_chunk,
|
||||
int64_t n_expert_chunk, int chunk) -> expert_chunk_result {
|
||||
ggml_tensor * selected_experts_moe = selected_chunk;
|
||||
if (longhaul) {
|
||||
selected_experts_moe = ggml_dup(ctx0, selected_chunk);
|
||||
ggml_format_name(selected_experts_moe, "longhaul.remap.%d.%d", il, chunk);
|
||||
ggml_build_forward_expand(gf, selected_experts_moe);
|
||||
}
|
||||
|
||||
if (gate_up_exps_b) {
|
||||
gate_up = ggml_add_id(ctx0, gate_up, gate_up_exps_b, selected_experts_moe);
|
||||
cb(gate_up, "ffn_moe_gate_up_biased", il);
|
||||
ggml_tensor * chunk_cur = moe_inp;
|
||||
if (weight_before_ffn) {
|
||||
// repeat cur to [n_embd, n_expert_chunk, n_tokens]
|
||||
ggml_tensor * repeated = ggml_repeat_4d(ctx0, chunk_cur, n_embd, n_expert_chunk, n_tokens, 1);
|
||||
chunk_cur = ggml_mul(ctx0, repeated, weights_chunk);
|
||||
cb(chunk_cur, "ffn_moe_weighted", il);
|
||||
}
|
||||
|
||||
const int64_t n_ff = gate_up->ne[0] / 2;
|
||||
cur = ggml_view_3d(ctx0, gate_up, n_ff, gate_up->ne[1], gate_up->ne[2], gate_up->nb[1], gate_up->nb[2], 0);
|
||||
cb(cur, "ffn_moe_gate", il);
|
||||
up = ggml_view_3d(ctx0, gate_up, n_ff, gate_up->ne[1], gate_up->ne[2], gate_up->nb[1], gate_up->nb[2], n_ff * gate_up->nb[0]);
|
||||
cb(up, "ffn_moe_up", il);
|
||||
} else {
|
||||
// separate gate and up path
|
||||
up = build_lora_mm_id(up_exps, cur, selected_experts_moe, up_exps_s); // [n_ff, n_expert_used, n_tokens]
|
||||
cb(up, "ffn_moe_up", il);
|
||||
ggml_tensor * up = nullptr;
|
||||
ggml_tensor * experts = nullptr;
|
||||
|
||||
if (up_exps_s) {
|
||||
cb(up, "ffn_moe_up_scaled", il);
|
||||
}
|
||||
if (gate_up_exps) {
|
||||
// merged gate_up path: one mul_mat_id, then split into gate and up views
|
||||
ggml_tensor * gate_up = build_lora_mm_id(gate_up_exps, chunk_cur, selected_experts_moe, up_exps_s);
|
||||
cb(gate_up, "ffn_moe_gate_up", il);
|
||||
|
||||
if (up_exps_b) {
|
||||
up = ggml_add_id(ctx0, up, up_exps_b, selected_experts_moe);
|
||||
cb(up, "ffn_moe_up_biased", il);
|
||||
}
|
||||
|
||||
if (gate_exps) {
|
||||
cur = build_lora_mm_id(gate_exps, cur, selected_experts_moe, gate_exps_s); // [n_ff, n_expert_used, n_tokens]
|
||||
cb(cur, "ffn_moe_gate", il);
|
||||
} else {
|
||||
cur = up;
|
||||
}
|
||||
|
||||
if (gate_exps_s) {
|
||||
cb(cur, "ffn_moe_gate_scaled", il);
|
||||
}
|
||||
|
||||
if (gate_exps_b) {
|
||||
cur = ggml_add_id(ctx0, cur, gate_exps_b, selected_experts_moe);
|
||||
cb(cur, "ffn_moe_gate_biased", il);
|
||||
}
|
||||
}
|
||||
|
||||
const bool has_gate = gate_exps || gate_up_exps;
|
||||
|
||||
switch (type_op) {
|
||||
case LLM_FFN_SILU:
|
||||
if (gate_exps) {
|
||||
if (il >= 0) {
|
||||
const float limit = hparams.swiglu_clamp_exp[il];
|
||||
constexpr float eps = 1e-6f;
|
||||
if (limit > eps) {
|
||||
up = ggml_clamp(ctx0, up, -limit, limit);
|
||||
cb(up, "ffn_moe_up_clamped", il);
|
||||
|
||||
if (arch == LLM_ARCH_DEEPSEEK4) {
|
||||
cur = ggml_clamp(ctx0, cur, -INFINITY, limit);
|
||||
cb(cur, "ffn_moe_gate_clamped", il);
|
||||
cur = ggml_swiglu_split(ctx0, cur, up);
|
||||
} else {
|
||||
ggml_tensor * gate_act = ggml_silu(ctx0, cur);
|
||||
cb(gate_act, "ffn_moe_silu", il);
|
||||
gate_act = ggml_clamp(ctx0, gate_act, -INFINITY, limit);
|
||||
cb(gate_act, "ffn_moe_silu_clamped", il);
|
||||
cur = ggml_mul(ctx0, gate_act, up);
|
||||
}
|
||||
cb(cur, "ffn_moe_swiglu_limited", il);
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (up_exps_s) {
|
||||
cb(gate_up, "ffn_moe_gate_up_scaled", il);
|
||||
}
|
||||
|
||||
if (has_gate) {
|
||||
cur = ggml_swiglu_split(ctx0, cur, up);
|
||||
cb(cur, "ffn_moe_swiglu", il);
|
||||
if (gate_up_exps_b) {
|
||||
gate_up = ggml_add_id(ctx0, gate_up, gate_up_exps_b, selected_experts_moe);
|
||||
cb(gate_up, "ffn_moe_gate_up_biased", il);
|
||||
}
|
||||
|
||||
const int64_t n_ff = gate_up->ne[0] / 2;
|
||||
chunk_cur = ggml_view_3d(ctx0, gate_up, n_ff, gate_up->ne[1], gate_up->ne[2],
|
||||
gate_up->nb[1], gate_up->nb[2], 0);
|
||||
cb(chunk_cur, "ffn_moe_gate", il);
|
||||
up = ggml_view_3d(ctx0, gate_up, n_ff, gate_up->ne[1], gate_up->ne[2],
|
||||
gate_up->nb[1], gate_up->nb[2], n_ff * gate_up->nb[0]);
|
||||
cb(up, "ffn_moe_up", il);
|
||||
} else {
|
||||
// separate gate and up path
|
||||
up = build_lora_mm_id(up_exps, chunk_cur, selected_experts_moe, up_exps_s);
|
||||
cb(up, "ffn_moe_up", il);
|
||||
|
||||
if (up_exps_s) {
|
||||
cb(up, "ffn_moe_up_scaled", il);
|
||||
}
|
||||
|
||||
if (up_exps_b) {
|
||||
up = ggml_add_id(ctx0, up, up_exps_b, selected_experts_moe);
|
||||
cb(up, "ffn_moe_up_biased", il);
|
||||
}
|
||||
|
||||
if (gate_exps) {
|
||||
chunk_cur = build_lora_mm_id(gate_exps, chunk_cur, selected_experts_moe, gate_exps_s);
|
||||
cb(chunk_cur, "ffn_moe_gate", il);
|
||||
} else {
|
||||
cur = ggml_silu(ctx0, cur);
|
||||
cb(cur, "ffn_moe_silu", il);
|
||||
} break;
|
||||
case LLM_FFN_GELU:
|
||||
if (has_gate) {
|
||||
cur = ggml_geglu_split(ctx0, cur, up);
|
||||
cb(cur, "ffn_moe_geglu", il);
|
||||
} else {
|
||||
cur = ggml_gelu(ctx0, cur);
|
||||
cb(cur, "ffn_moe_gelu", il);
|
||||
} break;
|
||||
case LLM_FFN_SWIGLU_OAI_MOE:
|
||||
{
|
||||
// TODO: move to hparams?
|
||||
constexpr float alpha = 1.702f;
|
||||
constexpr float limit = 7.0f;
|
||||
cur = ggml_swiglu_oai(ctx0, cur, up, alpha, limit);
|
||||
cb(cur, "ffn_moe_swiglu_oai", il);
|
||||
} break;
|
||||
case LLM_FFN_RELU:
|
||||
if (has_gate) {
|
||||
cur = ggml_reglu_split(ctx0, cur, up);
|
||||
cb(cur, "ffn_moe_reglu", il);
|
||||
} else {
|
||||
cur = ggml_relu(ctx0, cur);
|
||||
cb(cur, "ffn_moe_relu", il);
|
||||
} break;
|
||||
case LLM_FFN_RELU_SQR:
|
||||
if (has_gate) {
|
||||
// TODO: add support for gated squared relu
|
||||
GGML_ABORT("fatal error: gated squared relu not implemented");
|
||||
} else {
|
||||
cur = ggml_relu(ctx0, cur);
|
||||
cur = ggml_sqr(ctx0, cur);
|
||||
cb(cur, "ffn_moe_relu_sqr", il);
|
||||
} break;
|
||||
default:
|
||||
GGML_ABORT("fatal error");
|
||||
}
|
||||
chunk_cur = up;
|
||||
}
|
||||
|
||||
experts = build_lora_mm_id(down_exps, cur, selected_experts_moe, down_exps_s); // [n_embd, n_expert_used, n_tokens]
|
||||
cb(experts, "ffn_moe_down", il);
|
||||
if (gate_exps_s) {
|
||||
cb(chunk_cur, "ffn_moe_gate_scaled", il);
|
||||
}
|
||||
|
||||
if (down_exps_s) {
|
||||
cb(experts, "ffn_moe_down_scaled", il);
|
||||
}
|
||||
if (gate_exps_b) {
|
||||
chunk_cur = ggml_add_id(ctx0, chunk_cur, gate_exps_b, selected_experts_moe);
|
||||
cb(chunk_cur, "ffn_moe_gate_biased", il);
|
||||
}
|
||||
}
|
||||
|
||||
if (down_exps_b) {
|
||||
experts = ggml_add_id(ctx0, experts, down_exps_b, selected_experts_moe);
|
||||
cb(experts, "ffn_moe_down_biased", il);
|
||||
}
|
||||
const bool has_gate = gate_exps || gate_up_exps;
|
||||
|
||||
if (!weight_before_ffn) {
|
||||
experts = ggml_mul(ctx0, experts, weights);
|
||||
cb(experts, "ffn_moe_weighted", il);
|
||||
}
|
||||
switch (type_op) {
|
||||
case LLM_FFN_SILU:
|
||||
if (gate_exps) {
|
||||
if (il >= 0) {
|
||||
const float limit = hparams.swiglu_clamp_exp[il];
|
||||
constexpr float eps = 1e-6f;
|
||||
if (limit > eps) {
|
||||
up = ggml_clamp(ctx0, up, -limit, limit);
|
||||
cb(up, "ffn_moe_up_clamped", il);
|
||||
|
||||
ggml_build_forward_expand(gf, experts);
|
||||
if (arch == LLM_ARCH_DEEPSEEK4) {
|
||||
chunk_cur = ggml_clamp(ctx0, chunk_cur, -INFINITY, limit);
|
||||
cb(chunk_cur, "ffn_moe_gate_clamped", il);
|
||||
chunk_cur = ggml_swiglu_split(ctx0, chunk_cur, up);
|
||||
} else {
|
||||
ggml_tensor * gate_act = ggml_silu(ctx0, chunk_cur);
|
||||
cb(gate_act, "ffn_moe_silu", il);
|
||||
gate_act = ggml_clamp(ctx0, gate_act, -INFINITY, limit);
|
||||
cb(gate_act, "ffn_moe_silu_clamped", il);
|
||||
chunk_cur = ggml_mul(ctx0, gate_act, up);
|
||||
}
|
||||
cb(chunk_cur, "ffn_moe_swiglu_limited", il);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ggml_tensor * cur_experts[LLAMA_MAX_EXPERTS] = { nullptr };
|
||||
if (has_gate) {
|
||||
chunk_cur = ggml_swiglu_split(ctx0, chunk_cur, up);
|
||||
cb(chunk_cur, "ffn_moe_swiglu", il);
|
||||
} else {
|
||||
chunk_cur = ggml_silu(ctx0, chunk_cur);
|
||||
cb(chunk_cur, "ffn_moe_silu", il);
|
||||
} break;
|
||||
case LLM_FFN_GELU:
|
||||
if (has_gate) {
|
||||
chunk_cur = ggml_geglu_split(ctx0, chunk_cur, up);
|
||||
cb(chunk_cur, "ffn_moe_geglu", il);
|
||||
} else {
|
||||
chunk_cur = ggml_gelu(ctx0, chunk_cur);
|
||||
cb(chunk_cur, "ffn_moe_gelu", il);
|
||||
} break;
|
||||
case LLM_FFN_SWIGLU_OAI_MOE:
|
||||
{
|
||||
// TODO: move to hparams?
|
||||
constexpr float alpha = 1.702f;
|
||||
constexpr float limit = 7.0f;
|
||||
chunk_cur = ggml_swiglu_oai(ctx0, chunk_cur, up, alpha, limit);
|
||||
cb(chunk_cur, "ffn_moe_swiglu_oai", il);
|
||||
} break;
|
||||
case LLM_FFN_RELU:
|
||||
if (has_gate) {
|
||||
chunk_cur = ggml_reglu_split(ctx0, chunk_cur, up);
|
||||
cb(chunk_cur, "ffn_moe_reglu", il);
|
||||
} else {
|
||||
chunk_cur = ggml_relu(ctx0, chunk_cur);
|
||||
cb(chunk_cur, "ffn_moe_relu", il);
|
||||
} break;
|
||||
case LLM_FFN_RELU_SQR:
|
||||
if (has_gate) {
|
||||
// TODO: add support for gated squared relu
|
||||
GGML_ABORT("fatal error: gated squared relu not implemented");
|
||||
} else {
|
||||
chunk_cur = ggml_relu(ctx0, chunk_cur);
|
||||
chunk_cur = ggml_sqr(ctx0, chunk_cur);
|
||||
cb(chunk_cur, "ffn_moe_relu_sqr", il);
|
||||
} break;
|
||||
default:
|
||||
GGML_ABORT("fatal error");
|
||||
}
|
||||
|
||||
assert(n_expert_used > 0);
|
||||
experts = build_lora_mm_id(down_exps, chunk_cur, selected_experts_moe, down_exps_s);
|
||||
cb(experts, "ffn_moe_down", il);
|
||||
|
||||
// order the views before the adds
|
||||
for (uint32_t i = 0; i < hparams.n_expert_used; ++i) {
|
||||
cur_experts[i] = ggml_view_2d(ctx0, experts, n_embd, n_tokens, experts->nb[2], i*experts->nb[1]);
|
||||
if (down_exps_s) {
|
||||
cb(experts, "ffn_moe_down_scaled", il);
|
||||
}
|
||||
|
||||
ggml_build_forward_expand(gf, cur_experts[i]);
|
||||
}
|
||||
if (down_exps_b) {
|
||||
experts = ggml_add_id(ctx0, experts, down_exps_b, selected_experts_moe);
|
||||
cb(experts, "ffn_moe_down_biased", il);
|
||||
}
|
||||
|
||||
// aggregate experts
|
||||
// note: here we explicitly use hparams.n_expert_used instead of n_expert_used
|
||||
// to avoid potentially a large number of add nodes during warmup
|
||||
// ref: https://github.com/ggml-org/llama.cpp/pull/14753
|
||||
ggml_tensor * moe_out = cur_experts[0];
|
||||
if (!weight_before_ffn) {
|
||||
experts = ggml_mul(ctx0, experts, weights_chunk);
|
||||
cb(experts, "ffn_moe_weighted", il);
|
||||
}
|
||||
|
||||
for (uint32_t i = 1; i < hparams.n_expert_used; ++i) {
|
||||
moe_out = ggml_add(ctx0, moe_out, cur_experts[i]);
|
||||
ggml_build_forward_expand(gf, experts);
|
||||
|
||||
ggml_tensor * cur_experts[LLAMA_MAX_EXPERTS] = { nullptr };
|
||||
const int64_t n_expert_aggregate = longhaul
|
||||
? n_expert_chunk
|
||||
: std::min<int64_t>(n_expert_chunk, hparams.n_expert_used);
|
||||
GGML_ASSERT(n_expert_aggregate > 0 && n_expert_aggregate <= LLAMA_MAX_EXPERTS);
|
||||
|
||||
// order the views before the adds
|
||||
for (int64_t i = 0; i < n_expert_aggregate; ++i) {
|
||||
cur_experts[i] = ggml_view_2d(ctx0, experts, n_embd, n_tokens,
|
||||
experts->nb[2], i * experts->nb[1]);
|
||||
ggml_build_forward_expand(gf, cur_experts[i]);
|
||||
}
|
||||
|
||||
ggml_tensor * chunk_out = cur_experts[0];
|
||||
for (int64_t i = 1; i < n_expert_aggregate; ++i) {
|
||||
chunk_out = ggml_add(ctx0, chunk_out, cur_experts[i]);
|
||||
ggml_build_forward_expand(gf, chunk_out);
|
||||
}
|
||||
|
||||
if (n_expert_aggregate == 1) {
|
||||
// avoid returning a non-contiguous tensor
|
||||
chunk_out = ggml_cont(ctx0, chunk_out);
|
||||
}
|
||||
|
||||
ggml_tensor * release = nullptr;
|
||||
if (longhaul) {
|
||||
// Keep the marker on a separate tensor so callers can rename the
|
||||
// chunk output without preventing the release callback.
|
||||
release = ggml_cont(ctx0, chunk_out);
|
||||
ggml_format_name(release, "longhaul.release.%d.%d", il, chunk);
|
||||
ggml_build_forward_expand(gf, release);
|
||||
}
|
||||
|
||||
return { chunk_out, release };
|
||||
};
|
||||
|
||||
const int64_t chunk_size = longhaul
|
||||
? std::min<int64_t>(n_expert_used, longhaul->capacity())
|
||||
: n_expert_used;
|
||||
GGML_ASSERT(chunk_size > 0);
|
||||
|
||||
ggml_tensor * moe_out = nullptr;
|
||||
int chunk = 0;
|
||||
for (int64_t first = 0; first < n_expert_used; first += chunk_size, ++chunk) {
|
||||
const int64_t count = std::min<int64_t>(chunk_size, n_expert_used - first);
|
||||
|
||||
ggml_tensor * selected_chunk = selected_experts;
|
||||
ggml_tensor * weights_chunk = weights;
|
||||
if (count != n_expert_used) {
|
||||
selected_chunk = ggml_view_2d(ctx0, selected_experts, count, n_tokens,
|
||||
selected_experts->nb[1], first * selected_experts->nb[0]);
|
||||
weights_chunk = ggml_view_3d(ctx0, weights, 1, count, n_tokens,
|
||||
weights->nb[1], weights->nb[2], first * weights->nb[1]);
|
||||
}
|
||||
|
||||
const expert_chunk_result result =
|
||||
build_expert_chunk(selected_chunk, weights_chunk, count, chunk);
|
||||
// For staged execution, consume the release marker in the accumulated
|
||||
// output. This preserves the marker name and orders cache reuse before
|
||||
// the final MoE result can be produced.
|
||||
ggml_tensor * contribution = result.release && chunk_size < n_expert_used
|
||||
? result.release
|
||||
: result.output;
|
||||
moe_out = moe_out == nullptr ? contribution : ggml_add(ctx0, moe_out, contribution);
|
||||
ggml_build_forward_expand(gf, moe_out);
|
||||
}
|
||||
|
||||
if (hparams.n_expert_used == 1) {
|
||||
// avoid returning a non-contiguous tensor
|
||||
moe_out = ggml_cont(ctx0, moe_out);
|
||||
}
|
||||
|
||||
cb(moe_out, "ffn_moe_out", il);
|
||||
|
||||
if (longhaul) {
|
||||
// Keep the release marker separate from the returned MoE output.
|
||||
// Architecture graph builders commonly rename the returned tensor,
|
||||
// which would otherwise erase the callback marker and leave the
|
||||
// longhaul cache mutex locked at the next layer.
|
||||
ggml_tensor * release = ggml_cont(ctx0, moe_out);
|
||||
ggml_format_name(release, "longhaul.release.%d", il);
|
||||
ggml_build_forward_expand(gf, release);
|
||||
}
|
||||
|
||||
return moe_out;
|
||||
}
|
||||
|
||||
|
||||
@@ -113,6 +113,10 @@ void llama_longhaul_cache::release(int layer) {
|
||||
}
|
||||
}
|
||||
|
||||
uint32_t llama_longhaul_cache::capacity() const {
|
||||
return n_slots;
|
||||
}
|
||||
|
||||
uint32_t llama_longhaul_cache::max_ubatch(uint32_t n_expert_used) const {
|
||||
return std::max<uint32_t>(1, n_slots / n_expert_used);
|
||||
}
|
||||
|
||||
@@ -20,6 +20,7 @@ struct llama_longhaul_cache {
|
||||
bool remap(int layer, ggml_tensor * ids);
|
||||
void release(int layer);
|
||||
|
||||
uint32_t capacity() const;
|
||||
uint32_t max_ubatch(uint32_t n_expert_used) const;
|
||||
const std::string & error() const;
|
||||
bool failed() const;
|
||||
|
||||
+6
-7
@@ -1359,8 +1359,8 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
|
||||
#if !defined(__APPLE__)
|
||||
throw std::runtime_error("longhaul is only supported on macOS");
|
||||
#endif
|
||||
if (arch != LLM_ARCH_QWEN35MOE) {
|
||||
throw std::runtime_error("longhaul currently requires a qwen35moe model");
|
||||
if (arch != LLM_ARCH_QWEN35MOE && arch != LLM_ARCH_LAGUNA) {
|
||||
throw std::runtime_error("longhaul currently requires a qwen35moe or laguna model");
|
||||
}
|
||||
if (hparams.n_layer_nextn != 0) {
|
||||
throw std::runtime_error("longhaul does not support MTP tensors");
|
||||
@@ -1368,9 +1368,6 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
|
||||
if (params.check_tensors || params.no_alloc || params.vocab_only) {
|
||||
throw std::runtime_error("longhaul does not support check-tensors, no-alloc, or vocab-only loading");
|
||||
}
|
||||
if (ml.files.size() != 1) {
|
||||
throw std::runtime_error("longhaul currently requires a single-file GGUF");
|
||||
}
|
||||
for (int il = 0; il < n_layer_all; ++il) {
|
||||
ggml_backend_dev_t dev = pimpl->dev_layer[il].dev;
|
||||
if (ggml_backend_dev_type(dev) != GGML_BACKEND_DEVICE_TYPE_GPU ||
|
||||
@@ -1380,8 +1377,10 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
|
||||
}
|
||||
ml.configure_longhaul(params.longhaul_cache_bytes, n_expert, n_layer_all);
|
||||
if (ml.longhaul_slots < (size_t) n_expert_used) {
|
||||
throw std::runtime_error(format("longhaul cache has %zu slots per layer, but the model uses %lld experts",
|
||||
ml.longhaul_slots, (long long) n_expert_used));
|
||||
LLAMA_LOG_WARN("%s: longhaul cache has %zu slots for %lld selected experts; "
|
||||
"each MoE layer will run in %zu stages\n",
|
||||
__func__, ml.longhaul_slots, (long long) n_expert_used,
|
||||
((size_t) n_expert_used + ml.longhaul_slots - 1) / ml.longhaul_slots);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -127,12 +127,14 @@ void llama_model_laguna::load_arch_tensors(llama_model_loader & ml) {
|
||||
|
||||
if ((uint32_t)i >= hparams.n_layer_dense_lead) {
|
||||
// MoE layer
|
||||
const int expert_flags = params.load_mode == LLAMA_LOAD_MODE_LONGHAUL ? TENSOR_LONGHAUL : 0;
|
||||
|
||||
layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0);
|
||||
layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, 0);
|
||||
|
||||
layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, 0);
|
||||
layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, 0);
|
||||
layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, 0);
|
||||
layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, expert_flags);
|
||||
layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, expert_flags);
|
||||
layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, expert_flags);
|
||||
|
||||
// Always-on shared expert.
|
||||
layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), {n_embd, n_ff_shexp}, 0);
|
||||
|
||||
Reference in New Issue
Block a user