owen/longhaul.cpp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add Vulkan longhaul support

Browse Source
This commit is contained in:
Owen Qwen
2026-07-30 15:09:43 -05:00
parent a673afad85
commit fa404a9d97
13 changed files with 262 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/arg.cpp
+21 -9
View File
@@ -27,9 +27,11 @@
#include <algorithm>
#include <cinttypes>
#include <climits>
#include <cmath>
#include <cstdarg>
#include <filesystem>
#include <fstream>
#include <limits>
#include <list>
#include <regex>
#include <set>
@@ -56,6 +58,19 @@
using json = nlohmann::ordered_json;
using namespace common_arg_utils;
static uint64_t parse_longhaul_cache_gib(const std::string & value) {
size_t end = 0;
const double gib = std::stod(value, &end);
constexpr long double bytes_per_gib = 1024.0L * 1024.0L * 1024.0L;
const long double bytes = (long double) gib * bytes_per_gib;
if (end != value.size() || !std::isfinite(gib) || gib <= 0.0 ||
bytes < 1.0L || bytes > (long double) std::numeric_limits<uint64_t>::max()) {
throw std::invalid_argument("longhaul cache size must be a positive GiB value");
}
return (uint64_t) bytes;
}
static std::initializer_list<enum llama_example> mmproj_examples = {
LLAMA_EXAMPLE_MTMD,
LLAMA_EXAMPLE_SERVER,
@@ -815,7 +830,7 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context
parse_cli_args();
if (params.load_mode == LLAMA_LOAD_MODE_LONGHAUL && params.longhaul_cache_bytes == 0) {
throw std::invalid_argument("error: --longhaul requires --longhaul-cache N\n");
throw std::invalid_argument("error: --longhaul requires --longhaul-cache GiB\n");
}
postprocess_cpu_params(params.cpuparams, nullptr);
@@ -2631,19 +2646,16 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
).set_env("LLAMA_ARG_LOAD_MODE"));
add_opt(common_arg(
{"--longhaul"},
"stream routed Qwen3.5 MoE experts through a bounded Metal cache",
"stream routed Qwen3.x MoE experts through a bounded GPU cache",
[](common_params & params) {
params.load_mode = LLAMA_LOAD_MODE_LONGHAUL;
}
).set_env("LLAMA_ARG_LONGHAUL"));
add_opt(common_arg(
{"--longhaul-cache"}, "N",
"longhaul expert cache size in GiB",
[](common_params & params, int value) {
if (value <= 0) {
throw std::invalid_argument("longhaul cache size must be positive");
}
params.longhaul_cache_bytes = uint64_t(value) * 1024 * 1024 * 1024;
{"--longhaul-cache"}, "GiB",
"longhaul expert cache size in GiB; decimals are supported",
[](common_params & params, const std::string & value) {
params.longhaul_cache_bytes = parse_longhaul_cache_gib(value);
}
).set_env("LLAMA_ARG_LONGHAUL_CACHE"));
add_opt(common_arg(