owen/longhaul.cpp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add Vulkan longhaul support

Browse Source
This commit is contained in:
Owen Qwen
2026-07-30 15:09:43 -05:00
parent a673afad85
commit fa404a9d97
13 changed files with 262 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/llama-bench/README.md
+1 -1
View File
@@ -56,7 +56,7 @@ test parameters:
-ub, --ubatch-size <n> (default: 512)
-ctk, --cache-type-k <t> (default: f16)
-ctv, --cache-type-v <t> (default: f16)
--longhaul-cache <GiB> expert cache size for longhaul mode
--longhaul-cache <GiB> expert cache size for longhaul mode (decimals supported)
-t, --threads <n> (default: system dependent)
-C, --cpu-mask <hex,hex> (default: 0x0)
--cpu-strict <0|1> (default: 0)
tools/llama-bench/llama-bench.cpp
+20 -5
View File
@@ -10,6 +10,7 @@
#include <cstring>
#include <ctime>
#include <iterator>
#include <limits>
#include <map>
#include <numeric>
#include <regex>
@@ -37,6 +38,20 @@
#endif
// utils
static uint64_t parse_longhaul_cache_gib(const char * value) {
size_t end = 0;
const std::string text(value);
const double gib = std::stod(text, &end);
constexpr long double bytes_per_gib = 1024.0L * 1024.0L * 1024.0L;
const long double bytes = (long double) gib * bytes_per_gib;
if (end != text.size() || !std::isfinite(gib) || gib <= 0.0 ||
bytes < 1.0L || bytes > (long double) std::numeric_limits<uint64_t>::max()) {
throw std::invalid_argument("invalid longhaul cache size");
}
return (uint64_t) bytes;
}
static uint64_t get_time_ns() {
using clock = std::chrono::high_resolution_clock;
return std::chrono::nanoseconds(clock::now().time_since_epoch()).count();
@@ -462,7 +477,7 @@ static void print_usage(int /* argc */, char ** argv) {
printf(" -fa, --flash-attn <on|off|auto> (default: %s)\n", join(transform_to_str(cmd_params_defaults.flash_attn, llama_flash_attn_type_name), ",").c_str());
printf(" -dev, --device <dev0/dev1/...> (default: auto)\n");
printf(" -lm, --load-mode <none|mmap|mlock|mmap+mlock|dio|longhaul> (default: %s)\n", join(transform_to_str(cmd_params_defaults.load_mode, llama_load_mode_name), ",").c_str());
printf(" --longhaul-cache <GiB> expert cache size for longhaul mode\n");
printf(" --longhaul-cache <GiB> expert cache size for longhaul mode (decimals supported)\n");
printf(" -mmp, --mmap <0|1> (DEPRECATED IN FAVOUR OF --load-mode)\n");
printf(" -dio, --direct-io <0|1> (DEPRECATED IN FAVOUR OF --load-mode)\n");
printf(" -embd, --embeddings <0|1> (default: %s)\n", join(cmd_params_defaults.embeddings, ",").c_str());
@@ -795,12 +810,12 @@ static cmd_params parse_cmd_params(int argc, char ** argv) {
invalid_param = true;
break;
}
const uint64_t gib = std::stoull(argv[i]);
if (gib == 0 || gib > UINT64_MAX / 1024 / 1024 / 1024) {
try {
params.longhaul_cache_bytes = parse_longhaul_cache_gib(argv[i]);
} catch (const std::exception &) {
invalid_param = true;
break;
}
params.longhaul_cache_bytes = gib * 1024 * 1024 * 1024;
} else if (arg == "-mg" || arg == "--main-gpu") {
if (++i >= argc) {
invalid_param = true;
@@ -1154,7 +1169,7 @@ static cmd_params parse_cmd_params(int argc, char ** argv) {
}
if (std::find(params.load_mode.begin(), params.load_mode.end(), LLAMA_LOAD_MODE_LONGHAUL) != params.load_mode.end() &&
params.longhaul_cache_bytes == 0) {
fprintf(stderr, "error: longhaul load mode requires --longhaul-cache N\n");
fprintf(stderr, "error: longhaul load mode requires --longhaul-cache GiB\n");
exit(1);
}
if (params.main_gpu.empty()) {