owen/longhaul.cpp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add tokenizer longhaul cache

Browse Source
This commit is contained in:
Owen Qwen
2026-07-30 23:32:52 -05:00
parent a673afad85
commit e0665b1042
21 changed files with 1078 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/cli/README.md
+2
View File
@@ -35,6 +35,8 @@
| `--token-cache-size N` | shared tokenizer and tokenized-text cache size in MiB (default: 5120, 0 = disable)<br/>(env: LLAMA_ARG_TOKEN_CACHE_SIZE) |
| `--token-cache-dir PATH` | path to the shared tokenizer and tokenized-text cache database<br/>(env: LLAMA_ARG_TOKEN_CACHE_DIR) |
| `--token-cache-clear` | clear the shared tokenizer and tokenized-text cache before loading the model<br/>(env: LLAMA_ARG_TOKEN_CACHE_CLEAR) |
| `--tokenizer-longhaul` | build and page the Qwen3.5/Laguna tokenizer through a bounded cache<br/>(env: LLAMA_ARG_TOKENIZER_LONGHAUL) |
| `--tokenizer-longhaul-cache N` | tokenizer-longhaul RAM cache size in MiB<br/>(env: LLAMA_ARG_TOKENIZER_LONGHAUL_CACHE) |
| `-fa, --flash-attn [on\|off\|auto]` | set Flash Attention use ('on', 'off', or 'auto', default: 'auto')<br/>(env: LLAMA_ARG_FLASH_ATTN) |
| `-p, --prompt PROMPT` | prompt to start generation with; for system message, use -sys |
| `--perf, --no-perf` | whether to enable internal libllama performance timings (default: false)<br/>(env: LLAMA_ARG_PERF) |
tools/completion/README.md
+2
View File
@@ -118,6 +118,8 @@ llama-completion.exe -m models\gemma-1.1-7b-it.Q4_K_M.gguf --ignore-eos -n -1
| `--token-cache-size N` | shared tokenizer and tokenized-text cache size in MiB (default: 5120, 0 = disable)<br/>(env: LLAMA_ARG_TOKEN_CACHE_SIZE) |
| `--token-cache-dir PATH` | path to the shared tokenizer and tokenized-text cache database<br/>(env: LLAMA_ARG_TOKEN_CACHE_DIR) |
| `--token-cache-clear` | clear the shared tokenizer and tokenized-text cache before loading the model<br/>(env: LLAMA_ARG_TOKEN_CACHE_CLEAR) |
| `--tokenizer-longhaul` | build and page the Qwen3.5/Laguna tokenizer through a bounded cache<br/>(env: LLAMA_ARG_TOKENIZER_LONGHAUL) |
| `--tokenizer-longhaul-cache N` | tokenizer-longhaul RAM cache size in MiB<br/>(env: LLAMA_ARG_TOKENIZER_LONGHAUL_CACHE) |
| `-fa, --flash-attn [on\|off\|auto]` | set Flash Attention use ('on', 'off', or 'auto', default: 'auto')<br/>(env: LLAMA_ARG_FLASH_ATTN) |
| `-p, --prompt PROMPT` | prompt to start generation with; for system message, use -sys |
| `--perf, --no-perf` | whether to enable internal libllama performance timings (default: false)<br/>(env: LLAMA_ARG_PERF) |
tools/llama-bench/README.md
+2
View File
@@ -57,6 +57,8 @@ test parameters:
-ctk, --cache-type-k <t> (default: f16)
-ctv, --cache-type-v <t> (default: f16)
--longhaul-cache <GiB> expert cache size for longhaul mode
--tokenizer-longhaul page tokenizer indexes through a bounded cache
--tokenizer-longhaul-cache <MiB> tokenizer lookup cache size
-t, --threads <n> (default: system dependent)
-C, --cpu-mask <hex,hex> (default: 0x0)
--cpu-strict <0|1> (default: 0)
tools/llama-bench/llama-bench.cpp
+53 -3
View File
@@ -342,6 +342,8 @@ struct cmd_params {
std::vector<llama_split_mode> split_mode;
std::vector<llama_load_mode> load_mode;
uint64_t longhaul_cache_bytes;
bool tokenizer_longhaul;
uint64_t tokenizer_longhaul_cache_bytes;
std::vector<int> main_gpu;
std::vector<bool> no_kv_offload;
std::vector<llama_flash_attn_type> flash_attn;
@@ -387,6 +389,8 @@ static const cmd_params cmd_params_defaults = {
/* split_mode */ { LLAMA_SPLIT_MODE_LAYER },
/* load_mode */ { LLAMA_LOAD_MODE_MMAP },
/* longhaul_cache_bytes */ 0,
/* tokenizer_longhaul */ false,
/* tokenizer_longhaul_cache_bytes */ 0,
/* main_gpu */ { 0 },
/* no_kv_offload */ { false },
/* flash_attn */ { LLAMA_FLASH_ATTN_TYPE_AUTO },
@@ -463,6 +467,8 @@ static void print_usage(int /* argc */, char ** argv) {
printf(" -dev, --device <dev0/dev1/...> (default: auto)\n");
printf(" -lm, --load-mode <none|mmap|mlock|mmap+mlock|dio|longhaul> (default: %s)\n", join(transform_to_str(cmd_params_defaults.load_mode, llama_load_mode_name), ",").c_str());
printf(" --longhaul-cache <GiB> expert cache size for longhaul mode\n");
printf(" --tokenizer-longhaul page tokenizer indexes through a bounded cache\n");
printf(" --tokenizer-longhaul-cache <MiB> tokenizer lookup cache size (required with --tokenizer-longhaul)\n");
printf(" -mmp, --mmap <0|1> (DEPRECATED IN FAVOUR OF --load-mode)\n");
printf(" -dio, --direct-io <0|1> (DEPRECATED IN FAVOUR OF --load-mode)\n");
printf(" -embd, --embeddings <0|1> (default: %s)\n", join(cmd_params_defaults.embeddings, ",").c_str());
@@ -525,6 +531,8 @@ static cmd_params parse_cmd_params(int argc, char ** argv) {
params.no_warmup = cmd_params_defaults.no_warmup;
params.offline = cmd_params_defaults.offline;
params.longhaul_cache_bytes = cmd_params_defaults.longhaul_cache_bytes;
params.tokenizer_longhaul = cmd_params_defaults.tokenizer_longhaul;
params.tokenizer_longhaul_cache_bytes = cmd_params_defaults.tokenizer_longhaul_cache_bytes;
if (const char * env = getenv("HF_TOKEN")) {
params.hf_token = env;
@@ -801,6 +809,19 @@ static cmd_params parse_cmd_params(int argc, char ** argv) {
break;
}
params.longhaul_cache_bytes = gib * 1024 * 1024 * 1024;
} else if (arg == "--tokenizer-longhaul") {
params.tokenizer_longhaul = true;
} else if (arg == "--tokenizer-longhaul-cache") {
if (++i >= argc) {
invalid_param = true;
break;
}
const uint64_t mib = std::stoull(argv[i]);
if (mib == 0 || mib > UINT64_MAX / 1024 / 1024) {
invalid_param = true;
break;
}
params.tokenizer_longhaul_cache_bytes = mib * 1024 * 1024;
} else if (arg == "-mg" || arg == "--main-gpu") {
if (++i >= argc) {
invalid_param = true;
@@ -1157,6 +1178,10 @@ static cmd_params parse_cmd_params(int argc, char ** argv) {
fprintf(stderr, "error: longhaul load mode requires --longhaul-cache N\n");
exit(1);
}
if (params.tokenizer_longhaul != (params.tokenizer_longhaul_cache_bytes != 0)) {
fprintf(stderr, "error: --tokenizer-longhaul and --tokenizer-longhaul-cache must be used together\n");
exit(1);
}
if (params.main_gpu.empty()) {
params.main_gpu = cmd_params_defaults.main_gpu;
}
@@ -1224,6 +1249,8 @@ struct cmd_params_instance {
llama_split_mode split_mode;
llama_load_mode load_mode;
uint64_t longhaul_cache_bytes;
bool tokenizer_longhaul;
uint64_t tokenizer_longhaul_cache_bytes;
int main_gpu;
bool no_kv_offload;
llama_flash_attn_type flash_attn;
@@ -1246,6 +1273,8 @@ struct cmd_params_instance {
mparams.split_mode = split_mode;
mparams.load_mode = load_mode;
mparams.longhaul_cache_bytes = longhaul_cache_bytes;
mparams.tokenizer_longhaul = tokenizer_longhaul;
mparams.tokenizer_longhaul_cache_bytes = tokenizer_longhaul_cache_bytes;
mparams.main_gpu = main_gpu;
mparams.tensor_split = tensor_split.data();
mparams.no_host = no_host;
@@ -1294,6 +1323,8 @@ struct cmd_params_instance {
split_mode == other.split_mode &&
main_gpu == other.main_gpu && tensor_split == other.tensor_split &&
load_mode == other.load_mode && longhaul_cache_bytes == other.longhaul_cache_bytes &&
tokenizer_longhaul == other.tokenizer_longhaul &&
tokenizer_longhaul_cache_bytes == other.tokenizer_longhaul_cache_bytes &&
devices == other.devices && no_host == other.no_host &&
vec_tensor_buft_override_equal(tensor_buft_overrides, other.tensor_buft_overrides);
}
@@ -1368,6 +1399,8 @@ static std::vector<cmd_params_instance> get_cmd_params_instances(const cmd_param
/* .split_mode = */ sm,
/* .load_mode = */ lm,
/* .longhaul_cache_bytes = */ params.longhaul_cache_bytes,
/* .tokenizer_longhaul = */ params.tokenizer_longhaul,
/* .tokenizer_longhaul_cache_bytes = */ params.tokenizer_longhaul_cache_bytes,
/* .main_gpu = */ mg,
/* .no_kv_offload = */ nkvo,
/* .flash_attn = */ fa,
@@ -1405,6 +1438,8 @@ static std::vector<cmd_params_instance> get_cmd_params_instances(const cmd_param
/* .split_mode = */ sm,
/* .load_mode = */ lm,
/* .longhaul_cache_bytes = */ params.longhaul_cache_bytes,
/* .tokenizer_longhaul = */ params.tokenizer_longhaul,
/* .tokenizer_longhaul_cache_bytes = */ params.tokenizer_longhaul_cache_bytes,
/* .main_gpu = */ mg,
/* .no_kv_offload = */ nkvo,
/* .flash_attn = */ fa,
@@ -1442,6 +1477,8 @@ static std::vector<cmd_params_instance> get_cmd_params_instances(const cmd_param
/* .split_mode = */ sm,
/* .load_mode = */ lm,
/* .longhaul_cache_bytes = */ params.longhaul_cache_bytes,
/* .tokenizer_longhaul = */ params.tokenizer_longhaul,
/* .tokenizer_longhaul_cache_bytes = */ params.tokenizer_longhaul_cache_bytes,
/* .main_gpu = */ mg,
/* .no_kv_offload = */ nkvo,
/* .flash_attn = */ fa,
@@ -1484,6 +1521,8 @@ struct test {
llama_split_mode split_mode;
llama_load_mode load_mode;
uint64_t longhaul_cache_bytes;
bool tokenizer_longhaul;
uint64_t tokenizer_longhaul_cache_bytes;
int main_gpu;
bool no_kv_offload;
llama_flash_attn_type flash_attn;
@@ -1524,6 +1563,8 @@ struct test {
split_mode = inst.split_mode;
load_mode = inst.load_mode;
longhaul_cache_bytes = inst.longhaul_cache_bytes;
tokenizer_longhaul = inst.tokenizer_longhaul;
tokenizer_longhaul_cache_bytes = inst.tokenizer_longhaul_cache_bytes;
main_gpu = inst.main_gpu;
no_kv_offload = inst.no_kv_offload;
flash_attn = inst.flash_attn;
@@ -1591,7 +1632,8 @@ struct test {
"n_ubatch", "n_threads", "cpu_mask", "cpu_strict", "poll",
"type_k", "type_v", "n_gpu_layers", "n_cpu_moe", "split_mode",
"main_gpu", "no_kv_offload", "flash_attn", "devices", "tensor_split",
"tensor_buft_overrides", "load_mode", "longhaul_cache_bytes", "embeddings",
"tensor_buft_overrides", "load_mode", "longhaul_cache_bytes",
"tokenizer_longhaul", "tokenizer_longhaul_cache_bytes", "embeddings",
"no_op_offload", "no_host", "fit_target", "fit_min_ctx",
"n_prompt", "n_gen", "n_depth",
"test_time", "avg_ns", "stddev_ns", "avg_ts", "stddev_ts"
@@ -1607,11 +1649,11 @@ struct test {
field == "main_gpu" || field == "n_prompt" || field == "n_gen" || field == "n_depth" || field == "avg_ns" ||
field == "stddev_ns" || field == "no_op_offload" || field == "n_cpu_moe" ||
field == "fit_target" || field == "fit_min_ctx" || field == "flash_attn" ||
field == "longhaul_cache_bytes") {
field == "longhaul_cache_bytes" || field == "tokenizer_longhaul_cache_bytes") {
return INT;
}
if (field == "f16_kv" || field == "no_kv_offload" || field == "cpu_strict" ||
field == "embeddings" || field == "no_host") {
field == "embeddings" || field == "no_host" || field == "tokenizer_longhaul") {
return BOOL;
}
if (field == "avg_ts" || field == "stddev_ts") {
@@ -1688,6 +1730,8 @@ struct test {
tensor_buft_overrides_str,
llama_load_mode_name(load_mode),
std::to_string(longhaul_cache_bytes),
std::to_string(tokenizer_longhaul),
std::to_string(tokenizer_longhaul_cache_bytes),
std::to_string(embeddings),
std::to_string(no_op_offload),
std::to_string(no_host),
@@ -2005,6 +2049,12 @@ struct markdown_printer : public printer {
if (params.longhaul_cache_bytes != cmd_params_defaults.longhaul_cache_bytes) {
fields.emplace_back("longhaul_cache_bytes");
}
if (params.tokenizer_longhaul != cmd_params_defaults.tokenizer_longhaul) {
fields.emplace_back("tokenizer_longhaul");
}
if (params.tokenizer_longhaul_cache_bytes != cmd_params_defaults.tokenizer_longhaul_cache_bytes) {
fields.emplace_back("tokenizer_longhaul_cache_bytes");
}
if (params.embeddings.size() > 1 || params.embeddings != cmd_params_defaults.embeddings) {
fields.emplace_back("embeddings");
}
tools/server/README.md
+2
View File
@@ -56,6 +56,8 @@ For the full list of features, please refer to [server's changelog](https://gith
| `--token-cache-size N` | shared tokenizer and tokenized-text cache size in MiB (default: 5120, 0 = disable)<br/>(env: LLAMA_ARG_TOKEN_CACHE_SIZE) |
| `--token-cache-dir PATH` | path to the shared tokenizer and tokenized-text cache database<br/>(env: LLAMA_ARG_TOKEN_CACHE_DIR) |
| `--token-cache-clear` | clear the shared tokenizer and tokenized-text cache before loading the model<br/>(env: LLAMA_ARG_TOKEN_CACHE_CLEAR) |
| `--tokenizer-longhaul` | build and page the Qwen3.5/Laguna tokenizer through a bounded cache<br/>(env: LLAMA_ARG_TOKENIZER_LONGHAUL) |
| `--tokenizer-longhaul-cache N` | tokenizer-longhaul RAM cache size in MiB<br/>(env: LLAMA_ARG_TOKENIZER_LONGHAUL_CACHE) |
| `-fa, --flash-attn [on\|off\|auto]` | set Flash Attention use ('on', 'off', or 'auto', default: 'auto')<br/>(env: LLAMA_ARG_FLASH_ATTN) |
| `--perf, --no-perf` | whether to enable internal libllama performance timings (default: false)<br/>(env: LLAMA_ARG_PERF) |
| `-e, --escape, --no-escape` | whether to process escapes sequences (\n, \r, \t, \', \", \\) (default: true) |