Add tokenizer longhaul cache

common/arg.cpp

@@ -817,6 +817,15 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context
     if (params.load_mode == LLAMA_LOAD_MODE_LONGHAUL && params.longhaul_cache_bytes == 0) {
         throw std::invalid_argument("error: --longhaul requires --longhaul-cache N\n");
     }
+    if (params.tokenizer_longhaul != (params.tokenizer_longhaul_cache_bytes != 0)) {
+        throw std::invalid_argument(
+            params.tokenizer_longhaul
+                ? "error: --tokenizer-longhaul requires --tokenizer-longhaul-cache N\n"
+                : "error: --tokenizer-longhaul-cache requires --tokenizer-longhaul\n");
+    }
+    if (params.tokenizer_longhaul && params.token_cache_size_mib == 0) {
+        throw std::invalid_argument("error: --tokenizer-longhaul requires an enabled persistent token cache\n");
+    }
 
     postprocess_cpu_params(params.cpuparams,       nullptr);
     postprocess_cpu_params(params.cpuparams_batch, &params.cpuparams);
@@ -2646,6 +2655,23 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
             params.longhaul_cache_bytes = uint64_t(value) * 1024 * 1024 * 1024;
         }
     ).set_env("LLAMA_ARG_LONGHAUL_CACHE"));
+    add_opt(common_arg(
+        {"--tokenizer-longhaul"},
+        "build and page the Qwen3.5/Laguna tokenizer through a bounded cache",
+        [](common_params & params) {
+            params.tokenizer_longhaul = true;
+        }
+    ).set_env("LLAMA_ARG_TOKENIZER_LONGHAUL"));
+    add_opt(common_arg(
+        {"--tokenizer-longhaul-cache"}, "N",
+        "tokenizer-longhaul RAM cache size in MiB",
+        [](common_params & params, int value) {
+            if (value <= 0) {
+                throw std::invalid_argument("tokenizer-longhaul cache size must be positive");
+            }
+            params.tokenizer_longhaul_cache_bytes = uint64_t(value) * 1024 * 1024;
+        }
+    ).set_env("LLAMA_ARG_TOKENIZER_LONGHAUL_CACHE"));
     add_opt(common_arg(
         {"--numa"}, "TYPE",
         "attempt optimizations that help on some NUMA systems\n"