Add tokenizer longhaul cache

2026-07-30 23:32:52 -05:00
parent a673afad85
commit e0665b1042
21 changed files with 1078 additions and 23 deletions
@@ -486,6 +486,8 @@ struct common_params {
    enum llama_split_mode split_mode = LLAMA_SPLIT_MODE_LAYER; // how to split the model across GPUs
    enum llama_load_mode  load_mode  = LLAMA_LOAD_MODE_MMAP; // how to load the model
    uint64_t longhaul_cache_bytes = 0;
+    bool tokenizer_longhaul = false;
+    uint64_t tokenizer_longhaul_cache_bytes = 0;

    common_cpu_params cpuparams;
    common_cpu_params cpuparams_batch;