Add tokenizer longhaul cache

2026-07-30 23:32:52 -05:00
parent a673afad85
commit e0665b1042
21 changed files with 1078 additions and 23 deletions
@@ -86,7 +86,11 @@ struct llama_vocab {
    llama_vocab();
    ~llama_vocab();

-    void load(llama_model_loader & ml, const LLM_KV & kv);
+    void load(
+            llama_model_loader & ml,
+            const LLM_KV & kv,
+            bool tokenizer_longhaul = false,
+            uint64_t tokenizer_longhaul_cache_bytes = 0);

    std::string get_tokenizer_model() const;
    std::string get_tokenizer_pre() const;
@@ -181,7 +185,7 @@ struct llama_vocab {
                         bool   special) const;

    // use cached data
-    const std::string & token_to_piece(llama_token token) const;
+    std::string token_to_piece(llama_token token) const;

    int32_t detokenize(
            const llama_token * tokens,