stuff

include/llama.h

@@ -304,6 +304,26 @@ extern "C" {
         ggml_backend_buffer_type_t buft;
     };
 
+    struct llama_token_cache_params {
+        // NULL selects the platform llama.cpp cache directory.
+        const char * path;
+        // Shared logical entry budget. 0 disables the cache for this process.
+        uint64_t capacity_bytes;
+    };
+
+    struct llama_token_cache_stats {
+        uint64_t tokenizer_hits;
+        uint64_t tokenizer_misses;
+        uint64_t text_hits;
+        uint64_t text_misses;
+        uint64_t writes;
+        uint64_t dropped_writes;
+        uint64_t evictions;
+        uint64_t errors;
+        uint64_t entries;
+        uint64_t bytes_used;
+    };
+
     struct llama_model_params {
         // NULL-terminated list of devices to use for offloading (if NULL, all available devices are used)
         ggml_backend_dev_t * devices;
@@ -460,6 +480,12 @@ extern "C" {
     // Helpers for getting default parameters
     // TODO: update API to start accepting pointers to params structs (https://github.com/ggml-org/llama.cpp/discussions/9172)
     LLAMA_API struct llama_model_params          llama_model_default_params(void);
+    LLAMA_API struct llama_token_cache_params    llama_token_cache_default_params(void);
+    // Must be called before the first model load or tokenization operation.
+    LLAMA_API bool llama_token_cache_configure(struct llama_token_cache_params params);
+    LLAMA_API bool llama_token_cache_flush(void);
+    LLAMA_API bool llama_token_cache_clear(void);
+    LLAMA_API struct llama_token_cache_stats llama_token_cache_get_stats(void);
     LLAMA_API struct llama_context_params        llama_context_default_params(void);
     LLAMA_API struct llama_sampler_chain_params  llama_sampler_chain_default_params(void);
     LLAMA_API struct llama_model_quantize_params llama_model_quantize_default_params(void);