owen/inference-worker
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add caching support for models in start.sh and update documentation

Browse Source
This commit is contained in:
mags0ft
2025-12-26 14:49:03 +01:00
parent 0a59d57025
commit 0c814177fe
4 changed files with 104 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.runpod/hub.json
+22 -2
View File
@@ -16,11 +16,31 @@
"input": {
"name": "Command line arguments for llama-server",
"type": "string",
"description": "Launch command line arguments (argv) for the llama-server binary. Do not define the port.",
"default": "-hf unsloth/gemma-3-270m-it-GGUF:Q6_K --ctx-size 4096 -ngl 99",
"description": "Launch command line arguments (argv) for the llama-server binary. Do not define the port. If using caching, do not define -hf or -m here.",
"default": "-hf unsloth/gemma-3-270m-it-GGUF:Q6_K --ctx-size 4096 -ngl 999",
"advanced": false
}
},
{
"key": "LLAMA_CACHED_MODEL",
"input": {
"name": "Hugging Face Hub model name for cached model",
"type": "string",
"description": "Hugging Face Hub model name to use for the cached GGUF model. Leave empty to disable caching. Example: user/model-name",
"default": "",
"advanced": true
}
},
{
"key": "LLAMA_CACHED_GGUF_PATH",
"input": {
"name": "Path to GGUF file in the Hugging Face Hub model repository",
"type": "string",
"description": "Path to the GGUF file in the Hugging Face Hub model repository to use for caching. Example: model.gguf",
"default": "",
"advanced": true
}
},
{
"key": "MAX_CONCURRENCY",
"input": {