owen/inference-worker
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

update test timeout and default LLAMA_SERVER_CMD_ARGS for improved performance

Browse Source
This commit is contained in:
mags0ft
2025-12-25 19:17:31 +01:00
parent 889d3c5c96
commit 398b59c0a1
2 changed files with 5 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.runpod/tests.json
+2 -2
View File
@@ -5,7 +5,7 @@
"input": {
"prompt": "Hi! Who are you?"
},
"timeout": 120000
"timeout": 60000
}
],
"config": {
@@ -14,7 +14,7 @@
"env": [
{
"key": "LLAMA_SERVER_CMD_ARGS",
"value": "-hf unsloth/gemma-3-270m-it-GGUF:Q6_K --ctx-size 4096"
"value": "-hf unsloth/gemma-3-270m-it-GGUF:IQ2_XXS --ctx-size 512 -ngl 999"
}
],
"allowedCudaVersions": [
src/start.sh
+3 -3
View File
@@ -16,13 +16,13 @@ cleanup() {
# check if $LLAMA_SERVER_CMD_ARGS is set
if [ -z "$LLAMA_SERVER_CMD_ARGS" ]; then
echo "start.sh: Warning: LLAMA_SERVER_CMD_ARGS is not set. Defaulting to -hf unsloth/gemma-3-270m-it-GGUF:Q6_K --ctx-size 4096"
LLAMA_SERVER_CMD_ARGS="-hf unsloth/gemma-3-270m-it-GGUF:Q6_K --ctx-size 4096 -ngl 99"
echo "start.sh: Warning: LLAMA_SERVER_CMD_ARGS is not set. Defaulting to -hf unsloth/gemma-3-270m-it-GGUF:IQ2_XXS --ctx-size 512 -ngl 999"
LLAMA_SERVER_CMD_ARGS="-hf unsloth/gemma-3-270m-it-GGUF:IQ2_XXS --ctx-size 512 -ngl 999"
fi
# check if the substring /workspace is in LLAMA_SERVER_CMD_ARGS
if [[ "$LLAMA_SERVER_CMD_ARGS" != *"/workspace"* ]]; then
echo "start.sh: Tip: For reduced downloads and faster startup times, consider using a model stored in a network volume mounted to /workspace."
echo "start.sh: Tip: For reduced downloads and faster startup times, consider using a model stored in the RunPod cache."
fi
# check if the substring --port is in LLAMA_SERVER_CMD_ARGS and if yes, raise an error: