2 Commits
Author SHA1 Message Date
mags0ft 0a59d57025 rename tests.json temporarily to skip apparently buggy RunPod CI
The RunPod support told me to do this until the issues are resolved.
2025-12-26 11:37:23 +01:00
mags0ft 398b59c0a1 update test timeout and default LLAMA_SERVER_CMD_ARGS for improved performance 2025-12-25 19:17:31 +01:00
2 changed files with 5 additions and 5 deletions
+2 -2
View File
@@ -5,7 +5,7 @@
"input": { "input": {
"prompt": "Hi! Who are you?" "prompt": "Hi! Who are you?"
}, },
"timeout": 120000 "timeout": 60000
} }
], ],
"config": { "config": {
@@ -14,7 +14,7 @@
"env": [ "env": [
{ {
"key": "LLAMA_SERVER_CMD_ARGS", "key": "LLAMA_SERVER_CMD_ARGS",
"value": "-hf unsloth/gemma-3-270m-it-GGUF:Q6_K --ctx-size 4096" "value": "-hf unsloth/gemma-3-270m-it-GGUF:IQ2_XXS --ctx-size 512 -ngl 999"
} }
], ],
"allowedCudaVersions": [ "allowedCudaVersions": [
+3 -3
View File
@@ -16,13 +16,13 @@ cleanup() {
# check if $LLAMA_SERVER_CMD_ARGS is set # check if $LLAMA_SERVER_CMD_ARGS is set
if [ -z "$LLAMA_SERVER_CMD_ARGS" ]; then if [ -z "$LLAMA_SERVER_CMD_ARGS" ]; then
echo "start.sh: Warning: LLAMA_SERVER_CMD_ARGS is not set. Defaulting to -hf unsloth/gemma-3-270m-it-GGUF:Q6_K --ctx-size 4096" echo "start.sh: Warning: LLAMA_SERVER_CMD_ARGS is not set. Defaulting to -hf unsloth/gemma-3-270m-it-GGUF:IQ2_XXS --ctx-size 512 -ngl 999"
LLAMA_SERVER_CMD_ARGS="-hf unsloth/gemma-3-270m-it-GGUF:Q6_K --ctx-size 4096 -ngl 99" LLAMA_SERVER_CMD_ARGS="-hf unsloth/gemma-3-270m-it-GGUF:IQ2_XXS --ctx-size 512 -ngl 999"
fi fi
# check if the substring /workspace is in LLAMA_SERVER_CMD_ARGS # check if the substring /workspace is in LLAMA_SERVER_CMD_ARGS
if [[ "$LLAMA_SERVER_CMD_ARGS" != *"/workspace"* ]]; then if [[ "$LLAMA_SERVER_CMD_ARGS" != *"/workspace"* ]]; then
echo "start.sh: Tip: For reduced downloads and faster startup times, consider using a model stored in a network volume mounted to /workspace." echo "start.sh: Tip: For reduced downloads and faster startup times, consider using a model stored in the RunPod cache."
fi fi
# check if the substring --port is in LLAMA_SERVER_CMD_ARGS and if yes, raise an error: # check if the substring --port is in LLAMA_SERVER_CMD_ARGS and if yes, raise an error: