Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0a59d57025 | ||
|
|
398b59c0a1 |
@@ -5,7 +5,7 @@
|
|||||||
"input": {
|
"input": {
|
||||||
"prompt": "Hi! Who are you?"
|
"prompt": "Hi! Who are you?"
|
||||||
},
|
},
|
||||||
"timeout": 120000
|
"timeout": 60000
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"config": {
|
"config": {
|
||||||
@@ -14,7 +14,7 @@
|
|||||||
"env": [
|
"env": [
|
||||||
{
|
{
|
||||||
"key": "LLAMA_SERVER_CMD_ARGS",
|
"key": "LLAMA_SERVER_CMD_ARGS",
|
||||||
"value": "-hf unsloth/gemma-3-270m-it-GGUF:Q6_K --ctx-size 4096"
|
"value": "-hf unsloth/gemma-3-270m-it-GGUF:IQ2_XXS --ctx-size 512 -ngl 999"
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"allowedCudaVersions": [
|
"allowedCudaVersions": [
|
||||||
+3
-3
@@ -16,13 +16,13 @@ cleanup() {
|
|||||||
|
|
||||||
# check if $LLAMA_SERVER_CMD_ARGS is set
|
# check if $LLAMA_SERVER_CMD_ARGS is set
|
||||||
if [ -z "$LLAMA_SERVER_CMD_ARGS" ]; then
|
if [ -z "$LLAMA_SERVER_CMD_ARGS" ]; then
|
||||||
echo "start.sh: Warning: LLAMA_SERVER_CMD_ARGS is not set. Defaulting to -hf unsloth/gemma-3-270m-it-GGUF:Q6_K --ctx-size 4096"
|
echo "start.sh: Warning: LLAMA_SERVER_CMD_ARGS is not set. Defaulting to -hf unsloth/gemma-3-270m-it-GGUF:IQ2_XXS --ctx-size 512 -ngl 999"
|
||||||
LLAMA_SERVER_CMD_ARGS="-hf unsloth/gemma-3-270m-it-GGUF:Q6_K --ctx-size 4096 -ngl 99"
|
LLAMA_SERVER_CMD_ARGS="-hf unsloth/gemma-3-270m-it-GGUF:IQ2_XXS --ctx-size 512 -ngl 999"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# check if the substring /workspace is in LLAMA_SERVER_CMD_ARGS
|
# check if the substring /workspace is in LLAMA_SERVER_CMD_ARGS
|
||||||
if [[ "$LLAMA_SERVER_CMD_ARGS" != *"/workspace"* ]]; then
|
if [[ "$LLAMA_SERVER_CMD_ARGS" != *"/workspace"* ]]; then
|
||||||
echo "start.sh: Tip: For reduced downloads and faster startup times, consider using a model stored in a network volume mounted to /workspace."
|
echo "start.sh: Tip: For reduced downloads and faster startup times, consider using a model stored in the RunPod cache."
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# check if the substring --port is in LLAMA_SERVER_CMD_ARGS and if yes, raise an error:
|
# check if the substring --port is in LLAMA_SERVER_CMD_ARGS and if yes, raise an error:
|
||||||
|
|||||||
Reference in New Issue
Block a user