Add instructions for embeding a model

README.md

@@ -20,6 +20,10 @@ See the [test_inputs](./test_inputs) directory for example test requests.
 
 Streaming for openai requests are fully working.
 
+## Preload model into the docker image
+
+See the [embed_model](./embed_model/) directory for instructions.
+
 ## Licence
 
 This project is licensed under the Creative Commons Attribution 4.0 International License. You are free to use, share, and adapt the material for any purpose, even commercially, under the following terms:

embed_model/Dockerfile

@@ -0,0 +1,10 @@
+FROM svenbrnn/runpod-ollama:0.5.7
+
+ARG MODEL_NAME
+ENV MODEL_NAME=$MODEL_NAME
+ADD preload_model.sh /preload_model.sh
+RUN chmod +x /preload_model.sh && /preload_model.sh
+
+# Copy the model to the volume
+FROM svenbrnn/runpod-ollama:0.5.7
+COPY --from=0 /runpod-volume /runpod-volume

embed_model/README.md

@@ -0,0 +1,3 @@
+# Embed model into runpod-worker-ollama
+
+Run ``docker build --build-arg MODEL_NAME="<model-name>" -t <yourname>/<yourrepo>:<tag> .`` 

embed_model/preload_model.sh

@@ -0,0 +1,31 @@
+#!/bin/sh
+
+# Start the Ollama server in the background
+echo "Starting Ollama server to preload model: $MODEL_NAME"
+ollama serve &
+
+# Capture the PID of the Ollama server
+OLLAMA_PID=$!
+
+# Wait for the server to be ready (adjust if necessary)
+echo "Waiting for Ollama server to start..."
+sleep 5
+
+# Pull the specified model
+echo "Pulling model: $MODEL_NAME"
+if ollama pull "$MODEL_NAME"; then
+  echo "Successfully pulled model: $MODEL_NAME"
+else
+  echo "Failed to pull model: $MODEL_NAME"
+  kill $OLLAMA_PID
+  exit 1
+fi
+
+# Stop the Ollama server
+echo "Stopping Ollama server..."
+kill $OLLAMA_PID
+
+# Wait for the server to terminate
+wait $OLLAMA_PID
+
+echo "Model preloading complete."