alpayariyak
4cebe66b36
- You no longer need a linux-based machine or NVIDIA GPUs to build the worker. - Over 3x lighter Docker image size. - OpenAI Chat Completion output format (optional to use). - Extremely fast image build time. - Docker Secrets-protected Hugging Face token support for building the image with a model baked in without exposing your token. - Support for `n` and `best_of` sampling parameters, which allow you to generate multiple responses from a single prompt. - New environment variables for various configuration. - vLLM Version: 0.2.7
40 lines
1.2 KiB
Docker
40 lines
1.2 KiB
Docker
ARG WORKER_CUDA_VERSION=11.8.0
|
|
FROM runpod/worker-vllm:base-0.2.0-cuda${WORKER_CUDA_VERSION} AS vllm-base
|
|
|
|
RUN apt-get update -y \
|
|
&& apt-get install -y python3-pip
|
|
|
|
# Install Python dependencies
|
|
COPY builder/requirements.txt /requirements.txt
|
|
RUN --mount=type=cache,target=/root/.cache/pip \
|
|
python3 -m pip install --upgrade pip && \
|
|
python3 -m pip install --upgrade -r /requirements.txt
|
|
|
|
# Add source files
|
|
COPY src /src
|
|
|
|
# Setup for Option 2: Building the Image with the Model included
|
|
ARG MODEL_NAME=""
|
|
ARG MODEL_BASE_PATH="/runpod-volume"
|
|
ARG QUANTIZATION=""
|
|
|
|
ENV MODEL_BASE_PATH=$MODEL_BASE_PATH \
|
|
MODEL_NAME=$MODEL_NAME \
|
|
QUANTIZATION=$QUANTIZATION \
|
|
HF_DATASETS_CACHE="${MODEL_BASE_PATH}/huggingface-cache/datasets" \
|
|
HUGGINGFACE_HUB_CACHE="${MODEL_BASE_PATH}/huggingface-cache/hub" \
|
|
HF_HOME="${MODEL_BASE_PATH}/huggingface-cache/hub" \
|
|
HF_TRANSFER=1
|
|
|
|
RUN --mount=type=secret,id=HF_TOKEN,required=false \
|
|
if [ -f /run/secrets/HF_TOKEN ]; then \
|
|
export HF_TOKEN=$(cat /run/secrets/HF_TOKEN); \
|
|
fi && \
|
|
if [ -n "$MODEL_NAME" ]; then \
|
|
python3 /src/download_model.py --model $MODEL_NAME; \
|
|
fi
|
|
|
|
ENV PYTHONPATH="/:/vllm-installation"
|
|
|
|
# Start the handler
|
|
CMD ["python3", "/src/handler.py"] |