c45ac42acd
Release / release (push) Waiting to run
* VLLM upgrade to 0.12.0 and compatibility fixes * MAX_NUM_BATCHED_TOKENS fix and CUDA tester * Sys kill worker instead of marking as failed * upgrade to vllm 0.12.0 * Update to vllm 0.15.0 and lora fix * Update for HUB and removal of deprected env variables * reverted docker-bake changes * removed leftovers * Update src/handler.py Co-authored-by: Dj Isaac <contact@dejaydev.com> * Update src/utils.py Co-authored-by: Dj Isaac <contact@dejaydev.com> * Update src/handler.py Co-authored-by: Dj Isaac <contact@dejaydev.com> * Clean up of docs and comments in code * nit: lowercase p * nit: lowercase p --------- Co-authored-by: Dj Isaac <contact@dejaydev.com> Co-authored-by: chrisvela <chris.vela@runpod.io>
59 lines
1.9 KiB
Docker
59 lines
1.9 KiB
Docker
FROM nvidia/cuda:12.8.0-base-ubuntu22.04
|
|
|
|
RUN apt-get update -y \
|
|
&& apt-get install -y python3-pip
|
|
|
|
RUN ldconfig /usr/local/cuda-12.8/compat/
|
|
|
|
# Install vLLM with FlashInfer - use CUDA 12.8 PyTorch wheels (compatible with vLLM 0.15.0)
|
|
RUN python3 -m pip install --upgrade pip && \
|
|
python3 -m pip install "vllm[flashinfer]==0.15.0" --extra-index-url https://download.pytorch.org/whl/cu128
|
|
|
|
|
|
|
|
# Install additional Python dependencies (after vLLM to avoid PyTorch version conflicts)
|
|
COPY builder/requirements.txt /requirements.txt
|
|
RUN --mount=type=cache,target=/root/.cache/pip \
|
|
python3 -m pip install --upgrade -r /requirements.txt
|
|
|
|
# Setup for Option 2: Building the Image with the Model included
|
|
ARG MODEL_NAME=""
|
|
ARG TOKENIZER_NAME=""
|
|
ARG BASE_PATH="/runpod-volume"
|
|
ARG QUANTIZATION=""
|
|
ARG MODEL_REVISION=""
|
|
ARG TOKENIZER_REVISION=""
|
|
|
|
ENV MODEL_NAME=$MODEL_NAME \
|
|
MODEL_REVISION=$MODEL_REVISION \
|
|
TOKENIZER_NAME=$TOKENIZER_NAME \
|
|
TOKENIZER_REVISION=$TOKENIZER_REVISION \
|
|
BASE_PATH=$BASE_PATH \
|
|
QUANTIZATION=$QUANTIZATION \
|
|
HF_DATASETS_CACHE="${BASE_PATH}/huggingface-cache/datasets" \
|
|
HUGGINGFACE_HUB_CACHE="${BASE_PATH}/huggingface-cache/hub" \
|
|
HF_HOME="${BASE_PATH}/huggingface-cache/hub" \
|
|
HF_HUB_ENABLE_HF_TRANSFER=0 \
|
|
# Suppress Ray metrics agent warnings (not needed in containerized environments)
|
|
RAY_METRICS_EXPORT_ENABLED=0 \
|
|
RAY_DISABLE_USAGE_STATS=1 \
|
|
# Prevent rayon thread pool panic in containers where ulimit -u < nproc
|
|
# (tokenizers uses Rust's rayon which tries to spawn threads = CPU cores)
|
|
TOKENIZERS_PARALLELISM=false \
|
|
RAYON_NUM_THREADS=4
|
|
|
|
ENV PYTHONPATH="/:/vllm-workspace"
|
|
|
|
|
|
COPY src /src
|
|
RUN --mount=type=secret,id=HF_TOKEN,required=false \
|
|
if [ -f /run/secrets/HF_TOKEN ]; then \
|
|
export HF_TOKEN=$(cat /run/secrets/HF_TOKEN); \
|
|
fi && \
|
|
if [ -n "$MODEL_NAME" ]; then \
|
|
python3 /src/download_model.py; \
|
|
fi
|
|
|
|
# Start the handler
|
|
CMD ["python3", "/src/handler.py"]
|