owen/worker-vllm
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

0.3.3

Browse Source
This commit is contained in:
alpayariyak
2024-03-05 19:14:35 +00:00
parent d91ccb866f
commit db7167d57f
7 changed files with 20 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitmodules
+3
View File
@@ -0,0 +1,3 @@
[submodule "vllm-base-image/vllm"]
path = vllm-base-image/vllm
url = /devdisk/inference/worker-vllm/vllm-base-image/vllm
Dockerfile
+1 -1
View File
@@ -1,5 +1,5 @@
ARG WORKER_CUDA_VERSION=11.8.0
FROM runpod/worker-vllm:base-0.3.1-cuda${WORKER_CUDA_VERSION} AS vllm-base
FROM runpod/worker-vllm:base-0.3.2-cuda${WORKER_CUDA_VERSION} AS vllm-base
RUN apt-get update -y \
&& apt-get install -y python3-pip
README.md
+2 -2
View File
@@ -4,7 +4,7 @@
Deploy Blazing-fast LLMs powered by [vLLM](https://github.com/vllm-project/vllm) on RunPod Serverless in a few clicks.
<p>Worker Version: 0.3.1 | vLLM Version: 0.3.2</p>
<p>Worker Version: 0.3.2 | vLLM Version: 0.3.3</p>
[![CD | Docker-Build-Release](https://github.com/runpod-workers/worker-vllm/actions/workflows/docker-build-release.yml/badge.svg)](https://github.com/runpod-workers/worker-vllm/actions/workflows/docker-build-release.yml)
@@ -88,7 +88,7 @@ This table provides a quick reference to the image tags you should use based on
**LLM Settings**
| `MODEL_NAME`**\*** | - | `str` | Hugging Face Model Repository (e.g., `openchat/openchat-3.5-1210`). |
| `MODEL_REVISION` | `None` | `str` |Model revision(branch) to load. |
| `MAX_MODEL_LENGTH` | Model's maximum | `int` |Maximum number of tokens for the engine to handle per request. |
| `MAX_MODEL_LEN` | Model's maximum | `int` |Maximum number of tokens for the engine to handle per request. |
| `BASE_PATH` | `/runpod-volume` | `str` |Storage directory for Huggingface cache and model. Utilizes network storage if attached when pointed at `/runpod-volume`, which will have only one worker download the model once, which all workers will be able to load. If no network volume is present, creates a local directory within each worker. |
| `LOAD_FORMAT` | `auto` | `str` |Format to load model in. |
| `HF_TOKEN` | - | `str` |Hugging Face token for private and gated models. |
src/config.py
+1 -1
View File
@@ -39,7 +39,7 @@ class EngineConfig:
"trust_remote_code": bool(int(os.getenv("TRUST_REMOTE_CODE", 0))),
"gpu_memory_utilization": float(os.getenv("GPU_MEMORY_UTILIZATION", 0.95)),
"max_parallel_loading_workers": None if device_count() > 1 or not os.getenv("MAX_PARALLEL_LOADING_WORKERS") else int(os.getenv("MAX_PARALLEL_LOADING_WORKERS")),
"max_model_len": int(os.getenv("MAX_MODEL_LENGTH")) if os.getenv("MAX_MODEL_LENGTH") else None,
"max_model_len": int(os.getenv("MAX_MODEL_LEN")) if os.getenv("MAX_MODEL_LEN") else None,
"tensor_parallel_size": device_count(),
"seed": int(os.getenv("SEED")) if os.getenv("SEED") else None,
"kv_cache_dtype": os.getenv("KV_CACHE_DTYPE"),
vllm-base/Dockerfile → vllm-base-image/Dockerfile
+13 -34
View File
@@ -17,25 +17,16 @@ ARG WORKER_CUDA_VERSION
RUN apt-get update -y \
&& apt-get install -y python3-pip git
RUN if [ "${WORKER_CUDA_VERSION}" = "12.1.0" ]; then \
ldconfig /usr/local/cuda-12.1/compat/; \
fi
# Set working directory
WORKDIR /vllm-installation
# Install build and runtime dependencies
COPY vllm-${WORKER_CUDA_VERSION}/requirements.txt requirements.txt
COPY vllm/requirements-${WORKER_CUDA_VERSION}.txt requirements.txt
RUN --mount=type=cache,target=/root/.cache/pip \
pip install -r requirements.txt
RUN --mount=type=cache,target=/root/.cache/pip \
if [ "${WORKER_CUDA_VERSION}" = "11.8.0" ]; then \
pip install -U --force-reinstall torch==2.1.2 xformers==0.0.23.post1 --index-url https://download.pytorch.org/whl/cu118; \
fi
# Install development dependencies
COPY vllm-${WORKER_CUDA_VERSION}/requirements-dev.txt requirements-dev.txt
COPY vllm/requirements-dev.txt requirements-dev.txt
RUN --mount=type=cache,target=/root/.cache/pip \
pip install -r requirements-dev.txt
@@ -45,25 +36,15 @@ FROM dev AS build
ARG WORKER_CUDA_VERSION
# Install build dependencies
COPY vllm-${WORKER_CUDA_VERSION}/requirements-build.txt requirements-build.txt
COPY vllm/requirements-build.txt requirements-build.txt
RUN --mount=type=cache,target=/root/.cache/pip \
pip install -r requirements-build.txt
# Copy necessary files
COPY vllm-${WORKER_CUDA_VERSION}/csrc csrc
COPY vllm-${WORKER_CUDA_VERSION}/setup.py setup.py
COPY vllm-12.1.0/pyproject.toml pyproject.toml
COPY vllm-${WORKER_CUDA_VERSION}/vllm/__init__.py vllm/__init__.py
# Conditional installation based on CUDA version
RUN --mount=type=cache,target=/root/.cache/pip \
if [ "${WORKER_CUDA_VERSION}" = "11.8.0" ]; then \
pip install -U --force-reinstall torch==2.1.2 xformers==0.0.23.post1 --index-url https://download.pytorch.org/whl/cu118; \
rm pyproject.toml; \
elif [ "${WORKER_CUDA_VERSION}" != "12.1.0" ]; then \
echo "WORKER_CUDA_VERSION not supported"; \
exit 1; \
fi
COPY vllm/csrc csrc
COPY vllm/setup.py setup.py
COPY vllm/pyproject.toml pyproject.toml
COPY vllm/vllm/__init__.py vllm/__init__.py
# Set environment variables for building extensions
ARG torch_cuda_arch_list='7.0 7.5 8.0 8.6 8.9 9.0+PTX'
@@ -72,8 +53,10 @@ ARG max_jobs=48
ENV MAX_JOBS=${max_jobs}
ARG nvcc_threads=1024
ENV NVCC_THREADS=${nvcc_threads}
ENV WORKER_CUDA_VERSION=${WORKER_CUDA_VERSION}
ENV VLLM_INSTALL_PUNICA_KERNELS=0
# Build extensions
RUN ldconfig /usr/local/cuda-$(echo "$WORKER_CUDA_VERSION" | sed 's/\.0$//')/compat/
RUN python3 setup.py build_ext --inplace
FROM nvidia/cuda:${WORKER_CUDA_VERSION}-runtime-ubuntu22.04 AS vllm-base
@@ -88,19 +71,15 @@ RUN apt-get update -y \
# Set working directory
WORKDIR /vllm-installation
# Install runtime dependencies
COPY vllm-${WORKER_CUDA_VERSION}/requirements.txt requirements.txt
COPY vllm/requirements-${WORKER_CUDA_VERSION}.txt requirements.txt
RUN --mount=type=cache,target=/root/.cache/pip \
pip install -r requirements.txt
RUN --mount=type=cache,target=/root/.cache/pip \
if [ "${WORKER_CUDA_VERSION}" = "11.8.0" ]; then \
pip install -U --force-reinstall torch==2.1.2 xformers==0.0.23.post1 --index-url https://download.pytorch.org/whl/cu118; \
fi
# Copy built files from the build stage
COPY --from=build /vllm-installation/vllm/*.so /vllm-installation/vllm/
COPY vllm-${WORKER_CUDA_VERSION}/vllm vllm
COPY vllm/vllm vllm
# Set PYTHONPATH environment variable
ENV PYTHONPATH="/"
vllm-base/README.md → vllm-base-image/README.md
View File
vllm-base/download_required_files.sh
-12
View File
@@ -1,12 +0,0 @@
#!/bin/bash
git clone https://github.com/runpod/vllm-fork-for-sls-worker.git
cp -r vllm-fork-for-sls-worker vllm-12.1.0
cp -r vllm-fork-for-sls-worker vllm-11.8.0
rm -rf vllm-fork-for-sls-worker
cd vllm-11.8.0
git checkout cuda-11.8
echo "vLLM Base Image Builder Setup Complete."